code to iterate over a ziplist

[redis.git] / zipmap.c

diff --git a/zipmap.c b/zipmap.c
index 8a07b388ed5aabb6af3cd83a4bc175d1a6bf6f7b..7223678aab8aa815b6955078922fb482e7d52561 100644 (file)
--- a/zipmap.c
+++ b/zipmap.c
@@ -42,10 +42,11 @@
 
 /* Memory layout of a zipmap, for the map "foo" => "bar", "hello" => "world":
  *
 
 /* Memory layout of a zipmap, for the map "foo" => "bar", "hello" => "world":
  *

- * <status><len>"foo"<len><free>"bar"<len>"hello"<len><free>"world"
+ * <zmlen><len>"foo"<len><free>"bar"<len>"hello"<len><free>"world"

  *
  *

- * <status> is 1 byte status. Currently only 1 bit is used: if the least
- * significant bit is set, it means the zipmap needs to be defragmented.
+ * <zmlen> is 1 byte length that holds the current size of the zipmap.
+ * When the zipmap length is greater than or equal to 254, this value
+ * is not used and the zipmap needs to be traversed to find out the length.

  *
  * <len> is the length of the following string (key or value).
  * <len> lengths are encoded in a single value or in a 5 bytes value.
  *
  * <len> is the length of the following string (key or value).
  * <len> lengths are encoded in a single value or in a 5 bytes value.


@@ -62,22 +63,15 @@
  * or even in order to add a key/value pair if it fits.
  *
  * <free> is always an unsigned 8 bit number, because if after an
  * or even in order to add a key/value pair if it fits.
  *
  * <free> is always an unsigned 8 bit number, because if after an

- * update operation there are more than a few free bytes, they'll be converted
- * into empty space prefixed by the special value 254.
+ * update operation there are more than a few free bytes, the zipmap will be
+ * reallocated to make sure it is as small as possible.

  *
  * The most compact representation of the above two elements hash is actually:
  *
  *
  * The most compact representation of the above two elements hash is actually:
  *

- * "\x00\x03foo\x03\x00bar\x05hello\x05\x00world\xff"
+ * "\x02\x03foo\x03\x00bar\x05hello\x05\x00world\xff"

  *
  *

- * Empty space is marked using a 254 bytes + a <len> (coded as already
- * specified). The length includes the 254 bytes in the count and the
- * space taken by the <len> field. So for instance removing the "foo" key
- * from the zipmap above will lead to the following representation:
- *
- * "\x00\xfd\x10........\x05hello\x05\x00world\xff"
- *
- * Note that because empty space, keys, values, are all prefixed length
- * "objects", the lookup will take O(N) where N is the numeber of elements
+ * Note that because keys and values are prefixed length "objects",
+ * the lookup will take O(N) where N is the number of elements

  * in the zipmap and *not* the number of bytes needed to represent the zipmap.
  * This lowers the constant times considerably.
  */
  * in the zipmap and *not* the number of bytes needed to represent the zipmap.
  * This lowers the constant times considerably.
  */


@@ -86,54 +80,21 @@
 #include <string.h>
 #include <assert.h>
 #include "zmalloc.h"
 #include <string.h>
 #include <assert.h>
 #include "zmalloc.h"

-
-#define ZIPMAP_BIGLEN 254
-#define ZIPMAP_END 255
+#include "zip.c"

 
 /* The following defines the max value for the <free> field described in the
  * comments above, that is, the max number of trailing bytes in a value. */
 
 /* The following defines the max value for the <free> field described in the
  * comments above, that is, the max number of trailing bytes in a value. */

-#define ZIPMAP_VALUE_MAX_FREE 5
-
-/* The following macro returns the number of bytes needed to encode the length
- * for the integer value _l, that is, 1 byte for lengths < ZIPMAP_BIGLEN and
- * 5 bytes for all the other lengths. */
-#define ZIPMAP_LEN_BYTES(_l) (((_l) < ZIPMAP_BIGLEN) ? 1 : sizeof(unsigned int)+1)
+#define ZIPMAP_VALUE_MAX_FREE 4

 
 /* Create a new empty zipmap. */
 unsigned char *zipmapNew(void) {
     unsigned char *zm = zmalloc(2);
 
     zm[0] = 0; /* Length */
 
 /* Create a new empty zipmap. */
 unsigned char *zipmapNew(void) {
     unsigned char *zm = zmalloc(2);
 
     zm[0] = 0; /* Length */

-    zm[1] = ZIPMAP_END;
+    zm[1] = ZIP_END;

     return zm;
 }
 
     return zm;
 }
 

-/* Decode the encoded length pointed by 'p' */
-static unsigned int zipmapDecodeLength(unsigned char *p) {
-    unsigned int len = *p;
-
-    if (len < ZIPMAP_BIGLEN) return len;
-    memcpy(&len,p+1,sizeof(unsigned int));
-    return len;
-}
-
-/* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
- * the amount of bytes required to encode such a length. */
-static unsigned int zipmapEncodeLength(unsigned char *p, unsigned int len) {
-    if (p == NULL) {
-        return ZIPMAP_LEN_BYTES(len);
-    } else {
-        if (len < ZIPMAP_BIGLEN) {
-            p[0] = len;
-            return 1;
-        } else {
-            p[0] = ZIPMAP_BIGLEN;
-            memcpy(p+1,&len,sizeof(len));
-            return 1+sizeof(len);
-        }
-    }
-}
-

 /* Search for a matching key, returning a pointer to the entry inside the
  * zipmap. Returns NULL if the key is not found.
  *
 /* Search for a matching key, returning a pointer to the entry inside the
  * zipmap. Returns NULL if the key is not found.
  *


@@ -142,14 +103,15 @@ static unsigned int zipmapEncodeLength(unsigned char *p, unsigned int len) {
  * reallocate the original zipmap to make room for more entries. */
 static unsigned char *zipmapLookupRaw(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned int *totlen) {
     unsigned char *p = zm+1, *k = NULL;
  * reallocate the original zipmap to make room for more entries. */
 static unsigned char *zipmapLookupRaw(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned int *totlen) {
     unsigned char *p = zm+1, *k = NULL;

-    unsigned int l;
+    unsigned int l,llen;

 
 

-    while(*p != ZIPMAP_END) {
+    while(*p != ZIP_END) {

         unsigned char free;
 
         /* Match or skip the key */
         unsigned char free;
 
         /* Match or skip the key */

-        l = zipmapDecodeLength(p);
-        if (k == NULL && l == klen && !memcmp(p+1,key,l)) {
+        l = zipDecodeLength(p);
+        llen = zipEncodeLength(NULL,l);
+        if (k == NULL && l == klen && !memcmp(p+llen,key,l)) {

             /* Only return when the user doesn't care
              * for the total length of the zipmap. */
             if (totlen != NULL) {
             /* Only return when the user doesn't care
              * for the total length of the zipmap. */
             if (totlen != NULL) {


@@ -158,10 +120,10 @@ static unsigned char *zipmapLookupRaw(unsigned char *zm, unsigned char *key, uns
                 return p;
             }
         }
                 return p;
             }
         }

-        p += zipmapEncodeLength(NULL,l) + l;
+        p += llen+l;

         /* Skip the value as well */
         /* Skip the value as well */

-        l = zipmapDecodeLength(p);
-        p += zipmapEncodeLength(NULL,l);
+        l = zipDecodeLength(p);
+        p += zipEncodeLength(NULL,l);

         free = p[0];
         p += l+1+free; /* +1 to skip the free byte */
     }
         free = p[0];
         p += l+1+free; /* +1 to skip the free byte */
     }


@@ -173,25 +135,23 @@ static unsigned long zipmapRequiredLength(unsigned int klen, unsigned int vlen)
     unsigned int l;
 
     l = klen+vlen+3;
     unsigned int l;
 
     l = klen+vlen+3;

-    if (klen >= ZIPMAP_BIGLEN) l += 4;
-    if (vlen >= ZIPMAP_BIGLEN) l += 4;
+    if (klen >= ZIP_BIGLEN) l += 4;
+    if (vlen >= ZIP_BIGLEN) l += 4;

     return l;
 }
 
 /* Return the total amount used by a key (encoded length + payload) */
 static unsigned int zipmapRawKeyLength(unsigned char *p) {
     return l;
 }
 
 /* Return the total amount used by a key (encoded length + payload) */
 static unsigned int zipmapRawKeyLength(unsigned char *p) {

-    unsigned int l = zipmapDecodeLength(p);
-    
-    return zipmapEncodeLength(NULL,l) + l;
+    return zipRawEntryLength(p);

 }
 
 /* Return the total amount used by a value
  * (encoded length + single byte free count + payload) */
 static unsigned int zipmapRawValueLength(unsigned char *p) {
 }
 
 /* Return the total amount used by a value
  * (encoded length + single byte free count + payload) */
 static unsigned int zipmapRawValueLength(unsigned char *p) {

-    unsigned int l = zipmapDecodeLength(p);
+    unsigned int l = zipDecodeLength(p);

     unsigned int used;
     
     unsigned int used;
     

-    used = zipmapEncodeLength(NULL,l);
+    used = zipEncodeLength(NULL,l);

     used += p[used] + 1 + l;
     return used;
 }
     used += p[used] + 1 + l;
     return used;
 }


@@ -204,17 +164,11 @@ static unsigned int zipmapRawEntryLength(unsigned char *p) {
     return l + zipmapRawValueLength(p+l);
 }
 
     return l + zipmapRawValueLength(p+l);
 }
 

-static inline unsigned char *zipmapResize(unsigned char *zm, unsigned int len) {
-    zm = zrealloc(zm, len);
-    zm[len-1] = ZIPMAP_END;
-    return zm;
-}
-

 /* Set key to value, creating the key if it does not already exist.
  * If 'update' is not NULL, *update is set to 1 if the key was
  * already preset, otherwise to 0. */
 unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned char *val, unsigned int vlen, int *update) {
 /* Set key to value, creating the key if it does not already exist.
  * If 'update' is not NULL, *update is set to 1 if the key was
  * already preset, otherwise to 0. */
 unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned char *val, unsigned int vlen, int *update) {

-    unsigned int zmlen;
+    unsigned int zmlen, offset;

     unsigned int freelen, reqlen = zipmapRequiredLength(klen,vlen);
     unsigned int empty, vempty;
     unsigned char *p;
     unsigned int freelen, reqlen = zipmapRequiredLength(klen,vlen);
     unsigned int empty, vempty;
     unsigned char *p;


@@ -224,39 +178,46 @@ unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int kle
     p = zipmapLookupRaw(zm,key,klen,&zmlen);
     if (p == NULL) {
         /* Key not found: enlarge */
     p = zipmapLookupRaw(zm,key,klen,&zmlen);
     if (p == NULL) {
         /* Key not found: enlarge */

-        zm = zipmapResize(zm, zmlen+reqlen);
+        zm = zipResize(zm, zmlen+reqlen);

         p = zm+zmlen-1;
         zmlen = zmlen+reqlen;
 
         /* Increase zipmap length (this is an insert) */
         p = zm+zmlen-1;
         zmlen = zmlen+reqlen;
 
         /* Increase zipmap length (this is an insert) */

-        if (zm[0] < ZIPMAP_BIGLEN) zm[0]++;
+        if (zm[0] < ZIP_BIGLEN) zm[0]++;

     } else {
         /* Key found. Is there enough space for the new value? */
         /* Compute the total length: */
         if (update) *update = 1;
         freelen = zipmapRawEntryLength(p);
         if (freelen < reqlen) {
     } else {
         /* Key found. Is there enough space for the new value? */
         /* Compute the total length: */
         if (update) *update = 1;
         freelen = zipmapRawEntryLength(p);
         if (freelen < reqlen) {

-            /* Move remaining entries to the current position, so this
-             * pair can be appended. Note: the +1 in memmove is caused
-             * by the end-of-zipmap byte. */
-            memmove(p, p+freelen, zmlen-((p-zm)+freelen+1));
-            zm = zipmapResize(zm, zmlen-freelen+reqlen);
-            p = zm+zmlen-1-freelen;
-            zmlen = zmlen-1-freelen+reqlen;
+            /* Store the offset of this key within the current zipmap, so
+             * it can be resized. Then, move the tail backwards so this
+             * pair fits at the current position. */
+            offset = p-zm;
+            zm = zipResize(zm, zmlen-freelen+reqlen);
+            p = zm+offset;
+
+            /* The +1 in the number of bytes to be moved is caused by the
+             * end-of-zipmap byte. Note: the *original* zmlen is used. */
+            memmove(p+reqlen, p+freelen, zmlen-(offset+freelen+1));
+            zmlen = zmlen-freelen+reqlen;

             freelen = reqlen;
         }
     }
 
             freelen = reqlen;
         }
     }
 

-    /* Ok we have a suitable block where to write the new key/value
-     * entry. */
+    /* We now have a suitable block where the key/value entry can
+     * be written. If there is too much free space, move the tail
+     * of the zipmap a few bytes to the front and shrink the zipmap,
+     * as we want zipmaps to be very space efficient. */

     empty = freelen-reqlen;
     empty = freelen-reqlen;

-    /* If there is too much free space mark it as a free block instead
-     * of adding it as trailing empty space for the value, as we want
-     * zipmaps to be very space efficient. */

     if (empty >= ZIPMAP_VALUE_MAX_FREE) {
     if (empty >= ZIPMAP_VALUE_MAX_FREE) {

-        memmove(p+reqlen, p+freelen, zmlen-((p-zm)+freelen+1));
+        /* First, move the tail <empty> bytes to the front, then resize
+         * the zipmap to be <empty> bytes smaller. */
+        offset = p-zm;
+        memmove(p+reqlen, p+freelen, zmlen-(offset+freelen+1));

         zmlen -= empty;
         zmlen -= empty;

-        zm = zipmapResize(zm, zmlen);
+        zm = zipResize(zm, zmlen);
+        p = zm+offset;

         vempty = 0;
     } else {
         vempty = empty;
         vempty = 0;
     } else {
         vempty = empty;


@@ -264,11 +225,11 @@ unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int kle
 
     /* Just write the key + value and we are done. */
     /* Key: */
 
     /* Just write the key + value and we are done. */
     /* Key: */

-    p += zipmapEncodeLength(p,klen);
+    p += zipEncodeLength(p,klen);

     memcpy(p,key,klen);
     p += klen;
     /* Value: */
     memcpy(p,key,klen);
     p += klen;
     /* Value: */

-    p += zipmapEncodeLength(p,vlen);
+    p += zipEncodeLength(p,vlen);

     *p++ = vempty;
     memcpy(p,val,vlen);
     return zm;
     *p++ = vempty;
     memcpy(p,val,vlen);
     return zm;


@@ -282,10 +243,10 @@ unsigned char *zipmapDel(unsigned char *zm, unsigned char *key, unsigned int kle
     if (p) {
         freelen = zipmapRawEntryLength(p);
         memmove(p, p+freelen, zmlen-((p-zm)+freelen+1));
     if (p) {
         freelen = zipmapRawEntryLength(p);
         memmove(p, p+freelen, zmlen-((p-zm)+freelen+1));

-        zm = zipmapResize(zm, zmlen-freelen);
+        zm = zipResize(zm, zmlen-freelen);

 
         /* Decrease zipmap length */
 
         /* Decrease zipmap length */

-        if (zm[0] < ZIPMAP_BIGLEN) zm[0]--;
+        if (zm[0] < ZIP_BIGLEN) zm[0]--;

 
         if (deleted) *deleted = 1;
     } else {
 
         if (deleted) *deleted = 1;
     } else {


@@ -311,17 +272,17 @@ unsigned char *zipmapRewind(unsigned char *zm) {
  * }
  */
 unsigned char *zipmapNext(unsigned char *zm, unsigned char **key, unsigned int *klen, unsigned char **value, unsigned int *vlen) {
  * }
  */
 unsigned char *zipmapNext(unsigned char *zm, unsigned char **key, unsigned int *klen, unsigned char **value, unsigned int *vlen) {

-    if (zm[0] == ZIPMAP_END) return NULL;
+    if (zm[0] == ZIP_END) return NULL;

     if (key) {
         *key = zm;
     if (key) {
         *key = zm;

-        *klen = zipmapDecodeLength(zm);
-        *key += ZIPMAP_LEN_BYTES(*klen);
+        *klen = zipDecodeLength(zm);
+        *key += ZIP_LEN_BYTES(*klen);

     }
     zm += zipmapRawKeyLength(zm);
     if (value) {
         *value = zm+1;
     }
     zm += zipmapRawKeyLength(zm);
     if (value) {
         *value = zm+1;

-        *vlen = zipmapDecodeLength(zm);
-        *value += ZIPMAP_LEN_BYTES(*vlen);
+        *vlen = zipDecodeLength(zm);
+        *value += ZIP_LEN_BYTES(*vlen);

     }
     zm += zipmapRawValueLength(zm);
     return zm;
     }
     zm += zipmapRawValueLength(zm);
     return zm;


@@ -334,8 +295,8 @@ int zipmapGet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned
 
     if ((p = zipmapLookupRaw(zm,key,klen,NULL)) == NULL) return 0;
     p += zipmapRawKeyLength(p);
 
     if ((p = zipmapLookupRaw(zm,key,klen,NULL)) == NULL) return 0;
     p += zipmapRawKeyLength(p);

-    *vlen = zipmapDecodeLength(p);
-    *value = p + ZIPMAP_LEN_BYTES(*vlen) + 1;
+    *vlen = zipDecodeLength(p);
+    *value = p + ZIP_LEN_BYTES(*vlen) + 1;

     return 1;
 }
 
     return 1;
 }
 


@@ -347,14 +308,14 @@ int zipmapExists(unsigned char *zm, unsigned char *key, unsigned int klen) {
 /* Return the number of entries inside a zipmap */
 unsigned int zipmapLen(unsigned char *zm) {
     unsigned int len = 0;
 /* Return the number of entries inside a zipmap */
 unsigned int zipmapLen(unsigned char *zm) {
     unsigned int len = 0;

-    if (zm[0] < ZIPMAP_BIGLEN) {
+    if (zm[0] < ZIP_BIGLEN) {

         len = zm[0];
     } else {
         unsigned char *p = zipmapRewind(zm);
         while((p = zipmapNext(p,NULL,NULL,NULL,NULL)) != NULL) len++;
 
         /* Re-store length if small enough */
         len = zm[0];
     } else {
         unsigned char *p = zipmapRewind(zm);
         while((p = zipmapNext(p,NULL,NULL,NULL,NULL)) != NULL) len++;
 
         /* Re-store length if small enough */

-        if (len < ZIPMAP_BIGLEN) zm[0] = len;
+        if (len < ZIP_BIGLEN) zm[0] = len;

     }
     return len;
 }
     }
     return len;
 }


@@ -364,21 +325,21 @@ void zipmapRepr(unsigned char *p) {
 
     printf("{status %u}",*p++);
     while(1) {
 
     printf("{status %u}",*p++);
     while(1) {

-        if (p[0] == ZIPMAP_END) {
+        if (p[0] == ZIP_END) {

             printf("{end}");
             break;
         } else {
             unsigned char e;
 
             printf("{end}");
             break;
         } else {
             unsigned char e;
 

-            l = zipmapDecodeLength(p);
+            l = zipDecodeLength(p);

             printf("{key %u}",l);
             printf("{key %u}",l);

-            p += zipmapEncodeLength(NULL,l);
+            p += zipEncodeLength(NULL,l);

             fwrite(p,l,1,stdout);
             p += l;
 
             fwrite(p,l,1,stdout);
             p += l;
 

-            l = zipmapDecodeLength(p);
+            l = zipDecodeLength(p);

             printf("{value %u}",l);
             printf("{value %u}",l);

-            p += zipmapEncodeLength(NULL,l);
+            p += zipEncodeLength(NULL,l);

             e = *p++;
             fwrite(p,l,1,stdout);
             p += l+e;
             e = *p++;
             fwrite(p,l,1,stdout);
             p += l+e;


@@ -402,7 +363,6 @@ int main(void) {
     zm = zipmapSet(zm,(unsigned char*) "surname",7, (unsigned char*) "foo",3,NULL);
     zm = zipmapSet(zm,(unsigned char*) "age",3, (unsigned char*) "foo",3,NULL);
     zipmapRepr(zm);
     zm = zipmapSet(zm,(unsigned char*) "surname",7, (unsigned char*) "foo",3,NULL);
     zm = zipmapSet(zm,(unsigned char*) "age",3, (unsigned char*) "foo",3,NULL);
     zipmapRepr(zm);

-    exit(1);

 
     zm = zipmapSet(zm,(unsigned char*) "hello",5, (unsigned char*) "world!",6,NULL);
     zm = zipmapSet(zm,(unsigned char*) "foo",3, (unsigned char*) "bar",3,NULL);
 
     zm = zipmapSet(zm,(unsigned char*) "hello",5, (unsigned char*) "world!",6,NULL);
     zm = zipmapSet(zm,(unsigned char*) "foo",3, (unsigned char*) "bar",3,NULL);


@@ -415,6 +375,21 @@ int main(void) {
     zipmapRepr(zm);
     zm = zipmapDel(zm,(unsigned char*) "new",3,NULL);
     zipmapRepr(zm);
     zipmapRepr(zm);
     zm = zipmapDel(zm,(unsigned char*) "new",3,NULL);
     zipmapRepr(zm);

+
+    printf("\nLook up large key:\n");
+    {
+        unsigned char buf[512];
+        unsigned char *value;
+        unsigned int vlen, i;
+        for (i = 0; i < 512; i++) buf[i] = 'a';
+
+        zm = zipmapSet(zm,buf,512,(unsigned char*) "long",4,NULL);
+        if (zipmapGet(zm,buf,512,&value,&vlen)) {
+            printf("  <long key> is associated to the %d bytes value: %.*s\n",
+                vlen, vlen, value);
+        }
+    }
+

     printf("\nPerform a direct lookup:\n");
     {
         unsigned char *value;
     printf("\nPerform a direct lookup:\n");
     {
         unsigned char *value;