/* Memory layout of a zipmap, for the map "foo" => "bar", "hello" => "world":
*
- * <status><len>"foo"<len><free>"bar"<len>"hello"<len><free>"world"
+ * <zmlen><len>"foo"<len><free>"bar"<len>"hello"<len><free>"world"
*
- * <status> is 1 byte status. Currently only 1 bit is used: if the least
- * significant bit is set, it means the zipmap needs to be defragmented.
+ * <zmlen> is 1 byte length that holds the current size of the zipmap.
+ * When the zipmap length is greater than or equal to 254, this value
+ * is not used and the zipmap needs to be traversed to find out the length.
*
* <len> is the length of the following string (key or value).
* <len> lengths are encoded in a single value or in a 5 bytes value.
* or even in order to add a key/value pair if it fits.
*
* <free> is always an unsigned 8 bit number, because if after an
- * update operation there are more than a few free bytes, they'll be converted
- * into empty space prefixed by the special value 254.
+ * update operation there are more than a few free bytes, the zipmap will be
+ * reallocated to make sure it is as small as possible.
*
* The most compact representation of the above two elements hash is actually:
*
- * "\x00\x03foo\x03\x00bar\x05hello\x05\x00world\xff"
+ * "\x02\x03foo\x03\x00bar\x05hello\x05\x00world\xff"
*
- * Empty space is marked using a 254 bytes + a <len> (coded as already
- * specified). The length includes the 254 bytes in the count and the
- * space taken by the <len> field. So for instance removing the "foo" key
- * from the zipmap above will lead to the following representation:
- *
- * "\x00\xfd\x10........\x05hello\x05\x00world\xff"
- *
- * Note that because empty space, keys, values, are all prefixed length
- * "objects", the lookup will take O(N) where N is the numeber of elements
+ * Note that because keys and values are prefixed length "objects",
+ * the lookup will take O(N) where N is the number of elements
* in the zipmap and *not* the number of bytes needed to represent the zipmap.
* This lowers the constant times considerably.
*/
#include <assert.h>
#include "zmalloc.h"
-#define ZIPMAP_BIGLEN 253
-#define ZIPMAP_EMPTY 254
+#define ZIPMAP_BIGLEN 254
#define ZIPMAP_END 255
-#define ZIPMAP_STATUS_FRAGMENTED 1
-
/* The following defines the max value for the <free> field described in the
* comments above, that is, the max number of trailing bytes in a value. */
-#define ZIPMAP_VALUE_MAX_FREE 5
+#define ZIPMAP_VALUE_MAX_FREE 4
+
+/* The following macro returns the number of bytes needed to encode the length
+ * for the integer value _l, that is, 1 byte for lengths < ZIPMAP_BIGLEN and
+ * 5 bytes for all the other lengths. */
+#define ZIPMAP_LEN_BYTES(_l) (((_l) < ZIPMAP_BIGLEN) ? 1 : sizeof(unsigned int)+1)
/* Create a new empty zipmap. */
unsigned char *zipmapNew(void) {
unsigned char *zm = zmalloc(2);
- zm[0] = 0; /* Status */
+ zm[0] = 0; /* Length */
zm[1] = ZIPMAP_END;
return zm;
}
unsigned int len = *p;
if (len < ZIPMAP_BIGLEN) return len;
- memcpy(&len,p,sizeof(unsigned int));
+ memcpy(&len,p+1,sizeof(unsigned int));
return len;
}
* the amount of bytes required to encode such a length. */
static unsigned int zipmapEncodeLength(unsigned char *p, unsigned int len) {
if (p == NULL) {
- return (len < ZIPMAP_BIGLEN) ? 1 : 1+sizeof(unsigned int);
+ return ZIPMAP_LEN_BYTES(len);
} else {
if (len < ZIPMAP_BIGLEN) {
p[0] = len;
*
* If NULL is returned, and totlen is not NULL, it is set to the entire
* size of the zimap, so that the calling function will be able to
- * reallocate the original zipmap to make room for more entries.
- *
- * If NULL is returned, and freeoff and freelen are not NULL, they are set
- * to the offset of the first empty space that can hold '*freelen' bytes
- * (freelen is an integer pointer used both to signal the required length
- * and to get the reply from the function). If there is not a suitable
- * free space block to hold the requested bytes, *freelen is set to 0. */
-static unsigned char *zipmapLookupRaw(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned int *totlen, unsigned int *freeoff, unsigned int *freelen) {
- unsigned char *p = zm+1;
+ * reallocate the original zipmap to make room for more entries. */
+static unsigned char *zipmapLookupRaw(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned int *totlen) {
+ unsigned char *p = zm+1, *k = NULL;
unsigned int l;
- unsigned int reqfreelen = 0; /* initialized just to prevent warning */
- if (freelen) {
- reqfreelen = *freelen;
- *freelen = 0;
- assert(reqfreelen != 0);
- }
while(*p != ZIPMAP_END) {
- if (*p == ZIPMAP_EMPTY) {
- l = zipmapDecodeLength(p+1);
- /* if the user want a free space report, and this space is
- * enough, and we did't already found a suitable space... */
- if (freelen && l >= reqfreelen && *freelen == 0) {
- *freelen = l;
- *freeoff = p-zm;
+ unsigned char free;
+
+ /* Match or skip the key */
+ l = zipmapDecodeLength(p);
+ if (k == NULL && l == klen && !memcmp(p+1,key,l)) {
+ /* Only return when the user doesn't care
+ * for the total length of the zipmap. */
+ if (totlen != NULL) {
+ k = p;
+ } else {
+ return p;
}
- p += l;
- zm[0] |= ZIPMAP_STATUS_FRAGMENTED;
- } else {
- unsigned char free;
-
- /* Match or skip the key */
- l = zipmapDecodeLength(p);
- if (l == klen && !memcmp(p+1,key,l)) return p;
- p += zipmapEncodeLength(NULL,l) + l;
- /* Skip the value as well */
- l = zipmapDecodeLength(p);
- p += zipmapEncodeLength(NULL,l);
- free = p[0];
- p += l+1+free; /* +1 to skip the free byte */
}
+ p += zipmapEncodeLength(NULL,l) + l;
+ /* Skip the value as well */
+ l = zipmapDecodeLength(p);
+ p += zipmapEncodeLength(NULL,l);
+ free = p[0];
+ p += l+1+free; /* +1 to skip the free byte */
}
if (totlen != NULL) *totlen = (unsigned int)(p-zm)+1;
- return NULL;
+ return k;
}
static unsigned long zipmapRequiredLength(unsigned int klen, unsigned int vlen) {
* free space if any). */
static unsigned int zipmapRawEntryLength(unsigned char *p) {
unsigned int l = zipmapRawKeyLength(p);
-
return l + zipmapRawValueLength(p+l);
}
-/* Set key to value, creating the key if it does not already exist. */
-unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned char *val, unsigned int vlen) {
- unsigned int oldlen = 0, freeoff = 0, freelen;
- unsigned int reqlen = zipmapRequiredLength(klen,vlen);
+static inline unsigned char *zipmapResize(unsigned char *zm, unsigned int len) {
+ zm = zrealloc(zm, len);
+ zm[len-1] = ZIPMAP_END;
+ return zm;
+}
+
+/* Set key to value, creating the key if it does not already exist.
+ * If 'update' is not NULL, *update is set to 1 if the key was
+ * already preset, otherwise to 0. */
+unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned char *val, unsigned int vlen, int *update) {
+ unsigned int zmlen, offset;
+ unsigned int freelen, reqlen = zipmapRequiredLength(klen,vlen);
unsigned int empty, vempty;
unsigned char *p;
freelen = reqlen;
- p = zipmapLookupRaw(zm,key,klen,&oldlen,&freeoff,&freelen);
- if (p == NULL && freelen == 0) {
- printf("HERE oldlen:%u required:%u\n",oldlen,reqlen);
- /* Key not found, and not space for the new key. Enlarge */
- zm = zrealloc(zm,oldlen+reqlen);
- p = zm+oldlen-1;
- zm[oldlen+reqlen-1] = ZIPMAP_END;
- freelen = reqlen;
- printf("New total length is: %u\n", oldlen+reqlen);
- } else if (p == NULL) {
- /* Key not found, but there is enough free space. */
- p = zm+freeoff;
- /* note: freelen is already set in this case */
- } else {
- unsigned char *b = p;
+ if (update) *update = 0;
+ p = zipmapLookupRaw(zm,key,klen,&zmlen);
+ if (p == NULL) {
+ /* Key not found: enlarge */
+ zm = zipmapResize(zm, zmlen+reqlen);
+ p = zm+zmlen-1;
+ zmlen = zmlen+reqlen;
+ /* Increase zipmap length (this is an insert) */
+ if (zm[0] < ZIPMAP_BIGLEN) zm[0]++;
+ } else {
/* Key found. Is there enough space for the new value? */
/* Compute the total length: */
- freelen = zipmapRawKeyLength(b);
- b += freelen;
- freelen += zipmapRawValueLength(b);
+ if (update) *update = 1;
+ freelen = zipmapRawEntryLength(p);
if (freelen < reqlen) {
- /* Mark this entry as free and recurse */
- p[0] = ZIPMAP_EMPTY;
- zipmapEncodeLength(p+1,freelen);
- zm[0] |= ZIPMAP_STATUS_FRAGMENTED;
- return zipmapSet(zm,key,klen,val,vlen);
+ /* Store the offset of this key within the current zipmap, so
+ * it can be resized. Then, move the tail backwards so this
+ * pair fits at the current position. */
+ offset = p-zm;
+ zm = zipmapResize(zm, zmlen-freelen+reqlen);
+ p = zm+offset;
+
+ /* The +1 in the number of bytes to be moved is caused by the
+ * end-of-zipmap byte. Note: the *original* zmlen is used. */
+ memmove(p+reqlen, p+freelen, zmlen-(offset+freelen+1));
+ zmlen = zmlen-freelen+reqlen;
+ freelen = reqlen;
}
}
- /* Ok we have a suitable block where to write the new key/value
- * entry. */
+ /* We now have a suitable block where the key/value entry can
+ * be written. If there is too much free space, move the tail
+ * of the zipmap a few bytes to the front and shrink the zipmap,
+ * as we want zipmaps to be very space efficient. */
empty = freelen-reqlen;
- /* If there is too much free space mark it as a free block instead
- * of adding it as trailing empty space for the value, as we want
- * zipmaps to be very space efficient. */
- if (empty > ZIPMAP_VALUE_MAX_FREE) {
- unsigned char *e;
-
- e = p+reqlen;
- e[0] = ZIPMAP_EMPTY;
- zipmapEncodeLength(e+1,empty);
+ if (empty >= ZIPMAP_VALUE_MAX_FREE) {
+ /* First, move the tail <empty> bytes to the front, then resize
+ * the zipmap to be <empty> bytes smaller. */
+ offset = p-zm;
+ memmove(p+reqlen, p+freelen, zmlen-(offset+freelen+1));
+ zmlen -= empty;
+ zm = zipmapResize(zm, zmlen);
+ p = zm+offset;
vempty = 0;
- zm[0] |= ZIPMAP_STATUS_FRAGMENTED;
} else {
vempty = empty;
}
/* Remove the specified key. If 'deleted' is not NULL the pointed integer is
* set to 0 if the key was not found, to 1 if it was found and deleted. */
unsigned char *zipmapDel(unsigned char *zm, unsigned char *key, unsigned int klen, int *deleted) {
- unsigned char *p = zipmapLookupRaw(zm,key,klen,NULL,NULL,NULL);
+ unsigned int zmlen, freelen;
+ unsigned char *p = zipmapLookupRaw(zm,key,klen,&zmlen);
if (p) {
- unsigned int freelen = zipmapRawEntryLength(p);
+ freelen = zipmapRawEntryLength(p);
+ memmove(p, p+freelen, zmlen-((p-zm)+freelen+1));
+ zm = zipmapResize(zm, zmlen-freelen);
+
+ /* Decrease zipmap length */
+ if (zm[0] < ZIPMAP_BIGLEN) zm[0]--;
- p[0] = ZIPMAP_EMPTY;
- zipmapEncodeLength(p+1,freelen);
- zm[0] |= ZIPMAP_STATUS_FRAGMENTED;
if (deleted) *deleted = 1;
} else {
if (deleted) *deleted = 0;
return zm;
}
+/* Call it before to iterate trought elements via zipmapNext() */
+unsigned char *zipmapRewind(unsigned char *zm) {
+ return zm+1;
+}
+
/* This function is used to iterate through all the zipmap elements.
* In the first call the first argument is the pointer to the zipmap + 1.
* In the next calls what zipmapNext returns is used as first argument.
* Example:
*
- * unsigned char *i = my_zipmap+1;
+ * unsigned char *i = zipmapRewind(my_zipmap);
* while((i = zipmapNext(i,&key,&klen,&value,&vlen)) != NULL) {
* printf("%d bytes key at $p\n", klen, key);
* printf("%d bytes value at $p\n", vlen, value);
* }
*/
-unsigned char *zipmapNext(unsigned char *zm, unsigned char **key,
- unsigned int *klen, unsigned char **value, unsigned int *vlen) {
- while(zm[0] == ZIPMAP_EMPTY)
- zm += zipmapDecodeLength(zm+1);
+unsigned char *zipmapNext(unsigned char *zm, unsigned char **key, unsigned int *klen, unsigned char **value, unsigned int *vlen) {
if (zm[0] == ZIPMAP_END) return NULL;
if (key) {
*key = zm;
*klen = zipmapDecodeLength(zm);
- *key += (*klen < ZIPMAP_BIGLEN) ? 1 : sizeof(unsigned int);
+ *key += ZIPMAP_LEN_BYTES(*klen);
}
zm += zipmapRawKeyLength(zm);
if (value) {
*value = zm+1;
*vlen = zipmapDecodeLength(zm);
- *value += (*vlen < ZIPMAP_BIGLEN) ? 1 : sizeof(unsigned int);
+ *value += ZIPMAP_LEN_BYTES(*vlen);
}
zm += zipmapRawValueLength(zm);
return zm;
}
+/* Search a key and retrieve the pointer and len of the associated value.
+ * If the key is found the function returns 1, otherwise 0. */
+int zipmapGet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned char **value, unsigned int *vlen) {
+ unsigned char *p;
+
+ if ((p = zipmapLookupRaw(zm,key,klen,NULL)) == NULL) return 0;
+ p += zipmapRawKeyLength(p);
+ *vlen = zipmapDecodeLength(p);
+ *value = p + ZIPMAP_LEN_BYTES(*vlen) + 1;
+ return 1;
+}
+
+/* Return 1 if the key exists, otherwise 0 is returned. */
+int zipmapExists(unsigned char *zm, unsigned char *key, unsigned int klen) {
+ return zipmapLookupRaw(zm,key,klen,NULL) != NULL;
+}
+
+/* Return the number of entries inside a zipmap */
+unsigned int zipmapLen(unsigned char *zm) {
+ unsigned int len = 0;
+ if (zm[0] < ZIPMAP_BIGLEN) {
+ len = zm[0];
+ } else {
+ unsigned char *p = zipmapRewind(zm);
+ while((p = zipmapNext(p,NULL,NULL,NULL,NULL)) != NULL) len++;
+
+ /* Re-store length if small enough */
+ if (len < ZIPMAP_BIGLEN) zm[0] = len;
+ }
+ return len;
+}
+
void zipmapRepr(unsigned char *p) {
unsigned int l;
if (p[0] == ZIPMAP_END) {
printf("{end}");
break;
- } else if (p[0] == ZIPMAP_EMPTY) {
- l = zipmapDecodeLength(p+1);
- printf("{%u empty block}", l);
- p += l;
} else {
unsigned char e;
printf("\n");
}
+#ifdef ZIPMAP_TEST_MAIN
int main(void) {
unsigned char *zm;
zm = zipmapNew();
- zm = zipmapSet(zm,(unsigned char*) "hello",5, (unsigned char*) "world!",6);
- zm = zipmapSet(zm,(unsigned char*) "foo",3, (unsigned char*) "bar",3);
- zm = zipmapSet(zm,(unsigned char*) "foo",3, (unsigned char*) "!",1);
+
+ zm = zipmapSet(zm,(unsigned char*) "name",4, (unsigned char*) "foo",3,NULL);
+ zm = zipmapSet(zm,(unsigned char*) "surname",7, (unsigned char*) "foo",3,NULL);
+ zm = zipmapSet(zm,(unsigned char*) "age",3, (unsigned char*) "foo",3,NULL);
+ zipmapRepr(zm);
+ exit(1);
+
+ zm = zipmapSet(zm,(unsigned char*) "hello",5, (unsigned char*) "world!",6,NULL);
+ zm = zipmapSet(zm,(unsigned char*) "foo",3, (unsigned char*) "bar",3,NULL);
+ zm = zipmapSet(zm,(unsigned char*) "foo",3, (unsigned char*) "!",1,NULL);
zipmapRepr(zm);
- zm = zipmapSet(zm,(unsigned char*) "foo",3, (unsigned char*) "12345",5);
+ zm = zipmapSet(zm,(unsigned char*) "foo",3, (unsigned char*) "12345",5,NULL);
zipmapRepr(zm);
- zm = zipmapSet(zm,(unsigned char*) "new",3, (unsigned char*) "xx",2);
+ zm = zipmapSet(zm,(unsigned char*) "new",3, (unsigned char*) "xx",2,NULL);
+ zm = zipmapSet(zm,(unsigned char*) "noval",5, (unsigned char*) "",0,NULL);
zipmapRepr(zm);
zm = zipmapDel(zm,(unsigned char*) "new",3,NULL);
zipmapRepr(zm);
- printf("Iterate trought elements:\n");
+ printf("\nPerform a direct lookup:\n");
+ {
+ unsigned char *value;
+ unsigned int vlen;
+
+ if (zipmapGet(zm,(unsigned char*) "foo",3,&value,&vlen)) {
+ printf(" foo is associated to the %d bytes value: %.*s\n",
+ vlen, vlen, value);
+ }
+ }
+ printf("\nIterate trought elements:\n");
{
- unsigned char *i = zm+1;
+ unsigned char *i = zipmapRewind(zm);
unsigned char *key, *value;
unsigned int klen, vlen;
while((i = zipmapNext(i,&key,&klen,&value,&vlen)) != NULL) {
- printf("%d:%.*s => %d:%.*s\n", klen, klen, key, vlen, vlen, value);
+ printf(" %d:%.*s => %d:%.*s\n", klen, klen, key, vlen, vlen, value);
}
}
return 0;
}
+#endif
projects / redis.git / blobdiff