+/* String -> String Map data structure optimized for size.
+ * This file implements a data structure mapping strings to other strings
+ * implementing an O(n) lookup data structure designed to be very memory
+ * efficient.
+ *
+ * The Redis Hash type uses this data structure for hashes composed of a small
+ * number of elements, to switch to an hash table once a given number of
+ * elements is reached.
+ *
+ * Given that many times Redis Hashes are used to represent objects composed
+ * of few fields, this is a very big win in terms of used memory.
+ *
+ * --------------------------------------------------------------------------
+ *
+ * Copyright (c) 2009-2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Memory layout of a zipmap, for the map "foo" => "bar", "hello" => "world":
+ *
+ * <zmlen><len>"foo"<len><free>"bar"<len>"hello"<len><free>"world"
+ *
+ * <zmlen> is 1 byte length that holds the current size of the zipmap.
+ * When the zipmap length is greater than or equal to 254, this value
+ * is not used and the zipmap needs to be traversed to find out the length.
+ *
+ * <len> is the length of the following string (key or value).
+ * <len> lengths are encoded in a single value or in a 5 bytes value.
+ * If the first byte value (as an unsigned 8 bit value) is between 0 and
+ * 252, it's a single-byte length. If it is 253 then a four bytes unsigned
+ * integer follows (in the host byte ordering). A value fo 255 is used to
+ * signal the end of the hash. The special value 254 is used to mark
+ * empty space that can be used to add new key/value pairs.
+ *
+ * <free> is the number of free unused bytes
+ * after the string, resulting from modification of values associated to a
+ * key (for instance if "foo" is set to "bar', and later "foo" will be se to
+ * "hi", I'll have a free byte to use if the value will enlarge again later,
+ * or even in order to add a key/value pair if it fits.
+ *
+ * <free> is always an unsigned 8 bit number, because if after an
+ * update operation there are more than a few free bytes, the zipmap will be
+ * reallocated to make sure it is as small as possible.
+ *
+ * The most compact representation of the above two elements hash is actually:
+ *
+ * "\x02\x03foo\x03\x00bar\x05hello\x05\x00world\xff"
+ *
+ * Note that because keys and values are prefixed length "objects",
+ * the lookup will take O(N) where N is the number of elements
+ * in the zipmap and *not* the number of bytes needed to represent the zipmap.
+ * This lowers the constant times considerably.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include "zmalloc.h"
+
+#define ZIPMAP_BIGLEN 254
+#define ZIPMAP_END 255
+
+/* The following defines the max value for the <free> field described in the
+ * comments above, that is, the max number of trailing bytes in a value. */
+#define ZIPMAP_VALUE_MAX_FREE 4
+
+/* The following macro returns the number of bytes needed to encode the length
+ * for the integer value _l, that is, 1 byte for lengths < ZIPMAP_BIGLEN and
+ * 5 bytes for all the other lengths. */
+#define ZIPMAP_LEN_BYTES(_l) (((_l) < ZIPMAP_BIGLEN) ? 1 : sizeof(unsigned int)+1)
+
+/* Create a new empty zipmap. */
+unsigned char *zipmapNew(void) {
+ unsigned char *zm = zmalloc(2);
+
+ zm[0] = 0; /* Length */
+ zm[1] = ZIPMAP_END;
+ return zm;
+}
+
+/* Decode the encoded length pointed by 'p' */
+static unsigned int zipmapDecodeLength(unsigned char *p) {
+ unsigned int len = *p;
+
+ if (len < ZIPMAP_BIGLEN) return len;
+ memcpy(&len,p+1,sizeof(unsigned int));
+ return len;
+}
+
+/* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
+ * the amount of bytes required to encode such a length. */
+static unsigned int zipmapEncodeLength(unsigned char *p, unsigned int len) {
+ if (p == NULL) {
+ return ZIPMAP_LEN_BYTES(len);
+ } else {
+ if (len < ZIPMAP_BIGLEN) {
+ p[0] = len;
+ return 1;
+ } else {
+ p[0] = ZIPMAP_BIGLEN;
+ memcpy(p+1,&len,sizeof(len));
+ return 1+sizeof(len);
+ }
+ }
+}
+
+/* Search for a matching key, returning a pointer to the entry inside the
+ * zipmap. Returns NULL if the key is not found.
+ *
+ * If NULL is returned, and totlen is not NULL, it is set to the entire
+ * size of the zimap, so that the calling function will be able to
+ * reallocate the original zipmap to make room for more entries. */
+static unsigned char *zipmapLookupRaw(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned int *totlen) {
+ unsigned char *p = zm+1, *k = NULL;
+ unsigned int l,llen;
+
+ while(*p != ZIPMAP_END) {
+ unsigned char free;
+
+ /* Match or skip the key */
+ l = zipmapDecodeLength(p);
+ llen = zipmapEncodeLength(NULL,l);
+ if (k == NULL && l == klen && !memcmp(p+llen,key,l)) {
+ /* Only return when the user doesn't care
+ * for the total length of the zipmap. */
+ if (totlen != NULL) {
+ k = p;
+ } else {
+ return p;
+ }
+ }
+ p += llen+l;
+ /* Skip the value as well */
+ l = zipmapDecodeLength(p);
+ p += zipmapEncodeLength(NULL,l);
+ free = p[0];
+ p += l+1+free; /* +1 to skip the free byte */
+ }
+ if (totlen != NULL) *totlen = (unsigned int)(p-zm)+1;
+ return k;
+}
+
+static unsigned long zipmapRequiredLength(unsigned int klen, unsigned int vlen) {
+ unsigned int l;
+
+ l = klen+vlen+3;
+ if (klen >= ZIPMAP_BIGLEN) l += 4;
+ if (vlen >= ZIPMAP_BIGLEN) l += 4;
+ return l;
+}
+
+/* Return the total amount used by a key (encoded length + payload) */
+static unsigned int zipmapRawKeyLength(unsigned char *p) {
+ unsigned int l = zipmapDecodeLength(p);
+ return zipmapEncodeLength(NULL,l) + l;
+}
+
+/* Return the total amount used by a value
+ * (encoded length + single byte free count + payload) */
+static unsigned int zipmapRawValueLength(unsigned char *p) {
+ unsigned int l = zipmapDecodeLength(p);
+ unsigned int used;
+
+ used = zipmapEncodeLength(NULL,l);
+ used += p[used] + 1 + l;
+ return used;
+}
+
+/* If 'p' points to a key, this function returns the total amount of
+ * bytes used to store this entry (entry = key + associated value + trailing
+ * free space if any). */
+static unsigned int zipmapRawEntryLength(unsigned char *p) {
+ unsigned int l = zipmapRawKeyLength(p);
+ return l + zipmapRawValueLength(p+l);
+}
+
+static inline unsigned char *zipmapResize(unsigned char *zm, unsigned int len) {
+ zm = zrealloc(zm, len);
+ zm[len-1] = ZIPMAP_END;
+ return zm;
+}
+
+/* Set key to value, creating the key if it does not already exist.
+ * If 'update' is not NULL, *update is set to 1 if the key was
+ * already preset, otherwise to 0. */
+unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned char *val, unsigned int vlen, int *update) {
+ unsigned int zmlen, offset;
+ unsigned int freelen, reqlen = zipmapRequiredLength(klen,vlen);
+ unsigned int empty, vempty;
+ unsigned char *p;
+
+ freelen = reqlen;
+ if (update) *update = 0;
+ p = zipmapLookupRaw(zm,key,klen,&zmlen);
+ if (p == NULL) {
+ /* Key not found: enlarge */
+ zm = zipmapResize(zm, zmlen+reqlen);
+ p = zm+zmlen-1;
+ zmlen = zmlen+reqlen;
+
+ /* Increase zipmap length (this is an insert) */
+ if (zm[0] < ZIPMAP_BIGLEN) zm[0]++;
+ } else {
+ /* Key found. Is there enough space for the new value? */
+ /* Compute the total length: */
+ if (update) *update = 1;
+ freelen = zipmapRawEntryLength(p);
+ if (freelen < reqlen) {
+ /* Store the offset of this key within the current zipmap, so
+ * it can be resized. Then, move the tail backwards so this
+ * pair fits at the current position. */
+ offset = p-zm;
+ zm = zipmapResize(zm, zmlen-freelen+reqlen);
+ p = zm+offset;
+
+ /* The +1 in the number of bytes to be moved is caused by the
+ * end-of-zipmap byte. Note: the *original* zmlen is used. */
+ memmove(p+reqlen, p+freelen, zmlen-(offset+freelen+1));
+ zmlen = zmlen-freelen+reqlen;
+ freelen = reqlen;
+ }
+ }
+
+ /* We now have a suitable block where the key/value entry can
+ * be written. If there is too much free space, move the tail
+ * of the zipmap a few bytes to the front and shrink the zipmap,
+ * as we want zipmaps to be very space efficient. */
+ empty = freelen-reqlen;
+ if (empty >= ZIPMAP_VALUE_MAX_FREE) {
+ /* First, move the tail <empty> bytes to the front, then resize
+ * the zipmap to be <empty> bytes smaller. */
+ offset = p-zm;
+ memmove(p+reqlen, p+freelen, zmlen-(offset+freelen+1));
+ zmlen -= empty;
+ zm = zipmapResize(zm, zmlen);
+ p = zm+offset;
+ vempty = 0;
+ } else {
+ vempty = empty;
+ }
+
+ /* Just write the key + value and we are done. */
+ /* Key: */
+ p += zipmapEncodeLength(p,klen);
+ memcpy(p,key,klen);
+ p += klen;
+ /* Value: */
+ p += zipmapEncodeLength(p,vlen);
+ *p++ = vempty;
+ memcpy(p,val,vlen);
+ return zm;
+}
+
+/* Remove the specified key. If 'deleted' is not NULL the pointed integer is
+ * set to 0 if the key was not found, to 1 if it was found and deleted. */
+unsigned char *zipmapDel(unsigned char *zm, unsigned char *key, unsigned int klen, int *deleted) {
+ unsigned int zmlen, freelen;
+ unsigned char *p = zipmapLookupRaw(zm,key,klen,&zmlen);
+ if (p) {
+ freelen = zipmapRawEntryLength(p);
+ memmove(p, p+freelen, zmlen-((p-zm)+freelen+1));
+ zm = zipmapResize(zm, zmlen-freelen);
+
+ /* Decrease zipmap length */
+ if (zm[0] < ZIPMAP_BIGLEN) zm[0]--;
+
+ if (deleted) *deleted = 1;
+ } else {
+ if (deleted) *deleted = 0;
+ }
+ return zm;
+}
+
+/* Call it before to iterate trought elements via zipmapNext() */
+unsigned char *zipmapRewind(unsigned char *zm) {
+ return zm+1;
+}
+
+/* This function is used to iterate through all the zipmap elements.
+ * In the first call the first argument is the pointer to the zipmap + 1.
+ * In the next calls what zipmapNext returns is used as first argument.
+ * Example:
+ *
+ * unsigned char *i = zipmapRewind(my_zipmap);
+ * while((i = zipmapNext(i,&key,&klen,&value,&vlen)) != NULL) {
+ * printf("%d bytes key at $p\n", klen, key);
+ * printf("%d bytes value at $p\n", vlen, value);
+ * }
+ */
+unsigned char *zipmapNext(unsigned char *zm, unsigned char **key, unsigned int *klen, unsigned char **value, unsigned int *vlen) {
+ if (zm[0] == ZIPMAP_END) return NULL;
+ if (key) {
+ *key = zm;
+ *klen = zipmapDecodeLength(zm);
+ *key += ZIPMAP_LEN_BYTES(*klen);
+ }
+ zm += zipmapRawKeyLength(zm);
+ if (value) {
+ *value = zm+1;
+ *vlen = zipmapDecodeLength(zm);
+ *value += ZIPMAP_LEN_BYTES(*vlen);
+ }
+ zm += zipmapRawValueLength(zm);
+ return zm;
+}
+
+/* Search a key and retrieve the pointer and len of the associated value.
+ * If the key is found the function returns 1, otherwise 0. */
+int zipmapGet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned char **value, unsigned int *vlen) {
+ unsigned char *p;
+
+ if ((p = zipmapLookupRaw(zm,key,klen,NULL)) == NULL) return 0;
+ p += zipmapRawKeyLength(p);
+ *vlen = zipmapDecodeLength(p);
+ *value = p + ZIPMAP_LEN_BYTES(*vlen) + 1;
+ return 1;
+}
+
+/* Return 1 if the key exists, otherwise 0 is returned. */
+int zipmapExists(unsigned char *zm, unsigned char *key, unsigned int klen) {
+ return zipmapLookupRaw(zm,key,klen,NULL) != NULL;
+}
+
+/* Return the number of entries inside a zipmap */
+unsigned int zipmapLen(unsigned char *zm) {
+ unsigned int len = 0;
+ if (zm[0] < ZIPMAP_BIGLEN) {
+ len = zm[0];
+ } else {
+ unsigned char *p = zipmapRewind(zm);
+ while((p = zipmapNext(p,NULL,NULL,NULL,NULL)) != NULL) len++;
+
+ /* Re-store length if small enough */
+ if (len < ZIPMAP_BIGLEN) zm[0] = len;
+ }
+ return len;
+}
+
+void zipmapRepr(unsigned char *p) {
+ unsigned int l;
+
+ printf("{status %u}",*p++);
+ while(1) {
+ if (p[0] == ZIPMAP_END) {
+ printf("{end}");
+ break;
+ } else {
+ unsigned char e;
+
+ l = zipmapDecodeLength(p);
+ printf("{key %u}",l);
+ p += zipmapEncodeLength(NULL,l);
+ fwrite(p,l,1,stdout);
+ p += l;
+
+ l = zipmapDecodeLength(p);
+ printf("{value %u}",l);
+ p += zipmapEncodeLength(NULL,l);
+ e = *p++;
+ fwrite(p,l,1,stdout);
+ p += l+e;
+ if (e) {
+ printf("[");
+ while(e--) printf(".");
+ printf("]");
+ }
+ }
+ }
+ printf("\n");
+}
+
+#ifdef ZIPMAP_TEST_MAIN
+int main(void) {
+ unsigned char *zm;
+
+ zm = zipmapNew();
+
+ zm = zipmapSet(zm,(unsigned char*) "name",4, (unsigned char*) "foo",3,NULL);
+ zm = zipmapSet(zm,(unsigned char*) "surname",7, (unsigned char*) "foo",3,NULL);
+ zm = zipmapSet(zm,(unsigned char*) "age",3, (unsigned char*) "foo",3,NULL);
+ zipmapRepr(zm);
+
+ zm = zipmapSet(zm,(unsigned char*) "hello",5, (unsigned char*) "world!",6,NULL);
+ zm = zipmapSet(zm,(unsigned char*) "foo",3, (unsigned char*) "bar",3,NULL);
+ zm = zipmapSet(zm,(unsigned char*) "foo",3, (unsigned char*) "!",1,NULL);
+ zipmapRepr(zm);
+ zm = zipmapSet(zm,(unsigned char*) "foo",3, (unsigned char*) "12345",5,NULL);
+ zipmapRepr(zm);
+ zm = zipmapSet(zm,(unsigned char*) "new",3, (unsigned char*) "xx",2,NULL);
+ zm = zipmapSet(zm,(unsigned char*) "noval",5, (unsigned char*) "",0,NULL);
+ zipmapRepr(zm);
+ zm = zipmapDel(zm,(unsigned char*) "new",3,NULL);
+ zipmapRepr(zm);
+
+ printf("\nLook up large key:\n");
+ {
+ unsigned char buf[512];
+ unsigned char *value;
+ unsigned int vlen, i;
+ for (i = 0; i < 512; i++) buf[i] = 'a';
+
+ zm = zipmapSet(zm,buf,512,(unsigned char*) "long",4,NULL);
+ if (zipmapGet(zm,buf,512,&value,&vlen)) {
+ printf(" <long key> is associated to the %d bytes value: %.*s\n",
+ vlen, vlen, value);
+ }
+ }
+
+ printf("\nPerform a direct lookup:\n");
+ {
+ unsigned char *value;
+ unsigned int vlen;
+
+ if (zipmapGet(zm,(unsigned char*) "foo",3,&value,&vlen)) {
+ printf(" foo is associated to the %d bytes value: %.*s\n",
+ vlen, vlen, value);
+ }
+ }
+ printf("\nIterate trought elements:\n");
+ {
+ unsigned char *i = zipmapRewind(zm);
+ unsigned char *key, *value;
+ unsigned int klen, vlen;
+
+ while((i = zipmapNext(i,&key,&klen,&value,&vlen)) != NULL) {
+ printf(" %d:%.*s => %d:%.*s\n", klen, klen, key, vlen, vlen, value);
+ }
+ }
+ return 0;
+}
+#endif
projects / redis.git / blobdiff