special encoding limits redefined, and all the config options for special encodings...

[redis.git] / src / dict.c

diff --git a/src/dict.c b/src/dict.c
index d5010708c37ce6729e3068a170e30029f2fa1d38..9be7fb168ed00b8b633e06a47b0b74c87e6a70fd 100644 (file)
--- a/src/dict.c
+++ b/src/dict.c
@@ -42,6 +42,7 @@
 #include <assert.h>
 #include <limits.h>
 #include <sys/time.h>
 #include <assert.h>
 #include <limits.h>
 #include <sys/time.h>

+#include <ctype.h>

 
 #include "dict.h"
 #include "zmalloc.h"
 
 #include "dict.h"
 #include "zmalloc.h"


@@ -49,35 +50,13 @@
 /* Using dictEnableResize() / dictDisableResize() we make possible to
  * enable/disable resizing of the hash table as needed. This is very important
  * for Redis, as we use copy-on-write and don't want to move too much memory
 /* Using dictEnableResize() / dictDisableResize() we make possible to
  * enable/disable resizing of the hash table as needed. This is very important
  * for Redis, as we use copy-on-write and don't want to move too much memory

- * around when there is a child performing saving operations. */
+ * around when there is a child performing saving operations.
+ *
+ * Note that even when dict_can_resize is set to 0, not all resizes are
+ * prevented: an hash table is still allowed to grow if the ratio between
+ * the number of elements and the buckets > dict_force_resize_ratio. */

 static int dict_can_resize = 1;
 static int dict_can_resize = 1;

-
-/* ---------------------------- Utility funcitons --------------------------- */
-
-static void _dictPanic(const char *fmt, ...)
-{
-    va_list ap;
-
-    va_start(ap, fmt);
-    fprintf(stderr, "\nDICT LIBRARY PANIC: ");
-    vfprintf(stderr, fmt, ap);
-    fprintf(stderr, "\n\n");
-    va_end(ap);
-}
-
-/* ------------------------- Heap Management Wrappers------------------------ */
-
-static void *_dictAlloc(size_t size)
-{
-    void *p = zmalloc(size);
-    if (p == NULL)
-        _dictPanic("Out of memory");
-    return p;
-}
-
-static void _dictFree(void *ptr) {
-    zfree(ptr);
-}
+static unsigned int dict_force_resize_ratio = 5;

 
 /* -------------------------- private prototypes ---------------------------- */
 
 
 /* -------------------------- private prototypes ---------------------------- */
 


@@ -116,6 +95,15 @@ unsigned int dictGenHashFunction(const unsigned char *buf, int len) {
     return hash;
 }
 
     return hash;
 }
 

+/* And a case insensitive version */
+unsigned int dictGenCaseHashFunction(const unsigned char *buf, int len) {
+    unsigned int hash = 5381;
+
+    while (len--)
+        hash = ((hash << 5) + hash) + (tolower(*buf++)); /* hash * 33 + c */
+    return hash;
+}
+

 /* ----------------------------- API implementation ------------------------- */
 
 /* Reset an hashtable already initialized with ht_init().
 /* ----------------------------- API implementation ------------------------- */
 
 /* Reset an hashtable already initialized with ht_init().


@@ -132,7 +120,7 @@ static void _dictReset(dictht *ht)
 dict *dictCreate(dictType *type,
         void *privDataPtr)
 {
 dict *dictCreate(dictType *type,
         void *privDataPtr)
 {

-    dict *d = _dictAlloc(sizeof(*d));
+    dict *d = zmalloc(sizeof(*d));

 
     _dictInit(d,type,privDataPtr);
     return d;
 
     _dictInit(d,type,privDataPtr);
     return d;


@@ -152,7 +140,7 @@ int _dictInit(dict *d, dictType *type,
 }
 
 /* Resize the table to the minimal size that contains all the elements,
 }
 
 /* Resize the table to the minimal size that contains all the elements,

- * but with the invariant of a USER/BUCKETS ration near to <= 1 */
+ * but with the invariant of a USER/BUCKETS ratio near to <= 1 */

 int dictResize(dict *d)
 {
     int minimal;
 int dictResize(dict *d)
 {
     int minimal;


@@ -175,14 +163,12 @@ int dictExpand(dict *d, unsigned long size)
     if (dictIsRehashing(d) || d->ht[0].used > size)
         return DICT_ERR;
 
     if (dictIsRehashing(d) || d->ht[0].used > size)
         return DICT_ERR;
 

+    /* Allocate the new hashtable and initialize all pointers to NULL */

     n.size = realsize;
     n.sizemask = realsize-1;
     n.size = realsize;
     n.sizemask = realsize-1;

-    n.table = _dictAlloc(realsize*sizeof(dictEntry*));
+    n.table = zcalloc(realsize*sizeof(dictEntry*));

     n.used = 0;
 
     n.used = 0;
 

-    /* Initialize all the pointers to NULL */
-    memset(n.table, 0, realsize*sizeof(dictEntry*));
-

     /* Is this the first initialization? If so it's not really a rehashing
      * we just set the first hash table so that it can accept keys. */
     if (d->ht[0].table == NULL) {
     /* Is this the first initialization? If so it's not really a rehashing
      * we just set the first hash table so that it can accept keys. */
     if (d->ht[0].table == NULL) {


@@ -208,7 +194,7 @@ int dictRehash(dict *d, int n) {
 
         /* Check if we already rehashed the whole table... */
         if (d->ht[0].used == 0) {
 
         /* Check if we already rehashed the whole table... */
         if (d->ht[0].used == 0) {

-            _dictFree(d->ht[0].table);
+            zfree(d->ht[0].table);

             d->ht[0] = d->ht[1];
             _dictReset(&d->ht[1]);
             d->rehashidx = -1;
             d->ht[0] = d->ht[1];
             _dictReset(&d->ht[1]);
             d->rehashidx = -1;


@@ -285,7 +271,7 @@ int dictAdd(dict *d, void *key, void *val)
 
     /* Allocates the memory and stores key */
     ht = dictIsRehashing(d) ? &d->ht[1] : &d->ht[0];
 
     /* Allocates the memory and stores key */
     ht = dictIsRehashing(d) ? &d->ht[1] : &d->ht[0];

-    entry = _dictAlloc(sizeof(*entry));
+    entry = zmalloc(sizeof(*entry));

     entry->next = ht->table[index];
     ht->table[index] = entry;
     ht->used++;
     entry->next = ht->table[index];
     ht->table[index] = entry;
     ht->used++;


@@ -348,7 +334,7 @@ static int dictGenericDelete(dict *d, const void *key, int nofree)
                     dictFreeEntryKey(d, he);
                     dictFreeEntryVal(d, he);
                 }
                     dictFreeEntryKey(d, he);
                     dictFreeEntryVal(d, he);
                 }

-                _dictFree(he);
+                zfree(he);

                 d->ht[table].used--;
                 return DICT_OK;
             }
                 d->ht[table].used--;
                 return DICT_OK;
             }


@@ -382,13 +368,13 @@ int _dictClear(dict *d, dictht *ht)
             nextHe = he->next;
             dictFreeEntryKey(d, he);
             dictFreeEntryVal(d, he);
             nextHe = he->next;
             dictFreeEntryKey(d, he);
             dictFreeEntryVal(d, he);

-            _dictFree(he);
+            zfree(he);

             ht->used--;
             he = nextHe;
         }
     }
     /* Free the table and the allocated cache structure */
             ht->used--;
             he = nextHe;
         }
     }
     /* Free the table and the allocated cache structure */

-    _dictFree(ht->table);
+    zfree(ht->table);

     /* Re-initialize the table */
     _dictReset(ht);
     return DICT_OK; /* never fails */
     /* Re-initialize the table */
     _dictReset(ht);
     return DICT_OK; /* never fails */


@@ -399,7 +385,7 @@ void dictRelease(dict *d)
 {
     _dictClear(d,&d->ht[0]);
     _dictClear(d,&d->ht[1]);
 {
     _dictClear(d,&d->ht[0]);
     _dictClear(d,&d->ht[1]);

-    _dictFree(d);
+    zfree(d);

 }
 
 dictEntry *dictFind(dict *d, const void *key)
 }
 
 dictEntry *dictFind(dict *d, const void *key)


@@ -432,7 +418,7 @@ void *dictFetchValue(dict *d, const void *key) {
 
 dictIterator *dictGetIterator(dict *d)
 {
 
 dictIterator *dictGetIterator(dict *d)
 {

-    dictIterator *iter = _dictAlloc(sizeof(*iter));
+    dictIterator *iter = zmalloc(sizeof(*iter));

 
     iter->d = d;
     iter->table = 0;
 
     iter->d = d;
     iter->table = 0;


@@ -475,7 +461,7 @@ dictEntry *dictNext(dictIterator *iter)
 void dictReleaseIterator(dictIterator *iter)
 {
     if (!(iter->index == -1 && iter->table == 0)) iter->d->iterators--;
 void dictReleaseIterator(dictIterator *iter)
 {
     if (!(iter->index == -1 && iter->table == 0)) iter->d->iterators--;

-    _dictFree(iter);
+    zfree(iter);

 }
 
 /* Return a random entry from the hash table. Useful to
 }
 
 /* Return a random entry from the hash table. Useful to


@@ -522,14 +508,23 @@ dictEntry *dictGetRandomKey(dict *d)
 /* Expand the hash table if needed */
 static int _dictExpandIfNeeded(dict *d)
 {
 /* Expand the hash table if needed */
 static int _dictExpandIfNeeded(dict *d)
 {

-    /* If the hash table is empty expand it to the intial size,
-     * if the table is "full" dobule its size. */
+    /* Incremental rehashing already in progress. Return. */

     if (dictIsRehashing(d)) return DICT_OK;
     if (dictIsRehashing(d)) return DICT_OK;

-    if (d->ht[0].size == 0)
-        return dictExpand(d, DICT_HT_INITIAL_SIZE);
-    if (d->ht[0].used >= d->ht[0].size && dict_can_resize)
+
+    /* If the hash table is empty expand it to the intial size. */
+    if (d->ht[0].size == 0) return dictExpand(d, DICT_HT_INITIAL_SIZE);
+
+    /* If we reached the 1:1 ratio, and we are allowed to resize the hash
+     * table (global setting) or we should avoid it but the ratio between
+     * elements/buckets is over the "safe" threshold, we resize doubling
+     * the number of buckets. */
+    if (d->ht[0].used >= d->ht[0].size &&
+        (dict_can_resize ||
+         d->ht[0].used/d->ht[0].size > dict_force_resize_ratio))
+    {

         return dictExpand(d, ((d->ht[0].size > d->ht[0].used) ?
                                     d->ht[0].size : d->ht[0].used)*2);
         return dictExpand(d, ((d->ht[0].size > d->ht[0].used) ?
                                     d->ht[0].size : d->ht[0].used)*2);

+    }

     return DICT_OK;
 }
 
     return DICT_OK;
 }
 


@@ -644,6 +639,12 @@ void dictDisableResize(void) {
     dict_can_resize = 0;
 }
 
     dict_can_resize = 0;
 }
 

+#if 0
+
+/* The following are just example hash table types implementations.
+ * Not useful for Redis so they are commented out.
+ */
+

 /* ----------------------- StringCopy Hash Table Type ------------------------*/
 
 static unsigned int _dictStringCopyHTHashFunction(const void *key)
 /* ----------------------- StringCopy Hash Table Type ------------------------*/
 
 static unsigned int _dictStringCopyHTHashFunction(const void *key)


@@ -651,10 +652,10 @@ static unsigned int _dictStringCopyHTHashFunction(const void *key)
     return dictGenHashFunction(key, strlen(key));
 }
 
     return dictGenHashFunction(key, strlen(key));
 }
 

-static void *_dictStringCopyHTKeyDup(void *privdata, const void *key)
+static void *_dictStringDup(void *privdata, const void *key)

 {
     int len = strlen(key);
 {
     int len = strlen(key);

-    char *copy = _dictAlloc(len+1);
+    char *copy = zmalloc(len+1);

     DICT_NOTUSED(privdata);
 
     memcpy(copy, key, len);
     DICT_NOTUSED(privdata);
 
     memcpy(copy, key, len);


@@ -662,17 +663,6 @@ static void *_dictStringCopyHTKeyDup(void *privdata, const void *key)
     return copy;
 }
 
     return copy;
 }
 

-static void *_dictStringKeyValCopyHTValDup(void *privdata, const void *val)
-{
-    int len = strlen(val);
-    char *copy = _dictAlloc(len+1);
-    DICT_NOTUSED(privdata);
-
-    memcpy(copy, val, len);
-    copy[len] = '\0';
-    return copy;
-}
-

 static int _dictStringCopyHTKeyCompare(void *privdata, const void *key1,
         const void *key2)
 {
 static int _dictStringCopyHTKeyCompare(void *privdata, const void *key1,
         const void *key2)
 {


@@ -681,47 +671,41 @@ static int _dictStringCopyHTKeyCompare(void *privdata, const void *key1,
     return strcmp(key1, key2) == 0;
 }
 
     return strcmp(key1, key2) == 0;
 }
 

-static void _dictStringCopyHTKeyDestructor(void *privdata, void *key)
-{
-    DICT_NOTUSED(privdata);
-
-    _dictFree((void*)key); /* ATTENTION: const cast */
-}
-
-static void _dictStringKeyValCopyHTValDestructor(void *privdata, void *val)
+static void _dictStringDestructor(void *privdata, void *key)

 {
     DICT_NOTUSED(privdata);
 
 {
     DICT_NOTUSED(privdata);
 

-    _dictFree((void*)val); /* ATTENTION: const cast */
+    zfree(key);

 }
 
 dictType dictTypeHeapStringCopyKey = {
 }
 
 dictType dictTypeHeapStringCopyKey = {

-    _dictStringCopyHTHashFunction,        /* hash function */
-    _dictStringCopyHTKeyDup,              /* key dup */
-    NULL,                               /* val dup */
-    _dictStringCopyHTKeyCompare,          /* key compare */
-    _dictStringCopyHTKeyDestructor,       /* key destructor */
-    NULL                                /* val destructor */
+    _dictStringCopyHTHashFunction, /* hash function */
+    _dictStringDup,                /* key dup */
+    NULL,                          /* val dup */
+    _dictStringCopyHTKeyCompare,   /* key compare */
+    _dictStringDestructor,         /* key destructor */
+    NULL                           /* val destructor */

 };
 
 /* This is like StringCopy but does not auto-duplicate the key.
  * It's used for intepreter's shared strings. */
 dictType dictTypeHeapStrings = {
 };
 
 /* This is like StringCopy but does not auto-duplicate the key.
  * It's used for intepreter's shared strings. */
 dictType dictTypeHeapStrings = {

-    _dictStringCopyHTHashFunction,        /* hash function */
-    NULL,                               /* key dup */
-    NULL,                               /* val dup */
-    _dictStringCopyHTKeyCompare,          /* key compare */
-    _dictStringCopyHTKeyDestructor,       /* key destructor */
-    NULL                                /* val destructor */
+    _dictStringCopyHTHashFunction, /* hash function */
+    NULL,                          /* key dup */
+    NULL,                          /* val dup */
+    _dictStringCopyHTKeyCompare,   /* key compare */
+    _dictStringDestructor,         /* key destructor */
+    NULL                           /* val destructor */

 };
 
 /* This is like StringCopy but also automatically handle dynamic
  * allocated C strings as values. */
 dictType dictTypeHeapStringCopyKeyValue = {
 };
 
 /* This is like StringCopy but also automatically handle dynamic
  * allocated C strings as values. */
 dictType dictTypeHeapStringCopyKeyValue = {

-    _dictStringCopyHTHashFunction,        /* hash function */
-    _dictStringCopyHTKeyDup,              /* key dup */
-    _dictStringKeyValCopyHTValDup,        /* val dup */
-    _dictStringCopyHTKeyCompare,          /* key compare */
-    _dictStringCopyHTKeyDestructor,       /* key destructor */
-    _dictStringKeyValCopyHTValDestructor, /* val destructor */
+    _dictStringCopyHTHashFunction, /* hash function */
+    _dictStringDup,                /* key dup */
+    _dictStringDup,                /* val dup */
+    _dictStringCopyHTKeyCompare,   /* key compare */
+    _dictStringDestructor,         /* key destructor */
+    _dictStringDestructor,         /* val destructor */

 };
 };

+#endif