1 /* Hash Tables Implementation.
3 * This file implements in memory hash tables with insert/del/replace/find/
4 * get-random-element operations. Hash tables will auto resize if needed
5 * tables of power of two in size are used, collisions are handled by
6 * chaining. See the source code for more information... :)
8 * Copyright (c) 2006-2012, Salvatore Sanfilippo <antirez at gmail dot com>
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions are met:
14 * * Redistributions of source code must retain the above copyright notice,
15 * this list of conditions and the following disclaimer.
16 * * Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * * Neither the name of Redis nor the names of its contributors may be used
20 * to endorse or promote products derived from this software without
21 * specific prior written permission.
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33 * POSSIBILITY OF SUCH DAMAGE.
50 /* Using dictEnableResize() / dictDisableResize() we make possible to
51 * enable/disable resizing of the hash table as needed. This is very important
52 * for Redis, as we use copy-on-write and don't want to move too much memory
53 * around when there is a child performing saving operations.
55 * Note that even when dict_can_resize is set to 0, not all resizes are
56 * prevented: an hash table is still allowed to grow if the ratio between
57 * the number of elements and the buckets > dict_force_resize_ratio. */
58 static int dict_can_resize
= 1;
59 static unsigned int dict_force_resize_ratio
= 5;
61 /* -------------------------- private prototypes ---------------------------- */
63 static int _dictExpandIfNeeded(dict
*ht
);
64 static unsigned long _dictNextPower(unsigned long size
);
65 static int _dictKeyIndex(dict
*ht
, const void *key
);
66 static int _dictInit(dict
*ht
, dictType
*type
, void *privDataPtr
);
68 /* -------------------------- hash functions -------------------------------- */
70 /* Thomas Wang's 32 bit Mix Function */
71 unsigned int dictIntHashFunction(unsigned int key
)
82 /* Identity hash function for integer keys */
83 unsigned int dictIdentityHashFunction(unsigned int key
)
88 static uint32_t dict_hash_function_seed
= 5381;
90 void dictSetHashFunctionSeed(uint32_t seed
) {
91 dict_hash_function_seed
= seed
;
94 uint32_t dictGetHashFunctionSeed(void) {
95 return dict_hash_function_seed
;
98 /* MurmurHash2, by Austin Appleby
99 * Note - This code makes a few assumptions about how your machine behaves -
100 * 1. We can read a 4-byte value from any address without crashing
101 * 2. sizeof(int) == 4
103 * And it has a few limitations -
105 * 1. It will not work incrementally.
106 * 2. It will not produce the same results on little-endian and big-endian
109 unsigned int dictGenHashFunction(const void *key
, int len
) {
110 /* 'm' and 'r' are mixing constants generated offline.
111 They're not really 'magic', they just happen to work well. */
112 uint32_t seed
= dict_hash_function_seed
;
113 const uint32_t m
= 0x5bd1e995;
116 /* Initialize the hash to a 'random' value */
117 uint32_t h
= seed
^ len
;
119 /* Mix 4 bytes at a time into the hash */
120 const unsigned char *data
= (const unsigned char *)key
;
123 uint32_t k
= *(uint32_t*)data
;
136 /* Handle the last few bytes of the input array */
138 case 3: h
^= data
[2] << 16;
139 case 2: h
^= data
[1] << 8;
140 case 1: h
^= data
[0]; h
*= m
;
143 /* Do a few final mixes of the hash to ensure the last few
144 * bytes are well-incorporated. */
149 return (unsigned int)h
;
152 /* And a case insensitive hash function (based on djb hash) */
153 unsigned int dictGenCaseHashFunction(const unsigned char *buf
, int len
) {
154 unsigned int hash
= (unsigned int)dict_hash_function_seed
;
157 hash
= ((hash
<< 5) + hash
) + (tolower(*buf
++)); /* hash * 33 + c */
161 /* ----------------------------- API implementation ------------------------- */
163 /* Reset a hash table already initialized with ht_init().
164 * NOTE: This function should only be called by ht_destroy(). */
165 static void _dictReset(dictht
*ht
)
173 /* Create a new hash table */
174 dict
*dictCreate(dictType
*type
,
177 dict
*d
= zmalloc(sizeof(*d
));
179 _dictInit(d
,type
,privDataPtr
);
183 /* Initialize the hash table */
184 int _dictInit(dict
*d
, dictType
*type
,
187 _dictReset(&d
->ht
[0]);
188 _dictReset(&d
->ht
[1]);
190 d
->privdata
= privDataPtr
;
196 /* Resize the table to the minimal size that contains all the elements,
197 * but with the invariant of a USED/BUCKETS ratio near to <= 1 */
198 int dictResize(dict
*d
)
202 if (!dict_can_resize
|| dictIsRehashing(d
)) return DICT_ERR
;
203 minimal
= d
->ht
[0].used
;
204 if (minimal
< DICT_HT_INITIAL_SIZE
)
205 minimal
= DICT_HT_INITIAL_SIZE
;
206 return dictExpand(d
, minimal
);
209 /* Expand or create the hash table */
210 int dictExpand(dict
*d
, unsigned long size
)
212 dictht n
; /* the new hash table */
213 unsigned long realsize
= _dictNextPower(size
);
215 /* the size is invalid if it is smaller than the number of
216 * elements already inside the hash table */
217 if (dictIsRehashing(d
) || d
->ht
[0].used
> size
)
220 /* Allocate the new hash table and initialize all pointers to NULL */
222 n
.sizemask
= realsize
-1;
223 n
.table
= zcalloc(realsize
*sizeof(dictEntry
*));
226 /* Is this the first initialization? If so it's not really a rehashing
227 * we just set the first hash table so that it can accept keys. */
228 if (d
->ht
[0].table
== NULL
) {
233 /* Prepare a second hash table for incremental rehashing */
239 /* Performs N steps of incremental rehashing. Returns 1 if there are still
240 * keys to move from the old to the new hash table, otherwise 0 is returned.
241 * Note that a rehashing step consists in moving a bucket (that may have more
242 * thank one key as we use chaining) from the old to the new hash table. */
243 int dictRehash(dict
*d
, int n
) {
244 if (!dictIsRehashing(d
)) return 0;
247 dictEntry
*de
, *nextde
;
249 /* Check if we already rehashed the whole table... */
250 if (d
->ht
[0].used
== 0) {
251 zfree(d
->ht
[0].table
);
253 _dictReset(&d
->ht
[1]);
258 /* Note that rehashidx can't overflow as we are sure there are more
259 * elements because ht[0].used != 0 */
260 assert(d
->ht
[0].size
> (unsigned)d
->rehashidx
);
261 while(d
->ht
[0].table
[d
->rehashidx
] == NULL
) d
->rehashidx
++;
262 de
= d
->ht
[0].table
[d
->rehashidx
];
263 /* Move all the keys in this bucket from the old to the new hash HT */
268 /* Get the index in the new hash table */
269 h
= dictHashKey(d
, de
->key
) & d
->ht
[1].sizemask
;
270 de
->next
= d
->ht
[1].table
[h
];
271 d
->ht
[1].table
[h
] = de
;
276 d
->ht
[0].table
[d
->rehashidx
] = NULL
;
282 long long timeInMilliseconds(void) {
285 gettimeofday(&tv
,NULL
);
286 return (((long long)tv
.tv_sec
)*1000)+(tv
.tv_usec
/1000);
289 /* Rehash for an amount of time between ms milliseconds and ms+1 milliseconds */
290 int dictRehashMilliseconds(dict
*d
, int ms
) {
291 long long start
= timeInMilliseconds();
294 while(dictRehash(d
,100)) {
296 if (timeInMilliseconds()-start
> ms
) break;
301 /* This function performs just a step of rehashing, and only if there are
302 * no safe iterators bound to our hash table. When we have iterators in the
303 * middle of a rehashing we can't mess with the two hash tables otherwise
304 * some element can be missed or duplicated.
306 * This function is called by common lookup or update operations in the
307 * dictionary so that the hash table automatically migrates from H1 to H2
308 * while it is actively used. */
309 static void _dictRehashStep(dict
*d
) {
310 if (d
->iterators
== 0) dictRehash(d
,1);
313 /* Add an element to the target hash table */
314 int dictAdd(dict
*d
, void *key
, void *val
)
316 dictEntry
*entry
= dictAddRaw(d
,key
);
318 if (!entry
) return DICT_ERR
;
319 dictSetVal(d
, entry
, val
);
323 /* Low level add. This function adds the entry but instead of setting
324 * a value returns the dictEntry structure to the user, that will make
325 * sure to fill the value field as he wishes.
327 * This function is also directly exposed to user API to be called
328 * mainly in order to store non-pointers inside the hash value, example:
330 * entry = dictAddRaw(dict,mykey);
331 * if (entry != NULL) dictSetSignedIntegerVal(entry,1000);
335 * If key already exists NULL is returned.
336 * If key was added, the hash entry is returned to be manipulated by the caller.
338 dictEntry
*dictAddRaw(dict
*d
, void *key
)
344 if (dictIsRehashing(d
)) _dictRehashStep(d
);
346 /* Get the index of the new element, or -1 if
347 * the element already exists. */
348 if ((index
= _dictKeyIndex(d
, key
)) == -1)
351 /* Allocate the memory and store the new entry */
352 ht
= dictIsRehashing(d
) ? &d
->ht
[1] : &d
->ht
[0];
353 entry
= zmalloc(sizeof(*entry
));
354 entry
->next
= ht
->table
[index
];
355 ht
->table
[index
] = entry
;
358 /* Set the hash entry fields. */
359 dictSetKey(d
, entry
, key
);
363 /* Add an element, discarding the old if the key already exists.
364 * Return 1 if the key was added from scratch, 0 if there was already an
365 * element with such key and dictReplace() just performed a value update
367 int dictReplace(dict
*d
, void *key
, void *val
)
369 dictEntry
*entry
, auxentry
;
371 /* Try to add the element. If the key
372 * does not exists dictAdd will suceed. */
373 if (dictAdd(d
, key
, val
) == DICT_OK
)
375 /* It already exists, get the entry */
376 entry
= dictFind(d
, key
);
377 /* Set the new value and free the old one. Note that it is important
378 * to do that in this order, as the value may just be exactly the same
379 * as the previous one. In this context, think to reference counting,
380 * you want to increment (set), and then decrement (free), and not the
383 dictSetVal(d
, entry
, val
);
384 dictFreeVal(d
, &auxentry
);
388 /* dictReplaceRaw() is simply a version of dictAddRaw() that always
389 * returns the hash entry of the specified key, even if the key already
390 * exists and can't be added (in that case the entry of the already
391 * existing key is returned.)
393 * See dictAddRaw() for more information. */
394 dictEntry
*dictReplaceRaw(dict
*d
, void *key
) {
395 dictEntry
*entry
= dictFind(d
,key
);
397 return entry
? entry
: dictAddRaw(d
,key
);
400 /* Search and remove an element */
401 static int dictGenericDelete(dict
*d
, const void *key
, int nofree
)
404 dictEntry
*he
, *prevHe
;
407 if (d
->ht
[0].size
== 0) return DICT_ERR
; /* d->ht[0].table is NULL */
408 if (dictIsRehashing(d
)) _dictRehashStep(d
);
409 h
= dictHashKey(d
, key
);
411 for (table
= 0; table
<= 1; table
++) {
412 idx
= h
& d
->ht
[table
].sizemask
;
413 he
= d
->ht
[table
].table
[idx
];
416 if (dictCompareKeys(d
, key
, he
->key
)) {
417 /* Unlink the element from the list */
419 prevHe
->next
= he
->next
;
421 d
->ht
[table
].table
[idx
] = he
->next
;
433 if (!dictIsRehashing(d
)) break;
435 return DICT_ERR
; /* not found */
438 int dictDelete(dict
*ht
, const void *key
) {
439 return dictGenericDelete(ht
,key
,0);
442 int dictDeleteNoFree(dict
*ht
, const void *key
) {
443 return dictGenericDelete(ht
,key
,1);
446 /* Destroy an entire dictionary */
447 int _dictClear(dict
*d
, dictht
*ht
)
451 /* Free all the elements */
452 for (i
= 0; i
< ht
->size
&& ht
->used
> 0; i
++) {
453 dictEntry
*he
, *nextHe
;
455 if ((he
= ht
->table
[i
]) == NULL
) continue;
465 /* Free the table and the allocated cache structure */
467 /* Re-initialize the table */
469 return DICT_OK
; /* never fails */
472 /* Clear & Release the hash table */
473 void dictRelease(dict
*d
)
475 _dictClear(d
,&d
->ht
[0]);
476 _dictClear(d
,&d
->ht
[1]);
480 dictEntry
*dictFind(dict
*d
, const void *key
)
483 unsigned int h
, idx
, table
;
485 if (d
->ht
[0].size
== 0) return NULL
; /* We don't have a table at all */
486 if (dictIsRehashing(d
)) _dictRehashStep(d
);
487 h
= dictHashKey(d
, key
);
488 for (table
= 0; table
<= 1; table
++) {
489 idx
= h
& d
->ht
[table
].sizemask
;
490 he
= d
->ht
[table
].table
[idx
];
492 if (dictCompareKeys(d
, key
, he
->key
))
496 if (!dictIsRehashing(d
)) return NULL
;
501 void *dictFetchValue(dict
*d
, const void *key
) {
504 he
= dictFind(d
,key
);
505 return he
? dictGetVal(he
) : NULL
;
508 dictIterator
*dictGetIterator(dict
*d
)
510 dictIterator
*iter
= zmalloc(sizeof(*iter
));
517 iter
->nextEntry
= NULL
;
521 dictIterator
*dictGetSafeIterator(dict
*d
) {
522 dictIterator
*i
= dictGetIterator(d
);
528 dictEntry
*dictNext(dictIterator
*iter
)
531 if (iter
->entry
== NULL
) {
532 dictht
*ht
= &iter
->d
->ht
[iter
->table
];
533 if (iter
->safe
&& iter
->index
== -1 && iter
->table
== 0)
534 iter
->d
->iterators
++;
536 if (iter
->index
>= (signed) ht
->size
) {
537 if (dictIsRehashing(iter
->d
) && iter
->table
== 0) {
540 ht
= &iter
->d
->ht
[1];
545 iter
->entry
= ht
->table
[iter
->index
];
547 iter
->entry
= iter
->nextEntry
;
550 /* We need to save the 'next' here, the iterator user
551 * may delete the entry we are returning. */
552 iter
->nextEntry
= iter
->entry
->next
;
559 void dictReleaseIterator(dictIterator
*iter
)
561 if (iter
->safe
&& !(iter
->index
== -1 && iter
->table
== 0))
562 iter
->d
->iterators
--;
566 /* Return a random entry from the hash table. Useful to
567 * implement randomized algorithms */
568 dictEntry
*dictGetRandomKey(dict
*d
)
570 dictEntry
*he
, *orighe
;
572 int listlen
, listele
;
574 if (dictSize(d
) == 0) return NULL
;
575 if (dictIsRehashing(d
)) _dictRehashStep(d
);
576 if (dictIsRehashing(d
)) {
578 h
= random() % (d
->ht
[0].size
+d
->ht
[1].size
);
579 he
= (h
>= d
->ht
[0].size
) ? d
->ht
[1].table
[h
- d
->ht
[0].size
] :
584 h
= random() & d
->ht
[0].sizemask
;
585 he
= d
->ht
[0].table
[h
];
589 /* Now we found a non empty bucket, but it is a linked
590 * list and we need to get a random element from the list.
591 * The only sane way to do so is counting the elements and
592 * select a random index. */
599 listele
= random() % listlen
;
601 while(listele
--) he
= he
->next
;
605 /* ------------------------- private functions ------------------------------ */
607 /* Expand the hash table if needed */
608 static int _dictExpandIfNeeded(dict
*d
)
610 /* Incremental rehashing already in progress. Return. */
611 if (dictIsRehashing(d
)) return DICT_OK
;
613 /* If the hash table is empty expand it to the intial size. */
614 if (d
->ht
[0].size
== 0) return dictExpand(d
, DICT_HT_INITIAL_SIZE
);
616 /* If we reached the 1:1 ratio, and we are allowed to resize the hash
617 * table (global setting) or we should avoid it but the ratio between
618 * elements/buckets is over the "safe" threshold, we resize doubling
619 * the number of buckets. */
620 if (d
->ht
[0].used
>= d
->ht
[0].size
&&
622 d
->ht
[0].used
/d
->ht
[0].size
> dict_force_resize_ratio
))
624 return dictExpand(d
, ((d
->ht
[0].size
> d
->ht
[0].used
) ?
625 d
->ht
[0].size
: d
->ht
[0].used
)*2);
630 /* Our hash table capability is a power of two */
631 static unsigned long _dictNextPower(unsigned long size
)
633 unsigned long i
= DICT_HT_INITIAL_SIZE
;
635 if (size
>= LONG_MAX
) return LONG_MAX
;
643 /* Returns the index of a free slot that can be populated with
644 * an hash entry for the given 'key'.
645 * If the key already exists, -1 is returned.
647 * Note that if we are in the process of rehashing the hash table, the
648 * index is always returned in the context of the second (new) hash table. */
649 static int _dictKeyIndex(dict
*d
, const void *key
)
651 unsigned int h
, idx
, table
;
654 /* Expand the hash table if needed */
655 if (_dictExpandIfNeeded(d
) == DICT_ERR
)
657 /* Compute the key hash value */
658 h
= dictHashKey(d
, key
);
659 for (table
= 0; table
<= 1; table
++) {
660 idx
= h
& d
->ht
[table
].sizemask
;
661 /* Search if this slot does not already contain the given key */
662 he
= d
->ht
[table
].table
[idx
];
664 if (dictCompareKeys(d
, key
, he
->key
))
668 if (!dictIsRehashing(d
)) break;
673 void dictEmpty(dict
*d
) {
674 _dictClear(d
,&d
->ht
[0]);
675 _dictClear(d
,&d
->ht
[1]);
680 void dictEnableResize(void) {
684 void dictDisableResize(void) {
690 /* The following is code that we don't use for Redis currently, but that is part
693 /* ----------------------- Debugging ------------------------*/
695 #define DICT_STATS_VECTLEN 50
696 static void _dictPrintStatsHt(dictht
*ht
) {
697 unsigned long i
, slots
= 0, chainlen
, maxchainlen
= 0;
698 unsigned long totchainlen
= 0;
699 unsigned long clvector
[DICT_STATS_VECTLEN
];
702 printf("No stats available for empty dictionaries\n");
706 for (i
= 0; i
< DICT_STATS_VECTLEN
; i
++) clvector
[i
] = 0;
707 for (i
= 0; i
< ht
->size
; i
++) {
710 if (ht
->table
[i
] == NULL
) {
715 /* For each hash entry on this slot... */
722 clvector
[(chainlen
< DICT_STATS_VECTLEN
) ? chainlen
: (DICT_STATS_VECTLEN
-1)]++;
723 if (chainlen
> maxchainlen
) maxchainlen
= chainlen
;
724 totchainlen
+= chainlen
;
726 printf("Hash table stats:\n");
727 printf(" table size: %ld\n", ht
->size
);
728 printf(" number of elements: %ld\n", ht
->used
);
729 printf(" different slots: %ld\n", slots
);
730 printf(" max chain length: %ld\n", maxchainlen
);
731 printf(" avg chain length (counted): %.02f\n", (float)totchainlen
/slots
);
732 printf(" avg chain length (computed): %.02f\n", (float)ht
->used
/slots
);
733 printf(" Chain length distribution:\n");
734 for (i
= 0; i
< DICT_STATS_VECTLEN
-1; i
++) {
735 if (clvector
[i
] == 0) continue;
736 printf(" %s%ld: %ld (%.02f%%)\n",(i
== DICT_STATS_VECTLEN
-1)?">= ":"", i
, clvector
[i
], ((float)clvector
[i
]/ht
->size
)*100);
740 void dictPrintStats(dict
*d
) {
741 _dictPrintStatsHt(&d
->ht
[0]);
742 if (dictIsRehashing(d
)) {
743 printf("-- Rehashing into ht[1]:\n");
744 _dictPrintStatsHt(&d
->ht
[1]);
748 /* ----------------------- StringCopy Hash Table Type ------------------------*/
750 static unsigned int _dictStringCopyHTHashFunction(const void *key
)
752 return dictGenHashFunction(key
, strlen(key
));
755 static void *_dictStringDup(void *privdata
, const void *key
)
757 int len
= strlen(key
);
758 char *copy
= zmalloc(len
+1);
759 DICT_NOTUSED(privdata
);
761 memcpy(copy
, key
, len
);
766 static int _dictStringCopyHTKeyCompare(void *privdata
, const void *key1
,
769 DICT_NOTUSED(privdata
);
771 return strcmp(key1
, key2
) == 0;
774 static void _dictStringDestructor(void *privdata
, void *key
)
776 DICT_NOTUSED(privdata
);
781 dictType dictTypeHeapStringCopyKey
= {
782 _dictStringCopyHTHashFunction
, /* hash function */
783 _dictStringDup
, /* key dup */
785 _dictStringCopyHTKeyCompare
, /* key compare */
786 _dictStringDestructor
, /* key destructor */
787 NULL
/* val destructor */
790 /* This is like StringCopy but does not auto-duplicate the key.
791 * It's used for intepreter's shared strings. */
792 dictType dictTypeHeapStrings
= {
793 _dictStringCopyHTHashFunction
, /* hash function */
796 _dictStringCopyHTKeyCompare
, /* key compare */
797 _dictStringDestructor
, /* key destructor */
798 NULL
/* val destructor */
801 /* This is like StringCopy but also automatically handle dynamic
802 * allocated C strings as values. */
803 dictType dictTypeHeapStringCopyKeyValue
= {
804 _dictStringCopyHTHashFunction
, /* hash function */
805 _dictStringDup
, /* key dup */
806 _dictStringDup
, /* val dup */
807 _dictStringCopyHTKeyCompare
, /* key compare */
808 _dictStringDestructor
, /* key destructor */
809 _dictStringDestructor
, /* val destructor */