]>
git.saurik.com Git - redis.git/blob - ziplist.c
08253f37e63d7f6cf5d0e2dcbcbf426aedf59e8c
1 /* Memory layout of a ziplist, containing "foo", "bar", "quux":
2 * <zlbytes><zllen><len>"foo"<len>"bar"<len>"quux"
4 * <zlbytes> is an unsigned integer to hold the number of bytes that
5 * the ziplist occupies. This is stored to not have to traverse the ziplist
6 * to know the new length when pushing.
8 * <zllen> is the number of items in the ziplist. When this value is
9 * greater than 254, we need to traverse the entire list to know
10 * how many items it holds.
12 * <len> is the number of bytes occupied by a single entry. When this
13 * number is greater than 253, the length will occupy 5 bytes, where
14 * the extra bytes contain an unsigned integer to hold the length.
26 /* Important note: the ZIP_END value is used to depict the end of the
27 * ziplist structure. When a pointer contains an entry, the first couple
28 * of bytes contain the encoded length of the previous entry. This length
29 * is encoded as ZIP_ENC_RAW length, so the first two bits will contain 00
30 * and the byte will therefore never have a value of 255. */
32 #define ZIP_BIGLEN 254
36 #define ZIP_ENC_SHORT 1
38 #define ZIP_ENC_LLONG 3
39 #define ZIP_ENCODING(p) ((p)[0] >> 6)
41 /* Length encoding for raw entries */
42 #define ZIP_LEN_INLINE 0
43 #define ZIP_LEN_UINT16 1
44 #define ZIP_LEN_UINT32 2
47 #define ZIPLIST_BYTES(zl) (*((unsigned int*)(zl)))
48 #define ZIPLIST_TAIL_OFFSET(zl) (*((zl)+sizeof(unsigned int)))
49 #define ZIPLIST_LENGTH(zl) (*((zl)+2*sizeof(unsigned int)))
50 #define ZIPLIST_HEADER_SIZE (2*sizeof(unsigned int)+1)
51 #define ZIPLIST_INCR_LENGTH(zl,incr) { \
52 if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) ZIPLIST_LENGTH(zl)+=incr; }
54 typedef struct zlentry
{
55 unsigned int prevrawlensize
, prevrawlen
;
56 unsigned int lensize
, len
;
57 unsigned int headersize
;
58 unsigned char encoding
;
61 /* Return bytes needed to store integer encoded by 'encoding' */
62 static unsigned int zipEncodingSize(char encoding
) {
63 if (encoding
== ZIP_ENC_SHORT
) {
64 return sizeof(short int);
65 } else if (encoding
== ZIP_ENC_INT
) {
67 } else if (encoding
== ZIP_ENC_LLONG
) {
68 return sizeof(long long);
73 /* Decode the encoded length pointed by 'p'. If a pointer to 'lensize' is
74 * provided, it is set to the number of bytes required to encode the length. */
75 static unsigned int zipDecodeLength(unsigned char *p
, unsigned int *lensize
) {
76 unsigned char encoding
= ZIP_ENCODING(p
), lenenc
;
79 if (encoding
== ZIP_ENC_RAW
) {
80 lenenc
= (p
[0] >> 4) & 0x3;
81 if (lenenc
== ZIP_LEN_INLINE
) {
83 if (lensize
) *lensize
= 1;
84 } else if (lenenc
== ZIP_LEN_UINT16
) {
85 len
= p
[1] | (p
[2] << 8);
86 if (lensize
) *lensize
= 3;
88 len
= p
[1] | (p
[2] << 8) | (p
[3] << 16) | (p
[4] << 24);
89 if (lensize
) *lensize
= 5;
92 len
= zipEncodingSize(encoding
);
93 if (lensize
) *lensize
= 1;
98 /* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
99 * the amount of bytes required to encode such a length. */
100 static unsigned int zipEncodeLength(unsigned char *p
, char encoding
, unsigned int rawlen
) {
101 unsigned char len
= 1, lenenc
, buf
[5];
102 if (encoding
== ZIP_ENC_RAW
) {
105 lenenc
= ZIP_LEN_INLINE
;
107 } else if (rawlen
<= 0xffff) {
110 lenenc
= ZIP_LEN_UINT16
;
111 buf
[1] = (rawlen
) & 0xff;
112 buf
[2] = (rawlen
>> 8) & 0xff;
116 lenenc
= ZIP_LEN_UINT32
;
117 buf
[1] = (rawlen
) & 0xff;
118 buf
[2] = (rawlen
>> 8) & 0xff;
119 buf
[3] = (rawlen
>> 16) & 0xff;
120 buf
[4] = (rawlen
>> 24) & 0xff;
122 buf
[0] = (lenenc
<< 4) | (buf
[0] & 0xf);
126 /* Apparently we need to store the length in 'p' */
127 buf
[0] = (encoding
<< 6) | (buf
[0] & 0x3f);
132 /* Return the difference in number of bytes needed to store the new length
133 * "len" on the entry pointed to by "p". */
134 static int zipPrevLenByteDiff(unsigned char *p
, unsigned int len
) {
135 unsigned int prevlensize
;
136 zipDecodeLength(p
,&prevlensize
);
137 return zipEncodeLength(NULL
,ZIP_ENC_RAW
,len
)-prevlensize
;
140 /* Check if string pointed to by 'entry' can be encoded as an integer.
141 * Stores the integer value in 'v' and its encoding in 'encoding'.
142 * Warning: this function requires a NULL-terminated string! */
143 static int zipTryEncoding(unsigned char *entry
, long long *v
, char *encoding
) {
147 if (entry
[0] == '-' || (entry
[0] >= '0' && entry
[0] <= '9')) {
148 value
= strtoll((char*)entry
,&eptr
,10);
149 if (eptr
[0] != '\0') return 0;
150 if (value
>= SHRT_MIN
&& value
<= SHRT_MAX
) {
151 *encoding
= ZIP_ENC_SHORT
;
152 } else if (value
>= INT_MIN
&& value
<= INT_MAX
) {
153 *encoding
= ZIP_ENC_INT
;
155 *encoding
= ZIP_ENC_LLONG
;
163 /* Store integer 'value' at 'p', encoded as 'encoding' */
164 static void zipSaveInteger(unsigned char *p
, long long value
, char encoding
) {
168 if (encoding
== ZIP_ENC_SHORT
) {
170 memcpy(p
,&s
,sizeof(s
));
171 } else if (encoding
== ZIP_ENC_INT
) {
173 memcpy(p
,&i
,sizeof(i
));
174 } else if (encoding
== ZIP_ENC_LLONG
) {
176 memcpy(p
,&l
,sizeof(l
));
182 /* Read integer encoded as 'encoding' from 'p' */
183 static long long zipLoadInteger(unsigned char *p
, char encoding
) {
187 if (encoding
== ZIP_ENC_SHORT
) {
188 memcpy(&s
,p
,sizeof(s
));
190 } else if (encoding
== ZIP_ENC_INT
) {
191 memcpy(&i
,p
,sizeof(i
));
193 } else if (encoding
== ZIP_ENC_LLONG
) {
194 memcpy(&l
,p
,sizeof(l
));
202 /* Return a struct with all information about an entry. */
203 static zlentry
zipEntry(unsigned char *p
) {
205 e
.prevrawlen
= zipDecodeLength(p
,&e
.prevrawlensize
);
206 e
.len
= zipDecodeLength(p
+e
.prevrawlensize
,&e
.lensize
);
207 e
.headersize
= e
.prevrawlensize
+e
.lensize
;
208 e
.encoding
= ZIP_ENCODING(p
+e
.prevrawlensize
);
212 /* Return the total amount used by an entry (encoded length + payload). */
213 static unsigned int zipRawEntryLength(unsigned char *p
) {
214 unsigned int prevlensize
, lensize
, len
;
215 /* Byte-size of encoded length of previous entry */
216 zipDecodeLength(p
,&prevlensize
);
217 /* Encoded length of this entry's payload */
218 len
= zipDecodeLength(p
+prevlensize
, &lensize
);
219 return prevlensize
+lensize
+len
;
222 /* Create a new empty ziplist. */
223 unsigned char *ziplistNew(void) {
224 unsigned int bytes
= ZIPLIST_HEADER_SIZE
+1;
225 unsigned char *zl
= zmalloc(bytes
);
226 ZIPLIST_BYTES(zl
) = bytes
;
227 ZIPLIST_TAIL_OFFSET(zl
) = ZIPLIST_HEADER_SIZE
;
228 ZIPLIST_LENGTH(zl
) = 0;
229 zl
[bytes
-1] = ZIP_END
;
233 /* Resize the ziplist. */
234 static unsigned char *ziplistResize(unsigned char *zl
, unsigned int len
) {
235 zl
= zrealloc(zl
,len
);
236 ZIPLIST_BYTES(zl
) = len
;
241 static unsigned char *ziplistHead(unsigned char *zl
) {
242 return zl
+ZIPLIST_HEADER_SIZE
;
245 static unsigned char *ziplistTail(unsigned char *zl
) {
246 unsigned char *p
, *q
;
247 p
= q
= ziplistHead(zl
);
248 while (*p
!= ZIP_END
) {
250 p
+= zipRawEntryLength(p
);
255 unsigned char *ziplistPush(unsigned char *zl
, unsigned char *entry
, unsigned int elen
, int where
) {
256 unsigned int curlen
= ZIPLIST_BYTES(zl
), reqlen
, prevlen
;
257 unsigned char *p
, *curtail
;
258 char encoding
= ZIP_ENC_RAW
;
261 /* We need to store the length of the current tail when the list
262 * is non-empty and we push at the tail. */
263 curtail
= zl
+ZIPLIST_TAIL_OFFSET(zl
);
264 if (where
== ZIPLIST_TAIL
&& curtail
[0] != ZIP_END
) {
265 prevlen
= zipRawEntryLength(curtail
);
270 /* See if the entry can be encoded */
271 if (zipTryEncoding(entry
,&value
,&encoding
)) {
272 reqlen
= zipEncodingSize(encoding
);
277 /* We need space for both the length of the previous entry and
278 * the length of the payload. */
279 reqlen
+= zipEncodeLength(NULL
,ZIP_ENC_RAW
,prevlen
);
280 reqlen
+= zipEncodeLength(NULL
,encoding
,elen
);
282 /* Resize the ziplist and move if needed */
283 zl
= ziplistResize(zl
,curlen
+reqlen
);
284 if (where
== ZIPLIST_HEAD
) {
285 p
= zl
+ZIPLIST_HEADER_SIZE
;
287 /* Subtract one because of the ZIP_END bytes */
288 memmove(p
+reqlen
,p
,curlen
-ZIPLIST_HEADER_SIZE
-1);
294 /* Update tail offset if this is not the first element */
295 if (curtail
[0] != ZIP_END
) {
296 if (where
== ZIPLIST_HEAD
) {
297 ZIPLIST_TAIL_OFFSET(zl
) += reqlen
;
299 ZIPLIST_TAIL_OFFSET(zl
) += prevlen
;
303 /* Write the entry */
304 p
+= zipEncodeLength(p
,ZIP_ENC_RAW
,prevlen
);
305 p
+= zipEncodeLength(p
,encoding
,elen
);
306 if (encoding
!= ZIP_ENC_RAW
) {
307 zipSaveInteger(p
,value
,encoding
);
309 memcpy(p
,entry
,elen
);
311 ZIPLIST_INCR_LENGTH(zl
,1);
315 unsigned char *ziplistPop(unsigned char *zl
, sds
*target
, int where
) {
316 unsigned int curlen
= ZIPLIST_BYTES(zl
), rawlen
;
321 if (target
) *target
= NULL
;
323 /* Get pointer to element to remove */
324 p
= (where
== ZIPLIST_HEAD
) ? ziplistHead(zl
) : ziplistTail(zl
);
325 if (*p
== ZIP_END
) return zl
;
328 rawlen
= entry
.headersize
+entry
.len
;
330 if (entry
.encoding
== ZIP_ENC_RAW
) {
331 *target
= sdsnewlen(p
+entry
.headersize
,entry
.len
);
333 value
= zipLoadInteger(p
+entry
.headersize
,entry
.encoding
);
334 *target
= sdscatprintf(sdsempty(), "%lld", value
);
338 if (where
== ZIPLIST_HEAD
) {
339 /* The next entry will now be the head of the list */
340 if (p
[rawlen
] != ZIP_END
) {
341 /* Tricky: storing the length of the previous entry in the next
342 * entry (this previous length is always 0 when popping from the
343 * head), might reduce the number of bytes needed.
345 * In this special case (new length is 0), we know that the
346 * byte difference to store is always <= 0, which means that
347 * we always have space to store it. */
348 nextdiff
= zipPrevLenByteDiff(p
+rawlen
,0);
349 zipEncodeLength(p
+rawlen
-nextdiff
,ZIP_ENC_RAW
,0);
351 /* Move list to the front */
352 memmove(p
,p
+rawlen
-nextdiff
,curlen
-ZIPLIST_HEADER_SIZE
-rawlen
+nextdiff
);
354 /* Subtract the gained space from the tail offset */
355 ZIPLIST_TAIL_OFFSET(zl
) -= rawlen
+nextdiff
;
357 /* Subtract the length of the previous element from the tail offset. */
358 ZIPLIST_TAIL_OFFSET(zl
) -= entry
.prevrawlen
;
361 /* Resize and update length */
362 zl
= ziplistResize(zl
,curlen
-rawlen
+nextdiff
);
363 ZIPLIST_INCR_LENGTH(zl
,-1);
367 /* Returns an offset to use for iterating with ziplistNext. */
368 unsigned char *ziplistIndex(unsigned char *zl
, unsigned int index
) {
369 unsigned char *p
= zl
+ZIPLIST_HEADER_SIZE
;
371 for (; i
< index
; i
++) {
372 if (*p
== ZIP_END
) break;
373 p
+= zipRawEntryLength(p
);
378 /* Return pointer to next entry in ziplist. */
379 unsigned char *ziplistNext(unsigned char *p
) {
380 return *p
== ZIP_END
? p
: p
+zipRawEntryLength(p
);
383 /* Get entry pointer to by 'p' and store in either 'e' or 'v' depending
384 * on the encoding of the entry. 'e' is always set to NULL to be able
385 * to find out whether the string pointer or the integer value was set.
386 * Return 0 if 'p' points to the end of the zipmap, 1 otherwise. */
387 unsigned int ziplistGet(unsigned char *p
, unsigned char **e
, unsigned int *elen
, long long *v
) {
389 if (*p
== ZIP_END
) return 0;
393 if (entry
.encoding
== ZIP_ENC_RAW
) {
396 *e
= p
+entry
.headersize
;
400 *v
= zipLoadInteger(p
+entry
.headersize
,entry
.encoding
);
406 /* Delete a range of entries from the ziplist. */
407 unsigned char *ziplistDeleteRange(unsigned char *zl
, unsigned int index
, unsigned int num
) {
408 unsigned char *p
, *first
= ziplistIndex(zl
, index
);
409 unsigned int i
, totlen
, deleted
= 0;
410 for (p
= first
, i
= 0; *p
!= ZIP_END
&& i
< num
; i
++) {
411 p
+= zipRawEntryLength(p
);
417 /* Move current tail to the new tail when there *is* a tail */
418 if (*p
!= ZIP_END
) memmove(first
,p
,ZIPLIST_BYTES(zl
)-(p
-zl
)-1);
420 /* Resize and update length */
421 zl
= ziplistResize(zl
, ZIPLIST_BYTES(zl
)-totlen
);
422 ZIPLIST_INCR_LENGTH(zl
,-deleted
);
427 /* Delete a single entry from the ziplist, pointed to by *p.
428 * Also update *p in place, to be able to iterate over the
429 * ziplist, while deleting entries. */
430 unsigned char *ziplistDelete(unsigned char *zl
, unsigned char **p
) {
431 unsigned int offset
= *p
-zl
, tail
, len
;
432 len
= zipRawEntryLength(*p
);
433 tail
= ZIPLIST_BYTES(zl
)-offset
-len
-1;
435 /* Move current tail to the new tail when there *is* a tail */
436 if (tail
> 0) memmove(*p
,*p
+len
,tail
);
438 /* Resize and update length */
439 zl
= ziplistResize(zl
, ZIPLIST_BYTES(zl
)-len
);
440 ZIPLIST_INCR_LENGTH(zl
,-1);
442 /* Store new pointer to current element in p.
443 * This needs to be done because zl can change on realloc. */
448 /* Compare entry pointer to by 'p' with 'entry'. Return 1 if equal. */
449 unsigned int ziplistCompare(unsigned char *p
, unsigned char *s
, unsigned int slen
) {
451 unsigned char sencoding
;
453 if (*p
== ZIP_END
) return 0;
456 if (entry
.encoding
== ZIP_ENC_RAW
) {
458 if (entry
.len
== slen
) {
459 return memcmp(p
+entry
.headersize
,s
,slen
) == 0;
464 /* Try to compare encoded values */
465 if (zipTryEncoding(s
,&sval
,&sencoding
)) {
466 if (entry
.encoding
== sencoding
) {
467 val
= zipLoadInteger(p
+entry
.headersize
,entry
.encoding
);
475 /* Return length of ziplist. */
476 unsigned int ziplistLen(unsigned char *zl
) {
477 unsigned int len
= 0;
478 if (ZIPLIST_LENGTH(zl
) < ZIP_BIGLEN
) {
479 len
= ZIPLIST_LENGTH(zl
);
481 unsigned char *p
= zl
+ZIPLIST_HEADER_SIZE
;
482 while (*p
!= ZIP_END
) {
483 p
+= zipRawEntryLength(p
);
487 /* Re-store length if small enough */
488 if (len
< ZIP_BIGLEN
) ZIPLIST_LENGTH(zl
) = len
;
493 /* Return size in bytes of ziplist. */
494 unsigned int ziplistSize(unsigned char *zl
) {
495 return ZIPLIST_BYTES(zl
);
498 void ziplistRepr(unsigned char *zl
) {
499 unsigned char *p
, encoding
;
500 unsigned int prevrawlensize
, prevrawlen
, lensize
, len
;
502 printf("{total bytes %d} {length %u}\n",ZIPLIST_BYTES(zl
), ZIPLIST_LENGTH(zl
));
504 while(*p
!= ZIP_END
) {
505 prevrawlen
= zipDecodeLength(p
,&prevrawlensize
);
506 len
= zipDecodeLength(p
+prevrawlensize
,&lensize
);
507 printf("{offset %ld, header %u, payload %u} ",p
-zl
,prevrawlensize
+lensize
,len
);
508 encoding
= ZIP_ENCODING(p
+prevrawlensize
);
509 p
+= prevrawlensize
+lensize
;
510 if (encoding
== ZIP_ENC_RAW
) {
511 fwrite(p
,len
,1,stdout
);
513 printf("%lld", zipLoadInteger(p
,encoding
));
521 #ifdef ZIPLIST_TEST_MAIN
523 unsigned char *createList() {
524 unsigned char *zl
= ziplistNew();
525 zl
= ziplistPush(zl
, (unsigned char*)"foo", 3, ZIPLIST_TAIL
);
526 zl
= ziplistPush(zl
, (unsigned char*)"quux", 4, ZIPLIST_TAIL
);
527 zl
= ziplistPush(zl
, (unsigned char*)"hello", 5, ZIPLIST_HEAD
);
528 zl
= ziplistPush(zl
, (unsigned char*)"1024", 4, ZIPLIST_TAIL
);
532 unsigned char *createIntList() {
533 unsigned char *zl
= ziplistNew();
537 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_TAIL
);
538 sprintf(buf
, "128000");
539 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_TAIL
);
540 sprintf(buf
, "-100");
541 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_HEAD
);
542 sprintf(buf
, "4294967296");
543 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_HEAD
);
544 sprintf(buf
, "non integer");
545 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_TAIL
);
546 sprintf(buf
, "much much longer non integer");
547 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_TAIL
);
551 int main(int argc
, char **argv
) {
552 unsigned char *zl
, *p
, *q
, *entry
;
557 zl
= createIntList();
563 zl
= ziplistPop(zl
, &s
, ZIPLIST_TAIL
);
564 printf("Pop tail: %s (length %ld)\n", s
, sdslen(s
));
567 zl
= ziplistPop(zl
, &s
, ZIPLIST_HEAD
);
568 printf("Pop head: %s (length %ld)\n", s
, sdslen(s
));
571 zl
= ziplistPop(zl
, &s
, ZIPLIST_TAIL
);
572 printf("Pop tail: %s (length %ld)\n", s
, sdslen(s
));
575 zl
= ziplistPop(zl
, &s
, ZIPLIST_TAIL
);
576 printf("Pop tail: %s (length %ld)\n", s
, sdslen(s
));
579 printf("Iterate list from 0 to end:\n");
582 p
= ziplistIndex(zl
, 0);
583 while (ziplistGet(p
, &entry
, &elen
, &value
)) {
586 fwrite(entry
,elen
,1,stdout
);
588 printf("%lld", value
);
596 printf("Iterate list from 1 to end:\n");
599 p
= ziplistIndex(zl
, 1);
600 while (ziplistGet(p
, &entry
, &elen
, &value
)) {
603 fwrite(entry
,elen
,1,stdout
);
605 printf("%lld", value
);
613 printf("Iterate list from 2 to end:\n");
616 p
= ziplistIndex(zl
, 2);
617 while (ziplistGet(p
, &entry
, &elen
, &value
)) {
620 fwrite(entry
,elen
,1,stdout
);
622 printf("%lld", value
);
630 printf("Iterate starting out of range:\n");
633 p
= ziplistIndex(zl
, 4);
634 if (!ziplistGet(p
, &entry
, &elen
, &value
)) {
635 printf("No entry\n");
642 printf("Delete inclusive range 0,0:\n");
645 zl
= ziplistDeleteRange(zl
, 0, 1);
649 printf("Delete inclusive range 0,1:\n");
652 zl
= ziplistDeleteRange(zl
, 0, 2);
656 printf("Delete inclusive range 1,2:\n");
659 zl
= ziplistDeleteRange(zl
, 1, 2);
663 printf("Delete with start index out of range:\n");
666 zl
= ziplistDeleteRange(zl
, 5, 1);
670 printf("Delete with num overflow:\n");
673 zl
= ziplistDeleteRange(zl
, 1, 5);
677 printf("Delete foo while iterating:\n");
680 p
= ziplistIndex(zl
, 0);
681 while (ziplistGet(p
, &entry
, &elen
, &value
)) {
682 if (entry
&& strncmp("foo", entry
, elen
) == 0) {
683 printf("Delete foo\n");
684 zl
= ziplistDelete(zl
, &p
);
688 fwrite(entry
,elen
,1,stdout
);
690 printf("%lld", value
);
700 printf("Compare strings with ziplist entries:\n");
703 p
= ziplistIndex(zl
, 0);
704 if (!ziplistCompare(p
,"hello",5)) {
705 printf("ERROR: not \"hello\"\n");
708 if (ziplistCompare(p
,"hella",5)) {
709 printf("ERROR: \"hella\"\n");
713 p
= ziplistIndex(zl
, 3);
714 if (!ziplistCompare(p
,"1024",4)) {
715 printf("ERROR: not \"1024\"\n");
718 if (ziplistCompare(p
,"1025",4)) {
719 printf("ERROR: \"1025\"\n");