]>
git.saurik.com Git - redis.git/blob - ziplist.c
16e9dbc1ea5db807771cae45e53b4627ea9e0fd4
1 /* Memory layout of a ziplist, containing "foo", "bar", "quux":
2 * <zlbytes><zllen><len>"foo"<len>"bar"<len>"quux"
4 * <zlbytes> is an unsigned integer to hold the number of bytes that
5 * the ziplist occupies. This is stored to not have to traverse the ziplist
6 * to know the new length when pushing.
8 * <zllen> is the number of items in the ziplist. When this value is
9 * greater than 254, we need to traverse the entire list to know
10 * how many items it holds.
12 * <len> is the number of bytes occupied by a single entry. When this
13 * number is greater than 253, the length will occupy 5 bytes, where
14 * the extra bytes contain an unsigned integer to hold the length.
26 /* Important note: the ZIP_END value is used to depict the end of the
27 * ziplist structure. When a pointer contains an entry, the first couple
28 * of bytes contain the encoded length of the previous entry. This length
29 * is encoded as ZIP_ENC_RAW length, so the first two bits will contain 00
30 * and the byte will therefore never have a value of 255. */
32 #define ZIP_BIGLEN 254
36 #define ZIP_ENC_SHORT 1
38 #define ZIP_ENC_LLONG 3
39 #define ZIP_ENCODING(p) ((p)[0] >> 6)
41 /* Length encoding for raw entries */
42 #define ZIP_LEN_INLINE 0
43 #define ZIP_LEN_UINT16 1
44 #define ZIP_LEN_UINT32 2
47 #define ZIPLIST_BYTES(zl) (*((unsigned int*)(zl)))
48 #define ZIPLIST_TAIL_OFFSET(zl) (*((zl)+sizeof(unsigned int)))
49 #define ZIPLIST_LENGTH(zl) (*((zl)+2*sizeof(unsigned int)))
50 #define ZIPLIST_HEADER_SIZE (2*sizeof(unsigned int)+1)
51 #define ZIPLIST_INCR_LENGTH(zl,incr) { \
52 if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) ZIPLIST_LENGTH(zl)+=incr; }
54 typedef struct zlentry
{
55 unsigned int prevrawlensize
, prevrawlen
;
56 unsigned int lensize
, len
;
57 unsigned int headersize
;
58 unsigned char encoding
;
61 /* Return bytes needed to store integer encoded by 'encoding' */
62 static unsigned int zipEncodingSize(char encoding
) {
63 if (encoding
== ZIP_ENC_SHORT
) {
64 return sizeof(short int);
65 } else if (encoding
== ZIP_ENC_INT
) {
67 } else if (encoding
== ZIP_ENC_LLONG
) {
68 return sizeof(long long);
73 /* Decode the encoded length pointed by 'p'. If a pointer to 'lensize' is
74 * provided, it is set to the number of bytes required to encode the length. */
75 static unsigned int zipDecodeLength(unsigned char *p
, unsigned int *lensize
) {
76 unsigned char encoding
= ZIP_ENCODING(p
), lenenc
;
79 if (encoding
== ZIP_ENC_RAW
) {
80 lenenc
= (p
[0] >> 4) & 0x3;
81 if (lenenc
== ZIP_LEN_INLINE
) {
83 if (lensize
) *lensize
= 1;
84 } else if (lenenc
== ZIP_LEN_UINT16
) {
85 len
= p
[1] | (p
[2] << 8);
86 if (lensize
) *lensize
= 3;
88 len
= p
[1] | (p
[2] << 8) | (p
[3] << 16) | (p
[4] << 24);
89 if (lensize
) *lensize
= 5;
92 len
= zipEncodingSize(encoding
);
93 if (lensize
) *lensize
= 1;
98 /* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
99 * the amount of bytes required to encode such a length. */
100 static unsigned int zipEncodeLength(unsigned char *p
, char encoding
, unsigned int rawlen
) {
101 unsigned char len
= 1, lenenc
, buf
[5];
102 if (encoding
== ZIP_ENC_RAW
) {
105 lenenc
= ZIP_LEN_INLINE
;
107 } else if (rawlen
<= 0xffff) {
110 lenenc
= ZIP_LEN_UINT16
;
111 buf
[1] = (rawlen
) & 0xff;
112 buf
[2] = (rawlen
>> 8) & 0xff;
116 lenenc
= ZIP_LEN_UINT32
;
117 buf
[1] = (rawlen
) & 0xff;
118 buf
[2] = (rawlen
>> 8) & 0xff;
119 buf
[3] = (rawlen
>> 16) & 0xff;
120 buf
[4] = (rawlen
>> 24) & 0xff;
122 buf
[0] = (lenenc
<< 4) | (buf
[0] & 0xf);
126 /* Apparently we need to store the length in 'p' */
127 buf
[0] = (encoding
<< 6) | (buf
[0] & 0x3f);
132 /* Return the difference in number of bytes needed to store the new length
133 * "len" on the entry pointed to by "p". */
134 static int zipPrevLenByteDiff(unsigned char *p
, unsigned int len
) {
135 unsigned int prevlensize
;
136 zipDecodeLength(p
,&prevlensize
);
137 return zipEncodeLength(NULL
,ZIP_ENC_RAW
,len
)-prevlensize
;
140 /* Check if string pointed to by 'entry' can be encoded as an integer.
141 * Stores the integer value in 'v' and its encoding in 'encoding'.
142 * Warning: this function requires a NULL-terminated string! */
143 static int zipTryEncoding(unsigned char *entry
, long long *v
, char *encoding
) {
147 if (entry
[0] == '-' || (entry
[0] >= '0' && entry
[0] <= '9')) {
148 value
= strtoll((char*)entry
,&eptr
,10);
149 if (eptr
[0] != '\0') return 0;
150 if (value
>= SHRT_MIN
&& value
<= SHRT_MAX
) {
151 *encoding
= ZIP_ENC_SHORT
;
152 } else if (value
>= INT_MIN
&& value
<= INT_MAX
) {
153 *encoding
= ZIP_ENC_INT
;
155 *encoding
= ZIP_ENC_LLONG
;
163 /* Store integer 'value' at 'p', encoded as 'encoding' */
164 static void zipSaveInteger(unsigned char *p
, long long value
, char encoding
) {
168 if (encoding
== ZIP_ENC_SHORT
) {
170 memcpy(p
,&s
,sizeof(s
));
171 } else if (encoding
== ZIP_ENC_INT
) {
173 memcpy(p
,&i
,sizeof(i
));
174 } else if (encoding
== ZIP_ENC_LLONG
) {
176 memcpy(p
,&l
,sizeof(l
));
182 /* Read integer encoded as 'encoding' from 'p' */
183 static long long zipLoadInteger(unsigned char *p
, char encoding
) {
187 if (encoding
== ZIP_ENC_SHORT
) {
188 memcpy(&s
,p
,sizeof(s
));
190 } else if (encoding
== ZIP_ENC_INT
) {
191 memcpy(&i
,p
,sizeof(i
));
193 } else if (encoding
== ZIP_ENC_LLONG
) {
194 memcpy(&l
,p
,sizeof(l
));
202 /* Return a struct with all information about an entry. */
203 static zlentry
zipEntry(unsigned char *p
) {
205 e
.prevrawlen
= zipDecodeLength(p
,&e
.prevrawlensize
);
206 e
.len
= zipDecodeLength(p
+e
.prevrawlensize
,&e
.lensize
);
207 e
.headersize
= e
.prevrawlensize
+e
.lensize
;
208 e
.encoding
= ZIP_ENCODING(p
+e
.prevrawlensize
);
212 /* Return the total number of bytes used by the entry at "p". */
213 static unsigned int zipRawEntryLength(unsigned char *p
) {
214 zlentry e
= zipEntry(p
);
215 return e
.headersize
+ e
.len
;
218 /* Create a new empty ziplist. */
219 unsigned char *ziplistNew(void) {
220 unsigned int bytes
= ZIPLIST_HEADER_SIZE
+1;
221 unsigned char *zl
= zmalloc(bytes
);
222 ZIPLIST_BYTES(zl
) = bytes
;
223 ZIPLIST_TAIL_OFFSET(zl
) = ZIPLIST_HEADER_SIZE
;
224 ZIPLIST_LENGTH(zl
) = 0;
225 zl
[bytes
-1] = ZIP_END
;
229 /* Resize the ziplist. */
230 static unsigned char *ziplistResize(unsigned char *zl
, unsigned int len
) {
231 zl
= zrealloc(zl
,len
);
232 ZIPLIST_BYTES(zl
) = len
;
237 static unsigned char *ziplistHead(unsigned char *zl
) {
238 return zl
+ZIPLIST_HEADER_SIZE
;
241 static unsigned char *ziplistTail(unsigned char *zl
) {
242 unsigned char *p
, *q
;
243 p
= q
= ziplistHead(zl
);
244 while (*p
!= ZIP_END
) {
246 p
+= zipRawEntryLength(p
);
251 unsigned char *ziplistPush(unsigned char *zl
, unsigned char *entry
, unsigned int elen
, int where
) {
252 unsigned int curlen
= ZIPLIST_BYTES(zl
), reqlen
, prevlen
;
253 unsigned char *p
, *curtail
;
254 char encoding
= ZIP_ENC_RAW
;
257 /* We need to store the length of the current tail when the list
258 * is non-empty and we push at the tail. */
259 curtail
= zl
+ZIPLIST_TAIL_OFFSET(zl
);
260 if (where
== ZIPLIST_TAIL
&& curtail
[0] != ZIP_END
) {
261 prevlen
= zipRawEntryLength(curtail
);
266 /* See if the entry can be encoded */
267 if (zipTryEncoding(entry
,&value
,&encoding
)) {
268 reqlen
= zipEncodingSize(encoding
);
273 /* We need space for both the length of the previous entry and
274 * the length of the payload. */
275 reqlen
+= zipEncodeLength(NULL
,ZIP_ENC_RAW
,prevlen
);
276 reqlen
+= zipEncodeLength(NULL
,encoding
,elen
);
278 /* Resize the ziplist and move if needed */
279 zl
= ziplistResize(zl
,curlen
+reqlen
);
280 if (where
== ZIPLIST_HEAD
) {
281 p
= zl
+ZIPLIST_HEADER_SIZE
;
283 /* Subtract one because of the ZIP_END bytes */
284 memmove(p
+reqlen
,p
,curlen
-ZIPLIST_HEADER_SIZE
-1);
290 /* Update tail offset if this is not the first element */
291 if (curtail
[0] != ZIP_END
) {
292 if (where
== ZIPLIST_HEAD
) {
293 ZIPLIST_TAIL_OFFSET(zl
) += reqlen
;
295 ZIPLIST_TAIL_OFFSET(zl
) += prevlen
;
299 /* Write the entry */
300 p
+= zipEncodeLength(p
,ZIP_ENC_RAW
,prevlen
);
301 p
+= zipEncodeLength(p
,encoding
,elen
);
302 if (encoding
!= ZIP_ENC_RAW
) {
303 zipSaveInteger(p
,value
,encoding
);
305 memcpy(p
,entry
,elen
);
307 ZIPLIST_INCR_LENGTH(zl
,1);
311 unsigned char *ziplistPop(unsigned char *zl
, sds
*target
, int where
) {
312 unsigned int curlen
= ZIPLIST_BYTES(zl
), rawlen
;
317 if (target
) *target
= NULL
;
319 /* Get pointer to element to remove */
320 p
= (where
== ZIPLIST_HEAD
) ? ziplistHead(zl
) : ziplistTail(zl
);
321 if (*p
== ZIP_END
) return zl
;
324 rawlen
= entry
.headersize
+entry
.len
;
326 if (entry
.encoding
== ZIP_ENC_RAW
) {
327 *target
= sdsnewlen(p
+entry
.headersize
,entry
.len
);
329 value
= zipLoadInteger(p
+entry
.headersize
,entry
.encoding
);
330 *target
= sdscatprintf(sdsempty(), "%lld", value
);
334 if (where
== ZIPLIST_HEAD
) {
335 /* The next entry will now be the head of the list */
336 if (p
[rawlen
] != ZIP_END
) {
337 /* Tricky: storing the length of the previous entry in the next
338 * entry (this previous length is always 0 when popping from the
339 * head), might reduce the number of bytes needed.
341 * In this special case (new length is 0), we know that the
342 * byte difference to store is always <= 0, which means that
343 * we always have space to store it. */
344 nextdiff
= zipPrevLenByteDiff(p
+rawlen
,0);
345 zipEncodeLength(p
+rawlen
-nextdiff
,ZIP_ENC_RAW
,0);
347 /* Move list to the front */
348 memmove(p
,p
+rawlen
-nextdiff
,curlen
-ZIPLIST_HEADER_SIZE
-rawlen
+nextdiff
);
350 /* Subtract the gained space from the tail offset */
351 ZIPLIST_TAIL_OFFSET(zl
) -= rawlen
+nextdiff
;
353 /* Subtract the length of the previous element from the tail offset. */
354 ZIPLIST_TAIL_OFFSET(zl
) -= entry
.prevrawlen
;
357 /* Resize and update length */
358 zl
= ziplistResize(zl
,curlen
-rawlen
+nextdiff
);
359 ZIPLIST_INCR_LENGTH(zl
,-1);
363 /* Returns an offset to use for iterating with ziplistNext. */
364 unsigned char *ziplistIndex(unsigned char *zl
, unsigned int index
) {
365 unsigned char *p
= zl
+ZIPLIST_HEADER_SIZE
;
367 for (; i
< index
; i
++) {
368 if (*p
== ZIP_END
) break;
369 p
+= zipRawEntryLength(p
);
374 /* Return pointer to next entry in ziplist. */
375 unsigned char *ziplistNext(unsigned char *p
) {
376 return *p
== ZIP_END
? p
: p
+zipRawEntryLength(p
);
379 /* Get entry pointer to by 'p' and store in either 'e' or 'v' depending
380 * on the encoding of the entry. 'e' is always set to NULL to be able
381 * to find out whether the string pointer or the integer value was set.
382 * Return 0 if 'p' points to the end of the zipmap, 1 otherwise. */
383 unsigned int ziplistGet(unsigned char *p
, unsigned char **sstr
, unsigned int *slen
, long long *sval
) {
385 if (*p
== ZIP_END
) return 0;
386 if (sstr
) *sstr
= NULL
;
389 if (entry
.encoding
== ZIP_ENC_RAW
) {
392 *sstr
= p
+entry
.headersize
;
396 *sval
= zipLoadInteger(p
+entry
.headersize
,entry
.encoding
);
402 /* Delete a range of entries from the ziplist. */
403 unsigned char *ziplistDeleteRange(unsigned char *zl
, unsigned int index
, unsigned int num
) {
404 unsigned char *p
, *first
= ziplistIndex(zl
, index
);
405 unsigned int i
, totlen
, deleted
= 0;
406 for (p
= first
, i
= 0; *p
!= ZIP_END
&& i
< num
; i
++) {
407 p
+= zipRawEntryLength(p
);
413 /* Move current tail to the new tail when there *is* a tail */
414 if (*p
!= ZIP_END
) memmove(first
,p
,ZIPLIST_BYTES(zl
)-(p
-zl
)-1);
416 /* Resize and update length */
417 zl
= ziplistResize(zl
, ZIPLIST_BYTES(zl
)-totlen
);
418 ZIPLIST_INCR_LENGTH(zl
,-deleted
);
423 /* Delete a single entry from the ziplist, pointed to by *p.
424 * Also update *p in place, to be able to iterate over the
425 * ziplist, while deleting entries. */
426 unsigned char *ziplistDelete(unsigned char *zl
, unsigned char **p
) {
427 unsigned int offset
= *p
-zl
, tail
, len
;
428 len
= zipRawEntryLength(*p
);
429 tail
= ZIPLIST_BYTES(zl
)-offset
-len
-1;
431 /* Move current tail to the new tail when there *is* a tail */
432 if (tail
> 0) memmove(*p
,*p
+len
,tail
);
434 /* Resize and update length */
435 zl
= ziplistResize(zl
, ZIPLIST_BYTES(zl
)-len
);
436 ZIPLIST_INCR_LENGTH(zl
,-1);
438 /* Store new pointer to current element in p.
439 * This needs to be done because zl can change on realloc. */
444 /* Compare entry pointer to by 'p' with 'entry'. Return 1 if equal. */
445 unsigned int ziplistCompare(unsigned char *p
, unsigned char *sstr
, unsigned int slen
) {
447 unsigned char sencoding
;
449 if (*p
== ZIP_END
) return 0;
452 if (entry
.encoding
== ZIP_ENC_RAW
) {
454 if (entry
.len
== slen
) {
455 return memcmp(p
+entry
.headersize
,sstr
,slen
) == 0;
460 /* Try to compare encoded values */
461 if (zipTryEncoding(sstr
,&sval
,&sencoding
)) {
462 if (entry
.encoding
== sencoding
) {
463 val
= zipLoadInteger(p
+entry
.headersize
,entry
.encoding
);
471 /* Return length of ziplist. */
472 unsigned int ziplistLen(unsigned char *zl
) {
473 unsigned int len
= 0;
474 if (ZIPLIST_LENGTH(zl
) < ZIP_BIGLEN
) {
475 len
= ZIPLIST_LENGTH(zl
);
477 unsigned char *p
= zl
+ZIPLIST_HEADER_SIZE
;
478 while (*p
!= ZIP_END
) {
479 p
+= zipRawEntryLength(p
);
483 /* Re-store length if small enough */
484 if (len
< ZIP_BIGLEN
) ZIPLIST_LENGTH(zl
) = len
;
489 /* Return size in bytes of ziplist. */
490 unsigned int ziplistSize(unsigned char *zl
) {
491 return ZIPLIST_BYTES(zl
);
494 void ziplistRepr(unsigned char *zl
) {
498 printf("{total bytes %d} {length %u}\n",ZIPLIST_BYTES(zl
), ZIPLIST_LENGTH(zl
));
500 while(*p
!= ZIP_END
) {
502 printf("{offset %ld, header %u, payload %u} ",p
-zl
,entry
.headersize
,entry
.len
);
503 p
+= entry
.headersize
;
504 if (entry
.encoding
== ZIP_ENC_RAW
) {
505 fwrite(p
,entry
.len
,1,stdout
);
507 printf("%lld", zipLoadInteger(p
,entry
.encoding
));
515 #ifdef ZIPLIST_TEST_MAIN
517 unsigned char *createList() {
518 unsigned char *zl
= ziplistNew();
519 zl
= ziplistPush(zl
, (unsigned char*)"foo", 3, ZIPLIST_TAIL
);
520 zl
= ziplistPush(zl
, (unsigned char*)"quux", 4, ZIPLIST_TAIL
);
521 zl
= ziplistPush(zl
, (unsigned char*)"hello", 5, ZIPLIST_HEAD
);
522 zl
= ziplistPush(zl
, (unsigned char*)"1024", 4, ZIPLIST_TAIL
);
526 unsigned char *createIntList() {
527 unsigned char *zl
= ziplistNew();
531 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_TAIL
);
532 sprintf(buf
, "128000");
533 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_TAIL
);
534 sprintf(buf
, "-100");
535 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_HEAD
);
536 sprintf(buf
, "4294967296");
537 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_HEAD
);
538 sprintf(buf
, "non integer");
539 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_TAIL
);
540 sprintf(buf
, "much much longer non integer");
541 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_TAIL
);
545 int main(int argc
, char **argv
) {
546 unsigned char *zl
, *p
, *q
, *entry
;
551 zl
= createIntList();
557 zl
= ziplistPop(zl
, &s
, ZIPLIST_TAIL
);
558 printf("Pop tail: %s (length %ld)\n", s
, sdslen(s
));
561 zl
= ziplistPop(zl
, &s
, ZIPLIST_HEAD
);
562 printf("Pop head: %s (length %ld)\n", s
, sdslen(s
));
565 zl
= ziplistPop(zl
, &s
, ZIPLIST_TAIL
);
566 printf("Pop tail: %s (length %ld)\n", s
, sdslen(s
));
569 zl
= ziplistPop(zl
, &s
, ZIPLIST_TAIL
);
570 printf("Pop tail: %s (length %ld)\n", s
, sdslen(s
));
573 printf("Iterate list from 0 to end:\n");
576 p
= ziplistIndex(zl
, 0);
577 while (ziplistGet(p
, &entry
, &elen
, &value
)) {
580 fwrite(entry
,elen
,1,stdout
);
582 printf("%lld", value
);
590 printf("Iterate list from 1 to end:\n");
593 p
= ziplistIndex(zl
, 1);
594 while (ziplistGet(p
, &entry
, &elen
, &value
)) {
597 fwrite(entry
,elen
,1,stdout
);
599 printf("%lld", value
);
607 printf("Iterate list from 2 to end:\n");
610 p
= ziplistIndex(zl
, 2);
611 while (ziplistGet(p
, &entry
, &elen
, &value
)) {
614 fwrite(entry
,elen
,1,stdout
);
616 printf("%lld", value
);
624 printf("Iterate starting out of range:\n");
627 p
= ziplistIndex(zl
, 4);
628 if (!ziplistGet(p
, &entry
, &elen
, &value
)) {
629 printf("No entry\n");
636 printf("Delete inclusive range 0,0:\n");
639 zl
= ziplistDeleteRange(zl
, 0, 1);
643 printf("Delete inclusive range 0,1:\n");
646 zl
= ziplistDeleteRange(zl
, 0, 2);
650 printf("Delete inclusive range 1,2:\n");
653 zl
= ziplistDeleteRange(zl
, 1, 2);
657 printf("Delete with start index out of range:\n");
660 zl
= ziplistDeleteRange(zl
, 5, 1);
664 printf("Delete with num overflow:\n");
667 zl
= ziplistDeleteRange(zl
, 1, 5);
671 printf("Delete foo while iterating:\n");
674 p
= ziplistIndex(zl
, 0);
675 while (ziplistGet(p
, &entry
, &elen
, &value
)) {
676 if (entry
&& strncmp("foo", entry
, elen
) == 0) {
677 printf("Delete foo\n");
678 zl
= ziplistDelete(zl
, &p
);
682 fwrite(entry
,elen
,1,stdout
);
684 printf("%lld", value
);
694 printf("Compare strings with ziplist entries:\n");
697 p
= ziplistIndex(zl
, 0);
698 if (!ziplistCompare(p
,"hello",5)) {
699 printf("ERROR: not \"hello\"\n");
702 if (ziplistCompare(p
,"hella",5)) {
703 printf("ERROR: \"hella\"\n");
707 p
= ziplistIndex(zl
, 3);
708 if (!ziplistCompare(p
,"1024",4)) {
709 printf("ERROR: not \"1024\"\n");
712 if (ziplistCompare(p
,"1025",4)) {
713 printf("ERROR: \"1025\"\n");