]>
git.saurik.com Git - redis.git/blob - ziplist.c
e54f321143315e87568f9a5b0cf41e2f0241ea27
1 /* Memory layout of a ziplist, containing "foo", "bar", "quux":
2 * <zlbytes><zllen><len>"foo"<len>"bar"<len>"quux"
4 * <zlbytes> is an unsigned integer to hold the number of bytes that
5 * the ziplist occupies. This is stored to not have to traverse the ziplist
6 * to know the new length when pushing.
8 * <zllen> is the number of items in the ziplist. When this value is
9 * greater than 254, we need to traverse the entire list to know
10 * how many items it holds.
12 * <len> is the number of bytes occupied by a single entry. When this
13 * number is greater than 253, the length will occupy 5 bytes, where
14 * the extra bytes contain an unsigned integer to hold the length.
26 /* Important note: the ZIP_END value is used to depict the end of the
27 * ziplist structure. When a pointer contains an entry, the first couple
28 * of bytes contain the encoded length of the previous entry. This length
29 * is encoded as ZIP_ENC_RAW length, so the first two bits will contain 00
30 * and the byte will therefore never have a value of 255. */
32 #define ZIP_BIGLEN 254
36 #define ZIP_ENC_SHORT 1
38 #define ZIP_ENC_LLONG 3
39 #define ZIP_ENCODING(p) ((p)[0] >> 6)
41 /* Length encoding for raw entries */
42 #define ZIP_LEN_INLINE 0
43 #define ZIP_LEN_UINT16 1
44 #define ZIP_LEN_UINT32 2
47 #define ZIPLIST_BYTES(zl) (*((unsigned int*)(zl)))
48 #define ZIPLIST_TAIL_OFFSET(zl) (*((zl)+sizeof(unsigned int)))
49 #define ZIPLIST_LENGTH(zl) (*((zl)+2*sizeof(unsigned int)))
50 #define ZIPLIST_HEADER_SIZE (2*sizeof(unsigned int)+1)
51 #define ZIPLIST_INCR_LENGTH(zl,incr) { \
52 if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) ZIPLIST_LENGTH(zl)+=incr; }
54 typedef struct zlentry
{
55 unsigned int prevrawlensize
, prevrawlen
;
56 unsigned int lensize
, len
;
57 unsigned int headersize
;
58 unsigned char encoding
;
62 /* Return bytes needed to store integer encoded by 'encoding' */
63 static unsigned int zipEncodingSize(char encoding
) {
64 if (encoding
== ZIP_ENC_SHORT
) {
65 return sizeof(short int);
66 } else if (encoding
== ZIP_ENC_INT
) {
68 } else if (encoding
== ZIP_ENC_LLONG
) {
69 return sizeof(long long);
74 /* Decode the encoded length pointed by 'p'. If a pointer to 'lensize' is
75 * provided, it is set to the number of bytes required to encode the length. */
76 static unsigned int zipDecodeLength(unsigned char *p
, unsigned int *lensize
) {
77 unsigned char encoding
= ZIP_ENCODING(p
), lenenc
;
80 if (encoding
== ZIP_ENC_RAW
) {
81 lenenc
= (p
[0] >> 4) & 0x3;
82 if (lenenc
== ZIP_LEN_INLINE
) {
84 if (lensize
) *lensize
= 1;
85 } else if (lenenc
== ZIP_LEN_UINT16
) {
86 len
= p
[1] | (p
[2] << 8);
87 if (lensize
) *lensize
= 3;
89 len
= p
[1] | (p
[2] << 8) | (p
[3] << 16) | (p
[4] << 24);
90 if (lensize
) *lensize
= 5;
93 len
= zipEncodingSize(encoding
);
94 if (lensize
) *lensize
= 1;
99 /* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
100 * the amount of bytes required to encode such a length. */
101 static unsigned int zipEncodeLength(unsigned char *p
, char encoding
, unsigned int rawlen
) {
102 unsigned char len
= 1, lenenc
, buf
[5];
103 if (encoding
== ZIP_ENC_RAW
) {
106 lenenc
= ZIP_LEN_INLINE
;
108 } else if (rawlen
<= 0xffff) {
111 lenenc
= ZIP_LEN_UINT16
;
112 buf
[1] = (rawlen
) & 0xff;
113 buf
[2] = (rawlen
>> 8) & 0xff;
117 lenenc
= ZIP_LEN_UINT32
;
118 buf
[1] = (rawlen
) & 0xff;
119 buf
[2] = (rawlen
>> 8) & 0xff;
120 buf
[3] = (rawlen
>> 16) & 0xff;
121 buf
[4] = (rawlen
>> 24) & 0xff;
123 buf
[0] = (lenenc
<< 4) | (buf
[0] & 0xf);
127 /* Apparently we need to store the length in 'p' */
128 buf
[0] = (encoding
<< 6) | (buf
[0] & 0x3f);
133 /* Return the difference in number of bytes needed to store the new length
134 * "len" on the entry pointed to by "p". */
135 static int zipPrevLenByteDiff(unsigned char *p
, unsigned int len
) {
136 unsigned int prevlensize
;
137 zipDecodeLength(p
,&prevlensize
);
138 return zipEncodeLength(NULL
,ZIP_ENC_RAW
,len
)-prevlensize
;
141 /* Check if string pointed to by 'entry' can be encoded as an integer.
142 * Stores the integer value in 'v' and its encoding in 'encoding'.
143 * Warning: this function requires a NULL-terminated string! */
144 static int zipTryEncoding(unsigned char *entry
, long long *v
, char *encoding
) {
148 if (entry
[0] == '-' || (entry
[0] >= '0' && entry
[0] <= '9')) {
149 value
= strtoll((char*)entry
,&eptr
,10);
150 if (eptr
[0] != '\0') return 0;
151 if (value
>= SHRT_MIN
&& value
<= SHRT_MAX
) {
152 *encoding
= ZIP_ENC_SHORT
;
153 } else if (value
>= INT_MIN
&& value
<= INT_MAX
) {
154 *encoding
= ZIP_ENC_INT
;
156 *encoding
= ZIP_ENC_LLONG
;
164 /* Store integer 'value' at 'p', encoded as 'encoding' */
165 static void zipSaveInteger(unsigned char *p
, long long value
, char encoding
) {
169 if (encoding
== ZIP_ENC_SHORT
) {
171 memcpy(p
,&s
,sizeof(s
));
172 } else if (encoding
== ZIP_ENC_INT
) {
174 memcpy(p
,&i
,sizeof(i
));
175 } else if (encoding
== ZIP_ENC_LLONG
) {
177 memcpy(p
,&l
,sizeof(l
));
183 /* Read integer encoded as 'encoding' from 'p' */
184 static long long zipLoadInteger(unsigned char *p
, char encoding
) {
188 if (encoding
== ZIP_ENC_SHORT
) {
189 memcpy(&s
,p
,sizeof(s
));
191 } else if (encoding
== ZIP_ENC_INT
) {
192 memcpy(&i
,p
,sizeof(i
));
194 } else if (encoding
== ZIP_ENC_LLONG
) {
195 memcpy(&l
,p
,sizeof(l
));
203 /* Return a struct with all information about an entry. */
204 static zlentry
zipEntry(unsigned char *p
) {
206 e
.prevrawlen
= zipDecodeLength(p
,&e
.prevrawlensize
);
207 e
.len
= zipDecodeLength(p
+e
.prevrawlensize
,&e
.lensize
);
208 e
.headersize
= e
.prevrawlensize
+e
.lensize
;
209 e
.encoding
= ZIP_ENCODING(p
+e
.prevrawlensize
);
214 /* Return the total number of bytes used by the entry at "p". */
215 static unsigned int zipRawEntryLength(unsigned char *p
) {
216 zlentry e
= zipEntry(p
);
217 return e
.headersize
+ e
.len
;
220 /* Create a new empty ziplist. */
221 unsigned char *ziplistNew(void) {
222 unsigned int bytes
= ZIPLIST_HEADER_SIZE
+1;
223 unsigned char *zl
= zmalloc(bytes
);
224 ZIPLIST_BYTES(zl
) = bytes
;
225 ZIPLIST_TAIL_OFFSET(zl
) = ZIPLIST_HEADER_SIZE
;
226 ZIPLIST_LENGTH(zl
) = 0;
227 zl
[bytes
-1] = ZIP_END
;
231 /* Resize the ziplist. */
232 static unsigned char *ziplistResize(unsigned char *zl
, unsigned int len
) {
233 zl
= zrealloc(zl
,len
);
234 ZIPLIST_BYTES(zl
) = len
;
239 static unsigned char *ziplistHead(unsigned char *zl
) {
240 return zl
+ZIPLIST_HEADER_SIZE
;
243 static unsigned char *ziplistTail(unsigned char *zl
) {
244 unsigned char *p
, *q
;
245 p
= q
= ziplistHead(zl
);
246 while (*p
!= ZIP_END
) {
248 p
+= zipRawEntryLength(p
);
252 /* Delete "num" entries, starting at "p". Returns pointer to the ziplist. */
253 static unsigned char *__ziplistDelete(unsigned char *zl
, unsigned char *p
, int num
) {
254 unsigned int i
, totlen
, deleted
= 0;
256 zlentry first
= zipEntry(p
);
257 for (i
= 0; p
[0] != ZIP_END
&& i
< num
; i
++) {
258 p
+= zipRawEntryLength(p
);
264 if (p
[0] != ZIP_END
) {
265 /* Tricky: storing the prevlen in this entry might reduce or
266 * increase the number of bytes needed, compared to the current
267 * prevlen. Note that we can always store this length because
268 * it was previously stored by an entry that is being deleted. */
269 nextdiff
= zipPrevLenByteDiff(p
,first
.prevrawlen
);
270 zipEncodeLength(p
-nextdiff
,ZIP_ENC_RAW
,first
.prevrawlen
);
272 /* Update offset for tail */
273 ZIPLIST_TAIL_OFFSET(zl
) -= totlen
+nextdiff
;
275 /* Move tail to the front of the ziplist */
276 memmove(first
.p
,p
-nextdiff
,ZIPLIST_BYTES(zl
)-(p
-zl
)-1+nextdiff
);
278 /* The entire tail was deleted. No need to move memory. */
279 ZIPLIST_TAIL_OFFSET(zl
) = (first
.p
-zl
)-first
.prevrawlen
;
282 /* Resize and update length */
283 zl
= ziplistResize(zl
, ZIPLIST_BYTES(zl
)-totlen
+nextdiff
);
284 ZIPLIST_INCR_LENGTH(zl
,-deleted
);
289 /* Insert item at "p". */
290 static unsigned char *__ziplistInsert(unsigned char *zl
, unsigned char *p
, unsigned char *s
, unsigned int slen
) {
291 unsigned int curlen
= ZIPLIST_BYTES(zl
), reqlen
, prevlen
= 0;
292 unsigned int offset
, nextdiff
= 0;
294 char encoding
= ZIP_ENC_RAW
;
298 /* Find out prevlen for the entry that is inserted. */
299 if (p
[0] != ZIP_END
) {
301 prevlen
= entry
.prevrawlen
;
303 tail
= ziplistTail(zl
);
304 if (tail
[0] != ZIP_END
) {
305 prevlen
= zipRawEntryLength(tail
);
309 /* See if the entry can be encoded */
310 if (zipTryEncoding(s
,&value
,&encoding
)) {
311 reqlen
= zipEncodingSize(encoding
);
316 /* We need space for both the length of the previous entry and
317 * the length of the payload. */
318 reqlen
+= zipEncodeLength(NULL
,ZIP_ENC_RAW
,prevlen
);
319 reqlen
+= zipEncodeLength(NULL
,encoding
,slen
);
321 /* When the insert position is not equal to the tail, we need to
322 * make sure that the next entry can hold this entry's length in
323 * its prevlen field. */
324 nextdiff
= p
[0] != ZIP_END
? zipPrevLenByteDiff(p
,reqlen
) : 0;
326 /* Store offset because a realloc may change the address of zl. */
328 zl
= ziplistResize(zl
,curlen
+reqlen
+nextdiff
);
331 /* Apply memory move when necessary and update tail offset. */
332 if (p
[0] != ZIP_END
) {
333 /* Subtract one because of the ZIP_END bytes */
334 memmove(p
+reqlen
,p
-nextdiff
,curlen
-offset
-1+nextdiff
);
335 /* Encode this entry's raw length in the next entry. */
336 zipEncodeLength(p
+reqlen
,ZIP_ENC_RAW
,reqlen
);
337 /* Update offset for tail */
338 ZIPLIST_TAIL_OFFSET(zl
) += reqlen
+nextdiff
;
340 /* This element will be the new tail. */
341 ZIPLIST_TAIL_OFFSET(zl
) = p
-zl
;
344 /* Write the entry */
345 p
+= zipEncodeLength(p
,ZIP_ENC_RAW
,prevlen
);
346 p
+= zipEncodeLength(p
,encoding
,slen
);
347 if (encoding
!= ZIP_ENC_RAW
) {
348 zipSaveInteger(p
,value
,encoding
);
352 ZIPLIST_INCR_LENGTH(zl
,1);
356 unsigned char *ziplistPush(unsigned char *zl
, unsigned char *s
, unsigned int slen
, int where
) {
358 p
= (where
== ZIPLIST_HEAD
) ? ziplistHead(zl
) : (zl
+ZIPLIST_BYTES(zl
)-1);
359 return __ziplistInsert(zl
,p
,s
,slen
);
362 unsigned char *ziplistPop(unsigned char *zl
, sds
*target
, int where
) {
366 if (target
) *target
= NULL
;
368 /* Get pointer to element to remove */
369 p
= (where
== ZIPLIST_HEAD
) ? ziplistHead(zl
) : ziplistTail(zl
);
370 if (*p
== ZIP_END
) return zl
;
374 if (entry
.encoding
== ZIP_ENC_RAW
) {
375 *target
= sdsnewlen(p
+entry
.headersize
,entry
.len
);
377 value
= zipLoadInteger(p
+entry
.headersize
,entry
.encoding
);
378 *target
= sdscatprintf(sdsempty(), "%lld", value
);
382 zl
= __ziplistDelete(zl
,p
,1);
386 /* Returns an offset to use for iterating with ziplistNext. */
387 unsigned char *ziplistIndex(unsigned char *zl
, unsigned int index
) {
388 unsigned char *p
= zl
+ZIPLIST_HEADER_SIZE
;
390 for (; i
< index
; i
++) {
391 if (*p
== ZIP_END
) break;
392 p
+= zipRawEntryLength(p
);
397 /* Return pointer to next entry in ziplist. */
398 unsigned char *ziplistNext(unsigned char *p
) {
399 return *p
== ZIP_END
? p
: p
+zipRawEntryLength(p
);
402 /* Get entry pointer to by 'p' and store in either 'e' or 'v' depending
403 * on the encoding of the entry. 'e' is always set to NULL to be able
404 * to find out whether the string pointer or the integer value was set.
405 * Return 0 if 'p' points to the end of the zipmap, 1 otherwise. */
406 unsigned int ziplistGet(unsigned char *p
, unsigned char **sstr
, unsigned int *slen
, long long *sval
) {
408 if (*p
== ZIP_END
) return 0;
409 if (sstr
) *sstr
= NULL
;
412 if (entry
.encoding
== ZIP_ENC_RAW
) {
415 *sstr
= p
+entry
.headersize
;
419 *sval
= zipLoadInteger(p
+entry
.headersize
,entry
.encoding
);
425 /* Delete a range of entries from the ziplist. */
426 unsigned char *ziplistDeleteRange(unsigned char *zl
, unsigned int index
, unsigned int num
) {
427 unsigned char *p
= ziplistIndex(zl
,index
);
428 return __ziplistDelete(zl
,p
,num
);
431 /* Delete a single entry from the ziplist, pointed to by *p.
432 * Also update *p in place, to be able to iterate over the
433 * ziplist, while deleting entries. */
434 unsigned char *ziplistDelete(unsigned char *zl
, unsigned char **p
) {
435 unsigned int offset
= *p
-zl
;
436 zl
= __ziplistDelete(zl
,*p
,1);
438 /* Store pointer to current element in p, because ziplistDelete will
439 * do a realloc which might result in a different "zl"-pointer. */
444 /* Compare entry pointer to by 'p' with 'entry'. Return 1 if equal. */
445 unsigned int ziplistCompare(unsigned char *p
, unsigned char *sstr
, unsigned int slen
) {
447 unsigned char sencoding
;
449 if (*p
== ZIP_END
) return 0;
452 if (entry
.encoding
== ZIP_ENC_RAW
) {
454 if (entry
.len
== slen
) {
455 return memcmp(p
+entry
.headersize
,sstr
,slen
) == 0;
460 /* Try to compare encoded values */
461 if (zipTryEncoding(sstr
,&sval
,&sencoding
)) {
462 if (entry
.encoding
== sencoding
) {
463 val
= zipLoadInteger(p
+entry
.headersize
,entry
.encoding
);
471 /* Return length of ziplist. */
472 unsigned int ziplistLen(unsigned char *zl
) {
473 unsigned int len
= 0;
474 if (ZIPLIST_LENGTH(zl
) < ZIP_BIGLEN
) {
475 len
= ZIPLIST_LENGTH(zl
);
477 unsigned char *p
= zl
+ZIPLIST_HEADER_SIZE
;
478 while (*p
!= ZIP_END
) {
479 p
+= zipRawEntryLength(p
);
483 /* Re-store length if small enough */
484 if (len
< ZIP_BIGLEN
) ZIPLIST_LENGTH(zl
) = len
;
489 /* Return size in bytes of ziplist. */
490 unsigned int ziplistSize(unsigned char *zl
) {
491 return ZIPLIST_BYTES(zl
);
494 void ziplistRepr(unsigned char *zl
) {
498 printf("{total bytes %d} {length %u}\n",ZIPLIST_BYTES(zl
), ZIPLIST_LENGTH(zl
));
500 while(*p
!= ZIP_END
) {
502 printf("{offset %ld, header %u, payload %u} ",p
-zl
,entry
.headersize
,entry
.len
);
503 p
+= entry
.headersize
;
504 if (entry
.encoding
== ZIP_ENC_RAW
) {
505 fwrite(p
,entry
.len
,1,stdout
);
507 printf("%lld", zipLoadInteger(p
,entry
.encoding
));
515 #ifdef ZIPLIST_TEST_MAIN
517 unsigned char *createList() {
518 unsigned char *zl
= ziplistNew();
519 zl
= ziplistPush(zl
, (unsigned char*)"foo", 3, ZIPLIST_TAIL
);
520 zl
= ziplistPush(zl
, (unsigned char*)"quux", 4, ZIPLIST_TAIL
);
521 zl
= ziplistPush(zl
, (unsigned char*)"hello", 5, ZIPLIST_HEAD
);
522 zl
= ziplistPush(zl
, (unsigned char*)"1024", 4, ZIPLIST_TAIL
);
526 unsigned char *createIntList() {
527 unsigned char *zl
= ziplistNew();
531 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_TAIL
);
532 sprintf(buf
, "128000");
533 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_TAIL
);
534 sprintf(buf
, "-100");
535 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_HEAD
);
536 sprintf(buf
, "4294967296");
537 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_HEAD
);
538 sprintf(buf
, "non integer");
539 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_TAIL
);
540 sprintf(buf
, "much much longer non integer");
541 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_TAIL
);
545 int main(int argc
, char **argv
) {
546 unsigned char *zl
, *p
, *q
, *entry
;
551 zl
= createIntList();
557 zl
= ziplistPop(zl
, &s
, ZIPLIST_TAIL
);
558 printf("Pop tail: %s (length %ld)\n", s
, sdslen(s
));
561 zl
= ziplistPop(zl
, &s
, ZIPLIST_HEAD
);
562 printf("Pop head: %s (length %ld)\n", s
, sdslen(s
));
565 zl
= ziplistPop(zl
, &s
, ZIPLIST_TAIL
);
566 printf("Pop tail: %s (length %ld)\n", s
, sdslen(s
));
569 zl
= ziplistPop(zl
, &s
, ZIPLIST_TAIL
);
570 printf("Pop tail: %s (length %ld)\n", s
, sdslen(s
));
573 printf("Iterate list from 0 to end:\n");
576 p
= ziplistIndex(zl
, 0);
577 while (ziplistGet(p
, &entry
, &elen
, &value
)) {
580 fwrite(entry
,elen
,1,stdout
);
582 printf("%lld", value
);
590 printf("Iterate list from 1 to end:\n");
593 p
= ziplistIndex(zl
, 1);
594 while (ziplistGet(p
, &entry
, &elen
, &value
)) {
597 fwrite(entry
,elen
,1,stdout
);
599 printf("%lld", value
);
607 printf("Iterate list from 2 to end:\n");
610 p
= ziplistIndex(zl
, 2);
611 while (ziplistGet(p
, &entry
, &elen
, &value
)) {
614 fwrite(entry
,elen
,1,stdout
);
616 printf("%lld", value
);
624 printf("Iterate starting out of range:\n");
627 p
= ziplistIndex(zl
, 4);
628 if (!ziplistGet(p
, &entry
, &elen
, &value
)) {
629 printf("No entry\n");
636 printf("Delete inclusive range 0,0:\n");
639 zl
= ziplistDeleteRange(zl
, 0, 1);
643 printf("Delete inclusive range 0,1:\n");
646 zl
= ziplistDeleteRange(zl
, 0, 2);
650 printf("Delete inclusive range 1,2:\n");
653 zl
= ziplistDeleteRange(zl
, 1, 2);
657 printf("Delete with start index out of range:\n");
660 zl
= ziplistDeleteRange(zl
, 5, 1);
664 printf("Delete with num overflow:\n");
667 zl
= ziplistDeleteRange(zl
, 1, 5);
671 printf("Delete foo while iterating:\n");
674 p
= ziplistIndex(zl
, 0);
675 while (ziplistGet(p
, &entry
, &elen
, &value
)) {
676 if (entry
&& strncmp("foo", entry
, elen
) == 0) {
677 printf("Delete foo\n");
678 zl
= ziplistDelete(zl
, &p
);
682 fwrite(entry
,elen
,1,stdout
);
684 printf("%lld", value
);
694 printf("Compare strings with ziplist entries:\n");
697 p
= ziplistIndex(zl
, 0);
698 if (!ziplistCompare(p
,"hello",5)) {
699 printf("ERROR: not \"hello\"\n");
702 if (ziplistCompare(p
,"hella",5)) {
703 printf("ERROR: \"hella\"\n");
707 p
= ziplistIndex(zl
, 3);
708 if (!ziplistCompare(p
,"1024",4)) {
709 printf("ERROR: not \"1024\"\n");
712 if (ziplistCompare(p
,"1025",4)) {
713 printf("ERROR: \"1025\"\n");