]>
git.saurik.com Git - redis.git/blob - ziplist.c
adb7d0542b5210ec0a421c0c4417d046f9fb034e
1 /* Memory layout of a ziplist, containing "foo", "bar", "quux":
2 * <zlbytes><zllen><len>"foo"<len>"bar"<len>"quux"
4 * <zlbytes> is an unsigned integer to hold the number of bytes that
5 * the ziplist occupies. This is stored to not have to traverse the ziplist
6 * to know the new length when pushing.
8 * <zllen> is the number of items in the ziplist. When this value is
9 * greater than 254, we need to traverse the entire list to know
10 * how many items it holds.
12 * <len> is the number of bytes occupied by a single entry. When this
13 * number is greater than 253, the length will occupy 5 bytes, where
14 * the extra bytes contain an unsigned integer to hold the length.
26 /* Important note: the ZIP_END value is used to depict the end of the
27 * ziplist structure. When a pointer contains an entry, the first couple
28 * of bytes contain the encoded length of the previous entry. This length
29 * is encoded as ZIP_ENC_RAW length, so the first two bits will contain 00
30 * and the byte will therefore never have a value of 255. */
32 #define ZIP_BIGLEN 254
36 #define ZIP_ENC_SHORT 1
38 #define ZIP_ENC_LLONG 3
39 #define ZIP_ENCODING(p) ((p)[0] >> 6)
41 /* Length encoding for raw entries */
42 #define ZIP_LEN_INLINE 0
43 #define ZIP_LEN_UINT16 1
44 #define ZIP_LEN_UINT32 2
47 #define ZIPLIST_BYTES(zl) (*((unsigned int*)(zl)))
48 #define ZIPLIST_TAIL_OFFSET(zl) (*((zl)+sizeof(unsigned int)))
49 #define ZIPLIST_LENGTH(zl) (*((zl)+2*sizeof(unsigned int)))
50 #define ZIPLIST_HEADER_SIZE (2*sizeof(unsigned int)+1)
51 #define ZIPLIST_INCR_LENGTH(zl,incr) { \
52 if (ZIPLIST_LENGTH(zl) < ZIP_BIGLEN) ZIPLIST_LENGTH(zl)+=incr; }
54 /* Return bytes needed to store integer encoded by 'encoding' */
55 static unsigned int zipEncodingSize(char encoding
) {
56 if (encoding
== ZIP_ENC_SHORT
) {
57 return sizeof(short int);
58 } else if (encoding
== ZIP_ENC_INT
) {
60 } else if (encoding
== ZIP_ENC_LLONG
) {
61 return sizeof(long long);
66 /* Decode the encoded length pointed by 'p'. If a pointer to 'lensize' is
67 * provided, it is set to the number of bytes required to encode the length. */
68 static unsigned int zipDecodeLength(unsigned char *p
, unsigned int *lensize
) {
69 unsigned char encoding
= ZIP_ENCODING(p
), lenenc
;
72 if (encoding
== ZIP_ENC_RAW
) {
73 lenenc
= (p
[0] >> 4) & 0x3;
74 if (lenenc
== ZIP_LEN_INLINE
) {
76 if (lensize
) *lensize
= 1;
77 } else if (lenenc
== ZIP_LEN_UINT16
) {
78 len
= p
[1] | (p
[2] << 8);
79 if (lensize
) *lensize
= 3;
81 len
= p
[1] | (p
[2] << 8) | (p
[3] << 16) | (p
[4] << 24);
82 if (lensize
) *lensize
= 5;
85 len
= zipEncodingSize(encoding
);
86 if (lensize
) *lensize
= 1;
91 /* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
92 * the amount of bytes required to encode such a length. */
93 static unsigned int zipEncodeLength(unsigned char *p
, char encoding
, unsigned int rawlen
) {
94 unsigned char len
= 1, lenenc
, buf
[5];
95 if (encoding
== ZIP_ENC_RAW
) {
98 lenenc
= ZIP_LEN_INLINE
;
100 } else if (rawlen
<= 0xffff) {
103 lenenc
= ZIP_LEN_UINT16
;
104 buf
[1] = (rawlen
) & 0xff;
105 buf
[2] = (rawlen
>> 8) & 0xff;
109 lenenc
= ZIP_LEN_UINT32
;
110 buf
[1] = (rawlen
) & 0xff;
111 buf
[2] = (rawlen
>> 8) & 0xff;
112 buf
[3] = (rawlen
>> 16) & 0xff;
113 buf
[4] = (rawlen
>> 24) & 0xff;
115 buf
[0] = (lenenc
<< 4) | (buf
[0] & 0xf);
119 /* Apparently we need to store the length in 'p' */
120 buf
[0] = (encoding
<< 6) | (buf
[0] & 0x3f);
125 /* Return the difference in number of bytes needed to store the new length
126 * "len" on the entry pointed to by "p". */
127 static int zipPrevLenByteDiff(unsigned char *p
, unsigned int len
) {
128 unsigned int prevlensize
;
129 zipDecodeLength(p
,&prevlensize
);
130 return zipEncodeLength(NULL
,ZIP_ENC_RAW
,len
)-prevlensize
;
133 /* Check if string pointed to by 'entry' can be encoded as an integer.
134 * Stores the integer value in 'v' and its encoding in 'encoding'.
135 * Warning: this function requires a NULL-terminated string! */
136 static int zipTryEncoding(unsigned char *entry
, long long *v
, char *encoding
) {
140 if (entry
[0] == '-' || (entry
[0] >= '0' && entry
[0] <= '9')) {
141 value
= strtoll((char*)entry
,&eptr
,10);
142 if (eptr
[0] != '\0') return 0;
143 if (value
>= SHRT_MIN
&& value
<= SHRT_MAX
) {
144 *encoding
= ZIP_ENC_SHORT
;
145 } else if (value
>= INT_MIN
&& value
<= INT_MAX
) {
146 *encoding
= ZIP_ENC_INT
;
148 *encoding
= ZIP_ENC_LLONG
;
156 /* Store integer 'value' at 'p', encoded as 'encoding' */
157 static void zipSaveInteger(unsigned char *p
, long long value
, char encoding
) {
161 if (encoding
== ZIP_ENC_SHORT
) {
163 memcpy(p
,&s
,sizeof(s
));
164 } else if (encoding
== ZIP_ENC_INT
) {
166 memcpy(p
,&i
,sizeof(i
));
167 } else if (encoding
== ZIP_ENC_LLONG
) {
169 memcpy(p
,&l
,sizeof(l
));
175 /* Read integer encoded as 'encoding' from 'p' */
176 static long long zipLoadInteger(unsigned char *p
, char encoding
) {
180 if (encoding
== ZIP_ENC_SHORT
) {
181 memcpy(&s
,p
,sizeof(s
));
183 } else if (encoding
== ZIP_ENC_INT
) {
184 memcpy(&i
,p
,sizeof(i
));
186 } else if (encoding
== ZIP_ENC_LLONG
) {
187 memcpy(&l
,p
,sizeof(l
));
195 /* Return the total amount used by an entry (encoded length + payload). */
196 static unsigned int zipRawEntryLength(unsigned char *p
) {
197 unsigned int prevlensize
, lensize
, len
;
198 /* Byte-size of encoded length of previous entry */
199 zipDecodeLength(p
,&prevlensize
);
200 /* Encoded length of this entry's payload */
201 len
= zipDecodeLength(p
+prevlensize
, &lensize
);
202 return prevlensize
+lensize
+len
;
205 /* Create a new empty ziplist. */
206 unsigned char *ziplistNew(void) {
207 unsigned int bytes
= ZIPLIST_HEADER_SIZE
+1;
208 unsigned char *zl
= zmalloc(bytes
);
209 ZIPLIST_BYTES(zl
) = bytes
;
210 ZIPLIST_TAIL_OFFSET(zl
) = ZIPLIST_HEADER_SIZE
;
211 ZIPLIST_LENGTH(zl
) = 0;
212 zl
[bytes
-1] = ZIP_END
;
216 /* Resize the ziplist. */
217 static unsigned char *ziplistResize(unsigned char *zl
, unsigned int len
) {
218 zl
= zrealloc(zl
,len
);
219 ZIPLIST_BYTES(zl
) = len
;
224 static unsigned char *ziplistHead(unsigned char *zl
) {
225 return zl
+ZIPLIST_HEADER_SIZE
;
228 static unsigned char *ziplistTail(unsigned char *zl
) {
229 unsigned char *p
, *q
;
230 p
= q
= ziplistHead(zl
);
231 while (*p
!= ZIP_END
) {
233 p
+= zipRawEntryLength(p
);
238 unsigned char *ziplistPush(unsigned char *zl
, unsigned char *entry
, unsigned int elen
, int where
) {
239 unsigned int curlen
= ZIPLIST_BYTES(zl
), reqlen
, prevlen
;
240 unsigned char *p
, *curtail
;
241 char encoding
= ZIP_ENC_RAW
;
244 /* We need to store the length of the current tail when the list
245 * is non-empty and we push at the tail. */
246 curtail
= zl
+ZIPLIST_TAIL_OFFSET(zl
);
247 if (where
== ZIPLIST_TAIL
&& curtail
[0] != ZIP_END
) {
248 prevlen
= zipRawEntryLength(curtail
);
253 /* See if the entry can be encoded */
254 if (zipTryEncoding(entry
,&value
,&encoding
)) {
255 reqlen
= zipEncodingSize(encoding
);
260 /* We need space for both the length of the previous entry and
261 * the length of the payload. */
262 reqlen
+= zipEncodeLength(NULL
,ZIP_ENC_RAW
,prevlen
);
263 reqlen
+= zipEncodeLength(NULL
,encoding
,elen
);
265 /* Resize the ziplist and move if needed */
266 zl
= ziplistResize(zl
,curlen
+reqlen
);
267 if (where
== ZIPLIST_HEAD
) {
268 p
= zl
+ZIPLIST_HEADER_SIZE
;
270 /* Subtract one because of the ZIP_END bytes */
271 memmove(p
+reqlen
,p
,curlen
-ZIPLIST_HEADER_SIZE
-1);
277 /* Update tail offset if this is not the first element */
278 if (curtail
[0] != ZIP_END
) {
279 if (where
== ZIPLIST_HEAD
) {
280 ZIPLIST_TAIL_OFFSET(zl
) += reqlen
;
282 ZIPLIST_TAIL_OFFSET(zl
) += prevlen
;
286 /* Write the entry */
287 p
+= zipEncodeLength(p
,ZIP_ENC_RAW
,prevlen
);
288 p
+= zipEncodeLength(p
,encoding
,elen
);
289 if (encoding
!= ZIP_ENC_RAW
) {
290 zipSaveInteger(p
,value
,encoding
);
292 memcpy(p
,entry
,elen
);
294 ZIPLIST_INCR_LENGTH(zl
,1);
298 unsigned char *ziplistPop(unsigned char *zl
, sds
*target
, int where
) {
299 unsigned int curlen
= ZIPLIST_BYTES(zl
), rawlen
;
300 unsigned int prevrawlensize
, prevrawlen
, lensize
, len
, headerlen
;
302 unsigned char encoding
;
305 if (target
) *target
= NULL
;
307 /* Get pointer to element to remove */
308 p
= (where
== ZIPLIST_HEAD
) ? ziplistHead(zl
) : ziplistTail(zl
);
309 if (*p
== ZIP_END
) return zl
;
311 prevrawlen
= zipDecodeLength(p
,&prevrawlensize
);
312 len
= zipDecodeLength(p
+prevrawlensize
,&lensize
);
313 headerlen
= prevrawlensize
+lensize
;
314 rawlen
= headerlen
+len
;
316 encoding
= ZIP_ENCODING(p
+prevrawlensize
);
317 if (encoding
== ZIP_ENC_RAW
) {
318 *target
= sdsnewlen(p
+headerlen
,len
);
320 value
= zipLoadInteger(p
+headerlen
,encoding
);
321 *target
= sdscatprintf(sdsempty(), "%lld", value
);
325 if (where
== ZIPLIST_HEAD
) {
326 /* The next entry will now be the head of the list */
327 if (p
[rawlen
] != ZIP_END
) {
328 /* Tricky: storing the length of the previous entry in the next
329 * entry (this previous length is always 0 when popping from the
330 * head), might reduce the number of bytes needed.
332 * In this special case (new length is 0), we know that the
333 * byte difference to store is always <= 0, which means that
334 * we always have space to store it. */
335 nextdiff
= zipPrevLenByteDiff(p
+rawlen
,0);
336 zipEncodeLength(p
+rawlen
-nextdiff
,ZIP_ENC_RAW
,0);
338 /* Move list to the front */
339 memmove(p
,p
+rawlen
-nextdiff
,curlen
-ZIPLIST_HEADER_SIZE
-rawlen
+nextdiff
);
341 /* Subtract the gained space from the tail offset */
342 ZIPLIST_TAIL_OFFSET(zl
) -= rawlen
+nextdiff
;
344 /* Subtract the length of the previous element from the tail offset. */
345 ZIPLIST_TAIL_OFFSET(zl
) -= prevrawlen
;
348 /* Resize and update length */
349 zl
= ziplistResize(zl
,curlen
-rawlen
+nextdiff
);
350 ZIPLIST_INCR_LENGTH(zl
,-1);
354 /* Returns an offset to use for iterating with ziplistNext. */
355 unsigned char *ziplistIndex(unsigned char *zl
, unsigned int index
) {
356 unsigned char *p
= zl
+ZIPLIST_HEADER_SIZE
;
358 for (; i
< index
; i
++) {
359 if (*p
== ZIP_END
) break;
360 p
+= zipRawEntryLength(p
);
365 /* Return pointer to next entry in ziplist. */
366 unsigned char *ziplistNext(unsigned char *p
) {
367 return *p
== ZIP_END
? p
: p
+zipRawEntryLength(p
);
370 /* Get entry pointer to by 'p' and store in either 'e' or 'v' depending
371 * on the encoding of the entry. 'e' is always set to NULL to be able
372 * to find out whether the string pointer or the integer value was set.
373 * Return 0 if 'p' points to the end of the zipmap, 1 otherwise. */
374 unsigned int ziplistGet(unsigned char *p
, unsigned char **e
, unsigned int *elen
, long long *v
) {
375 unsigned int prevrawlensize
, lensize
, len
, headerlen
;
376 if (*p
== ZIP_END
) return 0;
379 zipDecodeLength(p
,&prevrawlensize
);
380 len
= zipDecodeLength(p
+prevrawlensize
,&lensize
);
381 headerlen
= prevrawlensize
+lensize
;
382 if (ZIP_ENCODING(p
+prevrawlensize
) == ZIP_ENC_RAW
) {
389 *v
= zipLoadInteger(p
+headerlen
,ZIP_ENCODING(p
+prevrawlensize
));
395 /* Delete a range of entries from the ziplist. */
396 unsigned char *ziplistDeleteRange(unsigned char *zl
, unsigned int index
, unsigned int num
) {
397 unsigned char *p
, *first
= ziplistIndex(zl
, index
);
398 unsigned int i
, totlen
, deleted
= 0;
399 for (p
= first
, i
= 0; *p
!= ZIP_END
&& i
< num
; i
++) {
400 p
+= zipRawEntryLength(p
);
406 /* Move current tail to the new tail when there *is* a tail */
407 if (*p
!= ZIP_END
) memmove(first
,p
,ZIPLIST_BYTES(zl
)-(p
-zl
)-1);
409 /* Resize and update length */
410 zl
= ziplistResize(zl
, ZIPLIST_BYTES(zl
)-totlen
);
411 ZIPLIST_INCR_LENGTH(zl
,-deleted
);
416 /* Delete a single entry from the ziplist, pointed to by *p.
417 * Also update *p in place, to be able to iterate over the
418 * ziplist, while deleting entries. */
419 unsigned char *ziplistDelete(unsigned char *zl
, unsigned char **p
) {
420 unsigned int offset
= *p
-zl
, tail
, len
;
421 len
= zipRawEntryLength(*p
);
422 tail
= ZIPLIST_BYTES(zl
)-offset
-len
-1;
424 /* Move current tail to the new tail when there *is* a tail */
425 if (tail
> 0) memmove(*p
,*p
+len
,tail
);
427 /* Resize and update length */
428 zl
= ziplistResize(zl
, ZIPLIST_BYTES(zl
)-len
);
429 ZIPLIST_INCR_LENGTH(zl
,-1);
431 /* Store new pointer to current element in p.
432 * This needs to be done because zl can change on realloc. */
437 /* Compare entry pointer to by 'p' with 'entry'. Return 1 if equal. */
438 unsigned int ziplistCompare(unsigned char *p
, unsigned char *entry
, unsigned int elen
) {
439 unsigned int prevrawlensize
, lensize
, len
, headerlen
;
442 if (*p
== ZIP_END
) return 0;
444 zipDecodeLength(p
,&prevrawlensize
);
445 len
= zipDecodeLength(p
+prevrawlensize
,&lensize
);
446 headerlen
= prevrawlensize
+lensize
;
447 if (ZIP_ENCODING(p
+prevrawlensize
) == ZIP_ENC_RAW
) {
450 return memcmp(p
+headerlen
,entry
,elen
) == 0;
455 if (zipTryEncoding(entry
,&eval
,&encoding
)) {
456 /* Do integer compare */
457 val
= zipLoadInteger(p
+headerlen
,ZIP_ENCODING(p
+prevrawlensize
));
460 /* Ziplist entry is integer encoded, but given entry is not. */
466 /* Return length of ziplist. */
467 unsigned int ziplistLen(unsigned char *zl
) {
468 unsigned int len
= 0;
469 if (ZIPLIST_LENGTH(zl
) < ZIP_BIGLEN
) {
470 len
= ZIPLIST_LENGTH(zl
);
472 unsigned char *p
= zl
+ZIPLIST_HEADER_SIZE
;
473 while (*p
!= ZIP_END
) {
474 p
+= zipRawEntryLength(p
);
478 /* Re-store length if small enough */
479 if (len
< ZIP_BIGLEN
) ZIPLIST_LENGTH(zl
) = len
;
484 /* Return size in bytes of ziplist. */
485 unsigned int ziplistSize(unsigned char *zl
) {
486 return ZIPLIST_BYTES(zl
);
489 void ziplistRepr(unsigned char *zl
) {
490 unsigned char *p
, encoding
;
491 unsigned int prevrawlensize
, prevrawlen
, lensize
, len
;
493 printf("{total bytes %d} {length %u}\n",ZIPLIST_BYTES(zl
), ZIPLIST_LENGTH(zl
));
495 while(*p
!= ZIP_END
) {
496 prevrawlen
= zipDecodeLength(p
,&prevrawlensize
);
497 len
= zipDecodeLength(p
+prevrawlensize
,&lensize
);
498 printf("{offset %ld, header %u, payload %u} ",p
-zl
,prevrawlensize
+lensize
,len
);
499 encoding
= ZIP_ENCODING(p
+prevrawlensize
);
500 p
+= prevrawlensize
+lensize
;
501 if (encoding
== ZIP_ENC_RAW
) {
502 fwrite(p
,len
,1,stdout
);
504 printf("%lld", zipLoadInteger(p
,encoding
));
512 #ifdef ZIPLIST_TEST_MAIN
514 unsigned char *createList() {
515 unsigned char *zl
= ziplistNew();
516 zl
= ziplistPush(zl
, (unsigned char*)"foo", 3, ZIPLIST_TAIL
);
517 zl
= ziplistPush(zl
, (unsigned char*)"quux", 4, ZIPLIST_TAIL
);
518 zl
= ziplistPush(zl
, (unsigned char*)"hello", 5, ZIPLIST_HEAD
);
519 zl
= ziplistPush(zl
, (unsigned char*)"1024", 4, ZIPLIST_TAIL
);
523 unsigned char *createIntList() {
524 unsigned char *zl
= ziplistNew();
528 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_TAIL
);
529 sprintf(buf
, "128000");
530 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_TAIL
);
531 sprintf(buf
, "-100");
532 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_HEAD
);
533 sprintf(buf
, "4294967296");
534 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_HEAD
);
535 sprintf(buf
, "non integer");
536 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_TAIL
);
537 sprintf(buf
, "much much longer non integer");
538 zl
= ziplistPush(zl
, buf
, strlen(buf
), ZIPLIST_TAIL
);
542 int main(int argc
, char **argv
) {
543 unsigned char *zl
, *p
, *q
, *entry
;
548 zl
= createIntList();
554 zl
= ziplistPop(zl
, &s
, ZIPLIST_TAIL
);
555 printf("Pop tail: %s (length %ld)\n", s
, sdslen(s
));
558 zl
= ziplistPop(zl
, &s
, ZIPLIST_HEAD
);
559 printf("Pop head: %s (length %ld)\n", s
, sdslen(s
));
562 zl
= ziplistPop(zl
, &s
, ZIPLIST_TAIL
);
563 printf("Pop tail: %s (length %ld)\n", s
, sdslen(s
));
566 zl
= ziplistPop(zl
, &s
, ZIPLIST_TAIL
);
567 printf("Pop tail: %s (length %ld)\n", s
, sdslen(s
));
570 printf("Iterate list from 0 to end:\n");
573 p
= ziplistIndex(zl
, 0);
574 while (ziplistGet(p
, &entry
, &elen
, &value
)) {
577 fwrite(entry
,elen
,1,stdout
);
579 printf("%lld", value
);
587 printf("Iterate list from 1 to end:\n");
590 p
= ziplistIndex(zl
, 1);
591 while (ziplistGet(p
, &entry
, &elen
, &value
)) {
594 fwrite(entry
,elen
,1,stdout
);
596 printf("%lld", value
);
604 printf("Iterate list from 2 to end:\n");
607 p
= ziplistIndex(zl
, 2);
608 while (ziplistGet(p
, &entry
, &elen
, &value
)) {
611 fwrite(entry
,elen
,1,stdout
);
613 printf("%lld", value
);
621 printf("Iterate starting out of range:\n");
624 p
= ziplistIndex(zl
, 4);
625 if (!ziplistGet(p
, &entry
, &elen
, &value
)) {
626 printf("No entry\n");
633 printf("Delete inclusive range 0,0:\n");
636 zl
= ziplistDeleteRange(zl
, 0, 1);
640 printf("Delete inclusive range 0,1:\n");
643 zl
= ziplistDeleteRange(zl
, 0, 2);
647 printf("Delete inclusive range 1,2:\n");
650 zl
= ziplistDeleteRange(zl
, 1, 2);
654 printf("Delete with start index out of range:\n");
657 zl
= ziplistDeleteRange(zl
, 5, 1);
661 printf("Delete with num overflow:\n");
664 zl
= ziplistDeleteRange(zl
, 1, 5);
668 printf("Delete foo while iterating:\n");
671 p
= ziplistIndex(zl
, 0);
672 while (ziplistGet(p
, &entry
, &elen
, &value
)) {
673 if (entry
&& strncmp("foo", entry
, elen
) == 0) {
674 printf("Delete foo\n");
675 zl
= ziplistDelete(zl
, &p
);
679 fwrite(entry
,elen
,1,stdout
);
681 printf("%lld", value
);
691 printf("Compare strings with ziplist entries:\n");
694 p
= ziplistIndex(zl
, 0);
695 if (!ziplistCompare(p
,"hello",5)) {
696 printf("ERROR: not \"hello\"\n");
699 if (ziplistCompare(p
,"hella",5)) {
700 printf("ERROR: \"hella\"\n");
704 p
= ziplistIndex(zl
, 3);
705 if (!ziplistCompare(p
,"1024",4)) {
706 printf("ERROR: not \"1024\"\n");
709 if (ziplistCompare(p
,"1025",4)) {
710 printf("ERROR: \"1025\"\n");