X-Git-Url: https://git.saurik.com/redis.git/blobdiff_plain/26f3388d27d1be9312d0244890db564c1b22ba0c..03af999cb0befed9d2556c2df1eb0511de2d96d2:/src/ziplist.c diff --git a/src/ziplist.c b/src/ziplist.c index 4f44bd58..55bb662b 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -68,6 +68,7 @@ #include #include "zmalloc.h" #include "ziplist.h" +#include "endian.h" int ll2string(char *s, size_t len, long long value); @@ -119,6 +120,7 @@ static unsigned int zipEntryEncoding(unsigned char *p) { return p[0] & 0xf0; } assert(NULL); + return 0; } /* Return bytes needed to store integer encoded by 'encoding' */ @@ -129,13 +131,14 @@ static unsigned int zipIntSize(unsigned char encoding) { case ZIP_INT_64B: return sizeof(int64_t); } assert(NULL); + return 0; } /* Decode the encoded length pointed by 'p'. If a pointer to 'lensize' is * provided, it is set to the number of bytes required to encode the length. */ static unsigned int zipDecodeLength(unsigned char *p, unsigned int *lensize) { unsigned char encoding = zipEntryEncoding(p); - unsigned int len; + unsigned int len = 0; if (ZIP_IS_STR(encoding)) { switch(encoding) { @@ -205,6 +208,7 @@ static unsigned int zipPrevDecodeLength(unsigned char *p, unsigned int *lensize) } else { if (lensize) *lensize = 1+sizeof(len); memcpy(&len,p+1,sizeof(len)); + memrev32ifbe(&len); } return len; } @@ -221,6 +225,7 @@ static unsigned int zipPrevEncodeLength(unsigned char *p, unsigned int len) { } else { p[0] = ZIP_BIGLEN; memcpy(p+1,&len,sizeof(len)); + memrev32ifbe(p+1); return 1+sizeof(len); } } @@ -232,6 +237,7 @@ static void zipPrevEncodeLengthForceLarge(unsigned char *p, unsigned int len) { if (p == NULL) return; p[0] = ZIP_BIGLEN; memcpy(p+1,&len,sizeof(len)); + memrev32ifbe(p+1); } /* Return the difference in number of bytes needed to store the new length @@ -285,12 +291,15 @@ static void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encodi if (encoding == ZIP_INT_16B) { i16 = value; memcpy(p,&i16,sizeof(i16)); + memrev16ifbe(p); } else if (encoding == ZIP_INT_32B) { i32 = value; memcpy(p,&i32,sizeof(i32)); + memrev32ifbe(p); } else if (encoding == ZIP_INT_64B) { i64 = value; memcpy(p,&i64,sizeof(i64)); + memrev64ifbe(p); } else { assert(NULL); } @@ -300,15 +309,18 @@ static void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encodi static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) { int16_t i16; int32_t i32; - int64_t i64, ret; + int64_t i64, ret = 0; if (encoding == ZIP_INT_16B) { memcpy(&i16,p,sizeof(i16)); + memrev16ifbe(&i16); ret = i16; } else if (encoding == ZIP_INT_32B) { memcpy(&i32,p,sizeof(i32)); + memrev16ifbe(&i32); ret = i32; } else if (encoding == ZIP_INT_64B) { memcpy(&i64,p,sizeof(i64)); + memrev16ifbe(&i64); ret = i64; } else { assert(NULL); @@ -409,6 +421,7 @@ static unsigned char *__ziplistCascadeUpdate(unsigned char *zl, unsigned char *p /* Advance the cursor */ p += rawlen; + curlen += extra; } else { if (next.prevrawlensize > rawlensize) { /* This would result in shrinking, which we want to avoid. @@ -594,7 +607,12 @@ unsigned char *ziplistIndex(unsigned char *zl, int index) { return (p[0] == ZIP_END || index > 0) ? NULL : p; } -/* Return pointer to next entry in ziplist. */ +/* Return pointer to next entry in ziplist. + * + * zl is the pointer to the ziplist + * p is the pointer to the current element + * + * The element after 'p' is returned, otherwise NULL if we are at the end. */ unsigned char *ziplistNext(unsigned char *zl, unsigned char *p) { ((void) zl); @@ -722,8 +740,8 @@ unsigned int ziplistLen(unsigned char *zl) { return len; } -/* Return size in bytes of ziplist. */ -unsigned int ziplistSize(unsigned char *zl) { +/* Return ziplist blob size in bytes. */ +size_t ziplistBlobLen(unsigned char *zl) { return ZIPLIST_BYTES(zl); } @@ -753,9 +771,9 @@ void ziplistRepr(unsigned char *zl) { "pls: %2u, " "payload %5u" "} ", - (long unsigned int)p, + (long unsigned)p, index, - p-zl, + (unsigned long) (p-zl), entry.headersize+entry.len, entry.headersize, entry.prevrawlen, @@ -764,10 +782,11 @@ void ziplistRepr(unsigned char *zl) { p += entry.headersize; if (ZIP_IS_STR(entry.encoding)) { if (entry.len > 40) { - fwrite(p,40,1,stdout); + if (fwrite(p,40,1,stdout) == 0) perror("fwrite"); printf("..."); } else { - fwrite(p,entry.len,1,stdout); + if (entry.len && + fwrite(p,entry.len,1,stdout) == 0) perror("fwrite"); } } else { printf("%lld", (long long) zipLoadInteger(p,entry.encoding)); @@ -781,6 +800,10 @@ void ziplistRepr(unsigned char *zl) { #ifdef ZIPLIST_TEST_MAIN #include +#include "adlist.h" +#include "sds.h" + +#define debug(f, ...) { if (DEBUG) printf(f, __VA_ARGS__); } unsigned char *createList() { unsigned char *zl = ziplistNew(); @@ -852,7 +875,7 @@ void pop(unsigned char *zl, int where) { printf("Pop tail: "); if (vstr) - fwrite(vstr,vlen,1,stdout); + if (vlen && fwrite(vstr,vlen,1,stdout) == 0) perror("fwrite"); else printf("%lld", vlong); @@ -864,6 +887,32 @@ void pop(unsigned char *zl, int where) { } } +void randstring(char *target, unsigned int min, unsigned int max) { + int p, len = min+rand()%(max-min+1); + int minval, maxval; + switch(rand() % 3) { + case 0: + minval = 0; + maxval = 255; + break; + case 1: + minval = 48; + maxval = 122; + break; + case 2: + minval = 48; + maxval = 52; + break; + default: + assert(NULL); + } + + while(p < len) + target[p++] = minval+rand()%(maxval-minval+1); + return; +} + + int main(int argc, char **argv) { unsigned char *zl, *p; unsigned char *entry; @@ -901,7 +950,7 @@ int main(int argc, char **argv) { return 1; } if (entry) { - fwrite(entry,elen,1,stdout); + if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite"); printf("\n"); } else { printf("%lld\n", value); @@ -931,7 +980,7 @@ int main(int argc, char **argv) { return 1; } if (entry) { - fwrite(entry,elen,1,stdout); + if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite"); printf("\n"); } else { printf("%lld\n", value); @@ -948,7 +997,7 @@ int main(int argc, char **argv) { return 1; } if (entry) { - fwrite(entry,elen,1,stdout); + if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite"); printf("\n"); } else { printf("%lld\n", value); @@ -976,7 +1025,7 @@ int main(int argc, char **argv) { while (ziplistGet(p, &entry, &elen, &value)) { printf("Entry: "); if (entry) { - fwrite(entry,elen,1,stdout); + if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite"); } else { printf("%lld", value); } @@ -993,7 +1042,7 @@ int main(int argc, char **argv) { while (ziplistGet(p, &entry, &elen, &value)) { printf("Entry: "); if (entry) { - fwrite(entry,elen,1,stdout); + if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite"); } else { printf("%lld", value); } @@ -1010,7 +1059,7 @@ int main(int argc, char **argv) { while (ziplistGet(p, &entry, &elen, &value)) { printf("Entry: "); if (entry) { - fwrite(entry,elen,1,stdout); + if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite"); } else { printf("%lld", value); } @@ -1039,7 +1088,7 @@ int main(int argc, char **argv) { while (ziplistGet(p, &entry, &elen, &value)) { printf("Entry: "); if (entry) { - fwrite(entry,elen,1,stdout); + if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite"); } else { printf("%lld", value); } @@ -1056,7 +1105,7 @@ int main(int argc, char **argv) { while (ziplistGet(p, &entry, &elen, &value)) { printf("Entry: "); if (entry) { - fwrite(entry,elen,1,stdout); + if (elen && fwrite(entry,elen,1,stdout) == 0) perror("fwrite"); } else { printf("%lld", value); } @@ -1113,7 +1162,8 @@ int main(int argc, char **argv) { } else { printf("Entry: "); if (entry) { - fwrite(entry,elen,1,stdout); + if (elen && fwrite(entry,elen,1,stdout) == 0) + perror("fwrite"); } else { printf("%lld",value); } @@ -1137,10 +1187,10 @@ int main(int argc, char **argv) { /* Pop values again and compare their value. */ p = ziplistIndex(zl,0); assert(ziplistGet(p,&entry,&elen,&value)); - assert(strncmp(v1,entry,elen) == 0); + assert(strncmp(v1,(char*)entry,elen) == 0); p = ziplistIndex(zl,1); assert(ziplistGet(p,&entry,&elen,&value)); - assert(strncmp(v2,entry,elen) == 0); + assert(strncmp(v2,(char*)entry,elen) == 0); printf("SUCCESS\n\n"); } @@ -1192,50 +1242,78 @@ int main(int argc, char **argv) { printf("Stress with random payloads of different encoding:\n"); { - int i, idx, where, len; - long long v; + int i,j,len,where; unsigned char *p; - char buf[0x4041]; /* max length of generated string */ - zl = ziplistNew(); - for (i = 0; i < 100000; i++) { - where = (rand() & 1) ? ZIPLIST_HEAD : ZIPLIST_TAIL; - if (rand() & 1) { - /* equally likely create a 16, 32 or 64 bit int */ - v = (rand() & INT16_MAX) + ((1ll << 32) >> ((rand() % 3)*16)); - v *= 2*(rand() & 1)-1; /* randomly flip sign */ - sprintf(buf, "%lld", v); + char buf[1024]; + list *ref; + listNode *refnode; + + /* Hold temp vars from ziplist */ + unsigned char *sstr; + unsigned int slen; + long long sval; + + /* In the regression for the cascade bug, it was triggered + * with a random seed of 2. */ + srand(2); + + for (i = 0; i < 20000; i++) { + zl = ziplistNew(); + ref = listCreate(); + listSetFreeMethod(ref,sdsfree); + len = rand() % 256; + + /* Create lists */ + for (j = 0; j < len; j++) { + where = (rand() & 1) ? ZIPLIST_HEAD : ZIPLIST_TAIL; + switch(rand() % 4) { + case 0: + sprintf(buf,"%lld",(0LL + rand()) >> 20); + break; + case 1: + sprintf(buf,"%lld",(0LL + rand())); + break; + case 2: + sprintf(buf,"%lld",(0LL + rand()) << 20); + break; + case 3: + randstring(buf,0,256); + break; + default: + assert(NULL); + } + + /* Add to ziplist */ zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), where); - } else { - /* equally likely generate 6, 14 or >14 bit length */ - v = rand() & 0x3f; - v += 0x4000 >> ((rand() % 3)*8); - memset(buf, 'x', v); - zl = ziplistPush(zl, (unsigned char*)buf, v, where); - } - /* delete a random element */ - if ((len = ziplistLen(zl)) >= 10) { - idx = rand() % len; - // printf("Delete index %d\n", idx); - // ziplistRepr(zl); - ziplistDeleteRange(zl, idx, 1); - // ziplistRepr(zl); - len--; + /* Add to reference list */ + if (where == ZIPLIST_HEAD) { + listAddNodeHead(ref,sdsnew(buf)); + } else if (where == ZIPLIST_TAIL) { + listAddNodeTail(ref,sdsnew(buf)); + } else { + assert(NULL); + } } - /* iterate from front to back */ - idx = 0; - p = ziplistIndex(zl, 0); - while((p = ziplistNext(zl,p))) - idx++; - assert(len == idx+1); - - /* iterate from back to front */ - idx = 0; - p = ziplistIndex(zl, -1); - while((p = ziplistPrev(zl,p))) - idx++; - assert(len == idx+1); + assert(listLength(ref) == ziplistLen(zl)); + for (j = 0; j < len; j++) { + /* Naive way to get elements, but similar to the stresser + * executed from the Tcl test suite. */ + p = ziplistIndex(zl,j); + refnode = listIndex(ref,j); + + assert(ziplistGet(p,&sstr,&slen,&sval)); + if (sstr == NULL) { + sprintf(buf,"%lld",sval); + } else { + memcpy(buf,sstr,slen); + buf[slen] = '\0'; + } + assert(strcmp(buf,listNodeValue(refnode)) == 0); + } + zfree(zl); + listRelease(ref); } printf("SUCCESS\n\n"); }