]>
git.saurik.com Git - redis.git/blob - src/redis-check-dump.c
5eac925ae98a358b12a5d6c74a4b8fee2deb9238
  14 #define REDIS_STRING 0 
  19 #define REDIS_HASH_ZIPMAP 9 
  20 #define REDIS_LIST_ZIPLIST 10 
  21 #define REDIS_SET_INTSET 11 
  22 #define REDIS_ZSET_ZIPLIST 12 
  23 #define REDIS_HASH_ZIPLIST 13 
  25 /* Objects encoding. Some kind of objects like Strings and Hashes can be 
  26  * internally represented in multiple ways. The 'encoding' field of the object 
  27  * is set to one of this fields for this object. */ 
  28 #define REDIS_ENCODING_RAW 0    /* Raw representation */ 
  29 #define REDIS_ENCODING_INT 1    /* Encoded as integer */ 
  30 #define REDIS_ENCODING_ZIPMAP 2 /* Encoded as zipmap */ 
  31 #define REDIS_ENCODING_HT 3     /* Encoded as an hash table */ 
  33 /* Object types only used for dumping to disk */ 
  34 #define REDIS_EXPIRETIME 253 
  35 #define REDIS_SELECTDB 254 
  38 /* Defines related to the dump file format. To store 32 bits lengths for short 
  39  * keys requires a lot of space, so we check the most significant 2 bits of 
  40  * the first byte to interpreter the length: 
  42  * 00|000000 => if the two MSB are 00 the len is the 6 bits of this byte 
  43  * 01|000000 00000000 =>  01, the len is 14 byes, 6 bits + 8 bits of next byte 
  44  * 10|000000 [32 bit integer] => if it's 01, a full 32 bit len will follow 
  45  * 11|000000 this means: specially encoded object will follow. The six bits 
  46  *           number specify the kind of object that follows. 
  47  *           See the REDIS_RDB_ENC_* defines. 
  49  * Lenghts up to 63 are stored using a single byte, most DB keys, and may 
  50  * values, will fit inside. */ 
  51 #define REDIS_RDB_6BITLEN 0 
  52 #define REDIS_RDB_14BITLEN 1 
  53 #define REDIS_RDB_32BITLEN 2 
  54 #define REDIS_RDB_ENCVAL 3 
  55 #define REDIS_RDB_LENERR UINT_MAX 
  57 /* When a length of a string object stored on disk has the first two bits 
  58  * set, the remaining two bits specify a special encoding for the object 
  59  * accordingly to the following defines: */ 
  60 #define REDIS_RDB_ENC_INT8 0        /* 8 bit signed integer */ 
  61 #define REDIS_RDB_ENC_INT16 1       /* 16 bit signed integer */ 
  62 #define REDIS_RDB_ENC_INT32 2       /* 32 bit signed integer */ 
  63 #define REDIS_RDB_ENC_LZF 3         /* string compressed with FASTLZ */ 
  65 #define ERROR(...) { \ 
  66     printf(__VA_ARGS__); \ 
  70 /* data type to hold offset in file and size */ 
  77 static unsigned char level 
= 0; 
  78 static pos positions
[16]; 
  80 #define CURR_OFFSET (positions[level].offset) 
  82 /* Hold a stack of errors */ 
  88 static errors_t errors
; 
  90 #define SHIFT_ERROR(provided_offset, ...) { \ 
  91     sprintf(errors.error[errors.level], __VA_ARGS__); \ 
  92     errors.offset[errors.level] = provided_offset; \ 
  96 /* Data type to hold opcode with optional key name an success status */ 
 103 /* Global vars that are actally used as constants. The following double 
 104  * values are used for double on-disk serialization, and are initialized 
 105  * at runtime to avoid strange compiler optimizations. */ 
 106 static double R_Zero
, R_PosInf
, R_NegInf
, R_Nan
; 
 108 /* store string types for output */ 
 109 static char types
[256][16]; 
 112 uint64_t crc64(uint64_t crc
, const unsigned char *s
, uint64_t l
); 
 114 /* when number of bytes to read is negative, do a peek */ 
 115 int readBytes(void *target
, long num
) { 
 116     char peek 
= (num 
< 0) ? 1 : 0; 
 117     num 
= (num 
< 0) ? -num 
: num
; 
 119     pos p 
= positions
[level
]; 
 120     if (p
.offset 
+ num 
> p
.size
) { 
 123         memcpy(target
, (void*)((size_t)p
.data 
+ p
.offset
), num
); 
 124         if (!peek
) positions
[level
].offset 
+= num
; 
 129 int processHeader() { 
 130     char buf
[10] = "_________"; 
 133     if (!readBytes(buf
, 9)) { 
 134         ERROR("Cannot read header\n"); 
 137     /* expect the first 5 bytes to equal REDIS */ 
 138     if (memcmp(buf
,"REDIS",5) != 0) { 
 139         ERROR("Wrong signature in header\n"); 
 142     dump_version 
= (int)strtol(buf 
+ 5, NULL
, 10); 
 143     if (dump_version 
< 1 || dump_version 
> 6) { 
 144         ERROR("Unknown RDB format version: %d\n", dump_version
); 
 149 int loadType(entry 
*e
) { 
 150     uint32_t offset 
= CURR_OFFSET
; 
 152     /* this byte needs to qualify as type */ 
 154     if (readBytes(&t
, 1)) { 
 155         if (t 
<= 4 || (t 
>=9 && t 
<= 12) || t 
>= 253) { 
 159             SHIFT_ERROR(offset
, "Unknown type (0x%02x)", t
); 
 162         SHIFT_ERROR(offset
, "Could not read type"); 
 171     if (readBytes(&t
, -1) && (t 
<= 4 || (t 
>=9 && t 
<= 12) || t 
>= 253)) 
 176 /* discard time, just consume the bytes */ 
 178     uint32_t offset 
= CURR_OFFSET
; 
 180     if (readBytes(t
, 4)) { 
 183         SHIFT_ERROR(offset
, "Could not read time"); 
 190 uint32_t loadLength(int *isencoded
) { 
 191     unsigned char buf
[2]; 
 195     if (isencoded
) *isencoded 
= 0; 
 196     if (!readBytes(buf
, 1)) return REDIS_RDB_LENERR
; 
 197     type 
= (buf
[0] & 0xC0) >> 6; 
 198     if (type 
== REDIS_RDB_6BITLEN
) { 
 199         /* Read a 6 bit len */ 
 200         return buf
[0] & 0x3F; 
 201     } else if (type 
== REDIS_RDB_ENCVAL
) { 
 202         /* Read a 6 bit len encoding type */ 
 203         if (isencoded
) *isencoded 
= 1; 
 204         return buf
[0] & 0x3F; 
 205     } else if (type 
== REDIS_RDB_14BITLEN
) { 
 206         /* Read a 14 bit len */ 
 207         if (!readBytes(buf
+1,1)) return REDIS_RDB_LENERR
; 
 208         return ((buf
[0] & 0x3F) << 8) | buf
[1]; 
 210         /* Read a 32 bit len */ 
 211         if (!readBytes(&len
, 4)) return REDIS_RDB_LENERR
; 
 212         return (unsigned int)ntohl(len
); 
 216 char *loadIntegerObject(int enctype
) { 
 217     uint32_t offset 
= CURR_OFFSET
; 
 218     unsigned char enc
[4]; 
 221     if (enctype 
== REDIS_RDB_ENC_INT8
) { 
 223         if (!readBytes(enc
, 1)) return NULL
; 
 226     } else if (enctype 
== REDIS_RDB_ENC_INT16
) { 
 228         if (!readBytes(enc
, 2)) return NULL
; 
 229         v 
= enc
[0]|(enc
[1]<<8); 
 231     } else if (enctype 
== REDIS_RDB_ENC_INT32
) { 
 233         if (!readBytes(enc
, 4)) return NULL
; 
 234         v 
= enc
[0]|(enc
[1]<<8)|(enc
[2]<<16)|(enc
[3]<<24); 
 237         SHIFT_ERROR(offset
, "Unknown integer encoding (0x%02x)", enctype
); 
 241     /* convert val into string */ 
 243     buf 
= malloc(sizeof(char) * 128); 
 244     sprintf(buf
, "%lld", val
); 
 248 char* loadLzfStringObject() { 
 249     unsigned int slen
, clen
; 
 252     if ((clen 
= loadLength(NULL
)) == REDIS_RDB_LENERR
) return NULL
; 
 253     if ((slen 
= loadLength(NULL
)) == REDIS_RDB_LENERR
) return NULL
; 
 256     if (!readBytes(c
, clen
)) { 
 262     if (lzf_decompress(c
,clen
,s
,slen
) == 0) { 
 271 /* returns NULL when not processable, char* when valid */ 
 272 char* loadStringObject() { 
 273     uint32_t offset 
= CURR_OFFSET
; 
 277     len 
= loadLength(&isencoded
); 
 280         case REDIS_RDB_ENC_INT8
: 
 281         case REDIS_RDB_ENC_INT16
: 
 282         case REDIS_RDB_ENC_INT32
: 
 283             return loadIntegerObject(len
); 
 284         case REDIS_RDB_ENC_LZF
: 
 285             return loadLzfStringObject(); 
 287             /* unknown encoding */ 
 288             SHIFT_ERROR(offset
, "Unknown string encoding (0x%02x)", len
); 
 293     if (len 
== REDIS_RDB_LENERR
) return NULL
; 
 295     char *buf 
= malloc(sizeof(char) * (len
+1)); 
 297     if (!readBytes(buf
, len
)) { 
 304 int processStringObject(char** store
) { 
 305     unsigned long offset 
= CURR_OFFSET
; 
 306     char *key 
= loadStringObject(); 
 308         SHIFT_ERROR(offset
, "Error reading string object"); 
 321 double* loadDoubleValue() { 
 326     if (!readBytes(&len
,1)) return NULL
; 
 328     val 
= malloc(sizeof(double)); 
 330     case 255: *val 
= R_NegInf
;  return val
; 
 331     case 254: *val 
= R_PosInf
;  return val
; 
 332     case 253: *val 
= R_Nan
;     return val
; 
 334         if (!readBytes(buf
, len
)) { 
 339         sscanf(buf
, "%lg", val
); 
 344 int processDoubleValue(double** store
) { 
 345     unsigned long offset 
= CURR_OFFSET
; 
 346     double *val 
= loadDoubleValue(); 
 348         SHIFT_ERROR(offset
, "Error reading double value"); 
 361 int loadPair(entry 
*e
) { 
 362     uint32_t offset 
= CURR_OFFSET
; 
 367     if (processStringObject(&key
)) { 
 370         SHIFT_ERROR(offset
, "Error reading entry key"); 
 375     if (e
->type 
== REDIS_LIST 
|| 
 376         e
->type 
== REDIS_SET  
|| 
 377         e
->type 
== REDIS_ZSET 
|| 
 378         e
->type 
== REDIS_HASH
) { 
 379         if ((length 
= loadLength(NULL
)) == REDIS_RDB_LENERR
) { 
 380             SHIFT_ERROR(offset
, "Error reading %s length", types
[e
->type
]); 
 387     case REDIS_HASH_ZIPMAP
: 
 388     case REDIS_LIST_ZIPLIST
: 
 389     case REDIS_SET_INTSET
: 
 390     case REDIS_ZSET_ZIPLIST
: 
 391     case REDIS_HASH_ZIPLIST
: 
 392         if (!processStringObject(NULL
)) { 
 393             SHIFT_ERROR(offset
, "Error reading entry value"); 
 399         for (i 
= 0; i 
< length
; i
++) { 
 400             offset 
= CURR_OFFSET
; 
 401             if (!processStringObject(NULL
)) { 
 402                 SHIFT_ERROR(offset
, "Error reading element at index %d (length: %d)", i
, length
); 
 408         for (i 
= 0; i 
< length
; i
++) { 
 409             offset 
= CURR_OFFSET
; 
 410             if (!processStringObject(NULL
)) { 
 411                 SHIFT_ERROR(offset
, "Error reading element key at index %d (length: %d)", i
, length
); 
 414             offset 
= CURR_OFFSET
; 
 415             if (!processDoubleValue(NULL
)) { 
 416                 SHIFT_ERROR(offset
, "Error reading element value at index %d (length: %d)", i
, length
); 
 422         for (i 
= 0; i 
< length
; i
++) { 
 423             offset 
= CURR_OFFSET
; 
 424             if (!processStringObject(NULL
)) { 
 425                 SHIFT_ERROR(offset
, "Error reading element key at index %d (length: %d)", i
, length
); 
 428             offset 
= CURR_OFFSET
; 
 429             if (!processStringObject(NULL
)) { 
 430                 SHIFT_ERROR(offset
, "Error reading element value at index %d (length: %d)", i
, length
); 
 436         SHIFT_ERROR(offset
, "Type not implemented"); 
 439     /* because we're done, we assume success */ 
 445     entry e 
= { NULL
, -1, 0 }; 
 446     uint32_t length
, offset
[4]; 
 448     /* reset error container */ 
 451     offset
[0] = CURR_OFFSET
; 
 456     offset
[1] = CURR_OFFSET
; 
 457     if (e
.type 
== REDIS_SELECTDB
) { 
 458         if ((length 
= loadLength(NULL
)) == REDIS_RDB_LENERR
) { 
 459             SHIFT_ERROR(offset
[1], "Error reading database number"); 
 463             SHIFT_ERROR(offset
[1], "Database number out of range (%d)", length
); 
 466     } else if (e
.type 
== REDIS_EOF
) { 
 467         if (positions
[level
].offset 
< positions
[level
].size
) { 
 468             SHIFT_ERROR(offset
[0], "Unexpected EOF"); 
 474         /* optionally consume expire */ 
 475         if (e
.type 
== REDIS_EXPIRETIME
) { 
 476             if (!processTime()) return e
; 
 477             if (!loadType(&e
)) return e
; 
 480         offset
[1] = CURR_OFFSET
; 
 482             SHIFT_ERROR(offset
[1], "Error for type %s", types
[e
.type
]); 
 487     /* all entries are followed by a valid type: 
 488      * e.g. a new entry, SELECTDB, EXPIRE, EOF */ 
 489     offset
[2] = CURR_OFFSET
; 
 490     if (peekType() == -1) { 
 491         SHIFT_ERROR(offset
[2], "Followed by invalid type"); 
 492         SHIFT_ERROR(offset
[0], "Error for type %s", types
[e
.type
]); 
 501 void printCentered(int indent
, int width
, char* body
) { 
 502     char head
[256], tail
[256]; 
 503     memset(head
, '\0', 256); 
 504     memset(tail
, '\0', 256); 
 506     memset(head
, '=', indent
); 
 507     memset(tail
, '=', width 
- 2 - indent 
- strlen(body
)); 
 508     printf("%s %s %s\n", head
, body
, tail
); 
 511 void printValid(uint64_t ops
, uint64_t bytes
) { 
 513     sprintf(body
, "Processed %llu valid opcodes (in %llu bytes)", 
 514         (unsigned long long) ops
, (unsigned long long) bytes
); 
 515     printCentered(4, 80, body
); 
 518 void printSkipped(uint64_t bytes
, uint64_t offset
) { 
 520     sprintf(body
, "Skipped %llu bytes (resuming at 0x%08llx)", 
 521         (unsigned long long) bytes
, (unsigned long long) offset
); 
 522     printCentered(4, 80, body
); 
 525 void printErrorStack(entry 
*e
) { 
 530         sprintf(body
, "Error trace"); 
 531     } else if (e
->type 
>= 253) { 
 532         sprintf(body
, "Error trace (%s)", types
[e
->type
]); 
 533     } else if (!e
->key
) { 
 534         sprintf(body
, "Error trace (%s: (unknown))", types
[e
->type
]); 
 537         strncpy(tmp
, e
->key
, 40); 
 539         /* display truncation at the last 3 chars */ 
 540         if (strlen(e
->key
) > 40) { 
 541             memset(&tmp
[37], '.', 3); 
 544         /* display unprintable characters as ? */ 
 545         for (i 
= 0; i 
< strlen(tmp
); i
++) { 
 546             if (tmp
[i
] <= 32) tmp
[i
] = '?'; 
 548         sprintf(body
, "Error trace (%s: %s)", types
[e
->type
], tmp
); 
 551     printCentered(4, 80, body
); 
 553     /* display error stack */ 
 554     for (i 
= 0; i 
< errors
.level
; i
++) { 
 555         printf("0x%08lx - %s\n", 
 556             (unsigned long) errors
.offset
[i
], errors
.error
[i
]); 
 561     uint64_t num_errors 
= 0, num_valid_ops 
= 0, num_valid_bytes 
= 0; 
 563     int dump_version 
= processHeader(); 
 565     /* Exclude the final checksum for RDB >= 5. Will be checked at the end. */ 
 566     if (dump_version 
>= 5) { 
 567         if (positions
[0].size 
< 8) { 
 568             printf("RDB version >= 5 but no room for checksum.\n"); 
 571         positions
[0].size 
-= 8;; 
 575     while(positions
[0].offset 
< positions
[0].size
) { 
 576         positions
[1] = positions
[0]; 
 579         if (!entry
.success
) { 
 580             printValid(num_valid_ops
, num_valid_bytes
); 
 581             printErrorStack(&entry
); 
 586             /* search for next valid entry */ 
 587             uint64_t offset 
= positions
[0].offset 
+ 1; 
 590             while (!entry
.success 
&& offset 
< positions
[0].size
) { 
 591                 positions
[1].offset 
= offset
; 
 593                 /* find 3 consecutive valid entries */ 
 594                 for (i 
= 0; i 
< 3; i
++) { 
 596                     if (!entry
.success
) break; 
 598                 /* check if we found 3 consecutive valid entries */ 
 604             /* print how many bytes we have skipped to find a new valid opcode */ 
 605             if (offset 
< positions
[0].size
) { 
 606                 printSkipped(offset 
- positions
[0].offset
, offset
); 
 609             positions
[0].offset 
= offset
; 
 612             num_valid_bytes 
+= positions
[1].offset 
- positions
[0].offset
; 
 614             /* advance position */ 
 615             positions
[0] = positions
[1]; 
 620     /* because there is another potential error, 
 621      * print how many valid ops we have processed */ 
 622     printValid(num_valid_ops
, num_valid_bytes
); 
 625     if (entry
.type 
!= REDIS_EOF
) { 
 626         /* last byte should be EOF, add error */ 
 628         SHIFT_ERROR(positions
[0].offset
, "Expected EOF, got %s", types
[entry
.type
]); 
 630         /* this is an EOF error so reset type */ 
 632         printErrorStack(&entry
); 
 637     /* Verify checksum */ 
 638     if (dump_version 
>= 5) { 
 639         uint64_t crc 
= crc64(0,positions
[0].data
,positions
[0].size
); 
 641         unsigned char *p 
= (unsigned char*)positions
[0].data
+positions
[0].size
; 
 642         crc2 
= ((uint64_t)p
[0] << 0) | 
 643                ((uint64_t)p
[1] << 8) | 
 644                ((uint64_t)p
[2] << 16) | 
 645                ((uint64_t)p
[3] << 24) | 
 646                ((uint64_t)p
[4] << 32) | 
 647                ((uint64_t)p
[5] << 40) | 
 648                ((uint64_t)p
[6] << 48) | 
 649                ((uint64_t)p
[7] << 56); 
 651             SHIFT_ERROR(positions
[0].offset
, "RDB CRC64 does not match."); 
 653             printf("CRC64 checksum is OK\n"); 
 657     /* print summary on errors */ 
 660         printf("Total unprocessable opcodes: %llu\n", 
 661             (unsigned long long) num_errors
); 
 665 int main(int argc
, char **argv
) { 
 666     /* expect the first argument to be the dump file */ 
 668         printf("Usage: %s <dump.rdb>\n", argv
[0]); 
 677     fd 
= open(argv
[1], O_RDONLY
); 
 679         ERROR("Cannot open file: %s\n", argv
[1]); 
 681     if (fstat(fd
, &stat
) == -1) { 
 682         ERROR("Cannot stat: %s\n", argv
[1]); 
 687     if (sizeof(size_t) == sizeof(int32_t) && size 
>= INT_MAX
) { 
 688         ERROR("Cannot check dump files >2GB on a 32-bit platform\n"); 
 691     data 
= mmap(NULL
, size
, PROT_READ
, MAP_SHARED
, fd
, 0); 
 692     if (data 
== MAP_FAILED
) { 
 693         ERROR("Cannot mmap: %s\n", argv
[1]); 
 696     /* Initialize static vars */ 
 697     positions
[0].data 
= data
; 
 698     positions
[0].size 
= size
; 
 699     positions
[0].offset 
= 0; 
 703     sprintf(types
[REDIS_STRING
], "STRING"); 
 704     sprintf(types
[REDIS_LIST
], "LIST"); 
 705     sprintf(types
[REDIS_SET
], "SET"); 
 706     sprintf(types
[REDIS_ZSET
], "ZSET"); 
 707     sprintf(types
[REDIS_HASH
], "HASH"); 
 709     /* Object types only used for dumping to disk */ 
 710     sprintf(types
[REDIS_EXPIRETIME
], "EXPIRETIME"); 
 711     sprintf(types
[REDIS_SELECTDB
], "SELECTDB"); 
 712     sprintf(types
[REDIS_EOF
], "EOF"); 
 714     /* Double constants initialization */ 
 716     R_PosInf 
= 1.0/R_Zero
; 
 717     R_NegInf 
= -1.0/R_Zero
; 
 718     R_Nan 
= R_Zero
/R_Zero
;