#define REDIS_SET 2
#define REDIS_ZSET 3
#define REDIS_HASH 4
+#define REDIS_HASH_ZIPMAP 9
+#define REDIS_LIST_ZIPLIST 10
+#define REDIS_SET_INTSET 11
+#define REDIS_ZSET_ZIPLIST 12
+#define REDIS_HASH_ZIPLIST 13
/* Objects encoding. Some kind of objects like Strings and Hashes can be
* internally represented in multiple ways. The 'encoding' field of the object
#define REDIS_ENCODING_HT 3 /* Encoded as an hash table */
/* Object types only used for dumping to disk */
+#define REDIS_EXPIRETIME_MS 252
#define REDIS_EXPIRETIME 253
#define REDIS_SELECTDB 254
#define REDIS_EOF 255
/* data type to hold offset in file and size */
typedef struct {
void *data;
- unsigned long size;
- unsigned long offset;
+ size_t size;
+ size_t offset;
} pos;
static unsigned char level = 0;
/* Hold a stack of errors */
typedef struct {
char error[16][1024];
- unsigned long offset[16];
- unsigned int level;
+ size_t offset[16];
+ size_t level;
} errors_t;
static errors_t errors;
/* store string types for output */
static char types[256][16];
+/* Prototypes */
+uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l);
+
+/* Return true if 't' is a valid object type. */
+int checkType(unsigned char t) {
+ /* In case a new object type is added, update the following
+ * condition as necessary. */
+ return
+ (t >= REDIS_HASH_ZIPMAP && t <= REDIS_HASH_ZIPLIST) ||
+ t <= REDIS_HASH ||
+ t >= REDIS_EXPIRETIME_MS;
+}
+
/* when number of bytes to read is negative, do a peek */
int readBytes(void *target, long num) {
char peek = (num < 0) ? 1 : 0;
if (p.offset + num > p.size) {
return 0;
} else {
- memcpy(target, (void*)((unsigned long)p.data + p.offset), num);
+ memcpy(target, (void*)((size_t)p.data + p.offset), num);
if (!peek) positions[level].offset += num;
}
return 1;
}
dump_version = (int)strtol(buf + 5, NULL, 10);
- if (dump_version != 1) {
+ if (dump_version < 1 || dump_version > 6) {
ERROR("Unknown RDB format version: %d\n", dump_version);
}
- return 1;
+ return dump_version;
}
int loadType(entry *e) {
/* this byte needs to qualify as type */
unsigned char t;
if (readBytes(&t, 1)) {
- if (t <= 4 || t >= 253) {
+ if (checkType(t)) {
e->type = t;
return 1;
} else {
int peekType() {
unsigned char t;
- if (readBytes(&t, -1) && (t <= 4 || t >= 253)) return t;
+ if (readBytes(&t, -1) && (checkType(t)))
+ return t;
return -1;
}
/* discard time, just consume the bytes */
-int processTime() {
+int processTime(int type) {
uint32_t offset = CURR_OFFSET;
- unsigned char t[4];
- if (readBytes(t, 4)) {
+ unsigned char t[8];
+ int timelen = (type == REDIS_EXPIRETIME_MS) ? 8 : 4;
+
+ if (readBytes(t,timelen)) {
return 1;
} else {
SHIFT_ERROR(offset, "Could not read time");
switch(e->type) {
case REDIS_STRING:
+ case REDIS_HASH_ZIPMAP:
+ case REDIS_LIST_ZIPLIST:
+ case REDIS_SET_INTSET:
+ case REDIS_ZSET_ZIPLIST:
+ case REDIS_HASH_ZIPLIST:
if (!processStringObject(NULL)) {
SHIFT_ERROR(offset, "Error reading entry value");
return 0;
return e;
} else {
/* optionally consume expire */
- if (e.type == REDIS_EXPIRETIME) {
- if (!processTime()) return e;
+ if (e.type == REDIS_EXPIRETIME ||
+ e.type == REDIS_EXPIRETIME_MS) {
+ if (!processTime(e.type)) return e;
if (!loadType(&e)) return e;
}
printf("%s %s %s\n", head, body, tail);
}
-void printValid(int ops, int bytes) {
+void printValid(uint64_t ops, uint64_t bytes) {
char body[80];
- sprintf(body, "Processed %d valid opcodes (in %d bytes)", ops, bytes);
+ sprintf(body, "Processed %llu valid opcodes (in %llu bytes)",
+ (unsigned long long) ops, (unsigned long long) bytes);
printCentered(4, 80, body);
}
-void printSkipped(int bytes, int offset) {
+void printSkipped(uint64_t bytes, uint64_t offset) {
char body[80];
- sprintf(body, "Skipped %d bytes (resuming at 0x%08x)", bytes, offset);
+ sprintf(body, "Skipped %llu bytes (resuming at 0x%08llx)",
+ (unsigned long long) bytes, (unsigned long long) offset);
printCentered(4, 80, body);
}
/* display error stack */
for (i = 0; i < errors.level; i++) {
- printf("0x%08lx - %s\n", errors.offset[i], errors.error[i]);
+ printf("0x%08lx - %s\n",
+ (unsigned long) errors.offset[i], errors.error[i]);
}
}
void process() {
- int i, num_errors = 0, num_valid_ops = 0, num_valid_bytes = 0;
+ uint64_t num_errors = 0, num_valid_ops = 0, num_valid_bytes = 0;
entry entry;
- processHeader();
+ int dump_version = processHeader();
+
+ /* Exclude the final checksum for RDB >= 5. Will be checked at the end. */
+ if (dump_version >= 5) {
+ if (positions[0].size < 8) {
+ printf("RDB version >= 5 but no room for checksum.\n");
+ exit(1);
+ }
+ positions[0].size -= 8;;
+ }
level = 1;
while(positions[0].offset < positions[0].size) {
num_valid_bytes = 0;
/* search for next valid entry */
- unsigned long offset = positions[0].offset + 1;
+ uint64_t offset = positions[0].offset + 1;
+ int i = 0;
+
while (!entry.success && offset < positions[0].size) {
positions[1].offset = offset;
/* advance position */
positions[0] = positions[1];
}
+ free(entry.key);
}
/* because there is another potential error,
num_errors++;
}
+ /* Verify checksum */
+ if (dump_version >= 5) {
+ uint64_t crc = crc64(0,positions[0].data,positions[0].size);
+ uint64_t crc2;
+ unsigned char *p = (unsigned char*)positions[0].data+positions[0].size;
+ crc2 = ((uint64_t)p[0] << 0) |
+ ((uint64_t)p[1] << 8) |
+ ((uint64_t)p[2] << 16) |
+ ((uint64_t)p[3] << 24) |
+ ((uint64_t)p[4] << 32) |
+ ((uint64_t)p[5] << 40) |
+ ((uint64_t)p[6] << 48) |
+ ((uint64_t)p[7] << 56);
+ if (crc != crc2) {
+ SHIFT_ERROR(positions[0].offset, "RDB CRC64 does not match.");
+ } else {
+ printf("CRC64 checksum is OK\n");
+ }
+ }
+
/* print summary on errors */
- if (num_errors > 0) {
+ if (num_errors) {
printf("\n");
- printf("Total unprocessable opcodes: %d\n", num_errors);
+ printf("Total unprocessable opcodes: %llu\n",
+ (unsigned long long) num_errors);
}
}
}
int fd;
- unsigned long size;
+ off_t size;
struct stat stat;
void *data;
size = stat.st_size;
}
+ if (sizeof(size_t) == sizeof(int32_t) && size >= INT_MAX) {
+ ERROR("Cannot check dump files >2GB on a 32-bit platform\n");
+ }
+
data = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0);
if (data == MAP_FAILED) {
ERROR("Cannot mmap: %s\n", argv[1]);
projects / redis.git / blobdiff