6372254c90d216269ff7a9c0421ee131e28ce70c
[redis.git] / src / rdb.c
1 #include "redis.h"
2 #include "lzf.h" /* LZF compression library */
3
4 #include <math.h>
5 #include <sys/types.h>
6 #include <sys/time.h>
7 #include <sys/resource.h>
8 #include <sys/wait.h>
9 #include <arpa/inet.h>
10 #include <sys/stat.h>
11
12 /* Convenience wrapper around fwrite, that returns the number of bytes written
13 * to the file instead of the number of objects (see fwrite(3)) and -1 in the
14 * case of an error. It also supports a NULL *fp to skip writing altogether
15 * instead of writing to /dev/null. */
16 static int rdbWriteRaw(FILE *fp, void *p, size_t len) {
17 if (fp != NULL && fwrite(p,len,1,fp) == 0) return -1;
18 return len;
19 }
20
21 int rdbSaveType(FILE *fp, unsigned char type) {
22 return rdbWriteRaw(fp,&type,1);
23 }
24
25 int rdbSaveTime(FILE *fp, time_t t) {
26 int32_t t32 = (int32_t) t;
27 return rdbWriteRaw(fp,&t32,4);
28 }
29
30 /* check rdbLoadLen() comments for more info */
31 int rdbSaveLen(FILE *fp, uint32_t len) {
32 unsigned char buf[2];
33 int nwritten;
34
35 if (len < (1<<6)) {
36 /* Save a 6 bit len */
37 buf[0] = (len&0xFF)|(REDIS_RDB_6BITLEN<<6);
38 if (rdbWriteRaw(fp,buf,1) == -1) return -1;
39 nwritten = 1;
40 } else if (len < (1<<14)) {
41 /* Save a 14 bit len */
42 buf[0] = ((len>>8)&0xFF)|(REDIS_RDB_14BITLEN<<6);
43 buf[1] = len&0xFF;
44 if (rdbWriteRaw(fp,buf,2) == -1) return -1;
45 nwritten = 2;
46 } else {
47 /* Save a 32 bit len */
48 buf[0] = (REDIS_RDB_32BITLEN<<6);
49 if (rdbWriteRaw(fp,buf,1) == -1) return -1;
50 len = htonl(len);
51 if (rdbWriteRaw(fp,&len,4) == -1) return -1;
52 nwritten = 1+4;
53 }
54 return nwritten;
55 }
56
57 /* Encode 'value' as an integer if possible (if integer will fit the
58 * supported range). If the function sucessful encoded the integer
59 * then the (up to 5 bytes) encoded representation is written in the
60 * string pointed by 'enc' and the length is returned. Otherwise
61 * 0 is returned. */
62 int rdbEncodeInteger(long long value, unsigned char *enc) {
63 /* Finally check if it fits in our ranges */
64 if (value >= -(1<<7) && value <= (1<<7)-1) {
65 enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT8;
66 enc[1] = value&0xFF;
67 return 2;
68 } else if (value >= -(1<<15) && value <= (1<<15)-1) {
69 enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT16;
70 enc[1] = value&0xFF;
71 enc[2] = (value>>8)&0xFF;
72 return 3;
73 } else if (value >= -((long long)1<<31) && value <= ((long long)1<<31)-1) {
74 enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT32;
75 enc[1] = value&0xFF;
76 enc[2] = (value>>8)&0xFF;
77 enc[3] = (value>>16)&0xFF;
78 enc[4] = (value>>24)&0xFF;
79 return 5;
80 } else {
81 return 0;
82 }
83 }
84
85 /* String objects in the form "2391" "-100" without any space and with a
86 * range of values that can fit in an 8, 16 or 32 bit signed value can be
87 * encoded as integers to save space */
88 int rdbTryIntegerEncoding(char *s, size_t len, unsigned char *enc) {
89 long long value;
90 char *endptr, buf[32];
91
92 /* Check if it's possible to encode this value as a number */
93 value = strtoll(s, &endptr, 10);
94 if (endptr[0] != '\0') return 0;
95 ll2string(buf,32,value);
96
97 /* If the number converted back into a string is not identical
98 * then it's not possible to encode the string as integer */
99 if (strlen(buf) != len || memcmp(buf,s,len)) return 0;
100
101 return rdbEncodeInteger(value,enc);
102 }
103
104 int rdbSaveLzfStringObject(FILE *fp, unsigned char *s, size_t len) {
105 size_t comprlen, outlen;
106 unsigned char byte;
107 int n, nwritten = 0;
108 void *out;
109
110 /* We require at least four bytes compression for this to be worth it */
111 if (len <= 4) return 0;
112 outlen = len-4;
113 if ((out = zmalloc(outlen+1)) == NULL) return 0;
114 comprlen = lzf_compress(s, len, out, outlen);
115 if (comprlen == 0) {
116 zfree(out);
117 return 0;
118 }
119 /* Data compressed! Let's save it on disk */
120 byte = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_LZF;
121 if ((n = rdbWriteRaw(fp,&byte,1)) == -1) goto writeerr;
122 nwritten += n;
123
124 if ((n = rdbSaveLen(fp,comprlen)) == -1) goto writeerr;
125 nwritten += n;
126
127 if ((n = rdbSaveLen(fp,len)) == -1) goto writeerr;
128 nwritten += n;
129
130 if ((n = rdbWriteRaw(fp,out,comprlen)) == -1) goto writeerr;
131 nwritten += n;
132
133 zfree(out);
134 return nwritten;
135
136 writeerr:
137 zfree(out);
138 return -1;
139 }
140
141 /* Save a string objet as [len][data] on disk. If the object is a string
142 * representation of an integer value we try to safe it in a special form */
143 int rdbSaveRawString(FILE *fp, unsigned char *s, size_t len) {
144 int enclen;
145 int n, nwritten = 0;
146
147 /* Try integer encoding */
148 if (len <= 11) {
149 unsigned char buf[5];
150 if ((enclen = rdbTryIntegerEncoding((char*)s,len,buf)) > 0) {
151 if (rdbWriteRaw(fp,buf,enclen) == -1) return -1;
152 return enclen;
153 }
154 }
155
156 /* Try LZF compression - under 20 bytes it's unable to compress even
157 * aaaaaaaaaaaaaaaaaa so skip it */
158 if (server.rdbcompression && len > 20) {
159 n = rdbSaveLzfStringObject(fp,s,len);
160 if (n == -1) return -1;
161 if (n > 0) return n;
162 /* Return value of 0 means data can't be compressed, save the old way */
163 }
164
165 /* Store verbatim */
166 if ((n = rdbSaveLen(fp,len)) == -1) return -1;
167 nwritten += n;
168 if (len > 0) {
169 if (rdbWriteRaw(fp,s,len) == -1) return -1;
170 nwritten += len;
171 }
172 return nwritten;
173 }
174
175 /* Save a long long value as either an encoded string or a string. */
176 int rdbSaveLongLongAsStringObject(FILE *fp, long long value) {
177 unsigned char buf[32];
178 int n, nwritten = 0;
179 int enclen = rdbEncodeInteger(value,buf);
180 if (enclen > 0) {
181 return rdbWriteRaw(fp,buf,enclen);
182 } else {
183 /* Encode as string */
184 enclen = ll2string((char*)buf,32,value);
185 redisAssert(enclen < 32);
186 if ((n = rdbSaveLen(fp,enclen)) == -1) return -1;
187 nwritten += n;
188 if ((n = rdbWriteRaw(fp,buf,enclen)) == -1) return -1;
189 nwritten += n;
190 }
191 return nwritten;
192 }
193
194 /* Like rdbSaveStringObjectRaw() but handle encoded objects */
195 int rdbSaveStringObject(FILE *fp, robj *obj) {
196 /* Avoid to decode the object, then encode it again, if the
197 * object is alrady integer encoded. */
198 if (obj->encoding == REDIS_ENCODING_INT) {
199 return rdbSaveLongLongAsStringObject(fp,(long)obj->ptr);
200 } else {
201 redisAssert(obj->encoding == REDIS_ENCODING_RAW);
202 return rdbSaveRawString(fp,obj->ptr,sdslen(obj->ptr));
203 }
204 }
205
206 /* Save a double value. Doubles are saved as strings prefixed by an unsigned
207 * 8 bit integer specifing the length of the representation.
208 * This 8 bit integer has special values in order to specify the following
209 * conditions:
210 * 253: not a number
211 * 254: + inf
212 * 255: - inf
213 */
214 int rdbSaveDoubleValue(FILE *fp, double val) {
215 unsigned char buf[128];
216 int len;
217
218 if (isnan(val)) {
219 buf[0] = 253;
220 len = 1;
221 } else if (!isfinite(val)) {
222 len = 1;
223 buf[0] = (val < 0) ? 255 : 254;
224 } else {
225 #if (DBL_MANT_DIG >= 52) && (LLONG_MAX == 0x7fffffffffffffffLL)
226 /* Check if the float is in a safe range to be casted into a
227 * long long. We are assuming that long long is 64 bit here.
228 * Also we are assuming that there are no implementations around where
229 * double has precision < 52 bit.
230 *
231 * Under this assumptions we test if a double is inside an interval
232 * where casting to long long is safe. Then using two castings we
233 * make sure the decimal part is zero. If all this is true we use
234 * integer printing function that is much faster. */
235 double min = -4503599627370495; /* (2^52)-1 */
236 double max = 4503599627370496; /* -(2^52) */
237 if (val > min && val < max && val == ((double)((long long)val)))
238 ll2string((char*)buf+1,sizeof(buf),(long long)val);
239 else
240 #endif
241 snprintf((char*)buf+1,sizeof(buf)-1,"%.17g",val);
242 buf[0] = strlen((char*)buf+1);
243 len = buf[0]+1;
244 }
245 return rdbWriteRaw(fp,buf,len);
246 }
247
248 /* Save a Redis object. */
249 int rdbSaveObject(FILE *fp, robj *o) {
250 int n, nwritten = 0;
251
252 if (o->type == REDIS_STRING) {
253 /* Save a string value */
254 if ((n = rdbSaveStringObject(fp,o)) == -1) return -1;
255 nwritten += n;
256 } else if (o->type == REDIS_LIST) {
257 /* Save a list value */
258 if (o->encoding == REDIS_ENCODING_ZIPLIST) {
259 unsigned char *p;
260 unsigned char *vstr;
261 unsigned int vlen;
262 long long vlong;
263
264 if ((n = rdbSaveLen(fp,ziplistLen(o->ptr))) == -1) return -1;
265 nwritten += n;
266
267 p = ziplistIndex(o->ptr,0);
268 while(ziplistGet(p,&vstr,&vlen,&vlong)) {
269 if (vstr) {
270 if ((n = rdbSaveRawString(fp,vstr,vlen)) == -1)
271 return -1;
272 nwritten += n;
273 } else {
274 if ((n = rdbSaveLongLongAsStringObject(fp,vlong)) == -1)
275 return -1;
276 nwritten += n;
277 }
278 p = ziplistNext(o->ptr,p);
279 }
280 } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
281 list *list = o->ptr;
282 listIter li;
283 listNode *ln;
284
285 if ((n = rdbSaveLen(fp,listLength(list))) == -1) return -1;
286 nwritten += n;
287
288 listRewind(list,&li);
289 while((ln = listNext(&li))) {
290 robj *eleobj = listNodeValue(ln);
291 if ((n = rdbSaveStringObject(fp,eleobj)) == -1) return -1;
292 nwritten += n;
293 }
294 } else {
295 redisPanic("Unknown list encoding");
296 }
297 } else if (o->type == REDIS_SET) {
298 /* Save a set value */
299 if (o->encoding == REDIS_ENCODING_HT) {
300 dict *set = o->ptr;
301 dictIterator *di = dictGetIterator(set);
302 dictEntry *de;
303
304 if ((n = rdbSaveLen(fp,dictSize(set))) == -1) return -1;
305 nwritten += n;
306
307 while((de = dictNext(di)) != NULL) {
308 robj *eleobj = dictGetEntryKey(de);
309 if ((n = rdbSaveStringObject(fp,eleobj)) == -1) return -1;
310 nwritten += n;
311 }
312 dictReleaseIterator(di);
313 } else if (o->encoding == REDIS_ENCODING_INTSET) {
314 intset *is = o->ptr;
315 int64_t llval;
316 int i = 0;
317
318 if ((n = rdbSaveLen(fp,intsetLen(is))) == -1) return -1;
319 nwritten += n;
320
321 while(intsetGet(is,i++,&llval)) {
322 if ((n = rdbSaveLongLongAsStringObject(fp,llval)) == -1) return -1;
323 nwritten += n;
324 }
325 } else {
326 redisPanic("Unknown set encoding");
327 }
328 } else if (o->type == REDIS_ZSET) {
329 /* Save a set value */
330 zset *zs = o->ptr;
331 dictIterator *di = dictGetIterator(zs->dict);
332 dictEntry *de;
333
334 if ((n = rdbSaveLen(fp,dictSize(zs->dict))) == -1) return -1;
335 nwritten += n;
336
337 while((de = dictNext(di)) != NULL) {
338 robj *eleobj = dictGetEntryKey(de);
339 double *score = dictGetEntryVal(de);
340
341 if ((n = rdbSaveStringObject(fp,eleobj)) == -1) return -1;
342 nwritten += n;
343 if ((n = rdbSaveDoubleValue(fp,*score)) == -1) return -1;
344 nwritten += n;
345 }
346 dictReleaseIterator(di);
347 } else if (o->type == REDIS_HASH) {
348 /* Save a hash value */
349 if (o->encoding == REDIS_ENCODING_ZIPMAP) {
350 unsigned char *p = zipmapRewind(o->ptr);
351 unsigned int count = zipmapLen(o->ptr);
352 unsigned char *key, *val;
353 unsigned int klen, vlen;
354
355 if ((n = rdbSaveLen(fp,count)) == -1) return -1;
356 nwritten += n;
357
358 while((p = zipmapNext(p,&key,&klen,&val,&vlen)) != NULL) {
359 if ((n = rdbSaveRawString(fp,key,klen)) == -1) return -1;
360 nwritten += n;
361 if ((n = rdbSaveRawString(fp,val,vlen)) == -1) return -1;
362 nwritten += n;
363 }
364 } else {
365 dictIterator *di = dictGetIterator(o->ptr);
366 dictEntry *de;
367
368 if ((n = rdbSaveLen(fp,dictSize((dict*)o->ptr))) == -1) return -1;
369 nwritten += n;
370
371 while((de = dictNext(di)) != NULL) {
372 robj *key = dictGetEntryKey(de);
373 robj *val = dictGetEntryVal(de);
374
375 if ((n = rdbSaveStringObject(fp,key)) == -1) return -1;
376 nwritten += n;
377 if ((n = rdbSaveStringObject(fp,val)) == -1) return -1;
378 nwritten += n;
379 }
380 dictReleaseIterator(di);
381 }
382 } else {
383 redisPanic("Unknown object type");
384 }
385 return nwritten;
386 }
387
388 /* Return the length the object will have on disk if saved with
389 * the rdbSaveObject() function. Currently we use a trick to get
390 * this length with very little changes to the code. In the future
391 * we could switch to a faster solution. */
392 off_t rdbSavedObjectLen(robj *o) {
393 int len = rdbSaveObject(NULL,o);
394 redisAssert(len != -1);
395 return len;
396 }
397
398 /* Save a key-value pair, with expire time, type, key, value.
399 * On error -1 is returned.
400 * On success if the key was actaully saved 1 is returned, otherwise 0
401 * is returned (the key was already expired). */
402 int rdbSaveKeyValuePair(FILE *fp, redisDb *db, robj *key, robj *val,
403 time_t now)
404 {
405 time_t expiretime;
406
407 expiretime = getExpire(db,key);
408
409 /* Save the expire time */
410 if (expiretime != -1) {
411 /* If this key is already expired skip it */
412 if (expiretime < now) return 0;
413 if (rdbSaveType(fp,REDIS_EXPIRETIME) == -1) return -1;
414 if (rdbSaveTime(fp,expiretime) == -1) return -1;
415 }
416 /* Save type, key, value */
417 if (rdbSaveType(fp,val->type) == -1) return -1;
418 if (rdbSaveStringObject(fp,key) == -1) return -1;
419 if (rdbSaveObject(fp,val) == -1) return -1;
420 return 1;
421 }
422
423 /* Save the DB on disk. Return REDIS_ERR on error, REDIS_OK on success */
424 int rdbSave(char *filename) {
425 dictIterator *di = NULL;
426 dictEntry *de;
427 FILE *fp;
428 char tmpfile[256];
429 int j;
430 time_t now = time(NULL);
431
432 if (server.ds_enabled) {
433 cacheForcePointInTime();
434 return dsRdbSave(filename);
435 }
436
437 snprintf(tmpfile,256,"temp-%d.rdb", (int) getpid());
438 fp = fopen(tmpfile,"w");
439 if (!fp) {
440 redisLog(REDIS_WARNING, "Failed opening .rdb for saving: %s",
441 strerror(errno));
442 return REDIS_ERR;
443 }
444 if (fwrite("REDIS0001",9,1,fp) == 0) goto werr;
445 for (j = 0; j < server.dbnum; j++) {
446 redisDb *db = server.db+j;
447 dict *d = db->dict;
448 if (dictSize(d) == 0) continue;
449 di = dictGetIterator(d);
450 if (!di) {
451 fclose(fp);
452 return REDIS_ERR;
453 }
454
455 /* Write the SELECT DB opcode */
456 if (rdbSaveType(fp,REDIS_SELECTDB) == -1) goto werr;
457 if (rdbSaveLen(fp,j) == -1) goto werr;
458
459 /* Iterate this DB writing every entry */
460 while((de = dictNext(di)) != NULL) {
461 sds keystr = dictGetEntryKey(de);
462 robj key, *o = dictGetEntryVal(de);
463
464 initStaticStringObject(key,keystr);
465 if (rdbSaveKeyValuePair(fp,db,&key,o,now) == -1) goto werr;
466 }
467 dictReleaseIterator(di);
468 }
469 /* EOF opcode */
470 if (rdbSaveType(fp,REDIS_EOF) == -1) goto werr;
471
472 /* Make sure data will not remain on the OS's output buffers */
473 fflush(fp);
474 fsync(fileno(fp));
475 fclose(fp);
476
477 /* Use RENAME to make sure the DB file is changed atomically only
478 * if the generate DB file is ok. */
479 if (rename(tmpfile,filename) == -1) {
480 redisLog(REDIS_WARNING,"Error moving temp DB file on the final destination: %s", strerror(errno));
481 unlink(tmpfile);
482 return REDIS_ERR;
483 }
484 redisLog(REDIS_NOTICE,"DB saved on disk");
485 server.dirty = 0;
486 server.lastsave = time(NULL);
487 return REDIS_OK;
488
489 werr:
490 fclose(fp);
491 unlink(tmpfile);
492 redisLog(REDIS_WARNING,"Write error saving DB on disk: %s", strerror(errno));
493 if (di) dictReleaseIterator(di);
494 return REDIS_ERR;
495 }
496
497 int rdbSaveBackground(char *filename) {
498 pid_t childpid;
499
500 if (server.bgsavechildpid != -1 ||
501 server.bgsavethread != (pthread_t) -1) return REDIS_ERR;
502
503 server.dirty_before_bgsave = server.dirty;
504
505 if (server.ds_enabled) {
506 cacheForcePointInTime();
507 return dsRdbSave(filename);
508 }
509
510 if ((childpid = fork()) == 0) {
511 int retval;
512
513 /* Child */
514 if (server.ipfd > 0) close(server.ipfd);
515 if (server.sofd > 0) close(server.sofd);
516 retval = rdbSave(filename);
517 _exit((retval == REDIS_OK) ? 0 : 1);
518 } else {
519 /* Parent */
520 if (childpid == -1) {
521 redisLog(REDIS_WARNING,"Can't save in background: fork: %s",
522 strerror(errno));
523 return REDIS_ERR;
524 }
525 redisLog(REDIS_NOTICE,"Background saving started by pid %d",childpid);
526 server.bgsavechildpid = childpid;
527 updateDictResizePolicy();
528 return REDIS_OK;
529 }
530 return REDIS_OK; /* unreached */
531 }
532
533 void rdbRemoveTempFile(pid_t childpid) {
534 char tmpfile[256];
535
536 snprintf(tmpfile,256,"temp-%d.rdb", (int) childpid);
537 unlink(tmpfile);
538 }
539
540 int rdbLoadType(FILE *fp) {
541 unsigned char type;
542 if (fread(&type,1,1,fp) == 0) return -1;
543 return type;
544 }
545
546 time_t rdbLoadTime(FILE *fp) {
547 int32_t t32;
548 if (fread(&t32,4,1,fp) == 0) return -1;
549 return (time_t) t32;
550 }
551
552 /* Load an encoded length from the DB, see the REDIS_RDB_* defines on the top
553 * of this file for a description of how this are stored on disk.
554 *
555 * isencoded is set to 1 if the readed length is not actually a length but
556 * an "encoding type", check the above comments for more info */
557 uint32_t rdbLoadLen(FILE *fp, int *isencoded) {
558 unsigned char buf[2];
559 uint32_t len;
560 int type;
561
562 if (isencoded) *isencoded = 0;
563 if (fread(buf,1,1,fp) == 0) return REDIS_RDB_LENERR;
564 type = (buf[0]&0xC0)>>6;
565 if (type == REDIS_RDB_6BITLEN) {
566 /* Read a 6 bit len */
567 return buf[0]&0x3F;
568 } else if (type == REDIS_RDB_ENCVAL) {
569 /* Read a 6 bit len encoding type */
570 if (isencoded) *isencoded = 1;
571 return buf[0]&0x3F;
572 } else if (type == REDIS_RDB_14BITLEN) {
573 /* Read a 14 bit len */
574 if (fread(buf+1,1,1,fp) == 0) return REDIS_RDB_LENERR;
575 return ((buf[0]&0x3F)<<8)|buf[1];
576 } else {
577 /* Read a 32 bit len */
578 if (fread(&len,4,1,fp) == 0) return REDIS_RDB_LENERR;
579 return ntohl(len);
580 }
581 }
582
583 /* Load an integer-encoded object from file 'fp', with the specified
584 * encoding type 'enctype'. If encode is true the function may return
585 * an integer-encoded object as reply, otherwise the returned object
586 * will always be encoded as a raw string. */
587 robj *rdbLoadIntegerObject(FILE *fp, int enctype, int encode) {
588 unsigned char enc[4];
589 long long val;
590
591 if (enctype == REDIS_RDB_ENC_INT8) {
592 if (fread(enc,1,1,fp) == 0) return NULL;
593 val = (signed char)enc[0];
594 } else if (enctype == REDIS_RDB_ENC_INT16) {
595 uint16_t v;
596 if (fread(enc,2,1,fp) == 0) return NULL;
597 v = enc[0]|(enc[1]<<8);
598 val = (int16_t)v;
599 } else if (enctype == REDIS_RDB_ENC_INT32) {
600 uint32_t v;
601 if (fread(enc,4,1,fp) == 0) return NULL;
602 v = enc[0]|(enc[1]<<8)|(enc[2]<<16)|(enc[3]<<24);
603 val = (int32_t)v;
604 } else {
605 val = 0; /* anti-warning */
606 redisPanic("Unknown RDB integer encoding type");
607 }
608 if (encode)
609 return createStringObjectFromLongLong(val);
610 else
611 return createObject(REDIS_STRING,sdsfromlonglong(val));
612 }
613
614 robj *rdbLoadLzfStringObject(FILE*fp) {
615 unsigned int len, clen;
616 unsigned char *c = NULL;
617 sds val = NULL;
618
619 if ((clen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
620 if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
621 if ((c = zmalloc(clen)) == NULL) goto err;
622 if ((val = sdsnewlen(NULL,len)) == NULL) goto err;
623 if (fread(c,clen,1,fp) == 0) goto err;
624 if (lzf_decompress(c,clen,val,len) == 0) goto err;
625 zfree(c);
626 return createObject(REDIS_STRING,val);
627 err:
628 zfree(c);
629 sdsfree(val);
630 return NULL;
631 }
632
633 robj *rdbGenericLoadStringObject(FILE*fp, int encode) {
634 int isencoded;
635 uint32_t len;
636 sds val;
637
638 len = rdbLoadLen(fp,&isencoded);
639 if (isencoded) {
640 switch(len) {
641 case REDIS_RDB_ENC_INT8:
642 case REDIS_RDB_ENC_INT16:
643 case REDIS_RDB_ENC_INT32:
644 return rdbLoadIntegerObject(fp,len,encode);
645 case REDIS_RDB_ENC_LZF:
646 return rdbLoadLzfStringObject(fp);
647 default:
648 redisPanic("Unknown RDB encoding type");
649 }
650 }
651
652 if (len == REDIS_RDB_LENERR) return NULL;
653 val = sdsnewlen(NULL,len);
654 if (len && fread(val,len,1,fp) == 0) {
655 sdsfree(val);
656 return NULL;
657 }
658 return createObject(REDIS_STRING,val);
659 }
660
661 robj *rdbLoadStringObject(FILE *fp) {
662 return rdbGenericLoadStringObject(fp,0);
663 }
664
665 robj *rdbLoadEncodedStringObject(FILE *fp) {
666 return rdbGenericLoadStringObject(fp,1);
667 }
668
669 /* For information about double serialization check rdbSaveDoubleValue() */
670 int rdbLoadDoubleValue(FILE *fp, double *val) {
671 char buf[128];
672 unsigned char len;
673
674 if (fread(&len,1,1,fp) == 0) return -1;
675 switch(len) {
676 case 255: *val = R_NegInf; return 0;
677 case 254: *val = R_PosInf; return 0;
678 case 253: *val = R_Nan; return 0;
679 default:
680 if (fread(buf,len,1,fp) == 0) return -1;
681 buf[len] = '\0';
682 sscanf(buf, "%lg", val);
683 return 0;
684 }
685 }
686
687 /* Load a Redis object of the specified type from the specified file.
688 * On success a newly allocated object is returned, otherwise NULL. */
689 robj *rdbLoadObject(int type, FILE *fp) {
690 robj *o, *ele, *dec;
691 size_t len;
692 unsigned int i;
693
694 redisLog(REDIS_DEBUG,"LOADING OBJECT %d (at %d)\n",type,ftell(fp));
695 if (type == REDIS_STRING) {
696 /* Read string value */
697 if ((o = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
698 o = tryObjectEncoding(o);
699 } else if (type == REDIS_LIST) {
700 /* Read list value */
701 if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
702
703 /* Use a real list when there are too many entries */
704 if (len > server.list_max_ziplist_entries) {
705 o = createListObject();
706 } else {
707 o = createZiplistObject();
708 }
709
710 /* Load every single element of the list */
711 while(len--) {
712 if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
713
714 /* If we are using a ziplist and the value is too big, convert
715 * the object to a real list. */
716 if (o->encoding == REDIS_ENCODING_ZIPLIST &&
717 ele->encoding == REDIS_ENCODING_RAW &&
718 sdslen(ele->ptr) > server.list_max_ziplist_value)
719 listTypeConvert(o,REDIS_ENCODING_LINKEDLIST);
720
721 if (o->encoding == REDIS_ENCODING_ZIPLIST) {
722 dec = getDecodedObject(ele);
723 o->ptr = ziplistPush(o->ptr,dec->ptr,sdslen(dec->ptr),REDIS_TAIL);
724 decrRefCount(dec);
725 decrRefCount(ele);
726 } else {
727 ele = tryObjectEncoding(ele);
728 listAddNodeTail(o->ptr,ele);
729 }
730 }
731 } else if (type == REDIS_SET) {
732 /* Read list/set value */
733 if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
734
735 /* Use a regular set when there are too many entries. */
736 if (len > server.set_max_intset_entries) {
737 o = createSetObject();
738 /* It's faster to expand the dict to the right size asap in order
739 * to avoid rehashing */
740 if (len > DICT_HT_INITIAL_SIZE)
741 dictExpand(o->ptr,len);
742 } else {
743 o = createIntsetObject();
744 }
745
746 /* Load every single element of the list/set */
747 for (i = 0; i < len; i++) {
748 long long llval;
749 if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
750 ele = tryObjectEncoding(ele);
751
752 if (o->encoding == REDIS_ENCODING_INTSET) {
753 /* Fetch integer value from element */
754 if (isObjectRepresentableAsLongLong(ele,&llval) == REDIS_OK) {
755 o->ptr = intsetAdd(o->ptr,llval,NULL);
756 } else {
757 setTypeConvert(o,REDIS_ENCODING_HT);
758 dictExpand(o->ptr,len);
759 }
760 }
761
762 /* This will also be called when the set was just converted
763 * to regular hashtable encoded set */
764 if (o->encoding == REDIS_ENCODING_HT) {
765 dictAdd((dict*)o->ptr,ele,NULL);
766 } else {
767 decrRefCount(ele);
768 }
769 }
770 } else if (type == REDIS_ZSET) {
771 /* Read list/set value */
772 size_t zsetlen;
773 zset *zs;
774
775 if ((zsetlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
776 o = createZsetObject();
777 zs = o->ptr;
778 /* Load every single element of the list/set */
779 while(zsetlen--) {
780 robj *ele;
781 double score;
782 zskiplistNode *znode;
783
784 if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
785 ele = tryObjectEncoding(ele);
786 if (rdbLoadDoubleValue(fp,&score) == -1) return NULL;
787 znode = zslInsert(zs->zsl,score,ele);
788 dictAdd(zs->dict,ele,&znode->score);
789 incrRefCount(ele); /* added to skiplist */
790 }
791 } else if (type == REDIS_HASH) {
792 size_t hashlen;
793
794 if ((hashlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL;
795 o = createHashObject();
796 /* Too many entries? Use an hash table. */
797 if (hashlen > server.hash_max_zipmap_entries)
798 convertToRealHash(o);
799 /* Load every key/value, then set it into the zipmap or hash
800 * table, as needed. */
801 while(hashlen--) {
802 robj *key, *val;
803
804 if ((key = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
805 if ((val = rdbLoadEncodedStringObject(fp)) == NULL) return NULL;
806 /* If we are using a zipmap and there are too big values
807 * the object is converted to real hash table encoding. */
808 if (o->encoding != REDIS_ENCODING_HT &&
809 ((key->encoding == REDIS_ENCODING_RAW &&
810 sdslen(key->ptr) > server.hash_max_zipmap_value) ||
811 (val->encoding == REDIS_ENCODING_RAW &&
812 sdslen(val->ptr) > server.hash_max_zipmap_value)))
813 {
814 convertToRealHash(o);
815 }
816
817 if (o->encoding == REDIS_ENCODING_ZIPMAP) {
818 unsigned char *zm = o->ptr;
819 robj *deckey, *decval;
820
821 /* We need raw string objects to add them to the zipmap */
822 deckey = getDecodedObject(key);
823 decval = getDecodedObject(val);
824 zm = zipmapSet(zm,deckey->ptr,sdslen(deckey->ptr),
825 decval->ptr,sdslen(decval->ptr),NULL);
826 o->ptr = zm;
827 decrRefCount(deckey);
828 decrRefCount(decval);
829 decrRefCount(key);
830 decrRefCount(val);
831 } else {
832 key = tryObjectEncoding(key);
833 val = tryObjectEncoding(val);
834 dictAdd((dict*)o->ptr,key,val);
835 }
836 }
837 } else {
838 redisPanic("Unknown object type");
839 }
840 return o;
841 }
842
843 /* Mark that we are loading in the global state and setup the fields
844 * needed to provide loading stats. */
845 void startLoading(FILE *fp) {
846 struct stat sb;
847
848 /* Load the DB */
849 server.loading = 1;
850 server.loading_start_time = time(NULL);
851 if (fstat(fileno(fp), &sb) == -1) {
852 server.loading_total_bytes = 1; /* just to avoid division by zero */
853 } else {
854 server.loading_total_bytes = sb.st_size;
855 }
856 }
857
858 /* Refresh the loading progress info */
859 void loadingProgress(off_t pos) {
860 server.loading_loaded_bytes = pos;
861 }
862
863 /* Loading finished */
864 void stopLoading(void) {
865 server.loading = 0;
866 }
867
868 int rdbLoad(char *filename) {
869 FILE *fp;
870 uint32_t dbid;
871 int type, retval, rdbver;
872 redisDb *db = server.db+0;
873 char buf[1024];
874 time_t expiretime, now = time(NULL);
875 long loops = 0;
876
877 fp = fopen(filename,"r");
878 if (!fp) return REDIS_ERR;
879 if (fread(buf,9,1,fp) == 0) goto eoferr;
880 buf[9] = '\0';
881 if (memcmp(buf,"REDIS",5) != 0) {
882 fclose(fp);
883 redisLog(REDIS_WARNING,"Wrong signature trying to load DB from file");
884 return REDIS_ERR;
885 }
886 rdbver = atoi(buf+5);
887 if (rdbver != 1) {
888 fclose(fp);
889 redisLog(REDIS_WARNING,"Can't handle RDB format version %d",rdbver);
890 return REDIS_ERR;
891 }
892
893 startLoading(fp);
894 while(1) {
895 robj *key, *val;
896 expiretime = -1;
897
898 /* Serve the clients from time to time */
899 if (!(loops++ % 1000)) {
900 loadingProgress(ftello(fp));
901 aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT);
902 }
903
904 /* Read type. */
905 if ((type = rdbLoadType(fp)) == -1) goto eoferr;
906 if (type == REDIS_EXPIRETIME) {
907 if ((expiretime = rdbLoadTime(fp)) == -1) goto eoferr;
908 /* We read the time so we need to read the object type again */
909 if ((type = rdbLoadType(fp)) == -1) goto eoferr;
910 }
911 if (type == REDIS_EOF) break;
912 /* Handle SELECT DB opcode as a special case */
913 if (type == REDIS_SELECTDB) {
914 if ((dbid = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR)
915 goto eoferr;
916 if (dbid >= (unsigned)server.dbnum) {
917 redisLog(REDIS_WARNING,"FATAL: Data file was created with a Redis server configured to handle more than %d databases. Exiting\n", server.dbnum);
918 exit(1);
919 }
920 db = server.db+dbid;
921 continue;
922 }
923 /* Read key */
924 if ((key = rdbLoadStringObject(fp)) == NULL) goto eoferr;
925 /* Read value */
926 if ((val = rdbLoadObject(type,fp)) == NULL) goto eoferr;
927 /* Check if the key already expired */
928 if (expiretime != -1 && expiretime < now) {
929 decrRefCount(key);
930 decrRefCount(val);
931 continue;
932 }
933 /* Add the new object in the hash table */
934 retval = dbAdd(db,key,val);
935 if (retval == REDIS_ERR) {
936 redisLog(REDIS_WARNING,"Loading DB, duplicated key (%s) found! Unrecoverable error, exiting now.", key->ptr);
937 exit(1);
938 }
939 /* Set the expire time if needed */
940 if (expiretime != -1) setExpire(db,key,expiretime);
941
942 decrRefCount(key);
943 }
944 fclose(fp);
945 stopLoading();
946 return REDIS_OK;
947
948 eoferr: /* unexpected end of file is handled here with a fatal exit */
949 redisLog(REDIS_WARNING,"Short read or OOM loading DB. Unrecoverable error, aborting now.");
950 exit(1);
951 return REDIS_ERR; /* Just to avoid warning */
952 }
953
954 /* A background saving child (BGSAVE) terminated its work. Handle this. */
955 void backgroundSaveDoneHandler(int exitcode, int bysignal) {
956 if (!bysignal && exitcode == 0) {
957 redisLog(REDIS_NOTICE,
958 "Background saving terminated with success");
959 server.dirty = server.dirty - server.dirty_before_bgsave;
960 server.lastsave = time(NULL);
961 } else if (!bysignal && exitcode != 0) {
962 redisLog(REDIS_WARNING, "Background saving error");
963 } else {
964 redisLog(REDIS_WARNING,
965 "Background saving terminated by signal %d", bysignal);
966 rdbRemoveTempFile(server.bgsavechildpid);
967 }
968 server.bgsavechildpid = -1;
969 server.bgsavethread = (pthread_t) -1;
970 server.bgsavethread_state = REDIS_BGSAVE_THREAD_UNACTIVE;
971 /* Possibly there are slaves waiting for a BGSAVE in order to be served
972 * (the first stage of SYNC is a bulk transfer of dump.rdb) */
973 updateSlavesWaitingBgsave(exitcode == 0 ? REDIS_OK : REDIS_ERR);
974 }
975
976 void saveCommand(redisClient *c) {
977 if (server.bgsavechildpid != -1 || server.bgsavethread != (pthread_t)-1) {
978 addReplyError(c,"Background save already in progress");
979 return;
980 }
981 if (rdbSave(server.dbfilename) == REDIS_OK) {
982 addReply(c,shared.ok);
983 } else {
984 addReply(c,shared.err);
985 }
986 }
987
988 void bgsaveCommand(redisClient *c) {
989 if (server.bgsavechildpid != -1 || server.bgsavethread != (pthread_t)-1) {
990 addReplyError(c,"Background save already in progress");
991 return;
992 }
993 if (rdbSaveBackground(server.dbfilename) == REDIS_OK) {
994 addReplyStatus(c,"Background saving started");
995 } else {
996 addReply(c,shared.err);
997 }
998 }