]>
Commit | Line | Data |
---|---|---|
e2641e09 | 1 | #include "redis.h" |
2 | #include "lzf.h" /* LZF compression library */ | |
3 | ||
4 | #include <math.h> | |
3688d7f3 | 5 | #include <sys/types.h> |
6 | #include <sys/time.h> | |
7 | #include <sys/resource.h> | |
8 | #include <sys/wait.h> | |
9 | #include <arpa/inet.h> | |
97e7f8ae | 10 | #include <sys/stat.h> |
e2641e09 | 11 | |
9a68cf91 PN |
12 | /* Convenience wrapper around fwrite, that returns the number of bytes written |
13 | * to the file instead of the number of objects (see fwrite(3)) and -1 in the | |
14 | * case of an error. It also supports a NULL *fp to skip writing altogether | |
15 | * instead of writing to /dev/null. */ | |
16 | static int rdbWriteRaw(FILE *fp, void *p, size_t len) { | |
17 | if (fp != NULL && fwrite(p,len,1,fp) == 0) return -1; | |
18 | return len; | |
19 | } | |
20 | ||
e2641e09 | 21 | int rdbSaveType(FILE *fp, unsigned char type) { |
9a68cf91 | 22 | return rdbWriteRaw(fp,&type,1); |
e2641e09 | 23 | } |
24 | ||
25 | int rdbSaveTime(FILE *fp, time_t t) { | |
26 | int32_t t32 = (int32_t) t; | |
9a68cf91 | 27 | return rdbWriteRaw(fp,&t32,4); |
e2641e09 | 28 | } |
29 | ||
30 | /* check rdbLoadLen() comments for more info */ | |
31 | int rdbSaveLen(FILE *fp, uint32_t len) { | |
32 | unsigned char buf[2]; | |
8a623a98 | 33 | int nwritten; |
e2641e09 | 34 | |
35 | if (len < (1<<6)) { | |
36 | /* Save a 6 bit len */ | |
37 | buf[0] = (len&0xFF)|(REDIS_RDB_6BITLEN<<6); | |
9a68cf91 | 38 | if (rdbWriteRaw(fp,buf,1) == -1) return -1; |
8a623a98 | 39 | nwritten = 1; |
e2641e09 | 40 | } else if (len < (1<<14)) { |
41 | /* Save a 14 bit len */ | |
42 | buf[0] = ((len>>8)&0xFF)|(REDIS_RDB_14BITLEN<<6); | |
43 | buf[1] = len&0xFF; | |
9a68cf91 | 44 | if (rdbWriteRaw(fp,buf,2) == -1) return -1; |
8a623a98 | 45 | nwritten = 2; |
e2641e09 | 46 | } else { |
47 | /* Save a 32 bit len */ | |
48 | buf[0] = (REDIS_RDB_32BITLEN<<6); | |
9a68cf91 | 49 | if (rdbWriteRaw(fp,buf,1) == -1) return -1; |
e2641e09 | 50 | len = htonl(len); |
9a68cf91 | 51 | if (rdbWriteRaw(fp,&len,4) == -1) return -1; |
8a623a98 | 52 | nwritten = 1+4; |
e2641e09 | 53 | } |
8a623a98 | 54 | return nwritten; |
e2641e09 | 55 | } |
56 | ||
57 | /* Encode 'value' as an integer if possible (if integer will fit the | |
58 | * supported range). If the function sucessful encoded the integer | |
59 | * then the (up to 5 bytes) encoded representation is written in the | |
60 | * string pointed by 'enc' and the length is returned. Otherwise | |
61 | * 0 is returned. */ | |
62 | int rdbEncodeInteger(long long value, unsigned char *enc) { | |
63 | /* Finally check if it fits in our ranges */ | |
64 | if (value >= -(1<<7) && value <= (1<<7)-1) { | |
65 | enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT8; | |
66 | enc[1] = value&0xFF; | |
67 | return 2; | |
68 | } else if (value >= -(1<<15) && value <= (1<<15)-1) { | |
69 | enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT16; | |
70 | enc[1] = value&0xFF; | |
71 | enc[2] = (value>>8)&0xFF; | |
72 | return 3; | |
73 | } else if (value >= -((long long)1<<31) && value <= ((long long)1<<31)-1) { | |
74 | enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT32; | |
75 | enc[1] = value&0xFF; | |
76 | enc[2] = (value>>8)&0xFF; | |
77 | enc[3] = (value>>16)&0xFF; | |
78 | enc[4] = (value>>24)&0xFF; | |
79 | return 5; | |
80 | } else { | |
81 | return 0; | |
82 | } | |
83 | } | |
84 | ||
85 | /* String objects in the form "2391" "-100" without any space and with a | |
86 | * range of values that can fit in an 8, 16 or 32 bit signed value can be | |
87 | * encoded as integers to save space */ | |
88 | int rdbTryIntegerEncoding(char *s, size_t len, unsigned char *enc) { | |
89 | long long value; | |
90 | char *endptr, buf[32]; | |
91 | ||
92 | /* Check if it's possible to encode this value as a number */ | |
93 | value = strtoll(s, &endptr, 10); | |
94 | if (endptr[0] != '\0') return 0; | |
95 | ll2string(buf,32,value); | |
96 | ||
97 | /* If the number converted back into a string is not identical | |
98 | * then it's not possible to encode the string as integer */ | |
99 | if (strlen(buf) != len || memcmp(buf,s,len)) return 0; | |
100 | ||
101 | return rdbEncodeInteger(value,enc); | |
102 | } | |
103 | ||
104 | int rdbSaveLzfStringObject(FILE *fp, unsigned char *s, size_t len) { | |
105 | size_t comprlen, outlen; | |
106 | unsigned char byte; | |
8a623a98 | 107 | int n, nwritten = 0; |
e2641e09 | 108 | void *out; |
109 | ||
110 | /* We require at least four bytes compression for this to be worth it */ | |
111 | if (len <= 4) return 0; | |
112 | outlen = len-4; | |
113 | if ((out = zmalloc(outlen+1)) == NULL) return 0; | |
114 | comprlen = lzf_compress(s, len, out, outlen); | |
115 | if (comprlen == 0) { | |
116 | zfree(out); | |
117 | return 0; | |
118 | } | |
119 | /* Data compressed! Let's save it on disk */ | |
120 | byte = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_LZF; | |
9a68cf91 PN |
121 | if ((n = rdbWriteRaw(fp,&byte,1)) == -1) goto writeerr; |
122 | nwritten += n; | |
8a623a98 PN |
123 | |
124 | if ((n = rdbSaveLen(fp,comprlen)) == -1) goto writeerr; | |
125 | nwritten += n; | |
126 | ||
127 | if ((n = rdbSaveLen(fp,len)) == -1) goto writeerr; | |
128 | nwritten += n; | |
129 | ||
9a68cf91 PN |
130 | if ((n = rdbWriteRaw(fp,out,comprlen)) == -1) goto writeerr; |
131 | nwritten += n; | |
8a623a98 | 132 | |
e2641e09 | 133 | zfree(out); |
8a623a98 | 134 | return nwritten; |
e2641e09 | 135 | |
136 | writeerr: | |
137 | zfree(out); | |
138 | return -1; | |
139 | } | |
140 | ||
141 | /* Save a string objet as [len][data] on disk. If the object is a string | |
142 | * representation of an integer value we try to safe it in a special form */ | |
143 | int rdbSaveRawString(FILE *fp, unsigned char *s, size_t len) { | |
144 | int enclen; | |
8a623a98 | 145 | int n, nwritten = 0; |
e2641e09 | 146 | |
147 | /* Try integer encoding */ | |
148 | if (len <= 11) { | |
149 | unsigned char buf[5]; | |
150 | if ((enclen = rdbTryIntegerEncoding((char*)s,len,buf)) > 0) { | |
9a68cf91 | 151 | if (rdbWriteRaw(fp,buf,enclen) == -1) return -1; |
8a623a98 | 152 | return enclen; |
e2641e09 | 153 | } |
154 | } | |
155 | ||
156 | /* Try LZF compression - under 20 bytes it's unable to compress even | |
157 | * aaaaaaaaaaaaaaaaaa so skip it */ | |
158 | if (server.rdbcompression && len > 20) { | |
8a623a98 PN |
159 | n = rdbSaveLzfStringObject(fp,s,len); |
160 | if (n == -1) return -1; | |
161 | if (n > 0) return n; | |
162 | /* Return value of 0 means data can't be compressed, save the old way */ | |
e2641e09 | 163 | } |
164 | ||
165 | /* Store verbatim */ | |
8a623a98 PN |
166 | if ((n = rdbSaveLen(fp,len)) == -1) return -1; |
167 | nwritten += n; | |
168 | if (len > 0) { | |
9a68cf91 | 169 | if (rdbWriteRaw(fp,s,len) == -1) return -1; |
8a623a98 PN |
170 | nwritten += len; |
171 | } | |
172 | return nwritten; | |
e2641e09 | 173 | } |
174 | ||
175 | /* Save a long long value as either an encoded string or a string. */ | |
176 | int rdbSaveLongLongAsStringObject(FILE *fp, long long value) { | |
177 | unsigned char buf[32]; | |
8a623a98 | 178 | int n, nwritten = 0; |
e2641e09 | 179 | int enclen = rdbEncodeInteger(value,buf); |
180 | if (enclen > 0) { | |
9a68cf91 | 181 | return rdbWriteRaw(fp,buf,enclen); |
e2641e09 | 182 | } else { |
183 | /* Encode as string */ | |
184 | enclen = ll2string((char*)buf,32,value); | |
185 | redisAssert(enclen < 32); | |
8a623a98 PN |
186 | if ((n = rdbSaveLen(fp,enclen)) == -1) return -1; |
187 | nwritten += n; | |
9a68cf91 PN |
188 | if ((n = rdbWriteRaw(fp,buf,enclen)) == -1) return -1; |
189 | nwritten += n; | |
e2641e09 | 190 | } |
8a623a98 | 191 | return nwritten; |
e2641e09 | 192 | } |
193 | ||
194 | /* Like rdbSaveStringObjectRaw() but handle encoded objects */ | |
195 | int rdbSaveStringObject(FILE *fp, robj *obj) { | |
196 | /* Avoid to decode the object, then encode it again, if the | |
197 | * object is alrady integer encoded. */ | |
198 | if (obj->encoding == REDIS_ENCODING_INT) { | |
199 | return rdbSaveLongLongAsStringObject(fp,(long)obj->ptr); | |
200 | } else { | |
201 | redisAssert(obj->encoding == REDIS_ENCODING_RAW); | |
202 | return rdbSaveRawString(fp,obj->ptr,sdslen(obj->ptr)); | |
203 | } | |
204 | } | |
205 | ||
206 | /* Save a double value. Doubles are saved as strings prefixed by an unsigned | |
207 | * 8 bit integer specifing the length of the representation. | |
208 | * This 8 bit integer has special values in order to specify the following | |
209 | * conditions: | |
210 | * 253: not a number | |
211 | * 254: + inf | |
212 | * 255: - inf | |
213 | */ | |
214 | int rdbSaveDoubleValue(FILE *fp, double val) { | |
215 | unsigned char buf[128]; | |
216 | int len; | |
217 | ||
218 | if (isnan(val)) { | |
219 | buf[0] = 253; | |
220 | len = 1; | |
221 | } else if (!isfinite(val)) { | |
222 | len = 1; | |
223 | buf[0] = (val < 0) ? 255 : 254; | |
224 | } else { | |
225 | #if (DBL_MANT_DIG >= 52) && (LLONG_MAX == 0x7fffffffffffffffLL) | |
226 | /* Check if the float is in a safe range to be casted into a | |
227 | * long long. We are assuming that long long is 64 bit here. | |
228 | * Also we are assuming that there are no implementations around where | |
229 | * double has precision < 52 bit. | |
230 | * | |
231 | * Under this assumptions we test if a double is inside an interval | |
232 | * where casting to long long is safe. Then using two castings we | |
233 | * make sure the decimal part is zero. If all this is true we use | |
234 | * integer printing function that is much faster. */ | |
235 | double min = -4503599627370495; /* (2^52)-1 */ | |
236 | double max = 4503599627370496; /* -(2^52) */ | |
237 | if (val > min && val < max && val == ((double)((long long)val))) | |
238 | ll2string((char*)buf+1,sizeof(buf),(long long)val); | |
239 | else | |
240 | #endif | |
241 | snprintf((char*)buf+1,sizeof(buf)-1,"%.17g",val); | |
242 | buf[0] = strlen((char*)buf+1); | |
243 | len = buf[0]+1; | |
244 | } | |
9a68cf91 | 245 | return rdbWriteRaw(fp,buf,len); |
e2641e09 | 246 | } |
247 | ||
248 | /* Save a Redis object. */ | |
249 | int rdbSaveObject(FILE *fp, robj *o) { | |
8a623a98 PN |
250 | int n, nwritten = 0; |
251 | ||
e2641e09 | 252 | if (o->type == REDIS_STRING) { |
253 | /* Save a string value */ | |
8a623a98 PN |
254 | if ((n = rdbSaveStringObject(fp,o)) == -1) return -1; |
255 | nwritten += n; | |
e2641e09 | 256 | } else if (o->type == REDIS_LIST) { |
257 | /* Save a list value */ | |
258 | if (o->encoding == REDIS_ENCODING_ZIPLIST) { | |
259 | unsigned char *p; | |
260 | unsigned char *vstr; | |
261 | unsigned int vlen; | |
262 | long long vlong; | |
263 | ||
8a623a98 PN |
264 | if ((n = rdbSaveLen(fp,ziplistLen(o->ptr))) == -1) return -1; |
265 | nwritten += n; | |
266 | ||
e2641e09 | 267 | p = ziplistIndex(o->ptr,0); |
268 | while(ziplistGet(p,&vstr,&vlen,&vlong)) { | |
269 | if (vstr) { | |
8a623a98 | 270 | if ((n = rdbSaveRawString(fp,vstr,vlen)) == -1) |
e2641e09 | 271 | return -1; |
8a623a98 | 272 | nwritten += n; |
e2641e09 | 273 | } else { |
8a623a98 | 274 | if ((n = rdbSaveLongLongAsStringObject(fp,vlong)) == -1) |
e2641e09 | 275 | return -1; |
8a623a98 | 276 | nwritten += n; |
e2641e09 | 277 | } |
278 | p = ziplistNext(o->ptr,p); | |
279 | } | |
280 | } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) { | |
281 | list *list = o->ptr; | |
282 | listIter li; | |
283 | listNode *ln; | |
284 | ||
8a623a98 PN |
285 | if ((n = rdbSaveLen(fp,listLength(list))) == -1) return -1; |
286 | nwritten += n; | |
287 | ||
e2641e09 | 288 | listRewind(list,&li); |
289 | while((ln = listNext(&li))) { | |
290 | robj *eleobj = listNodeValue(ln); | |
8a623a98 PN |
291 | if ((n = rdbSaveStringObject(fp,eleobj)) == -1) return -1; |
292 | nwritten += n; | |
e2641e09 | 293 | } |
294 | } else { | |
295 | redisPanic("Unknown list encoding"); | |
296 | } | |
297 | } else if (o->type == REDIS_SET) { | |
298 | /* Save a set value */ | |
96ffb2fe PN |
299 | if (o->encoding == REDIS_ENCODING_HT) { |
300 | dict *set = o->ptr; | |
301 | dictIterator *di = dictGetIterator(set); | |
302 | dictEntry *de; | |
e2641e09 | 303 | |
8a623a98 PN |
304 | if ((n = rdbSaveLen(fp,dictSize(set))) == -1) return -1; |
305 | nwritten += n; | |
306 | ||
96ffb2fe PN |
307 | while((de = dictNext(di)) != NULL) { |
308 | robj *eleobj = dictGetEntryKey(de); | |
8a623a98 PN |
309 | if ((n = rdbSaveStringObject(fp,eleobj)) == -1) return -1; |
310 | nwritten += n; | |
96ffb2fe PN |
311 | } |
312 | dictReleaseIterator(di); | |
313 | } else if (o->encoding == REDIS_ENCODING_INTSET) { | |
314 | intset *is = o->ptr; | |
23c64fe5 | 315 | int64_t llval; |
96ffb2fe PN |
316 | int i = 0; |
317 | ||
8a623a98 PN |
318 | if ((n = rdbSaveLen(fp,intsetLen(is))) == -1) return -1; |
319 | nwritten += n; | |
320 | ||
96ffb2fe | 321 | while(intsetGet(is,i++,&llval)) { |
8a623a98 PN |
322 | if ((n = rdbSaveLongLongAsStringObject(fp,llval)) == -1) return -1; |
323 | nwritten += n; | |
96ffb2fe PN |
324 | } |
325 | } else { | |
326 | redisPanic("Unknown set encoding"); | |
e2641e09 | 327 | } |
e2641e09 | 328 | } else if (o->type == REDIS_ZSET) { |
329 | /* Save a set value */ | |
330 | zset *zs = o->ptr; | |
331 | dictIterator *di = dictGetIterator(zs->dict); | |
332 | dictEntry *de; | |
333 | ||
8a623a98 PN |
334 | if ((n = rdbSaveLen(fp,dictSize(zs->dict))) == -1) return -1; |
335 | nwritten += n; | |
336 | ||
e2641e09 | 337 | while((de = dictNext(di)) != NULL) { |
338 | robj *eleobj = dictGetEntryKey(de); | |
339 | double *score = dictGetEntryVal(de); | |
340 | ||
8a623a98 PN |
341 | if ((n = rdbSaveStringObject(fp,eleobj)) == -1) return -1; |
342 | nwritten += n; | |
343 | if ((n = rdbSaveDoubleValue(fp,*score)) == -1) return -1; | |
344 | nwritten += n; | |
e2641e09 | 345 | } |
346 | dictReleaseIterator(di); | |
347 | } else if (o->type == REDIS_HASH) { | |
348 | /* Save a hash value */ | |
349 | if (o->encoding == REDIS_ENCODING_ZIPMAP) { | |
350 | unsigned char *p = zipmapRewind(o->ptr); | |
351 | unsigned int count = zipmapLen(o->ptr); | |
352 | unsigned char *key, *val; | |
353 | unsigned int klen, vlen; | |
354 | ||
8a623a98 PN |
355 | if ((n = rdbSaveLen(fp,count)) == -1) return -1; |
356 | nwritten += n; | |
357 | ||
e2641e09 | 358 | while((p = zipmapNext(p,&key,&klen,&val,&vlen)) != NULL) { |
8a623a98 PN |
359 | if ((n = rdbSaveRawString(fp,key,klen)) == -1) return -1; |
360 | nwritten += n; | |
361 | if ((n = rdbSaveRawString(fp,val,vlen)) == -1) return -1; | |
362 | nwritten += n; | |
e2641e09 | 363 | } |
364 | } else { | |
365 | dictIterator *di = dictGetIterator(o->ptr); | |
366 | dictEntry *de; | |
367 | ||
8a623a98 PN |
368 | if ((n = rdbSaveLen(fp,dictSize((dict*)o->ptr))) == -1) return -1; |
369 | nwritten += n; | |
370 | ||
e2641e09 | 371 | while((de = dictNext(di)) != NULL) { |
372 | robj *key = dictGetEntryKey(de); | |
373 | robj *val = dictGetEntryVal(de); | |
374 | ||
8a623a98 PN |
375 | if ((n = rdbSaveStringObject(fp,key)) == -1) return -1; |
376 | nwritten += n; | |
377 | if ((n = rdbSaveStringObject(fp,val)) == -1) return -1; | |
378 | nwritten += n; | |
e2641e09 | 379 | } |
380 | dictReleaseIterator(di); | |
381 | } | |
382 | } else { | |
383 | redisPanic("Unknown object type"); | |
384 | } | |
8a623a98 | 385 | return nwritten; |
e2641e09 | 386 | } |
387 | ||
388 | /* Return the length the object will have on disk if saved with | |
389 | * the rdbSaveObject() function. Currently we use a trick to get | |
390 | * this length with very little changes to the code. In the future | |
391 | * we could switch to a faster solution. */ | |
bd70a5f5 PN |
392 | off_t rdbSavedObjectLen(robj *o) { |
393 | int len = rdbSaveObject(NULL,o); | |
394 | redisAssert(len != -1); | |
395 | return len; | |
e2641e09 | 396 | } |
397 | ||
4ab98823 | 398 | /* Save a key-value pair, with expire time, type, key, value. |
399 | * On error -1 is returned. | |
400 | * On success if the key was actaully saved 1 is returned, otherwise 0 | |
401 | * is returned (the key was already expired). */ | |
402 | int rdbSaveKeyValuePair(FILE *fp, redisDb *db, robj *key, robj *val, | |
403 | time_t now) | |
404 | { | |
405 | time_t expiretime; | |
406 | ||
1fce3201 | 407 | expiretime = getExpire(db,key); |
4ab98823 | 408 | |
409 | /* Save the expire time */ | |
410 | if (expiretime != -1) { | |
411 | /* If this key is already expired skip it */ | |
412 | if (expiretime < now) return 0; | |
413 | if (rdbSaveType(fp,REDIS_EXPIRETIME) == -1) return -1; | |
414 | if (rdbSaveTime(fp,expiretime) == -1) return -1; | |
415 | } | |
416 | /* Save type, key, value */ | |
417 | if (rdbSaveType(fp,val->type) == -1) return -1; | |
1fce3201 | 418 | if (rdbSaveStringObject(fp,key) == -1) return -1; |
4ab98823 | 419 | if (rdbSaveObject(fp,val) == -1) return -1; |
420 | return 1; | |
421 | } | |
422 | ||
e2641e09 | 423 | /* Save the DB on disk. Return REDIS_ERR on error, REDIS_OK on success */ |
424 | int rdbSave(char *filename) { | |
425 | dictIterator *di = NULL; | |
426 | dictEntry *de; | |
427 | FILE *fp; | |
428 | char tmpfile[256]; | |
429 | int j; | |
430 | time_t now = time(NULL); | |
431 | ||
16d77878 | 432 | /* FIXME: implement .rdb save for disk store properly */ |
433 | redisAssert(server.ds_enabled == 0); | |
e2641e09 | 434 | |
435 | snprintf(tmpfile,256,"temp-%d.rdb", (int) getpid()); | |
436 | fp = fopen(tmpfile,"w"); | |
437 | if (!fp) { | |
438 | redisLog(REDIS_WARNING, "Failed saving the DB: %s", strerror(errno)); | |
439 | return REDIS_ERR; | |
440 | } | |
441 | if (fwrite("REDIS0001",9,1,fp) == 0) goto werr; | |
442 | for (j = 0; j < server.dbnum; j++) { | |
443 | redisDb *db = server.db+j; | |
444 | dict *d = db->dict; | |
445 | if (dictSize(d) == 0) continue; | |
446 | di = dictGetIterator(d); | |
447 | if (!di) { | |
448 | fclose(fp); | |
449 | return REDIS_ERR; | |
450 | } | |
451 | ||
452 | /* Write the SELECT DB opcode */ | |
453 | if (rdbSaveType(fp,REDIS_SELECTDB) == -1) goto werr; | |
454 | if (rdbSaveLen(fp,j) == -1) goto werr; | |
455 | ||
456 | /* Iterate this DB writing every entry */ | |
457 | while((de = dictNext(di)) != NULL) { | |
458 | sds keystr = dictGetEntryKey(de); | |
459 | robj key, *o = dictGetEntryVal(de); | |
e2641e09 | 460 | |
461 | initStaticStringObject(key,keystr); | |
1fce3201 | 462 | if (rdbSaveKeyValuePair(fp,db,&key,o,now) == -1) goto werr; |
e2641e09 | 463 | } |
464 | dictReleaseIterator(di); | |
465 | } | |
466 | /* EOF opcode */ | |
467 | if (rdbSaveType(fp,REDIS_EOF) == -1) goto werr; | |
468 | ||
469 | /* Make sure data will not remain on the OS's output buffers */ | |
470 | fflush(fp); | |
471 | fsync(fileno(fp)); | |
472 | fclose(fp); | |
473 | ||
474 | /* Use RENAME to make sure the DB file is changed atomically only | |
475 | * if the generate DB file is ok. */ | |
476 | if (rename(tmpfile,filename) == -1) { | |
477 | redisLog(REDIS_WARNING,"Error moving temp DB file on the final destination: %s", strerror(errno)); | |
478 | unlink(tmpfile); | |
479 | return REDIS_ERR; | |
480 | } | |
481 | redisLog(REDIS_NOTICE,"DB saved on disk"); | |
482 | server.dirty = 0; | |
483 | server.lastsave = time(NULL); | |
484 | return REDIS_OK; | |
485 | ||
486 | werr: | |
487 | fclose(fp); | |
488 | unlink(tmpfile); | |
489 | redisLog(REDIS_WARNING,"Write error saving DB on disk: %s", strerror(errno)); | |
490 | if (di) dictReleaseIterator(di); | |
491 | return REDIS_ERR; | |
492 | } | |
493 | ||
494 | int rdbSaveBackground(char *filename) { | |
495 | pid_t childpid; | |
496 | ||
497 | if (server.bgsavechildpid != -1) return REDIS_ERR; | |
16d77878 | 498 | redisAssert(server.ds_enabled == 0); |
2f6b31c3 | 499 | server.dirty_before_bgsave = server.dirty; |
e2641e09 | 500 | if ((childpid = fork()) == 0) { |
501 | /* Child */ | |
a5639e7d PN |
502 | if (server.ipfd > 0) close(server.ipfd); |
503 | if (server.sofd > 0) close(server.sofd); | |
e2641e09 | 504 | if (rdbSave(filename) == REDIS_OK) { |
505 | _exit(0); | |
506 | } else { | |
507 | _exit(1); | |
508 | } | |
509 | } else { | |
510 | /* Parent */ | |
511 | if (childpid == -1) { | |
512 | redisLog(REDIS_WARNING,"Can't save in background: fork: %s", | |
513 | strerror(errno)); | |
514 | return REDIS_ERR; | |
515 | } | |
516 | redisLog(REDIS_NOTICE,"Background saving started by pid %d",childpid); | |
517 | server.bgsavechildpid = childpid; | |
518 | updateDictResizePolicy(); | |
519 | return REDIS_OK; | |
520 | } | |
521 | return REDIS_OK; /* unreached */ | |
522 | } | |
523 | ||
524 | void rdbRemoveTempFile(pid_t childpid) { | |
525 | char tmpfile[256]; | |
526 | ||
527 | snprintf(tmpfile,256,"temp-%d.rdb", (int) childpid); | |
528 | unlink(tmpfile); | |
529 | } | |
530 | ||
531 | int rdbLoadType(FILE *fp) { | |
532 | unsigned char type; | |
533 | if (fread(&type,1,1,fp) == 0) return -1; | |
534 | return type; | |
535 | } | |
536 | ||
537 | time_t rdbLoadTime(FILE *fp) { | |
538 | int32_t t32; | |
539 | if (fread(&t32,4,1,fp) == 0) return -1; | |
540 | return (time_t) t32; | |
541 | } | |
542 | ||
543 | /* Load an encoded length from the DB, see the REDIS_RDB_* defines on the top | |
544 | * of this file for a description of how this are stored on disk. | |
545 | * | |
546 | * isencoded is set to 1 if the readed length is not actually a length but | |
547 | * an "encoding type", check the above comments for more info */ | |
548 | uint32_t rdbLoadLen(FILE *fp, int *isencoded) { | |
549 | unsigned char buf[2]; | |
550 | uint32_t len; | |
551 | int type; | |
552 | ||
553 | if (isencoded) *isencoded = 0; | |
554 | if (fread(buf,1,1,fp) == 0) return REDIS_RDB_LENERR; | |
555 | type = (buf[0]&0xC0)>>6; | |
556 | if (type == REDIS_RDB_6BITLEN) { | |
557 | /* Read a 6 bit len */ | |
558 | return buf[0]&0x3F; | |
559 | } else if (type == REDIS_RDB_ENCVAL) { | |
560 | /* Read a 6 bit len encoding type */ | |
561 | if (isencoded) *isencoded = 1; | |
562 | return buf[0]&0x3F; | |
563 | } else if (type == REDIS_RDB_14BITLEN) { | |
564 | /* Read a 14 bit len */ | |
565 | if (fread(buf+1,1,1,fp) == 0) return REDIS_RDB_LENERR; | |
566 | return ((buf[0]&0x3F)<<8)|buf[1]; | |
567 | } else { | |
568 | /* Read a 32 bit len */ | |
569 | if (fread(&len,4,1,fp) == 0) return REDIS_RDB_LENERR; | |
570 | return ntohl(len); | |
571 | } | |
572 | } | |
573 | ||
574 | /* Load an integer-encoded object from file 'fp', with the specified | |
575 | * encoding type 'enctype'. If encode is true the function may return | |
576 | * an integer-encoded object as reply, otherwise the returned object | |
577 | * will always be encoded as a raw string. */ | |
578 | robj *rdbLoadIntegerObject(FILE *fp, int enctype, int encode) { | |
579 | unsigned char enc[4]; | |
580 | long long val; | |
581 | ||
582 | if (enctype == REDIS_RDB_ENC_INT8) { | |
583 | if (fread(enc,1,1,fp) == 0) return NULL; | |
584 | val = (signed char)enc[0]; | |
585 | } else if (enctype == REDIS_RDB_ENC_INT16) { | |
586 | uint16_t v; | |
587 | if (fread(enc,2,1,fp) == 0) return NULL; | |
588 | v = enc[0]|(enc[1]<<8); | |
589 | val = (int16_t)v; | |
590 | } else if (enctype == REDIS_RDB_ENC_INT32) { | |
591 | uint32_t v; | |
592 | if (fread(enc,4,1,fp) == 0) return NULL; | |
593 | v = enc[0]|(enc[1]<<8)|(enc[2]<<16)|(enc[3]<<24); | |
594 | val = (int32_t)v; | |
595 | } else { | |
596 | val = 0; /* anti-warning */ | |
597 | redisPanic("Unknown RDB integer encoding type"); | |
598 | } | |
599 | if (encode) | |
600 | return createStringObjectFromLongLong(val); | |
601 | else | |
602 | return createObject(REDIS_STRING,sdsfromlonglong(val)); | |
603 | } | |
604 | ||
605 | robj *rdbLoadLzfStringObject(FILE*fp) { | |
606 | unsigned int len, clen; | |
607 | unsigned char *c = NULL; | |
608 | sds val = NULL; | |
609 | ||
610 | if ((clen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; | |
611 | if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; | |
612 | if ((c = zmalloc(clen)) == NULL) goto err; | |
613 | if ((val = sdsnewlen(NULL,len)) == NULL) goto err; | |
614 | if (fread(c,clen,1,fp) == 0) goto err; | |
615 | if (lzf_decompress(c,clen,val,len) == 0) goto err; | |
616 | zfree(c); | |
617 | return createObject(REDIS_STRING,val); | |
618 | err: | |
619 | zfree(c); | |
620 | sdsfree(val); | |
621 | return NULL; | |
622 | } | |
623 | ||
624 | robj *rdbGenericLoadStringObject(FILE*fp, int encode) { | |
625 | int isencoded; | |
626 | uint32_t len; | |
627 | sds val; | |
628 | ||
629 | len = rdbLoadLen(fp,&isencoded); | |
630 | if (isencoded) { | |
631 | switch(len) { | |
632 | case REDIS_RDB_ENC_INT8: | |
633 | case REDIS_RDB_ENC_INT16: | |
634 | case REDIS_RDB_ENC_INT32: | |
635 | return rdbLoadIntegerObject(fp,len,encode); | |
636 | case REDIS_RDB_ENC_LZF: | |
637 | return rdbLoadLzfStringObject(fp); | |
638 | default: | |
639 | redisPanic("Unknown RDB encoding type"); | |
640 | } | |
641 | } | |
642 | ||
643 | if (len == REDIS_RDB_LENERR) return NULL; | |
644 | val = sdsnewlen(NULL,len); | |
645 | if (len && fread(val,len,1,fp) == 0) { | |
646 | sdsfree(val); | |
647 | return NULL; | |
648 | } | |
649 | return createObject(REDIS_STRING,val); | |
650 | } | |
651 | ||
652 | robj *rdbLoadStringObject(FILE *fp) { | |
653 | return rdbGenericLoadStringObject(fp,0); | |
654 | } | |
655 | ||
656 | robj *rdbLoadEncodedStringObject(FILE *fp) { | |
657 | return rdbGenericLoadStringObject(fp,1); | |
658 | } | |
659 | ||
660 | /* For information about double serialization check rdbSaveDoubleValue() */ | |
661 | int rdbLoadDoubleValue(FILE *fp, double *val) { | |
662 | char buf[128]; | |
663 | unsigned char len; | |
664 | ||
665 | if (fread(&len,1,1,fp) == 0) return -1; | |
666 | switch(len) { | |
667 | case 255: *val = R_NegInf; return 0; | |
668 | case 254: *val = R_PosInf; return 0; | |
669 | case 253: *val = R_Nan; return 0; | |
670 | default: | |
671 | if (fread(buf,len,1,fp) == 0) return -1; | |
672 | buf[len] = '\0'; | |
673 | sscanf(buf, "%lg", val); | |
674 | return 0; | |
675 | } | |
676 | } | |
677 | ||
678 | /* Load a Redis object of the specified type from the specified file. | |
679 | * On success a newly allocated object is returned, otherwise NULL. */ | |
680 | robj *rdbLoadObject(int type, FILE *fp) { | |
681 | robj *o, *ele, *dec; | |
682 | size_t len; | |
96ffb2fe | 683 | unsigned int i; |
e2641e09 | 684 | |
685 | redisLog(REDIS_DEBUG,"LOADING OBJECT %d (at %d)\n",type,ftell(fp)); | |
686 | if (type == REDIS_STRING) { | |
687 | /* Read string value */ | |
688 | if ((o = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; | |
689 | o = tryObjectEncoding(o); | |
690 | } else if (type == REDIS_LIST) { | |
691 | /* Read list value */ | |
692 | if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; | |
693 | ||
694 | /* Use a real list when there are too many entries */ | |
695 | if (len > server.list_max_ziplist_entries) { | |
696 | o = createListObject(); | |
697 | } else { | |
698 | o = createZiplistObject(); | |
699 | } | |
700 | ||
701 | /* Load every single element of the list */ | |
702 | while(len--) { | |
703 | if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; | |
704 | ||
705 | /* If we are using a ziplist and the value is too big, convert | |
706 | * the object to a real list. */ | |
707 | if (o->encoding == REDIS_ENCODING_ZIPLIST && | |
708 | ele->encoding == REDIS_ENCODING_RAW && | |
709 | sdslen(ele->ptr) > server.list_max_ziplist_value) | |
710 | listTypeConvert(o,REDIS_ENCODING_LINKEDLIST); | |
711 | ||
712 | if (o->encoding == REDIS_ENCODING_ZIPLIST) { | |
713 | dec = getDecodedObject(ele); | |
714 | o->ptr = ziplistPush(o->ptr,dec->ptr,sdslen(dec->ptr),REDIS_TAIL); | |
715 | decrRefCount(dec); | |
716 | decrRefCount(ele); | |
717 | } else { | |
718 | ele = tryObjectEncoding(ele); | |
719 | listAddNodeTail(o->ptr,ele); | |
720 | } | |
721 | } | |
722 | } else if (type == REDIS_SET) { | |
723 | /* Read list/set value */ | |
724 | if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; | |
96ffb2fe PN |
725 | |
726 | /* Use a regular set when there are too many entries. */ | |
727 | if (len > server.set_max_intset_entries) { | |
728 | o = createSetObject(); | |
729 | /* It's faster to expand the dict to the right size asap in order | |
730 | * to avoid rehashing */ | |
731 | if (len > DICT_HT_INITIAL_SIZE) | |
732 | dictExpand(o->ptr,len); | |
733 | } else { | |
734 | o = createIntsetObject(); | |
735 | } | |
736 | ||
e2641e09 | 737 | /* Load every single element of the list/set */ |
96ffb2fe PN |
738 | for (i = 0; i < len; i++) { |
739 | long long llval; | |
e2641e09 | 740 | if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; |
741 | ele = tryObjectEncoding(ele); | |
96ffb2fe PN |
742 | |
743 | if (o->encoding == REDIS_ENCODING_INTSET) { | |
744 | /* Fetch integer value from element */ | |
2df84b72 | 745 | if (isObjectRepresentableAsLongLong(ele,&llval) == REDIS_OK) { |
96ffb2fe PN |
746 | o->ptr = intsetAdd(o->ptr,llval,NULL); |
747 | } else { | |
748 | setTypeConvert(o,REDIS_ENCODING_HT); | |
749 | dictExpand(o->ptr,len); | |
750 | } | |
751 | } | |
752 | ||
753 | /* This will also be called when the set was just converted | |
754 | * to regular hashtable encoded set */ | |
755 | if (o->encoding == REDIS_ENCODING_HT) { | |
756 | dictAdd((dict*)o->ptr,ele,NULL); | |
bad7d097 | 757 | } else { |
758 | decrRefCount(ele); | |
96ffb2fe | 759 | } |
e2641e09 | 760 | } |
761 | } else if (type == REDIS_ZSET) { | |
762 | /* Read list/set value */ | |
763 | size_t zsetlen; | |
764 | zset *zs; | |
765 | ||
766 | if ((zsetlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; | |
767 | o = createZsetObject(); | |
768 | zs = o->ptr; | |
769 | /* Load every single element of the list/set */ | |
770 | while(zsetlen--) { | |
771 | robj *ele; | |
56e52b69 PN |
772 | double score; |
773 | zskiplistNode *znode; | |
e2641e09 | 774 | |
775 | if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; | |
776 | ele = tryObjectEncoding(ele); | |
56e52b69 PN |
777 | if (rdbLoadDoubleValue(fp,&score) == -1) return NULL; |
778 | znode = zslInsert(zs->zsl,score,ele); | |
779 | dictAdd(zs->dict,ele,&znode->score); | |
e2641e09 | 780 | incrRefCount(ele); /* added to skiplist */ |
781 | } | |
782 | } else if (type == REDIS_HASH) { | |
783 | size_t hashlen; | |
784 | ||
785 | if ((hashlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; | |
786 | o = createHashObject(); | |
787 | /* Too many entries? Use an hash table. */ | |
788 | if (hashlen > server.hash_max_zipmap_entries) | |
789 | convertToRealHash(o); | |
790 | /* Load every key/value, then set it into the zipmap or hash | |
791 | * table, as needed. */ | |
792 | while(hashlen--) { | |
793 | robj *key, *val; | |
794 | ||
795 | if ((key = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; | |
796 | if ((val = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; | |
797 | /* If we are using a zipmap and there are too big values | |
798 | * the object is converted to real hash table encoding. */ | |
799 | if (o->encoding != REDIS_ENCODING_HT && | |
800 | ((key->encoding == REDIS_ENCODING_RAW && | |
801 | sdslen(key->ptr) > server.hash_max_zipmap_value) || | |
802 | (val->encoding == REDIS_ENCODING_RAW && | |
803 | sdslen(val->ptr) > server.hash_max_zipmap_value))) | |
804 | { | |
805 | convertToRealHash(o); | |
806 | } | |
807 | ||
808 | if (o->encoding == REDIS_ENCODING_ZIPMAP) { | |
809 | unsigned char *zm = o->ptr; | |
810 | robj *deckey, *decval; | |
811 | ||
812 | /* We need raw string objects to add them to the zipmap */ | |
813 | deckey = getDecodedObject(key); | |
814 | decval = getDecodedObject(val); | |
815 | zm = zipmapSet(zm,deckey->ptr,sdslen(deckey->ptr), | |
816 | decval->ptr,sdslen(decval->ptr),NULL); | |
817 | o->ptr = zm; | |
818 | decrRefCount(deckey); | |
819 | decrRefCount(decval); | |
820 | decrRefCount(key); | |
821 | decrRefCount(val); | |
822 | } else { | |
823 | key = tryObjectEncoding(key); | |
824 | val = tryObjectEncoding(val); | |
825 | dictAdd((dict*)o->ptr,key,val); | |
826 | } | |
827 | } | |
828 | } else { | |
829 | redisPanic("Unknown object type"); | |
830 | } | |
831 | return o; | |
832 | } | |
833 | ||
97e7f8ae | 834 | /* Mark that we are loading in the global state and setup the fields |
835 | * needed to provide loading stats. */ | |
836 | void startLoading(FILE *fp) { | |
837 | struct stat sb; | |
838 | ||
839 | /* Load the DB */ | |
840 | server.loading = 1; | |
841 | server.loading_start_time = time(NULL); | |
842 | if (fstat(fileno(fp), &sb) == -1) { | |
843 | server.loading_total_bytes = 1; /* just to avoid division by zero */ | |
844 | } else { | |
845 | server.loading_total_bytes = sb.st_size; | |
846 | } | |
847 | } | |
848 | ||
849 | /* Refresh the loading progress info */ | |
850 | void loadingProgress(off_t pos) { | |
851 | server.loading_loaded_bytes = pos; | |
852 | } | |
853 | ||
854 | /* Loading finished */ | |
855 | void stopLoading(void) { | |
856 | server.loading = 0; | |
857 | } | |
858 | ||
e2641e09 | 859 | int rdbLoad(char *filename) { |
860 | FILE *fp; | |
861 | uint32_t dbid; | |
862 | int type, retval, rdbver; | |
e2641e09 | 863 | redisDb *db = server.db+0; |
864 | char buf[1024]; | |
865 | time_t expiretime, now = time(NULL); | |
97e7f8ae | 866 | long loops = 0; |
e2641e09 | 867 | |
868 | fp = fopen(filename,"r"); | |
869 | if (!fp) return REDIS_ERR; | |
870 | if (fread(buf,9,1,fp) == 0) goto eoferr; | |
871 | buf[9] = '\0'; | |
872 | if (memcmp(buf,"REDIS",5) != 0) { | |
873 | fclose(fp); | |
874 | redisLog(REDIS_WARNING,"Wrong signature trying to load DB from file"); | |
875 | return REDIS_ERR; | |
876 | } | |
877 | rdbver = atoi(buf+5); | |
878 | if (rdbver != 1) { | |
879 | fclose(fp); | |
880 | redisLog(REDIS_WARNING,"Can't handle RDB format version %d",rdbver); | |
881 | return REDIS_ERR; | |
882 | } | |
97e7f8ae | 883 | |
884 | startLoading(fp); | |
e2641e09 | 885 | while(1) { |
886 | robj *key, *val; | |
e2641e09 | 887 | expiretime = -1; |
97e7f8ae | 888 | |
889 | /* Serve the clients from time to time */ | |
890 | if (!(loops++ % 1000)) { | |
891 | loadingProgress(ftello(fp)); | |
892 | aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT); | |
893 | } | |
894 | ||
e2641e09 | 895 | /* Read type. */ |
896 | if ((type = rdbLoadType(fp)) == -1) goto eoferr; | |
897 | if (type == REDIS_EXPIRETIME) { | |
898 | if ((expiretime = rdbLoadTime(fp)) == -1) goto eoferr; | |
899 | /* We read the time so we need to read the object type again */ | |
900 | if ((type = rdbLoadType(fp)) == -1) goto eoferr; | |
901 | } | |
902 | if (type == REDIS_EOF) break; | |
903 | /* Handle SELECT DB opcode as a special case */ | |
904 | if (type == REDIS_SELECTDB) { | |
905 | if ((dbid = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) | |
906 | goto eoferr; | |
907 | if (dbid >= (unsigned)server.dbnum) { | |
908 | redisLog(REDIS_WARNING,"FATAL: Data file was created with a Redis server configured to handle more than %d databases. Exiting\n", server.dbnum); | |
909 | exit(1); | |
910 | } | |
911 | db = server.db+dbid; | |
912 | continue; | |
913 | } | |
914 | /* Read key */ | |
915 | if ((key = rdbLoadStringObject(fp)) == NULL) goto eoferr; | |
916 | /* Read value */ | |
917 | if ((val = rdbLoadObject(type,fp)) == NULL) goto eoferr; | |
918 | /* Check if the key already expired */ | |
919 | if (expiretime != -1 && expiretime < now) { | |
920 | decrRefCount(key); | |
921 | decrRefCount(val); | |
922 | continue; | |
923 | } | |
924 | /* Add the new object in the hash table */ | |
925 | retval = dbAdd(db,key,val); | |
926 | if (retval == REDIS_ERR) { | |
927 | redisLog(REDIS_WARNING,"Loading DB, duplicated key (%s) found! Unrecoverable error, exiting now.", key->ptr); | |
928 | exit(1); | |
929 | } | |
930 | /* Set the expire time if needed */ | |
931 | if (expiretime != -1) setExpire(db,key,expiretime); | |
932 | ||
e2641e09 | 933 | decrRefCount(key); |
e2641e09 | 934 | } |
935 | fclose(fp); | |
97e7f8ae | 936 | stopLoading(); |
e2641e09 | 937 | return REDIS_OK; |
938 | ||
939 | eoferr: /* unexpected end of file is handled here with a fatal exit */ | |
940 | redisLog(REDIS_WARNING,"Short read or OOM loading DB. Unrecoverable error, aborting now."); | |
941 | exit(1); | |
942 | return REDIS_ERR; /* Just to avoid warning */ | |
943 | } | |
944 | ||
945 | /* A background saving child (BGSAVE) terminated its work. Handle this. */ | |
946 | void backgroundSaveDoneHandler(int statloc) { | |
947 | int exitcode = WEXITSTATUS(statloc); | |
948 | int bysignal = WIFSIGNALED(statloc); | |
949 | ||
950 | if (!bysignal && exitcode == 0) { | |
951 | redisLog(REDIS_NOTICE, | |
952 | "Background saving terminated with success"); | |
2f6b31c3 | 953 | server.dirty = server.dirty - server.dirty_before_bgsave; |
e2641e09 | 954 | server.lastsave = time(NULL); |
955 | } else if (!bysignal && exitcode != 0) { | |
956 | redisLog(REDIS_WARNING, "Background saving error"); | |
957 | } else { | |
958 | redisLog(REDIS_WARNING, | |
959 | "Background saving terminated by signal %d", WTERMSIG(statloc)); | |
960 | rdbRemoveTempFile(server.bgsavechildpid); | |
961 | } | |
962 | server.bgsavechildpid = -1; | |
963 | /* Possibly there are slaves waiting for a BGSAVE in order to be served | |
964 | * (the first stage of SYNC is a bulk transfer of dump.rdb) */ | |
965 | updateSlavesWaitingBgsave(exitcode == 0 ? REDIS_OK : REDIS_ERR); | |
966 | } |