8 #include <sys/resource.h>
11 /* Called when the user switches from "appendonly yes" to "appendonly no"
12 * at runtime using the CONFIG command. */
13 void stopAppendOnly(void) {
14 flushAppendOnlyFile();
15 aof_fsync(server
.appendfd
);
16 close(server
.appendfd
);
19 server
.appendseldb
= -1;
20 server
.appendonly
= 0;
21 /* rewrite operation in progress? kill it, wait child exit */
22 if (server
.bgsavechildpid
!= -1) {
25 if (kill(server
.bgsavechildpid
,SIGKILL
) != -1)
26 wait3(&statloc
,0,NULL
);
27 /* reset the buffer accumulating changes while the child saves */
28 sdsfree(server
.bgrewritebuf
);
29 server
.bgrewritebuf
= sdsempty();
30 server
.bgsavechildpid
= -1;
34 /* Called when the user switches from "appendonly no" to "appendonly yes"
35 * at runtime using the CONFIG command. */
36 int startAppendOnly(void) {
37 server
.appendonly
= 1;
38 server
.lastfsync
= time(NULL
);
39 server
.appendfd
= open(server
.appendfilename
,O_WRONLY
|O_APPEND
|O_CREAT
,0644);
40 if (server
.appendfd
== -1) {
41 redisLog(REDIS_WARNING
,"Used tried to switch on AOF via CONFIG, but I can't open the AOF file: %s",strerror(errno
));
44 if (rewriteAppendOnlyFileBackground() == REDIS_ERR
) {
45 server
.appendonly
= 0;
46 close(server
.appendfd
);
47 redisLog(REDIS_WARNING
,"Used tried to switch on AOF via CONFIG, I can't trigger a background AOF rewrite operation. Check the above logs for more info about the error.",strerror(errno
));
53 /* Write the append only file buffer on disk.
55 * Since we are required to write the AOF before replying to the client,
56 * and the only way the client socket can get a write is entering when the
57 * the event loop, we accumulate all the AOF writes in a memory
58 * buffer and write it on disk using this function just before entering
59 * the event loop again. */
60 void flushAppendOnlyFile(void) {
64 if (sdslen(server
.aofbuf
) == 0) return;
66 /* We want to perform a single write. This should be guaranteed atomic
67 * at least if the filesystem we are writing is a real physical one.
68 * While this will save us against the server being killed I don't think
69 * there is much to do about the whole server stopping for power problems
71 nwritten
= write(server
.appendfd
,server
.aofbuf
,sdslen(server
.aofbuf
));
72 if (nwritten
!= (signed)sdslen(server
.aofbuf
)) {
73 /* Ooops, we are in troubles. The best thing to do for now is
74 * aborting instead of giving the illusion that everything is
75 * working as expected. */
77 redisLog(REDIS_WARNING
,"Exiting on error writing to the append-only file: %s",strerror(errno
));
79 redisLog(REDIS_WARNING
,"Exiting on short write while writing to the append-only file: %s",strerror(errno
));
83 sdsfree(server
.aofbuf
);
84 server
.aofbuf
= sdsempty();
86 /* Don't Fsync if no-appendfsync-on-rewrite is set to yes and we have
87 * childs performing heavy I/O on disk. */
88 if (server
.no_appendfsync_on_rewrite
&&
89 (server
.bgrewritechildpid
!= -1 || server
.bgsavechildpid
!= -1))
93 if (server
.appendfsync
== APPENDFSYNC_ALWAYS
||
94 (server
.appendfsync
== APPENDFSYNC_EVERYSEC
&&
95 now
-server
.lastfsync
> 1))
97 /* aof_fsync is defined as fdatasync() for Linux in order to avoid
98 * flushing metadata. */
99 aof_fsync(server
.appendfd
); /* Let's try to get this data on the disk */
100 server
.lastfsync
= now
;
104 sds
catAppendOnlyGenericCommand(sds buf
, int argc
, robj
**argv
) {
106 buf
= sdscatprintf(buf
,"*%d\r\n",argc
);
107 for (j
= 0; j
< argc
; j
++) {
108 robj
*o
= getDecodedObject(argv
[j
]);
109 buf
= sdscatprintf(buf
,"$%lu\r\n",(unsigned long)sdslen(o
->ptr
));
110 buf
= sdscatlen(buf
,o
->ptr
,sdslen(o
->ptr
));
111 buf
= sdscatlen(buf
,"\r\n",2);
117 sds
catAppendOnlyExpireAtCommand(sds buf
, robj
*key
, robj
*seconds
) {
122 /* Make sure we can use strtol */
123 seconds
= getDecodedObject(seconds
);
124 when
= time(NULL
)+strtol(seconds
->ptr
,NULL
,10);
125 decrRefCount(seconds
);
127 argv
[0] = createStringObject("EXPIREAT",8);
129 argv
[2] = createObject(REDIS_STRING
,
130 sdscatprintf(sdsempty(),"%ld",when
));
131 buf
= catAppendOnlyGenericCommand(buf
, argc
, argv
);
132 decrRefCount(argv
[0]);
133 decrRefCount(argv
[2]);
137 void feedAppendOnlyFile(struct redisCommand
*cmd
, int dictid
, robj
**argv
, int argc
) {
138 sds buf
= sdsempty();
141 /* The DB this command was targetting is not the same as the last command
142 * we appendend. To issue a SELECT command is needed. */
143 if (dictid
!= server
.appendseldb
) {
146 snprintf(seldb
,sizeof(seldb
),"%d",dictid
);
147 buf
= sdscatprintf(buf
,"*2\r\n$6\r\nSELECT\r\n$%lu\r\n%s\r\n",
148 (unsigned long)strlen(seldb
),seldb
);
149 server
.appendseldb
= dictid
;
152 if (cmd
->proc
== expireCommand
) {
153 /* Translate EXPIRE into EXPIREAT */
154 buf
= catAppendOnlyExpireAtCommand(buf
,argv
[1],argv
[2]);
155 } else if (cmd
->proc
== setexCommand
) {
156 /* Translate SETEX to SET and EXPIREAT */
157 tmpargv
[0] = createStringObject("SET",3);
158 tmpargv
[1] = argv
[1];
159 tmpargv
[2] = argv
[3];
160 buf
= catAppendOnlyGenericCommand(buf
,3,tmpargv
);
161 decrRefCount(tmpargv
[0]);
162 buf
= catAppendOnlyExpireAtCommand(buf
,argv
[1],argv
[2]);
164 buf
= catAppendOnlyGenericCommand(buf
,argc
,argv
);
167 /* Append to the AOF buffer. This will be flushed on disk just before
168 * of re-entering the event loop, so before the client will get a
169 * positive reply about the operation performed. */
170 server
.aofbuf
= sdscatlen(server
.aofbuf
,buf
,sdslen(buf
));
172 /* If a background append only file rewriting is in progress we want to
173 * accumulate the differences between the child DB and the current one
174 * in a buffer, so that when the child process will do its work we
175 * can append the differences to the new append only file. */
176 if (server
.bgrewritechildpid
!= -1)
177 server
.bgrewritebuf
= sdscatlen(server
.bgrewritebuf
,buf
,sdslen(buf
));
182 /* In Redis commands are always executed in the context of a client, so in
183 * order to load the append only file we need to create a fake client. */
184 struct redisClient
*createFakeClient(void) {
185 struct redisClient
*c
= zmalloc(sizeof(*c
));
189 c
->querybuf
= sdsempty();
194 /* We set the fake client as a slave waiting for the synchronization
195 * so that Redis will not try to send replies to this client. */
196 c
->replstate
= REDIS_REPL_WAIT_BGSAVE_START
;
197 c
->reply
= listCreate();
198 c
->watched_keys
= listCreate();
199 listSetFreeMethod(c
->reply
,decrRefCount
);
200 listSetDupMethod(c
->reply
,dupClientReplyValue
);
201 initClientMultiState(c
);
205 void freeFakeClient(struct redisClient
*c
) {
206 sdsfree(c
->querybuf
);
207 listRelease(c
->reply
);
208 listRelease(c
->watched_keys
);
209 freeClientMultiState(c
);
213 /* Replay the append log file. On error REDIS_OK is returned. On non fatal
214 * error (the append only file is zero-length) REDIS_ERR is returned. On
215 * fatal error an error message is logged and the program exists. */
216 int loadAppendOnlyFile(char *filename
) {
217 struct redisClient
*fakeClient
;
218 FILE *fp
= fopen(filename
,"r");
219 struct redis_stat sb
;
220 int appendonly
= server
.appendonly
;
223 if (redis_fstat(fileno(fp
),&sb
) != -1 && sb
.st_size
== 0)
227 redisLog(REDIS_WARNING
,"Fatal error: can't open the append log file for reading: %s",strerror(errno
));
231 /* Temporarily disable AOF, to prevent EXEC from feeding a MULTI
232 * to the same file we're about to read. */
233 server
.appendonly
= 0;
235 fakeClient
= createFakeClient();
244 struct redisCommand
*cmd
;
246 /* Serve the clients from time to time */
247 if (!(loops
++ % 1000)) {
248 loadingProgress(ftello(fp
));
249 aeProcessEvents(server
.el
, AE_FILE_EVENTS
|AE_DONT_WAIT
);
252 if (fgets(buf
,sizeof(buf
),fp
) == NULL
) {
258 if (buf
[0] != '*') goto fmterr
;
260 argv
= zmalloc(sizeof(robj
*)*argc
);
261 for (j
= 0; j
< argc
; j
++) {
262 if (fgets(buf
,sizeof(buf
),fp
) == NULL
) goto readerr
;
263 if (buf
[0] != '$') goto fmterr
;
264 len
= strtol(buf
+1,NULL
,10);
265 argsds
= sdsnewlen(NULL
,len
);
266 if (len
&& fread(argsds
,len
,1,fp
) == 0) goto fmterr
;
267 argv
[j
] = createObject(REDIS_STRING
,argsds
);
268 if (fread(buf
,2,1,fp
) == 0) goto fmterr
; /* discard CRLF */
272 cmd
= lookupCommand(argv
[0]->ptr
);
274 redisLog(REDIS_WARNING
,"Unknown command '%s' reading the append only file", argv
[0]->ptr
);
277 /* Run the command in the context of a fake client */
278 fakeClient
->argc
= argc
;
279 fakeClient
->argv
= argv
;
280 cmd
->proc(fakeClient
);
282 /* The fake client should not have a reply */
283 redisAssert(fakeClient
->bufpos
== 0 && listLength(fakeClient
->reply
) == 0);
285 /* Clean up, ready for the next command */
286 for (j
= 0; j
< argc
; j
++) decrRefCount(argv
[j
]);
290 /* This point can only be reached when EOF is reached without errors.
291 * If the client is in the middle of a MULTI/EXEC, log error and quit. */
292 if (fakeClient
->flags
& REDIS_MULTI
) goto readerr
;
295 freeFakeClient(fakeClient
);
296 server
.appendonly
= appendonly
;
302 redisLog(REDIS_WARNING
,"Unexpected end of file reading the append only file");
304 redisLog(REDIS_WARNING
,"Unrecoverable error reading the append only file: %s", strerror(errno
));
308 redisLog(REDIS_WARNING
,"Bad file format reading the append only file: make a backup of your AOF file, then use ./redis-check-aof --fix <filename>");
312 /* Write a sequence of commands able to fully rebuild the dataset into
313 * "filename". Used both by REWRITEAOF and BGREWRITEAOF. */
314 int rewriteAppendOnlyFile(char *filename
) {
315 dictIterator
*di
= NULL
;
320 time_t now
= time(NULL
);
322 /* Note that we have to use a different temp name here compared to the
323 * one used by rewriteAppendOnlyFileBackground() function. */
324 snprintf(tmpfile
,256,"temp-rewriteaof-%d.aof", (int) getpid());
325 fp
= fopen(tmpfile
,"w");
327 redisLog(REDIS_WARNING
, "Failed rewriting the append only file: %s", strerror(errno
));
330 for (j
= 0; j
< server
.dbnum
; j
++) {
331 char selectcmd
[] = "*2\r\n$6\r\nSELECT\r\n";
332 redisDb
*db
= server
.db
+j
;
334 if (dictSize(d
) == 0) continue;
335 di
= dictGetIterator(d
);
341 /* SELECT the new DB */
342 if (fwrite(selectcmd
,sizeof(selectcmd
)-1,1,fp
) == 0) goto werr
;
343 if (fwriteBulkLongLong(fp
,j
) == 0) goto werr
;
345 /* Iterate this DB writing every entry */
346 while((de
= dictNext(di
)) != NULL
) {
347 sds keystr
= dictGetEntryKey(de
);
351 keystr
= dictGetEntryKey(de
);
352 o
= dictGetEntryVal(de
);
353 initStaticStringObject(key
,keystr
);
355 expiretime
= getExpire(db
,&key
);
357 /* Save the key and associated value */
358 if (o
->type
== REDIS_STRING
) {
359 /* Emit a SET command */
360 char cmd
[]="*3\r\n$3\r\nSET\r\n";
361 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
363 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
364 if (fwriteBulkObject(fp
,o
) == 0) goto werr
;
365 } else if (o
->type
== REDIS_LIST
) {
366 /* Emit the RPUSHes needed to rebuild the list */
367 char cmd
[]="*3\r\n$5\r\nRPUSH\r\n";
368 if (o
->encoding
== REDIS_ENCODING_ZIPLIST
) {
369 unsigned char *zl
= o
->ptr
;
370 unsigned char *p
= ziplistIndex(zl
,0);
375 while(ziplistGet(p
,&vstr
,&vlen
,&vlong
)) {
376 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
377 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
379 if (fwriteBulkString(fp
,(char*)vstr
,vlen
) == 0)
382 if (fwriteBulkLongLong(fp
,vlong
) == 0)
385 p
= ziplistNext(zl
,p
);
387 } else if (o
->encoding
== REDIS_ENCODING_LINKEDLIST
) {
392 listRewind(list
,&li
);
393 while((ln
= listNext(&li
))) {
394 robj
*eleobj
= listNodeValue(ln
);
396 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
397 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
398 if (fwriteBulkObject(fp
,eleobj
) == 0) goto werr
;
401 redisPanic("Unknown list encoding");
403 } else if (o
->type
== REDIS_SET
) {
404 char cmd
[]="*3\r\n$4\r\nSADD\r\n";
406 /* Emit the SADDs needed to rebuild the set */
407 if (o
->encoding
== REDIS_ENCODING_INTSET
) {
410 while(intsetGet(o
->ptr
,ii
++,&llval
)) {
411 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
412 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
413 if (fwriteBulkLongLong(fp
,llval
) == 0) goto werr
;
415 } else if (o
->encoding
== REDIS_ENCODING_HT
) {
416 dictIterator
*di
= dictGetIterator(o
->ptr
);
418 while((de
= dictNext(di
)) != NULL
) {
419 robj
*eleobj
= dictGetEntryKey(de
);
420 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
421 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
422 if (fwriteBulkObject(fp
,eleobj
) == 0) goto werr
;
424 dictReleaseIterator(di
);
426 redisPanic("Unknown set encoding");
428 } else if (o
->type
== REDIS_ZSET
) {
429 /* Emit the ZADDs needed to rebuild the sorted set */
431 dictIterator
*di
= dictGetIterator(zs
->dict
);
434 while((de
= dictNext(di
)) != NULL
) {
435 char cmd
[]="*4\r\n$4\r\nZADD\r\n";
436 robj
*eleobj
= dictGetEntryKey(de
);
437 double *score
= dictGetEntryVal(de
);
439 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
440 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
441 if (fwriteBulkDouble(fp
,*score
) == 0) goto werr
;
442 if (fwriteBulkObject(fp
,eleobj
) == 0) goto werr
;
444 dictReleaseIterator(di
);
445 } else if (o
->type
== REDIS_HASH
) {
446 char cmd
[]="*4\r\n$4\r\nHSET\r\n";
448 /* Emit the HSETs needed to rebuild the hash */
449 if (o
->encoding
== REDIS_ENCODING_ZIPMAP
) {
450 unsigned char *p
= zipmapRewind(o
->ptr
);
451 unsigned char *field
, *val
;
452 unsigned int flen
, vlen
;
454 while((p
= zipmapNext(p
,&field
,&flen
,&val
,&vlen
)) != NULL
) {
455 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
456 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
457 if (fwriteBulkString(fp
,(char*)field
,flen
) == 0)
459 if (fwriteBulkString(fp
,(char*)val
,vlen
) == 0)
463 dictIterator
*di
= dictGetIterator(o
->ptr
);
466 while((de
= dictNext(di
)) != NULL
) {
467 robj
*field
= dictGetEntryKey(de
);
468 robj
*val
= dictGetEntryVal(de
);
470 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
471 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
472 if (fwriteBulkObject(fp
,field
) == 0) goto werr
;
473 if (fwriteBulkObject(fp
,val
) == 0) goto werr
;
475 dictReleaseIterator(di
);
478 redisPanic("Unknown object type");
480 /* Save the expire time */
481 if (expiretime
!= -1) {
482 char cmd
[]="*3\r\n$8\r\nEXPIREAT\r\n";
483 /* If this key is already expired skip it */
484 if (expiretime
< now
) continue;
485 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
486 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
487 if (fwriteBulkLongLong(fp
,expiretime
) == 0) goto werr
;
490 dictReleaseIterator(di
);
493 /* Make sure data will not remain on the OS's output buffers */
495 aof_fsync(fileno(fp
));
498 /* Use RENAME to make sure the DB file is changed atomically only
499 * if the generate DB file is ok. */
500 if (rename(tmpfile
,filename
) == -1) {
501 redisLog(REDIS_WARNING
,"Error moving temp append only file on the final destination: %s", strerror(errno
));
505 redisLog(REDIS_NOTICE
,"SYNC append only file rewrite performed");
511 redisLog(REDIS_WARNING
,"Write error writing append only file on disk: %s", strerror(errno
));
512 if (di
) dictReleaseIterator(di
);
516 /* This is how rewriting of the append only file in background works:
518 * 1) The user calls BGREWRITEAOF
519 * 2) Redis calls this function, that forks():
520 * 2a) the child rewrite the append only file in a temp file.
521 * 2b) the parent accumulates differences in server.bgrewritebuf.
522 * 3) When the child finished '2a' exists.
523 * 4) The parent will trap the exit code, if it's OK, will append the
524 * data accumulated into server.bgrewritebuf into the temp file, and
525 * finally will rename(2) the temp file in the actual file name.
526 * The the new file is reopened as the new append only file. Profit!
528 int rewriteAppendOnlyFileBackground(void) {
531 if (server
.bgrewritechildpid
!= -1) return REDIS_ERR
;
532 if (server
.ds_enabled
!= 0) {
533 redisLog(REDIS_WARNING
,"BGREWRITEAOF called with diskstore enabled: AOF is not supported when diskstore is enabled. Operation not performed.");
536 if ((childpid
= fork()) == 0) {
540 if (server
.ipfd
> 0) close(server
.ipfd
);
541 if (server
.sofd
> 0) close(server
.sofd
);
542 snprintf(tmpfile
,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
543 if (rewriteAppendOnlyFile(tmpfile
) == REDIS_OK
) {
550 if (childpid
== -1) {
551 redisLog(REDIS_WARNING
,
552 "Can't rewrite append only file in background: fork: %s",
556 redisLog(REDIS_NOTICE
,
557 "Background append only file rewriting started by pid %d",childpid
);
558 server
.bgrewritechildpid
= childpid
;
559 updateDictResizePolicy();
560 /* We set appendseldb to -1 in order to force the next call to the
561 * feedAppendOnlyFile() to issue a SELECT command, so the differences
562 * accumulated by the parent into server.bgrewritebuf will start
563 * with a SELECT statement and it will be safe to merge. */
564 server
.appendseldb
= -1;
567 return REDIS_OK
; /* unreached */
570 void bgrewriteaofCommand(redisClient
*c
) {
571 if (server
.bgrewritechildpid
!= -1) {
572 addReplyError(c
,"Background append only file rewriting already in progress");
575 if (rewriteAppendOnlyFileBackground() == REDIS_OK
) {
576 addReplyStatus(c
,"Background append only file rewriting started");
578 addReply(c
,shared
.err
);
582 void aofRemoveTempFile(pid_t childpid
) {
585 snprintf(tmpfile
,256,"temp-rewriteaof-bg-%d.aof", (int) childpid
);
589 /* A background append only file rewriting (BGREWRITEAOF) terminated its work.
591 void backgroundRewriteDoneHandler(int exitcode
, int bysignal
) {
592 if (!bysignal
&& exitcode
== 0) {
596 redisLog(REDIS_NOTICE
,
597 "Background append only file rewriting terminated with success");
598 /* Now it's time to flush the differences accumulated by the parent */
599 snprintf(tmpfile
,256,"temp-rewriteaof-bg-%d.aof", (int) server
.bgrewritechildpid
);
600 fd
= open(tmpfile
,O_WRONLY
|O_APPEND
);
602 redisLog(REDIS_WARNING
, "Not able to open the temp append only file produced by the child: %s", strerror(errno
));
605 /* Flush our data... */
606 if (write(fd
,server
.bgrewritebuf
,sdslen(server
.bgrewritebuf
)) !=
607 (signed) sdslen(server
.bgrewritebuf
)) {
608 redisLog(REDIS_WARNING
, "Error or short write trying to flush the parent diff of the append log file in the child temp file: %s", strerror(errno
));
612 redisLog(REDIS_NOTICE
,"Parent diff flushed into the new append log file with success (%lu bytes)",sdslen(server
.bgrewritebuf
));
613 /* Now our work is to rename the temp file into the stable file. And
614 * switch the file descriptor used by the server for append only. */
615 if (rename(tmpfile
,server
.appendfilename
) == -1) {
616 redisLog(REDIS_WARNING
,"Can't rename the temp append only file into the stable one: %s", strerror(errno
));
620 /* Mission completed... almost */
621 redisLog(REDIS_NOTICE
,"Append only file successfully rewritten.");
622 if (server
.appendfd
!= -1) {
623 /* If append only is actually enabled... */
624 close(server
.appendfd
);
625 server
.appendfd
= fd
;
626 if (server
.appendfsync
!= APPENDFSYNC_NO
) aof_fsync(fd
);
627 server
.appendseldb
= -1; /* Make sure it will issue SELECT */
628 redisLog(REDIS_NOTICE
,"The new append only file was selected for future appends.");
630 /* If append only is disabled we just generate a dump in this
631 * format. Why not? */
634 } else if (!bysignal
&& exitcode
!= 0) {
635 redisLog(REDIS_WARNING
, "Background append only file rewriting error");
637 redisLog(REDIS_WARNING
,
638 "Background append only file rewriting terminated by signal %d",
642 sdsfree(server
.bgrewritebuf
);
643 server
.bgrewritebuf
= sdsempty();
644 aofRemoveTempFile(server
.bgrewritechildpid
);
645 server
.bgrewritechildpid
= -1;