8 #include <sys/resource.h>
11 /* Called when the user switches from "appendonly yes" to "appendonly no"
12 * at runtime using the CONFIG command. */
13 void stopAppendOnly(void) {
14 flushAppendOnlyFile();
15 aof_fsync(server
.appendfd
);
16 close(server
.appendfd
);
19 server
.appendseldb
= -1;
20 server
.appendonly
= 0;
21 /* rewrite operation in progress? kill it, wait child exit */
22 if (server
.bgsavechildpid
!= -1) {
25 if (kill(server
.bgsavechildpid
,SIGKILL
) != -1)
26 wait3(&statloc
,0,NULL
);
27 /* reset the buffer accumulating changes while the child saves */
28 sdsfree(server
.bgrewritebuf
);
29 server
.bgrewritebuf
= sdsempty();
30 server
.bgsavechildpid
= -1;
34 /* Called when the user switches from "appendonly no" to "appendonly yes"
35 * at runtime using the CONFIG command. */
36 int startAppendOnly(void) {
37 server
.appendonly
= 1;
38 server
.lastfsync
= time(NULL
);
39 server
.appendfd
= open(server
.appendfilename
,O_WRONLY
|O_APPEND
|O_CREAT
,0644);
40 if (server
.appendfd
== -1) {
41 redisLog(REDIS_WARNING
,"Used tried to switch on AOF via CONFIG, but I can't open the AOF file: %s",strerror(errno
));
44 if (rewriteAppendOnlyFileBackground() == REDIS_ERR
) {
45 server
.appendonly
= 0;
46 close(server
.appendfd
);
47 redisLog(REDIS_WARNING
,"Used tried to switch on AOF via CONFIG, I can't trigger a background AOF rewrite operation. Check the above logs for more info about the error.",strerror(errno
));
53 /* Write the append only file buffer on disk.
55 * Since we are required to write the AOF before replying to the client,
56 * and the only way the client socket can get a write is entering when the
57 * the event loop, we accumulate all the AOF writes in a memory
58 * buffer and write it on disk using this function just before entering
59 * the event loop again. */
60 void flushAppendOnlyFile(void) {
64 if (sdslen(server
.aofbuf
) == 0) return;
66 /* We want to perform a single write. This should be guaranteed atomic
67 * at least if the filesystem we are writing is a real physical one.
68 * While this will save us against the server being killed I don't think
69 * there is much to do about the whole server stopping for power problems
71 nwritten
= write(server
.appendfd
,server
.aofbuf
,sdslen(server
.aofbuf
));
72 if (nwritten
!= (signed)sdslen(server
.aofbuf
)) {
73 /* Ooops, we are in troubles. The best thing to do for now is
74 * aborting instead of giving the illusion that everything is
75 * working as expected. */
77 redisLog(REDIS_WARNING
,"Exiting on error writing to the append-only file: %s",strerror(errno
));
79 redisLog(REDIS_WARNING
,"Exiting on short write while writing to the append-only file: %s",strerror(errno
));
83 sdsfree(server
.aofbuf
);
84 server
.aofbuf
= sdsempty();
86 /* Don't Fsync if no-appendfsync-on-rewrite is set to yes and we have
87 * childs performing heavy I/O on disk. */
88 if (server
.no_appendfsync_on_rewrite
&&
89 (server
.bgrewritechildpid
!= -1 || server
.bgsavechildpid
!= -1))
93 if (server
.appendfsync
== APPENDFSYNC_ALWAYS
||
94 (server
.appendfsync
== APPENDFSYNC_EVERYSEC
&&
95 now
-server
.lastfsync
> 1))
97 /* aof_fsync is defined as fdatasync() for Linux in order to avoid
98 * flushing metadata. */
99 aof_fsync(server
.appendfd
); /* Let's try to get this data on the disk */
100 server
.lastfsync
= now
;
104 sds
catAppendOnlyGenericCommand(sds buf
, int argc
, robj
**argv
) {
106 buf
= sdscatprintf(buf
,"*%d\r\n",argc
);
107 for (j
= 0; j
< argc
; j
++) {
108 robj
*o
= getDecodedObject(argv
[j
]);
109 buf
= sdscatprintf(buf
,"$%lu\r\n",(unsigned long)sdslen(o
->ptr
));
110 buf
= sdscatlen(buf
,o
->ptr
,sdslen(o
->ptr
));
111 buf
= sdscatlen(buf
,"\r\n",2);
117 sds
catAppendOnlyExpireAtCommand(sds buf
, robj
*key
, robj
*seconds
) {
122 /* Make sure we can use strtol */
123 seconds
= getDecodedObject(seconds
);
124 when
= time(NULL
)+strtol(seconds
->ptr
,NULL
,10);
125 decrRefCount(seconds
);
127 argv
[0] = createStringObject("EXPIREAT",8);
129 argv
[2] = createObject(REDIS_STRING
,
130 sdscatprintf(sdsempty(),"%ld",when
));
131 buf
= catAppendOnlyGenericCommand(buf
, argc
, argv
);
132 decrRefCount(argv
[0]);
133 decrRefCount(argv
[2]);
137 void feedAppendOnlyFile(struct redisCommand
*cmd
, int dictid
, robj
**argv
, int argc
) {
138 sds buf
= sdsempty();
141 /* The DB this command was targetting is not the same as the last command
142 * we appendend. To issue a SELECT command is needed. */
143 if (dictid
!= server
.appendseldb
) {
146 snprintf(seldb
,sizeof(seldb
),"%d",dictid
);
147 buf
= sdscatprintf(buf
,"*2\r\n$6\r\nSELECT\r\n$%lu\r\n%s\r\n",
148 (unsigned long)strlen(seldb
),seldb
);
149 server
.appendseldb
= dictid
;
152 if (cmd
->proc
== expireCommand
) {
153 /* Translate EXPIRE into EXPIREAT */
154 buf
= catAppendOnlyExpireAtCommand(buf
,argv
[1],argv
[2]);
155 } else if (cmd
->proc
== setexCommand
) {
156 /* Translate SETEX to SET and EXPIREAT */
157 tmpargv
[0] = createStringObject("SET",3);
158 tmpargv
[1] = argv
[1];
159 tmpargv
[2] = argv
[3];
160 buf
= catAppendOnlyGenericCommand(buf
,3,tmpargv
);
161 decrRefCount(tmpargv
[0]);
162 buf
= catAppendOnlyExpireAtCommand(buf
,argv
[1],argv
[2]);
164 buf
= catAppendOnlyGenericCommand(buf
,argc
,argv
);
167 /* Append to the AOF buffer. This will be flushed on disk just before
168 * of re-entering the event loop, so before the client will get a
169 * positive reply about the operation performed. */
170 server
.aofbuf
= sdscatlen(server
.aofbuf
,buf
,sdslen(buf
));
172 /* If a background append only file rewriting is in progress we want to
173 * accumulate the differences between the child DB and the current one
174 * in a buffer, so that when the child process will do its work we
175 * can append the differences to the new append only file. */
176 if (server
.bgrewritechildpid
!= -1)
177 server
.bgrewritebuf
= sdscatlen(server
.bgrewritebuf
,buf
,sdslen(buf
));
182 /* In Redis commands are always executed in the context of a client, so in
183 * order to load the append only file we need to create a fake client. */
184 struct redisClient
*createFakeClient(void) {
185 struct redisClient
*c
= zmalloc(sizeof(*c
));
189 c
->querybuf
= sdsempty();
194 /* We set the fake client as a slave waiting for the synchronization
195 * so that Redis will not try to send replies to this client. */
196 c
->replstate
= REDIS_REPL_WAIT_BGSAVE_START
;
197 c
->reply
= listCreate();
198 c
->watched_keys
= listCreate();
199 listSetFreeMethod(c
->reply
,decrRefCount
);
200 listSetDupMethod(c
->reply
,dupClientReplyValue
);
201 initClientMultiState(c
);
205 void freeFakeClient(struct redisClient
*c
) {
206 sdsfree(c
->querybuf
);
207 listRelease(c
->reply
);
208 listRelease(c
->watched_keys
);
209 freeClientMultiState(c
);
213 /* Replay the append log file. On error REDIS_OK is returned. On non fatal
214 * error (the append only file is zero-length) REDIS_ERR is returned. On
215 * fatal error an error message is logged and the program exists. */
216 int loadAppendOnlyFile(char *filename
) {
217 struct redisClient
*fakeClient
;
218 FILE *fp
= fopen(filename
,"r");
219 struct redis_stat sb
;
220 int appendonly
= server
.appendonly
;
223 if (fp
&& redis_fstat(fileno(fp
),&sb
) != -1 && sb
.st_size
== 0) {
229 redisLog(REDIS_WARNING
,"Fatal error: can't open the append log file for reading: %s",strerror(errno
));
233 /* Temporarily disable AOF, to prevent EXEC from feeding a MULTI
234 * to the same file we're about to read. */
235 server
.appendonly
= 0;
237 fakeClient
= createFakeClient();
246 struct redisCommand
*cmd
;
248 /* Serve the clients from time to time */
249 if (!(loops
++ % 1000)) {
250 loadingProgress(ftello(fp
));
251 aeProcessEvents(server
.el
, AE_FILE_EVENTS
|AE_DONT_WAIT
);
254 if (fgets(buf
,sizeof(buf
),fp
) == NULL
) {
260 if (buf
[0] != '*') goto fmterr
;
262 argv
= zmalloc(sizeof(robj
*)*argc
);
263 for (j
= 0; j
< argc
; j
++) {
264 if (fgets(buf
,sizeof(buf
),fp
) == NULL
) goto readerr
;
265 if (buf
[0] != '$') goto fmterr
;
266 len
= strtol(buf
+1,NULL
,10);
267 argsds
= sdsnewlen(NULL
,len
);
268 if (len
&& fread(argsds
,len
,1,fp
) == 0) goto fmterr
;
269 argv
[j
] = createObject(REDIS_STRING
,argsds
);
270 if (fread(buf
,2,1,fp
) == 0) goto fmterr
; /* discard CRLF */
274 cmd
= lookupCommand(argv
[0]->ptr
);
276 redisLog(REDIS_WARNING
,"Unknown command '%s' reading the append only file", argv
[0]->ptr
);
279 /* Run the command in the context of a fake client */
280 fakeClient
->argc
= argc
;
281 fakeClient
->argv
= argv
;
282 cmd
->proc(fakeClient
);
284 /* The fake client should not have a reply */
285 redisAssert(fakeClient
->bufpos
== 0 && listLength(fakeClient
->reply
) == 0);
287 /* Clean up, ready for the next command */
288 for (j
= 0; j
< argc
; j
++) decrRefCount(argv
[j
]);
292 /* This point can only be reached when EOF is reached without errors.
293 * If the client is in the middle of a MULTI/EXEC, log error and quit. */
294 if (fakeClient
->flags
& REDIS_MULTI
) goto readerr
;
297 freeFakeClient(fakeClient
);
298 server
.appendonly
= appendonly
;
304 redisLog(REDIS_WARNING
,"Unexpected end of file reading the append only file");
306 redisLog(REDIS_WARNING
,"Unrecoverable error reading the append only file: %s", strerror(errno
));
310 redisLog(REDIS_WARNING
,"Bad file format reading the append only file: make a backup of your AOF file, then use ./redis-check-aof --fix <filename>");
314 /* Write a sequence of commands able to fully rebuild the dataset into
315 * "filename". Used both by REWRITEAOF and BGREWRITEAOF. */
316 int rewriteAppendOnlyFile(char *filename
) {
317 dictIterator
*di
= NULL
;
322 time_t now
= time(NULL
);
324 /* Note that we have to use a different temp name here compared to the
325 * one used by rewriteAppendOnlyFileBackground() function. */
326 snprintf(tmpfile
,256,"temp-rewriteaof-%d.aof", (int) getpid());
327 fp
= fopen(tmpfile
,"w");
329 redisLog(REDIS_WARNING
, "Failed rewriting the append only file: %s", strerror(errno
));
332 for (j
= 0; j
< server
.dbnum
; j
++) {
333 char selectcmd
[] = "*2\r\n$6\r\nSELECT\r\n";
334 redisDb
*db
= server
.db
+j
;
336 if (dictSize(d
) == 0) continue;
337 di
= dictGetIterator(d
);
343 /* SELECT the new DB */
344 if (fwrite(selectcmd
,sizeof(selectcmd
)-1,1,fp
) == 0) goto werr
;
345 if (fwriteBulkLongLong(fp
,j
) == 0) goto werr
;
347 /* Iterate this DB writing every entry */
348 while((de
= dictNext(di
)) != NULL
) {
349 sds keystr
= dictGetEntryKey(de
);
353 keystr
= dictGetEntryKey(de
);
354 o
= dictGetEntryVal(de
);
355 initStaticStringObject(key
,keystr
);
357 expiretime
= getExpire(db
,&key
);
359 /* Save the key and associated value */
360 if (o
->type
== REDIS_STRING
) {
361 /* Emit a SET command */
362 char cmd
[]="*3\r\n$3\r\nSET\r\n";
363 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
365 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
366 if (fwriteBulkObject(fp
,o
) == 0) goto werr
;
367 } else if (o
->type
== REDIS_LIST
) {
368 /* Emit the RPUSHes needed to rebuild the list */
369 char cmd
[]="*3\r\n$5\r\nRPUSH\r\n";
370 if (o
->encoding
== REDIS_ENCODING_ZIPLIST
) {
371 unsigned char *zl
= o
->ptr
;
372 unsigned char *p
= ziplistIndex(zl
,0);
377 while(ziplistGet(p
,&vstr
,&vlen
,&vlong
)) {
378 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
379 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
381 if (fwriteBulkString(fp
,(char*)vstr
,vlen
) == 0)
384 if (fwriteBulkLongLong(fp
,vlong
) == 0)
387 p
= ziplistNext(zl
,p
);
389 } else if (o
->encoding
== REDIS_ENCODING_LINKEDLIST
) {
394 listRewind(list
,&li
);
395 while((ln
= listNext(&li
))) {
396 robj
*eleobj
= listNodeValue(ln
);
398 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
399 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
400 if (fwriteBulkObject(fp
,eleobj
) == 0) goto werr
;
403 redisPanic("Unknown list encoding");
405 } else if (o
->type
== REDIS_SET
) {
406 char cmd
[]="*3\r\n$4\r\nSADD\r\n";
408 /* Emit the SADDs needed to rebuild the set */
409 if (o
->encoding
== REDIS_ENCODING_INTSET
) {
412 while(intsetGet(o
->ptr
,ii
++,&llval
)) {
413 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
414 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
415 if (fwriteBulkLongLong(fp
,llval
) == 0) goto werr
;
417 } else if (o
->encoding
== REDIS_ENCODING_HT
) {
418 dictIterator
*di
= dictGetIterator(o
->ptr
);
420 while((de
= dictNext(di
)) != NULL
) {
421 robj
*eleobj
= dictGetEntryKey(de
);
422 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
423 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
424 if (fwriteBulkObject(fp
,eleobj
) == 0) goto werr
;
426 dictReleaseIterator(di
);
428 redisPanic("Unknown set encoding");
430 } else if (o
->type
== REDIS_ZSET
) {
431 /* Emit the ZADDs needed to rebuild the sorted set */
433 dictIterator
*di
= dictGetIterator(zs
->dict
);
436 while((de
= dictNext(di
)) != NULL
) {
437 char cmd
[]="*4\r\n$4\r\nZADD\r\n";
438 robj
*eleobj
= dictGetEntryKey(de
);
439 double *score
= dictGetEntryVal(de
);
441 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
442 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
443 if (fwriteBulkDouble(fp
,*score
) == 0) goto werr
;
444 if (fwriteBulkObject(fp
,eleobj
) == 0) goto werr
;
446 dictReleaseIterator(di
);
447 } else if (o
->type
== REDIS_HASH
) {
448 char cmd
[]="*4\r\n$4\r\nHSET\r\n";
450 /* Emit the HSETs needed to rebuild the hash */
451 if (o
->encoding
== REDIS_ENCODING_ZIPMAP
) {
452 unsigned char *p
= zipmapRewind(o
->ptr
);
453 unsigned char *field
, *val
;
454 unsigned int flen
, vlen
;
456 while((p
= zipmapNext(p
,&field
,&flen
,&val
,&vlen
)) != NULL
) {
457 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
458 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
459 if (fwriteBulkString(fp
,(char*)field
,flen
) == 0)
461 if (fwriteBulkString(fp
,(char*)val
,vlen
) == 0)
465 dictIterator
*di
= dictGetIterator(o
->ptr
);
468 while((de
= dictNext(di
)) != NULL
) {
469 robj
*field
= dictGetEntryKey(de
);
470 robj
*val
= dictGetEntryVal(de
);
472 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
473 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
474 if (fwriteBulkObject(fp
,field
) == 0) goto werr
;
475 if (fwriteBulkObject(fp
,val
) == 0) goto werr
;
477 dictReleaseIterator(di
);
480 redisPanic("Unknown object type");
482 /* Save the expire time */
483 if (expiretime
!= -1) {
484 char cmd
[]="*3\r\n$8\r\nEXPIREAT\r\n";
485 /* If this key is already expired skip it */
486 if (expiretime
< now
) continue;
487 if (fwrite(cmd
,sizeof(cmd
)-1,1,fp
) == 0) goto werr
;
488 if (fwriteBulkObject(fp
,&key
) == 0) goto werr
;
489 if (fwriteBulkLongLong(fp
,expiretime
) == 0) goto werr
;
492 dictReleaseIterator(di
);
495 /* Make sure data will not remain on the OS's output buffers */
497 aof_fsync(fileno(fp
));
500 /* Use RENAME to make sure the DB file is changed atomically only
501 * if the generate DB file is ok. */
502 if (rename(tmpfile
,filename
) == -1) {
503 redisLog(REDIS_WARNING
,"Error moving temp append only file on the final destination: %s", strerror(errno
));
507 redisLog(REDIS_NOTICE
,"SYNC append only file rewrite performed");
513 redisLog(REDIS_WARNING
,"Write error writing append only file on disk: %s", strerror(errno
));
514 if (di
) dictReleaseIterator(di
);
518 /* This is how rewriting of the append only file in background works:
520 * 1) The user calls BGREWRITEAOF
521 * 2) Redis calls this function, that forks():
522 * 2a) the child rewrite the append only file in a temp file.
523 * 2b) the parent accumulates differences in server.bgrewritebuf.
524 * 3) When the child finished '2a' exists.
525 * 4) The parent will trap the exit code, if it's OK, will append the
526 * data accumulated into server.bgrewritebuf into the temp file, and
527 * finally will rename(2) the temp file in the actual file name.
528 * The the new file is reopened as the new append only file. Profit!
530 int rewriteAppendOnlyFileBackground(void) {
533 if (server
.bgrewritechildpid
!= -1) return REDIS_ERR
;
534 if (server
.ds_enabled
!= 0) {
535 redisLog(REDIS_WARNING
,"BGREWRITEAOF called with diskstore enabled: AOF is not supported when diskstore is enabled. Operation not performed.");
538 if ((childpid
= fork()) == 0) {
542 if (server
.ipfd
> 0) close(server
.ipfd
);
543 if (server
.sofd
> 0) close(server
.sofd
);
544 snprintf(tmpfile
,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
545 if (rewriteAppendOnlyFile(tmpfile
) == REDIS_OK
) {
552 if (childpid
== -1) {
553 redisLog(REDIS_WARNING
,
554 "Can't rewrite append only file in background: fork: %s",
558 redisLog(REDIS_NOTICE
,
559 "Background append only file rewriting started by pid %d",childpid
);
560 server
.bgrewritechildpid
= childpid
;
561 updateDictResizePolicy();
562 /* We set appendseldb to -1 in order to force the next call to the
563 * feedAppendOnlyFile() to issue a SELECT command, so the differences
564 * accumulated by the parent into server.bgrewritebuf will start
565 * with a SELECT statement and it will be safe to merge. */
566 server
.appendseldb
= -1;
569 return REDIS_OK
; /* unreached */
572 void bgrewriteaofCommand(redisClient
*c
) {
573 if (server
.bgrewritechildpid
!= -1) {
574 addReplyError(c
,"Background append only file rewriting already in progress");
577 if (rewriteAppendOnlyFileBackground() == REDIS_OK
) {
578 addReplyStatus(c
,"Background append only file rewriting started");
580 addReply(c
,shared
.err
);
584 void aofRemoveTempFile(pid_t childpid
) {
587 snprintf(tmpfile
,256,"temp-rewriteaof-bg-%d.aof", (int) childpid
);
591 /* A background append only file rewriting (BGREWRITEAOF) terminated its work.
593 void backgroundRewriteDoneHandler(int exitcode
, int bysignal
) {
594 if (!bysignal
&& exitcode
== 0) {
598 redisLog(REDIS_NOTICE
,
599 "Background append only file rewriting terminated with success");
600 /* Now it's time to flush the differences accumulated by the parent */
601 snprintf(tmpfile
,256,"temp-rewriteaof-bg-%d.aof", (int) server
.bgrewritechildpid
);
602 fd
= open(tmpfile
,O_WRONLY
|O_APPEND
);
604 redisLog(REDIS_WARNING
, "Not able to open the temp append only file produced by the child: %s", strerror(errno
));
607 /* Flush our data... */
608 if (write(fd
,server
.bgrewritebuf
,sdslen(server
.bgrewritebuf
)) !=
609 (signed) sdslen(server
.bgrewritebuf
)) {
610 redisLog(REDIS_WARNING
, "Error or short write trying to flush the parent diff of the append log file in the child temp file: %s", strerror(errno
));
614 redisLog(REDIS_NOTICE
,"Parent diff flushed into the new append log file with success (%lu bytes)",sdslen(server
.bgrewritebuf
));
615 /* Now our work is to rename the temp file into the stable file. And
616 * switch the file descriptor used by the server for append only. */
617 if (rename(tmpfile
,server
.appendfilename
) == -1) {
618 redisLog(REDIS_WARNING
,"Can't rename the temp append only file into the stable one: %s", strerror(errno
));
622 /* Mission completed... almost */
623 redisLog(REDIS_NOTICE
,"Append only file successfully rewritten.");
624 if (server
.appendfd
!= -1) {
625 /* If append only is actually enabled... */
626 close(server
.appendfd
);
627 server
.appendfd
= fd
;
628 if (server
.appendfsync
!= APPENDFSYNC_NO
) aof_fsync(fd
);
629 server
.appendseldb
= -1; /* Make sure it will issue SELECT */
630 redisLog(REDIS_NOTICE
,"The new append only file was selected for future appends.");
632 /* If append only is disabled we just generate a dump in this
633 * format. Why not? */
636 } else if (!bysignal
&& exitcode
!= 0) {
637 redisLog(REDIS_WARNING
, "Background append only file rewriting error");
639 redisLog(REDIS_WARNING
,
640 "Background append only file rewriting terminated by signal %d",
644 sdsfree(server
.bgrewritebuf
);
645 server
.bgrewritebuf
= sdsempty();
646 aofRemoveTempFile(server
.bgrewritechildpid
);
647 server
.bgrewritechildpid
= -1;