Re-use AOF buffer when it is small enough

[redis.git] / src / redis.c

diff --git a/src/redis.c b/src/redis.c
index 89efffbabd844aec36ee4d06b220918a2c0acf5d..77f4666c2d4aba2c55cd9da77881adabc882178c 100644 (file)
--- a/src/redis.c
+++ b/src/redis.c
@@ -28,6 +28,7 @@
  */
 
 #include "redis.h"
  */
 
 #include "redis.h"

+#include "slowlog.h"

 
 #ifdef HAVE_BACKTRACE
 #include <execinfo.h>
 
 #ifdef HAVE_BACKTRACE
 #include <execinfo.h>


@@ -192,7 +193,10 @@ struct redisCommand redisCommandTable[] = {
     {"migrate",migrateCommand,6,0,NULL,0,0,0,0,0},
     {"dump",dumpCommand,2,0,NULL,0,0,0,0,0},
     {"object",objectCommand,-2,0,NULL,0,0,0,0,0},
     {"migrate",migrateCommand,6,0,NULL,0,0,0,0,0},
     {"dump",dumpCommand,2,0,NULL,0,0,0,0,0},
     {"object",objectCommand,-2,0,NULL,0,0,0,0,0},

-    {"client",clientCommand,-2,0,NULL,0,0,0,0,0}
+    {"client",clientCommand,-2,0,NULL,0,0,0,0,0},
+    {"eval",evalCommand,-3,REDIS_CMD_DENYOOM,zunionInterGetKeys,0,0,0,0,0},
+    {"evalsha",evalShaCommand,-3,REDIS_CMD_DENYOOM,zunionInterGetKeys,0,0,0,0,0},
+    {"slowlog",slowlogCommand,-2,0,NULL,0,0,0,0,0}

 };
 
 /*============================ Utility functions ============================ */
 };
 
 /*============================ Utility functions ============================ */


@@ -379,7 +383,7 @@ unsigned int dictEncObjHash(const void *key) {
     }
 }
 
     }
 }
 

-/* Sets type and diskstore negative caching hash table */
+/* Sets type hash table */

 dictType setDictType = {
     dictEncObjHash,            /* hash function */
     NULL,                      /* key dup */
 dictType setDictType = {
     dictEncObjHash,            /* hash function */
     NULL,                      /* key dup */


@@ -571,6 +575,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
      * in objects at every object access, and accuracy is not needed.
      * To access a global var is faster than calling time(NULL) */
     server.unixtime = time(NULL);
      * in objects at every object access, and accuracy is not needed.
      * To access a global var is faster than calling time(NULL) */
     server.unixtime = time(NULL);

+

     /* We have just 22 bits per object for LRU information.
      * So we use an (eventually wrapping) LRU clock with 10 seconds resolution.
      * 2^22 bits with 10 seconds resoluton is more or less 1.5 years.
     /* We have just 22 bits per object for LRU information.
      * So we use an (eventually wrapping) LRU clock with 10 seconds resolution.
      * 2^22 bits with 10 seconds resoluton is more or less 1.5 years.


@@ -681,7 +686,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
              server.auto_aofrewrite_perc &&
              server.appendonly_current_size > server.auto_aofrewrite_min_size)
          {
              server.auto_aofrewrite_perc &&
              server.appendonly_current_size > server.auto_aofrewrite_min_size)
          {

-            int base = server.auto_aofrewrite_base_size ?
+            long long base = server.auto_aofrewrite_base_size ?

                             server.auto_aofrewrite_base_size : 1;
             long long growth = (server.appendonly_current_size*100/base) - 100;
             if (growth >= server.auto_aofrewrite_perc) {
                             server.auto_aofrewrite_base_size : 1;
             long long growth = (server.appendonly_current_size*100/base) - 100;
             if (growth >= server.auto_aofrewrite_perc) {


@@ -759,6 +764,8 @@ void createSharedObjects(void) {
         "-ERR source and destination objects are the same\r\n"));
     shared.outofrangeerr = createObject(REDIS_STRING,sdsnew(
         "-ERR index out of range\r\n"));
         "-ERR source and destination objects are the same\r\n"));
     shared.outofrangeerr = createObject(REDIS_STRING,sdsnew(
         "-ERR index out of range\r\n"));

+    shared.noscripterr = createObject(REDIS_STRING,sdsnew(
+        "-NOSCRIPT No matching script. Please use EVAL.\r\n"));

     shared.loadingerr = createObject(REDIS_STRING,sdsnew(
         "-LOADING Redis is loading the dataset in memory\r\n"));
     shared.space = createObject(REDIS_STRING,sdsnew(" "));
     shared.loadingerr = createObject(REDIS_STRING,sdsnew(
         "-LOADING Redis is loading the dataset in memory\r\n"));
     shared.space = createObject(REDIS_STRING,sdsnew(" "));


@@ -835,6 +842,7 @@ void initServerConfig() {
     server.shutdown_asap = 0;
     server.cluster_enabled = 0;
     server.cluster.configfile = zstrdup("nodes.conf");
     server.shutdown_asap = 0;
     server.cluster_enabled = 0;
     server.cluster.configfile = zstrdup("nodes.conf");

+    server.lua_time_limit = REDIS_LUA_TIME_LIMIT;

 
     updateLRUClock();
     resetServerSaveParams();
 
     updateLRUClock();
     resetServerSaveParams();


@@ -866,6 +874,10 @@ void initServerConfig() {
     populateCommandTable();
     server.delCommand = lookupCommandByCString("del");
     server.multiCommand = lookupCommandByCString("multi");
     populateCommandTable();
     server.delCommand = lookupCommandByCString("del");
     server.multiCommand = lookupCommandByCString("multi");

+    
+    /* Slow log */
+    server.slowlog_log_slower_than = REDIS_SLOWLOG_LOG_SLOWER_THAN;
+    server.slowlog_max_len = REDIS_SLOWLOG_MAX_LEN;

 }
 
 void initServer() {
 }
 
 void initServer() {


@@ -952,6 +964,8 @@ void initServer() {
     }
 
     if (server.cluster_enabled) clusterInit();
     }
 
     if (server.cluster_enabled) clusterInit();

+    scriptingInit();
+    slowlogInit();

     srand(time(NULL)^getpid());
 }
 
     srand(time(NULL)^getpid());
 }
 


@@ -998,18 +1012,20 @@ struct redisCommand *lookupCommandByCString(char *s) {
 }
 
 /* Call() is the core of Redis execution of a command */
 }
 
 /* Call() is the core of Redis execution of a command */

-void call(redisClient *c, struct redisCommand *cmd) {
-    long long dirty, start = ustime();
+void call(redisClient *c) {
+    long long dirty, start = ustime(), duration;

 
     dirty = server.dirty;
 
     dirty = server.dirty;

-    cmd->proc(c);
+    c->cmd->proc(c);

     dirty = server.dirty-dirty;
     dirty = server.dirty-dirty;

-    cmd->microseconds += ustime()-start;
-    cmd->calls++;
+    duration = ustime()-start;
+    c->cmd->microseconds += duration;
+    slowlogPushEntryIfNeeded(c->argv,c->argc,duration);
+    c->cmd->calls++;

 
     if (server.appendonly && dirty)
 
     if (server.appendonly && dirty)

-        feedAppendOnlyFile(cmd,c->db->id,c->argv,c->argc);
-    if ((dirty || cmd->flags & REDIS_CMD_FORCE_REPLICATION) &&
+        feedAppendOnlyFile(c->cmd,c->db->id,c->argv,c->argc);
+    if ((dirty || c->cmd->flags & REDIS_CMD_FORCE_REPLICATION) &&

         listLength(server.slaves))
         replicationFeedSlaves(server.slaves,c->db->id,c->argv,c->argc);
     if (listLength(server.monitors))
         listLength(server.slaves))
         replicationFeedSlaves(server.slaves,c->db->id,c->argv,c->argc);
     if (listLength(server.monitors))


@@ -1026,8 +1042,6 @@ void call(redisClient *c, struct redisCommand *cmd) {
  * and other operations can be performed by the caller. Otherwise
  * if 0 is returned the client was destroied (i.e. after QUIT). */
 int processCommand(redisClient *c) {
  * and other operations can be performed by the caller. Otherwise
  * if 0 is returned the client was destroied (i.e. after QUIT). */
 int processCommand(redisClient *c) {

-    struct redisCommand *cmd;
-

     /* The QUIT command is handled separately. Normal command procs will
      * go through checking for replication and QUIT will cause trouble
      * when FORCE_REPLICATION is enabled and would be implemented in
     /* The QUIT command is handled separately. Normal command procs will
      * go through checking for replication and QUIT will cause trouble
      * when FORCE_REPLICATION is enabled and would be implemented in


@@ -1039,28 +1053,29 @@ int processCommand(redisClient *c) {
     }
 
     /* Now lookup the command and check ASAP about trivial error conditions
     }
 
     /* Now lookup the command and check ASAP about trivial error conditions

-     * such wrong arity, bad command name and so forth. */
-    cmd = lookupCommand(c->argv[0]->ptr);
-    if (!cmd) {
+     * such as wrong arity, bad command name and so forth. */
+    c->cmd = lookupCommand(c->argv[0]->ptr);
+    if (!c->cmd) {

         addReplyErrorFormat(c,"unknown command '%s'",
             (char*)c->argv[0]->ptr);
         return REDIS_OK;
         addReplyErrorFormat(c,"unknown command '%s'",
             (char*)c->argv[0]->ptr);
         return REDIS_OK;

-    } else if ((cmd->arity > 0 && cmd->arity != c->argc) ||
-               (c->argc < -cmd->arity)) {
+    } else if ((c->cmd->arity > 0 && c->cmd->arity != c->argc) ||
+               (c->argc < -c->cmd->arity)) {

         addReplyErrorFormat(c,"wrong number of arguments for '%s' command",
         addReplyErrorFormat(c,"wrong number of arguments for '%s' command",

-            cmd->name);
+            c->cmd->name);

         return REDIS_OK;
     }
 
     /* Check if the user is authenticated */
         return REDIS_OK;
     }
 
     /* Check if the user is authenticated */

-    if (server.requirepass && !c->authenticated && cmd->proc != authCommand) {
+    if (server.requirepass && !c->authenticated && c->cmd->proc != authCommand)
+    {

         addReplyError(c,"operation not permitted");
         return REDIS_OK;
     }
 
     /* If cluster is enabled, redirect here */
     if (server.cluster_enabled &&
         addReplyError(c,"operation not permitted");
         return REDIS_OK;
     }
 
     /* If cluster is enabled, redirect here */
     if (server.cluster_enabled &&

-                !(cmd->getkeys_proc == NULL && cmd->firstkey == 0)) {
+                !(c->cmd->getkeys_proc == NULL && c->cmd->firstkey == 0)) {

         int hashslot;
 
         if (server.cluster.state != REDIS_CLUSTER_OK) {
         int hashslot;
 
         if (server.cluster.state != REDIS_CLUSTER_OK) {


@@ -1068,7 +1083,7 @@ int processCommand(redisClient *c) {
             return REDIS_OK;
         } else {
             int ask;
             return REDIS_OK;
         } else {
             int ask;

-            clusterNode *n = getNodeByQuery(c,cmd,c->argv,c->argc,&hashslot,&ask);
+            clusterNode *n = getNodeByQuery(c,c->cmd,c->argv,c->argc,&hashslot,&ask);

             if (n == NULL) {
                 addReplyError(c,"Multi keys request invalid in cluster");
                 return REDIS_OK;
             if (n == NULL) {
                 addReplyError(c,"Multi keys request invalid in cluster");
                 return REDIS_OK;


@@ -1087,7 +1102,7 @@ int processCommand(redisClient *c) {
      * keys in the dataset). If there are not the only thing we can do
      * is returning an error. */
     if (server.maxmemory) freeMemoryIfNeeded();
      * keys in the dataset). If there are not the only thing we can do
      * is returning an error. */
     if (server.maxmemory) freeMemoryIfNeeded();

-    if (server.maxmemory && (cmd->flags & REDIS_CMD_DENYOOM) &&
+    if (server.maxmemory && (c->cmd->flags & REDIS_CMD_DENYOOM) &&

         zmalloc_used_memory() > server.maxmemory)
     {
         addReplyError(c,"command not allowed when used memory > 'maxmemory'");
         zmalloc_used_memory() > server.maxmemory)
     {
         addReplyError(c,"command not allowed when used memory > 'maxmemory'");


@@ -1097,8 +1112,10 @@ int processCommand(redisClient *c) {
     /* Only allow SUBSCRIBE and UNSUBSCRIBE in the context of Pub/Sub */
     if ((dictSize(c->pubsub_channels) > 0 || listLength(c->pubsub_patterns) > 0)
         &&
     /* Only allow SUBSCRIBE and UNSUBSCRIBE in the context of Pub/Sub */
     if ((dictSize(c->pubsub_channels) > 0 || listLength(c->pubsub_patterns) > 0)
         &&

-        cmd->proc != subscribeCommand && cmd->proc != unsubscribeCommand &&
-        cmd->proc != psubscribeCommand && cmd->proc != punsubscribeCommand) {
+        c->cmd->proc != subscribeCommand &&
+        c->cmd->proc != unsubscribeCommand &&
+        c->cmd->proc != psubscribeCommand &&
+        c->cmd->proc != punsubscribeCommand) {

         addReplyError(c,"only (P)SUBSCRIBE / (P)UNSUBSCRIBE / QUIT allowed in this context");
         return REDIS_OK;
     }
         addReplyError(c,"only (P)SUBSCRIBE / (P)UNSUBSCRIBE / QUIT allowed in this context");
         return REDIS_OK;
     }


@@ -1107,7 +1124,7 @@ int processCommand(redisClient *c) {
      * we are a slave with a broken link with master. */
     if (server.masterhost && server.replstate != REDIS_REPL_CONNECTED &&
         server.repl_serve_stale_data == 0 &&
      * we are a slave with a broken link with master. */
     if (server.masterhost && server.replstate != REDIS_REPL_CONNECTED &&
         server.repl_serve_stale_data == 0 &&

-        cmd->proc != infoCommand && cmd->proc != slaveofCommand)
+        c->cmd->proc != infoCommand && c->cmd->proc != slaveofCommand)

     {
         addReplyError(c,
             "link with MASTER is down and slave-serve-stale-data is set to no");
     {
         addReplyError(c,
             "link with MASTER is down and slave-serve-stale-data is set to no");


@@ -1115,20 +1132,20 @@ int processCommand(redisClient *c) {
     }
 
     /* Loading DB? Return an error if the command is not INFO */
     }
 
     /* Loading DB? Return an error if the command is not INFO */

-    if (server.loading && cmd->proc != infoCommand) {
+    if (server.loading && c->cmd->proc != infoCommand) {

         addReply(c, shared.loadingerr);
         return REDIS_OK;
     }
 
     /* Exec the command */
     if (c->flags & REDIS_MULTI &&
         addReply(c, shared.loadingerr);
         return REDIS_OK;
     }
 
     /* Exec the command */
     if (c->flags & REDIS_MULTI &&

-        cmd->proc != execCommand && cmd->proc != discardCommand &&
-        cmd->proc != multiCommand && cmd->proc != watchCommand)
+        c->cmd->proc != execCommand && c->cmd->proc != discardCommand &&
+        c->cmd->proc != multiCommand && c->cmd->proc != watchCommand)

     {
     {

-        queueMultiCommand(c,cmd);
+        queueMultiCommand(c);

         addReply(c,shared.queued);
     } else {
         addReply(c,shared.queued);
     } else {

-        call(c,cmd);
+        call(c);

     }
     return REDIS_OK;
 }
     }
     return REDIS_OK;
 }


@@ -1136,19 +1153,29 @@ int processCommand(redisClient *c) {
 /*================================== Shutdown =============================== */
 
 int prepareForShutdown() {
 /*================================== Shutdown =============================== */
 
 int prepareForShutdown() {

-    redisLog(REDIS_WARNING,"User requested shutdown, saving DB...");
+    redisLog(REDIS_WARNING,"User requested shutdown...");

     /* Kill the saving child if there is a background saving in progress.
        We want to avoid race conditions, for instance our saving child may
        overwrite the synchronous saving did by SHUTDOWN. */
     if (server.bgsavechildpid != -1) {
     /* Kill the saving child if there is a background saving in progress.
        We want to avoid race conditions, for instance our saving child may
        overwrite the synchronous saving did by SHUTDOWN. */
     if (server.bgsavechildpid != -1) {

-        redisLog(REDIS_WARNING,"There is a live saving child. Killing it!");
+        redisLog(REDIS_WARNING,"There is a child saving an .rdb. Killing it!");

         kill(server.bgsavechildpid,SIGKILL);
         rdbRemoveTempFile(server.bgsavechildpid);
     }
     if (server.appendonly) {
         kill(server.bgsavechildpid,SIGKILL);
         rdbRemoveTempFile(server.bgsavechildpid);
     }
     if (server.appendonly) {

+        /* Kill the AOF saving child as the AOF we already have may be longer
+         * but contains the full dataset anyway. */
+        if (server.bgrewritechildpid != -1) {
+            redisLog(REDIS_WARNING,
+                "There is a child rewriting the AOF. Killing it!");
+            kill(server.bgrewritechildpid,SIGKILL);
+        }

         /* Append only file: fsync() the AOF and exit */
         /* Append only file: fsync() the AOF and exit */

+        redisLog(REDIS_NOTICE,"Calling fsync() on the AOF file.");

         aof_fsync(server.appendfd);
         aof_fsync(server.appendfd);

-    } else if (server.saveparamslen > 0) {
+    }
+    if (server.saveparamslen > 0) {
+        redisLog(REDIS_NOTICE,"Saving the final RDB snapshot before exiting.");

         /* Snapshotting. Perform a SYNC SAVE and exit */
         if (rdbSave(server.dbfilename) != REDIS_OK) {
             /* Ooops.. error saving! The best we can do is to continue
         /* Snapshotting. Perform a SYNC SAVE and exit */
         if (rdbSave(server.dbfilename) != REDIS_OK) {
             /* Ooops.. error saving! The best we can do is to continue


@@ -1156,14 +1183,19 @@ int prepareForShutdown() {
              * in the next cron() Redis will be notified that the background
              * saving aborted, handling special stuff like slaves pending for
              * synchronization... */
              * in the next cron() Redis will be notified that the background
              * saving aborted, handling special stuff like slaves pending for
              * synchronization... */

-            redisLog(REDIS_WARNING,"Error trying to save the DB, can't exit");
+            redisLog(REDIS_WARNING,"Error trying to save the DB, can't exit.");

             return REDIS_ERR;
         }
             return REDIS_ERR;
         }

-    } else {
-        redisLog(REDIS_WARNING,"Not saving DB.");

     }
     }

-    if (server.daemonize) unlink(server.pidfile);
-    redisLog(REDIS_WARNING,"Server exit now, bye bye...");
+    if (server.daemonize) {
+        redisLog(REDIS_NOTICE,"Removing the pid file.");
+        unlink(server.pidfile);
+    }
+    /* Close the listening sockets. Apparently this allows faster restarts. */
+    if (server.ipfd != -1) close(server.ipfd);
+    if (server.sofd != -1) close(server.sofd);
+
+    redisLog(REDIS_WARNING,"Redis is now ready to exit, bye bye...");

     return REDIS_OK;
 }
 
     return REDIS_OK;
 }
 


@@ -1285,6 +1317,7 @@ sds genRedisInfoString(char *section) {
             "used_memory_rss:%zu\r\n"
             "used_memory_peak:%zu\r\n"
             "used_memory_peak_human:%s\r\n"
             "used_memory_rss:%zu\r\n"
             "used_memory_peak:%zu\r\n"
             "used_memory_peak_human:%s\r\n"

+            "used_memory_lua:%lld\r\n"

             "mem_fragmentation_ratio:%.2f\r\n"
             "mem_allocator:%s\r\n",
             zmalloc_used_memory(),
             "mem_fragmentation_ratio:%.2f\r\n"
             "mem_allocator:%s\r\n",
             zmalloc_used_memory(),


@@ -1292,27 +1325,12 @@ sds genRedisInfoString(char *section) {
             zmalloc_get_rss(),
             server.stat_peak_memory,
             peak_hmem,
             zmalloc_get_rss(),
             server.stat_peak_memory,
             peak_hmem,

+            ((long long)lua_gc(server.lua,LUA_GCCOUNT,0))*1024LL,

             zmalloc_get_fragmentation_ratio(),
             ZMALLOC_LIB
             );
     }
 
             zmalloc_get_fragmentation_ratio(),
             ZMALLOC_LIB
             );
     }
 

-    /* Allocation statistics */
-    if (allsections || !strcasecmp(section,"allocstats")) {
-        if (sections++) info = sdscat(info,"\r\n");
-        info = sdscat(info, "# Allocstats\r\nallocation_stats:");
-        for (j = 0; j <= ZMALLOC_MAX_ALLOC_STAT; j++) {
-            size_t count = zmalloc_allocations_for_size(j);
-            if (count) {
-                if (info[sdslen(info)-1] != ':') info = sdscatlen(info,",",1);
-                info = sdscatprintf(info,"%s%d=%zu",
-                    (j == ZMALLOC_MAX_ALLOC_STAT) ? ">=" : "",
-                    j,count);
-            }
-        }
-        info = sdscat(info,"\r\n");
-    }
-

     /* Persistence */
     if (allsections || defsections || !strcasecmp(section,"persistence")) {
         if (sections++) info = sdscat(info,"\r\n");
     /* Persistence */
     if (allsections || defsections || !strcasecmp(section,"persistence")) {
         if (sections++) info = sdscat(info,"\r\n");


@@ -1448,8 +1466,8 @@ sds genRedisInfoString(char *section) {
         "# CPU\r\n"
         "used_cpu_sys:%.2f\r\n"
         "used_cpu_user:%.2f\r\n"
         "# CPU\r\n"
         "used_cpu_sys:%.2f\r\n"
         "used_cpu_user:%.2f\r\n"

-        "used_cpu_sys_childrens:%.2f\r\n"
-        "used_cpu_user_childrens:%.2f\r\n",
+        "used_cpu_sys_children:%.2f\r\n"
+        "used_cpu_user_children:%.2f\r\n",

         (float)self_ru.ru_utime.tv_sec+(float)self_ru.ru_utime.tv_usec/1000000,
         (float)self_ru.ru_stime.tv_sec+(float)self_ru.ru_stime.tv_usec/1000000,
         (float)c_ru.ru_utime.tv_sec+(float)c_ru.ru_utime.tv_usec/1000000,
         (float)self_ru.ru_utime.tv_sec+(float)self_ru.ru_utime.tv_usec/1000000,
         (float)self_ru.ru_stime.tv_sec+(float)self_ru.ru_stime.tv_usec/1000000,
         (float)c_ru.ru_utime.tv_sec+(float)c_ru.ru_utime.tv_usec/1000000,