X-Git-Url: https://git.saurik.com/redis.git/blobdiff_plain/39ca1713d7400cf33e2cbc502478055a78cc03c1..32f62ed6d0f1cad1cc22a55d934ff4487ebd7f82:/src/redis.c?ds=sidebyside diff --git a/src/redis.c b/src/redis.c index 344c60c4..4e004296 100644 --- a/src/redis.c +++ b/src/redis.c @@ -31,11 +31,6 @@ #include "slowlog.h" #include "bio.h" -#ifdef HAVE_BACKTRACE -#include -#include -#endif /* HAVE_BACKTRACE */ - #include #include #include @@ -107,7 +102,10 @@ struct redisCommand *commandTable; * s: command not allowed in scripts. * R: random command. Command is not deterministic, that is, the same command * with the same arguments, with the same key space, may have different - * results. For instance SPOP and RANDOMKEY are two random commands. */ + * results. For instance SPOP and RANDOMKEY are two random commands. + * S: Sort command output array if called from script, so that the output + * is deterministic. + */ struct redisCommand redisCommandTable[] = { {"get",getCommand,2,"r",0,NULL,1,1,1,0,0}, {"set",setCommand,3,"wm",0,noPreloadGetKeys,1,1,1,0,0}, @@ -150,13 +148,13 @@ struct redisCommand redisCommandTable[] = { {"scard",scardCommand,2,"r",0,NULL,1,1,1,0,0}, {"spop",spopCommand,2,"wRs",0,NULL,1,1,1,0,0}, {"srandmember",srandmemberCommand,2,"rR",0,NULL,1,1,1,0,0}, - {"sinter",sinterCommand,-2,"r",0,NULL,1,-1,1,0,0}, + {"sinter",sinterCommand,-2,"rS",0,NULL,1,-1,1,0,0}, {"sinterstore",sinterstoreCommand,-3,"wm",0,NULL,1,-1,1,0,0}, - {"sunion",sunionCommand,-2,"r",0,NULL,1,-1,1,0,0}, + {"sunion",sunionCommand,-2,"rS",0,NULL,1,-1,1,0,0}, {"sunionstore",sunionstoreCommand,-3,"wm",0,NULL,1,-1,1,0,0}, - {"sdiff",sdiffCommand,-2,"r",0,NULL,1,-1,1,0,0}, + {"sdiff",sdiffCommand,-2,"rS",0,NULL,1,-1,1,0,0}, {"sdiffstore",sdiffstoreCommand,-3,"wm",0,NULL,1,-1,1,0,0}, - {"smembers",sinterCommand,2,"r",0,NULL,1,1,1,0,0}, + {"smembers",sinterCommand,2,"rS",0,NULL,1,1,1,0,0}, {"zadd",zaddCommand,-4,"wm",0,NULL,1,1,1,0,0}, {"zincrby",zincrbyCommand,4,"wm",0,NULL,1,1,1,0,0}, {"zrem",zremCommand,-3,"w",0,NULL,1,1,1,0,0}, @@ -182,8 +180,8 @@ struct redisCommand redisCommandTable[] = { {"hincrbyfloat",hincrbyfloatCommand,4,"wm",0,NULL,1,1,1,0,0}, {"hdel",hdelCommand,-3,"w",0,NULL,1,1,1,0,0}, {"hlen",hlenCommand,2,"r",0,NULL,1,1,1,0,0}, - {"hkeys",hkeysCommand,2,"r",0,NULL,1,1,1,0,0}, - {"hvals",hvalsCommand,2,"r",0,NULL,1,1,1,0,0}, + {"hkeys",hkeysCommand,2,"rS",0,NULL,1,1,1,0,0}, + {"hvals",hvalsCommand,2,"rS",0,NULL,1,1,1,0,0}, {"hgetall",hgetallCommand,2,"r",0,NULL,1,1,1,0,0}, {"hexists",hexistsCommand,3,"r",0,NULL,1,1,1,0,0}, {"incrby",incrbyCommand,3,"wm",0,NULL,1,1,1,0,0}, @@ -201,7 +199,7 @@ struct redisCommand redisCommandTable[] = { {"expireat",expireatCommand,3,"w",0,NULL,1,1,1,0,0}, {"pexpire",pexpireCommand,3,"w",0,NULL,1,1,1,0,0}, {"pexpireat",pexpireatCommand,3,"w",0,NULL,1,1,1,0,0}, - {"keys",keysCommand,2,"r",0,NULL,0,0,0,0,0}, + {"keys",keysCommand,2,"rS",0,NULL,0,0,0,0,0}, {"dbsize",dbsizeCommand,1,"r",0,NULL,0,0,0,0,0}, {"auth",authCommand,2,"rs",0,NULL,0,0,0,0,0}, {"ping",pingCommand,1,"r",0,NULL,0,0,0,0,0}, @@ -218,7 +216,7 @@ struct redisCommand redisCommandTable[] = { {"sync",syncCommand,1,"ars",0,NULL,0,0,0,0,0}, {"flushdb",flushdbCommand,1,"w",0,NULL,0,0,0,0,0}, {"flushall",flushallCommand,1,"w",0,NULL,0,0,0,0,0}, - {"sort",sortCommand,-2,"wm",0,NULL,1,1,1,0,0}, + {"sort",sortCommand,-2,"wmS",0,NULL,1,1,1,0,0}, {"info",infoCommand,-1,"r",0,NULL,0,0,0,0,0}, {"monitor",monitorCommand,1,"ars",0,NULL,0,0,0,0,0}, {"ttl",ttlCommand,2,"r",0,NULL,1,1,1,0,0}, @@ -234,17 +232,16 @@ struct redisCommand redisCommandTable[] = { {"publish",publishCommand,3,"rpf",0,NULL,0,0,0,0,0}, {"watch",watchCommand,-2,"rs",0,noPreloadGetKeys,1,-1,1,0,0}, {"unwatch",unwatchCommand,1,"rs",0,NULL,0,0,0,0,0}, - {"cluster",clusterCommand,-2,"ar",0,NULL,0,0,0,0,0}, {"restore",restoreCommand,4,"awm",0,NULL,1,1,1,0,0}, {"migrate",migrateCommand,6,"aw",0,NULL,0,0,0,0,0}, - {"asking",askingCommand,1,"r",0,NULL,0,0,0,0,0}, {"dump",dumpCommand,2,"ar",0,NULL,1,1,1,0,0}, {"object",objectCommand,-2,"r",0,NULL,2,2,2,0,0}, {"client",clientCommand,-2,"ar",0,NULL,0,0,0,0,0}, {"eval",evalCommand,-3,"wms",0,zunionInterGetKeys,0,0,0,0,0}, {"evalsha",evalShaCommand,-3,"wms",0,zunionInterGetKeys,0,0,0,0,0}, {"slowlog",slowlogCommand,-2,"r",0,NULL,0,0,0,0,0}, - {"script",scriptCommand,-2,"ras",0,NULL,0,0,0,0,0} + {"script",scriptCommand,-2,"ras",0,NULL,0,0,0,0,0}, + {"time",timeCommand,1,"rR",0,NULL,0,0,0,0,0} }; /*============================ Utility functions ============================ */ @@ -508,17 +505,6 @@ dictType keylistDictType = { dictListDestructor /* val destructor */ }; -/* Cluster nodes hash table, mapping nodes addresses 1.2.3.4:6379 to - * clusterNode structures. */ -dictType clusterNodesDictType = { - dictSdsHash, /* hash function */ - NULL, /* key dup */ - NULL, /* val dup */ - dictSdsKeyCompare, /* key compare */ - dictSdsDestructor, /* key destructor */ - NULL /* val destructor */ -}; - int htNeedsResize(dict *dict) { long long size, used; @@ -563,7 +549,7 @@ void incrementallyRehash(void) { * for dict.c to resize the hash tables accordingly to the fact we have o not * running childs. */ void updateDictResizePolicy(void) { - if (server.bgsavechildpid == -1 && server.bgrewritechildpid == -1) + if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) dictEnableResize(); else dictDisableResize(); @@ -617,6 +603,110 @@ void updateLRUClock(void) { REDIS_LRU_CLOCK_MAX; } + +/* Add a sample to the operations per second array of samples. */ +void trackOperationsPerSecond(void) { + long long t = mstime() - server.ops_sec_last_sample_time; + long long ops = server.stat_numcommands - server.ops_sec_last_sample_ops; + long long ops_sec; + + ops_sec = t > 0 ? (ops*1000/t) : 0; + + server.ops_sec_samples[server.ops_sec_idx] = ops_sec; + server.ops_sec_idx = (server.ops_sec_idx+1) % REDIS_OPS_SEC_SAMPLES; + server.ops_sec_last_sample_time = mstime(); + server.ops_sec_last_sample_ops = server.stat_numcommands; +} + +/* Return the mean of all the samples. */ +long long getOperationsPerSecond(void) { + int j; + long long sum = 0; + + for (j = 0; j < REDIS_OPS_SEC_SAMPLES; j++) + sum += server.ops_sec_samples[j]; + return sum / REDIS_OPS_SEC_SAMPLES; +} + +/* Check for timeouts. Returns non-zero if the client was terminated */ +int clientsCronHandleTimeout(redisClient *c) { + time_t now = server.unixtime; + + if (server.maxidletime && + !(c->flags & REDIS_SLAVE) && /* no timeout for slaves */ + !(c->flags & REDIS_MASTER) && /* no timeout for masters */ + !(c->flags & REDIS_BLOCKED) && /* no timeout for BLPOP */ + dictSize(c->pubsub_channels) == 0 && /* no timeout for pubsub */ + listLength(c->pubsub_patterns) == 0 && + (now - c->lastinteraction > server.maxidletime)) + { + redisLog(REDIS_VERBOSE,"Closing idle client"); + freeClient(c); + return 1; + } else if (c->flags & REDIS_BLOCKED) { + if (c->bpop.timeout != 0 && c->bpop.timeout < now) { + addReply(c,shared.nullmultibulk); + unblockClientWaitingData(c); + } + } + return 0; +} + +/* The client query buffer is an sds.c string that can end with a lot of + * free space not used, this function reclaims space if needed. + * + * The funciton always returns 0 as it never terminates the client. */ +int clientsCronResizeQueryBuffer(redisClient *c) { + size_t querybuf_size = sdsAllocSize(c->querybuf); + time_t idletime = server.unixtime - c->lastinteraction; + + /* There are two conditions to resize the query buffer: + * 1) Query buffer is > BIG_ARG and too big for latest peak. + * 2) Client is inactive and the buffer is bigger than 1k. */ + if (((querybuf_size > REDIS_MBULK_BIG_ARG) && + (querybuf_size/(c->querybuf_peak+1)) > 2) || + (querybuf_size > 1024 && idletime > 2)) + { + /* Only resize the query buffer if it is actually wasting space. */ + if (sdsavail(c->querybuf) > 1024) { + c->querybuf = sdsRemoveFreeSpace(c->querybuf); + } + } + /* Reset the peak again to capture the peak memory usage in the next + * cycle. */ + c->querybuf_peak = 0; + return 0; +} + +void clientsCron(void) { + /* Make sure to process at least 1/100 of clients per call. + * Since this function is called 10 times per second we are sure that + * in the worst case we process all the clients in 10 seconds. + * In normal conditions (a reasonable number of clients) we process + * all the clients in a shorter time. */ + int numclients = listLength(server.clients); + int iterations = numclients/100; + + if (iterations < 50) + iterations = (numclients < 50) ? numclients : 50; + while(listLength(server.clients) && iterations--) { + redisClient *c; + listNode *head; + + /* Rotate the list, take the current head, process. + * This way if the client must be removed from the list it's the + * first element and we don't incur into O(N) computation. */ + listRotate(server.clients); + head = listFirst(server.clients); + c = listNodeValue(head); + /* The following functions do different service checks on the client. + * The protocol is that they return non-zero if the client was + * terminated. */ + if (clientsCronHandleTimeout(c)) continue; + if (clientsCronResizeQueryBuffer(c)) continue; + } +} + int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { int j, loops = server.cronloops; REDIS_NOTUSED(eventLoop); @@ -629,6 +719,8 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { * To access a global var is faster than calling time(NULL) */ server.unixtime = time(NULL); + trackOperationsPerSecond(); + /* We have just 22 bits per object for LRU information. * So we use an (eventually wrapping) LRU clock with 10 seconds resolution. * 2^22 bits with 10 seconds resoluton is more or less 1.5 years. @@ -673,7 +765,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { * if we resize the HT while there is the saving child at work actually * a lot of memory movements in the parent will cause a lot of pages * copied. */ - if (server.bgsavechildpid == -1 && server.bgrewritechildpid == -1) { + if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) { if (!(loops % 10)) tryResizeHashTables(); if (server.activerehashing) incrementallyRehash(); } @@ -686,20 +778,19 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { zmalloc_used_memory()); } - /* Close connections of timedout clients */ - if ((server.maxidletime && !(loops % 100)) || server.bpop_blocked_clients) - closeTimedoutClients(); + /* We need to do a few operations on clients asynchronously. */ + clientsCron(); /* Start a scheduled AOF rewrite if this was requested by the user while * a BGSAVE was in progress. */ - if (server.bgsavechildpid == -1 && server.bgrewritechildpid == -1 && - server.aofrewrite_scheduled) + if (server.rdb_child_pid == -1 && server.aof_child_pid == -1 && + server.aof_rewrite_scheduled) { rewriteAppendOnlyFileBackground(); } /* Check if a background saving or AOF rewrite in progress terminated. */ - if (server.bgsavechildpid != -1 || server.bgrewritechildpid != -1) { + if (server.rdb_child_pid != -1 || server.aof_child_pid != -1) { int statloc; pid_t pid; @@ -709,7 +800,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { if (WIFSIGNALED(statloc)) bysignal = WTERMSIG(statloc); - if (pid == server.bgsavechildpid) { + if (pid == server.rdb_child_pid) { backgroundSaveDoneHandler(exitcode,bysignal); } else { backgroundRewriteDoneHandler(exitcode,bysignal); @@ -728,21 +819,21 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { now-server.lastsave > sp->seconds) { redisLog(REDIS_NOTICE,"%d changes in %d seconds. Saving...", sp->changes, sp->seconds); - rdbSaveBackground(server.dbfilename); + rdbSaveBackground(server.rdb_filename); break; } } /* Trigger an AOF rewrite if needed */ - if (server.bgsavechildpid == -1 && - server.bgrewritechildpid == -1 && - server.auto_aofrewrite_perc && - server.appendonly_current_size > server.auto_aofrewrite_min_size) + if (server.rdb_child_pid == -1 && + server.aof_child_pid == -1 && + server.aof_rewrite_perc && + server.aof_current_size > server.aof_rewrite_min_size) { - long long base = server.auto_aofrewrite_base_size ? - server.auto_aofrewrite_base_size : 1; - long long growth = (server.appendonly_current_size*100/base) - 100; - if (growth >= server.auto_aofrewrite_perc) { + long long base = server.aof_rewrite_base_size ? + server.aof_rewrite_base_size : 1; + long long growth = (server.aof_current_size*100/base) - 100; + if (growth >= server.aof_rewrite_perc) { redisLog(REDIS_NOTICE,"Starting automatic rewriting of AOF on %lld%% growth",growth); rewriteAppendOnlyFileBackground(); } @@ -759,13 +850,13 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { * in order to guarantee a strict consistency. */ if (server.masterhost == NULL) activeExpireCycle(); + /* Close clients that need to be closed asynchronous */ + freeClientsInAsyncFreeQueue(); + /* Replication cron function -- used to reconnect to master and * to detect transfer failures. */ if (!(loops % 10)) replicationCron(); - /* Run other sub-systems specific cron jobs */ - if (server.cluster_enabled && !(loops % 10)) clusterCron(); - server.cronloops++; return 100; } @@ -787,8 +878,11 @@ void beforeSleep(struct aeEventLoop *eventLoop) { c->flags &= ~REDIS_UNBLOCKED; /* Process remaining data in the input buffer. */ - if (c->querybuf && sdslen(c->querybuf) > 0) + if (c->querybuf && sdslen(c->querybuf) > 0) { + server.current_client = c; processInputBuffer(c); + server.current_client = NULL; + } } /* Write the AOF buffer on disk */ @@ -828,6 +922,8 @@ void createSharedObjects(void) { "-LOADING Redis is loading the dataset in memory\r\n")); shared.slowscripterr = createObject(REDIS_STRING,sdsnew( "-BUSY Redis is busy running a script. You can only call SCRIPT KILL or SHUTDOWN NOSAVE.\r\n")); + shared.bgsaveerr = createObject(REDIS_STRING,sdsnew( + "-MISCONF Redis is configured to save RDB snapshots, but is currently not able to persist on disk. Write commands are disabled. Please check Redis logs for details about the error.\r\n")); shared.space = createObject(REDIS_STRING,sdsnew(" ")); shared.colon = createObject(REDIS_STRING,sdsnew(":")); shared.plus = createObject(REDIS_STRING,sdsnew("+")); @@ -847,15 +943,25 @@ void createSharedObjects(void) { shared.unsubscribebulk = createStringObject("$11\r\nunsubscribe\r\n",18); shared.psubscribebulk = createStringObject("$10\r\npsubscribe\r\n",17); shared.punsubscribebulk = createStringObject("$12\r\npunsubscribe\r\n",19); - shared.mbulk3 = createStringObject("*3\r\n",4); - shared.mbulk4 = createStringObject("*4\r\n",4); + shared.del = createStringObject("DEL",3); + shared.rpop = createStringObject("RPOP",4); + shared.lpop = createStringObject("LPOP",4); for (j = 0; j < REDIS_SHARED_INTEGERS; j++) { shared.integers[j] = createObject(REDIS_STRING,(void*)(long)j); shared.integers[j]->encoding = REDIS_ENCODING_INT; } + for (j = 0; j < REDIS_SHARED_BULKHDR_LEN; j++) { + shared.mbulkhdr[j] = createObject(REDIS_STRING, + sdscatprintf(sdsempty(),"*%d\r\n",j)); + shared.bulkhdr[j] = createObject(REDIS_STRING, + sdscatprintf(sdsempty(),"$%d\r\n",j)); + } } void initServerConfig() { + getRandomHexChars(server.runid,REDIS_RUN_ID_SIZE); + server.runid[REDIS_RUN_ID_SIZE] = '\0'; + server.arch_bits = (sizeof(long) == 8) ? 64 : 32; server.port = REDIS_SERVERPORT; server.bindaddr = NULL; server.unixsocket = NULL; @@ -873,30 +979,30 @@ void initServerConfig() { server.syslog_ident = zstrdup("redis"); server.syslog_facility = LOG_LOCAL0; server.daemonize = 0; - server.appendonly = 0; - server.appendfsync = APPENDFSYNC_EVERYSEC; - server.no_appendfsync_on_rewrite = 0; - server.auto_aofrewrite_perc = REDIS_AUTO_AOFREWRITE_PERC; - server.auto_aofrewrite_min_size = REDIS_AUTO_AOFREWRITE_MIN_SIZE; - server.auto_aofrewrite_base_size = 0; - server.aofrewrite_scheduled = 0; - server.lastfsync = time(NULL); - server.appendfd = -1; - server.appendseldb = -1; /* Make sure the first time will not match */ + server.aof_state = REDIS_AOF_OFF; + server.aof_fsync = AOF_FSYNC_EVERYSEC; + server.aof_no_fsync_on_rewrite = 0; + server.aof_rewrite_perc = REDIS_AOF_REWRITE_PERC; + server.aof_rewrite_min_size = REDIS_AOF_REWRITE_MIN_SIZE; + server.aof_rewrite_base_size = 0; + server.aof_rewrite_scheduled = 0; + server.aof_last_fsync = time(NULL); + server.aof_fd = -1; + server.aof_selected_db = -1; /* Make sure the first time will not match */ server.aof_flush_postponed_start = 0; server.pidfile = zstrdup("/var/run/redis.pid"); - server.dbfilename = zstrdup("dump.rdb"); - server.appendfilename = zstrdup("appendonly.aof"); + server.rdb_filename = zstrdup("dump.rdb"); + server.aof_filename = zstrdup("appendonly.aof"); server.requirepass = NULL; - server.rdbcompression = 1; + server.rdb_compression = 1; server.activerehashing = 1; server.maxclients = REDIS_MAX_CLIENTS; server.bpop_blocked_clients = 0; server.maxmemory = 0; server.maxmemory_policy = REDIS_MAXMEMORY_VOLATILE_LRU; server.maxmemory_samples = 3; - server.hash_max_zipmap_entries = REDIS_HASH_MAX_ZIPMAP_ENTRIES; - server.hash_max_zipmap_value = REDIS_HASH_MAX_ZIPMAP_VALUE; + server.hash_max_ziplist_entries = REDIS_HASH_MAX_ZIPLIST_ENTRIES; + server.hash_max_ziplist_value = REDIS_HASH_MAX_ZIPLIST_VALUE; server.list_max_ziplist_entries = REDIS_LIST_MAX_ZIPLIST_ENTRIES; server.list_max_ziplist_value = REDIS_LIST_MAX_ZIPLIST_VALUE; server.set_max_intset_entries = REDIS_SET_MAX_INTSET_ENTRIES; @@ -905,8 +1011,6 @@ void initServerConfig() { server.shutdown_asap = 0; server.repl_ping_slave_period = REDIS_REPL_PING_SLAVE_PERIOD; server.repl_timeout = REDIS_REPL_TIMEOUT; - server.cluster_enabled = 0; - server.cluster.configfile = zstrdup("nodes.conf"); server.lua_caller = NULL; server.lua_time_limit = REDIS_LUA_TIME_LIMIT; server.lua_client = NULL; @@ -919,16 +1023,26 @@ void initServerConfig() { appendServerSaveParams(300,100); /* save after 5 minutes and 100 changes */ appendServerSaveParams(60,10000); /* save after 1 minute and 10000 changes */ /* Replication related */ - server.isslave = 0; server.masterauth = NULL; server.masterhost = NULL; server.masterport = 6379; server.master = NULL; - server.replstate = REDIS_REPL_NONE; + server.repl_state = REDIS_REPL_NONE; server.repl_syncio_timeout = REDIS_REPL_SYNCIO_TIMEOUT; server.repl_serve_stale_data = 1; server.repl_down_since = -1; + /* Client output buffer limits */ + server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_NORMAL].hard_limit_bytes = 0; + server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_NORMAL].soft_limit_bytes = 0; + server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_NORMAL].soft_limit_seconds = 0; + server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_SLAVE].hard_limit_bytes = 1024*1024*256; + server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_SLAVE].soft_limit_bytes = 1024*1024*64; + server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_SLAVE].soft_limit_seconds = 60; + server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_PUBSUB].hard_limit_bytes = 1024*1024*32; + server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_PUBSUB].soft_limit_bytes = 1024*1024*8; + server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_PUBSUB].soft_limit_seconds = 60; + /* Double constants initialization */ R_Zero = 0.0; R_PosInf = 1.0/R_Zero; @@ -942,6 +1056,7 @@ void initServerConfig() { populateCommandTable(); server.delCommand = lookupCommandByCString("del"); server.multiCommand = lookupCommandByCString("multi"); + server.lpushCommand = lookupCommandByCString("lpush"); /* Slow log */ server.slowlog_log_slower_than = REDIS_SLOWLOG_LOG_SLOWER_THAN; @@ -954,6 +1069,43 @@ void initServerConfig() { server.bug_report_start = 0; } +/* This function will try to raise the max number of open files accordingly to + * the configured max number of clients. It will also account for 32 additional + * file descriptors as we need a few more for persistence, listening + * sockets, log files and so forth. + * + * If it will not be possible to set the limit accordingly to the configured + * max number of clients, the function will do the reverse setting + * server.maxclients to the value that we can actually handle. */ +void adjustOpenFilesLimit(void) { + rlim_t maxfiles = server.maxclients+32; + struct rlimit limit; + + if (maxfiles < 1024) maxfiles = 1024; + if (getrlimit(RLIMIT_NOFILE,&limit) == -1) { + redisLog(REDIS_WARNING,"Unable to obtain the current NOFILE limit (%s), assuming 1024 and setting the max clients configuration accordingly.", + strerror(errno)); + server.maxclients = 1024-32; + } else { + rlim_t oldlimit = limit.rlim_cur; + + /* Set the max number of files if the current limit is not enough + * for our needs. */ + if (oldlimit < maxfiles) { + limit.rlim_cur = maxfiles; + limit.rlim_max = maxfiles; + if (setrlimit(RLIMIT_NOFILE,&limit) == -1) { + server.maxclients = oldlimit-32; + redisLog(REDIS_WARNING,"Unable to set the max number of files limit to %d (%s), setting the max clients configuration to %d.", + (int) maxfiles, strerror(errno), (int) server.maxclients); + } else { + redisLog(REDIS_NOTICE,"Max number of open files set to %d", + (int) maxfiles); + } + } + } +} + void initServer() { int j; @@ -966,13 +1118,16 @@ void initServer() { server.syslog_facility); } + server.current_client = NULL; server.clients = listCreate(); + server.clients_to_close = listCreate(); server.slaves = listCreate(); server.monitors = listCreate(); server.unblocked_clients = listCreate(); createSharedObjects(); - server.el = aeCreateEventLoop(); + adjustOpenFilesLimit(); + server.el = aeCreateEventLoop(server.maxclients+1024); server.db = zmalloc(sizeof(redisDb)*server.dbnum); if (server.port != 0) { @@ -1007,10 +1162,10 @@ void initServer() { listSetFreeMethod(server.pubsub_patterns,freePubsubPattern); listSetMatchMethod(server.pubsub_patterns,listMatchPubsubPattern); server.cronloops = 0; - server.bgsavechildpid = -1; - server.bgrewritechildpid = -1; - server.bgrewritebuf = sdsempty(); - server.aofbuf = sdsempty(); + server.rdb_child_pid = -1; + server.aof_child_pid = -1; + server.aof_rewrite_buf = sdsempty(); + server.aof_buf = sdsempty(); server.lastsave = time(NULL); server.dirty = 0; server.stat_numcommands = 0; @@ -1023,60 +1178,42 @@ void initServer() { server.stat_peak_memory = 0; server.stat_fork_time = 0; server.stat_rejected_conn = 0; + memset(server.ops_sec_samples,0,sizeof(server.ops_sec_samples)); + server.ops_sec_idx = 0; + server.ops_sec_last_sample_time = mstime(); + server.ops_sec_last_sample_ops = 0; server.unixtime = time(NULL); + server.lastbgsave_status = REDIS_OK; + server.stop_writes_on_bgsave_err = 1; aeCreateTimeEvent(server.el, 1, serverCron, NULL, NULL); if (server.ipfd > 0 && aeCreateFileEvent(server.el,server.ipfd,AE_READABLE, acceptTcpHandler,NULL) == AE_ERR) oom("creating file event"); if (server.sofd > 0 && aeCreateFileEvent(server.el,server.sofd,AE_READABLE, acceptUnixHandler,NULL) == AE_ERR) oom("creating file event"); - if (server.appendonly) { - server.appendfd = open(server.appendfilename,O_WRONLY|O_APPEND|O_CREAT,0644); - if (server.appendfd == -1) { + if (server.aof_state == REDIS_AOF_ON) { + server.aof_fd = open(server.aof_filename, + O_WRONLY|O_APPEND|O_CREAT,0644); + if (server.aof_fd == -1) { redisLog(REDIS_WARNING, "Can't open the append-only file: %s", strerror(errno)); exit(1); } } - if (server.cluster_enabled) clusterInit(); + /* 32 bit instances are limited to 4GB of address space, so if there is + * no explicit limit in the user provided configuration we set a limit + * at 3.5GB using maxmemory with 'noeviction' policy'. This saves + * useless crashes of the Redis instance. */ + if (server.arch_bits == 32 && server.maxmemory == 0) { + redisLog(REDIS_WARNING,"Warning: 32 bit instance detected but no memory limit set. Setting 3.5 GB maxmemory limit with 'noeviction' policy now."); + server.maxmemory = 3584LL*(1024*1024); /* 3584 MB = 3.5 GB */ + server.maxmemory_policy = REDIS_MAXMEMORY_NO_EVICTION; + } + scriptingInit(); slowlogInit(); bioInit(); - srand(time(NULL)^getpid()); - - /* Try to raise the max number of open files accordingly to the - * configured max number of clients. Also account for 32 additional - * file descriptors as we need a few more for persistence, listening - * sockets, log files and so forth. */ - { - rlim_t maxfiles = server.maxclients+32; - struct rlimit limit; - - if (maxfiles < 1024) maxfiles = 1024; - if (getrlimit(RLIMIT_NOFILE,&limit) == -1) { - redisLog(REDIS_WARNING,"Unable to obtain the current NOFILE limit (%s), assuming 1024 and setting the max clients configuration accordingly.", - strerror(errno)); - server.maxclients = 1024-32; - } else { - rlim_t oldlimit = limit.rlim_cur; - - /* Set the max number of files if the current limit is not enough - * for our needs. */ - if (oldlimit < maxfiles) { - limit.rlim_cur = maxfiles; - limit.rlim_max = maxfiles; - if (setrlimit(RLIMIT_NOFILE,&limit) == -1) { - server.maxclients = oldlimit-32; - redisLog(REDIS_WARNING,"Unable to set the max number of files limit to %d (%s), setting the max clients configuration to %d.", - (int) maxfiles, strerror(errno), (int) server.maxclients); - } else { - redisLog(REDIS_NOTICE,"Max number of open files set to %d", - (int) maxfiles); - } - } - } - } } /* Populates the Redis Command Table starting from the hard coded list @@ -1100,6 +1237,7 @@ void populateCommandTable(void) { case 'f': c->flags |= REDIS_CMD_FORCE_REPLICATION; break; case 's': c->flags |= REDIS_CMD_NOSCRIPT; break; case 'R': c->flags |= REDIS_CMD_RANDOM; break; + case 'S': c->flags |= REDIS_CMD_SORT_FOR_SCRIPT; break; default: redisPanic("Unsupported command flag"); break; } f++; @@ -1122,6 +1260,43 @@ void resetCommandTableStats(void) { } } +/* ========================== Redis OP Array API ============================ */ + +void redisOpArrayInit(redisOpArray *oa) { + oa->ops = NULL; + oa->numops = 0; +} + +int redisOpArrayAppend(redisOpArray *oa, struct redisCommand *cmd, int dbid, + robj **argv, int argc, int target) +{ + redisOp *op; + + oa->ops = zrealloc(oa->ops,sizeof(redisOp)*(oa->numops+1)); + op = oa->ops+oa->numops; + op->cmd = cmd; + op->dbid = dbid; + op->argv = argv; + op->argc = argc; + op->target = target; + oa->numops++; + return oa->numops; +} + +void redisOpArrayFree(redisOpArray *oa) { + while(oa->numops) { + int j; + redisOp *op; + + oa->numops--; + op = oa->ops+oa->numops; + for (j = 0; j < op->argc; j++) + decrRefCount(op->argv[j]); + zfree(op->argv); + } + zfree(oa->ops); +} + /* ====================== Commands lookup and execution ===================== */ struct redisCommand *lookupCommand(sds name) { @@ -1137,25 +1312,84 @@ struct redisCommand *lookupCommandByCString(char *s) { return cmd; } +/* Propagate the specified command (in the context of the specified database id) + * to AOF, Slaves and Monitors. + * + * flags are an xor between: + * + REDIS_PROPAGATE_NONE (no propagation of command at all) + * + REDIS_PROPAGATE_AOF (propagate into the AOF file if is enabled) + * + REDIS_PROPAGATE_REPL (propagate into the replication link) + */ +void propagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, + int flags) +{ + if (server.aof_state != REDIS_AOF_OFF && flags & REDIS_PROPAGATE_AOF) + feedAppendOnlyFile(cmd,dbid,argv,argc); + if (flags & REDIS_PROPAGATE_REPL && listLength(server.slaves)) + replicationFeedSlaves(server.slaves,dbid,argv,argc); +} + +/* Used inside commands to schedule the propagation of additional commands + * after the current command is propagated to AOF / Replication. */ +void alsoPropagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, + int target) +{ + redisOpArrayAppend(&server.also_propagate,cmd,dbid,argv,argc,target); +} + /* Call() is the core of Redis execution of a command */ -void call(redisClient *c) { +void call(redisClient *c, int flags) { long long dirty, start = ustime(), duration; + /* Sent the command to clients in MONITOR mode, only if the commands are + * not geneated from reading an AOF. */ + if (listLength(server.monitors) && !server.loading) + replicationFeedMonitors(c,server.monitors,c->db->id,c->argv,c->argc); + + /* Call the command. */ + redisOpArrayInit(&server.also_propagate); dirty = server.dirty; c->cmd->proc(c); dirty = server.dirty-dirty; duration = ustime()-start; - c->cmd->microseconds += duration; - slowlogPushEntryIfNeeded(c->argv,c->argc,duration); - c->cmd->calls++; - - if (server.appendonly && dirty > 0) - feedAppendOnlyFile(c->cmd,c->db->id,c->argv,c->argc); - if ((dirty > 0 || c->cmd->flags & REDIS_CMD_FORCE_REPLICATION) && - listLength(server.slaves)) - replicationFeedSlaves(server.slaves,c->db->id,c->argv,c->argc); - if (listLength(server.monitors)) - replicationFeedMonitors(server.monitors,c->db->id,c->argv,c->argc); + + /* When EVAL is called loading the AOF we don't want commands called + * from Lua to go into the slowlog or to populate statistics. */ + if (server.loading && c->flags & REDIS_LUA_CLIENT) + flags &= ~(REDIS_CALL_SLOWLOG | REDIS_CALL_STATS); + + /* Log the command into the Slow log if needed, and populate the + * per-command statistics that we show in INFO commandstats. */ + if (flags & REDIS_CALL_SLOWLOG) + slowlogPushEntryIfNeeded(c->argv,c->argc,duration); + if (flags & REDIS_CALL_STATS) { + c->cmd->microseconds += duration; + c->cmd->calls++; + } + + /* Propagate the command into the AOF and replication link */ + if (flags & REDIS_CALL_PROPAGATE) { + int flags = REDIS_PROPAGATE_NONE; + + if (c->cmd->flags & REDIS_CMD_FORCE_REPLICATION) + flags |= REDIS_PROPAGATE_REPL; + if (dirty) + flags |= (REDIS_PROPAGATE_REPL | REDIS_PROPAGATE_AOF); + if (flags != REDIS_PROPAGATE_NONE) + propagate(c->cmd,c->db->id,c->argv,c->argc,flags); + } + /* Commands such as LPUSH or BRPOPLPUSH may propagate an additional + * PUSH command. */ + if (server.also_propagate.numops) { + int j; + redisOp *rop; + + for (j = 0; j < server.also_propagate.numops; j++) { + rop = &server.also_propagate.ops[j]; + propagate(rop->cmd, rop->dbid, rop->argv, rop->argc, rop->target); + } + redisOpArrayFree(&server.also_propagate); + } server.stat_numcommands++; } @@ -1199,39 +1433,27 @@ int processCommand(redisClient *c) { return REDIS_OK; } - /* If cluster is enabled, redirect here */ - if (server.cluster_enabled && - !(c->cmd->getkeys_proc == NULL && c->cmd->firstkey == 0)) { - int hashslot; - - if (server.cluster.state != REDIS_CLUSTER_OK) { - addReplyError(c,"The cluster is down. Check with CLUSTER INFO for more information"); - return REDIS_OK; - } else { - int ask; - clusterNode *n = getNodeByQuery(c,c->cmd,c->argv,c->argc,&hashslot,&ask); - if (n == NULL) { - addReplyError(c,"Multi keys request invalid in cluster"); - return REDIS_OK; - } else if (n != server.cluster.myself) { - addReplySds(c,sdscatprintf(sdsempty(), - "-%s %d %s:%d\r\n", ask ? "ASK" : "MOVED", - hashslot,n->ip,n->port)); - return REDIS_OK; - } - } - } - /* Handle the maxmemory directive. * * First we try to free some memory if possible (if there are volatile * keys in the dataset). If there are not the only thing we can do * is returning an error. */ - if (server.maxmemory) freeMemoryIfNeeded(); - if (server.maxmemory && (c->cmd->flags & REDIS_CMD_DENYOOM) && - zmalloc_used_memory() > server.maxmemory) + if (server.maxmemory) { + int retval = freeMemoryIfNeeded(); + if ((c->cmd->flags & REDIS_CMD_DENYOOM) && retval == REDIS_ERR) { + addReplyError(c, + "command not allowed when used memory > 'maxmemory'"); + return REDIS_OK; + } + } + + /* Don't accept write commands if there are problems persisting on disk. */ + if (server.stop_writes_on_bgsave_err && + server.saveparamslen > 0 + && server.lastbgsave_status == REDIS_ERR && + c->cmd->flags & REDIS_CMD_WRITE) { - addReplyError(c,"command not allowed when used memory > 'maxmemory'"); + addReply(c, shared.bgsaveerr); return REDIS_OK; } @@ -1248,7 +1470,7 @@ int processCommand(redisClient *c) { /* Only allow INFO and SLAVEOF when slave-serve-stale-data is no and * we are a slave with a broken link with master. */ - if (server.masterhost && server.replstate != REDIS_REPL_CONNECTED && + if (server.masterhost && server.repl_state != REDIS_REPL_CONNECTED && server.repl_serve_stale_data == 0 && c->cmd->proc != infoCommand && c->cmd->proc != slaveofCommand) { @@ -1284,7 +1506,7 @@ int processCommand(redisClient *c) { queueMultiCommand(c); addReply(c,shared.queued); } else { - call(c); + call(c,REDIS_CALL_FULL); } return REDIS_OK; } @@ -1299,27 +1521,27 @@ int prepareForShutdown(int flags) { /* Kill the saving child if there is a background saving in progress. We want to avoid race conditions, for instance our saving child may overwrite the synchronous saving did by SHUTDOWN. */ - if (server.bgsavechildpid != -1) { + if (server.rdb_child_pid != -1) { redisLog(REDIS_WARNING,"There is a child saving an .rdb. Killing it!"); - kill(server.bgsavechildpid,SIGKILL); - rdbRemoveTempFile(server.bgsavechildpid); + kill(server.rdb_child_pid,SIGKILL); + rdbRemoveTempFile(server.rdb_child_pid); } - if (server.appendonly) { + if (server.aof_state != REDIS_AOF_OFF) { /* Kill the AOF saving child as the AOF we already have may be longer * but contains the full dataset anyway. */ - if (server.bgrewritechildpid != -1) { + if (server.aof_child_pid != -1) { redisLog(REDIS_WARNING, "There is a child rewriting the AOF. Killing it!"); - kill(server.bgrewritechildpid,SIGKILL); + kill(server.aof_child_pid,SIGKILL); } /* Append only file: fsync() the AOF and exit */ redisLog(REDIS_NOTICE,"Calling fsync() on the AOF file."); - aof_fsync(server.appendfd); + aof_fsync(server.aof_fd); } if ((server.saveparamslen > 0 && !nosave) || save) { redisLog(REDIS_NOTICE,"Saving the final RDB snapshot before exiting."); /* Snapshotting. Perform a SYNC SAVE and exit */ - if (rdbSave(server.dbfilename) != REDIS_OK) { + if (rdbSave(server.rdb_filename) != REDIS_OK) { /* Ooops.. error saving! The best we can do is to continue * operating. Note that if there was a background saving process, * in the next cron() Redis will be notified that the background @@ -1367,6 +1589,17 @@ void echoCommand(redisClient *c) { addReplyBulk(c,c->argv[1]); } +void timeCommand(redisClient *c) { + struct timeval tv; + + /* gettimeofday() can only fail if &tv is a bad addresss so we + * don't check for errors. */ + gettimeofday(&tv,NULL); + addReplyMultiBulkLen(c,2); + addReplyBulkLongLong(c,tv.tv_sec); + addReplyBulkLongLong(c,tv.tv_usec); +} + /* Convert an amount of bytes into a human readable string in the form * of 100B, 2G, 100M, 4K, and so forth. */ void bytesToHuman(char *s, unsigned long long n) { @@ -1417,9 +1650,11 @@ sds genRedisInfoString(char *section) { "redis_version:%s\r\n" "redis_git_sha1:%s\r\n" "redis_git_dirty:%d\r\n" - "arch_bits:%s\r\n" + "arch_bits:%d\r\n" "multiplexing_api:%s\r\n" + "gcc_version:%d.%d.%d\r\n" "process_id:%ld\r\n" + "run_id:%s\r\n" "tcp_port:%d\r\n" "uptime_in_seconds:%ld\r\n" "uptime_in_days:%ld\r\n" @@ -1427,9 +1662,15 @@ sds genRedisInfoString(char *section) { REDIS_VERSION, redisGitSHA1(), strtol(redisGitDirty(),NULL,10) > 0, - (sizeof(long) == 8) ? "64" : "32", + server.arch_bits, aeGetApiName(), +#ifdef __GNUC__ + __GNUC__,__GNUC_MINOR__,__GNUC_PATCHLEVEL__, +#else + 0,0,0, +#endif (long) getpid(), + server.runid, server.port, uptime, uptime/(3600*24), @@ -1441,7 +1682,7 @@ sds genRedisInfoString(char *section) { if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# Clients\r\n" - "connected_clients:%d\r\n" + "connected_clients:%lu\r\n" "client_longest_output_list:%lu\r\n" "client_biggest_input_buf:%lu\r\n" "blocked_clients:%d\r\n", @@ -1489,25 +1730,27 @@ sds genRedisInfoString(char *section) { "changes_since_last_save:%lld\r\n" "bgsave_in_progress:%d\r\n" "last_save_time:%ld\r\n" + "last_bgsave_status:%s\r\n" "bgrewriteaof_in_progress:%d\r\n", server.loading, - server.appendonly, + server.aof_state != REDIS_AOF_OFF, server.dirty, - server.bgsavechildpid != -1, + server.rdb_child_pid != -1, server.lastsave, - server.bgrewritechildpid != -1); + server.lastbgsave_status == REDIS_OK ? "ok" : "err", + server.aof_child_pid != -1); - if (server.appendonly) { + if (server.aof_state != REDIS_AOF_OFF) { info = sdscatprintf(info, "aof_current_size:%lld\r\n" "aof_base_size:%lld\r\n" "aof_pending_rewrite:%d\r\n" "aof_buffer_length:%zu\r\n" "aof_pending_bio_fsync:%llu\r\n", - (long long) server.appendonly_current_size, - (long long) server.auto_aofrewrite_base_size, - server.aofrewrite_scheduled, - sdslen(server.aofbuf), + (long long) server.aof_current_size, + (long long) server.aof_rewrite_base_size, + server.aof_rewrite_scheduled, + sdslen(server.aof_buf), bioPendingJobsOfType(REDIS_BIO_AOF_FSYNC)); } @@ -1550,16 +1793,18 @@ sds genRedisInfoString(char *section) { "# Stats\r\n" "total_connections_received:%lld\r\n" "total_commands_processed:%lld\r\n" + "instantaneous_ops_per_sec:%lld\r\n" "rejected_connections:%lld\r\n" "expired_keys:%lld\r\n" "evicted_keys:%lld\r\n" "keyspace_hits:%lld\r\n" "keyspace_misses:%lld\r\n" "pubsub_channels:%ld\r\n" - "pubsub_patterns:%u\r\n" + "pubsub_patterns:%lu\r\n" "latest_fork_usec:%lld\r\n", server.stat_numconnections, server.stat_numcommands, + getOperationsPerSecond(), server.stat_rejected_conn, server.stat_expiredkeys, server.stat_evictedkeys, @@ -1586,14 +1831,14 @@ sds genRedisInfoString(char *section) { "master_sync_in_progress:%d\r\n" ,server.masterhost, server.masterport, - (server.replstate == REDIS_REPL_CONNECTED) ? + (server.repl_state == REDIS_REPL_CONNECTED) ? "up" : "down", server.master ? ((int)(time(NULL)-server.master->lastinteraction)) : -1, - server.replstate == REDIS_REPL_TRANSFER + server.repl_state == REDIS_REPL_TRANSFER ); - if (server.replstate == REDIS_REPL_TRANSFER) { + if (server.repl_state == REDIS_REPL_TRANSFER) { info = sdscatprintf(info, "master_sync_left_bytes:%ld\r\n" "master_sync_last_io_seconds_ago:%d\r\n" @@ -1602,15 +1847,46 @@ sds genRedisInfoString(char *section) { ); } - if (server.replstate != REDIS_REPL_CONNECTED) { + if (server.repl_state != REDIS_REPL_CONNECTED) { info = sdscatprintf(info, "master_link_down_since_seconds:%ld\r\n", (long)time(NULL)-server.repl_down_since); } } info = sdscatprintf(info, - "connected_slaves:%d\r\n", + "connected_slaves:%lu\r\n", listLength(server.slaves)); + if (listLength(server.slaves)) { + int slaveid = 0; + listNode *ln; + listIter li; + + listRewind(server.slaves,&li); + while((ln = listNext(&li))) { + redisClient *slave = listNodeValue(ln); + char *state = NULL; + char ip[32]; + int port; + + if (anetPeerToString(slave->fd,ip,&port) == -1) continue; + switch(slave->replstate) { + case REDIS_REPL_WAIT_BGSAVE_START: + case REDIS_REPL_WAIT_BGSAVE_END: + state = "wait_bgsave"; + break; + case REDIS_REPL_SEND_BULK: + state = "send_bulk"; + break; + case REDIS_REPL_ONLINE: + state = "online"; + break; + } + if (state == NULL) continue; + info = sdscatprintf(info,"slave%d:%s,%d,%s\r\n", + slaveid,ip,port,state); + slaveid++; + } + } } /* CPU */ @@ -1644,15 +1920,6 @@ sds genRedisInfoString(char *section) { } } - /* Clusetr */ - if (allsections || defsections || !strcasecmp(section,"cluster")) { - if (sections++) info = sdscat(info,"\r\n"); - info = sdscatprintf(info, - "# Cluster\r\n" - "cluster_enabled:%d\r\n", - server.cluster_enabled); - } - /* Key space */ if (allsections || defsections || !strcasecmp(section,"keyspace")) { if (sections++) info = sdscat(info,"\r\n"); @@ -1698,23 +1965,57 @@ void monitorCommand(redisClient *c) { /* ============================ Maxmemory directive ======================== */ /* This function gets called when 'maxmemory' is set on the config file to limit - * the max memory used by the server, and we are out of memory. - * This function will try to, in order: + * the max memory used by the server, before processing a command. * - * - Free objects from the free list - * - Try to remove keys with an EXPIRE set + * The goal of the function is to free enough memory to keep Redis under the + * configured memory limit. * - * It is not possible to free enough memory to reach used-memory < maxmemory - * the server will start refusing commands that will enlarge even more the - * memory usage. + * The function starts calculating how many bytes should be freed to keep + * Redis under the limit, and enters a loop selecting the best keys to + * evict accordingly to the configured policy. + * + * If all the bytes needed to return back under the limit were freed the + * function returns REDIS_OK, otherwise REDIS_ERR is returned, and the caller + * should block the execution of commands that will result in more memory + * used by the server. */ -void freeMemoryIfNeeded(void) { - /* Remove keys accordingly to the active policy as long as we are - * over the memory limit. */ - if (server.maxmemory_policy == REDIS_MAXMEMORY_NO_EVICTION) return; +int freeMemoryIfNeeded(void) { + size_t mem_used, mem_tofree, mem_freed; + int slaves = listLength(server.slaves); + + /* Remove the size of slaves output buffers and AOF buffer from the + * count of used memory. */ + mem_used = zmalloc_used_memory(); + if (slaves) { + listIter li; + listNode *ln; + + listRewind(server.slaves,&li); + while((ln = listNext(&li))) { + redisClient *slave = listNodeValue(ln); + unsigned long obuf_bytes = getClientOutputBufferMemoryUsage(slave); + if (obuf_bytes > mem_used) + mem_used = 0; + else + mem_used -= obuf_bytes; + } + } + if (server.aof_state != REDIS_AOF_OFF) { + mem_used -= sdslen(server.aof_buf); + mem_used -= sdslen(server.aof_rewrite_buf); + } + + /* Check if we are over the memory limit. */ + if (mem_used <= server.maxmemory) return REDIS_OK; + + if (server.maxmemory_policy == REDIS_MAXMEMORY_NO_EVICTION) + return REDIS_ERR; /* We need to free memory, but policy forbids. */ - while (server.maxmemory && zmalloc_used_memory() > server.maxmemory) { - int j, k, freed = 0; + /* Compute how much memory we need to free. */ + mem_tofree = mem_used - server.maxmemory; + mem_freed = 0; + while (mem_freed < mem_tofree) { + int j, k, keys_freed = 0; for (j = 0; j < server.dbnum; j++) { long bestval = 0; /* just to prevent warning */ @@ -1787,16 +2088,36 @@ void freeMemoryIfNeeded(void) { /* Finally remove the selected key. */ if (bestkey) { + long long delta; + robj *keyobj = createStringObject(bestkey,sdslen(bestkey)); propagateExpire(db,keyobj); + /* We compute the amount of memory freed by dbDelete() alone. + * It is possible that actually the memory needed to propagate + * the DEL in AOF and replication link is greater than the one + * we are freeing removing the key, but we can't account for + * that otherwise we would never exit the loop. + * + * AOF and Output buffer memory will be freed eventually so + * we only care about memory used by the key space. */ + delta = (long long) zmalloc_used_memory(); dbDelete(db,keyobj); + delta -= (long long) zmalloc_used_memory(); + mem_freed += delta; server.stat_evictedkeys++; decrRefCount(keyobj); - freed++; + keys_freed++; + + /* When the memory to free starts to be big enough, we may + * start spending so much time here that is impossible to + * deliver data to the slaves fast enough, so we force the + * transmission here inside the loop. */ + if (slaves) flushSlavesOutputBuffers(); } } - if (!freed) return; /* nothing to free... */ + if (!keys_freed) return REDIS_ERR; /* nothing to free... */ } + return REDIS_OK; } /* =================================== Main! ================================ */ @@ -1856,8 +2177,17 @@ void version() { } void usage() { - fprintf(stderr,"Usage: ./redis-server [/path/to/redis.conf]\n"); + fprintf(stderr,"Usage: ./redis-server [/path/to/redis.conf] [options]\n"); fprintf(stderr," ./redis-server - (read config from stdin)\n"); + fprintf(stderr," ./redis-server -v or --version\n"); + fprintf(stderr," ./redis-server -h or --help\n"); + fprintf(stderr," ./redis-server --test-memory \n\n"); + fprintf(stderr,"Examples:\n"); + fprintf(stderr," ./redis-server (run the server with default conf)\n"); + fprintf(stderr," ./redis-server /etc/redis/6379.conf\n"); + fprintf(stderr," ./redis-server --port 7777\n"); + fprintf(stderr," ./redis-server --port 7777 --slaveof 127.0.0.1 8888\n"); + fprintf(stderr," ./redis-server /etc/myredis.conf --loglevel verbose\n"); exit(1); } @@ -1870,7 +2200,7 @@ void redisAsciiArt(void) { redisGitSHA1(), strtol(redisGitDirty(),NULL,10) > 0, (sizeof(long) == 8) ? "64" : "32", - server.cluster_enabled ? "cluster" : "stand alone", + "stand alone", server.port, (long) getpid() ); @@ -1878,102 +2208,6 @@ void redisAsciiArt(void) { zfree(buf); } -#ifdef HAVE_BACKTRACE -static void *getMcontextEip(ucontext_t *uc) { -#if defined(__FreeBSD__) - return (void*) uc->uc_mcontext.mc_eip; -#elif defined(__dietlibc__) - return (void*) uc->uc_mcontext.eip; -#elif defined(__APPLE__) && !defined(MAC_OS_X_VERSION_10_6) - #if __x86_64__ - return (void*) uc->uc_mcontext->__ss.__rip; - #elif __i386__ - return (void*) uc->uc_mcontext->__ss.__eip; - #else - return (void*) uc->uc_mcontext->__ss.__srr0; - #endif -#elif defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_6) - #if defined(_STRUCT_X86_THREAD_STATE64) && !defined(__i386__) - return (void*) uc->uc_mcontext->__ss.__rip; - #else - return (void*) uc->uc_mcontext->__ss.__eip; - #endif -#elif defined(__i386__) - return (void*) uc->uc_mcontext.gregs[14]; /* Linux 32 */ -#elif defined(__X86_64__) || defined(__x86_64__) - return (void*) uc->uc_mcontext.gregs[16]; /* Linux 64 */ -#elif defined(__ia64__) /* Linux IA64 */ - return (void*) uc->uc_mcontext.sc_ip; -#else - return NULL; -#endif -} - -void bugReportStart(void) { - if (server.bug_report_start == 0) { - redisLog(REDIS_WARNING, - "=== REDIS BUG REPORT START: Cut & paste starting from here ==="); - server.bug_report_start = 1; - } -} - -static void sigsegvHandler(int sig, siginfo_t *info, void *secret) { - void *trace[100]; - char **messages = NULL; - int i, trace_size = 0; - ucontext_t *uc = (ucontext_t*) secret; - sds infostring, clients; - struct sigaction act; - REDIS_NOTUSED(info); - - bugReportStart(); - redisLog(REDIS_WARNING, - " Redis %s crashed by signal: %d", REDIS_VERSION, sig); - redisLog(REDIS_WARNING, - " Failed assertion: %s (%s:%d)", server.assert_failed, - server.assert_file, server.assert_line); - - /* Generate the stack trace */ - trace_size = backtrace(trace, 100); - - /* overwrite sigaction with caller's address */ - if (getMcontextEip(uc) != NULL) { - trace[1] = getMcontextEip(uc); - } - messages = backtrace_symbols(trace, trace_size); - redisLog(REDIS_WARNING, "--- STACK TRACE"); - for (i=1; i= 2) { + int j = 1; /* First option to parse in argv[] */ + sds options = sdsempty(); + char *configfile = NULL; + + /* Handle special options --help and --version */ if (strcmp(argv[1], "-v") == 0 || strcmp(argv[1], "--version") == 0) version(); - if (strcmp(argv[1], "--help") == 0) usage(); + if (strcmp(argv[1], "--help") == 0 || + strcmp(argv[1], "-h") == 0) usage(); + if (strcmp(argv[1], "--test-memory") == 0) { + if (argc == 3) { + memtest(atoi(argv[2]),300); + exit(0); + } else { + fprintf(stderr,"Please specify the amount of memory to test in megabytes.\n"); + fprintf(stderr,"Example: ./redis-server --test-memory 4096\n\n"); + exit(1); + } + } + + /* First argument is the config file name? */ + if (argv[j][0] != '-' || argv[j][1] != '-') + configfile = argv[j++]; + /* All the other options are parsed and conceptually appended to the + * configuration file. For instance --port 6380 will generate the + * string "port 6380\n" to be parsed after the actual file name + * is parsed, if any. */ + while(j != argc) { + if (argv[j][0] == '-' && argv[j][1] == '-') { + /* Option name */ + if (sdslen(options)) options = sdscat(options,"\n"); + options = sdscat(options,argv[j]+2); + options = sdscat(options," "); + } else { + /* Option argument */ + options = sdscatrepr(options,argv[j],strlen(argv[j])); + options = sdscat(options," "); + } + j++; + } resetServerSaveParams(); - loadServerConfig(argv[1]); - } else if ((argc > 2)) { - usage(); + loadServerConfig(configfile,options); + sdsfree(options); } else { redisLog(REDIS_WARNING,"Warning: no config file specified, using the default config. In order to specify a config file use 'redis-server /path/to/redis.conf'"); } @@ -2023,16 +2301,16 @@ int main(int argc, char **argv) { initServer(); if (server.daemonize) createPidFile(); redisAsciiArt(); - redisLog(REDIS_NOTICE,"Server started, Redis version " REDIS_VERSION); + redisLog(REDIS_WARNING,"Server started, Redis version " REDIS_VERSION); #ifdef __linux__ linuxOvercommitMemoryWarning(); #endif start = ustime(); - if (server.appendonly) { - if (loadAppendOnlyFile(server.appendfilename) == REDIS_OK) + if (server.aof_state == REDIS_AOF_ON) { + if (loadAppendOnlyFile(server.aof_filename) == REDIS_OK) redisLog(REDIS_NOTICE,"DB loaded from append only file: %.3f seconds",(float)(ustime()-start)/1000000); } else { - if (rdbLoad(server.dbfilename) == REDIS_OK) { + if (rdbLoad(server.rdb_filename) == REDIS_OK) { redisLog(REDIS_NOTICE,"DB loaded from disk: %.3f seconds", (float)(ustime()-start)/1000000); } else if (errno != ENOENT) {