From: antirez Date: Tue, 12 Jul 2011 10:39:16 +0000 (+0200) Subject: master branch merged into scripting. X-Git-Url: https://git.saurik.com/redis.git/commitdiff_plain/0681c5ad844cefefbe62f30df6587c0cbec3272e?hp=-c master branch merged into scripting. --- 0681c5ad844cefefbe62f30df6587c0cbec3272e diff --combined Makefile index 20c4f86e,44df36f5..691843d4 --- a/Makefile +++ b/Makefile @@@ -12,8 -12,7 +12,9 @@@ clean cd src && $(MAKE) $@ cd deps/hiredis && $(MAKE) $@ cd deps/linenoise && $(MAKE) $@ + cd deps/jemalloc && $(MAKE) distclean + cd deps/lua && $(MAKE) $@ + -(cd deps/jemalloc && $(MAKE) distclean) $(TARGETS): cd src && $(MAKE) $@ diff --combined redis.conf index 098c28da,1a051e4d..9e9eac5f --- a/redis.conf +++ b/redis.conf @@@ -312,13 -312,30 +312,37 @@@ no-appendfsync-on-rewrite n auto-aof-rewrite-percentage 100 auto-aof-rewrite-min-size 64mb +################################ LUA SCRIPTING ############################### + +# Max execution time of a Lua script in milliseconds. +# This prevents that a programming error generating an infinite loop will block +# your server forever. Set it to 0 or a negative value for unlimited execution. +lua-time-limit 60000 + + ################################## SLOW LOG ################################### + + # The Redis Slow Log is a system to log queries that exceeded a specified + # execution time. The execution time does not include the I/O operations + # like talking with the client, sending the reply and so forth, + # but just the time needed to actually execute the command (this is the only + # stage of command execution where the thread is blocked and can not serve + # other requests in the meantime). + # + # You can configure the slow log with two parameters: one tells Redis + # what is the execution time, in microseconds, to exceed in order for the + # command to get logged, and the other parameter is the length of the + # slow log. When a new command is logged the oldest one is removed from the + # queue of logged commands. + + # The following time is expressed in microseconds, so 1000000 is equivalent + # to one second. Note that a negative number disables the slow log, while + # a value of zero forces the logging of every command. + slowlog-log-slower-than 10000 + + # There is no limit to this length. Just be aware that it will consume memory. + # You can reclaim memory used by the slow log with SLOWLOG RESET. + slowlog-max-len 1024 + ############################### ADVANCED CONFIG ############################### # Hashes are encoded in a special way (much more memory efficient) when they diff --combined src/Makefile index 7e4b829d,c8647b7d..84cb1cda --- a/src/Makefile +++ b/src/Makefile @@@ -61,7 -61,7 +61,7 @@@ QUIET_CC = @printf ' %b %b\n' $(CCCO QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR); endif - OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endian.o scripting.o -OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endian.o slowlog.o ++OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endian.o slowlog.o scripting.o BENCHOBJ = ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o CHECKDUMPOBJ = redis-check-dump.o lzf_c.o lzf_d.o @@@ -86,38 -86,37 +86,37 @@@ ae_kqueue.o: ae_kqueue. ae_select.o: ae_select.c anet.o: anet.c fmacros.h anet.h aof.o: aof.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h cluster.o: cluster.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h \ + slowlog.h config.o: config.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h crc16.o: crc16.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h db.o: db.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h debug.o: debug.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h sha1.h - dict.o: dict.c fmacros.h dict.h zmalloc.h - diskstore.o: diskstore.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h \ + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h \ sha1.h - dscache.o: dscache.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + dict.o: dict.c fmacros.h dict.h zmalloc.h endian.o: endian.c intset.o: intset.c intset.h zmalloc.h endian.h lzf_c.o: lzf_c.c lzfP.h lzf_d.o: lzf_d.c lzfP.h multi.o: multi.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h networking.o: networking.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h \ + slowlog.h object.o: object.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h pqsort.o: pqsort.c pubsub.o: pubsub.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h rdb.o: rdb.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h lzf.h + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h \ + lzf.h redis-benchmark.o: redis-benchmark.c fmacros.h ae.h \ ../deps/hiredis/hiredis.h sds.h adlist.h zmalloc.h redis-check-aof.o: redis-check-aof.c fmacros.h config.h @@@ -125,27 -124,33 +124,33 @@@ redis-check-dump.o: redis-check-dump.c redis-cli.o: redis-cli.c fmacros.h version.h ../deps/hiredis/hiredis.h \ sds.h zmalloc.h ../deps/linenoise/linenoise.h help.h redis.o: redis.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h \ + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h \ asciilogo.h release.o: release.c release.h replication.o: replication.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h \ + slowlog.h sds.o: sds.c sds.h zmalloc.h sha1.o: sha1.c sha1.h config.h + slowlog.o: slowlog.c redis.h fmacros.h config.h ae.h sds.h dict.h \ + adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h \ + slowlog.h sort.o: sort.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h pqsort.h + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h \ + pqsort.h syncio.o: syncio.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h t_hash.o: t_hash.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h t_list.o: t_list.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h t_set.o: t_set.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h t_string.o: t_string.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h \ + slowlog.h t_zset.o: t_zset.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h util.o: util.c fmacros.h util.h ziplist.o: ziplist.c zmalloc.h util.h ziplist.h endian.h zipmap.o: zipmap.c zmalloc.h endian.h @@@ -158,14 -163,12 +163,14 @@@ dependencies @cd ../deps/hiredis && $(MAKE) static ARCH="$(ARCH)" @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)linenoise$(ENDCOLOR) @cd ../deps/linenoise && $(MAKE) ARCH="$(ARCH)" + @echo $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)Lua ansi$(ENDCOLOR) + @cd ../deps/lua && $(MAKE) ARCH="$(ARCH)" ansi ../deps/jemalloc/lib/libjemalloc.a: cd ../deps/jemalloc && ./configure $(JEMALLOC_CFLAGS) --with-jemalloc-prefix=je_ --enable-cc-silence && $(MAKE) lib/libjemalloc.a redis-server: $(OBJ) - $(QUIET_CC)$(CC) -o $(PRGNAME) $(CCOPT) $(DEBUG) $(OBJ) $(CCLINK) $(ALLOC_LINK) + $(QUIET_CC)$(CC) -o $(PRGNAME) $(CCOPT) $(DEBUG) $(OBJ) $(CCLINK) $(ALLOC_LINK) ../deps/lua/src/liblua.a redis-benchmark: dependencies $(BENCHOBJ) @cd ../deps/hiredis && $(MAKE) static @@@ -189,7 -192,7 +194,7 @@@ redis-check-aof: $(CHECKAOFOBJ # Because the jemalloc.h header is generated as a part of the jemalloc build # process, building it should complete before building any other object. %.o: %.c $(ALLOC_DEP) - $(QUIET_CC)$(CC) -c $(CFLAGS) $(DEBUG) $(COMPILE_TIME) $< + $(QUIET_CC)$(CC) -c $(CFLAGS) $(DEBUG) $(COMPILE_TIME) -I../deps/lua/src $< clean: rm -rf $(PRGNAME) $(BENCHPRGNAME) $(CLIPRGNAME) $(CHECKDUMPPRGNAME) $(CHECKAOFPRGNAME) *.o *.gcda *.gcno *.gcov @@@ -197,8 -200,8 +202,8 @@@ dep: $(CC) -MM *.c -I ../deps/hiredis -I ../deps/linenoise - test: redis-server - @(cd ..; (which tclsh >/dev/null && tclsh tests/test_helper.tcl --tags "${TAGS}" --file "${FILE}") || echo "You need to install Tcl in order to run tests.") + test: redis-server redis-check-aof + @(cd ..; (which tclsh8.5 >/dev/null && tclsh8.5 tests/test_helper.tcl --tags "${TAGS}") || echo "You need to install Tcl (tclsh8.5) in order to run tests.") bench: ./redis-benchmark diff --combined src/config.c index 88a00d38,e36f588a..5442e036 --- a/src/config.c +++ b/src/config.c @@@ -296,8 -296,12 +296,14 @@@ void loadServerConfig(char *filename) } else if (!strcasecmp(argv[0],"cluster-config-file") && argc == 2) { zfree(server.cluster.configfile); server.cluster.configfile = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { + server.lua_time_limit = strtoll(argv[1],NULL,10); + } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && + argc == 2) + { + server.slowlog_log_slower_than = strtoll(argv[1],NULL,10); + } else if (!strcasecmp(argv[0],"slowlog-max-len") && argc == 2) { + server.slowlog_max_len = strtoll(argv[1],NULL,10); } else { err = "Bad directive or wrong number of arguments"; goto loaderr; } @@@ -468,9 -472,12 +474,15 @@@ void configSetCommand(redisClient *c) } else if (!strcasecmp(c->argv[2]->ptr,"zset-max-ziplist-value")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; server.zset_max_ziplist_value = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"lua-time-limit")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; + server.lua_time_limit = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"slowlog-log-slower-than")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR) goto badfmt; + server.slowlog_log_slower_than = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"slowlog-max-len")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; + server.slowlog_max_len = (unsigned)ll; } else { addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s", (char*)c->argv[2]->ptr); @@@ -642,9 -649,14 +654,17 @@@ void configGetCommand(redisClient *c) addReplyBulkLongLong(c,server.zset_max_ziplist_value); matches++; } + if (stringmatch(pattern,"lua-time-limit",0)) { + addReplyBulkCString(c,"lua-time-limit"); + addReplyBulkLongLong(c,server.lua_time_limit); + if (stringmatch(pattern,"slowlog-log-slower-than",0)) { + addReplyBulkCString(c,"slowlog-log-slower-than"); + addReplyBulkLongLong(c,server.slowlog_log_slower_than); + matches++; + } + if (stringmatch(pattern,"slowlog-max-len",0)) { + addReplyBulkCString(c,"slowlog-max-len"); + addReplyBulkLongLong(c,server.slowlog_max_len); matches++; } setDeferredMultiBulkLength(c,replylen,matches*2); diff --combined src/networking.c index b95ef946,7319bd90..a4eee464 --- a/src/networking.c +++ b/src/networking.c @@@ -14,20 -14,14 +14,20 @@@ redisClient *createClient(int fd) redisClient *c = zmalloc(sizeof(redisClient)); c->bufpos = 0; - anetNonBlock(NULL,fd); - anetTcpNoDelay(NULL,fd); - if (aeCreateFileEvent(server.el,fd,AE_READABLE, - readQueryFromClient, c) == AE_ERR) - { - close(fd); - zfree(c); - return NULL; + /* passing -1 as fd it is possible to create a non connected client. + * This is useful since all the Redis commands needs to be executed + * in the context of a client. When commands are executed in other + * contexts (for instance a Lua script) we need a non connected client. */ + if (fd != -1) { + anetNonBlock(NULL,fd); + anetTcpNoDelay(NULL,fd); + if (aeCreateFileEvent(server.el,fd,AE_READABLE, + readQueryFromClient, c) == AE_ERR) + { + close(fd); + zfree(c); + return NULL; + } } selectDb(c,0); @@@ -36,6 -30,7 +36,7 @@@ c->reqtype = 0; c->argc = 0; c->argv = NULL; + c->cmd = NULL; c->multibulklen = 0; c->bulklen = -1; c->sentlen = 0; @@@ -57,7 -52,7 +58,7 @@@ c->pubsub_patterns = listCreate(); listSetFreeMethod(c->pubsub_patterns,decrRefCount); listSetMatchMethod(c->pubsub_patterns,listMatchObjects); - listAddNodeTail(server.clients,c); + if (fd != -1) listAddNodeTail(server.clients,c); initClientMultiState(c); return c; } @@@ -65,7 -60,6 +66,7 @@@ /* Set the event loop to listen for write events on the client's socket. * Typically gets called every time a reply is built. */ int _installWriteEvent(redisClient *c) { + if (c->flags & REDIS_LUA_CLIENT) return REDIS_OK; if (c->fd <= 0) return REDIS_ERR; if (c->bufpos == 0 && listLength(c->reply) == 0 && (c->replstate == REDIS_REPL_NONE || @@@ -246,17 -240,10 +247,17 @@@ void addReplyError(redisClient *c, cha } void addReplyErrorFormat(redisClient *c, const char *fmt, ...) { + size_t l, j; va_list ap; va_start(ap,fmt); sds s = sdscatvprintf(sdsempty(),fmt,ap); va_end(ap); + /* Make sure there are no newlines in the string, otherwise invalid protocol + * is emitted. */ + l = sdslen(s); + for (j = 0; j < l; j++) { + if (s[j] == '\r' || s[j] == '\n') s[j] = ' '; + } _addReplyError(c,s,sdslen(s)); sdsfree(s); } @@@ -461,6 -448,7 +462,7 @@@ static void freeClientArgv(redisClient for (j = 0; j < c->argc; j++) decrRefCount(c->argv[j]); c->argc = 0; + c->cmd = NULL; } void freeClient(redisClient *c) { @@@ -961,5 -949,7 +963,7 @@@ void rewriteClientCommandVector(redisCl /* Replace argv and argc with our new versions. */ c->argv = argv; c->argc = argc; + c->cmd = lookupCommand(c->argv[0]->ptr); + redisAssert(c->cmd != NULL); va_end(ap); } diff --combined src/redis.c index f4e3f623,0990d98a..7e9c6fd5 --- a/src/redis.c +++ b/src/redis.c @@@ -28,6 -28,7 +28,7 @@@ */ #include "redis.h" + #include "slowlog.h" #ifdef HAVE_BACKTRACE #include @@@ -193,8 -194,7 +194,9 @@@ struct redisCommand redisCommandTable[ {"dump",dumpCommand,2,0,NULL,0,0,0,0,0}, {"object",objectCommand,-2,0,NULL,0,0,0,0,0}, {"client",clientCommand,-2,0,NULL,0,0,0,0,0}, + {"eval",evalCommand,-3,REDIS_CMD_DENYOOM,zunionInterGetKeys,0,0,0,0,0}, - {"evalsha",evalShaCommand,-3,REDIS_CMD_DENYOOM,zunionInterGetKeys,0,0,0,0,0} ++ {"evalsha",evalShaCommand,-3,REDIS_CMD_DENYOOM,zunionInterGetKeys,0,0,0,0,0}, + {"slowlog",slowlogCommand,-2,0,NULL,0,0,0,0,0} }; /*============================ Utility functions ============================ */ @@@ -761,8 -761,6 +763,8 @@@ void createSharedObjects(void) "-ERR source and destination objects are the same\r\n")); shared.outofrangeerr = createObject(REDIS_STRING,sdsnew( "-ERR index out of range\r\n")); + shared.noscripterr = createObject(REDIS_STRING,sdsnew( + "-NOSCRIPT No matching script. Please use EVAL.\r\n")); shared.loadingerr = createObject(REDIS_STRING,sdsnew( "-LOADING Redis is loading the dataset in memory\r\n")); shared.space = createObject(REDIS_STRING,sdsnew(" ")); @@@ -839,7 -837,6 +841,7 @@@ void initServerConfig() server.shutdown_asap = 0; server.cluster_enabled = 0; server.cluster.configfile = zstrdup("nodes.conf"); + server.lua_time_limit = REDIS_LUA_TIME_LIMIT; updateLRUClock(); resetServerSaveParams(); @@@ -871,6 -868,10 +873,10 @@@ populateCommandTable(); server.delCommand = lookupCommandByCString("del"); server.multiCommand = lookupCommandByCString("multi"); + + /* Slow log */ + server.slowlog_log_slower_than = REDIS_SLOWLOG_LOG_SLOWER_THAN; + server.slowlog_max_len = REDIS_SLOWLOG_MAX_LEN; } void initServer() { @@@ -957,7 -958,7 +963,8 @@@ } if (server.cluster_enabled) clusterInit(); + scriptingInit(); + slowlogInit(); srand(time(NULL)^getpid()); } @@@ -1004,18 -1005,20 +1011,20 @@@ struct redisCommand *lookupCommandByCSt } /* Call() is the core of Redis execution of a command */ - void call(redisClient *c, struct redisCommand *cmd) { - long long dirty, start = ustime(); + void call(redisClient *c) { + long long dirty, start = ustime(), duration; dirty = server.dirty; - cmd->proc(c); + c->cmd->proc(c); dirty = server.dirty-dirty; - cmd->microseconds += ustime()-start; - cmd->calls++; + duration = ustime()-start; + c->cmd->microseconds += duration; + slowlogPushEntryIfNeeded(c->argv,c->argc,duration); + c->cmd->calls++; if (server.appendonly && dirty) - feedAppendOnlyFile(cmd,c->db->id,c->argv,c->argc); - if ((dirty || cmd->flags & REDIS_CMD_FORCE_REPLICATION) && + feedAppendOnlyFile(c->cmd,c->db->id,c->argv,c->argc); + if ((dirty || c->cmd->flags & REDIS_CMD_FORCE_REPLICATION) && listLength(server.slaves)) replicationFeedSlaves(server.slaves,c->db->id,c->argv,c->argc); if (listLength(server.monitors)) @@@ -1032,8 -1035,6 +1041,6 @@@ * and other operations can be performed by the caller. Otherwise * if 0 is returned the client was destroied (i.e. after QUIT). */ int processCommand(redisClient *c) { - struct redisCommand *cmd; - /* The QUIT command is handled separately. Normal command procs will * go through checking for replication and QUIT will cause trouble * when FORCE_REPLICATION is enabled and would be implemented in @@@ -1045,28 -1046,29 +1052,29 @@@ } /* Now lookup the command and check ASAP about trivial error conditions - * such wrong arity, bad command name and so forth. */ - cmd = lookupCommand(c->argv[0]->ptr); - if (!cmd) { + * such as wrong arity, bad command name and so forth. */ + c->cmd = lookupCommand(c->argv[0]->ptr); + if (!c->cmd) { addReplyErrorFormat(c,"unknown command '%s'", (char*)c->argv[0]->ptr); return REDIS_OK; - } else if ((cmd->arity > 0 && cmd->arity != c->argc) || - (c->argc < -cmd->arity)) { + } else if ((c->cmd->arity > 0 && c->cmd->arity != c->argc) || + (c->argc < -c->cmd->arity)) { addReplyErrorFormat(c,"wrong number of arguments for '%s' command", - cmd->name); + c->cmd->name); return REDIS_OK; } /* Check if the user is authenticated */ - if (server.requirepass && !c->authenticated && cmd->proc != authCommand) { + if (server.requirepass && !c->authenticated && c->cmd->proc != authCommand) + { addReplyError(c,"operation not permitted"); return REDIS_OK; } /* If cluster is enabled, redirect here */ if (server.cluster_enabled && - !(cmd->getkeys_proc == NULL && cmd->firstkey == 0)) { + !(c->cmd->getkeys_proc == NULL && c->cmd->firstkey == 0)) { int hashslot; if (server.cluster.state != REDIS_CLUSTER_OK) { @@@ -1074,7 -1076,7 +1082,7 @@@ return REDIS_OK; } else { int ask; - clusterNode *n = getNodeByQuery(c,cmd,c->argv,c->argc,&hashslot,&ask); + clusterNode *n = getNodeByQuery(c,c->cmd,c->argv,c->argc,&hashslot,&ask); if (n == NULL) { addReplyError(c,"Multi keys request invalid in cluster"); return REDIS_OK; @@@ -1093,7 -1095,7 +1101,7 @@@ * keys in the dataset). If there are not the only thing we can do * is returning an error. */ if (server.maxmemory) freeMemoryIfNeeded(); - if (server.maxmemory && (cmd->flags & REDIS_CMD_DENYOOM) && + if (server.maxmemory && (c->cmd->flags & REDIS_CMD_DENYOOM) && zmalloc_used_memory() > server.maxmemory) { addReplyError(c,"command not allowed when used memory > 'maxmemory'"); @@@ -1103,8 -1105,10 +1111,10 @@@ /* Only allow SUBSCRIBE and UNSUBSCRIBE in the context of Pub/Sub */ if ((dictSize(c->pubsub_channels) > 0 || listLength(c->pubsub_patterns) > 0) && - cmd->proc != subscribeCommand && cmd->proc != unsubscribeCommand && - cmd->proc != psubscribeCommand && cmd->proc != punsubscribeCommand) { + c->cmd->proc != subscribeCommand && + c->cmd->proc != unsubscribeCommand && + c->cmd->proc != psubscribeCommand && + c->cmd->proc != punsubscribeCommand) { addReplyError(c,"only (P)SUBSCRIBE / (P)UNSUBSCRIBE / QUIT allowed in this context"); return REDIS_OK; } @@@ -1113,7 -1117,7 +1123,7 @@@ * we are a slave with a broken link with master. */ if (server.masterhost && server.replstate != REDIS_REPL_CONNECTED && server.repl_serve_stale_data == 0 && - cmd->proc != infoCommand && cmd->proc != slaveofCommand) + c->cmd->proc != infoCommand && c->cmd->proc != slaveofCommand) { addReplyError(c, "link with MASTER is down and slave-serve-stale-data is set to no"); @@@ -1121,20 -1125,20 +1131,20 @@@ } /* Loading DB? Return an error if the command is not INFO */ - if (server.loading && cmd->proc != infoCommand) { + if (server.loading && c->cmd->proc != infoCommand) { addReply(c, shared.loadingerr); return REDIS_OK; } /* Exec the command */ if (c->flags & REDIS_MULTI && - cmd->proc != execCommand && cmd->proc != discardCommand && - cmd->proc != multiCommand && cmd->proc != watchCommand) + c->cmd->proc != execCommand && c->cmd->proc != discardCommand && + c->cmd->proc != multiCommand && c->cmd->proc != watchCommand) { - queueMultiCommand(c,cmd); + queueMultiCommand(c); addReply(c,shared.queued); } else { - call(c,cmd); + call(c); } return REDIS_OK; } @@@ -1291,7 -1295,6 +1301,7 @@@ sds genRedisInfoString(char *section) "used_memory_rss:%zu\r\n" "used_memory_peak:%zu\r\n" "used_memory_peak_human:%s\r\n" + "used_memory_lua:%lld\r\n" "mem_fragmentation_ratio:%.2f\r\n" "mem_allocator:%s\r\n", zmalloc_used_memory(), @@@ -1299,28 -1302,11 +1309,12 @@@ zmalloc_get_rss(), server.stat_peak_memory, peak_hmem, + ((long long)lua_gc(server.lua,LUA_GCCOUNT,0))*1024LL, zmalloc_get_fragmentation_ratio(), ZMALLOC_LIB ); } - /* Allocation statistics */ - if (allsections || !strcasecmp(section,"allocstats")) { - if (sections++) info = sdscat(info,"\r\n"); - info = sdscat(info, "# Allocstats\r\nallocation_stats:"); - for (j = 0; j <= ZMALLOC_MAX_ALLOC_STAT; j++) { - size_t count = zmalloc_allocations_for_size(j); - if (count) { - if (info[sdslen(info)-1] != ':') info = sdscatlen(info,",",1); - info = sdscatprintf(info,"%s%d=%zu", - (j == ZMALLOC_MAX_ALLOC_STAT) ? ">=" : "", - j,count); - } - } - info = sdscat(info,"\r\n"); - } - /* Persistence */ if (allsections || defsections || !strcasecmp(section,"persistence")) { if (sections++) info = sdscat(info,"\r\n"); @@@ -1456,8 -1442,8 +1450,8 @@@ "# CPU\r\n" "used_cpu_sys:%.2f\r\n" "used_cpu_user:%.2f\r\n" - "used_cpu_sys_childrens:%.2f\r\n" - "used_cpu_user_childrens:%.2f\r\n", + "used_cpu_sys_children:%.2f\r\n" + "used_cpu_user_children:%.2f\r\n", (float)self_ru.ru_utime.tv_sec+(float)self_ru.ru_utime.tv_usec/1000000, (float)self_ru.ru_stime.tv_sec+(float)self_ru.ru_stime.tv_usec/1000000, (float)c_ru.ru_utime.tv_sec+(float)c_ru.ru_utime.tv_usec/1000000, diff --combined src/redis.h index 9775e298,693347ef..1d094c1d --- a/src/redis.h +++ b/src/redis.h @@@ -19,19 -19,18 +19,19 @@@ #include #include #include +#include - #include "ae.h" /* Event driven programming library */ - #include "sds.h" /* Dynamic safe strings */ - #include "dict.h" /* Hash tables */ - #include "adlist.h" /* Linked lists */ + #include "ae.h" /* Event driven programming library */ + #include "sds.h" /* Dynamic safe strings */ + #include "dict.h" /* Hash tables */ + #include "adlist.h" /* Linked lists */ #include "zmalloc.h" /* total memory usage aware version of malloc/free */ - #include "anet.h" /* Networking the easy way */ - #include "zipmap.h" /* Compact string -> string data structure */ + #include "anet.h" /* Networking the easy way */ + #include "zipmap.h" /* Compact string -> string data structure */ #include "ziplist.h" /* Compact list data structure */ - #include "intset.h" /* Compact integer set structure */ - #include "version.h" - #include "util.h" + #include "intset.h" /* Compact integer set structure */ + #include "version.h" /* Version macro */ + #include "util.h" /* Misc functions useful in many places */ /* Error codes */ #define REDIS_OK 0 @@@ -53,6 -52,8 +53,8 @@@ #define REDIS_MAX_LOGMSG_LEN 1024 /* Default maximum length of syslog messages */ #define REDIS_AUTO_AOFREWRITE_PERC 100 #define REDIS_AUTO_AOFREWRITE_MIN_SIZE (1024*1024) + #define REDIS_SLOWLOG_LOG_SLOWER_THAN 10000 + #define REDIS_SLOWLOG_MAX_LEN 64 /* Hash table parameters */ #define REDIS_HT_MINFILL 10 /* Minimal hash table fill 10% */ @@@ -135,7 -136,6 +137,7 @@@ #define REDIS_CLOSE_AFTER_REPLY 128 /* Close after writing entire reply. */ #define REDIS_UNBLOCKED 256 /* This client was unblocked and is stored in server.unblocked_clients */ +#define REDIS_LUA_CLIENT 512 /* This is a non connected client used by Lua */ /* Client request types */ #define REDIS_REQ_INLINE 1 @@@ -210,9 -210,6 +212,9 @@@ #define REDIS_MAXMEMORY_ALLKEYS_RANDOM 4 #define REDIS_MAXMEMORY_NO_EVICTION 5 +/* Scripting */ +#define REDIS_LUA_TIME_LIMIT 60000 /* milliseconds */ + /* We can print the stacktrace, so our assert is defined this way: */ #define redisAssert(_e) ((_e)?(void)0 : (_redisAssert(#_e,__FILE__,__LINE__),_exit(1))) #define redisPanic(_e) _redisPanic(#_e,__FILE__,__LINE__),_exit(1) @@@ -312,6 -309,7 +314,7 @@@ typedef struct redisClient sds querybuf; int argc; robj **argv; + struct redisCommand *cmd; int reqtype; int multibulklen; /* number of multi bulk arguments left to read */ long bulklen; /* length of bulk argument in multi bulk request */ @@@ -347,7 -345,7 +350,7 @@@ struct sharedObjectsStruct robj *crlf, *ok, *err, *emptybulk, *czero, *cone, *cnegone, *pong, *space, *colon, *nullbulk, *nullmultibulk, *queued, *emptymultibulk, *wrongtypeerr, *nokeyerr, *syntaxerr, *sameobjecterr, - *outofrangeerr, *loadingerr, *plus, + *outofrangeerr, *noscripterr, *loadingerr, *plus, *select0, *select1, *select2, *select3, *select4, *select5, *select6, *select7, *select8, *select9, *messagebulk, *pmessagebulk, *subscribebulk, *unsubscribebulk, *mbulk3, @@@ -530,6 -528,10 +533,10 @@@ struct redisServer long long stat_keyspace_misses; /* number of failed lookups of keys */ size_t stat_peak_memory; /* max used memory record */ long long stat_fork_time; /* time needed to perform latets fork() */ + list *slowlog; + long long slowlog_entry_id; + long long slowlog_log_slower_than; + unsigned long slowlog_max_len; /* Configuration */ int verbosity; int maxidletime; @@@ -638,11 -640,6 +645,11 @@@ /* Cluster */ int cluster_enabled; clusterState cluster; + /* Scripting */ + lua_State *lua; + redisClient *lua_client; + long long lua_time_limit; + long long lua_time_start; }; typedef struct pubsubPattern { @@@ -807,7 -804,7 +814,7 @@@ void popGenericCommand(redisClient *c, void unwatchAllKeys(redisClient *c); void initClientMultiState(redisClient *c); void freeClientMultiState(redisClient *c); - void queueMultiCommand(redisClient *c, struct redisCommand *cmd); + void queueMultiCommand(redisClient *c); void touchWatchedKey(redisDb *db, robj *key); void touchWatchedKeysOnFlush(int dbid); @@@ -918,10 -915,9 +925,10 @@@ int processCommand(redisClient *c) void setupSignalHandlers(void); struct redisCommand *lookupCommand(sds name); struct redisCommand *lookupCommandByCString(char *s); - void call(redisClient *c, struct redisCommand *cmd); + void call(redisClient *c); int prepareForShutdown(); void redisLog(int level, const char *fmt, ...); +void redisLogRaw(int level, const char *msg); void usage(); void updateDictResizePolicy(void); int htNeedsResize(dict *dict); @@@ -1011,9 -1007,6 +1018,9 @@@ int clusterAddNode(clusterNode *node) void clusterCron(void); clusterNode *getNodeByQuery(redisClient *c, struct redisCommand *cmd, robj **argv, int argc, int *hashslot, int *ask); +/* Scripting */ +void scriptingInit(void); + /* Git SHA1 */ char *redisGitSHA1(void); char *redisGitDirty(void); @@@ -1142,8 -1135,6 +1149,8 @@@ void migrateCommand(redisClient *c) void dumpCommand(redisClient *c); void objectCommand(redisClient *c); void clientCommand(redisClient *c); +void evalCommand(redisClient *c); +void evalShaCommand(redisClient *c); #if defined(__GNUC__) void *calloc(size_t count, size_t size) __attribute__ ((deprecated)); diff --combined tests/test_helper.tcl index f505f26a,fbd9d3b3..559d0264 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@@ -9,6 -9,32 +9,33 @@@ source tests/support/tmpfile.tc source tests/support/test.tcl source tests/support/util.tcl + set ::all_tests { + unit/printver + unit/auth + unit/protocol + unit/basic + unit/type/list + unit/type/list-2 + unit/type/list-3 + unit/type/set + unit/type/zset + unit/type/hash + unit/sort + unit/expire + unit/other + unit/cas + unit/quit + integration/replication + integration/replication-2 + integration/replication-3 + integration/aof + unit/pubsub + unit/slowlog ++ unit/scripting + } + # Index to the next test to run in the ::all_tests list. + set ::next_test 0 + set ::host 127.0.0.1 set ::port 16379 set ::traceleaks 0 @@@ -19,12 -45,22 +46,22 @@@ set ::allowtags { set ::external 0; # If "1" this means, we are running against external instance set ::file ""; # If set, runs only the tests in this comma separated list set ::curfile ""; # Hold the filename of the current suite - set ::diskstore 0; # Don't touch this by hand. The test itself will toggle it. + set ::accurate 0; # If true runs fuzz tests with more iterations + set ::force_failure 0 + + # Set to 1 when we are running in client mode. The Redis test uses a + # server-client model to run tests simultaneously. The server instance + # runs the specified number of client instances that will actually run tests. + # The server is responsible of showing the result to the user, and exit with + # the appropriate exit code depending on the test outcome. + set ::client 0 + set ::numclients 16 proc execute_tests name { set path "tests/$name.tcl" set ::curfile $path source $path + send_data_packet $::test_server_fd done "$name" } # Setup a list to hold a stack of server configs. When calls to start_server @@@ -104,93 -140,191 +141,191 @@@ proc s {args} } proc cleanup {} { - puts "Cleanup: warning may take some time..." + puts -nonewline "Cleanup: may take some time... " + flush stdout catch {exec rm -rf {*}[glob tests/tmp/redis.conf.*]} catch {exec rm -rf {*}[glob tests/tmp/server.*]} + puts "OK" } - proc execute_everything {} { - if 0 { - # Use this when hacking on new tests. - set ::verbose 1 - execute_tests "unit/first" - return - } - - execute_tests "unit/printver" - execute_tests "unit/auth" - execute_tests "unit/protocol" - execute_tests "unit/basic" - execute_tests "unit/type/list" - execute_tests "unit/type/set" - execute_tests "unit/type/zset" - execute_tests "unit/type/hash" - execute_tests "unit/sort" - execute_tests "unit/expire" - execute_tests "unit/other" - execute_tests "unit/cas" - execute_tests "unit/quit" - execute_tests "integration/replication" - execute_tests "integration/aof" - # execute_tests "integration/redis-cli" - execute_tests "unit/pubsub" - execute_tests "unit/scripting" - - return; # No diskstore tests for now... - # run tests with diskstore enabled - puts "\nRunning diskstore tests... this is slow, press Ctrl+C if not interested.." - set ::diskstore 1 - lappend ::denytags nodiskstore - set ::global_overrides {diskstore-enabled yes} - execute_tests "unit/protocol" - execute_tests "unit/basic" - execute_tests "unit/type/list" - execute_tests "unit/type/set" - execute_tests "unit/type/zset" - execute_tests "unit/type/hash" - execute_tests "unit/sort" - execute_tests "unit/expire" - execute_tests "unit/other" - execute_tests "unit/cas" - } - - proc main {} { + proc test_server_main {} { cleanup + # Open a listening socket, trying different ports in order to find a + # non busy one. + set port 11111 + while 1 { + puts "Starting test server at port $port" + if {[catch {socket -server accept_test_clients $port} e]} { + if {[string match {*address already in use*} $e]} { + if {$port == 20000} { + puts "Can't find an available TCP port for test server." + exit 1 + } else { + incr port + } + } else { + puts "Fatal error starting test server: $e" + exit 1 + } + } else { + break + } + } + + # Start the client instances + set ::clients_pids {} + for {set j 0} {$j < $::numclients} {incr j} { + set p [exec tclsh8.5 [info script] {*}$::argv \ + --client $port --port [expr {$::port+($j*10)}] &] + lappend ::clients_pids $p + } - if {[string length $::file] > 0} { - foreach {file} [split $::file ,] { - execute_tests $file + # Setup global state for the test server + set ::idle_clients {} + set ::active_clients {} + array set ::clients_start_time {} + set ::clients_time_history {} + set ::failed_tests {} + + # Enter the event loop to handle clients I/O + after 100 test_server_cron + vwait forever + } + + # This function gets called 10 times per second, for now does nothing but + # may be used in the future in order to detect test clients taking too much + # time to execute the task. + proc test_server_cron {} { + } + + proc accept_test_clients {fd addr port} { + fileevent $fd readable [list read_from_test_client $fd] + } + + # This is the readable handler of our test server. Clients send us messages + # in the form of a status code such and additional data. Supported + # status types are: + # + # ready: the client is ready to execute the command. Only sent at client + # startup. The server will queue the client FD in the list of idle + # clients. + # testing: just used to signal that a given test started. + # ok: a test was executed with success. + # err: a test was executed with an error. + # exception: there was a runtime exception while executing the test. + # done: all the specified test file was processed, this test client is + # ready to accept a new task. + proc read_from_test_client fd { + set bytes [gets $fd] + set payload [read $fd $bytes] + foreach {status data} $payload break + if {$status eq {ready}} { + puts "\[$status\]: $data" + signal_idle_client $fd + } elseif {$status eq {done}} { + set elapsed [expr {[clock seconds]-$::clients_start_time($fd)}] + puts "\[[colorstr yellow $status]\]: $data ($elapsed seconds)" + puts "+++ [expr {[llength $::active_clients]-1}] units still in execution." + lappend ::clients_time_history $elapsed $data + signal_idle_client $fd + } elseif {$status eq {ok}} { + puts "\[[colorstr green $status]\]: $data" + } elseif {$status eq {err}} { + set err "\[[colorstr red $status]\]: $data" + puts $err + lappend ::failed_tests $err + } elseif {$status eq {exception}} { + puts "\[[colorstr red $status]\]: $data" + foreach p $::clients_pids { + catch {exec kill -9 $p} } + exit 1 + } elseif {$status eq {testing}} { + # No op } else { - execute_everything + puts "\[$status\]: $data" } + } - cleanup - puts "\n[expr $::num_tests] tests, $::num_passed passed, $::num_failed failed\n" - if {$::num_failed > 0} { - set curheader "" - puts "Failures:" - foreach {test} $::tests_failed { - set header [lindex $test 0] - append header " (" - append header [join [lindex $test 1] ","] - append header ")" - - if {$curheader ne $header} { - set curheader $header - puts "\n$curheader:" - } - - set name [lindex $test 2] - set msg [lindex $test 3] - puts "- $name: $msg" + # A new client is idle. Remove it from the list of active clients and + # if there are still test units to run, launch them. + proc signal_idle_client fd { + # Remove this fd from the list of active clients. + set ::active_clients \ + [lsearch -all -inline -not -exact $::active_clients $fd] + # New unit to process? + if {$::next_test != [llength $::all_tests]} { + puts [colorstr bold-white "Testing [lindex $::all_tests $::next_test]"] + set ::clients_start_time($fd) [clock seconds] + send_data_packet $fd run [lindex $::all_tests $::next_test] + lappend ::active_clients $fd + incr ::next_test + } else { + lappend ::idle_clients $fd + if {[llength $::active_clients] == 0} { + the_end } + } + } - puts "" + # The the_end funciton gets called when all the test units were already + # executed, so the test finished. + proc the_end {} { + # TODO: print the status, exit with the rigth exit code. + puts "\n The End\n" + puts "Execution time of different units:" + foreach {time name} $::clients_time_history { + puts " $time seconds - $name" + } + if {[llength $::failed_tests]} { + puts "\n[colorstr bold-red {!!! WARNING}] The following tests failed:\n" + foreach failed $::failed_tests { + puts "*** $failed" + } + cleanup exit 1 + } else { + puts "\n[colorstr bold-white {\o/}] [colorstr bold-green {All tests passed without errors!}]\n" + cleanup + exit 0 } } + # The client is not even driven (the test server is instead) as we just need + # to read the command, execute, reply... all this in a loop. + proc test_client_main server_port { + set ::test_server_fd [socket localhost $server_port] + send_data_packet $::test_server_fd ready [pid] + while 1 { + set bytes [gets $::test_server_fd] + set payload [read $::test_server_fd $bytes] + foreach {cmd data} $payload break + if {$cmd eq {run}} { + execute_tests $data + } else { + error "Unknown test client command: $cmd" + } + } + } + + proc send_data_packet {fd status data} { + set payload [list $status $data] + puts $fd [string length $payload] + puts -nonewline $fd $payload + flush $fd + } + + proc print_help_screen {} { + puts [join { + "--valgrind Run the test over valgrind." + "--accurate Run slow randomized tests for more iterations." + "--single Just execute the specified unit (see next option)." + "--list-tests List all the available test units." + "--force-failure Force the execution of a test that always fails." + "--help Print this help screen." + } "\n"] + } + # parse arguments for {set j 0} {$j < [llength $argv]} {incr j} { set opt [lindex $argv $j] @@@ -206,9 -340,6 +341,6 @@@ incr j } elseif {$opt eq {--valgrind}} { set ::valgrind 1 - } elseif {$opt eq {--file}} { - set ::file $arg - incr j } elseif {$opt eq {--host}} { set ::external 1 set ::host $arg @@@ -216,20 -347,47 +348,47 @@@ } elseif {$opt eq {--port}} { set ::port $arg incr j - } elseif {$opt eq {--verbose}} { - set ::verbose 1 + } elseif {$opt eq {--accurate}} { + set ::accurate 1 + } elseif {$opt eq {--force-failure}} { + set ::force_failure 1 + } elseif {$opt eq {--single}} { + set ::all_tests $arg + incr j + } elseif {$opt eq {--list-tests}} { + foreach t $::all_tests { + puts $t + } + exit 0 + } elseif {$opt eq {--client}} { + set ::client 1 + set ::test_server_port $arg + incr j + } elseif {$opt eq {--help}} { + print_help_screen + exit 0 } else { puts "Wrong argument: $opt" exit 1 } } - if {[catch { main } err]} { - if {[string length $err] > 0} { - # only display error when not generated by the test suite - if {$err ne "exception"} { - puts $::errorInfo + if {$::client} { + if {[catch { test_client_main $::test_server_port } err]} { + set estr "Executing test client: $err.\n$::errorInfo" + if {[catch {send_data_packet $::test_server_fd exception $estr}]} { + puts $estr } exit 1 } + } else { + if {[catch { test_server_main } err]} { + if {[string length $err] > 0} { + # only display error when not generated by the test suite + if {$err ne "exception"} { + puts $::errorInfo + } + exit 1 + } + } }