From: Salvatore Sanfilippo Date: Fri, 18 Nov 2011 13:34:56 +0000 (-0800) Subject: Merge pull request #173 from jasondavies/typo X-Git-Url: https://git.saurik.com/redis.git/commitdiff_plain/794e6ce14ff85f00b0d3e86365e9cc45a90d3bcf?hp=b91cbf66bb527fbc7eef8079e9153fa5357186e2 Merge pull request #173 from jasondavies/typo Fix some minor typos. --- diff --git a/.gitignore b/.gitignore index 262faef3..1f480bd4 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ src/redis.conf deps/lua/src/lua deps/lua/src/luac deps/lua/src/liblua.a +.make-* diff --git a/README b/README index e3054887..3920b534 100644 --- a/README +++ b/README @@ -26,18 +26,19 @@ NOTE: if after building Redis with a 32 bit target you need to rebuild it Allocator --------- -By default Redis compiles and links against jemalloc under Linux, since -glibc malloc() has memory fragmentation problems. +Selecting a non-default memory allocator when building Redis is done by setting +the `MALLOC` environment variable. Redis is compiled and linked against libc +malloc by default, with the exception of jemalloc being the default on Linux +systems. This default was picked because jemalloc has proven to have fewer +fragmentation problems than libc malloc. -To force a libc malloc() build use: +To force compiling against libc malloc, use: - % make FORCE_LIBC_MALLOC=yes + % make MALLOC=libc -In all the other non Linux systems the libc malloc() is used by default. +To compile against jemalloc on Mac OS X systems, use: -On Mac OS X you can force a jemalloc based build using the following: - - % make USE_JEMALLOC=yes + % make MALLOC=jemalloc Verbose build ------------- diff --git a/deps/Makefile b/deps/Makefile new file mode 100644 index 00000000..b881c814 --- /dev/null +++ b/deps/Makefile @@ -0,0 +1,59 @@ +# Redis dependency Makefile + +UNAME_S:=$(shell sh -c 'uname -s 2> /dev/null || echo not') + +LUA_CFLAGS=-O2 -Wall $(ARCH) +ifeq ($(UNAME_S),SunOS) + # Make isinf() available + LUA_CFLAGS+= -D__C99FEATURES__=1 +endif + +JEMALLOC_CFLAGS= +ifeq ($(ARCH),-m32) + JEMALLOC_CFLAGS+=CFLAGS="-std=gnu99 -Wall -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -m32" +endif + +CCCOLOR="\033[34m" +LINKCOLOR="\033[34;1m" +SRCCOLOR="\033[33m" +BINCOLOR="\033[37;1m" +MAKECOLOR="\033[32;1m" +ENDCOLOR="\033[0m" + +default: + @echo "Explicit target required" + +# Clean everything when ARCH is different +ifneq ($(shell sh -c '[ -f .make-arch ] && cat .make-arch'), $(ARCH)) +.make-arch: distclean +else +.make-arch: +endif + +.make-arch: + -(echo $(ARCH) > .make-arch) + +distclean: + -(cd hiredis && $(MAKE) clean) > /dev/null || true + -(cd linenoise && $(MAKE) clean) > /dev/null || true + -(cd lua && $(MAKE) clean) > /dev/null || true + -(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true + -(rm -f .make-arch) + +hiredis: .make-arch + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)hiredis$(ENDCOLOR) + cd hiredis && $(MAKE) static ARCH="$(ARCH)" + +linenoise: .make-arch + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)linenoise$(ENDCOLOR) + cd linenoise && $(MAKE) ARCH="$(ARCH)" + +lua: .make-arch + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)lua$(ENDCOLOR) + cd lua && $(MAKE) CFLAGS="$(LUA_CFLAGS)" MYLDFLAGS="$(ARCH)" ansi + +jemalloc: .make-arch + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)jemalloc$(ENDCOLOR) + cd jemalloc && ./configure $(JEMALLOC_CFLAGS) --with-jemalloc-prefix=je_ --enable-cc-silence && $(MAKE) lib/libjemalloc.a + +.PHONY: default conditional_clean hiredis linenoise lua jemalloc diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c index b27c63b8..976e94f9 100644 --- a/deps/hiredis/hiredis.c +++ b/deps/hiredis/hiredis.c @@ -520,13 +520,14 @@ void redisReplyReaderFeed(void *reader, const char *buf, size_t len) { /* Copy the provided buffer. */ if (buf != NULL && len >= 1) { +#if 0 /* Destroy internal buffer when it is empty and is quite large. */ if (r->len == 0 && sdsavail(r->buf) > 16*1024) { sdsfree(r->buf); r->buf = sdsempty(); r->pos = 0; } - +#endif r->buf = sdscatlen(r->buf,buf,len); r->len = sdslen(r->buf); } @@ -901,7 +902,7 @@ static void __redisCreateReplyReader(redisContext *c) { * After this function is called, you may use redisContextReadReply to * see if there is a reply available. */ int redisBufferRead(redisContext *c) { - char buf[2048]; + char buf[1024*16]; int nread = read(c->fd,buf,sizeof(buf)); if (nread == -1) { if (errno == EAGAIN && !(c->flags & REDIS_BLOCK)) { diff --git a/deps/lua/Makefile b/deps/lua/Makefile index eed4aedc..6e78f66f 100644 --- a/deps/lua/Makefile +++ b/deps/lua/Makefile @@ -53,7 +53,7 @@ R= 5.1.4 all: $(PLAT) $(PLATS) clean: - cd src && $(MAKE) ARCH="$(ARCH)" $@ + cd src && $(MAKE) $@ test: dummy src/lua test/hello.lua diff --git a/deps/lua/src/Makefile b/deps/lua/src/Makefile index 01088abc..ff666160 100644 --- a/deps/lua/src/Makefile +++ b/deps/lua/src/Makefile @@ -8,7 +8,7 @@ PLAT= none CC= gcc -CFLAGS= -O2 -Wall $(MYCFLAGS) $(ARCH) +CFLAGS= -O2 -Wall $(MYCFLAGS) AR= ar rcu RANLIB= ranlib RM= rm -f @@ -52,10 +52,10 @@ $(LUA_A): $(CORE_O) $(LIB_O) $(RANLIB) $@ $(LUA_T): $(LUA_O) $(LUA_A) - $(CC) -o $@ $(MYLDFLAGS) $(LUA_O) $(LUA_A) $(LIBS) $(ARCH) + $(CC) -o $@ $(MYLDFLAGS) $(LUA_O) $(LUA_A) $(LIBS) $(LUAC_T): $(LUAC_O) $(LUA_A) - $(CC) -o $@ $(MYLDFLAGS) $(LUAC_O) $(LUA_A) $(LIBS) $(ARCH) + $(CC) -o $@ $(MYLDFLAGS) $(LUAC_O) $(LUA_A) $(LIBS) clean: $(RM) $(ALL_T) $(ALL_O) @@ -84,7 +84,7 @@ aix: $(MAKE) all CC="xlc" CFLAGS="-O2 -DLUA_USE_POSIX -DLUA_USE_DLOPEN" MYLIBS="-ldl" MYLDFLAGS="-brtl -bexpall" ansi: - $(MAKE) all MYCFLAGS=-DLUA_ANSI ARCH="$(ARCH)" + $(MAKE) all MYCFLAGS=-DLUA_ANSI bsd: $(MAKE) all MYCFLAGS="-DLUA_USE_POSIX -DLUA_USE_DLOPEN" MYLIBS="-Wl,-E" diff --git a/src/Makefile b/src/Makefile index 04f2a68c..659d1d7f 100644 --- a/src/Makefile +++ b/src/Makefile @@ -5,18 +5,9 @@ release_hdr := $(shell sh -c './mkreleasehdr.sh') uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') OPTIMIZATION?=-O2 - -LUA_CFLAGS=-O2 -Wall - -ifeq ($(uname_S),Linux) - ifneq ($(FORCE_LIBC_MALLOC),yes) - USE_JEMALLOC=yes - endif -endif +DEPENDENCY_TARGETS=hiredis linenoise lua ifeq ($(uname_S),SunOS) - # make isinf() available - LUA_CFLAGS+=-D__C99FEATURES__=1 CFLAGS?=-std=c99 -pedantic $(OPTIMIZATION) -Wall -W -D__EXTENSIONS__ -D_XPG6 CCLINK?=-ldl -lnsl -lsocket -lm -lpthread DEBUG?=-g -ggdb @@ -26,27 +17,44 @@ else DEBUG?=-g -rdynamic -ggdb endif +# Default allocator +ifeq ($(uname_S),Linux) + MALLOC?=jemalloc +else + MALLOC?=libc +endif + +# Backwards compatibility for selecting an allocator ifeq ($(USE_TCMALLOC),yes) - ALLOD_DEPS= + MALLOC=tcmalloc +endif + +ifeq ($(USE_TCMALLOC_MINIMAL),yes) + MALLOC=tcmalloc_minimal +endif + +ifeq ($(USE_JEMALLOC),yes) + MALLOC=jemalloc +endif + +ifeq ($(MALLOC),tcmalloc) ALLOC_LINK=-ltcmalloc ALLOC_FLAGS=-DUSE_TCMALLOC endif -ifeq ($(USE_TCMALLOC_MINIMAL),yes) - ALLOD_DEPS= +ifeq ($(MALLOC),tcmalloc_minimal) ALLOC_LINK=-ltcmalloc_minimal ALLOC_FLAGS=-DUSE_TCMALLOC endif -ifeq ($(USE_JEMALLOC),yes) - ALLOC_DEP=../deps/jemalloc/lib/libjemalloc.a - ALLOC_LINK=$(ALLOC_DEP) -ldl +ifeq ($(MALLOC),jemalloc) + ALLOC_LINK=../deps/jemalloc/lib/libjemalloc.a -ldl ALLOC_FLAGS=-DUSE_JEMALLOC -I../deps/jemalloc/include + DEPENDENCY_TARGETS+= jemalloc endif CCLINK+= $(ALLOC_LINK) CFLAGS+= $(ALLOC_FLAGS) - CCOPT= $(CFLAGS) $(ARCH) $(PROF) PREFIX= /usr/local @@ -159,52 +167,66 @@ ziplist.o: ziplist.c zmalloc.h util.h ziplist.h endian.h zipmap.o: zipmap.c zmalloc.h endian.h zmalloc.o: zmalloc.c config.h zmalloc.h -.PHONY: dependencies +# Clean local objects when ARCH is different +ifneq ($(shell sh -c '[ -f .make-arch ] && cat .make-arch'), $(ARCH)) +.make-arch: clean +else +.make-arch: +endif -dependencies: - @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)hiredis$(ENDCOLOR) - @cd ../deps/hiredis && $(MAKE) static ARCH="$(ARCH)" - @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)linenoise$(ENDCOLOR) - @cd ../deps/linenoise && $(MAKE) ARCH="$(ARCH)" - @echo $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)Lua ansi$(ENDCOLOR) - @cd ../deps/lua && $(MAKE) ARCH="$(ARCH)" CFLAGS="$(LUA_CFLAGS)" ansi +.make-arch: + -(cd ../deps && make $(DEPENDENCY_TARGETS) ARCH="$(ARCH)") + -(echo $(ARCH) > .make-arch) -../deps/jemalloc/lib/libjemalloc.a: - cd ../deps/jemalloc && ./configure $(JEMALLOC_CFLAGS) --with-jemalloc-prefix=je_ --enable-cc-silence && $(MAKE) lib/libjemalloc.a +# Clean local objects when allocator changes +ifneq ($(shell sh -c '[ -f .make-malloc ] && cat .make-malloc'), $(MALLOC)) +.make-malloc: clean +else +.make-malloc: +endif + +.make-malloc: + -(echo $(MALLOC) > .make-malloc) -redis-server: dependencies $(OBJ) - $(QUIET_LINK)$(CC) -o $(PRGNAME) $(CCOPT) $(DEBUG) $(OBJ) $(CCLINK) $(ALLOC_LINK) ../deps/lua/src/liblua.a +# Union of make-prerequisites +.make-prerequisites: .make-arch .make-malloc + @touch $@ -redis-benchmark: dependencies $(BENCHOBJ) - @cd ../deps/hiredis && $(MAKE) static - $(QUIET_LINK)$(CC) -o $(BENCHPRGNAME) $(CCOPT) $(DEBUG) $(BENCHOBJ) ../deps/hiredis/libhiredis.a $(CCLINK) $(ALLOC_LINK) +redis-server: .make-prerequisites $(OBJ) + $(QUIET_LINK)$(CC) -o $(PRGNAME) $(CCOPT) $(DEBUG) $(OBJ) ../deps/lua/src/liblua.a $(CCLINK) -redis-benchmark.o: +redis-benchmark: .make-prerequisites $(BENCHOBJ) + $(QUIET_LINK)$(CC) -o $(BENCHPRGNAME) $(CCOPT) $(DEBUG) $(BENCHOBJ) ../deps/hiredis/libhiredis.a $(CCLINK) + +redis-benchmark.o: redis-benchmark.c .make-prerequisites $(QUIET_CC)$(CC) -c $(CFLAGS) -I../deps/hiredis $(DEBUG) $(COMPILE_TIME) $< -redis-cli: dependencies $(CLIOBJ) - $(QUIET_LINK)$(CC) -o $(CLIPRGNAME) $(CCOPT) $(DEBUG) $(CLIOBJ) ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(CCLINK) $(ALLOC_LINK) +redis-cli: .make-prerequisites $(CLIOBJ) + $(QUIET_LINK)$(CC) -o $(CLIPRGNAME) $(CCOPT) $(DEBUG) $(CLIOBJ) ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(CCLINK) -redis-cli.o: +redis-cli.o: redis-cli.c .make-prerequisites $(QUIET_CC)$(CC) -c $(CFLAGS) -I../deps/hiredis -I../deps/linenoise $(DEBUG) $(COMPILE_TIME) $< -redis-check-dump: $(CHECKDUMPOBJ) - $(QUIET_LINK)$(CC) -o $(CHECKDUMPPRGNAME) $(CCOPT) $(DEBUG) $(CHECKDUMPOBJ) $(CCLINK) $(ALLOC_LINK) +redis-check-dump: .make-prerequisites $(CHECKDUMPOBJ) + $(QUIET_LINK)$(CC) -o $(CHECKDUMPPRGNAME) $(CCOPT) $(DEBUG) $(CHECKDUMPOBJ) $(CCLINK) -redis-check-aof: $(CHECKAOFOBJ) - $(QUIET_LINK)$(CC) -o $(CHECKAOFPRGNAME) $(CCOPT) $(DEBUG) $(CHECKAOFOBJ) $(CCLINK) $(ALLOC_LINK) +redis-check-aof: .make-prerequisites $(CHECKAOFOBJ) + $(QUIET_LINK)$(CC) -o $(CHECKAOFPRGNAME) $(CCOPT) $(DEBUG) $(CHECKAOFOBJ) $(CCLINK) # Because the jemalloc.h header is generated as a part of the jemalloc build -# process, building it should complete before building any other object. -%.o: %.c $(ALLOC_DEP) +# process, building it should complete before building any other object. Instead of +# depending on a single artifact, simply build all dependencies first. +%.o: %.c .make-prerequisites $(QUIET_CC)$(CC) -c $(CFLAGS) $(DEBUG) $(COMPILE_TIME) -I../deps/lua/src $< +.PHONY: all clean distclean + clean: rm -rf $(PRGNAME) $(BENCHPRGNAME) $(CLIPRGNAME) $(CHECKDUMPPRGNAME) $(CHECKAOFPRGNAME) *.o *.gcda *.gcno *.gcov - cd ../deps/hiredis && $(MAKE) $@ - cd ../deps/linenoise && $(MAKE) $@ - cd ../deps/lua && $(MAKE) $@ - -(cd ../deps/jemalloc && $(MAKE) distclean) + +distclean: clean + -(cd ../deps && $(MAKE) distclean) + -(rm -f .make-arch .make-malloc) dep: $(CC) -MM *.c -I ../deps/hiredis -I ../deps/linenoise diff --git a/src/aof.c b/src/aof.c index 66516f78..4a463bde 100644 --- a/src/aof.c +++ b/src/aof.c @@ -181,21 +181,38 @@ sds catAppendOnlyGenericCommand(sds dst, int argc, robj **argv) { return dst; } -sds catAppendOnlyExpireAtCommand(sds buf, robj *key, robj *seconds) { - int argc = 3; - long when; +/* Create the sds representation of an PEXPIREAT command, using + * 'seconds' as time to live and 'cmd' to understand what command + * we are translating into a PEXPIREAT. + * + * This command is used in order to translate EXPIRE and PEXPIRE commands + * into PEXPIREAT command so that we retain precision in the append only + * file, and the time is always absolute and not relative. */ +sds catAppendOnlyExpireAtCommand(sds buf, struct redisCommand *cmd, robj *key, robj *seconds) { + long long when; robj *argv[3]; /* Make sure we can use strtol */ seconds = getDecodedObject(seconds); - when = time(NULL)+strtol(seconds->ptr,NULL,10); + when = strtoll(seconds->ptr,NULL,10); + /* Convert argument into milliseconds for EXPIRE, SETEX, EXPIREAT */ + if (cmd->proc == expireCommand || cmd->proc == setexCommand || + cmd->proc == expireatCommand) + { + when *= 1000; + } + /* Convert into absolute time for EXPIRE, PEXPIRE, SETEX, PSETEX */ + if (cmd->proc == expireCommand || cmd->proc == pexpireCommand || + cmd->proc == setexCommand || cmd->proc == psetexCommand) + { + when += mstime(); + } decrRefCount(seconds); - argv[0] = createStringObject("EXPIREAT",8); + argv[0] = createStringObject("PEXPIREAT",9); argv[1] = key; - argv[2] = createObject(REDIS_STRING, - sdscatprintf(sdsempty(),"%ld",when)); - buf = catAppendOnlyGenericCommand(buf, argc, argv); + argv[2] = createStringObjectFromLongLong(when); + buf = catAppendOnlyGenericCommand(buf, 3, argv); decrRefCount(argv[0]); decrRefCount(argv[2]); return buf; @@ -216,18 +233,22 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a server.appendseldb = dictid; } - if (cmd->proc == expireCommand) { - /* Translate EXPIRE into EXPIREAT */ - buf = catAppendOnlyExpireAtCommand(buf,argv[1],argv[2]); - } else if (cmd->proc == setexCommand) { - /* Translate SETEX to SET and EXPIREAT */ + if (cmd->proc == expireCommand || cmd->proc == pexpireCommand || + cmd->proc == expireatCommand) { + /* Translate EXPIRE/PEXPIRE/EXPIREAT into PEXPIREAT */ + buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]); + } else if (cmd->proc == setexCommand || cmd->proc == psetexCommand) { + /* Translate SETEX/PSETEX to SET and PEXPIREAT */ tmpargv[0] = createStringObject("SET",3); tmpargv[1] = argv[1]; tmpargv[2] = argv[3]; buf = catAppendOnlyGenericCommand(buf,3,tmpargv); decrRefCount(tmpargv[0]); - buf = catAppendOnlyExpireAtCommand(buf,argv[1],argv[2]); + buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]); } else { + /* All the other commands don't need translation or need the + * same translation already operated in the command vector + * for the replication itself. */ buf = catAppendOnlyGenericCommand(buf,argc,argv); } @@ -410,7 +431,7 @@ int rewriteAppendOnlyFile(char *filename) { FILE *fp; char tmpfile[256]; int j; - time_t now = time(NULL); + long long now = mstime(); /* Note that we have to use a different temp name here compared to the * one used by rewriteAppendOnlyFileBackground() function. */ @@ -441,10 +462,10 @@ int rewriteAppendOnlyFile(char *filename) { while((de = dictNext(di)) != NULL) { sds keystr; robj key, *o; - time_t expiretime; + long long expiretime; - keystr = dictGetEntryKey(de); - o = dictGetEntryVal(de); + keystr = dictGetKey(de); + o = dictGetVal(de); initStaticStringObject(key,keystr); expiretime = getExpire(db,&key); @@ -511,7 +532,7 @@ int rewriteAppendOnlyFile(char *filename) { dictIterator *di = dictGetIterator(o->ptr); dictEntry *de; while((de = dictNext(di)) != NULL) { - robj *eleobj = dictGetEntryKey(de); + robj *eleobj = dictGetKey(de); if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr; if (rioWriteBulkObject(&aof,&key) == 0) goto werr; if (rioWriteBulkObject(&aof,eleobj) == 0) goto werr; @@ -559,8 +580,8 @@ int rewriteAppendOnlyFile(char *filename) { dictEntry *de; while((de = dictNext(di)) != NULL) { - robj *eleobj = dictGetEntryKey(de); - double *score = dictGetEntryVal(de); + robj *eleobj = dictGetKey(de); + double *score = dictGetVal(de); if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr; if (rioWriteBulkObject(&aof,&key) == 0) goto werr; @@ -593,8 +614,8 @@ int rewriteAppendOnlyFile(char *filename) { dictEntry *de; while((de = dictNext(di)) != NULL) { - robj *field = dictGetEntryKey(de); - robj *val = dictGetEntryVal(de); + robj *field = dictGetKey(de); + robj *val = dictGetVal(de); if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr; if (rioWriteBulkObject(&aof,&key) == 0) goto werr; @@ -608,7 +629,7 @@ int rewriteAppendOnlyFile(char *filename) { } /* Save the expire time */ if (expiretime != -1) { - char cmd[]="*3\r\n$8\r\nEXPIREAT\r\n"; + char cmd[]="*3\r\n$9\r\nPEXPIREAT\r\n"; /* If this key is already expired skip it */ if (expiretime < now) continue; if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr; diff --git a/src/cluster.c b/src/cluster.c index fadce42e..4ccff657 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -377,7 +377,7 @@ clusterNode *clusterLookupNode(char *name) { de = dictFind(server.cluster.nodes,s); sdsfree(s); if (de == NULL) return NULL; - return dictGetEntryVal(de); + return dictGetVal(de); } /* This is only used after the handshake. When we connect a given IP/PORT @@ -793,7 +793,7 @@ void clusterBroadcastMessage(void *buf, size_t len) { di = dictGetIterator(server.cluster.nodes); while((de = dictNext(di)) != NULL) { - clusterNode *node = dictGetEntryVal(de); + clusterNode *node = dictGetVal(de); if (!node->link) continue; if (node->flags & (REDIS_NODE_MYSELF|REDIS_NODE_NOADDR)) continue; @@ -849,7 +849,7 @@ void clusterSendPing(clusterLink *link, int type) { /* Populate the gossip fields */ while(freshnodes > 0 && gossipcount < 3) { struct dictEntry *de = dictGetRandomKey(server.cluster.nodes); - clusterNode *this = dictGetEntryVal(de); + clusterNode *this = dictGetVal(de); clusterMsgDataGossip *gossip; int j; @@ -970,7 +970,7 @@ void clusterCron(void) { /* Check if we have disconnected nodes and reestablish the connection. */ di = dictGetIterator(server.cluster.nodes); while((de = dictNext(di)) != NULL) { - clusterNode *node = dictGetEntryVal(de); + clusterNode *node = dictGetVal(de); if (node->flags & (REDIS_NODE_MYSELF|REDIS_NODE_NOADDR)) continue; if (node->link == NULL) { @@ -1005,7 +1005,7 @@ void clusterCron(void) { * the oldest ping_sent time */ for (j = 0; j < 5; j++) { de = dictGetRandomKey(server.cluster.nodes); - clusterNode *this = dictGetEntryVal(de); + clusterNode *this = dictGetVal(de); if (this->link == NULL) continue; if (this->flags & (REDIS_NODE_MYSELF|REDIS_NODE_HANDSHAKE)) continue; @@ -1022,7 +1022,7 @@ void clusterCron(void) { /* Iterate nodes to check if we need to flag something as failing */ di = dictGetIterator(server.cluster.nodes); while((de = dictNext(di)) != NULL) { - clusterNode *node = dictGetEntryVal(de); + clusterNode *node = dictGetVal(de); int delay; if (node->flags & @@ -1153,7 +1153,7 @@ sds clusterGenNodesDescription(void) { di = dictGetIterator(server.cluster.nodes); while((de = dictNext(di)) != NULL) { - clusterNode *node = dictGetEntryVal(de); + clusterNode *node = dictGetVal(de); /* Node coordinates */ ci = sdscatprintf(ci,"%.40s %s:%d ", diff --git a/src/db.c b/src/db.c index cc9810b6..3135795d 100644 --- a/src/db.c +++ b/src/db.c @@ -1,6 +1,7 @@ #include "redis.h" #include +#include void SlotToKeyAdd(robj *key); void SlotToKeyDel(robj *key); @@ -34,7 +35,7 @@ void SlotToKeyDel(robj *key); robj *lookupKey(redisDb *db, robj *key) { dictEntry *de = dictFind(db->dict,key->ptr); if (de) { - robj *val = dictGetEntryVal(de); + robj *val = dictGetVal(de); /* Update the access time for the aging algorithm. * Don't do it if we have a saving child, as this will trigger @@ -130,7 +131,7 @@ robj *dbRandomKey(redisDb *db) { de = dictGetRandomKey(db->dict); if (de == NULL) return NULL; - key = dictGetEntryKey(de); + key = dictGetKey(de); keyobj = createStringObject(key,sdslen(key)); if (dictFind(db->expires,key)) { if (expireIfNeeded(db,keyobj)) { @@ -282,7 +283,7 @@ void keysCommand(redisClient *c) { di = dictGetIterator(c->db->dict); allkeys = (pattern[0] == '*' && pattern[1] == '\0'); while((de = dictNext(di)) != NULL) { - sds key = dictGetEntryKey(de); + sds key = dictGetKey(de); robj *keyobj; if (allkeys || stringmatchlen(pattern,plen,key,sdslen(key),0)) { @@ -327,14 +328,28 @@ void typeCommand(redisClient *c) { } void shutdownCommand(redisClient *c) { - if (prepareForShutdown() == REDIS_OK) - exit(0); + int flags = 0; + + if (c->argc > 2) { + addReply(c,shared.syntaxerr); + return; + } else if (c->argc == 2) { + if (!strcasecmp(c->argv[1]->ptr,"nosave")) { + flags |= REDIS_SHUTDOWN_NOSAVE; + } else if (!strcasecmp(c->argv[1]->ptr,"save")) { + flags |= REDIS_SHUTDOWN_SAVE; + } else { + addReply(c,shared.syntaxerr); + return; + } + } + if (prepareForShutdown(flags) == REDIS_OK) exit(0); addReplyError(c,"Errors trying to SHUTDOWN. Check logs."); } void renameGenericCommand(redisClient *c, int nx) { robj *o; - time_t expire; + long long expire; /* To use the same key as src and dst is probably an error */ if (sdscmp(c->argv[1]->ptr,c->argv[2]->ptr) == 0) { @@ -432,18 +447,19 @@ int removeExpire(redisDb *db, robj *key) { return dictDelete(db->expires,key->ptr) == DICT_OK; } -void setExpire(redisDb *db, robj *key, time_t when) { - dictEntry *de; +void setExpire(redisDb *db, robj *key, long long when) { + dictEntry *kde, *de; /* Reuse the sds from the main dict in the expire dict */ - de = dictFind(db->dict,key->ptr); - redisAssertWithInfo(NULL,key,de != NULL); - dictReplace(db->expires,dictGetEntryKey(de),(void*)when); + kde = dictFind(db->dict,key->ptr); + redisAssertWithInfo(NULL,key,kde != NULL); + de = dictReplaceRaw(db->expires,dictGetKey(kde)); + dictSetSignedIntegerVal(de,when); } /* Return the expire time of the specified key, or -1 if no expire * is associated with this key (i.e. the key is non volatile) */ -time_t getExpire(redisDb *db, robj *key) { +long long getExpire(redisDb *db, robj *key) { dictEntry *de; /* No expire? return ASAP */ @@ -453,7 +469,7 @@ time_t getExpire(redisDb *db, robj *key) { /* The entry was found in the expire dict, this means it should also * be present in the main dict (safety check). */ redisAssertWithInfo(NULL,key,dictFind(db->dict,key->ptr) != NULL); - return (time_t) dictGetEntryVal(de); + return dictGetSignedIntegerVal(de); } /* Propagate expires into slaves and the AOF file. @@ -481,7 +497,7 @@ void propagateExpire(redisDb *db, robj *key) { } int expireIfNeeded(redisDb *db, robj *key) { - time_t when = getExpire(db,key); + long long when = getExpire(db,key); if (when < 0) return 0; /* No expire for this key */ @@ -500,7 +516,7 @@ int expireIfNeeded(redisDb *db, robj *key) { } /* Return when this key has not expired */ - if (time(NULL) <= when) return 0; + if (mstime() <= when) return 0; /* Delete the key */ server.stat_expiredkeys++; @@ -512,13 +528,24 @@ int expireIfNeeded(redisDb *db, robj *key) { * Expires Commands *----------------------------------------------------------------------------*/ -void expireGenericCommand(redisClient *c, robj *key, robj *param, long offset) { +/* Given an string object return true if it contains exactly the "ms" + * or "MS" string. This is used in order to check if the last argument + * of EXPIRE, EXPIREAT or TTL is "ms" to switch into millisecond input/output */ +int stringObjectEqualsMs(robj *a) { + char *arg = a->ptr; + return tolower(arg[0]) == 'm' && tolower(arg[1]) == 's' && arg[2] == '\0'; +} + +void expireGenericCommand(redisClient *c, long long offset, int unit) { dictEntry *de; - long seconds; + robj *key = c->argv[1], *param = c->argv[2]; + long long milliseconds; - if (getLongFromObjectOrReply(c, param, &seconds, NULL) != REDIS_OK) return; + if (getLongLongFromObjectOrReply(c, param, &milliseconds, NULL) != REDIS_OK) + return; - seconds -= offset; + if (unit == UNIT_SECONDS) milliseconds *= 1000; + milliseconds -= offset; de = dictFind(c->db->dict,key->ptr); if (de == NULL) { @@ -531,7 +558,7 @@ void expireGenericCommand(redisClient *c, robj *key, robj *param, long offset) { * * Instead we take the other branch of the IF statement setting an expire * (possibly in the past) and wait for an explicit DEL from the master. */ - if (seconds <= 0 && !server.loading && !server.masterhost) { + if (milliseconds <= 0 && !server.loading && !server.masterhost) { robj *aux; redisAssertWithInfo(c,key,dbDelete(c->db,key)); @@ -545,7 +572,7 @@ void expireGenericCommand(redisClient *c, robj *key, robj *param, long offset) { addReply(c, shared.cone); return; } else { - time_t when = time(NULL)+seconds; + long long when = mstime()+milliseconds; setExpire(c->db,key,when); addReply(c,shared.cone); signalModifiedKey(c->db,key); @@ -555,22 +582,42 @@ void expireGenericCommand(redisClient *c, robj *key, robj *param, long offset) { } void expireCommand(redisClient *c) { - expireGenericCommand(c,c->argv[1],c->argv[2],0); + expireGenericCommand(c,0,UNIT_SECONDS); } void expireatCommand(redisClient *c) { - expireGenericCommand(c,c->argv[1],c->argv[2],time(NULL)); + expireGenericCommand(c,mstime(),UNIT_SECONDS); } -void ttlCommand(redisClient *c) { - time_t expire, ttl = -1; +void pexpireCommand(redisClient *c) { + expireGenericCommand(c,0,UNIT_MILLISECONDS); +} + +void pexpireatCommand(redisClient *c) { + expireGenericCommand(c,mstime(),UNIT_MILLISECONDS); +} + +void ttlGenericCommand(redisClient *c, int output_ms) { + long long expire, ttl = -1; expire = getExpire(c->db,c->argv[1]); if (expire != -1) { - ttl = (expire-time(NULL)); + ttl = expire-mstime(); if (ttl < 0) ttl = -1; } - addReplyLongLong(c,(long long)ttl); + if (ttl == -1) { + addReplyLongLong(c,-1); + } else { + addReplyLongLong(c,output_ms ? ttl : ((ttl+500)/1000)); + } +} + +void ttlCommand(redisClient *c) { + ttlGenericCommand(c, 0); +} + +void pttlCommand(redisClient *c) { + ttlGenericCommand(c, 1); } void persistCommand(redisClient *c) { diff --git a/src/debug.c b/src/debug.c index 7751adf9..376e0712 100644 --- a/src/debug.c +++ b/src/debug.c @@ -91,16 +91,16 @@ void computeDatasetDigest(unsigned char *final) { while((de = dictNext(di)) != NULL) { sds key; robj *keyobj, *o; - time_t expiretime; + long long expiretime; memset(digest,0,20); /* This key-val digest */ - key = dictGetEntryKey(de); + key = dictGetKey(de); keyobj = createStringObject(key,sdslen(key)); mixDigest(digest,key,sdslen(key)); /* Make sure the key is loaded if VM is active */ - o = dictGetEntryVal(de); + o = dictGetVal(de); aux = htonl(o->type); mixDigest(digest,&aux,sizeof(aux)); @@ -165,8 +165,8 @@ void computeDatasetDigest(unsigned char *final) { dictEntry *de; while((de = dictNext(di)) != NULL) { - robj *eleobj = dictGetEntryKey(de); - double *score = dictGetEntryVal(de); + robj *eleobj = dictGetKey(de); + double *score = dictGetVal(de); snprintf(buf,sizeof(buf),"%.17g",*score); memset(eledigest,0,20); @@ -244,7 +244,7 @@ void debugCommand(redisClient *c) { addReply(c,shared.nokeyerr); return; } - val = dictGetEntryVal(de); + val = dictGetVal(de); strenc = strEncoding(val->encoding); addReplyStatusFormat(c, diff --git a/src/dict.c b/src/dict.c index 24001fdd..a573bcd6 100644 --- a/src/dict.c +++ b/src/dict.c @@ -258,6 +258,30 @@ static void _dictRehashStep(dict *d) { /* Add an element to the target hash table */ int dictAdd(dict *d, void *key, void *val) +{ + dictEntry *entry = dictAddRaw(d,key); + + if (!entry) return DICT_ERR; + dictSetVal(d, entry, val); + return DICT_OK; +} + +/* Low level add. This function adds the entry but instead of setting + * a value returns the dictEntry structure to the user, that will make + * sure to fill the value field as he wishes. + * + * This function is also directly expoed to user API to be called + * mainly in order to store non-pointers inside the hash value, example: + * + * entry = dictAddRaw(dict,mykey); + * if (entry != NULL) dictSetSignedIntegerVal(entry,1000); + * + * Return values: + * + * If key already exists NULL is returned. + * If key was added, the hash entry is returned to be manipulated by the caller. + */ +dictEntry *dictAddRaw(dict *d, void *key) { int index; dictEntry *entry; @@ -268,9 +292,9 @@ int dictAdd(dict *d, void *key, void *val) /* Get the index of the new element, or -1 if * the element already exists. */ if ((index = _dictKeyIndex(d, key)) == -1) - return DICT_ERR; + return NULL; - /* Allocates the memory and stores key */ + /* Allocate the memory and store the new entry */ ht = dictIsRehashing(d) ? &d->ht[1] : &d->ht[0]; entry = zmalloc(sizeof(*entry)); entry->next = ht->table[index]; @@ -278,9 +302,8 @@ int dictAdd(dict *d, void *key, void *val) ht->used++; /* Set the hash entry fields. */ - dictSetHashKey(d, entry, key); - dictSetHashVal(d, entry, val); - return DICT_OK; + dictSetKey(d, entry, key); + return entry; } /* Add an element, discarding the old if the key already exists. @@ -297,18 +320,29 @@ int dictReplace(dict *d, void *key, void *val) return 1; /* It already exists, get the entry */ entry = dictFind(d, key); - /* Free the old value and set the new one */ /* Set the new value and free the old one. Note that it is important * to do that in this order, as the value may just be exactly the same * as the previous one. In this context, think to reference counting, * you want to increment (set), and then decrement (free), and not the * reverse. */ auxentry = *entry; - dictSetHashVal(d, entry, val); - dictFreeEntryVal(d, &auxentry); + dictSetVal(d, entry, val); + dictFreeVal(d, &auxentry); return 0; } +/* dictReplaceRaw() is simply a version of dictAddRaw() that always + * returns the hash entry of the specified key, even if the key already + * exists and can't be added (in that case the entry of the already + * existing key is returned.) + * + * See dictAddRaw() for more information. */ +dictEntry *dictReplaceRaw(dict *d, void *key) { + dictEntry *entry = dictFind(d,key); + + return entry ? entry : dictAddRaw(d,key); +} + /* Search and remove an element */ static int dictGenericDelete(dict *d, const void *key, int nofree) { @@ -325,15 +359,15 @@ static int dictGenericDelete(dict *d, const void *key, int nofree) he = d->ht[table].table[idx]; prevHe = NULL; while(he) { - if (dictCompareHashKeys(d, key, he->key)) { + if (dictCompareKeys(d, key, he->key)) { /* Unlink the element from the list */ if (prevHe) prevHe->next = he->next; else d->ht[table].table[idx] = he->next; if (!nofree) { - dictFreeEntryKey(d, he); - dictFreeEntryVal(d, he); + dictFreeKey(d, he); + dictFreeVal(d, he); } zfree(he); d->ht[table].used--; @@ -367,8 +401,8 @@ int _dictClear(dict *d, dictht *ht) if ((he = ht->table[i]) == NULL) continue; while(he) { nextHe = he->next; - dictFreeEntryKey(d, he); - dictFreeEntryVal(d, he); + dictFreeKey(d, he); + dictFreeVal(d, he); zfree(he); ht->used--; he = nextHe; @@ -401,7 +435,7 @@ dictEntry *dictFind(dict *d, const void *key) idx = h & d->ht[table].sizemask; he = d->ht[table].table[idx]; while(he) { - if (dictCompareHashKeys(d, key, he->key)) + if (dictCompareKeys(d, key, he->key)) return he; he = he->next; } @@ -414,7 +448,7 @@ void *dictFetchValue(dict *d, const void *key) { dictEntry *he; he = dictFind(d,key); - return he ? dictGetEntryVal(he) : NULL; + return he ? dictGetVal(he) : NULL; } dictIterator *dictGetIterator(dict *d) @@ -573,7 +607,7 @@ static int _dictKeyIndex(dict *d, const void *key) /* Search if this slot does not already contain the given key */ he = d->ht[table].table[idx]; while(he) { - if (dictCompareHashKeys(d, key, he->key)) + if (dictCompareKeys(d, key, he->key)) return -1; he = he->next; } diff --git a/src/dict.h b/src/dict.h index 74bcd2aa..76451047 100644 --- a/src/dict.h +++ b/src/dict.h @@ -33,6 +33,8 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#include + #ifndef __DICT_H #define __DICT_H @@ -44,7 +46,11 @@ typedef struct dictEntry { void *key; - void *val; + union { + void *val; + uint64_t u64; + int64_t s64; + } v; struct dictEntry *next; } dictEntry; @@ -88,37 +94,44 @@ typedef struct dictIterator { #define DICT_HT_INITIAL_SIZE 4 /* ------------------------------- Macros ------------------------------------*/ -#define dictFreeEntryVal(d, entry) \ +#define dictFreeVal(d, entry) \ if ((d)->type->valDestructor) \ - (d)->type->valDestructor((d)->privdata, (entry)->val) + (d)->type->valDestructor((d)->privdata, (entry)->v.val) -#define dictSetHashVal(d, entry, _val_) do { \ +#define dictSetVal(d, entry, _val_) do { \ if ((d)->type->valDup) \ - entry->val = (d)->type->valDup((d)->privdata, _val_); \ + entry->v.val = (d)->type->valDup((d)->privdata, _val_); \ else \ - entry->val = (_val_); \ + entry->v.val = (_val_); \ } while(0) -#define dictFreeEntryKey(d, entry) \ +#define dictSetSignedIntegerVal(entry, _val_) \ + do { entry->v.s64 = _val_; } while(0) + +#define dictSetUnsignedIntegerVal(entry, _val_) \ + do { entry->v.u64 = _val_; } while(0) + +#define dictFreeKey(d, entry) \ if ((d)->type->keyDestructor) \ (d)->type->keyDestructor((d)->privdata, (entry)->key) -#define dictSetHashKey(d, entry, _key_) do { \ +#define dictSetKey(d, entry, _key_) do { \ if ((d)->type->keyDup) \ entry->key = (d)->type->keyDup((d)->privdata, _key_); \ else \ entry->key = (_key_); \ } while(0) -#define dictCompareHashKeys(d, key1, key2) \ +#define dictCompareKeys(d, key1, key2) \ (((d)->type->keyCompare) ? \ (d)->type->keyCompare((d)->privdata, key1, key2) : \ (key1) == (key2)) #define dictHashKey(d, key) (d)->type->hashFunction(key) - -#define dictGetEntryKey(he) ((he)->key) -#define dictGetEntryVal(he) ((he)->val) +#define dictGetKey(he) ((he)->key) +#define dictGetVal(he) ((he)->v.val) +#define dictGetSignedIntegerVal(he) ((he)->v.s64) +#define dictGetUnsignedIntegerVal(he) ((he)->v.u64) #define dictSlots(d) ((d)->ht[0].size+(d)->ht[1].size) #define dictSize(d) ((d)->ht[0].used+(d)->ht[1].used) #define dictIsRehashing(ht) ((ht)->rehashidx != -1) @@ -127,7 +140,9 @@ typedef struct dictIterator { dict *dictCreate(dictType *type, void *privDataPtr); int dictExpand(dict *d, unsigned long size); int dictAdd(dict *d, void *key, void *val); +dictEntry *dictAddRaw(dict *d, void *key); int dictReplace(dict *d, void *key, void *val); +dictEntry *dictReplaceRaw(dict *d, void *key); int dictDelete(dict *d, const void *key); int dictDeleteNoFree(dict *d, const void *key); void dictRelease(dict *d); diff --git a/src/networking.c b/src/networking.c index 862e69f4..edd7891d 100644 --- a/src/networking.c +++ b/src/networking.c @@ -767,6 +767,17 @@ int processMultibulkBuffer(redisClient *c) { } pos += newline-(c->querybuf+pos)+2; + if (ll >= REDIS_MBULK_BIG_ARG) { + /* If we are going to read a large object from network + * try to make it likely that it will start at c->querybuf + * boundary so that we can optimized object creation + * avoiding a large copy of data. */ + c->querybuf = sdsrange(c->querybuf,pos,-1); + pos = 0; + /* Hint the sds library about the amount of bytes this string is + * going to contain. */ + c->querybuf = sdsMakeRoomFor(c->querybuf,ll+2); + } c->bulklen = ll; } @@ -775,15 +786,32 @@ int processMultibulkBuffer(redisClient *c) { /* Not enough data (+2 == trailing \r\n) */ break; } else { - c->argv[c->argc++] = createStringObject(c->querybuf+pos,c->bulklen); - pos += c->bulklen+2; + /* Optimization: if the buffer contanins JUST our bulk element + * instead of creating a new object by *copying* the sds we + * just use the current sds string. */ + if (pos == 0 && + c->bulklen >= REDIS_MBULK_BIG_ARG && + (signed) sdslen(c->querybuf) == c->bulklen+2) + { + c->argv[c->argc++] = createObject(REDIS_STRING,c->querybuf); + sdsIncrLen(c->querybuf,-2); /* remove CRLF */ + c->querybuf = sdsempty(); + /* Assume that if we saw a fat argument we'll see another one + * likely... */ + c->querybuf = sdsMakeRoomFor(c->querybuf,c->bulklen+2); + pos = 0; + } else { + c->argv[c->argc++] = + createStringObject(c->querybuf+pos,c->bulklen); + pos += c->bulklen+2; + } c->bulklen = -1; c->multibulklen--; } } /* Trim to pos */ - c->querybuf = sdsrange(c->querybuf,pos,-1); + if (pos) c->querybuf = sdsrange(c->querybuf,pos,-1); /* We're done when c->multibulk == 0 */ if (c->multibulklen == 0) { @@ -833,12 +861,29 @@ void processInputBuffer(redisClient *c) { void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { redisClient *c = (redisClient*) privdata; - char buf[REDIS_IOBUF_LEN]; - int nread; + int nread, readlen; + size_t qblen; REDIS_NOTUSED(el); REDIS_NOTUSED(mask); - nread = read(fd, buf, REDIS_IOBUF_LEN); + readlen = REDIS_IOBUF_LEN; + /* If this is a multi bulk request, and we are processing a bulk reply + * that is large enough, try to maximize the probabilty that the query + * buffer contains excatly the SDS string representing the object, even + * at the risk of requring more read(2) calls. This way the function + * processMultiBulkBuffer() can avoid copying buffers to create the + * Redis Object representing the argument. */ + if (c->reqtype == REDIS_REQ_MULTIBULK && c->multibulklen && c->bulklen != -1 + && c->bulklen >= REDIS_MBULK_BIG_ARG) + { + int remaining = (unsigned)(c->bulklen+2)-sdslen(c->querybuf); + + if (remaining < readlen) readlen = remaining; + } + + qblen = sdslen(c->querybuf); + c->querybuf = sdsMakeRoomFor(c->querybuf, readlen); + nread = read(fd, c->querybuf+qblen, readlen); if (nread == -1) { if (errno == EAGAIN) { nread = 0; @@ -853,7 +898,7 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { return; } if (nread) { - c->querybuf = sdscatlen(c->querybuf,buf,nread); + sdsIncrLen(c->querybuf,nread); c->lastinteraction = time(NULL); } else { return; diff --git a/src/object.c b/src/object.c index 23462a5b..0711afed 100644 --- a/src/object.c +++ b/src/object.c @@ -1,5 +1,6 @@ #include "redis.h" #include +#include robj *createObject(int type, void *ptr) { robj *o = zmalloc(sizeof(*o)); @@ -44,6 +45,21 @@ robj *createStringObjectFromLongLong(long long value) { return o; } +/* Note: this function is defined into object.c since here it is where it + * belongs but it is actually designed to be used just for INCRBYFLOAT */ +robj *createStringObjectFromLongDouble(long double value) { + char buf[256]; + int len; + + /* We use 17 digits precision since with 128 bit floats that precision + * after rouding is able to represent most small decimal numbers in a way + * that is "non surprising" for the user (that is, most small decimal + * numbers will be represented in a way that when converted back into + * a string are exactly the same as what the user typed.) */ + len = snprintf(buf,sizeof(buf),"%.17Lg", value); + return createStringObject(buf,len); +} + robj *dupStringObject(robj *o) { redisAssertWithInfo(NULL,o,o->encoding == REDIS_ENCODING_RAW); return createStringObject(o->ptr,sdslen(o->ptr)); @@ -350,15 +366,17 @@ int getDoubleFromObject(robj *o, double *target) { } else { redisAssertWithInfo(NULL,o,o->type == REDIS_STRING); if (o->encoding == REDIS_ENCODING_RAW) { + errno = 0; value = strtod(o->ptr, &eptr); - if (eptr[0] != '\0' || isnan(value)) return REDIS_ERR; + if (isspace(((char*)o->ptr)[0]) || eptr[0] != '\0' || + errno == ERANGE || isnan(value)) + return REDIS_ERR; } else if (o->encoding == REDIS_ENCODING_INT) { value = (long)o->ptr; } else { redisPanic("Unknown string encoding"); } } - *target = value; return REDIS_OK; } @@ -369,11 +387,48 @@ int getDoubleFromObjectOrReply(redisClient *c, robj *o, double *target, const ch if (msg != NULL) { addReplyError(c,(char*)msg); } else { - addReplyError(c,"value is not a double"); + addReplyError(c,"value is not a valid float"); } return REDIS_ERR; } + *target = value; + return REDIS_OK; +} + +int getLongDoubleFromObject(robj *o, long double *target) { + long double value; + char *eptr; + if (o == NULL) { + value = 0; + } else { + redisAssertWithInfo(NULL,o,o->type == REDIS_STRING); + if (o->encoding == REDIS_ENCODING_RAW) { + errno = 0; + value = strtold(o->ptr, &eptr); + if (isspace(((char*)o->ptr)[0]) || eptr[0] != '\0' || + errno == ERANGE || isnan(value)) + return REDIS_ERR; + } else if (o->encoding == REDIS_ENCODING_INT) { + value = (long)o->ptr; + } else { + redisPanic("Unknown string encoding"); + } + } + *target = value; + return REDIS_OK; +} + +int getLongDoubleFromObjectOrReply(redisClient *c, robj *o, long double *target, const char *msg) { + long double value; + if (getLongDoubleFromObject(o, &value) != REDIS_OK) { + if (msg != NULL) { + addReplyError(c,(char*)msg); + } else { + addReplyError(c,"value is not a valid float"); + } + return REDIS_ERR; + } *target = value; return REDIS_OK; } @@ -387,9 +442,10 @@ int getLongLongFromObject(robj *o, long long *target) { } else { redisAssertWithInfo(NULL,o,o->type == REDIS_STRING); if (o->encoding == REDIS_ENCODING_RAW) { + errno = 0; value = strtoll(o->ptr, &eptr, 10); - if (eptr[0] != '\0') return REDIS_ERR; - if (errno == ERANGE && (value == LLONG_MIN || value == LLONG_MAX)) + if (isspace(((char*)o->ptr)[0]) || eptr[0] != '\0' || + errno == ERANGE) return REDIS_ERR; } else if (o->encoding == REDIS_ENCODING_INT) { value = (long)o->ptr; @@ -397,7 +453,6 @@ int getLongLongFromObject(robj *o, long long *target) { redisPanic("Unknown string encoding"); } } - if (target) *target = value; return REDIS_OK; } @@ -412,7 +467,6 @@ int getLongLongFromObjectOrReply(redisClient *c, robj *o, long long *target, con } return REDIS_ERR; } - *target = value; return REDIS_OK; } @@ -429,7 +483,6 @@ int getLongFromObjectOrReply(redisClient *c, robj *o, long *target, const char * } return REDIS_ERR; } - *target = value; return REDIS_OK; } @@ -465,7 +518,7 @@ robj *objectCommandLookup(redisClient *c, robj *key) { dictEntry *de; if ((de = dictFind(c->db->dict,key->ptr)) == NULL) return NULL; - return (robj*) dictGetEntryVal(de); + return (robj*) dictGetVal(de); } robj *objectCommandLookupOrReply(redisClient *c, robj *key, robj *reply) { diff --git a/src/pubsub.c b/src/pubsub.c index af37df5c..27e6f9a5 100644 --- a/src/pubsub.c +++ b/src/pubsub.c @@ -36,7 +36,7 @@ int pubsubSubscribeChannel(redisClient *c, robj *channel) { dictAdd(server.pubsub_channels,channel,clients); incrRefCount(channel); } else { - clients = dictGetEntryVal(de); + clients = dictGetVal(de); } listAddNodeTail(clients,c); } @@ -64,7 +64,7 @@ int pubsubUnsubscribeChannel(redisClient *c, robj *channel, int notify) { /* Remove the client from the channel -> clients list hash table */ de = dictFind(server.pubsub_channels,channel); redisAssertWithInfo(c,NULL,de != NULL); - clients = dictGetEntryVal(de); + clients = dictGetVal(de); ln = listSearchKey(clients,c); redisAssertWithInfo(c,NULL,ln != NULL); listDelNode(clients,ln); @@ -146,7 +146,7 @@ int pubsubUnsubscribeAllChannels(redisClient *c, int notify) { int count = 0; while((de = dictNext(di)) != NULL) { - robj *channel = dictGetEntryKey(de); + robj *channel = dictGetKey(de); count += pubsubUnsubscribeChannel(c,channel,notify); } @@ -180,7 +180,7 @@ int pubsubPublishMessage(robj *channel, robj *message) { /* Send to clients listening for that channel */ de = dictFind(server.pubsub_channels,channel); if (de) { - list *list = dictGetEntryVal(de); + list *list = dictGetVal(de); listNode *ln; listIter li; diff --git a/src/rdb.c b/src/rdb.c index d2d54807..2c0feb6d 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -36,6 +36,17 @@ time_t rdbLoadTime(rio *rdb) { return (time_t)t32; } +int rdbSaveMillisecondTime(rio *rdb, long long t) { + int64_t t64 = (int64_t) t; + return rdbWriteRaw(rdb,&t64,8); +} + +long long rdbLoadMillisecondTime(rio *rdb) { + int64_t t64; + if (rioRead(rdb,&t64,8) == 0) return -1; + return (long long)t64; +} + /* Saves an encoded length. The first two bits in the first byte are used to * hold the encoding type. See the REDIS_RDB_* definitions for more information * on the types of encoding. */ @@ -476,7 +487,7 @@ int rdbSaveObject(rio *rdb, robj *o) { nwritten += n; while((de = dictNext(di)) != NULL) { - robj *eleobj = dictGetEntryKey(de); + robj *eleobj = dictGetKey(de); if ((n = rdbSaveStringObject(rdb,eleobj)) == -1) return -1; nwritten += n; } @@ -505,8 +516,8 @@ int rdbSaveObject(rio *rdb, robj *o) { nwritten += n; while((de = dictNext(di)) != NULL) { - robj *eleobj = dictGetEntryKey(de); - double *score = dictGetEntryVal(de); + robj *eleobj = dictGetKey(de); + double *score = dictGetVal(de); if ((n = rdbSaveStringObject(rdb,eleobj)) == -1) return -1; nwritten += n; @@ -532,8 +543,8 @@ int rdbSaveObject(rio *rdb, robj *o) { nwritten += n; while((de = dictNext(di)) != NULL) { - robj *key = dictGetEntryKey(de); - robj *val = dictGetEntryVal(de); + robj *key = dictGetKey(de); + robj *val = dictGetVal(de); if ((n = rdbSaveStringObject(rdb,key)) == -1) return -1; nwritten += n; @@ -563,14 +574,14 @@ off_t rdbSavedObjectLen(robj *o) { * On success if the key was actaully saved 1 is returned, otherwise 0 * is returned (the key was already expired). */ int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, - time_t expiretime, time_t now) + long long expiretime, long long now) { /* Save the expire time */ if (expiretime != -1) { /* If this key is already expired skip it */ if (expiretime < now) return 0; - if (rdbSaveType(rdb,REDIS_RDB_OPCODE_EXPIRETIME) == -1) return -1; - if (rdbSaveTime(rdb,expiretime) == -1) return -1; + if (rdbSaveType(rdb,REDIS_RDB_OPCODE_EXPIRETIME_MS) == -1) return -1; + if (rdbSaveMillisecondTime(rdb,expiretime) == -1) return -1; } /* Save type, key, value */ @@ -586,7 +597,7 @@ int rdbSave(char *filename) { dictEntry *de; char tmpfile[256]; int j; - time_t now = time(NULL); + long long now = mstime(); FILE *fp; rio rdb; @@ -599,7 +610,7 @@ int rdbSave(char *filename) { } rioInitWithFile(&rdb,fp); - if (rdbWriteRaw(&rdb,"REDIS0002",9) == -1) goto werr; + if (rdbWriteRaw(&rdb,"REDIS0003",9) == -1) goto werr; for (j = 0; j < server.dbnum; j++) { redisDb *db = server.db+j; @@ -617,9 +628,9 @@ int rdbSave(char *filename) { /* Iterate this DB writing every entry */ while((de = dictNext(di)) != NULL) { - sds keystr = dictGetEntryKey(de); - robj key, *o = dictGetEntryVal(de); - time_t expire; + sds keystr = dictGetKey(de); + robj key, *o = dictGetVal(de); + long long expire; initStaticStringObject(key,keystr); expire = getExpire(db,&key); @@ -942,7 +953,7 @@ int rdbLoad(char *filename) { int type, rdbver; redisDb *db = server.db+0; char buf[1024]; - time_t expiretime, now = time(NULL); + long long expiretime, now = mstime(); long loops = 0; FILE *fp; rio rdb; @@ -962,7 +973,7 @@ int rdbLoad(char *filename) { return REDIS_ERR; } rdbver = atoi(buf+5); - if (rdbver < 1 || rdbver > 2) { + if (rdbver < 1 || rdbver > 3) { fclose(fp); redisLog(REDIS_WARNING,"Can't handle RDB format version %d",rdbver); errno = EINVAL; @@ -986,6 +997,15 @@ int rdbLoad(char *filename) { if ((expiretime = rdbLoadTime(&rdb)) == -1) goto eoferr; /* We read the time so we need to read the object type again. */ if ((type = rdbLoadType(&rdb)) == -1) goto eoferr; + /* the EXPIRETIME opcode specifies time in seconds, so convert + * into milliesconds. */ + expiretime *= 1000; + } else if (type == REDIS_RDB_OPCODE_EXPIRETIME_MS) { + /* Milliseconds precision expire times introduced with RDB + * version 3. */ + if ((expiretime = rdbLoadMillisecondTime(&rdb)) == -1) goto eoferr; + /* We read the time so we need to read the object type again. */ + if ((type = rdbLoadType(&rdb)) == -1) goto eoferr; } if (type == REDIS_RDB_OPCODE_EOF) diff --git a/src/rdb.h b/src/rdb.h index fec16ffb..827947b4 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -52,6 +52,7 @@ #define rdbIsObjectType(t) ((t >= 0 && t <= 4) || (t >= 9 && t <= 12)) /* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */ +#define REDIS_RDB_OPCODE_EXPIRETIME_MS 252 #define REDIS_RDB_OPCODE_EXPIRETIME 253 #define REDIS_RDB_OPCODE_SELECTDB 254 #define REDIS_RDB_OPCODE_EOF 255 @@ -76,7 +77,7 @@ off_t rdbSavedObjectLen(robj *o); off_t rdbSavedObjectPages(robj *o); robj *rdbLoadObject(int type, rio *rdb); void backgroundSaveDoneHandler(int exitcode, int bysignal); -int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, time_t expireitme, time_t now); +int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime, long long now); robj *rdbLoadStringObject(rio *rdb); #endif diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index e4a40e13..b22322f4 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -68,8 +68,10 @@ static struct config { const char *title; list *clients; int quiet; + int csv; int loop; int idlemode; + char *tests; } config; typedef struct _client { @@ -295,7 +297,7 @@ static void showLatencyReport(void) { float perc, reqpersec; reqpersec = (float)config.requests_finished/((float)config.totlatency/1000); - if (!config.quiet) { + if (!config.quiet && !config.csv) { printf("====== %s ======\n", config.title); printf(" %d requests completed in %.2f seconds\n", config.requests_finished, (float)config.totlatency/1000); @@ -313,6 +315,8 @@ static void showLatencyReport(void) { } } printf("%.2f requests per second\n\n", reqpersec); + } else if (config.csv) { + printf("\"%s\",\"%.2f\"\n", config.title, reqpersec); } else { printf("%s: %.2f requests per second\n", config.title, reqpersec); } @@ -367,7 +371,7 @@ int parseOptions(int argc, const char **argv) { if (lastarg) goto invalid; config.datasize = atoi(argv[++i]); if (config.datasize < 1) config.datasize=1; - if (config.datasize > 1024*1024) config.datasize = 1024*1024; + if (config.datasize > 1024*1024*1024) config.datasize = 1024*1024*1024; } else if (!strcmp(argv[i],"-r")) { if (lastarg) goto invalid; config.randomkeys = 1; @@ -376,10 +380,23 @@ int parseOptions(int argc, const char **argv) { config.randomkeys_keyspacelen = 0; } else if (!strcmp(argv[i],"-q")) { config.quiet = 1; + } else if (!strcmp(argv[i],"--csv")) { + config.csv = 1; } else if (!strcmp(argv[i],"-l")) { config.loop = 1; } else if (!strcmp(argv[i],"-I")) { config.idlemode = 1; + } else if (!strcmp(argv[i],"-t")) { + if (lastarg) goto invalid; + /* We get the list of tests to run as a string in the form + * get,set,lrange,...,test_N. Then we add a comma before and + * after the string in order to make sure that searching + * for ",testname," will always get a match if the test is + * enabled. */ + config.tests = sdsnew(","); + config.tests = sdscat(config.tests,(char*)argv[++i]); + config.tests = sdscat(config.tests,","); + sdstolower(config.tests); } else if (!strcmp(argv[i],"--help")) { exit_status = 0; goto usage; @@ -398,24 +415,38 @@ invalid: printf("Invalid option \"%s\" or option argument missing\n\n",argv[i]); usage: - printf("Usage: redis-benchmark [-h ] [-p ] [-c ] [-n [-k ]\n\n"); - printf(" -h Server hostname (default 127.0.0.1)\n"); - printf(" -p Server port (default 6379)\n"); - printf(" -s Server socket (overrides host and port)\n"); - printf(" -c Number of parallel connections (default 50)\n"); - printf(" -n Total number of requests (default 10000)\n"); - printf(" -d Data size of SET/GET value in bytes (default 2)\n"); - printf(" -k 1=keep alive 0=reconnect (default 1)\n"); - printf(" -r Use random keys for SET/GET/INCR, random values for SADD\n"); - printf(" Using this option the benchmark will get/set keys\n"); - printf(" in the form mykey_rand000000012456 instead of constant\n"); - printf(" keys, the argument determines the max\n"); - printf(" number of values for the random number. For instance\n"); - printf(" if set to 10 only rand000000000000 - rand000000000009\n"); - printf(" range will be allowed.\n"); - printf(" -q Quiet. Just show query/sec values\n"); - printf(" -l Loop. Run the tests forever\n"); - printf(" -I Idle mode. Just open N idle connections and wait.\n"); + printf( +"Usage: redis-benchmark [-h ] [-p ] [-c ] [-n [-k ]\n\n" +" -h Server hostname (default 127.0.0.1)\n" +" -p Server port (default 6379)\n" +" -s Server socket (overrides host and port)\n" +" -c Number of parallel connections (default 50)\n" +" -n Total number of requests (default 10000)\n" +" -d Data size of SET/GET value in bytes (default 2)\n" +" -k 1=keep alive 0=reconnect (default 1)\n" +" -r Use random keys for SET/GET/INCR, random values for SADD\n" +" Using this option the benchmark will get/set keys\n" +" in the form mykey_rand000000012456 instead of constant\n" +" keys, the argument determines the max\n" +" number of values for the random number. For instance\n" +" if set to 10 only rand000000000000 - rand000000000009\n" +" range will be allowed.\n" +" -q Quiet. Just show query/sec values\n" +" --csv Output in CSV format\n" +" -l Loop. Run the tests forever\n" +" -t Only run the comma separated list of tests. The test\n" +" names are the same as the ones produced as output.\n" +" -I Idle mode. Just open N idle connections and wait.\n\n" +"Examples:\n\n" +" Run the benchmark with the default configuration against 127.0.0.1:6379:\n" +" $ redis-benchmark\n\n" +" Use 20 parallel clients, for a total of 100k requests, against 192.168.1.1:\n" +" $ redis-benchmark -h 192.168.1.1 -p 6379 -n 100000 -c 20\n\n" +" Fill 127.0.0.1:6379 with about 1 million keys only using the SET test:\n" +" $ redis-benchmark -t set -n 1000000 -r 100000000\n\n" +" Benchmark 127.0.0.1:6379 for a few commands producing CSV output:\n" +" $ redis-benchmark -t ping,set,get -n 100000 --csv\n\n" + ); exit(exit_status); } @@ -424,6 +455,7 @@ int showThroughput(struct aeEventLoop *eventLoop, long long id, void *clientData REDIS_NOTUSED(id); REDIS_NOTUSED(clientData); + if (config.csv) return 250; float dt = (float)(mstime()-config.start)/1000.0; float rps = (float)config.requests_finished/dt; printf("%s: %.2f\r", config.title, rps); @@ -431,6 +463,20 @@ int showThroughput(struct aeEventLoop *eventLoop, long long id, void *clientData return 250; /* every 250ms */ } +/* Return true if the named test was selected using the -t command line + * switch, or if all the tests are selected (no -t passed by user). */ +int test_is_selected(char *name) { + char buf[256]; + int l = strlen(name); + + if (config.tests == NULL) return 1; + buf[0] = ','; + memcpy(buf+1,name,l); + buf[l+1] = ','; + buf[l+2] = '\0'; + return strstr(config.tests,buf) != NULL; +} + int main(int argc, const char **argv) { int i; char *data, *cmd; @@ -451,6 +497,7 @@ int main(int argc, const char **argv) { config.randomkeys = 0; config.randomkeys_keyspacelen = 0; config.quiet = 0; + config.csv = 0; config.loop = 0; config.idlemode = 0; config.latency = NULL; @@ -458,6 +505,7 @@ int main(int argc, const char **argv) { config.hostip = "127.0.0.1"; config.hostport = 6379; config.hostsocket = NULL; + config.tests = NULL; i = parseOptions(argc,argv); argc -= i; @@ -500,71 +548,106 @@ int main(int argc, const char **argv) { memset(data,'x',config.datasize); data[config.datasize] = '\0'; - benchmark("PING (inline)","PING\r\n",6); + if (test_is_selected("ping_inline") || test_is_selected("ping")) + benchmark("PING_INLINE","PING\r\n",6); - len = redisFormatCommand(&cmd,"PING"); - benchmark("PING",cmd,len); - free(cmd); + if (test_is_selected("ping_mbulk") || test_is_selected("ping")) { + len = redisFormatCommand(&cmd,"PING"); + benchmark("PING_BULK",cmd,len); + free(cmd); + } - const char *argv[21]; - argv[0] = "MSET"; - for (i = 1; i < 21; i += 2) { - argv[i] = "foo:rand:000000000000"; - argv[i+1] = data; + if (test_is_selected("set")) { + len = redisFormatCommand(&cmd,"SET foo:rand:000000000000 %s",data); + benchmark("SET",cmd,len); + free(cmd); } - len = redisFormatCommandArgv(&cmd,21,argv,NULL); - benchmark("MSET (10 keys)",cmd,len); - free(cmd); - len = redisFormatCommand(&cmd,"SET foo:rand:000000000000 %s",data); - benchmark("SET",cmd,len); - free(cmd); + if (test_is_selected("get")) { + len = redisFormatCommand(&cmd,"GET foo:rand:000000000000"); + benchmark("GET",cmd,len); + free(cmd); + } - len = redisFormatCommand(&cmd,"GET foo:rand:000000000000"); - benchmark("GET",cmd,len); - free(cmd); + if (test_is_selected("incr")) { + len = redisFormatCommand(&cmd,"INCR counter:rand:000000000000"); + benchmark("INCR",cmd,len); + free(cmd); + } - len = redisFormatCommand(&cmd,"INCR counter:rand:000000000000"); - benchmark("INCR",cmd,len); - free(cmd); + if (test_is_selected("lpush")) { + len = redisFormatCommand(&cmd,"LPUSH mylist %s",data); + benchmark("LPUSH",cmd,len); + free(cmd); + } - len = redisFormatCommand(&cmd,"LPUSH mylist %s",data); - benchmark("LPUSH",cmd,len); - free(cmd); + if (test_is_selected("lpop")) { + len = redisFormatCommand(&cmd,"LPOP mylist"); + benchmark("LPOP",cmd,len); + free(cmd); + } - len = redisFormatCommand(&cmd,"LPOP mylist"); - benchmark("LPOP",cmd,len); - free(cmd); + if (test_is_selected("sadd")) { + len = redisFormatCommand(&cmd, + "SADD myset counter:rand:000000000000"); + benchmark("SADD",cmd,len); + free(cmd); + } - len = redisFormatCommand(&cmd,"SADD myset counter:rand:000000000000"); - benchmark("SADD",cmd,len); - free(cmd); + if (test_is_selected("spop")) { + len = redisFormatCommand(&cmd,"SPOP myset"); + benchmark("SPOP",cmd,len); + free(cmd); + } - len = redisFormatCommand(&cmd,"SPOP myset"); - benchmark("SPOP",cmd,len); - free(cmd); + if (test_is_selected("lrange") || + test_is_selected("lrange_100") || + test_is_selected("lrange_300") || + test_is_selected("lrange_500") || + test_is_selected("lrange_600")) + { + len = redisFormatCommand(&cmd,"LPUSH mylist %s",data); + benchmark("LPUSH (needed to benchmark LRANGE)",cmd,len); + free(cmd); + } - len = redisFormatCommand(&cmd,"LPUSH mylist %s",data); - benchmark("LPUSH (again, in order to bench LRANGE)",cmd,len); - free(cmd); + if (test_is_selected("lrange") || test_is_selected("lrange_100")) { + len = redisFormatCommand(&cmd,"LRANGE mylist 0 99"); + benchmark("LRANGE_100 (first 100 elements)",cmd,len); + free(cmd); + } - len = redisFormatCommand(&cmd,"LRANGE mylist 0 99"); - benchmark("LRANGE (first 100 elements)",cmd,len); - free(cmd); + if (test_is_selected("lrange") || test_is_selected("lrange_300")) { + len = redisFormatCommand(&cmd,"LRANGE mylist 0 299"); + benchmark("LRANGE_300 (first 300 elements)",cmd,len); + free(cmd); + } - len = redisFormatCommand(&cmd,"LRANGE mylist 0 299"); - benchmark("LRANGE (first 300 elements)",cmd,len); - free(cmd); + if (test_is_selected("lrange") || test_is_selected("lrange_500")) { + len = redisFormatCommand(&cmd,"LRANGE mylist 0 449"); + benchmark("LRANGE_500 (first 450 elements)",cmd,len); + free(cmd); + } - len = redisFormatCommand(&cmd,"LRANGE mylist 0 449"); - benchmark("LRANGE (first 450 elements)",cmd,len); - free(cmd); + if (test_is_selected("lrange") || test_is_selected("lrange_600")) { + len = redisFormatCommand(&cmd,"LRANGE mylist 0 599"); + benchmark("LRANGE_600 (first 600 elements)",cmd,len); + free(cmd); + } - len = redisFormatCommand(&cmd,"LRANGE mylist 0 599"); - benchmark("LRANGE (first 600 elements)",cmd,len); - free(cmd); + if (test_is_selected("mset")) { + const char *argv[21]; + argv[0] = "MSET"; + for (i = 1; i < 21; i += 2) { + argv[i] = "foo:rand:000000000000"; + argv[i+1] = data; + } + len = redisFormatCommandArgv(&cmd,21,argv,NULL); + benchmark("MSET (10 keys)",cmd,len); + free(cmd); + } - printf("\n"); + if (!config.csv) printf("\n"); } while(config.loop); return 0; diff --git a/src/redis-trib.rb b/src/redis-trib.rb index fa0e8c6d..473e4922 100755 --- a/src/redis-trib.rb +++ b/src/redis-trib.rb @@ -319,7 +319,7 @@ class RedisTrib # divisibility. Like we have 3 nodes and need to get 10 slots, we take # 4 from the first, and 3 from the rest. So the biggest is always the first. sources = sources.sort{|a,b| b.slots.length <=> a.slots.length} - source_tot_slots = sources.inject {|a,b| a.slots.length+b.slots.length} + source_tot_slots = sources.inject(0) {|sum,source| sum+source.slots.length} sources.each_with_index{|s,i| # Every node will provide a number of slots proportional to the # slots it has assigned. diff --git a/src/redis.c b/src/redis.c index 832a9f59..91e1e10c 100644 --- a/src/redis.c +++ b/src/redis.c @@ -91,6 +91,7 @@ struct redisCommand redisCommandTable[] = { {"set",setCommand,3,"wm",0,noPreloadGetKeys,1,1,1,0,0}, {"setnx",setnxCommand,3,"wm",0,noPreloadGetKeys,1,1,1,0,0}, {"setex",setexCommand,4,"wm",0,noPreloadGetKeys,2,2,1,0,0}, + {"psetex",psetexCommand,4,"wm",0,noPreloadGetKeys,2,2,1,0,0}, {"append",appendCommand,3,"wm",0,NULL,1,1,1,0,0}, {"strlen",strlenCommand,2,"r",0,NULL,1,1,1,0,0}, {"del",delCommand,-2,"w",0,noPreloadGetKeys,1,-1,1,0,0}, @@ -110,9 +111,9 @@ struct redisCommand redisCommandTable[] = { {"linsert",linsertCommand,5,"wm",0,NULL,1,1,1,0,0}, {"rpop",rpopCommand,2,"w",0,NULL,1,1,1,0,0}, {"lpop",lpopCommand,2,"w",0,NULL,1,1,1,0,0}, - {"brpop",brpopCommand,-3,"w",0,NULL,1,1,1,0,0}, - {"brpoplpush",brpoplpushCommand,4,"wm",0,NULL,1,2,1,0,0}, - {"blpop",blpopCommand,-3,"w",0,NULL,1,-2,1,0,0}, + {"brpop",brpopCommand,-3,"ws",0,NULL,1,1,1,0,0}, + {"brpoplpush",brpoplpushCommand,4,"wms",0,NULL,1,2,1,0,0}, + {"blpop",blpopCommand,-3,"ws",0,NULL,1,-2,1,0,0}, {"llen",llenCommand,2,"r",0,NULL,1,1,1,0,0}, {"lindex",lindexCommand,3,"r",0,NULL,1,1,1,0,0}, {"lset",lsetCommand,4,"wm",0,NULL,1,1,1,0,0}, @@ -156,6 +157,7 @@ struct redisCommand redisCommandTable[] = { {"hmset",hmsetCommand,-4,"wm",0,NULL,1,1,1,0,0}, {"hmget",hmgetCommand,-3,"r",0,NULL,1,1,1,0,0}, {"hincrby",hincrbyCommand,4,"wm",0,NULL,1,1,1,0,0}, + {"hincrbyfloat",hincrbyfloatCommand,4,"wm",0,NULL,1,1,1,0,0}, {"hdel",hdelCommand,-3,"w",0,NULL,1,1,1,0,0}, {"hlen",hlenCommand,2,"r",0,NULL,1,1,1,0,0}, {"hkeys",hkeysCommand,2,"r",0,NULL,1,1,1,0,0}, @@ -164,6 +166,7 @@ struct redisCommand redisCommandTable[] = { {"hexists",hexistsCommand,3,"r",0,NULL,1,1,1,0,0}, {"incrby",incrbyCommand,3,"wm",0,NULL,1,1,1,0,0}, {"decrby",decrbyCommand,3,"wm",0,NULL,1,1,1,0,0}, + {"incrbyfloat",incrbyfloatCommand,3,"wm",0,NULL,1,1,1,0,0}, {"getset",getsetCommand,3,"wm",0,NULL,1,1,1,0,0}, {"mset",msetCommand,-3,"wm",0,NULL,1,-1,2,0,0}, {"msetnx",msetnxCommand,-3,"wm",0,NULL,1,-1,2,0,0}, @@ -174,15 +177,17 @@ struct redisCommand redisCommandTable[] = { {"renamenx",renamenxCommand,3,"w",0,renameGetKeys,1,2,1,0,0}, {"expire",expireCommand,3,"w",0,NULL,1,1,1,0,0}, {"expireat",expireatCommand,3,"w",0,NULL,1,1,1,0,0}, + {"pexpire",pexpireCommand,3,"w",0,NULL,1,1,1,0,0}, + {"pexpireat",pexpireatCommand,3,"w",0,NULL,1,1,1,0,0}, {"keys",keysCommand,2,"r",0,NULL,0,0,0,0,0}, {"dbsize",dbsizeCommand,1,"r",0,NULL,0,0,0,0,0}, - {"auth",authCommand,2,"r",0,NULL,0,0,0,0,0}, + {"auth",authCommand,2,"rs",0,NULL,0,0,0,0,0}, {"ping",pingCommand,1,"r",0,NULL,0,0,0,0,0}, {"echo",echoCommand,2,"r",0,NULL,0,0,0,0,0}, - {"save",saveCommand,1,"ar",0,NULL,0,0,0,0,0}, + {"save",saveCommand,1,"ars",0,NULL,0,0,0,0,0}, {"bgsave",bgsaveCommand,1,"ar",0,NULL,0,0,0,0,0}, {"bgrewriteaof",bgrewriteaofCommand,1,"ar",0,NULL,0,0,0,0,0}, - {"shutdown",shutdownCommand,1,"ar",0,NULL,0,0,0,0,0}, + {"shutdown",shutdownCommand,-1,"ar",0,NULL,0,0,0,0,0}, {"lastsave",lastsaveCommand,1,"r",0,NULL,0,0,0,0,0}, {"type",typeCommand,2,"r",0,NULL,1,1,1,0,0}, {"multi",multiCommand,1,"rs",0,NULL,0,0,0,0,0}, @@ -195,9 +200,10 @@ struct redisCommand redisCommandTable[] = { {"info",infoCommand,-1,"r",0,NULL,0,0,0,0,0}, {"monitor",monitorCommand,1,"ars",0,NULL,0,0,0,0,0}, {"ttl",ttlCommand,2,"r",0,NULL,1,1,1,0,0}, + {"pttl",pttlCommand,2,"r",0,NULL,1,1,1,0,0}, {"persist",persistCommand,2,"w",0,NULL,1,1,1,0,0}, {"slaveof",slaveofCommand,3,"aws",0,NULL,0,0,0,0,0}, - {"debug",debugCommand,-2,"aw",0,NULL,0,0,0,0,0}, + {"debug",debugCommand,-2,"aws",0,NULL,0,0,0,0,0}, {"config",configCommand,-2,"ar",0,NULL,0,0,0,0,0}, {"subscribe",subscribeCommand,-2,"rps",0,NULL,0,0,0,0,0}, {"unsubscribe",unsubscribeCommand,-1,"rps",0,NULL,0,0,0,0,0}, @@ -288,6 +294,11 @@ long long ustime(void) { return ust; } +/* Return the UNIX time in milliseconds */ +long long mstime(void) { + return ustime()/1000; +} + /*====================== Hash table type implementation ==================== */ /* This is an hash table type that uses the SDS dynamic strings libary as @@ -553,19 +564,19 @@ void activeExpireCycle(void) { * of the keys were expired. */ do { long num = dictSize(db->expires); - time_t now = time(NULL); + long long now = mstime(); expired = 0; if (num > REDIS_EXPIRELOOKUPS_PER_CRON) num = REDIS_EXPIRELOOKUPS_PER_CRON; while (num--) { dictEntry *de; - time_t t; + long long t; if ((de = dictGetRandomKey(db->expires)) == NULL) break; - t = (time_t) dictGetEntryVal(de); + t = dictGetSignedIntegerVal(de); if (now > t) { - sds key = dictGetEntryKey(de); + sds key = dictGetKey(de); robj *keyobj = createStringObject(key,sdslen(key)); propagateExpire(db,keyobj); @@ -617,7 +628,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* We received a SIGTERM, shutting down here in a safe way, as it is * not ok doing so inside the signal handler. */ if (server.shutdown_asap) { - if (prepareForShutdown() == REDIS_OK) exit(0); + if (prepareForShutdown(0) == REDIS_OK) exit(0); redisLog(REDIS_WARNING,"SIGTERM received but errors trying to shut down the server, check the logs for more information"); } @@ -794,7 +805,7 @@ void createSharedObjects(void) { shared.loadingerr = createObject(REDIS_STRING,sdsnew( "-LOADING Redis is loading the dataset in memory\r\n")); shared.slowscripterr = createObject(REDIS_STRING,sdsnew( - "-BUSY Redis is busy running a script. Please wait or stop the server with SHUTDOWN.\r\n")); + "-BUSY Redis is busy running a script. You can only call SCRIPT KILL or SHUTDOWN NOSAVE.\r\n")); shared.space = createObject(REDIS_STRING,sdsnew(" ")); shared.colon = createObject(REDIS_STRING,sdsnew(":")); shared.plus = createObject(REDIS_STRING,sdsnew("+")); @@ -873,6 +884,7 @@ void initServerConfig() { server.repl_timeout = REDIS_REPL_TIMEOUT; server.cluster_enabled = 0; server.cluster.configfile = zstrdup("nodes.conf"); + server.lua_caller = NULL; server.lua_time_limit = REDIS_LUA_TIME_LIMIT; server.lua_client = NULL; server.lua_timedout = 0; @@ -1221,8 +1233,15 @@ int processCommand(redisClient *c) { return REDIS_OK; } - /* Lua script too slow? */ - if (server.lua_timedout && c->cmd->proc != shutdownCommand) { + /* Lua script too slow? Only allow SHUTDOWN NOSAVE and SCRIPT KILL. */ + if (server.lua_timedout && + !(c->cmd->proc != shutdownCommand && + c->argc == 2 && + tolower(((char*)c->argv[1]->ptr)[0]) == 'n') && + !(c->cmd->proc == scriptCommand && + c->argc == 2 && + tolower(((char*)c->argv[1]->ptr)[0]) == 'k')) + { addReply(c, shared.slowscripterr); return REDIS_OK; } @@ -1242,7 +1261,10 @@ int processCommand(redisClient *c) { /*================================== Shutdown =============================== */ -int prepareForShutdown() { +int prepareForShutdown(int flags) { + int save = flags & REDIS_SHUTDOWN_SAVE; + int nosave = flags & REDIS_SHUTDOWN_NOSAVE; + redisLog(REDIS_WARNING,"User requested shutdown..."); /* Kill the saving child if there is a background saving in progress. We want to avoid race conditions, for instance our saving child may @@ -1264,7 +1286,7 @@ int prepareForShutdown() { redisLog(REDIS_NOTICE,"Calling fsync() on the AOF file."); aof_fsync(server.appendfd); } - if (server.saveparamslen > 0) { + if ((server.saveparamslen > 0 && !nosave) || save) { redisLog(REDIS_NOTICE,"Saving the final RDB snapshot before exiting."); /* Snapshotting. Perform a SYNC SAVE and exit */ if (rdbSave(server.dbfilename) != REDIS_OK) { @@ -1679,7 +1701,7 @@ void freeMemoryIfNeeded(void) { server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_RANDOM) { de = dictGetRandomKey(dict); - bestkey = dictGetEntryKey(de); + bestkey = dictGetKey(de); } /* volatile-lru and allkeys-lru policy */ @@ -1692,12 +1714,12 @@ void freeMemoryIfNeeded(void) { robj *o; de = dictGetRandomKey(dict); - thiskey = dictGetEntryKey(de); + thiskey = dictGetKey(de); /* When policy is volatile-lru we need an additonal lookup * to locate the real key, as dict is set to db->expires. */ if (server.maxmemory_policy == REDIS_MAXMEMORY_VOLATILE_LRU) de = dictFind(db->dict, thiskey); - o = dictGetEntryVal(de); + o = dictGetVal(de); thisval = estimateObjectIdleTime(o); /* Higher idle time is better candidate for deletion */ @@ -1715,8 +1737,8 @@ void freeMemoryIfNeeded(void) { long thisval; de = dictGetRandomKey(dict); - thiskey = dictGetEntryKey(de); - thisval = (long) dictGetEntryVal(de); + thiskey = dictGetKey(de); + thisval = (long) dictGetVal(de); /* Expire sooner (minor expire unix timestamp) is better * candidate for deletion */ diff --git a/src/redis.h b/src/redis.h index 883db1f5..d532e385 100644 --- a/src/redis.h +++ b/src/redis.h @@ -39,8 +39,8 @@ /* Static server configuration */ #define REDIS_SERVERPORT 6379 /* TCP port */ -#define REDIS_MAXIDLETIME (60*5) /* default client timeout */ -#define REDIS_IOBUF_LEN 1024 +#define REDIS_MAXIDLETIME 0 /* default client timeout: infinite */ +#define REDIS_IOBUF_LEN (1024*16) #define REDIS_LOADBUF_LEN 1024 #define REDIS_DEFAULT_DBNUM 16 #define REDIS_CONFIGLINE_MAX 1024 @@ -49,7 +49,7 @@ #define REDIS_MAX_WRITE_PER_EVENT (1024*64) #define REDIS_REQUEST_MAX_SIZE (1024*1024*256) /* max bytes in inline command */ #define REDIS_SHARED_INTEGERS 10000 -#define REDIS_REPLY_CHUNK_BYTES (5*1500) /* 5 TCP packets with default MTU */ +#define REDIS_REPLY_CHUNK_BYTES (16*1024) /* 16k output buffer */ #define REDIS_MAX_LOGMSG_LEN 1024 /* Default maximum length of syslog messages */ #define REDIS_AUTO_AOFREWRITE_PERC 100 #define REDIS_AUTO_AOFREWRITE_MIN_SIZE (1024*1024) @@ -59,6 +59,7 @@ #define REDIS_REPL_TIMEOUT 60 #define REDIS_REPL_PING_SLAVE_PERIOD 10 +#define REDIS_MBULK_BIG_ARG (1024*32) /* Hash table parameters */ #define REDIS_HT_MINFILL 10 /* Minimal hash table fill 10% */ @@ -94,11 +95,6 @@ #define REDIS_ENCODING_INTSET 6 /* Encoded as intset */ #define REDIS_ENCODING_SKIPLIST 7 /* Encoded as skiplist */ -/* Object types only used for dumping to disk */ -#define REDIS_EXPIRETIME 253 -#define REDIS_SELECTDB 254 -#define REDIS_EOF 255 - /* Defines related to the dump file format. To store 32 bits lengths for short * keys requires a lot of space, so we check the most significant 2 bits of * the first byte to interpreter the length: @@ -215,6 +211,15 @@ /* Scripting */ #define REDIS_LUA_TIME_LIMIT 5000 /* milliseconds */ +/* Units */ +#define UNIT_SECONDS 0 +#define UNIT_MILLISECONDS 1 + +/* SHUTDOWN flags */ +#define REDIS_SHUTDOWN_SAVE 1 /* Force SAVE on SHUTDOWN even if no save + points are configured. */ +#define REDIS_SHUTDOWN_NOSAVE 2 /* Don't SAVE on SHUTDOWN. */ + /* We can print the stacktrace, so our assert is defined this way: */ #define redisAssertWithInfo(_c,_o,_e) ((_e)?(void)0 : (_redisAssertWithInfo(_c,_o,#_e,__FILE__,__LINE__),_exit(1))) #define redisAssert(_e) ((_e)?(void)0 : (_redisAssert(#_e,__FILE__,__LINE__),_exit(1))) @@ -538,7 +543,7 @@ struct redisServer { off_t auto_aofrewrite_base_size;/* AOF size on latest startup or rewrite. */ off_t appendonly_current_size; /* AOF current size. */ int aofrewrite_scheduled; /* Rewrite once BGSAVE terminates. */ - int shutdown_asap; + int shutdown_asap; /* SHUTDOWN needed */ int activerehashing; char *requirepass; /* Persistence */ @@ -615,13 +620,17 @@ struct redisServer { /* Scripting */ lua_State *lua; /* The Lua interpreter. We use just one for all clients */ redisClient *lua_client; /* The "fake client" to query Redis from Lua */ + redisClient *lua_caller; /* The client running EVAL right now, or NULL */ dict *lua_scripts; /* A dictionary of SHA1 -> Lua scripts */ long long lua_time_limit; long long lua_time_start; + int lua_write_dirty; /* True if a write command was called during the + execution of the current script. */ int lua_random_dirty; /* True if a random command was called during the - exection of the current script. */ + execution of the current script. */ int lua_timedout; /* True if we reached the time limit for script execution. */ + int lua_kill; /* Kill the script if true. */ }; typedef struct pubsubPattern { @@ -725,6 +734,7 @@ dictType hashDictType; /* Utils */ long long ustime(void); +long long mstime(void); /* networking.c -- Networking and Client related operations */ redisClient *createClient(int fd); @@ -810,6 +820,7 @@ robj *tryObjectEncoding(robj *o); robj *getDecodedObject(robj *o); size_t stringObjectLen(robj *o); robj *createStringObjectFromLongLong(long long value); +robj *createStringObjectFromLongDouble(long double value); robj *createListObject(void); robj *createZiplistObject(void); robj *createSetObject(void); @@ -822,6 +833,8 @@ int checkType(redisClient *c, robj *o, int type); int getLongLongFromObjectOrReply(redisClient *c, robj *o, long long *target, const char *msg); int getDoubleFromObjectOrReply(redisClient *c, robj *o, double *target, const char *msg); int getLongLongFromObject(robj *o, long long *target); +int getLongDoubleFromObject(robj *o, long double *target); +int getLongDoubleFromObjectOrReply(redisClient *c, robj *o, long double *target, const char *msg); char *strEncoding(int encoding); int compareStringObjects(robj *a, robj *b); int equalStringObjects(robj *a, robj *b); @@ -939,8 +952,8 @@ void resetServerSaveParams(); int removeExpire(redisDb *db, robj *key); void propagateExpire(redisDb *db, robj *key); int expireIfNeeded(redisDb *db, robj *key); -time_t getExpire(redisDb *db, robj *key); -void setExpire(redisDb *db, robj *key, time_t when); +long long getExpire(redisDb *db, robj *key); +void setExpire(redisDb *db, robj *key, long long when); robj *lookupKey(redisDb *db, robj *key); robj *lookupKeyRead(redisDb *db, robj *key); robj *lookupKeyWrite(redisDb *db, robj *key); @@ -991,6 +1004,7 @@ void echoCommand(redisClient *c); void setCommand(redisClient *c); void setnxCommand(redisClient *c); void setexCommand(redisClient *c); +void psetexCommand(redisClient *c); void getCommand(redisClient *c); void delCommand(redisClient *c); void existsCommand(redisClient *c); @@ -1002,6 +1016,7 @@ void incrCommand(redisClient *c); void decrCommand(redisClient *c); void incrbyCommand(redisClient *c); void decrbyCommand(redisClient *c); +void incrbyfloatCommand(redisClient *c); void selectCommand(redisClient *c); void randomkeyCommand(redisClient *c); void keysCommand(redisClient *c); @@ -1051,8 +1066,11 @@ void mgetCommand(redisClient *c); void monitorCommand(redisClient *c); void expireCommand(redisClient *c); void expireatCommand(redisClient *c); +void pexpireCommand(redisClient *c); +void pexpireatCommand(redisClient *c); void getsetCommand(redisClient *c); void ttlCommand(redisClient *c); +void pttlCommand(redisClient *c); void persistCommand(redisClient *c); void slaveofCommand(redisClient *c); void debugCommand(redisClient *c); @@ -1095,6 +1113,7 @@ void hgetallCommand(redisClient *c); void hexistsCommand(redisClient *c); void configCommand(redisClient *c); void hincrbyCommand(redisClient *c); +void hincrbyfloatCommand(redisClient *c); void subscribeCommand(redisClient *c); void unsubscribeCommand(redisClient *c); void psubscribeCommand(redisClient *c); diff --git a/src/scripting.c b/src/scripting.c index 0b548873..1503c3c9 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -187,6 +187,7 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { } if (cmd->flags & REDIS_CMD_RANDOM) server.lua_random_dirty = 1; + if (cmd->flags & REDIS_CMD_WRITE) server.lua_write_dirty = 1; /* Run the command */ cmd->proc(c); @@ -277,11 +278,22 @@ void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { elapsed = (ustime()/1000) - server.lua_time_start; if (elapsed >= server.lua_time_limit && server.lua_timedout == 0) { - redisLog(REDIS_WARNING,"Lua slow script detected: still in execution after %lld milliseconds. You can shut down the server using the SHUTDOWN command.",elapsed); + redisLog(REDIS_WARNING,"Lua slow script detected: still in execution after %lld milliseconds. You can try killing the script using the SCRIPT KILL command.",elapsed); server.lua_timedout = 1; + /* Once the script timeouts we reenter the event loop to permit others + * to call SCRIPT KILL or SHUTDOWN NOSAVE if needed. For this reason + * we need to mask the client executing the script from the event loop. + * If we don't do that the client may disconnect and could no longer be + * here when the EVAL command will return. */ + aeDeleteFileEvent(server.el, server.lua_caller->fd, AE_READABLE); } if (server.lua_timedout) aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT); + if (server.lua_kill) { + redisLog(REDIS_WARNING,"Lua script killed by user with SCRIPT KILL."); + lua_pushstring(lua,"Script killed by user with SCRIPT KILL..."); + lua_error(lua); + } } void luaLoadLib(lua_State *lua, const char *libname, lua_CFunction luafunc) { @@ -553,6 +565,7 @@ void evalGenericCommand(redisClient *c, int evalsha) { * Thanks to this flag we'll raise an error every time a write command * is called after a random command was used. */ server.lua_random_dirty = 0; + server.lua_write_dirty = 0; /* Get the number of arguments that are keys */ if (getLongLongFromObjectOrReply(c,c->argv[2],&numkeys,NULL) != REDIS_OK) @@ -610,7 +623,6 @@ void evalGenericCommand(redisClient *c, int evalsha) { * make the Lua script execution slower. */ if (server.lua_time_limit > 0 && server.masterhost == NULL) { lua_sethook(lua,luaMaskCountHook,LUA_MASKCOUNT,100000); - server.lua_time_start = ustime()/1000; } else { lua_sethook(lua,luaMaskCountHook,0,0); } @@ -618,8 +630,18 @@ void evalGenericCommand(redisClient *c, int evalsha) { /* At this point whatever this script was never seen before or if it was * already defined, we can call it. We have zero arguments and expect * a single return value. */ + server.lua_caller = c; + server.lua_time_start = ustime()/1000; + server.lua_kill = 0; if (lua_pcall(lua,0,1,0)) { - server.lua_timedout = 0; + if (server.lua_timedout) { + server.lua_timedout = 0; + /* Restore the readable handler that was unregistered when the + * script timeout was detected. */ + aeCreateFileEvent(server.el,c->fd,AE_READABLE, + readQueryFromClient,c); + } + server.lua_caller = NULL; selectDb(c,server.lua_client->db->id); /* set DB ID from Lua client */ addReplyErrorFormat(c,"Error running script (call to %s): %s\n", funcname, lua_tostring(lua,-1)); @@ -628,6 +650,7 @@ void evalGenericCommand(redisClient *c, int evalsha) { return; } server.lua_timedout = 0; + server.lua_caller = NULL; selectDb(c,server.lua_client->db->id); /* set DB ID from Lua client */ luaReplyToRedisReply(c,lua); lua_gc(lua,LUA_GCSTEP,1); @@ -743,6 +766,15 @@ void scriptCommand(redisClient *c) { } addReplyBulkCBuffer(c,funcname+2,40); sdsfree(sha); + } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"kill")) { + if (server.lua_caller == NULL) { + addReplyError(c,"No scripts in execution right now."); + } else if (server.lua_write_dirty) { + addReplyError(c, "Sorry the script already executed write commands against the dataset. You can either wait the script termination or kill the server in an hard way using the SHUTDOWN NOSAVE command."); + } else { + server.lua_kill = 1; + addReply(c,shared.ok); + } } else { addReplyError(c, "Unknown SCRIPT subcommand or wrong # of args."); } diff --git a/src/sds.c b/src/sds.c index 2104eb36..c3a0ccb9 100644 --- a/src/sds.c +++ b/src/sds.c @@ -40,6 +40,7 @@ #include #include #include +#include #include "sds.h" #include "zmalloc.h" @@ -101,7 +102,13 @@ void sdsclear(sds s) { sh->buf[0] = '\0'; } -static sds sdsMakeRoomFor(sds s, size_t addlen) { +/* Enlarge the free space at the end of the sds string so that the caller + * is sure that after calling this function can overwrite up to addlen + * bytes after the end of the string, plus one more byte for nul term. + * + * Note: this does not change the *size* of the sds string as returned + * by sdslen(), but only the free buffer space we have. */ +sds sdsMakeRoomFor(sds s, size_t addlen) { struct sdshdr *sh, *newsh; size_t free = sdsavail(s); size_t len, newlen; @@ -121,6 +128,37 @@ static sds sdsMakeRoomFor(sds s, size_t addlen) { return newsh->buf; } +/* Increment the sds length and decrements the left free space at the + * end of the string accordingly to 'incr'. Also set the null term + * in the new end of the string. + * + * This function is used in order to fix the string length after the + * user calls sdsMakeRoomFor(), writes something after the end of + * the current string, and finally needs to set the new length. + * + * Note: it is possible to use a negative increment in order to + * right-trim the string. + * + * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the + * following schema to cat bytes coming from the kerenl to the end of an + * sds string new things without copying into an intermediate buffer: + * + * oldlen = sdslen(s); + * s = sdsMakeRoomFor(s, BUFFER_SIZE); + * nread = read(fd, s+oldlen, BUFFER_SIZE); + * ... check for nread <= 0 and handle it ... + * sdsIncrLen(s, nhread); + */ +void sdsIncrLen(sds s, int incr) { + struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); + + assert(sh->free >= incr); + sh->len += incr; + sh->free -= incr; + assert(sh->free >= 0); + s[sh->len] = '\0'; +} + /* Grow the sds to have the specified length. Bytes that were not part of * the original length of the sds will be set to zero. */ sds sdsgrowzero(sds s, size_t len) { @@ -609,6 +647,7 @@ sds sdsmapchars(sds s, char *from, char *to, size_t setlen) { int main(void) { { + struct sdshdr *sh; sds x = sdsnew("foo"), y; test_cond("Create a string and obtain the length", @@ -688,7 +727,26 @@ int main(void) { x = sdsnew("aar"); y = sdsnew("bar"); test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0) + + { + int oldfree; + + sdsfree(x); + x = sdsnew("0"); + sh = (void*) (x-(sizeof(struct sdshdr))); + test_cond("sdsnew() free/len buffers", sh->len == 1 && sh->free == 0); + x = sdsMakeRoomFor(x,1); + sh = (void*) (x-(sizeof(struct sdshdr))); + test_cond("sdsMakeRoomFor()", sh->len == 1 && sh->free > 0); + oldfree = sh->free; + x[1] = '1'; + sdsIncrLen(x,1); + test_cond("sdsIncrLen() -- content", x[0] == '0' && x[1] == '1'); + test_cond("sdsIncrLen() -- len", sh->len == 2); + test_cond("sdsIncrLen() -- free", sh->free == oldfree-1); + } } test_report() + return 0; } #endif diff --git a/src/sds.h b/src/sds.h index 6e5684ee..eff1b03e 100644 --- a/src/sds.h +++ b/src/sds.h @@ -88,4 +88,8 @@ sds *sdssplitargs(char *line, int *argc); void sdssplitargs_free(sds *argv, int argc); sds sdsmapchars(sds s, char *from, char *to, size_t setlen); +/* Low level functions exposed to the user API */ +sds sdsMakeRoomFor(sds s, size_t addlen); +void sdsIncrLen(sds s, int incr); + #endif diff --git a/src/sort.c b/src/sort.c index da31b1b5..f70810b9 100644 --- a/src/sort.c +++ b/src/sort.c @@ -238,7 +238,7 @@ void sortCommand(redisClient *c) { dictEntry *setele; di = dictGetIterator(set); while((setele = dictNext(di)) != NULL) { - vector[j].obj = dictGetEntryKey(setele); + vector[j].obj = dictGetKey(setele); vector[j].u.score = 0; vector[j].u.cmpobj = NULL; j++; diff --git a/src/t_hash.c b/src/t_hash.c index 3ccdfd14..8ee5485c 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -55,7 +55,7 @@ int hashTypeGet(robj *o, robj *key, robj **objval, unsigned char **v, } else { dictEntry *de = dictFind(o->ptr,key); if (de == NULL) return -1; - *objval = dictGetEntryVal(de); + *objval = dictGetVal(de); } return o->encoding; } @@ -206,9 +206,9 @@ int hashTypeCurrent(hashTypeIterator *hi, int what, robj **objval, unsigned char } } else { if (what & REDIS_HASH_KEY) - *objval = dictGetEntryKey(hi->de); + *objval = dictGetKey(hi->de); else - *objval = dictGetEntryVal(hi->de); + *objval = dictGetVal(hi->de); } return hi->encoding; } @@ -346,6 +346,33 @@ void hincrbyCommand(redisClient *c) { server.dirty++; } +void hincrbyfloatCommand(redisClient *c) { + double long value, incr; + robj *o, *current, *new; + + if (getLongDoubleFromObjectOrReply(c,c->argv[3],&incr,NULL) != REDIS_OK) return; + if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return; + if ((current = hashTypeGetObject(o,c->argv[2])) != NULL) { + if (getLongDoubleFromObjectOrReply(c,current,&value, + "hash value is not a valid float") != REDIS_OK) { + decrRefCount(current); + return; + } + decrRefCount(current); + } else { + value = 0; + } + + value += incr; + new = createStringObjectFromLongDouble(value); + hashTypeTryObjectEncoding(o,&c->argv[2],NULL); + hashTypeSet(o,c->argv[2],new); + addReplyBulk(c,new); + decrRefCount(new); + signalModifiedKey(c->db,c->argv[1]); + server.dirty++; +} + void hgetCommand(redisClient *c) { robj *o, *value; unsigned char *v; diff --git a/src/t_list.c b/src/t_list.c index c5e3df61..21fd1c2f 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -776,7 +776,7 @@ void blockForKeys(redisClient *c, robj **keys, int numkeys, time_t timeout, robj incrRefCount(keys[j]); redisAssertWithInfo(c,keys[j],retval == DICT_OK); } else { - l = dictGetEntryVal(de); + l = dictGetVal(de); } listAddNodeTail(l,c); } @@ -797,7 +797,7 @@ void unblockClientWaitingData(redisClient *c) { /* Remove this client from the list of clients waiting for this key. */ de = dictFind(c->db->blocking_keys,c->bpop.keys[j]); redisAssertWithInfo(c,c->bpop.keys[j],de != NULL); - l = dictGetEntryVal(de); + l = dictGetVal(de); listDelNode(l,listSearchKey(l,c)); /* If the list is empty we need to remove it to avoid wasting memory */ if (listLength(l) == 0) @@ -836,7 +836,7 @@ int handleClientsWaitingListPush(redisClient *c, robj *key, robj *ele) { de = dictFind(c->db->blocking_keys,key); if (de == NULL) return 0; - clients = dictGetEntryVal(de); + clients = dictGetVal(de); numclients = listLength(clients); /* Try to handle the push as long as there are clients waiting for a push. diff --git a/src/t_set.c b/src/t_set.c index db54ffe7..3cf1cf00 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -115,7 +115,7 @@ int setTypeNext(setTypeIterator *si, robj **objele, int64_t *llele) { if (si->encoding == REDIS_ENCODING_HT) { dictEntry *de = dictNext(si->di); if (de == NULL) return -1; - *objele = dictGetEntryKey(de); + *objele = dictGetKey(de); } else if (si->encoding == REDIS_ENCODING_INTSET) { if (!intsetGet(si->subject->ptr,si->ii++,llele)) return -1; @@ -165,7 +165,7 @@ robj *setTypeNextObject(setTypeIterator *si) { int setTypeRandomElement(robj *setobj, robj **objele, int64_t *llele) { if (setobj->encoding == REDIS_ENCODING_HT) { dictEntry *de = dictGetRandomKey(setobj->ptr); - *objele = dictGetEntryKey(de); + *objele = dictGetKey(de); } else if (setobj->encoding == REDIS_ENCODING_INTSET) { *llele = intsetRandom(setobj->ptr); } else { diff --git a/src/t_string.c b/src/t_string.c index e0b9b263..2bd1646e 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -1,4 +1,5 @@ #include "redis.h" +#include /* isnan(), isinf() */ /*----------------------------------------------------------------------------- * String Commands @@ -12,16 +13,17 @@ static int checkStringLength(redisClient *c, long long size) { return REDIS_OK; } -void setGenericCommand(redisClient *c, int nx, robj *key, robj *val, robj *expire) { - long seconds = 0; /* initialized to avoid an harmness warning */ +void setGenericCommand(redisClient *c, int nx, robj *key, robj *val, robj *expire, int unit) { + long long milliseconds = 0; /* initialized to avoid an harmness warning */ if (expire) { - if (getLongFromObjectOrReply(c, expire, &seconds, NULL) != REDIS_OK) + if (getLongLongFromObjectOrReply(c, expire, &milliseconds, NULL) != REDIS_OK) return; - if (seconds <= 0) { + if (milliseconds <= 0) { addReplyError(c,"invalid expire time in SETEX"); return; } + if (unit == UNIT_SECONDS) milliseconds *= 1000; } if (lookupKeyWrite(c->db,key) != NULL && nx) { @@ -30,23 +32,28 @@ void setGenericCommand(redisClient *c, int nx, robj *key, robj *val, robj *expir } setKey(c->db,key,val); server.dirty++; - if (expire) setExpire(c->db,key,time(NULL)+seconds); + if (expire) setExpire(c->db,key,mstime()+milliseconds); addReply(c, nx ? shared.cone : shared.ok); } void setCommand(redisClient *c) { c->argv[2] = tryObjectEncoding(c->argv[2]); - setGenericCommand(c,0,c->argv[1],c->argv[2],NULL); + setGenericCommand(c,0,c->argv[1],c->argv[2],NULL,0); } void setnxCommand(redisClient *c) { c->argv[2] = tryObjectEncoding(c->argv[2]); - setGenericCommand(c,1,c->argv[1],c->argv[2],NULL); + setGenericCommand(c,1,c->argv[1],c->argv[2],NULL,0); } void setexCommand(redisClient *c) { c->argv[3] = tryObjectEncoding(c->argv[3]); - setGenericCommand(c,0,c->argv[1],c->argv[3],c->argv[2]); + setGenericCommand(c,0,c->argv[1],c->argv[3],c->argv[2],UNIT_SECONDS); +} + +void psetexCommand(redisClient *c) { + c->argv[3] = tryObjectEncoding(c->argv[3]); + setGenericCommand(c,0,c->argv[1],c->argv[3],c->argv[2],UNIT_MILLISECONDS); } int getGenericCommand(redisClient *c) { @@ -376,6 +383,39 @@ void decrbyCommand(redisClient *c) { incrDecrCommand(c,-incr); } +void incrbyfloatCommand(redisClient *c) { + long double incr, value; + robj *o, *new, *aux; + + o = lookupKeyWrite(c->db,c->argv[1]); + if (o != NULL && checkType(c,o,REDIS_STRING)) return; + if (getLongDoubleFromObjectOrReply(c,o,&value,NULL) != REDIS_OK || + getLongDoubleFromObjectOrReply(c,c->argv[2],&incr,NULL) != REDIS_OK) + return; + + value += incr; + if (isnan(value) || isinf(value)) { + addReplyError(c,"increment would produce NaN or Infinity"); + return; + } + new = createStringObjectFromLongDouble(value); + if (o) + dbOverwrite(c->db,c->argv[1],new); + else + dbAdd(c->db,c->argv[1],new); + signalModifiedKey(c->db,c->argv[1]); + server.dirty++; + addReplyBulk(c,new); + + /* Always replicate INCRBYFLOAT as a SET command with the final value + * in order to make sure that differences in float pricision or formatting + * will not create differences in replicas or after an AOF restart. */ + aux = createStringObject("SET",3); + rewriteClientCommandArgument(c,0,aux); + decrRefCount(aux); + rewriteClientCommandArgument(c,2,new); +} + void appendCommand(redisClient *c) { size_t totlen; robj *o, *append; diff --git a/src/t_zset.c b/src/t_zset.c index 5dab64dc..ccf9962a 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -900,8 +900,8 @@ void zaddGenericCommand(redisClient *c, int incr) { ele = c->argv[3+j*2] = tryObjectEncoding(c->argv[3+j*2]); de = dictFind(zs->dict,ele); if (de != NULL) { - curobj = dictGetEntryKey(de); - curscore = *(double*)dictGetEntryVal(de); + curobj = dictGetKey(de); + curscore = *(double*)dictGetVal(de); if (incr) { score += curscore; @@ -921,7 +921,7 @@ void zaddGenericCommand(redisClient *c, int incr) { redisAssertWithInfo(c,curobj,zslDelete(zs->zsl,curscore,curobj)); znode = zslInsert(zs->zsl,score,curobj); incrRefCount(curobj); /* Re-inserted in skiplist. */ - dictGetEntryVal(de) = &znode->score; /* Update score ptr. */ + dictGetVal(de) = &znode->score; /* Update score ptr. */ signalModifiedKey(c->db,key); server.dirty++; @@ -987,7 +987,7 @@ void zremCommand(redisClient *c) { deleted++; /* Delete from the skiplist */ - score = *(double*)dictGetEntryVal(de); + score = *(double*)dictGetVal(de); redisAssertWithInfo(c,c->argv[j],zslDelete(zs->zsl,score,c->argv[j])); /* Delete from the hash table */ @@ -1018,7 +1018,7 @@ void zremrangebyscoreCommand(redisClient *c) { /* Parse the range arguments. */ if (zslParseRange(c->argv[2],c->argv[3],&range) != REDIS_OK) { - addReplyError(c,"min or max is not a double"); + addReplyError(c,"min or max is not a float"); return; } @@ -1263,7 +1263,7 @@ int zuiNext(zsetopsrc *op, zsetopval *val) { } else if (op->encoding == REDIS_ENCODING_HT) { if (it->ht.de == NULL) return 0; - val->ele = dictGetEntryKey(it->ht.de); + val->ele = dictGetKey(it->ht.de); val->score = 1.0; /* Move to next element. */ @@ -1397,7 +1397,7 @@ int zuiFind(zsetopsrc *op, zsetopval *val, double *score) { } else if (op->encoding == REDIS_ENCODING_SKIPLIST) { dictEntry *de; if ((de = dictFind(it->sl.zs->dict,val->ele)) != NULL) { - *score = *(double*)dictGetEntryVal(de); + *score = *(double*)dictGetVal(de); return 1; } else { return 0; @@ -1417,7 +1417,7 @@ int zuiCompareByCardinality(const void *s1, const void *s2) { #define REDIS_AGGR_SUM 1 #define REDIS_AGGR_MIN 2 #define REDIS_AGGR_MAX 3 -#define zunionInterDictValue(_e) (dictGetEntryVal(_e) == NULL ? 1.0 : *(double*)dictGetEntryVal(_e)) +#define zunionInterDictValue(_e) (dictGetVal(_e) == NULL ? 1.0 : *(double*)dictGetVal(_e)) inline static void zunionInterAggregate(double *target, double val, int aggregate) { if (aggregate == REDIS_AGGR_SUM) { @@ -1493,7 +1493,7 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { j++; remaining--; for (i = 0; i < setnum; i++, j++, remaining--) { if (getDoubleFromObjectOrReply(c,c->argv[j],&src[i].weight, - "weight value is not a double") != REDIS_OK) + "weight value is not a float") != REDIS_OK) { zfree(src); return; @@ -1777,7 +1777,7 @@ void genericZrangebyscoreCommand(redisClient *c, int reverse) { } if (zslParseRange(c->argv[minidx],c->argv[maxidx],&range) != REDIS_OK) { - addReplyError(c,"min or max is not a double"); + addReplyError(c,"min or max is not a float"); return; } @@ -1959,7 +1959,7 @@ void zcountCommand(redisClient *c) { /* Parse the range arguments */ if (zslParseRange(c->argv[2],c->argv[3],&range) != REDIS_OK) { - addReplyError(c,"min or max is not a double"); + addReplyError(c,"min or max is not a float"); return; } @@ -2058,7 +2058,7 @@ void zscoreCommand(redisClient *c) { c->argv[2] = tryObjectEncoding(c->argv[2]); de = dictFind(zs->dict,c->argv[2]); if (de != NULL) { - score = *(double*)dictGetEntryVal(de); + score = *(double*)dictGetVal(de); addReplyDouble(c,score); } else { addReply(c,shared.nullbulk); @@ -2114,7 +2114,7 @@ void zrankGenericCommand(redisClient *c, int reverse) { ele = c->argv[2] = tryObjectEncoding(c->argv[2]); de = dictFind(zs->dict,ele); if (de != NULL) { - score = *(double*)dictGetEntryVal(de); + score = *(double*)dictGetVal(de); rank = zslGetRank(zsl,score,ele); redisAssertWithInfo(c,ele,rank); /* Existing elements always have a rank. */ if (reverse) diff --git a/src/testhelp.h b/src/testhelp.h index d699f2ae..807a86e9 100644 --- a/src/testhelp.h +++ b/src/testhelp.h @@ -48,6 +48,7 @@ int __test_num = 0; __test_num-__failed_tests, __failed_tests); \ if (__failed_tests) { \ printf("=== WARNING === We have failed tests here...\n"); \ + exit(1); \ } \ } while(0); diff --git a/src/zmalloc.c b/src/zmalloc.c index 5408c2fa..56b9140c 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -38,7 +38,7 @@ #ifdef HAVE_MALLOC_SIZE #define PREFIX_SIZE (0) #else -#if defined(__sun) +#if defined(__sun) || defined(__sparc) || defined(__sparc__) #define PREFIX_SIZE (sizeof(long long)) #else #define PREFIX_SIZE (sizeof(size_t)) diff --git a/tests/support/util.tcl b/tests/support/util.tcl index a39a2134..675d57f7 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -294,3 +294,7 @@ proc csvdump r { proc csvstring s { return "\"$s\"" } + +proc roundFloat f { + format "%.10g" $f +} diff --git a/tests/unit/basic.tcl b/tests/unit/basic.tcl index 86645e95..4210f48b 100644 --- a/tests/unit/basic.tcl +++ b/tests/unit/basic.tcl @@ -120,7 +120,19 @@ start_server {tags {"basic"}} { r incrby novar 17179869184 } {34359738368} - test {INCR fails against key with spaces (no integer encoded)} { + test {INCR fails against key with spaces (left)} { + r set novar " 11" + catch {r incr novar} err + format $err + } {ERR*} + + test {INCR fails against key with spaces (right)} { + r set novar "11 " + catch {r incr novar} err + format $err + } {ERR*} + + test {INCR fails against key with spaces (both)} { r set novar " 11 " catch {r incr novar} err format $err @@ -138,6 +150,74 @@ start_server {tags {"basic"}} { r decrby novar 17179869185 } {-1} + test {INCRBYFLOAT against non existing key} { + r del novar + list [roundFloat [r incrbyfloat novar 1]] \ + [roundFloat [r get novar]] \ + [roundFloat [r incrbyfloat novar 0.25]] \ + [roundFloat [r get novar]] + } {1 1 1.25 1.25} + + test {INCRBYFLOAT against key originally set with SET} { + r set novar 1.5 + roundFloat [r incrbyfloat novar 1.5] + } {3} + + test {INCRBYFLOAT over 32bit value} { + r set novar 17179869184 + r incrbyfloat novar 1.5 + } {17179869185.5} + + test {INCRBYFLOAT over 32bit value with over 32bit increment} { + r set novar 17179869184 + r incrbyfloat novar 17179869184 + } {34359738368} + + test {INCRBYFLOAT fails against key with spaces (left)} { + set err {} + r set novar " 11" + catch {r incrbyfloat novar 1.0} err + format $err + } {ERR*valid*} + + test {INCRBYFLOAT fails against key with spaces (right)} { + set err {} + r set novar "11 " + catch {r incrbyfloat novar 1.0} err + format $err + } {ERR*valid*} + + test {INCRBYFLOAT fails against key with spaces (both)} { + set err {} + r set novar " 11 " + catch {r incrbyfloat novar 1.0} err + format $err + } {ERR*valid*} + + test {INCRBYFLOAT fails against a key holding a list} { + r del mylist + set err {} + r rpush mylist 1 + catch {r incrbyfloat mylist 1.0} err + r del mylist + format $err + } {ERR*kind*} + + test {INCRBYFLOAT does not allow NaN or Infinity} { + r set foo 0 + set err {} + catch {r incrbyfloat foo +inf} err + set err + # p.s. no way I can force NaN to test it from the API because + # there is no way to increment / decrement by infinity nor to + # perform divisions. + } {ERR*would produce*} + + test {INCRBYFLOAT decrement} { + r set foo 1 + roundFloat [r incrbyfloat foo -1.1] + } {-0.1} + test "SETNX target key missing" { r del novar assert_equal 1 [r setnx novar foobared] diff --git a/tests/unit/expire.tcl b/tests/unit/expire.tcl index 415a0f53..b88ff821 100644 --- a/tests/unit/expire.tcl +++ b/tests/unit/expire.tcl @@ -5,7 +5,7 @@ start_server {tags {"expire"}} { set v2 [r ttl x] set v3 [r expire x 10] set v4 [r ttl x] - r expire x 4 + r expire x 2 list $v1 $v2 $v3 $v4 } {1 [45] 1 10} @@ -14,8 +14,8 @@ start_server {tags {"expire"}} { } {foobar} tags {"slow"} { - test {EXPIRE - After 6 seconds the key should no longer be here} { - after 6000 + test {EXPIRE - After 2.1 seconds the key should no longer be here} { + after 2100 list [r get x] [r exists x] } {{} 0} } @@ -51,7 +51,7 @@ start_server {tags {"expire"}} { tags {"slow"} { test {SETEX - Wait for the key to expire} { - after 3000 + after 1100 r get y } {} } @@ -71,4 +71,74 @@ start_server {tags {"expire"}} { r set x foo list [r persist foo] [r persist nokeyatall] } {0 0} + + test {EXPIRE pricision is now the millisecond} { + # This test is very likely to do a false positive if the + # server is under pressure, so if it does not work give it a few more + # chances. + for {set j 0} {$j < 3} {incr j} { + r del x + r setex x 1 somevalue + after 997 + set a [r get x] + after 1002 + set b [r get x] + if {$a eq {somevalue} && $b eq {}} break + } + list $a $b + } {somevalue {}} + + test {PEXPIRE/PSETEX/PEXPIREAT can set sub-second expires} { + # This test is very likely to do a false positive if the + # server is under pressure, so if it does not work give it a few more + # chances. + for {set j 0} {$j < 3} {incr j} { + r del x y z + r psetex x 100 somevalue + after 97 + set a [r get x] + after 102 + set b [r get x] + + r set x somevalue + r pexpire x 100 + after 97 + set c [r get x] + after 102 + set d [r get x] + + r set x somevalue + r pexpireat x [expr ([clock seconds]*1000)+100] + after 97 + set e [r get x] + after 102 + set f [r get x] + + if {$a eq {somevalue} && $b eq {} && + $c eq {somevalue} && $d eq {} && + $e eq {somevalue} && $f eq {}} break + } + list $a $b + } {somevalue {}} + + test {PTTL returns millisecond time to live} { + r del x + r setex x 1 somevalue + set ttl [r pttl x] + assert {$ttl > 900 && $ttl <= 1000} + } + + test {Redis should actively expire keys incrementally} { + r flushdb + r psetex key1 500 a + r psetex key2 500 a + r psetex key3 500 a + set size1 [r dbsize] + # Redis expires random keys ten times every second so we are + # fairly sure that all the three keys should be evicted after + # one second. + after 1000 + set size2 [r dbsize] + list $size1 $size2 + } {3 0} } diff --git a/tests/unit/other.tcl b/tests/unit/other.tcl index 702c291f..bb65570a 100644 --- a/tests/unit/other.tcl +++ b/tests/unit/other.tcl @@ -107,7 +107,7 @@ start_server {tags {"other"}} { } } - test {EXPIRES after a reload (snapshot + append only file)} { + test {EXPIRES after a reload (snapshot + append only file rewrite)} { r flushdb r set x 10 r expire x 1000 @@ -123,6 +123,39 @@ start_server {tags {"other"}} { list $e1 $e2 } {1 1} + test {EXPIRES after AOF reload (without rewrite)} { + r flushdb + r config set appendonly yes + r set x somevalue + r expire x 1000 + r setex y 2000 somevalue + r set z somevalue + r expireat z [expr {[clock seconds]+3000}] + + # Milliseconds variants + r set px somevalue + r pexpire px 1000000 + r psetex py 2000000 somevalue + r set pz somevalue + r pexpireat pz [expr {([clock seconds]+3000)*1000}] + + # Reload and check + r debug loadaof + set ttl [r ttl x] + assert {$ttl > 900 && $ttl <= 1000} + set ttl [r ttl y] + assert {$ttl > 1900 && $ttl <= 2000} + set ttl [r ttl z] + assert {$ttl > 2900 && $ttl <= 3000} + set ttl [r ttl px] + assert {$ttl > 900 && $ttl <= 1000} + set ttl [r ttl py] + assert {$ttl > 1900 && $ttl <= 2000} + set ttl [r ttl pz] + assert {$ttl > 2900 && $ttl <= 3000} + r config set appendonly no + } + tags {protocol} { test {PIPELINING stresser (also a regression for the old epoll bug)} { set fd2 [socket $::host $::port] diff --git a/tests/unit/type/hash.tcl b/tests/unit/type/hash.tcl index 718bc04a..04a5f4c7 100644 --- a/tests/unit/type/hash.tcl +++ b/tests/unit/type/hash.tcl @@ -298,9 +298,9 @@ start_server {tags {"hash"}} { list [r hincrby smallhash tmp 17179869184] [r hincrby bighash tmp 17179869184] } {34359738368 34359738368} - test {HINCRBY fails against hash value with spaces} { - r hset smallhash str " 11 " - r hset bighash str " 11 " + test {HINCRBY fails against hash value with spaces (left)} { + r hset smallhash str " 11" + r hset bighash str " 11" catch {r hincrby smallhash str 1} smallerr catch {r hincrby smallhash str 1} bigerr set rv {} @@ -308,6 +308,80 @@ start_server {tags {"hash"}} { lappend rv [string match "ERR*not an integer*" $bigerr] } {1 1} + test {HINCRBY fails against hash value with spaces (right)} { + r hset smallhash str "11 " + r hset bighash str "11 " + catch {r hincrby smallhash str 1} smallerr + catch {r hincrby smallhash str 1} bigerr + set rv {} + lappend rv [string match "ERR*not an integer*" $smallerr] + lappend rv [string match "ERR*not an integer*" $bigerr] + } {1 1} + + test {HINCRBYFLOAT against non existing database key} { + r del htest + list [r hincrbyfloat htest foo 2.5] + } {2.5} + + test {HINCRBYFLOAT against non existing hash key} { + set rv {} + r hdel smallhash tmp + r hdel bighash tmp + lappend rv [roundFloat [r hincrbyfloat smallhash tmp 2.5]] + lappend rv [roundFloat [r hget smallhash tmp]] + lappend rv [roundFloat [r hincrbyfloat bighash tmp 2.5]] + lappend rv [roundFloat [r hget bighash tmp]] + } {2.5 2.5 2.5 2.5} + + test {HINCRBYFLOAT against hash key created by hincrby itself} { + set rv {} + lappend rv [roundFloat [r hincrbyfloat smallhash tmp 3.5]] + lappend rv [roundFloat [r hget smallhash tmp]] + lappend rv [roundFloat [r hincrbyfloat bighash tmp 3.5]] + lappend rv [roundFloat [r hget bighash tmp]] + } {6 6 6 6} + + test {HINCRBYFLOAT against hash key originally set with HSET} { + r hset smallhash tmp 100 + r hset bighash tmp 100 + list [roundFloat [r hincrbyfloat smallhash tmp 2.5]] \ + [roundFloat [r hincrbyfloat bighash tmp 2.5]] + } {102.5 102.5} + + test {HINCRBYFLOAT over 32bit value} { + r hset smallhash tmp 17179869184 + r hset bighash tmp 17179869184 + list [r hincrbyfloat smallhash tmp 1] \ + [r hincrbyfloat bighash tmp 1] + } {17179869185 17179869185} + + test {HINCRBYFLOAT over 32bit value with over 32bit increment} { + r hset smallhash tmp 17179869184 + r hset bighash tmp 17179869184 + list [r hincrbyfloat smallhash tmp 17179869184] \ + [r hincrbyfloat bighash tmp 17179869184] + } {34359738368 34359738368} + + test {HINCRBYFLOAT fails against hash value with spaces (left)} { + r hset smallhash str " 11" + r hset bighash str " 11" + catch {r hincrbyfloat smallhash str 1} smallerr + catch {r hincrbyfloat smallhash str 1} bigerr + set rv {} + lappend rv [string match "ERR*not*float*" $smallerr] + lappend rv [string match "ERR*not*float*" $bigerr] + } {1 1} + + test {HINCRBYFLOAT fails against hash value with spaces (right)} { + r hset smallhash str "11 " + r hset bighash str "11 " + catch {r hincrbyfloat smallhash str 1} smallerr + catch {r hincrbyfloat smallhash str 1} bigerr + set rv {} + lappend rv [string match "ERR*not*float*" $smallerr] + lappend rv [string match "ERR*not*float*" $bigerr] + } {1 1} + test {Hash zipmap regression test for large keys} { r hset hash kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk a r hset hash kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk b diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index 41f5f588..aa43f3bf 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -36,11 +36,11 @@ start_server {tags {"zset"}} { } test "ZSET element can't be set to NaN with ZADD - $encoding" { - assert_error "*not a double*" {r zadd myzset nan abc} + assert_error "*not*float*" {r zadd myzset nan abc} } test "ZSET element can't be set to NaN with ZINCRBY" { - assert_error "*not a double*" {r zadd myzset nan abc} + assert_error "*not*float*" {r zadd myzset nan abc} } test "ZINCRBY calls leading to NaN result in error" { @@ -60,7 +60,7 @@ start_server {tags {"zset"}} { test {ZADD - Variadic version does not add nothing on single parsing err} { r del myzset catch {r zadd myzset 10 a 20 b 30.badscore c} e - assert_match {*ERR*not*double*} $e + assert_match {*ERR*not*float*} $e r exists myzset } {0} @@ -291,9 +291,9 @@ start_server {tags {"zset"}} { } test "ZRANGEBYSCORE with non-value min or max" { - assert_error "*not a double*" {r zrangebyscore fooz str 1} - assert_error "*not a double*" {r zrangebyscore fooz 1 str} - assert_error "*not a double*" {r zrangebyscore fooz 1 NaN} + assert_error "*not*float*" {r zrangebyscore fooz str 1} + assert_error "*not*float*" {r zrangebyscore fooz 1 str} + assert_error "*not*float*" {r zrangebyscore fooz 1 NaN} } test "ZREMRANGEBYSCORE basics" { @@ -353,9 +353,9 @@ start_server {tags {"zset"}} { } test "ZREMRANGEBYSCORE with non-value min or max" { - assert_error "*not a double*" {r zremrangebyscore fooz str 1} - assert_error "*not a double*" {r zremrangebyscore fooz 1 str} - assert_error "*not a double*" {r zremrangebyscore fooz 1 NaN} + assert_error "*not*float*" {r zremrangebyscore fooz str 1} + assert_error "*not*float*" {r zremrangebyscore fooz 1 str} + assert_error "*not*float*" {r zremrangebyscore fooz 1 NaN} } test "ZREMRANGEBYRANK basics" { @@ -501,7 +501,7 @@ start_server {tags {"zset"}} { r zadd zsetinf1 1.0 key r zadd zsetinf2 1.0 key - assert_error "*weight value is not a double*" { + assert_error "*weight*not*float*" { r $cmd zsetinf3 2 zsetinf1 zsetinf2 weights nan nan } } diff --git a/utils/speed-regression.tcl b/utils/speed-regression.tcl new file mode 100755 index 00000000..86a7d8d8 --- /dev/null +++ b/utils/speed-regression.tcl @@ -0,0 +1,130 @@ +#!/usr/bin/env tclsh8.5 +# Copyright (C) 2011 Salvatore Sanfilippo +# Released under the BSD license like Redis itself + +source ../tests/support/redis.tcl +set ::port 12123 +set ::tests {PING,SET,GET,INCR,LPUSH,LPOP,SADD,SPOP,LRANGE_100,LRANGE_600,MSET} +set ::datasize 16 +set ::requests 100000 + +proc run-tests branches { + set runs {} + set branch_id 0 + foreach b $branches { + cd ../src + puts "Benchmarking $b" + exec -ignorestderr git checkout $b 2> /dev/null + exec -ignorestderr make clean 2> /dev/null + puts " compiling..." + exec -ignorestderr make 2> /dev/null + + if {$branch_id == 0} { + puts " copy redis-benchmark from unstable to /tmp..." + exec -ignorestderr cp ./redis-benchmark /tmp + incr branch_id + continue + } + + # Start the Redis server + puts " starting the server... [exec ./redis-server -v]" + set pids [exec echo "port $::port\nloglevel warning\n" | ./redis-server - > /dev/null 2> /dev/null &] + puts " pids: $pids" + after 1000 + puts " running the benchmark" + + set r [redis 127.0.0.1 $::port] + set i [$r info] + puts " redis INFO shows version: [lindex [split $i] 0]" + $r close + + set output [exec /tmp/redis-benchmark -n $::requests -t $::tests -d $::datasize --csv -p $::port] + lappend runs $b $output + puts " killing server..." + catch {exec kill -9 [lindex $pids 0]} + catch {exec kill -9 [lindex $pids 1]} + incr branch_id + } + return $runs +} + +proc get-result-with-name {output name} { + foreach line [split $output "\n"] { + lassign [split $line ","] key value + set key [string tolower [string range $key 1 end-1]] + set value [string range $value 1 end-1] + if {$key eq [string tolower $name]} { + return $value + } + } + return "n/a" +} + +proc get-test-names output { + set names {} + foreach line [split $output "\n"] { + lassign [split $line ","] key value + set key [string tolower [string range $key 1 end-1]] + lappend names $key + } + return $names +} + +proc combine-results {results} { + set tests [get-test-names [lindex $results 1]] + foreach test $tests { + puts $test + foreach {branch output} $results { + puts [format "%-20s %s" \ + $branch [get-result-with-name $output $test]] + } + puts {} + } +} + +proc main {} { + # Note: the first branch is only used in order to get the redis-benchmark + # executable. Tests are performed starting from the second branch. + set branches { + slowset 2.2.0 2.4.0 unstable slowset + } + set results [run-tests $branches] + puts "\n" + puts "# Test results: datasize=$::datasize requests=$::requests" + puts [combine-results $results] +} + +# Force the user to run the script from the 'utils' directory. +if {![file exists speed-regression.tcl]} { + puts "Please make sure to run speed-regression.tcl while inside /utils." + puts "Example: cd utils; ./speed-regression.tcl" + exit 1 +} + +# Make sure there is not already a server runnign on port 12123 +set is_not_running [catch {set r [redis 127.0.0.1 $::port]}] +if {!$is_not_running} { + puts "Sorry, you have a running server on port $::port" + exit 1 +} + +# parse arguments +for {set j 0} {$j < [llength $argv]} {incr j} { + set opt [lindex $argv $j] + set arg [lindex $argv [expr $j+1]] + if {$opt eq {--tests}} { + set ::tests $arg + incr j + } elseif {$opt eq {--datasize}} { + set ::datasize $arg + incr j + } elseif {$opt eq {--requests}} { + set ::requests $arg + incr j + } else { + puts "Wrong argument: $opt" + exit 1 + } +} + +main