From: antirez Date: Thu, 23 Dec 2010 12:18:07 +0000 (+0100) Subject: Merge remote branch 'pietern/bench-fix' X-Git-Url: https://git.saurik.com/redis.git/commitdiff_plain/04a2ade90dd15db2a1e685e2e045b8c0181b9eea?hp=2380388974bd0256e1b123d608940355144e9fd4 Merge remote branch 'pietern/bench-fix' --- diff --git a/.gitignore b/.gitignore index 9d3423fe..90c94621 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ redis-benchmark redis-check-dump redis-check-aof doc-tools -mkrelease.sh release myredis.conf misc/* diff --git a/TODO b/TODO index 830149b3..9e6b0561 100644 --- a/TODO +++ b/TODO @@ -1,66 +1,44 @@ -Redis TODO and Roadmap ----------------------- +Redis TODO +---------- -VERSION 2.2 TODO (Optimizations and latency) -============================================ - -* Support for syslog(3). -* Change the implementation of ZCOUNT to use the augmented skiplist in order to be much faster. -* Add an explicit test for MULTI/EXEC reloaded in the AOF. -* Command table -> hash table, with support for command renaming +WARNING: are you a possible Redis contributor? + Before implementing what is listed what is listed in this file + please drop a message in the Redis google group or chat with + antirez or pietern on irc.freenode.org #redis to check if the work + is already in progress and if the feature is still interesting for + us, and *how* exactly this can be implemented to have good changes + of a merge. Otherwise it is probably wasted work! Thank you VM TODO ======= * Use multiple open FDs against the VM file, one for thread. * Check what happens performance-wise if instead of creating threads again and again the same threads are reused forever. Note: this requires a way to disable this clients in the child, but waiting for empty new jobs queue can be enough. +* mmap the swap file. +* Use just a single IO Job to swap out a key, and add a mutex so that pages in the page table can be marked as used and scanned from the thread itself. -STRING COMMANDS -=============== - -* Implement STRLEN, PEEK, POKE, SETBIT, GETBIT - -OTHER IMPORTANT THINGS THAT WILL BE ADDED BUT I'M NOT SURE WHEN -=============================================================== - -BIG ONES: +REPLICATION +=========== -* BRPOPLPUSH -* Specially encoded memory-saving integer sets. -* A command to export a JSON dump (there should be mostly working patch needing major reworking). -* Specially encoded sets of integers (this includes a big refactoring providing an higher level layer for Sets manipulation) +* PING between master and slave from time to time, so we can subject the +master-slave link to timeout, and detect when the connection is gone even +if the socket is still up. -SMALL ONES: - -* If sizeof(double) == sizeof(void*) we could store the double value of sorted sets directly in place of the pointer instead of allocating it in the heap. -* Delete on writes against expire policy should only happen after argument parsing for commands doing their own arg parsing stuff. -* Give errors when incrementing a key that does not look like an integer, when providing as a sorted set score something can't be parsed as a double, and so forth. -* MSADD (n keys) (n values). See this thread in the Redis google group: http://groups.google.com/group/redis-db/browse_thread/thread/e766d84eb375cd41 -* Don't save empty lists / sets / zsets on disk with snapshotting. -* Remove keys when a list / set / zset reaches length of 0. -* An option to exec a command slave-side if the master connection is lost: even cooler: if the script returns "0" the slave elects itself as master, otherwise continue trying to reconnect. -* PING the master from time to time to check if it's gone. - -THE "MAYBE" TODO LIST: things that may or may not get implemented -================================================================= - -Most of this can be seen just as proposals, the fact they are in this list -it's not a guarantee they'll ever get implemented ;) +OPTIMIZATIONS +============= * SORT: Don't copy the list into a vector when BY argument is constant. * Write the hash table size of every db in the dump, so that Redis can resize the hash table just one time when loading a big DB. -* Byte Array type (BA prefixed commands): BASETBIT BAGETBIT BASETU8 U16 U32 U64 S8 S16 S32 S64, ability to atomically INCRBY all the base types. BARANGE to get a range of bytes as a bulk value, BASETRANGE to set a range of bytes. -* Read-only mode. -* Kill the delete-on-write behavior of expires, replicating DELs -* Multiple BY in SORT. +* Read-only mode for slaves. KNOWN BUGS ========== -* LRANGE and other commands are using 32 bit integers for ranges, and overflows are not detected. So LRANGE mylist 0 23498204823094823904823904 will have random effects. - -REDIS CLI TODO -============== +* What happens in the following scenario: + 1) We are reading an AOF file. + 2) SETEX FOO 5 BAR + 3) APPEND FOO ZAP + What happens if between 1 and 2 for some reason (system under huge load + or alike) too many time passes? We should prevent expires while the + AOF is loading. -* Computer parsable output generation -* Memoize return values so that they can be used later as arguments, like $1 diff --git a/src/Makefile b/src/Makefile index 08f93f7a..17fc435b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -8,9 +8,11 @@ OPTIMIZATION?=-O2 ifeq ($(uname_S),SunOS) CFLAGS?= -std=c99 -pedantic $(OPTIMIZATION) -Wall -W -D__EXTENSIONS__ -D_XPG6 CCLINK?= -ldl -lnsl -lsocket -lm -lpthread + DEBUG?= -g -ggdb else CFLAGS?= -std=c99 -pedantic $(OPTIMIZATION) -Wall -W $(ARCH) $(PROF) CCLINK?= -lm -pthread + DEBUG?= -g -rdynamic -ggdb endif ifeq ($(USE_TCMALLOC),yes) @@ -18,7 +20,6 @@ ifeq ($(USE_TCMALLOC),yes) CFLAGS+= -DUSE_TCMALLOC endif CCOPT= $(CFLAGS) $(CCLINK) $(ARCH) $(PROF) -DEBUG?= -g -rdynamic -ggdb PREFIX= /usr/local INSTALL_BIN= $(PREFIX)/bin diff --git a/src/anet.h b/src/anet.h index bef0adcf..118b4dda 100644 --- a/src/anet.h +++ b/src/anet.h @@ -35,6 +35,10 @@ #define ANET_ERR -1 #define ANET_ERR_LEN 256 +#if defined(__sun) +#define AF_LOCAL AF_UNIX +#endif + int anetTcpConnect(char *err, char *addr, int port); int anetTcpNonBlockConnect(char *err, char *addr, int port); int anetUnixConnect(char *err, char *path); diff --git a/src/help.h b/src/help.h index 121d9dfa..51613c9b 100644 --- a/src/help.h +++ b/src/help.h @@ -1,4 +1,4 @@ -/* Automatically generated by utils/generate-command-help.rb, do not edit. */ +/* Automatically generated by generate-command-help.rb, do not edit. */ #ifndef __REDIS_HELP_H #define __REDIS_HELP_H @@ -53,11 +53,21 @@ struct commandHelp { "Remove and get the last element in a list, or block until one is available", 2, "1.3.1" }, + { "BRPOPLPUSH", + "source destination timeout", + "Pop a value from a list, push it to another list and return it; or block until one is available", + 2, + "2.1.7" }, { "CONFIG GET", "parameter", "Get the value of a configuration parameter", 9, "2.0" }, + { "CONFIG RESETSTAT", + "-", + "Reset the stats returned by INFO", + 9, + "2.0" }, { "CONFIG SET", "parameter value", "Set a configuration parameter to the given value", @@ -79,7 +89,7 @@ struct commandHelp { 9, "0.101" }, { "DECR", - "key decrement", + "key", "Decrement the integer value of a key by one", 1, "0.07" }, @@ -138,6 +148,11 @@ struct commandHelp { "Get the value of a key", 1, "0.07" }, + { "GETBIT", + "key offset", + "Returns the bit value at offset in the string value stored at key", + 1, + "2.1.8" }, { "GETSET", "key value", "Set the string value of a key and return its old value", @@ -344,12 +359,12 @@ struct commandHelp { 0, "0.07" }, { "RENAME", - "old new", + "key newkey", "Rename a key", 0, "0.07" }, { "RENAMENX", - "old new", + "key newkey", "Rename a key, only if the new key does not exist", 0, "0.07" }, @@ -408,8 +423,13 @@ struct commandHelp { "Set the string value of a key", 1, "0.07" }, + { "SETBIT", + "key offset value", + "Sets or clears the bit at offset in the string value stored at key", + 1, + "2.1.8" }, { "SETEX", - "key timestamp value", + "key seconds value", "Set the value and expiration of a key", 1, "1.3.10" }, @@ -418,6 +438,11 @@ struct commandHelp { "Set the value of a key, only if the key does not exist", 1, "0.07" }, + { "SETRANGE", + "key offset value", + "Overwrite part of a string at key starting at the specified offset", + 1, + "2.1.8" }, { "SHUTDOWN", "-", "Synchronously save the dataset to disk and then shut down the server", @@ -454,7 +479,7 @@ struct commandHelp { 3, "0.091" }, { "SORT", - "key [BY pattern] [LIMIT start count] [GET pattern [GET pattern ...]] [ASC|DESC] [ALPHA] [STORE destination]", + "key [BY pattern] [LIMIT offset count] [GET pattern [GET pattern ...]] [ASC|DESC] [ALPHA] [STORE destination]", "Sort the elements in a list, set or sorted set", 0, "0.07" }, @@ -484,7 +509,7 @@ struct commandHelp { 6, "1.3.8" }, { "SUBSTR", - "key start stop", + "key start end", "Get a substring of the string stored at a key", 1, "1.3.4" }, @@ -549,17 +574,17 @@ struct commandHelp { 4, "1.1" }, { "ZINTERSTORE", - "destination key [key ...] [WEIGHTS weight] [AGGREGATE SUM|MIN|MAX]", + "destination numkeys key [key ...] [WEIGHTS weight] [AGGREGATE SUM|MIN|MAX]", "Intersect multiple sorted sets and store the resulting sorted set in a new key", 4, "1.3.10" }, { "ZRANGE", - "key start stop", + "key start stop [WITHSCORES]", "Return a range of members in a sorted set, by index", 4, "1.1" }, { "ZRANGEBYSCORE", - "key min max", + "key min max [WITHSCORES] [LIMIT offset count]", "Return a range of members in a sorted set, by score", 4, "1.050" }, @@ -584,10 +609,15 @@ struct commandHelp { 4, "1.1" }, { "ZREVRANGE", - "key start stop", + "key start stop [WITHSCORES]", "Return a range of members in a sorted set, by index, with scores ordered from high to low", 4, "1.1" }, + { "ZREVRANGEBYSCORE", + "key max min [WITHSCORES] [LIMIT offset count]", + "Return a range of members in a sorted set, by score, with scores ordered from high to low", + 4, + "2.1.6" }, { "ZREVRANK", "key member", "Determine the index of a member in a sorted set, with scores ordered from high to low", @@ -599,7 +629,7 @@ struct commandHelp { 4, "1.1" }, { "ZUNIONSTORE", - "destination key [key ...] [WEIGHTS weight] [AGGREGATE SUM|MIN|MAX]", + "destination numkeys key [key ...] [WEIGHTS weight] [AGGREGATE SUM|MIN|MAX]", "Add multiple sorted sets and store the resulting sorted set in a new key", 4, "1.3.10" } diff --git a/src/redis-cli.c b/src/redis-cli.c index ca70c947..08b3a713 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -414,10 +414,11 @@ static sds cliFormatReplyRaw(redisReply *r) { } static int cliReadReply(int output_raw_strings) { + void *_reply; redisReply *reply; sds out; - if (redisGetReply(context,(void**)&reply) != REDIS_OK) { + if (redisGetReply(context,&_reply) != REDIS_OK) { if (config.shutdown) return REDIS_OK; if (config.interactive) { @@ -431,6 +432,7 @@ static int cliReadReply(int output_raw_strings) { return REDIS_ERR; /* avoid compiler warning */ } + reply = (redisReply*)_reply; if (output_raw_strings) { out = cliFormatReplyRaw(reply); } else { diff --git a/src/redis.c b/src/redis.c index fb6eb469..8f4ca961 100644 --- a/src/redis.c +++ b/src/redis.c @@ -891,6 +891,7 @@ void initServer() { server.stat_numcommands = 0; server.stat_numconnections = 0; server.stat_expiredkeys = 0; + server.stat_evictedkeys = 0; server.stat_starttime = time(NULL); server.stat_keyspace_misses = 0; server.stat_keyspace_hits = 0; @@ -1177,6 +1178,7 @@ sds genRedisInfoString(void) { "total_connections_received:%lld\r\n" "total_commands_processed:%lld\r\n" "expired_keys:%lld\r\n" + "evicted_keys:%lld\r\n" "keyspace_hits:%lld\r\n" "keyspace_misses:%lld\r\n" "hash_max_zipmap_entries:%zu\r\n" @@ -1219,6 +1221,7 @@ sds genRedisInfoString(void) { server.stat_numconnections, server.stat_numcommands, server.stat_expiredkeys, + server.stat_evictedkeys, server.stat_keyspace_hits, server.stat_keyspace_misses, server.hash_max_zipmap_entries, @@ -1436,44 +1439,13 @@ void freeMemoryIfNeeded(void) { if (bestkey) { robj *keyobj = createStringObject(bestkey,sdslen(bestkey)); dbDelete(db,keyobj); - server.stat_expiredkeys++; + server.stat_evictedkeys++; decrRefCount(keyobj); freed++; } } if (!freed) return; /* nothing to free... */ } - - while(0) { - int j, k, freed = 0; - for (j = 0; j < server.dbnum; j++) { - int minttl = -1; - sds minkey = NULL; - robj *keyobj = NULL; - struct dictEntry *de; - - if (dictSize(server.db[j].expires)) { - freed = 1; - /* From a sample of three keys drop the one nearest to - * the natural expire */ - for (k = 0; k < 3; k++) { - time_t t; - - de = dictGetRandomKey(server.db[j].expires); - t = (time_t) dictGetEntryVal(de); - if (minttl == -1 || t < minttl) { - minkey = dictGetEntryKey(de); - minttl = t; - } - } - keyobj = createStringObject(minkey,sdslen(minkey)); - dbDelete(server.db+j,keyobj); - server.stat_expiredkeys++; - decrRefCount(keyobj); - } - } - if (!freed) return; /* nothing to free... */ - } } /* =================================== Main! ================================ */ @@ -1504,7 +1476,7 @@ void createPidFile(void) { /* Try to write the pid file in a best-effort way. */ FILE *fp = fopen(server.pidfile,"w"); if (fp) { - fprintf(fp,"%d\n",getpid()); + fprintf(fp,"%d\n",(int)getpid()); fclose(fp); } } diff --git a/src/redis.h b/src/redis.h index 8495a87e..bc1a58c1 100644 --- a/src/redis.h +++ b/src/redis.h @@ -387,6 +387,7 @@ struct redisServer { long long stat_numcommands; /* number of processed commands */ long long stat_numconnections; /* number of connections received */ long long stat_expiredkeys; /* number of expired keys */ + long long stat_evictedkeys; /* number of evicted keys (maxmemory) */ long long stat_keyspace_hits; /* number of successful lookups of keys */ long long stat_keyspace_misses; /* number of failed lookups of keys */ /* Configuration */ diff --git a/src/t_string.c b/src/t_string.c index eb080c88..58834289 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -1,4 +1,3 @@ -#include #include "redis.h" /*----------------------------------------------------------------------------- @@ -346,14 +345,19 @@ void msetnxCommand(redisClient *c) { } void incrDecrCommand(redisClient *c, long long incr) { - long long value; + long long value, oldvalue; robj *o; o = lookupKeyWrite(c->db,c->argv[1]); if (o != NULL && checkType(c,o,REDIS_STRING)) return; if (getLongLongFromObjectOrReply(c,o,&value,NULL) != REDIS_OK) return; + oldvalue = value; value += incr; + if ((incr < 0 && value > oldvalue) || (incr > 0 && value < oldvalue)) { + addReplyError(c,"increment or decrement would overflow"); + return; + } o = createStringObjectFromLongLong(value); dbReplace(c->db,c->argv[1],o); touchWatchedKey(c->db,c->argv[1]); diff --git a/src/ziplist.c b/src/ziplist.c index a1a63478..233fabef 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -119,6 +119,7 @@ static unsigned int zipEntryEncoding(unsigned char *p) { return p[0] & 0xf0; } assert(NULL); + return 0; } /* Return bytes needed to store integer encoded by 'encoding' */ @@ -129,13 +130,14 @@ static unsigned int zipIntSize(unsigned char encoding) { case ZIP_INT_64B: return sizeof(int64_t); } assert(NULL); + return 0; } /* Decode the encoded length pointed by 'p'. If a pointer to 'lensize' is * provided, it is set to the number of bytes required to encode the length. */ static unsigned int zipDecodeLength(unsigned char *p, unsigned int *lensize) { unsigned char encoding = zipEntryEncoding(p); - unsigned int len; + unsigned int len = 0; if (ZIP_IS_STR(encoding)) { switch(encoding) { @@ -300,7 +302,7 @@ static void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encodi static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) { int16_t i16; int32_t i32; - int64_t i64, ret; + int64_t i64, ret = 0; if (encoding == ZIP_INT_16B) { memcpy(&i16,p,sizeof(i16)); ret = i16; diff --git a/utils/generate-command-help.rb b/utils/generate-command-help.rb index f730eaf1..96cccc2b 100755 --- a/utils/generate-command-help.rb +++ b/utils/generate-command-help.rb @@ -42,6 +42,7 @@ end def commands return @commands if @commands + require "rubygems" require "net/http" require "net/https" require "json" diff --git a/utils/mkrelease.sh b/utils/mkrelease.sh new file mode 100755 index 00000000..2316fff9 --- /dev/null +++ b/utils/mkrelease.sh @@ -0,0 +1,14 @@ +#!/bin/sh +if [ $# != "1" ] +then + echo "Usage: ./mkrelease.sh " + exit 1 +fi + +TAG=$1 +TARNAME="redis-${TAG}.tar" +echo "Generating /tmp/${TARNAME}" +git archive $TAG --prefix redis-${TAG}/ > /tmp/$TARNAME || exit 1 +echo "Gizipping the archive" +rm -f /tmp/$TARNAME.gz +gzip -9 /tmp/$TARNAME