X-Git-Url: https://git.saurik.com/redis.git/blobdiff_plain/a4460ef43f9625e6c09cdc327614d1b47c043489..51829ed3f09bb4381f075900521bcaafee8819bb:/redis.c diff --git a/redis.c b/redis.c index 856a6cac..a3341f6d 100644 --- a/redis.c +++ b/redis.c @@ -27,7 +27,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#define REDIS_VERSION "0.091" +#define REDIS_VERSION "0.100" #include "fmacros.h" @@ -67,7 +67,7 @@ #define REDIS_MAXIDLETIME (60*5) /* default client timeout */ #define REDIS_IOBUF_LEN 1024 #define REDIS_LOADBUF_LEN 1024 -#define REDIS_MAX_ARGS 16 +#define REDIS_STATIC_ARGS 4 #define REDIS_DEFAULT_DBNUM 16 #define REDIS_CONFIGLINE_MAX 1024 #define REDIS_OBJFREELIST_MAX 1000000 /* Max number of objects to cache */ @@ -183,7 +183,7 @@ typedef struct redisClient { redisDb *db; int dictid; sds querybuf; - robj *argv[REDIS_MAX_ARGS]; + robj **argv; int argc; int bulklen; /* bulk read len. -1 if not in bulk read mode */ list *reply; @@ -348,6 +348,8 @@ static void sinterCommand(redisClient *c); static void sinterstoreCommand(redisClient *c); static void sunionCommand(redisClient *c); static void sunionstoreCommand(redisClient *c); +static void sdiffCommand(redisClient *c); +static void sdiffstoreCommand(redisClient *c); static void syncCommand(redisClient *c); static void flushdbCommand(redisClient *c); static void flushallCommand(redisClient *c); @@ -367,7 +369,7 @@ static struct redisCommand cmdTable[] = { {"get",getCommand,2,REDIS_CMD_INLINE}, {"set",setCommand,3,REDIS_CMD_BULK}, {"setnx",setnxCommand,3,REDIS_CMD_BULK}, - {"del",delCommand,2,REDIS_CMD_INLINE}, + {"del",delCommand,-2,REDIS_CMD_INLINE}, {"exists",existsCommand,2,REDIS_CMD_INLINE}, {"incr",incrCommand,2,REDIS_CMD_INLINE}, {"decr",decrCommand,2,REDIS_CMD_INLINE}, @@ -391,6 +393,8 @@ static struct redisCommand cmdTable[] = { {"sinterstore",sinterstoreCommand,-3,REDIS_CMD_INLINE}, {"sunion",sunionCommand,-2,REDIS_CMD_INLINE}, {"sunionstore",sunionstoreCommand,-3,REDIS_CMD_INLINE}, + {"sdiff",sdiffCommand,-2,REDIS_CMD_INLINE}, + {"sdiffstore",sdiffstoreCommand,-3,REDIS_CMD_INLINE}, {"smembers",sinterCommand,2,REDIS_CMD_INLINE}, {"incrby",incrbyCommand,3,REDIS_CMD_INLINE}, {"decrby",decrbyCommand,3,REDIS_CMD_INLINE}, @@ -556,7 +560,12 @@ void redisLog(int level, const char *fmt, ...) va_start(ap, fmt); if (level >= server.verbosity) { char *c = ".-*"; - fprintf(fp,"%c ",c[level]); + char buf[64]; + time_t now; + + now = time(NULL); + strftime(buf,64,"%d %b %H:%M:%S",gmtime(&now)); + fprintf(fp,"%s %c ",buf,c[level]); vfprintf(fp, fmt, ap); fprintf(fp,"\n"); fflush(fp); @@ -652,6 +661,25 @@ void closeTimedoutClients(void) { } } +/* If the percentage of used slots in the HT reaches REDIS_HT_MINFILL + * we resize the hash table to save memory */ +void tryResizeHashTables(void) { + int j; + + for (j = 0; j < server.dbnum; j++) { + long long size, used; + + size = dictSlots(server.db[j].dict); + used = dictSize(server.db[j].dict); + if (size && used && size > REDIS_HT_MINSLOTS && + (used*100/size < REDIS_HT_MINFILL)) { + redisLog(REDIS_NOTICE,"The hash table %d is too sparse, resize it...",j); + dictResize(server.db[j].dict); + redisLog(REDIS_NOTICE,"Hash table %d resized.",j); + } + } +} + int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { int j, loops = server.cronloops++; REDIS_NOTUSED(eventLoop); @@ -661,10 +689,9 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Update the global state with the amount of used memory */ server.usedmemory = zmalloc_used_memory(); - /* If the percentage of used slots in the HT reaches REDIS_HT_MINFILL - * we resize the hash table to save memory */ + /* Show some info about non-empty databases */ for (j = 0; j < server.dbnum; j++) { - int size, used, vkeys; + long long size, used, vkeys; size = dictSlots(server.db[j].dict); used = dictSize(server.db[j].dict); @@ -673,14 +700,16 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { redisLog(REDIS_DEBUG,"DB %d: %d keys (%d volatile) in %d slots HT.",j,used,vkeys,size); /* dictPrintStats(server.dict); */ } - if (size && used && size > REDIS_HT_MINSLOTS && - (used*100/size < REDIS_HT_MINFILL)) { - redisLog(REDIS_NOTICE,"The hash table %d is too sparse, resize it...",j); - dictResize(server.db[j].dict); - redisLog(REDIS_NOTICE,"Hash table %d resized.",j); - } } + /* We don't want to resize the hash tables while a bacground saving + * is in progress: the saving child is created using fork() that is + * implemented with a copy-on-write semantic in most modern systems, so + * if we resize the HT while there is the saving child at work actually + * a lot of memory movements in the parent will cause a lot of pages + * copied. */ + if (!server.bgsaveinprogress) tryResizeHashTables(); + /* Show information about connected clients */ if (!(loops % 5)) { redisLog(REDIS_DEBUG,"%d clients connected (%d slaves), %zu bytes in use", @@ -1058,6 +1087,7 @@ static void freeClient(redisClient *c) { server.master = NULL; server.replstate = REDIS_REPL_CONNECT; } + zfree(c->argv); zfree(c); } @@ -1237,8 +1267,17 @@ static int processCommand(redisClient *c) { static void replicationFeedSlaves(list *slaves, struct redisCommand *cmd, int dictid, robj **argv, int argc) { listNode *ln; - robj *outv[REDIS_MAX_ARGS*4]; /* enough room for args, spaces, newlines */ int outc = 0, j; + robj **outv; + /* (args*2)+1 is enough room for args, spaces, newlines */ + robj *static_outv[REDIS_STATIC_ARGS*2+1]; + + if (argc <= REDIS_STATIC_ARGS) { + outv = static_outv; + } else { + outv = zmalloc(sizeof(robj*)*(argc*2+1)); + if (!outv) oom("replicationFeedSlaves"); + } for (j = 0; j < argc; j++) { if (j != 0) outv[outc++] = shared.space; @@ -1292,6 +1331,7 @@ static void replicationFeedSlaves(list *slaves, struct redisCommand *cmd, int di for (j = 0; j < outc; j++) addReply(slave,outv[j]); } for (j = 0; j < outc; j++) decrRefCount(outv[j]); + if (outv != static_outv) zfree(outv); } static void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { @@ -1349,9 +1389,14 @@ again: return; } argv = sdssplitlen(query,sdslen(query)," ",1,&argc); - sdsfree(query); if (argv == NULL) oom("sdssplitlen"); - for (j = 0; j < argc && j < REDIS_MAX_ARGS; j++) { + sdsfree(query); + + if (c->argv) zfree(c->argv); + c->argv = zmalloc(sizeof(robj*)*argc); + if (c->argv == NULL) oom("allocating arguments list for client"); + + for (j = 0; j < argc; j++) { if (sdslen(argv[j])) { c->argv[c->argc] = createObject(REDIS_STRING,argv[j]); c->argc++; @@ -1410,6 +1455,7 @@ static redisClient *createClient(int fd) { c->fd = fd; c->querybuf = sdsempty(); c->argc = 0; + c->argv = NULL; c->bulklen = -1; c->sentlen = 0; c->flags = 0; @@ -1704,7 +1750,7 @@ static int rdbSaveLzfStringObject(FILE *fp, robj *obj) { /* We require at least four bytes compression for this to be worth it */ outlen = sdslen(obj->ptr)-4; if (outlen <= 0) return 0; - if ((out = zmalloc(outlen)) == NULL) return 0; + if ((out = zmalloc(outlen+1)) == NULL) return 0; comprlen = lzf_compress(obj->ptr, sdslen(obj->ptr), out, outlen); if (comprlen == 0) { zfree(out); @@ -1741,7 +1787,7 @@ static int rdbSaveStringObject(FILE *fp, robj *obj) { /* Try LZF compression - under 20 bytes it's unable to compress even * aaaaaaaaaaaaaaaaaa so skip it */ - if (len > 20) { + if (1 && len > 20) { int retval; retval = rdbSaveLzfStringObject(fp,obj); @@ -1879,6 +1925,11 @@ static int rdbSaveBackground(char *filename) { } } else { /* Parent */ + if (childpid == -1) { + redisLog(REDIS_WARNING,"Can't save in background: fork: %s", + strerror(errno)); + return REDIS_ERR; + } redisLog(REDIS_NOTICE,"Background saving started by pid %d",childpid); server.bgsaveinprogress = 1; return REDIS_OK; @@ -1970,6 +2021,7 @@ static robj *rdbLoadLzfStringObject(FILE*fp, int rdbver) { if ((val = sdsnewlen(NULL,len)) == NULL) goto err; if (fread(c,clen,1,fp) == 0) goto err; if (lzf_decompress(c,clen,val,len) == 0) goto err; + zfree(c); return createObject(REDIS_STRING,val); err: zfree(c); @@ -2263,11 +2315,24 @@ static void decrbyCommand(redisClient *c) { /* ========================= Type agnostic commands ========================= */ static void delCommand(redisClient *c) { - if (deleteKey(c->db,c->argv[1])) { - server.dirty++; - addReply(c,shared.cone); - } else { + int deleted = 0, j; + + for (j = 1; j < c->argc; j++) { + if (deleteKey(c->db,c->argv[j])) { + server.dirty++; + deleted++; + } + } + switch(deleted) { + case 0: addReply(c,shared.czero); + break; + case 1: + addReply(c,shared.cone); + break; + default: + addReplySds(c,sdscatprintf(sdsempty(),":%d\r\n",deleted)); + break; } } @@ -2392,8 +2457,9 @@ static void shutdownCommand(redisClient *c) { /* XXX: TODO kill the child if there is a bgsave in progress */ if (rdbSave(server.dbfilename) == REDIS_OK) { if (server.daemonize) { - unlink(server.pidfile); + unlink(server.pidfile); } + redisLog(REDIS_WARNING,"%zu bytes used at exit",zmalloc_used_memory()); redisLog(REDIS_WARNING,"Server exit now, bye bye..."); exit(1); } else { @@ -2995,14 +3061,17 @@ static void sinterstoreCommand(redisClient *c) { sinterGenericCommand(c,c->argv+2,c->argc-2,c->argv[1]); } -static void sunionGenericCommand(redisClient *c, robj **setskeys, int setsnum, robj *dstkey) { +#define REDIS_OP_UNION 0 +#define REDIS_OP_DIFF 1 + +static void sunionDiffGenericCommand(redisClient *c, robj **setskeys, int setsnum, robj *dstkey, int op) { dict **dv = zmalloc(sizeof(dict*)*setsnum); dictIterator *di; dictEntry *de; - robj *lenobj = NULL, *dstset = NULL; + robj *dstset = NULL; int j, cardinality = 0; - if (!dv) oom("sunionCommand"); + if (!dv) oom("sunionDiffGenericCommand"); for (j = 0; j < setsnum; j++) { robj *setobj; @@ -3031,11 +3100,7 @@ static void sunionGenericCommand(redisClient *c, robj **setskeys, int setsnum, r * the intersection set size, so we use a trick, append an empty object * to the output list and save the pointer to later modify it with the * right length */ - if (!dstkey) { - lenobj = createObject(REDIS_STRING,NULL); - addReply(c,lenobj); - decrRefCount(lenobj); - } else { + if (dstkey) { /* If we have a target key where to store the resulting set * create this key with an empty set inside */ deleteKey(c->db,dstkey); @@ -3047,6 +3112,7 @@ static void sunionGenericCommand(redisClient *c, robj **setskeys, int setsnum, r /* Iterate all the elements of all the sets, add every element a single * time to the result set */ for (j = 0; j < setsnum; j++) { + if (op == REDIS_OP_DIFF && j == 0 && !dv[j]) break; /* result set is empty */ if (!dv[j]) continue; /* non existing keys are like empty sets */ di = dictGetIterator(dv[j]); @@ -3057,22 +3123,41 @@ static void sunionGenericCommand(redisClient *c, robj **setskeys, int setsnum, r /* dictAdd will not add the same element multiple times */ ele = dictGetEntryKey(de); - if (dictAdd(dstset->ptr,ele,NULL) == DICT_OK) { - incrRefCount(ele); - if (!dstkey) { - addReplySds(c,sdscatprintf(sdsempty(), - "$%d\r\n",sdslen(ele->ptr))); - addReply(c,ele); - addReply(c,shared.crlf); + if (op == REDIS_OP_UNION || j == 0) { + if (dictAdd(dstset->ptr,ele,NULL) == DICT_OK) { + incrRefCount(ele); cardinality++; } + } else if (op == REDIS_OP_DIFF) { + if (dictDelete(dstset->ptr,ele) == DICT_OK) { + cardinality--; + } } } dictReleaseIterator(di); + + if (op == REDIS_OP_DIFF && cardinality == 0) break; /* result set is empty */ } + /* Output the content of the resulting set, if not in STORE mode */ + if (!dstkey) { + addReplySds(c,sdscatprintf(sdsempty(),"*%d\r\n",cardinality)); + di = dictGetIterator(dstset->ptr); + if (!di) oom("dictGetIterator"); + while((de = dictNext(di)) != NULL) { + robj *ele; + + ele = dictGetEntryKey(de); + addReplySds(c,sdscatprintf(sdsempty(), + "$%d\r\n",sdslen(ele->ptr))); + addReply(c,ele); + addReply(c,shared.crlf); + } + dictReleaseIterator(di); + } + + /* Cleanup */ if (!dstkey) { - lenobj->ptr = sdscatprintf(sdsempty(),"*%d\r\n",cardinality); decrRefCount(dstset); } else { addReply(c,shared.ok); @@ -3082,11 +3167,19 @@ static void sunionGenericCommand(redisClient *c, robj **setskeys, int setsnum, r } static void sunionCommand(redisClient *c) { - sunionGenericCommand(c,c->argv+1,c->argc-1,NULL); + sunionDiffGenericCommand(c,c->argv+1,c->argc-1,NULL,REDIS_OP_UNION); } static void sunionstoreCommand(redisClient *c) { - sunionGenericCommand(c,c->argv+2,c->argc-2,c->argv[1]); + sunionDiffGenericCommand(c,c->argv+2,c->argc-2,c->argv[1],REDIS_OP_UNION); +} + +static void sdiffCommand(redisClient *c) { + sunionDiffGenericCommand(c,c->argv+1,c->argc-1,NULL,REDIS_OP_DIFF); +} + +static void sdiffstoreCommand(redisClient *c) { + sunionDiffGenericCommand(c,c->argv+2,c->argc-2,c->argv[1],REDIS_OP_DIFF); } static void flushdbCommand(redisClient *c) { @@ -3809,6 +3902,28 @@ static int syncWithMaster(void) { /* =================================== Main! ================================ */ +#ifdef __linux__ +int linuxOvercommitMemoryValue(void) { + FILE *fp = fopen("/proc/sys/vm/overcommit_memory","r"); + char buf[64]; + + if (!fp) return -1; + if (fgets(buf,64,fp) == NULL) { + fclose(fp); + return -1; + } + fclose(fp); + + return atoi(buf); +} + +void linuxOvercommitMemoryWarning(void) { + if (linuxOvercommitMemoryValue() == 0) { + redisLog(REDIS_WARNING,"WARNING overcommit_memory is set to 0! Background save may fail under low condition memory. To fix this issue add 'echo 1 > /proc/sys/vm/overcommit_memory' in your init scripts."); + } +} +#endif /* __linux__ */ + static void daemonize(void) { int fd; FILE *fp; @@ -3834,6 +3949,10 @@ static void daemonize(void) { } int main(int argc, char **argv) { +#ifdef __linux__ + linuxOvercommitMemoryWarning(); +#endif + initServerConfig(); if (argc == 2) { ResetServerSaveParams();