From: antirez Date: Thu, 13 May 2010 12:24:41 +0000 (+0200) Subject: conflicts resolved X-Git-Url: https://git.saurik.com/redis.git/commitdiff_plain/9682a58fca1c53cecbf1b7a65bcf343f1f53c098?hp=9376e434f08003678e8042855bc5c35d406b8db2 conflicts resolved --- diff --git a/.gitignore b/.gitignore index 1b0b8578..5463a140 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ redis-cli redis-server redis-benchmark redis-check-dump +redis-check-aof doc-tools mkrelease.sh release diff --git a/Makefile b/Makefile index 13b6bf45..0d7cd28f 100644 --- a/Makefile +++ b/Makefile @@ -20,13 +20,15 @@ OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort BENCHOBJ = ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o linenoise.o CHECKDUMPOBJ = redis-check-dump.o lzf_c.o lzf_d.o +CHECKAOFOBJ = redis-check-aof.o PRGNAME = redis-server BENCHPRGNAME = redis-benchmark CLIPRGNAME = redis-cli CHECKDUMPPRGNAME = redis-check-dump +CHECKAOFPRGNAME = redis-check-aof -all: redis-server redis-benchmark redis-cli redis-check-dump +all: redis-server redis-benchmark redis-cli redis-check-dump redis-check-aof # Deps (use make dep to generate this) adlist.o: adlist.c adlist.h zmalloc.h @@ -68,11 +70,14 @@ redis-cli: $(CLIOBJ) redis-check-dump: $(CHECKDUMPOBJ) $(CC) -o $(CHECKDUMPPRGNAME) $(CCOPT) $(DEBUG) $(CHECKDUMPOBJ) +redis-check-aof: $(CHECKAOFOBJ) + $(CC) -o $(CHECKAOFPRGNAME) $(CCOPT) $(DEBUG) $(CHECKAOFOBJ) + .c.o: $(CC) -c $(CFLAGS) $(DEBUG) $(COMPILE_TIME) $< clean: - rm -rf $(PRGNAME) $(BENCHPRGNAME) $(CLIPRGNAME) $(CHECKDUMPPRGNAME) *.o *.gcda *.gcno *.gcov + rm -rf $(PRGNAME) $(BENCHPRGNAME) $(CLIPRGNAME) $(CHECKDUMPPRGNAME) $(CHECKAOFPRGNAME) *.o *.gcda *.gcno *.gcov dep: $(CC) -MM *.c diff --git a/TODO b/TODO index 5e1de7a5..5ca3cada 100644 --- a/TODO +++ b/TODO @@ -1,27 +1,18 @@ Redis TODO and Roadmap -VERSION 2.0 TODO +VERSION 2.2 TODO ================ * BRPOPLPUSH -* List ops like L/RPUSH L/RPOP should return the new list length. * Save dataset / fsync() on SIGTERM -* MULTI/EXEC should support the "EXEC FSYNC" form? -* BLPOP & C. tests (write a non blocking Tcl client as first step) -* Once ZRANK is implemented, change the implementation of ZCOUNT to use the augmented skiplist in order to be much faster. +* Change the implementation of ZCOUNT to use the augmented skiplist in order to be much faster. * Write doc for ZCOUNT, and for open / closed intervals of sorted sets range operations. Virtual Memory sub-TODO: -* Check if the page selection algorithm is working well -* Divide swappability of objects by refcount * Use multiple open FDs against the VM file, one for thread. * vm-min-age option -* Make sure objects loaded from the VM are specially encoded when possible. * Check what happens performance-wise if instead to create threads again and again the same threads are reused forever. Note: this requires a way to disable this clients in the child, but waiting for empty new jobs queue can be enough. -* Sets of integers are slow to load, for a number of reasons. Fix it. (use slow_sets.rdb file for debugging). (p.s. this was now partially fixed). -* Hashes (GET/SET/DEL/INCRBY/EXISTS/FIELDS/LEN/MSET/MGET). Special encoding for hashes with less than N elements. -* Write documentation for APPEND * Implement LEN, PEEK, POKE, SETBIT, GETBIT VERSION 2.2 TODO (Fault tolerant sharding) diff --git a/redis-check-aof.c b/redis-check-aof.c new file mode 100644 index 00000000..ff0d1f82 --- /dev/null +++ b/redis-check-aof.c @@ -0,0 +1,185 @@ +#include "fmacros.h" +#include +#include +#include +#include +#include +#include "config.h" + +#define ERROR(...) { \ + char __buf[1024]; \ + sprintf(__buf, __VA_ARGS__); \ + sprintf(error, "0x%08lx: %s", epos, __buf); \ +} + +static char error[1024]; +static long epos; + +int consumeNewline(char *buf) { + if (strncmp(buf,"\r\n",2) != 0) { + ERROR("Expected \\r\\n, got: %02x%02x",buf[0],buf[1]); + return 0; + } + return 1; +} + +int readLong(FILE *fp, char prefix, long *target) { + char buf[128], *eptr; + epos = ftell(fp); + if (fgets(buf,sizeof(buf),fp) == NULL) { + return 0; + } + if (buf[0] != prefix) { + ERROR("Expected prefix '%c', got: '%c'",buf[0],prefix); + return 0; + } + *target = strtol(buf+1,&eptr,10); + return consumeNewline(eptr); +} + +int readBytes(FILE *fp, char *target, long length) { + long real; + epos = ftell(fp); + real = fread(target,1,length,fp); + if (real != length) { + ERROR("Expected to read %ld bytes, got %ld bytes",length,real); + return 0; + } + return 1; +} + +int readString(FILE *fp, char** target) { + long len; + *target = NULL; + if (!readLong(fp,'$',&len)) { + return 0; + } + + /* Increase length to also consume \r\n */ + len += 2; + *target = (char*)malloc(len); + if (!readBytes(fp,*target,len)) { + return 0; + } + if (!consumeNewline(*target+len-2)) { + return 0; + } + (*target)[len-2] = '\0'; + return 1; +} + +int readArgc(FILE *fp, long *target) { + return readLong(fp,'*',target); +} + +long process(FILE *fp) { + long argc, pos = 0; + int i, multi = 0; + char *str; + + while(1) { + if (!multi) pos = ftell(fp); + if (!readArgc(fp, &argc)) break; + + for (i = 0; i < argc; i++) { + if (!readString(fp,&str)) break; + if (i == 0) { + if (strcasecmp(str, "multi") == 0) { + if (multi++) { + ERROR("Unexpected MULTI"); + break; + } + } else if (strcasecmp(str, "exec") == 0) { + if (--multi) { + ERROR("Unexpected EXEC"); + break; + } + } + } + free(str); + } + + /* Stop if the loop did not finish */ + if (i < argc) { + if (str) free(str); + break; + } + } + + if (feof(fp) && multi && strlen(error) == 0) { + ERROR("Reached EOF before reading EXEC for MULTI"); + } + if (strlen(error) > 0) { + printf("%s\n", error); + } + return pos; +} + +int main(int argc, char **argv) { + char *filename; + int fix = 0; + + if (argc < 2) { + printf("Usage: %s [--fix] \n", argv[0]); + exit(1); + } else if (argc == 2) { + filename = argv[1]; + } else if (argc == 3) { + if (strcmp(argv[1],"--fix") != 0) { + printf("Invalid argument: %s\n", argv[1]); + exit(1); + } + filename = argv[2]; + fix = 1; + } else { + printf("Invalid arguments\n"); + exit(1); + } + + FILE *fp = fopen(filename,"r+"); + if (fp == NULL) { + printf("Cannot open file: %s\n", filename); + exit(1); + } + + struct redis_stat sb; + if (redis_fstat(fileno(fp),&sb) == -1) { + printf("Cannot stat file: %s\n", filename); + exit(1); + } + + long size = sb.st_size; + if (size == 0) { + printf("Empty file: %s\n", filename); + exit(1); + } + + long pos = process(fp); + long diff = size-pos; + if (diff > 0) { + if (fix) { + char buf[2]; + printf("This will shrink the AOF from %ld bytes, with %ld bytes, to %ld bytes\n",size,diff,pos); + printf("Continue? [y/N]: "); + if (fgets(buf,sizeof(buf),stdin) == NULL || + strncasecmp(buf,"y",1) != 0) { + printf("Aborting...\n"); + exit(1); + } + if (ftruncate(fileno(fp), pos) == -1) { + printf("Failed to truncate AOF\n"); + exit(1); + } else { + printf("Successfully truncated AOF\n"); + } + } else { + printf("AOF is not valid\n"); + exit(1); + } + } else { + printf("AOF is valid\n"); + } + + fclose(fp); + return 0; +} diff --git a/redis.c b/redis.c index f52dbb82..45caae8f 100644 --- a/redis.c +++ b/redis.c @@ -57,6 +57,7 @@ #include #include #include +#include #include #include @@ -624,8 +625,10 @@ static int pubsubUnsubscribeAllPatterns(redisClient *c, int notify); static void freePubsubPattern(void *p); static int listMatchPubsubPattern(void *a, void *b); static int compareStringObjects(robj *a, robj *b); +static int equalStringObjects(robj *a, robj *b); static void usage(); static int rewriteAppendOnlyFileBackground(void); +static int vmSwapObjectBlocking(robj *key, robj *val); static void authCommand(redisClient *c); static void pingCommand(redisClient *c); @@ -1023,6 +1026,30 @@ static long long memtoll(const char *p, int *err) { return val*mul; } +/* Convert a long long into a string. Returns the number of + * characters needed to represent the number, that can be shorter if passed + * buffer length is not enough to store the whole number. */ +static int ll2string(char *s, size_t len, long long value) { + char buf[32], *p; + unsigned long long v; + size_t l; + + if (len == 0) return 0; + v = (value < 0) ? -value : value; + p = buf+31; /* point to the last character */ + do { + *p-- = '0'+(v%10); + v /= 10; + } while(v); + if (value < 0) *p-- = '-'; + p++; + l = 32-(p-buf); + if (l+1 > len) l = len-1; /* Make sure it fits, including the nul term */ + memcpy(s,p,l); + s[l] = '\0'; + return l; +} + static void redisLog(int level, const char *fmt, ...) { va_list ap; FILE *fp; @@ -1105,8 +1132,8 @@ static int dictEncObjKeyCompare(void *privdata, const void *key1, int cmp; if (o1->encoding == REDIS_ENCODING_INT && - o2->encoding == REDIS_ENCODING_INT && - o1->ptr == o2->ptr) return 1; + o2->encoding == REDIS_ENCODING_INT) + return o1->ptr == o2->ptr; o1 = getDecodedObject(o1); o2 = getDecodedObject(o2); @@ -1126,7 +1153,7 @@ static unsigned int dictEncObjHash(const void *key) { char buf[32]; int len; - len = snprintf(buf,32,"%ld",(long)o->ptr); + len = ll2string(buf,32,(long)o->ptr); return dictGenHashFunction((unsigned char*)buf, len); } else { unsigned int hash; @@ -1636,7 +1663,7 @@ static void initServerConfig() { server.glueoutputbuf = 1; server.daemonize = 0; server.appendonly = 0; - server.appendfsync = APPENDFSYNC_ALWAYS; + server.appendfsync = APPENDFSYNC_EVERYSEC; server.lastfsync = time(NULL); server.appendfd = -1; server.appendseldb = -1; /* Make sure the first time will not match */ @@ -2660,7 +2687,7 @@ static void *dupClientReplyValue(void *o) { } static int listMatchObjects(void *a, void *b) { - return compareStringObjects(a,b) == 0; + return equalStringObjects(a,b); } static redisClient *createClient(int fd) { @@ -2900,7 +2927,7 @@ static robj *createStringObjectFromLongLong(long long value) { o->encoding = REDIS_ENCODING_INT; o->ptr = (void*)((long)value); } else { - o->ptr = sdscatprintf(sdsempty(),"%lld",value); + o = createObject(REDIS_STRING,sdsfromlonglong(value)); } } return o; @@ -3115,7 +3142,7 @@ static int isStringRepresentableAsLong(sds s, long *longval) { value = strtol(s, &endptr, 10); if (endptr[0] != '\0') return REDIS_ERR; - slen = snprintf(buf,32,"%ld",value); + slen = ll2string(buf,32,value); /* If the number converted back into a string is not identical * then it's not possible to encode the string as integer */ @@ -3168,7 +3195,7 @@ static robj *getDecodedObject(robj *o) { if (o->type == REDIS_STRING && o->encoding == REDIS_ENCODING_INT) { char buf[32]; - snprintf(buf,32,"%ld",(long)o->ptr); + ll2string(buf,32,(long)o->ptr); dec = createStringObject(buf,strlen(buf)); return dec; } else { @@ -3178,7 +3205,7 @@ static robj *getDecodedObject(robj *o) { /* Compare two string objects via strcmp() or alike. * Note that the objects may be integer-encoded. In such a case we - * use snprintf() to get a string representation of the numbers on the stack + * use ll2string() to get a string representation of the numbers on the stack * and compare the strings, it's much faster than calling getDecodedObject(). * * Important note: if objects are not integer encoded, but binary-safe strings, @@ -3191,14 +3218,14 @@ static int compareStringObjects(robj *a, robj *b) { if (a == b) return 0; if (a->encoding != REDIS_ENCODING_RAW) { - snprintf(bufa,sizeof(bufa),"%ld",(long) a->ptr); + ll2string(bufa,sizeof(bufa),(long) a->ptr); astr = bufa; bothsds = 0; } else { astr = a->ptr; } if (b->encoding != REDIS_ENCODING_RAW) { - snprintf(bufb,sizeof(bufb),"%ld",(long) b->ptr); + ll2string(bufb,sizeof(bufb),(long) b->ptr); bstr = bufb; bothsds = 0; } else { @@ -3207,6 +3234,18 @@ static int compareStringObjects(robj *a, robj *b) { return bothsds ? sdscmp(astr,bstr) : strcmp(astr,bstr); } +/* Equal string objects return 1 if the two objects are the same from the + * point of view of a string comparison, otherwise 0 is returned. Note that + * this function is faster then checking for (compareStringObject(a,b) == 0) + * because it can perform some more optimization. */ +static int equalStringObjects(robj *a, robj *b) { + if (a->encoding != REDIS_ENCODING_RAW && b->encoding != REDIS_ENCODING_RAW){ + return a->ptr == b->ptr; + } else { + return compareStringObjects(a,b) == 0; + } +} + static size_t stringObjectLen(robj *o) { redisAssert(o->type == REDIS_STRING); if (o->encoding == REDIS_ENCODING_RAW) { @@ -3214,7 +3253,7 @@ static size_t stringObjectLen(robj *o) { } else { char buf[32]; - return snprintf(buf,32,"%ld",(long)o->ptr); + return ll2string(buf,32,(long)o->ptr); } } @@ -3345,22 +3384,12 @@ static int rdbSaveLen(FILE *fp, uint32_t len) { return 0; } -/* String objects in the form "2391" "-100" without any space and with a - * range of values that can fit in an 8, 16 or 32 bit signed value can be - * encoded as integers to save space */ -static int rdbTryIntegerEncoding(char *s, size_t len, unsigned char *enc) { - long long value; - char *endptr, buf[32]; - - /* Check if it's possible to encode this value as a number */ - value = strtoll(s, &endptr, 10); - if (endptr[0] != '\0') return 0; - snprintf(buf,32,"%lld",value); - - /* If the number converted back into a string is not identical - * then it's not possible to encode the string as integer */ - if (strlen(buf) != len || memcmp(buf,s,len)) return 0; - +/* Encode 'value' as an integer if possible (if integer will fit the + * supported range). If the function sucessful encoded the integer + * then the (up to 5 bytes) encoded representation is written in the + * string pointed by 'enc' and the length is returned. Otherwise + * 0 is returned. */ +static int rdbEncodeInteger(long long value, unsigned char *enc) { /* Finally check if it fits in our ranges */ if (value >= -(1<<7) && value <= (1<<7)-1) { enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT8; @@ -3383,6 +3412,25 @@ static int rdbTryIntegerEncoding(char *s, size_t len, unsigned char *enc) { } } +/* String objects in the form "2391" "-100" without any space and with a + * range of values that can fit in an 8, 16 or 32 bit signed value can be + * encoded as integers to save space */ +static int rdbTryIntegerEncoding(char *s, size_t len, unsigned char *enc) { + long long value; + char *endptr, buf[32]; + + /* Check if it's possible to encode this value as a number */ + value = strtoll(s, &endptr, 10); + if (endptr[0] != '\0') return 0; + ll2string(buf,32,value); + + /* If the number converted back into a string is not identical + * then it's not possible to encode the string as integer */ + if (strlen(buf) != len || memcmp(buf,s,len)) return 0; + + return rdbEncodeInteger(value,enc); +} + static int rdbSaveLzfStringObject(FILE *fp, unsigned char *s, size_t len) { size_t comprlen, outlen; unsigned char byte; @@ -3446,6 +3494,21 @@ static int rdbSaveRawString(FILE *fp, unsigned char *s, size_t len) { static int rdbSaveStringObject(FILE *fp, robj *obj) { int retval; + /* Avoid to decode the object, then encode it again, if the + * object is alrady integer encoded. */ + if (obj->encoding == REDIS_ENCODING_INT) { + long val = (long) obj->ptr; + unsigned char buf[5]; + int enclen; + + if ((enclen = rdbEncodeInteger(val,buf)) > 0) { + if (fwrite(buf,enclen,1,fp) == 0) return -1; + return 0; + } + /* otherwise... fall throught and continue with the usual + * code path. */ + } + /* Avoid incr/decr ref count business when possible. * This plays well with copy-on-write given that we are probably * in a child process (BGSAVE). Also this makes sure key objects @@ -3480,7 +3543,23 @@ static int rdbSaveDoubleValue(FILE *fp, double val) { len = 1; buf[0] = (val < 0) ? 255 : 254; } else { - snprintf((char*)buf+1,sizeof(buf)-1,"%.17g",val); +#if (DBL_MANT_DIG >= 52) && (LLONG_MAX == 0x7fffffffffffffffLL) + /* Check if the float is in a safe range to be casted into a + * long long. We are assuming that long long is 64 bit here. + * Also we are assuming that there are no implementations around where + * double has precision < 52 bit. + * + * Under this assumptions we test if a double is inside an interval + * where casting to long long is safe. Then using two castings we + * make sure the decimal part is zero. If all this is true we use + * integer printing function that is much faster. */ + double min = -4503599627370495; /* (2^52)-1 */ + double max = 4503599627370496; /* -(2^52) */ + if (val > min && val < max && val == ((double)((long long)val))) + ll2string((char*)buf+1,sizeof(buf),(long long)val); + else +#endif + snprintf((char*)buf+1,sizeof(buf)-1,"%.17g",val); buf[0] = strlen((char*)buf+1); len = buf[0]+1; } @@ -3764,7 +3843,11 @@ static uint32_t rdbLoadLen(FILE *fp, int *isencoded) { } } -static robj *rdbLoadIntegerObject(FILE *fp, int enctype) { +/* Load an integer-encoded object from file 'fp', with the specified + * encoding type 'enctype'. If encode is true the function may return + * an integer-encoded object as reply, otherwise the returned object + * will always be encoded as a raw string. */ +static robj *rdbLoadIntegerObject(FILE *fp, int enctype, int encode) { unsigned char enc[4]; long long val; @@ -3785,7 +3868,10 @@ static robj *rdbLoadIntegerObject(FILE *fp, int enctype) { val = 0; /* anti-warning */ redisPanic("Unknown RDB integer encoding type"); } - return createObject(REDIS_STRING,sdscatprintf(sdsempty(),"%lld",val)); + if (encode) + return createStringObjectFromLongLong(val); + else + return createObject(REDIS_STRING,sdsfromlonglong(val)); } static robj *rdbLoadLzfStringObject(FILE*fp) { @@ -3807,7 +3893,7 @@ err: return NULL; } -static robj *rdbLoadStringObject(FILE*fp) { +static robj *rdbGenericLoadStringObject(FILE*fp, int encode) { int isencoded; uint32_t len; sds val; @@ -3818,7 +3904,7 @@ static robj *rdbLoadStringObject(FILE*fp) { case REDIS_RDB_ENC_INT8: case REDIS_RDB_ENC_INT16: case REDIS_RDB_ENC_INT32: - return rdbLoadIntegerObject(fp,len); + return rdbLoadIntegerObject(fp,len,encode); case REDIS_RDB_ENC_LZF: return rdbLoadLzfStringObject(fp); default: @@ -3835,6 +3921,14 @@ static robj *rdbLoadStringObject(FILE*fp) { return createObject(REDIS_STRING,val); } +static robj *rdbLoadStringObject(FILE *fp) { + return rdbGenericLoadStringObject(fp,0); +} + +static robj *rdbLoadEncodedStringObject(FILE *fp) { + return rdbGenericLoadStringObject(fp,1); +} + /* For information about double serialization check rdbSaveDoubleValue() */ static int rdbLoadDoubleValue(FILE *fp, double *val) { char buf[128]; @@ -3861,7 +3955,7 @@ static robj *rdbLoadObject(int type, FILE *fp) { redisLog(REDIS_DEBUG,"LOADING OBJECT %d (at %d)\n",type,ftell(fp)); if (type == REDIS_STRING) { /* Read string value */ - if ((o = rdbLoadStringObject(fp)) == NULL) return NULL; + if ((o = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; o = tryObjectEncoding(o); } else if (type == REDIS_LIST || type == REDIS_SET) { /* Read list/set value */ @@ -3877,7 +3971,7 @@ static robj *rdbLoadObject(int type, FILE *fp) { while(listlen--) { robj *ele; - if ((ele = rdbLoadStringObject(fp)) == NULL) return NULL; + if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; ele = tryObjectEncoding(ele); if (type == REDIS_LIST) { listAddNodeTail((list*)o->ptr,ele); @@ -3898,7 +3992,7 @@ static robj *rdbLoadObject(int type, FILE *fp) { robj *ele; double *score = zmalloc(sizeof(double)); - if ((ele = rdbLoadStringObject(fp)) == NULL) return NULL; + if ((ele = rdbLoadEncodedStringObject(fp)) == NULL) return NULL; ele = tryObjectEncoding(ele); if (rdbLoadDoubleValue(fp,score) == -1) return NULL; dictAdd(zs->dict,ele,score); @@ -3951,13 +4045,13 @@ static robj *rdbLoadObject(int type, FILE *fp) { static int rdbLoad(char *filename) { FILE *fp; - robj *keyobj = NULL; uint32_t dbid; int type, retval, rdbver; + int swap_all_values = 0; dict *d = server.db[0].dict; redisDb *db = server.db+0; char buf[1024]; - time_t expiretime = -1, now = time(NULL); + time_t expiretime, now = time(NULL); long long loadedkeys = 0; fp = fopen(filename,"r"); @@ -3976,8 +4070,9 @@ static int rdbLoad(char *filename) { return REDIS_ERR; } while(1) { - robj *o; + robj *key, *val; + expiretime = -1; /* Read type. */ if ((type = rdbLoadType(fp)) == -1) goto eoferr; if (type == REDIS_EXPIRETIME) { @@ -3999,36 +4094,61 @@ static int rdbLoad(char *filename) { continue; } /* Read key */ - if ((keyobj = rdbLoadStringObject(fp)) == NULL) goto eoferr; + if ((key = rdbLoadStringObject(fp)) == NULL) goto eoferr; /* Read value */ - if ((o = rdbLoadObject(type,fp)) == NULL) goto eoferr; + if ((val = rdbLoadObject(type,fp)) == NULL) goto eoferr; + /* Check if the key already expired */ + if (expiretime != -1 && expiretime < now) { + decrRefCount(key); + decrRefCount(val); + continue; + } /* Add the new object in the hash table */ - retval = dictAdd(d,keyobj,o); + retval = dictAdd(d,key,val); if (retval == DICT_ERR) { - redisLog(REDIS_WARNING,"Loading DB, duplicated key (%s) found! Unrecoverable error, exiting now.", keyobj->ptr); + redisLog(REDIS_WARNING,"Loading DB, duplicated key (%s) found! Unrecoverable error, exiting now.", key->ptr); exit(1); } + loadedkeys++; /* Set the expire time if needed */ - if (expiretime != -1) { - setExpire(db,keyobj,expiretime); - /* Delete this key if already expired */ - if (expiretime < now) deleteKey(db,keyobj); - expiretime = -1; - } - keyobj = o = NULL; + if (expiretime != -1) setExpire(db,key,expiretime); + /* Handle swapping while loading big datasets when VM is on */ - loadedkeys++; - if (server.vm_enabled && (loadedkeys % 5000) == 0) { + + /* If we detecter we are hopeless about fitting something in memory + * we just swap every new key on disk. Directly... + * Note that's important to check for this condition before resorting + * to random sampling, otherwise we may try to swap already + * swapped keys. */ + if (swap_all_values) { + dictEntry *de = dictFind(d,key); + + /* de may be NULL since the key already expired */ + if (de) { + key = dictGetEntryKey(de); + val = dictGetEntryVal(de); + + if (vmSwapObjectBlocking(key,val) == REDIS_OK) { + dictGetEntryVal(de) = NULL; + } + } + continue; + } + + /* If we have still some hope of having some value fitting memory + * then we try random sampling. */ + if (!swap_all_values && server.vm_enabled && (loadedkeys % 5000) == 0) { while (zmalloc_used_memory() > server.vm_max_memory) { if (vmSwapOneObjectBlocking() == REDIS_ERR) break; } + if (zmalloc_used_memory() > server.vm_max_memory) + swap_all_values = 1; /* We are already using too much mem */ } } fclose(fp); return REDIS_OK; eoferr: /* unexpected end of file is handled here with a fatal exit */ - if (keyobj) decrRefCount(keyobj); redisLog(REDIS_WARNING,"Short read or OOM loading DB. Unrecoverable error, aborting now."); exit(1); return REDIS_ERR; /* Just to avoid warning */ @@ -4209,8 +4329,8 @@ static void incrDecrCommand(redisClient *c, long long incr) { robj *o; o = lookupKeyWrite(c->db,c->argv[1]); - - if (getLongLongFromObjectOrReply(c, o, &value, NULL) != REDIS_OK) return; + if (o != NULL && checkType(c,o,REDIS_STRING)) return; + if (getLongLongFromObjectOrReply(c,o,&value,NULL) != REDIS_OK) return; value += incr; o = createObject(REDIS_STRING,sdscatprintf(sdsempty(),"%lld",value)); @@ -4833,7 +4953,7 @@ static void lremCommand(redisClient *c) { robj *ele = listNodeValue(ln); next = fromtail ? ln->prev : ln->next; - if (compareStringObjects(ele,c->argv[3]) == 0) { + if (equalStringObjects(ele,c->argv[3])) { listDelNode(list,ln); server.dirty++; removed++; @@ -5437,7 +5557,7 @@ static int zslDelete(zskiplist *zsl, double score, robj *obj) { /* We may have multiple elements with the same score, what we need * is to find the element with both the right score and object. */ x = x->forward[0]; - if (x && score == x->score && compareStringObjects(x->obj,obj) == 0) { + if (x && score == x->score && equalStringObjects(x->obj,obj)) { zslDeleteNode(zsl, x, update); zslFreeNode(x); return 1; @@ -5542,7 +5662,7 @@ static unsigned long zslGetRank(zskiplist *zsl, double score, robj *o) { } /* x might be equal to zsl->header, so test if obj is non-NULL */ - if (x->obj && compareStringObjects(x->obj,o) == 0) { + if (x->obj && equalStringObjects(x->obj,o)) { return rank; } } @@ -9659,7 +9779,7 @@ static int dontWaitForSwappedKey(redisClient *c, robj *key) { /* Remove the key from the list of keys this client is waiting for. */ listRewind(c->io_keys,&li); while ((ln = listNext(&li)) != NULL) { - if (compareStringObjects(ln->value,key) == 0) { + if (equalStringObjects(ln->value,key)) { listDelNode(c->io_keys,ln); break; } @@ -9719,6 +9839,50 @@ static void configSetCommand(redisClient *c) { server.masterauth = zstrdup(o->ptr); } else if (!strcasecmp(c->argv[2]->ptr,"maxmemory")) { server.maxmemory = strtoll(o->ptr, NULL, 10); + } else if (!strcasecmp(c->argv[2]->ptr,"appendfsync")) { + if (!strcasecmp(o->ptr,"no")) { + server.appendfsync = APPENDFSYNC_NO; + } else if (!strcasecmp(o->ptr,"everysec")) { + server.appendfsync = APPENDFSYNC_EVERYSEC; + } else if (!strcasecmp(o->ptr,"always")) { + server.appendfsync = APPENDFSYNC_ALWAYS; + } else { + goto badfmt; + } + } else if (!strcasecmp(c->argv[2]->ptr,"save")) { + int vlen, j; + sds *v = sdssplitlen(o->ptr,sdslen(o->ptr)," ",1,&vlen); + + /* Perform sanity check before setting the new config: + * - Even number of args + * - Seconds >= 1, changes >= 0 */ + if (vlen & 1) { + sdsfreesplitres(v,vlen); + goto badfmt; + } + for (j = 0; j < vlen; j++) { + char *eptr; + long val; + + val = strtoll(v[j], &eptr, 10); + if (eptr[0] != '\0' || + ((j & 1) == 0 && val < 1) || + ((j & 1) == 1 && val < 0)) { + sdsfreesplitres(v,vlen); + goto badfmt; + } + } + /* Finally set the new config */ + resetServerSaveParams(); + for (j = 0; j < vlen; j += 2) { + time_t seconds; + int changes; + + seconds = strtoll(v[j],NULL,10); + changes = strtoll(v[j+1],NULL,10); + appendServerSaveParams(seconds, changes); + } + sdsfreesplitres(v,vlen); } else { addReplySds(c,sdscatprintf(sdsempty(), "-ERR not supported CONFIG parameter %s\r\n", @@ -9728,6 +9892,14 @@ static void configSetCommand(redisClient *c) { } decrRefCount(o); addReply(c,shared.ok); + return; + +badfmt: /* Bad format errors */ + addReplySds(c,sdscatprintf(sdsempty(), + "-ERR invalid argument '%s' for CONFIG SET '%s'\r\n", + (char*)o->ptr, + (char*)c->argv[2]->ptr)); + decrRefCount(o); } static void configGetCommand(redisClient *c) { @@ -9762,6 +9934,35 @@ static void configGetCommand(redisClient *c) { addReplyBulkCString(c,buf); matches++; } + if (stringmatch(pattern,"appendfsync",0)) { + char *policy; + + switch(server.appendfsync) { + case APPENDFSYNC_NO: policy = "no"; break; + case APPENDFSYNC_EVERYSEC: policy = "everysec"; break; + case APPENDFSYNC_ALWAYS: policy = "always"; break; + default: policy = "unknown"; break; /* too harmless to panic */ + } + addReplyBulkCString(c,"appendfsync"); + addReplyBulkCString(c,policy); + matches++; + } + if (stringmatch(pattern,"save",0)) { + sds buf = sdsempty(); + int j; + + for (j = 0; j < server.saveparamslen; j++) { + buf = sdscatprintf(buf,"%ld %d", + server.saveparams[j].seconds, + server.saveparams[j].changes); + if (j != server.saveparamslen-1) + buf = sdscatlen(buf," ",1); + } + addReplyBulkCString(c,"save"); + addReplyBulkCString(c,buf); + sdsfree(buf); + matches++; + } decrRefCount(o); lenobj->ptr = sdscatprintf(sdsempty(),"*%d\r\n",matches*2); } @@ -9805,7 +10006,7 @@ static int listMatchPubsubPattern(void *a, void *b) { pubsubPattern *pa = a, *pb = b; return (pa->client == pb->client) && - (compareStringObjects(pa->pattern,pb->pattern) == 0); + (equalStringObjects(pa->pattern,pb->pattern)); } /* Subscribe a client to a channel. Returns 1 if the operation succeeded, or @@ -10142,6 +10343,25 @@ static void debugCommand(redisClient *c) { } else { addReply(c,shared.err); } + } else if (!strcasecmp(c->argv[1]->ptr,"populate") && c->argc == 3) { + long keys, j; + robj *key, *val; + char buf[128]; + + if (getLongFromObjectOrReply(c, c->argv[2], &keys, NULL) != REDIS_OK) + return; + for (j = 0; j < keys; j++) { + snprintf(buf,sizeof(buf),"key:%lu",j); + key = createStringObject(buf,strlen(buf)); + if (lookupKeyRead(c->db,key) != NULL) { + decrRefCount(key); + continue; + } + snprintf(buf,sizeof(buf),"value:%lu",j); + val = createStringObject(buf,strlen(buf)); + dictAdd(c->db->dict,key,val); + } + addReply(c,shared.ok); } else { addReplySds(c,sdsnew( "-ERR Syntax error, try DEBUG [SEGFAULT|OBJECT |SWAPIN |SWAPOUT |RELOAD]\r\n")); diff --git a/redis.conf b/redis.conf index 4d3689ed..c48a2fb8 100644 --- a/redis.conf +++ b/redis.conf @@ -112,6 +112,10 @@ dir ./ # # This should stay commented out for backward compatibility and because most # people do not need auth (e.g. they run their own servers). +# +# Warning: since Redis is pretty fast an outside user can try up to +# 150k passwords per second against a good box. This means that you should +# use a very strong password otherwise it will be very easy to break. # # requirepass foobared diff --git a/sds.c b/sds.c index 8d1edcb7..feb1a621 100644 --- a/sds.c +++ b/sds.c @@ -335,3 +335,25 @@ cleanup: } #endif } + +void sdsfreesplitres(sds *tokens, int count) { + if (!tokens) return; + while(count--) + sdsfree(tokens[count]); + zfree(tokens); +} + +sds sdsfromlonglong(long long value) { + char buf[32], *p; + unsigned long long v; + + v = (value < 0) ? -value : value; + p = buf+31; /* point to the last character */ + do { + *p-- = '0'+(v%10); + v /= 10; + } while(v); + if (value < 0) *p-- = '-'; + p++; + return sdsnewlen(p,32-(p-buf)); +} diff --git a/sds.h b/sds.h index c8b42f77..8b632ff9 100644 --- a/sds.h +++ b/sds.h @@ -65,7 +65,9 @@ sds sdsrange(sds s, long start, long end); void sdsupdatelen(sds s); int sdscmp(sds s1, sds s2); sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count); +void sdsfreesplitres(sds *tokens, int count); void sdstolower(sds s); void sdstoupper(sds s); +sds sdsfromlonglong(long long value); #endif diff --git a/test-redis.tcl b/test-redis.tcl index d4a9ecdb..a3ced320 100644 --- a/test-redis.tcl +++ b/test-redis.tcl @@ -214,7 +214,7 @@ proc datasetDigest r { if {[$r zcard $k] == 0} { set aux {} } else { - set aux [::sha1::sha1 -hex [$r zrange $k 0 -1]] + set aux [::sha1::sha1 -hex [$r zrange $k 0 -1 withscores]] } } {hash} { if {[$r hlen $k] == 0} { @@ -373,7 +373,7 @@ proc main {} { test {INCR fails against a key holding a list} { $r rpush mylist 1 - catch {$r incr novar} err + catch {$r incr mylist} err $r rpop mylist format $err } {ERR*}