From: antirez Date: Tue, 4 Jan 2011 18:07:15 +0000 (+0100) Subject: Merge branch 'master' into unstable X-Git-Url: https://git.saurik.com/redis.git/commitdiff_plain/6eaad66373982aa9c1c0ff6b584c542670548d0e?hp=-c Merge branch 'master' into unstable --- 6eaad66373982aa9c1c0ff6b584c542670548d0e diff --combined src/Makefile index fe71b6cd,17fc435b..4d0f86db --- a/src/Makefile +++ b/src/Makefile @@@ -8,9 -8,11 +8,11 @@@ OPTIMIZATION?=-O ifeq ($(uname_S),SunOS) CFLAGS?= -std=c99 -pedantic $(OPTIMIZATION) -Wall -W -D__EXTENSIONS__ -D_XPG6 CCLINK?= -ldl -lnsl -lsocket -lm -lpthread + DEBUG?= -g -ggdb else CFLAGS?= -std=c99 -pedantic $(OPTIMIZATION) -Wall -W $(ARCH) $(PROF) CCLINK?= -lm -pthread + DEBUG?= -g -rdynamic -ggdb endif ifeq ($(USE_TCMALLOC),yes) @@@ -18,13 -20,12 +20,12 @@@ CFLAGS+= -DUSE_TCMALLOC endif CCOPT= $(CFLAGS) $(CCLINK) $(ARCH) $(PROF) - DEBUG?= -g -rdynamic -ggdb PREFIX= /usr/local INSTALL_BIN= $(PREFIX)/bin INSTALL= cp -p -OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o vm.o pubsub.o multi.o debug.o sort.o intset.o syncio.o +OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o dscache.o pubsub.o multi.o debug.o sort.o intset.o syncio.o diskstore.o BENCHOBJ = ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o CHECKDUMPOBJ = redis-check-dump.o lzf_c.o lzf_d.o @@@ -50,6 -51,7 +51,6 @@@ ae_select.o: ae_select. anet.o: anet.c fmacros.h anet.h aof.o: aof.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h -chprgname.o: chprgname.c config.o: config.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h db.o: db.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ @@@ -57,10 -59,6 +58,10 @@@ debug.o: debug.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h sha1.h dict.o: dict.c fmacros.h dict.h zmalloc.h +diskstore.o: diskstore.c redis.h fmacros.h config.h ae.h sds.h dict.h \ + adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h +dscache.o: dscache.c redis.h fmacros.h config.h ae.h sds.h dict.h \ + adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h intset.o: intset.c intset.h zmalloc.h lzf_c.o: lzf_c.c lzfP.h lzf_d.o: lzf_d.c lzfP.h @@@ -75,12 -73,11 +76,12 @@@ pubsub.o: pubsub.c redis.h fmacros.h co zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h rdb.o: rdb.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h lzf.h -redis-benchmark.o: redis-benchmark.c fmacros.h ae.h anet.h sds.h adlist.h \ - zmalloc.h +redis-benchmark.o: redis-benchmark.c fmacros.h ae.h \ + ../deps/hiredis/hiredis.h sds.h adlist.h zmalloc.h redis-check-aof.o: redis-check-aof.c fmacros.h config.h redis-check-dump.o: redis-check-dump.c lzf.h -redis-cli.o: redis-cli.c fmacros.h version.h sds.h adlist.h zmalloc.h +redis-cli.o: redis-cli.c fmacros.h version.h ../deps/hiredis/hiredis.h \ + sds.h zmalloc.h ../deps/linenoise/linenoise.h help.h redis.o: redis.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h release.o: release.c release.h @@@ -90,8 -87,7 +91,8 @@@ sds.o: sds.c sds.h zmalloc. sha1.o: sha1.c sha1.h sort.o: sort.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h pqsort.h -syncio.o: syncio.c +syncio.o: syncio.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h t_hash.o: t_hash.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h t_list.o: t_list.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ @@@ -104,6 -100,8 +105,6 @@@ t_zset.o: t_zset.c redis.h fmacros.h co zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.o: util.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h -vm.o: vm.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h ziplist.o: ziplist.c zmalloc.h ziplist.h zipmap.o: zipmap.c zmalloc.h zmalloc.o: zmalloc.c config.h @@@ -141,7 -139,7 +142,7 @@@ clean rm -rf $(PRGNAME) $(BENCHPRGNAME) $(CLIPRGNAME) $(CHECKDUMPPRGNAME) $(CHECKAOFPRGNAME) *.o *.gcda *.gcno *.gcov dep: - $(CC) -MM *.c + $(CC) -MM *.c -I ../deps/hiredis -I ../deps/linenoise test: redis-server (cd ..; tclsh8.5 tests/test_helper.tcl --tags "${TAGS}" --file "${FILE}") diff --combined src/redis.c index c567710e,8f4ca961..91371a00 --- a/src/redis.c +++ b/src/redis.c @@@ -345,7 -345,7 +345,7 @@@ unsigned int dictEncObjHash(const void } } -/* Sets type */ +/* Sets type and diskstore negative caching hash table */ dictType setDictType = { dictEncObjHash, /* hash function */ NULL, /* key dup */ @@@ -596,7 -596,7 +596,7 @@@ int serverCron(struct aeEventLoop *even } updateDictResizePolicy(); } - } else { + } else if (!server.ds_enabled) { /* If there is not a background saving in progress check if * we have to save now */ time_t now = time(NULL); @@@ -618,9 -618,28 +618,9 @@@ * in order to guarantee a strict consistency. */ if (server.masterhost == NULL) activeExpireCycle(); - /* Swap a few keys on disk if we are over the memory limit and VM - * is enbled. Try to free objects from the free list first. */ - if (vmCanSwapOut()) { - while (server.vm_enabled && zmalloc_used_memory() > - server.vm_max_memory) - { - int retval = (server.vm_max_threads == 0) ? - vmSwapOneObjectBlocking() : - vmSwapOneObjectThreaded(); - if (retval == REDIS_ERR && !(loops % 300) && - zmalloc_used_memory() > - (server.vm_max_memory+server.vm_max_memory/10)) - { - redisLog(REDIS_WARNING,"WARNING: vm-max-memory limit exceeded by more than 10%% but unable to swap more objects out!"); - } - /* Note that when using threade I/O we free just one object, - * because anyway when the I/O thread in charge to swap this - * object out will finish, the handler of completed jobs - * will try to swap more objects if we are still out of memory. */ - if (retval == REDIS_ERR || server.vm_max_threads > 0) break; - } - } + /* Remove a few cached objects from memory if we are over the + * configured memory limit */ + if (server.ds_enabled) cacheCron(); /* Replication cron function -- used to reconnect to master and * to detect transfer failures. */ @@@ -637,8 -656,8 +637,8 @@@ void beforeSleep(struct aeEventLoop *ev listNode *ln; redisClient *c; - /* Awake clients that got all the swapped keys they requested */ - if (server.vm_enabled && listLength(server.io_ready_clients)) { + /* Awake clients that got all the on disk keys they requested */ + if (server.ds_enabled && listLength(server.io_ready_clients)) { listIter li; listRewind(server.io_ready_clients,&li); @@@ -649,7 -668,7 +649,7 @@@ /* Resume the client. */ listDelNode(server.io_ready_clients,ln); c->flags &= (~REDIS_IO_WAIT); - server.vm_blocked_clients--; + server.cache_blocked_clients--; aeCreateFileEvent(server.el, c->fd, AE_READABLE, readQueryFromClient, c); cmd = lookupCommand(c->argv[0]->ptr); @@@ -768,17 -787,19 +768,17 @@@ void initServerConfig() server.maxmemory = 0; server.maxmemory_policy = REDIS_MAXMEMORY_VOLATILE_LRU; server.maxmemory_samples = 3; - server.vm_enabled = 0; - server.vm_swap_file = zstrdup("/tmp/redis-%p.vm"); - server.vm_page_size = 256; /* 256 bytes per page */ - server.vm_pages = 1024*1024*100; /* 104 millions of pages */ - server.vm_max_memory = 1024LL*1024*1024*1; /* 1 GB of RAM */ - server.vm_max_threads = 4; - server.vm_blocked_clients = 0; + server.ds_enabled = 0; + server.ds_path = sdsnew("/tmp/redis.ds"); + server.cache_max_memory = 64LL*1024*1024; /* 64 MB of RAM */ + server.cache_blocked_clients = 0; server.hash_max_zipmap_entries = REDIS_HASH_MAX_ZIPMAP_ENTRIES; server.hash_max_zipmap_value = REDIS_HASH_MAX_ZIPMAP_VALUE; server.list_max_ziplist_entries = REDIS_LIST_MAX_ZIPLIST_ENTRIES; server.list_max_ziplist_value = REDIS_LIST_MAX_ZIPLIST_VALUE; server.set_max_intset_entries = REDIS_SET_MAX_INTSET_ENTRIES; server.shutdown_asap = 0; + server.cache_flush_delay = 0; updateLRUClock(); resetServerSaveParams(); @@@ -827,8 -848,6 +827,8 @@@ void initServer() server.slaves = listCreate(); server.monitors = listCreate(); server.unblocked_clients = listCreate(); + server.cache_io_queue = listCreate(); + createSharedObjects(); server.el = aeCreateEventLoop(); server.db = zmalloc(sizeof(redisDb)*server.dbnum); @@@ -854,11 -873,8 +854,11 @@@ server.db[j].expires = dictCreate(&keyptrDictType,NULL); server.db[j].blocking_keys = dictCreate(&keylistDictType,NULL); server.db[j].watched_keys = dictCreate(&keylistDictType,NULL); - if (server.vm_enabled) + if (server.ds_enabled) { server.db[j].io_keys = dictCreate(&keylistDictType,NULL); + server.db[j].io_negcache = dictCreate(&setDictType,NULL); + server.db[j].io_queued = dictCreate(&setDictType,NULL); + } server.db[j].id = j; } server.pubsub_channels = dictCreate(&keylistDictType,NULL); @@@ -895,7 -911,7 +895,7 @@@ } } - if (server.vm_enabled) vmInit(); + if (server.ds_enabled) dsInit(); } /* Populates the Redis Command Table starting from the hard coded list @@@ -1034,8 -1050,8 +1034,8 @@@ int processCommand(redisClient *c) queueMultiCommand(c,cmd); addReply(c,shared.queued); } else { - if (server.vm_enabled && server.vm_max_threads > 0 && - blockClientOnSwappedKeys(c,cmd)) return REDIS_ERR; + if (server.ds_enabled && blockClientOnSwappedKeys(c,cmd)) + return REDIS_ERR; call(c,cmd); } return REDIS_OK; @@@ -1053,11 -1069,10 +1053,11 @@@ int prepareForShutdown() kill(server.bgsavechildpid,SIGKILL); rdbRemoveTempFile(server.bgsavechildpid); } - if (server.appendonly) { + if (server.ds_enabled) { + /* FIXME: flush all objects on disk */ + } else if (server.appendonly) { /* Append only file: fsync() the AOF and exit */ aof_fsync(server.appendfd); - if (server.vm_enabled) unlink(server.vm_swap_file); } else if (server.saveparamslen > 0) { /* Snapshotting. Perform a SYNC SAVE and exit */ if (rdbSave(server.dbfilename) != REDIS_OK) { @@@ -1170,7 -1185,7 +1170,7 @@@ sds genRedisInfoString(void) "hash_max_zipmap_value:%zu\r\n" "pubsub_channels:%ld\r\n" "pubsub_patterns:%u\r\n" - "vm_enabled:%d\r\n" + "ds_enabled:%d\r\n" "role:%s\r\n" ,REDIS_VERSION, redisGitSHA1(), @@@ -1213,7 -1228,7 +1213,7 @@@ server.hash_max_zipmap_value, dictSize(server.pubsub_channels), listLength(server.pubsub_patterns), - server.vm_enabled != 0, + server.ds_enabled != 0, server.masterhost == NULL ? "master" : "slave" ); if (server.masterhost) { @@@ -1240,13 -1255,33 +1240,13 @@@ ); } } - if (server.vm_enabled) { + if (server.ds_enabled) { lockThreadedIO(); info = sdscatprintf(info, - "vm_conf_max_memory:%llu\r\n" - "vm_conf_page_size:%llu\r\n" - "vm_conf_pages:%llu\r\n" - "vm_stats_used_pages:%llu\r\n" - "vm_stats_swapped_objects:%llu\r\n" - "vm_stats_swappin_count:%llu\r\n" - "vm_stats_swappout_count:%llu\r\n" - "vm_stats_io_newjobs_len:%lu\r\n" - "vm_stats_io_processing_len:%lu\r\n" - "vm_stats_io_processed_len:%lu\r\n" - "vm_stats_io_active_threads:%lu\r\n" - "vm_stats_blocked_clients:%lu\r\n" - ,(unsigned long long) server.vm_max_memory, - (unsigned long long) server.vm_page_size, - (unsigned long long) server.vm_pages, - (unsigned long long) server.vm_stats_used_pages, - (unsigned long long) server.vm_stats_swapped_objects, - (unsigned long long) server.vm_stats_swapins, - (unsigned long long) server.vm_stats_swapouts, - (unsigned long) listLength(server.io_newjobs), - (unsigned long) listLength(server.io_processing), - (unsigned long) listLength(server.io_processed), - (unsigned long) server.io_active_threads, - (unsigned long) server.vm_blocked_clients + "cache_max_memory:%llu\r\n" + "cache_blocked_clients:%lu\r\n" + ,(unsigned long long) server.cache_max_memory, + (unsigned long) server.cache_blocked_clients ); unlockThreadedIO(); } @@@ -1441,7 -1476,7 +1441,7 @@@ void createPidFile(void) /* Try to write the pid file in a best-effort way. */ FILE *fp = fopen(server.pidfile,"w"); if (fp) { - fprintf(fp,"%d\n",getpid()); + fprintf(fp,"%d\n",(int)getpid()); fclose(fp); } } @@@ -1498,9 -1533,7 +1498,9 @@@ int main(int argc, char **argv) linuxOvercommitMemoryWarning(); #endif start = time(NULL); - if (server.appendonly) { + if (server.ds_enabled) { + redisLog(REDIS_NOTICE,"DB not loaded (running with disk back end)"); + } else if (server.appendonly) { if (loadAppendOnlyFile(server.appendfilename) == REDIS_OK) redisLog(REDIS_NOTICE,"DB loaded from append only file: %ld seconds",time(NULL)-start); } else { diff --combined src/t_string.c index 2d7785ff,58834289..da065e88 --- a/src/t_string.c +++ b/src/t_string.c @@@ -1,4 -1,3 +1,3 @@@ - #include #include "redis.h" /*----------------------------------------------------------------------------- @@@ -38,7 -37,7 +37,7 @@@ void setGenericCommand(redisClient *c, } else { incrRefCount(val); } - touchWatchedKey(c->db,key); + signalModifiedKey(c->db,key); server.dirty++; removeExpire(c->db,key); if (expire) setExpire(c->db,key,time(NULL)+seconds); @@@ -84,7 -83,7 +83,7 @@@ void getsetCommand(redisClient *c) c->argv[2] = tryObjectEncoding(c->argv[2]); dbReplace(c->db,c->argv[1],c->argv[2]); incrRefCount(c->argv[2]); - touchWatchedKey(c->db,c->argv[1]); + signalModifiedKey(c->db,c->argv[1]); server.dirty++; removeExpire(c->db,c->argv[1]); } @@@ -156,7 -155,7 +155,7 @@@ void setbitCommand(redisClient *c) byteval &= ~(1 << bit); byteval |= ((on & 0x1) << bit); ((char*)o->ptr)[byte] = byteval; - touchWatchedKey(c->db,c->argv[1]); + signalModifiedKey(c->db,c->argv[1]); server.dirty++; addReply(c, bitval ? shared.cone : shared.czero); } @@@ -244,7 -243,7 +243,7 @@@ void setrangeCommand(redisClient *c) if (sdslen(value) > 0) { o->ptr = sdsgrowzero(o->ptr,offset+sdslen(value)); memcpy((char*)o->ptr+offset,value,sdslen(value)); - touchWatchedKey(c->db,c->argv[1]); + signalModifiedKey(c->db,c->argv[1]); server.dirty++; } addReplyLongLong(c,sdslen(o->ptr)); @@@ -331,7 -330,7 +330,7 @@@ void msetGenericCommand(redisClient *c dbReplace(c->db,c->argv[j],c->argv[j+1]); incrRefCount(c->argv[j+1]); removeExpire(c->db,c->argv[j]); - touchWatchedKey(c->db,c->argv[j]); + signalModifiedKey(c->db,c->argv[j]); } server.dirty += (c->argc-1)/2; addReply(c, nx ? shared.cone : shared.ok); @@@ -361,7 -360,7 +360,7 @@@ void incrDecrCommand(redisClient *c, lo } o = createStringObjectFromLongLong(value); dbReplace(c->db,c->argv[1],o); - touchWatchedKey(c->db,c->argv[1]); + signalModifiedKey(c->db,c->argv[1]); server.dirty++; addReply(c,shared.colon); addReply(c,o); @@@ -424,7 -423,7 +423,7 @@@ void appendCommand(redisClient *c) o->ptr = sdscatlen(o->ptr,append->ptr,sdslen(append->ptr)); totlen = sdslen(o->ptr); } - touchWatchedKey(c->db,c->argv[1]); + signalModifiedKey(c->db,c->argv[1]); server.dirty++; addReplyLongLong(c,totlen); }