From 33388d43042273bcf96b3e364fb1432f50c2db08 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 28 Dec 2010 14:42:09 +0100 Subject: [PATCH] added diskstore.c in Makefile and prototypes in redis.h --- src/Makefile | 21 +++++++------ src/{vm.c => dscache.c} | 67 +++++++++++++++++++++++++++++++++++++++++ src/redis.h | 9 +++++- 3 files changed, 87 insertions(+), 10 deletions(-) rename src/{vm.c => dscache.c} (92%) diff --git a/src/Makefile b/src/Makefile index 08f93f7a..fe71b6cd 100644 --- a/src/Makefile +++ b/src/Makefile @@ -24,7 +24,7 @@ PREFIX= /usr/local INSTALL_BIN= $(PREFIX)/bin INSTALL= cp -p -OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o vm.o pubsub.o multi.o debug.o sort.o intset.o syncio.o +OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o dscache.o pubsub.o multi.o debug.o sort.o intset.o syncio.o diskstore.o BENCHOBJ = ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o CHECKDUMPOBJ = redis-check-dump.o lzf_c.o lzf_d.o @@ -50,7 +50,6 @@ ae_select.o: ae_select.c anet.o: anet.c fmacros.h anet.h aof.o: aof.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h -chprgname.o: chprgname.c config.o: config.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h db.o: db.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ @@ -58,6 +57,10 @@ db.o: db.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ debug.o: debug.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h sha1.h dict.o: dict.c fmacros.h dict.h zmalloc.h +diskstore.o: diskstore.c redis.h fmacros.h config.h ae.h sds.h dict.h \ + adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h +dscache.o: dscache.c redis.h fmacros.h config.h ae.h sds.h dict.h \ + adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h intset.o: intset.c intset.h zmalloc.h lzf_c.o: lzf_c.c lzfP.h lzf_d.o: lzf_d.c lzfP.h @@ -72,11 +75,12 @@ pubsub.o: pubsub.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h rdb.o: rdb.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h lzf.h -redis-benchmark.o: redis-benchmark.c fmacros.h ae.h anet.h sds.h adlist.h \ - zmalloc.h +redis-benchmark.o: redis-benchmark.c fmacros.h ae.h \ + ../deps/hiredis/hiredis.h sds.h adlist.h zmalloc.h redis-check-aof.o: redis-check-aof.c fmacros.h config.h redis-check-dump.o: redis-check-dump.c lzf.h -redis-cli.o: redis-cli.c fmacros.h version.h sds.h adlist.h zmalloc.h +redis-cli.o: redis-cli.c fmacros.h version.h ../deps/hiredis/hiredis.h \ + sds.h zmalloc.h ../deps/linenoise/linenoise.h help.h redis.o: redis.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h release.o: release.c release.h @@ -86,7 +90,8 @@ sds.o: sds.c sds.h zmalloc.h sha1.o: sha1.c sha1.h sort.o: sort.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h pqsort.h -syncio.o: syncio.c +syncio.o: syncio.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h t_hash.o: t_hash.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h t_list.o: t_list.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ @@ -99,8 +104,6 @@ t_zset.o: t_zset.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.o: util.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h -vm.o: vm.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h ziplist.o: ziplist.c zmalloc.h ziplist.h zipmap.o: zipmap.c zmalloc.h zmalloc.o: zmalloc.c config.h @@ -138,7 +141,7 @@ clean: rm -rf $(PRGNAME) $(BENCHPRGNAME) $(CLIPRGNAME) $(CHECKDUMPPRGNAME) $(CHECKAOFPRGNAME) *.o *.gcda *.gcno *.gcov dep: - $(CC) -MM *.c + $(CC) -MM *.c -I ../deps/hiredis -I ../deps/linenoise test: redis-server (cd ..; tclsh8.5 tests/test_helper.tcl --tags "${TAGS}" --file "${FILE}") diff --git a/src/vm.c b/src/dscache.c similarity index 92% rename from src/vm.c rename to src/dscache.c index ac0d92e3..b21868c3 100644 --- a/src/vm.c +++ b/src/dscache.c @@ -5,6 +5,73 @@ #include #include +/* dscache.c - Disk store cache for disk store backend. + * + * When Redis is configured for using disk as backend instead of memory, the + * memory is used as a cache, so that recently accessed keys are taken in + * memory for fast read and write operations. + * + * Modified keys are marked to be flushed on disk, and will be flushed + * as long as the maxium configured flush time elapsed. + * + * This file implements the whole caching subsystem and contains further + * documentation. */ + +/* TODO: + * + * - The WATCH helper will be used to signal the cache system + * we need to flush a given key/dbid into disk, adding this key/dbid + * pair into a server.ds_cache_dirty linked list AND hash table (so that we + * don't add the same thing multiple times). + * + * - cron() checks if there are elements on this list. When there are things + * to flush, we create an IO Job for the I/O thread. + * FIXME: how to mark this key as "busy"? With VM we used to change the + * object->storage field, but this time we need this to work with every + * kind of object, including shared ones. One possibility is just killing + * object sharing at all. So let's assume this will be our solution. + * + * So we set keys that are in the process of being saved as + * object->storage = REDIS_STORAGE_SAVING; + * + * - This is what we do on key lookup: + * 1) The key already exists in memory. object->storage == REDIS_DS_MEMORY. + * We don't do nothing special, lookup, return value object pointer. + * 2) The key is in memory but object->storage == REDIS_DS_SAVING. + * This is an explicit lookup so we have to abort the saving operation. + * We kill the IO Job, set the storage to == REDIS_DB_MEMORY but + * re-queue the object in the server.ds_cache_dirty list. + * + * Btw here we need some protection against the problem of continuously + * writing against a value having the effect of this value to be never + * saved on disk. That is, at some point we need to block and write it + * if there is too much delay. + * 3) The key is not in memory. We block to load the key from disk. + * Of course the key may not be present at all on the disk store as well, + * in such case we just detect this condition and continue, returning + * NULL from lookup. + * + * - Preloading of needed keys: + * 1) As it was done with VM, also with this new system we try preloading + * keys a client is going to use. We block the client, load keys + * using the I/O thread, unblock the client. Same code as VM more or less. + * + * - Transfering keys from memory to disk. + * Again while in cron() we detect our memory limit was reached. What we + * do is transfering random keys that are not set as dirty on disk, using + * LRU to select the key. + * If this is not enough to return again under the memory limits we also + * start to flush keys that need to be synched on disk synchronously, + * removing it from the memory. + * + * - IO thread operations are no longer stopped for sync loading/saving of + * things. When a key is found to be in the process of being saved or + * loaded we simply wait for the IO thread to end its work. + * + * Otherwise if there is to load a key without any IO thread operation + * just started it is blocking-loaded in the lookup function. + */ + /* Virtual Memory is composed mainly of two subsystems: * - Blocking Virutal Memory * - Threaded Virtual Memory I/O diff --git a/src/redis.h b/src/redis.h index bc1a58c1..6e8b40b9 100644 --- a/src/redis.h +++ b/src/redis.h @@ -792,7 +792,14 @@ int htNeedsResize(dict *dict); void oom(const char *msg); void populateCommandTable(void); -/* Virtual Memory */ +/* Disk store */ +int dsOpen(void); +int dsClose(void); +int dsSet(redisDb *db, robj *key, robj *val); +robj *dsGet(redisDb *db, robj *key); +int dsExists(redisDb *db, robj *key); + +/* Disk Store Cache */ void vmInit(void); void vmMarkPagesFree(off_t page, off_t count); robj *vmLoadObject(robj *o); -- 2.45.2