X-Git-Url: https://git.saurik.com/redis.git/blobdiff_plain/454d4e43b5ba15b8e79dcc1f67d5754d4faf80cb..b3e3d0d78e3b74328f89697ad4f63a07e8acf373:/redis.c?ds=sidebyside diff --git a/redis.c b/redis.c index db25135a..f489072e 100644 --- a/redis.c +++ b/redis.c @@ -27,7 +27,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#define REDIS_VERSION "1.1.91" +#define REDIS_VERSION "1.3.2" #include "fmacros.h" #include "config.h" @@ -59,6 +59,7 @@ #include #include #include +#include #if defined(__sun) #include "solarisfixes.h" @@ -152,11 +153,26 @@ #define REDIS_RDB_ENC_INT32 2 /* 32 bit signed integer */ #define REDIS_RDB_ENC_LZF 3 /* string compressed with FASTLZ */ +/* Virtual memory object->where field. */ +#define REDIS_VM_MEMORY 0 /* The object is on memory */ +#define REDIS_VM_SWAPPED 1 /* The object is on disk */ +#define REDIS_VM_SWAPPING 2 /* Redis is swapping this object on disk */ +#define REDIS_VM_LOADING 3 /* Redis is loading this object from disk */ + +/* Virtual memory static configuration stuff. + * Check vmFindContiguousPages() to know more about this magic numbers. */ +#define REDIS_VM_MAX_NEAR_PAGES 65536 +#define REDIS_VM_MAX_RANDOM_JUMP 4096 +#define REDIS_VM_MAX_THREADS 32 + /* Client flags */ #define REDIS_CLOSE 1 /* This client connection should be closed ASAP */ #define REDIS_SLAVE 2 /* This client is a slave server */ #define REDIS_MASTER 4 /* This client is a master server */ #define REDIS_MONITOR 8 /* This client is a slave monitor, see MONITOR */ +#define REDIS_MULTI 16 /* This client is in a MULTI context */ +#define REDIS_BLOCKED 32 /* The client is waiting in a blocking operation */ +#define REDIS_IO_WAIT 64 /* The client is waiting for Virtual Memory I/O */ /* Slave replication state - slave side */ #define REDIS_REPL_NONE 0 /* No active replication */ @@ -184,8 +200,9 @@ /* Log levels */ #define REDIS_DEBUG 0 -#define REDIS_NOTICE 1 -#define REDIS_WARNING 2 +#define REDIS_VERBOSE 1 +#define REDIS_NOTICE 2 +#define REDIS_WARNING 3 /* Anti-warning macro... */ #define REDIS_NOTUSED(V) ((void) V) @@ -205,12 +222,29 @@ static void _redisAssert(char *estr); /*================================= Data types ============================== */ /* A redis object, that is a type able to hold a string / list / set */ + +/* The VM object structure */ +struct redisObjectVM { + off_t page; /* the page at witch the object is stored on disk */ + off_t usedpages; /* number of pages used on disk */ + time_t atime; /* Last access time */ +} vm; + +/* The actual Redis Object */ typedef struct redisObject { void *ptr; unsigned char type; unsigned char encoding; - unsigned char notused[2]; + unsigned char storage; /* If this object is a key, where is the value? + * REDIS_VM_MEMORY, REDIS_VM_SWAPPED, ... */ + unsigned char vtype; /* If this object is a key, and value is swapped out, + * this is the type of the swapped out object. */ int refcount; + /* VM fields, this are only allocated if VM is active, otherwise the + * object allocation function will just allocate + * sizeof(redisObjct) minus sizeof(redisObjectVM), so using + * Redis without VM active will not have any overhead. */ + struct redisObjectVM vm; } robj; /* Macro used to initalize a Redis object allocated on the stack. @@ -222,14 +256,28 @@ typedef struct redisObject { _var.type = REDIS_STRING; \ _var.encoding = REDIS_ENCODING_RAW; \ _var.ptr = _ptr; \ + if (server.vm_enabled) _var.storage = REDIS_VM_MEMORY; \ } while(0); typedef struct redisDb { - dict *dict; - dict *expires; + dict *dict; /* The keyspace for this DB */ + dict *expires; /* Timeout of keys with a timeout set */ + dict *blockingkeys; /* Keys with clients waiting for data (BLPOP) */ int id; } redisDb; +/* Client MULTI/EXEC state */ +typedef struct multiCmd { + robj **argv; + int argc; + struct redisCommand *cmd; +} multiCmd; + +typedef struct multiState { + multiCmd *commands; /* Array of MULTI commands */ + int count; /* Total number of MULTI commands */ +} multiState; + /* With multiplexing we need to take per-clinet state. * Clients are taken in a liked list. */ typedef struct redisClient { @@ -245,12 +293,21 @@ typedef struct redisClient { int sentlen; time_t lastinteraction; /* time of the last interaction, used for timeout */ int flags; /* REDIS_CLOSE | REDIS_SLAVE | REDIS_MONITOR */ + /* REDIS_MULTI */ int slaveseldb; /* slave selected db, if this client is a slave */ int authenticated; /* when requirepass is non-NULL */ int replstate; /* replication state if this is a slave */ int repldbfd; /* replication DB file descriptor */ - long repldboff; /* replication DB file offset */ + long repldboff; /* replication DB file offset */ off_t repldbsize; /* replication DB file size */ + multiState mstate; /* MULTI/EXEC state */ + robj **blockingkeys; /* The key we waiting to terminate a blocking + * operation such as BLPOP. Otherwise NULL. */ + int blockingkeysnum; /* Number of blocking keys */ + time_t blockingto; /* Blocking operation timeout. If UNIX current time + * is >= blockingto then the operation timed out. */ + list *io_keys; /* Keys this client is waiting to be loaded from the + * swap file in order to continue. */ } redisClient; struct saveparam { @@ -263,7 +320,7 @@ struct redisServer { int port; int fd; redisDb *db; - dict *sharingpool; + dict *sharingpool; /* Poll used for object sharing */ unsigned int sharingpoolsize; long long dirty; /* changes to DB from the last save */ list *clients; @@ -301,6 +358,7 @@ struct redisServer { char *appendfilename; char *requirepass; int shareobjects; + int rdbcompression; /* Replication related */ int isslave; char *masterauth; @@ -309,12 +367,48 @@ struct redisServer { redisClient *master; /* client that is master for this slave */ int replstate; unsigned int maxclients; - unsigned long maxmemory; + unsigned long long maxmemory; + unsigned int blockedclients; /* Sort parameters - qsort_r() is only available under BSD so we * have to take this state global, in order to pass it to sortCompare() */ int sort_desc; int sort_alpha; int sort_bypattern; + /* Virtual memory configuration */ + int vm_enabled; + off_t vm_page_size; + off_t vm_pages; + unsigned long long vm_max_memory; + /* Virtual memory state */ + FILE *vm_fp; + int vm_fd; + off_t vm_next_page; /* Next probably empty page */ + off_t vm_near_pages; /* Number of pages allocated sequentially */ + unsigned char *vm_bitmap; /* Bitmap of free/used pages */ + time_t unixtime; /* Unix time sampled every second. */ + /* Virtual memory I/O threads stuff */ + /* An I/O thread process an element taken from the io_jobs queue and + * put the result of the operation in the io_done list. While the + * job is being processed, it's put on io_processing queue. */ + list *io_newjobs; /* List of VM I/O jobs yet to be processed */ + list *io_processing; /* List of VM I/O jobs being processed */ + list *io_processed; /* List of VM I/O jobs already processed */ + list *io_clients; /* All the clients waiting for SWAP I/O operations */ + pthread_mutex_t io_mutex; /* lock to access io_jobs/io_done/io_thread_job */ + int io_active_threads; /* Number of running I/O threads */ + int vm_max_threads; /* Max number of I/O threads running at the same time */ + /* Our main thread is blocked on the event loop, locking for sockets ready + * to be read or written, so when a threaded I/O operation is ready to be + * processed by the main thread, the I/O thread will use a unix pipe to + * awake the main thread. The followings are the two pipe FDs. */ + int io_ready_pipe_read; + int io_ready_pipe_write; + /* Virtual memory stats */ + unsigned long long vm_stats_used_pages; + unsigned long long vm_stats_swapped_objects; + unsigned long long vm_stats_swapouts; + unsigned long long vm_stats_swapins; + FILE *devnull; }; typedef void redisCommandProc(redisClient *c); @@ -367,7 +461,7 @@ typedef struct zset { struct sharedObjectsStruct { robj *crlf, *ok, *err, *emptybulk, *czero, *cone, *pong, *space, - *colon, *nullbulk, *nullmultibulk, + *colon, *nullbulk, *nullmultibulk, *queued, *emptymultibulk, *wrongtypeerr, *nokeyerr, *syntaxerr, *sameobjecterr, *outofrangeerr, *plus, *select0, *select1, *select2, *select3, *select4, @@ -380,6 +474,22 @@ struct sharedObjectsStruct { static double R_Zero, R_PosInf, R_NegInf, R_Nan; +/* VM threaded I/O request message */ +#define REDIS_IOJOB_LOAD 0 /* Load from disk to memory */ +#define REDIS_IOJOB_PREPARE_SWAP 1 /* Compute needed pages */ +#define REDIS_IOJOB_DO_SWAP 2 /* Swap from memory to disk */ +typedef struct iojon { + int type; /* Request type, REDIS_IOJOB_* */ + redisDb *db;/* Redis database */ + robj *key; /* This I/O request is about swapping this key */ + robj *val; /* the value to swap for REDIS_IOREQ_*_SWAP, otherwise this + * field is populated by the I/O thread for REDIS_IOREQ_LOAD. */ + off_t page; /* Swap page where to read/write the object */ + off_t pages; /* Swap pages needed to safe object. PREPARE_SWAP return val */ + int canceled; /* True if this command was canceled by blocking side of VM */ + pthread_t thread; /* ID of the thread processing this entry */ +} iojob; + /*================================ Prototypes =============================== */ static void freeStringObject(robj *o); @@ -394,6 +504,7 @@ static void addReplySds(redisClient *c, sds s); static void incrRefCount(robj *o); static int rdbSaveBackground(char *filename); static robj *createStringObject(char *ptr, size_t len); +static robj *dupStringObject(robj *o); static void replicationFeedSlaves(list *slaves, struct redisCommand *cmd, int dictid, robj **argv, int argc); static void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int argc); static int syncWithMaster(void); @@ -403,6 +514,7 @@ static robj *getDecodedObject(robj *o); static int removeExpire(redisDb *db, robj *key); static int expireIfNeeded(redisDb *db, robj *key); static int deleteIfVolatile(redisDb *db, robj *key); +static int deleteIfSwapped(redisDb *db, robj *key); static int deleteKey(redisDb *db, robj *key); static time_t getExpire(redisDb *db, robj *key); static int setExpire(redisDb *db, robj *key, time_t when); @@ -418,6 +530,27 @@ static zskiplist *zslCreate(void); static void zslFree(zskiplist *zsl); static void zslInsert(zskiplist *zsl, double score, robj *obj); static void sendReplyToClientWritev(aeEventLoop *el, int fd, void *privdata, int mask); +static void initClientMultiState(redisClient *c); +static void freeClientMultiState(redisClient *c); +static void queueMultiCommand(redisClient *c, struct redisCommand *cmd); +static void unblockClient(redisClient *c); +static int handleClientsWaitingListPush(redisClient *c, robj *key, robj *ele); +static void vmInit(void); +static void vmMarkPagesFree(off_t page, off_t count); +static robj *vmLoadObject(robj *key); +static robj *vmPreviewObject(robj *key); +static int vmSwapOneObjectBlocking(void); +static int vmSwapOneObjectThreaded(void); +static int vmCanSwapOut(void); +static void freeOneObjectFromFreelist(void); +static void acceptHandler(aeEventLoop *el, int fd, void *privdata, int mask); +static void vmThreadedIOCompletedJob(aeEventLoop *el, int fd, void *privdata, int mask); +static void vmCancelThreadedIOJob(robj *o); +static void lockThreadedIO(void); +static void unlockThreadedIO(void); +static int vmSwapObjectThreaded(robj *key, robj *val, redisDb *db); +static void freeIOJob(iojob *j); +static void queueIOJob(iojob *j); static void authCommand(redisClient *c); static void pingCommand(redisClient *c); @@ -492,6 +625,10 @@ static void zcardCommand(redisClient *c); static void zremCommand(redisClient *c); static void zscoreCommand(redisClient *c); static void zremrangebyscoreCommand(redisClient *c); +static void multiCommand(redisClient *c); +static void execCommand(redisClient *c); +static void blpopCommand(redisClient *c); +static void brpopCommand(redisClient *c); /*================================= Globals ================================= */ @@ -510,6 +647,8 @@ static struct redisCommand cmdTable[] = { {"lpush",lpushCommand,3,REDIS_CMD_BULK|REDIS_CMD_DENYOOM}, {"rpop",rpopCommand,2,REDIS_CMD_INLINE}, {"lpop",lpopCommand,2,REDIS_CMD_INLINE}, + {"brpop",brpopCommand,-3,REDIS_CMD_INLINE}, + {"blpop",blpopCommand,-3,REDIS_CMD_INLINE}, {"llen",llenCommand,2,REDIS_CMD_INLINE}, {"lindex",lindexCommand,3,REDIS_CMD_INLINE}, {"lset",lsetCommand,4,REDIS_CMD_BULK|REDIS_CMD_DENYOOM}, @@ -535,9 +674,9 @@ static struct redisCommand cmdTable[] = { {"zincrby",zincrbyCommand,4,REDIS_CMD_BULK|REDIS_CMD_DENYOOM}, {"zrem",zremCommand,3,REDIS_CMD_BULK}, {"zremrangebyscore",zremrangebyscoreCommand,4,REDIS_CMD_INLINE}, - {"zrange",zrangeCommand,4,REDIS_CMD_INLINE}, + {"zrange",zrangeCommand,-4,REDIS_CMD_INLINE}, {"zrangebyscore",zrangebyscoreCommand,-4,REDIS_CMD_INLINE}, - {"zrevrange",zrevrangeCommand,4,REDIS_CMD_INLINE}, + {"zrevrange",zrevrangeCommand,-4,REDIS_CMD_INLINE}, {"zcard",zcardCommand,2,REDIS_CMD_INLINE}, {"zscore",zscoreCommand,3,REDIS_CMD_BULK|REDIS_CMD_DENYOOM}, {"incrby",incrbyCommand,3,REDIS_CMD_INLINE|REDIS_CMD_DENYOOM}, @@ -563,6 +702,8 @@ static struct redisCommand cmdTable[] = { {"shutdown",shutdownCommand,1,REDIS_CMD_INLINE}, {"lastsave",lastsaveCommand,1,REDIS_CMD_INLINE}, {"type",typeCommand,2,REDIS_CMD_INLINE}, + {"multi",multiCommand,1,REDIS_CMD_INLINE}, + {"exec",execCommand,1,REDIS_CMD_INLINE}, {"sync",syncCommand,1,REDIS_CMD_INLINE}, {"flushdb",flushdbCommand,1,REDIS_CMD_INLINE}, {"flushall",flushallCommand,1,REDIS_CMD_INLINE}, @@ -737,6 +878,12 @@ static void dictVanillaFree(void *privdata, void *val) zfree(val); } +static void dictListDestructor(void *privdata, void *val) +{ + DICT_NOTUSED(privdata); + listRelease((list*)val); +} + static int sdsDictKeyCompare(void *privdata, const void *key1, const void *key2) { @@ -753,6 +900,7 @@ static void dictRedisObjectDestructor(void *privdata, void *val) { DICT_NOTUSED(privdata); + if (val == NULL) return; /* Values of swapped out keys as set to NULL */ decrRefCount(val); } @@ -818,6 +966,17 @@ static dictType hashDictType = { dictRedisObjectDestructor /* val destructor */ }; +/* Keylist hash table type has unencoded redis objects as keys and + * lists as values. It's used for blocking operations (BLPOP) */ +static dictType keylistDictType = { + dictObjHash, /* hash function */ + NULL, /* key dup */ + NULL, /* val dup */ + dictObjKeyCompare, /* key compare */ + dictRedisObjectDestructor, /* key destructor */ + dictListDestructor /* val destructor */ +}; + /* ========================= Random utility functions ======================= */ /* Redis generally does not try to recover from out of memory conditions @@ -840,11 +999,18 @@ static void closeTimedoutClients(void) { listRewind(server.clients); while ((ln = listYield(server.clients)) != NULL) { c = listNodeValue(ln); - if (!(c->flags & REDIS_SLAVE) && /* no timeout for slaves */ + if (server.maxidletime && + !(c->flags & REDIS_SLAVE) && /* no timeout for slaves */ !(c->flags & REDIS_MASTER) && /* no timeout for masters */ - (now - c->lastinteraction > server.maxidletime)) { - redisLog(REDIS_DEBUG,"Closing idle client"); + (now - c->lastinteraction > server.maxidletime)) + { + redisLog(REDIS_VERBOSE,"Closing idle client"); freeClient(c); + } else if (c->flags & REDIS_BLOCKED) { + if (c->blockingto != 0 && c->blockingto < now) { + addReply(c,shared.nullmultibulk); + unblockClient(c); + } } } } @@ -865,9 +1031,9 @@ static void tryResizeHashTables(void) { for (j = 0; j < server.dbnum; j++) { if (htNeedsResize(server.db[j].dict)) { - redisLog(REDIS_DEBUG,"The hash table %d is too sparse, resize it...",j); + redisLog(REDIS_VERBOSE,"The hash table %d is too sparse, resize it...",j); dictResize(server.db[j].dict); - redisLog(REDIS_DEBUG,"Hash table %d resized.",j); + redisLog(REDIS_VERBOSE,"Hash table %d resized.",j); } if (htNeedsResize(server.db[j].expires)) dictResize(server.db[j].expires); @@ -923,7 +1089,7 @@ void backgroundRewriteDoneHandler(int statloc) { close(fd); goto cleanup; } - redisLog(REDIS_WARNING,"Parent diff flushed into the new append log file with success"); + redisLog(REDIS_NOTICE,"Parent diff flushed into the new append log file with success (%lu bytes)",sdslen(server.bgrewritebuf)); /* Now our work is to rename the temp file into the stable file. And * switch the file descriptor used by the server for append only. */ if (rename(tmpfile,server.appendfilename) == -1) { @@ -964,6 +1130,12 @@ static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientD REDIS_NOTUSED(id); REDIS_NOTUSED(clientData); + /* We take a cached value of the unix time in the global state because + * with virtual memory and aging there is to store the current time + * in objects at every object access, and accuracy is not needed. + * To access a global var is faster than calling time(NULL) */ + server.unixtime = time(NULL); + /* Update the global state with the amount of used memory */ server.usedmemory = zmalloc_used_memory(); @@ -975,7 +1147,7 @@ static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientD used = dictSize(server.db[j].dict); vkeys = dictSize(server.db[j].expires); if (!(loops % 5) && (used || vkeys)) { - redisLog(REDIS_DEBUG,"DB %d: %lld keys (%lld volatile) in %lld slots HT.",j,used,vkeys,size); + redisLog(REDIS_VERBOSE,"DB %d: %lld keys (%lld volatile) in %lld slots HT.",j,used,vkeys,size); /* dictPrintStats(server.dict); */ } } @@ -990,7 +1162,7 @@ static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientD /* Show information about connected clients */ if (!(loops % 5)) { - redisLog(REDIS_DEBUG,"%d clients connected (%d slaves), %zu bytes in use, %d shared objects", + redisLog(REDIS_VERBOSE,"%d clients connected (%d slaves), %zu bytes in use, %d shared objects", listLength(server.clients)-listLength(server.slaves), listLength(server.slaves), server.usedmemory, @@ -998,7 +1170,7 @@ static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientD } /* Close connections of timedout clients */ - if (server.maxidletime && !(loops % 10)) + if ((server.maxidletime && !(loops % 10)) || server.blockedclients) closeTimedoutClients(); /* Check if a background saving or AOF rewrite in progress terminated */ @@ -1041,7 +1213,7 @@ static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientD /* Continue to expire if at the end of the cycle more than 25% * of the keys were expired. */ do { - int num = dictSize(db->expires); + long num = dictSize(db->expires); time_t now = time(NULL); expired = 0; @@ -1061,6 +1233,30 @@ static int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientD } while (expired > REDIS_EXPIRELOOKUPS_PER_CRON/4); } + /* Swap a few keys on disk if we are over the memory limit and VM + * is enbled. Try to free objects from the free list first. */ + if (vmCanSwapOut()) { + while (server.vm_enabled && zmalloc_used_memory() > + server.vm_max_memory) + { + if (listLength(server.objfreelist)) { + freeOneObjectFromFreelist(); + } else { + if (vmSwapOneObjectThreaded() == REDIS_ERR) { + if ((loops % 30) == 0 && zmalloc_used_memory() > + (server.vm_max_memory+server.vm_max_memory/10)) { + redisLog(REDIS_WARNING,"WARNING: vm-max-memory limit exceeded by more than 10%% but unable to swap more objects out!"); + } + } + /* Note that we freed just one object, because anyway when + * the I/O thread in charge to swap this object out will + * do its work, the handler of completed jobs will try to swap + * more objects if we are out of memory. */ + break; + } + } + } + /* Check if we should connect to a MASTER */ if (server.replstate == REDIS_REPL_CONNECT) { redisLog(REDIS_NOTICE,"Connecting to MASTER..."); @@ -1081,8 +1277,8 @@ static void createSharedObjects(void) { shared.nullbulk = createObject(REDIS_STRING,sdsnew("$-1\r\n")); shared.nullmultibulk = createObject(REDIS_STRING,sdsnew("*-1\r\n")); shared.emptymultibulk = createObject(REDIS_STRING,sdsnew("*0\r\n")); - /* no such key */ shared.pong = createObject(REDIS_STRING,sdsnew("+PONG\r\n")); + shared.queued = createObject(REDIS_STRING,sdsnew("+QUEUED\r\n")); shared.wrongtypeerr = createObject(REDIS_STRING,sdsnew( "-ERR Operation against a key holding the wrong kind of value\r\n")); shared.nokeyerr = createObject(REDIS_STRING,sdsnew( @@ -1124,7 +1320,7 @@ static void resetServerSaveParams() { static void initServerConfig() { server.dbnum = REDIS_DEFAULT_DBNUM; server.port = REDIS_SERVERPORT; - server.verbosity = REDIS_DEBUG; + server.verbosity = REDIS_VERBOSE; server.maxidletime = REDIS_MAXIDLETIME; server.saveparams = NULL; server.logfile = NULL; /* NULL = log on standard output */ @@ -1141,9 +1337,17 @@ static void initServerConfig() { server.appendfilename = "appendonly.aof"; server.requirepass = NULL; server.shareobjects = 0; + server.rdbcompression = 1; server.sharingpoolsize = 1024; server.maxclients = 0; + server.blockedclients = 0; server.maxmemory = 0; + server.vm_enabled = 0; + server.vm_page_size = 256; /* 256 bytes per page */ + server.vm_pages = 1024*1024*100; /* 104 millions of pages */ + server.vm_max_memory = 1024LL*1024*1024*1; /* 1 GB of RAM */ + server.vm_max_threads = 4; + resetServerSaveParams(); appendServerSaveParams(60*60,1); /* save after 1 hour and 1 change */ @@ -1171,6 +1375,11 @@ static void initServer() { signal(SIGPIPE, SIG_IGN); setupSigSegvAction(); + server.devnull = fopen("/dev/null","w"); + if (server.devnull == NULL) { + redisLog(REDIS_WARNING, "Can't open /dev/null: %s", server.neterr); + exit(1); + } server.clients = listCreate(); server.slaves = listCreate(); server.monitors = listCreate(); @@ -1187,6 +1396,7 @@ static void initServer() { for (j = 0; j < server.dbnum; j++) { server.db[j].dict = dictCreate(&hashDictType,NULL); server.db[j].expires = dictCreate(&setDictType,NULL); + server.db[j].blockingkeys = dictCreate(&keylistDictType,NULL); server.db[j].id = j; } server.cronloops = 0; @@ -1199,7 +1409,10 @@ static void initServer() { server.stat_numcommands = 0; server.stat_numconnections = 0; server.stat_starttime = time(NULL); + server.unixtime = time(NULL); aeCreateTimeEvent(server.el, 1, serverCron, NULL, NULL); + if (aeCreateFileEvent(server.el, server.fd, AE_READABLE, + acceptHandler, NULL) == AE_ERR) oom("creating file event"); if (server.appendonly) { server.appendfd = open(server.appendfilename,O_WRONLY|O_APPEND|O_CREAT,0644); @@ -1209,6 +1422,8 @@ static void initServer() { exit(1); } } + + if (server.vm_enabled) vmInit(); } /* Empty the whole database */ @@ -1293,6 +1508,7 @@ static void loadServerConfig(char *filename) { } } else if (!strcasecmp(argv[0],"loglevel") && argc == 2) { if (!strcasecmp(argv[1],"debug")) server.verbosity = REDIS_DEBUG; + else if (!strcasecmp(argv[1],"verbose")) server.verbosity = REDIS_VERBOSE; else if (!strcasecmp(argv[1],"notice")) server.verbosity = REDIS_NOTICE; else if (!strcasecmp(argv[1],"warning")) server.verbosity = REDIS_WARNING; else { @@ -1341,6 +1557,10 @@ static void loadServerConfig(char *filename) { if ((server.shareobjects = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } + } else if (!strcasecmp(argv[0],"rdbcompression") && argc == 2) { + if ((server.rdbcompression = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"shareobjectspoolsize") && argc == 2) { server.sharingpoolsize = atoi(argv[1]); if (server.sharingpoolsize < 1) { @@ -1371,6 +1591,18 @@ static void loadServerConfig(char *filename) { server.pidfile = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"dbfilename") && argc == 2) { server.dbfilename = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"vm-enabled") && argc == 2) { + if ((server.vm_enabled = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } + } else if (!strcasecmp(argv[0],"vm-max-memory") && argc == 2) { + server.vm_max_memory = strtoll(argv[1], NULL, 10); + } else if (!strcasecmp(argv[0],"vm-page-size") && argc == 2) { + server.vm_page_size = strtoll(argv[1], NULL, 10); + } else if (!strcasecmp(argv[0],"vm-pages") && argc == 2) { + server.vm_pages = strtoll(argv[1], NULL, 10); + } else if (!strcasecmp(argv[0],"vm-max-threads") && argc == 2) { + server.vm_max_threads = strtoll(argv[1], NULL, 10); } else { err = "Bad directive or wrong number of arguments"; goto loaderr; } @@ -1404,15 +1636,33 @@ static void freeClientArgv(redisClient *c) { static void freeClient(redisClient *c) { listNode *ln; + /* Note that if the client we are freeing is blocked into a blocking + * call, we have to set querybuf to NULL *before* to call unblockClient() + * to avoid processInputBuffer() will get called. Also it is important + * to remove the file events after this, because this call adds + * the READABLE event. */ + sdsfree(c->querybuf); + c->querybuf = NULL; + if (c->flags & REDIS_BLOCKED) + unblockClient(c); + aeDeleteFileEvent(server.el,c->fd,AE_READABLE); aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); - sdsfree(c->querybuf); listRelease(c->reply); freeClientArgv(c); close(c->fd); + /* Remove from the list of clients */ ln = listSearchKey(server.clients,c); redisAssert(ln != NULL); listDelNode(server.clients,ln); + /* Remove from the list of clients waiting for VM operations */ + if (server.vm_enabled && listLength(c->io_keys)) { + ln = listSearchKey(server.io_clients,c); + if (ln) listDelNode(server.io_clients,ln); + listRelease(c->io_keys); + } + listRelease(c->io_keys); + /* Other cleanup */ if (c->flags & REDIS_SLAVE) { if (c->replstate == REDIS_REPL_SEND_BULK && c->repldbfd != -1) close(c->repldbfd); @@ -1427,6 +1677,7 @@ static void freeClient(redisClient *c) { } zfree(c->argv); zfree(c->mbargv); + freeClientMultiState(c); zfree(c); } @@ -1510,7 +1761,7 @@ static void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) if (errno == EAGAIN) { nwritten = 0; } else { - redisLog(REDIS_DEBUG, + redisLog(REDIS_VERBOSE, "Error writing to client: %s", strerror(errno)); freeClient(c); return; @@ -1563,7 +1814,7 @@ static void sendReplyToClientWritev(aeEventLoop *el, int fd, void *privdata, int /* write all collected blocks at once */ if((nwritten = writev(fd, iov, ion)) < 0) { if (errno != EAGAIN) { - redisLog(REDIS_DEBUG, + redisLog(REDIS_VERBOSE, "Error writing to client: %s", strerror(errno)); freeClient(c); return; @@ -1617,6 +1868,21 @@ static void resetClient(redisClient *c) { c->multibulk = 0; } +/* Call() is the core of Redis execution of a command */ +static void call(redisClient *c, struct redisCommand *cmd) { + long long dirty; + + dirty = server.dirty; + cmd->proc(c); + if (server.appendonly && server.dirty-dirty) + feedAppendOnlyFile(cmd,c->db->id,c->argv,c->argc); + if (server.dirty-dirty && listLength(server.slaves)) + replicationFeedSlaves(server.slaves,cmd,c->db->id,c->argv,c->argc); + if (listLength(server.monitors)) + replicationFeedSlaves(server.monitors,cmd,c->db->id,c->argv,c->argc); + server.stat_numcommands++; +} + /* If this function gets called we already read a whole * command, argments are in the client argv/argc fields. * processCommand() execute the command or prepare the @@ -1627,7 +1893,6 @@ static void resetClient(redisClient *c) { * if 0 is returned the client was destroied (i.e. after QUIT). */ static int processCommand(redisClient *c) { struct redisCommand *cmd; - long long dirty; /* Free some memory if needed (maxmemory setting) */ if (server.maxmemory) freeMemoryIfNeeded(); @@ -1708,7 +1973,9 @@ static int processCommand(redisClient *c) { } cmd = lookupCommand(c->argv[0]->ptr); if (!cmd) { - addReplySds(c,sdsnew("-ERR unknown command\r\n")); + addReplySds(c, + sdscatprintf(sdsempty(), "-ERR unknown command '%s'\r\n", + (char*)c->argv[0]->ptr)); resetClient(c); return 1; } else if ((cmd->arity > 0 && cmd->arity != c->argc) || @@ -1766,15 +2033,12 @@ static int processCommand(redisClient *c) { } /* Exec the command */ - dirty = server.dirty; - cmd->proc(c); - if (server.appendonly && server.dirty-dirty) - feedAppendOnlyFile(cmd,c->db->id,c->argv,c->argc); - if (server.dirty-dirty && listLength(server.slaves)) - replicationFeedSlaves(server.slaves,cmd,c->db->id,c->argv,c->argc); - if (listLength(server.monitors)) - replicationFeedSlaves(server.monitors,cmd,c->db->id,c->argv,c->argc); - server.stat_numcommands++; + if (c->flags & REDIS_MULTI && cmd->proc != execCommand) { + queueMultiCommand(c,cmd); + addReply(c,shared.queued); + } else { + call(c,cmd); + } /* Prepare the client for the next command */ if (c->flags & REDIS_CLOSE) { @@ -1856,6 +2120,13 @@ static void replicationFeedSlaves(list *slaves, struct redisCommand *cmd, int di static void processInputBuffer(redisClient *c) { again: + /* Before to process the input buffer, make sure the client is not + * waitig for a blocking operation such as BLPOP. Note that the first + * iteration the client is never blocked, otherwise the processInputBuffer + * would not be called at all, but after the execution of the first commands + * in the input buffer the client may be blocked, and the "goto again" + * will try to reiterate. The following line will make it return asap. */ + if (c->flags & REDIS_BLOCKED || c->flags & REDIS_IO_WAIT) return; if (c->bulklen == -1) { /* Read the first line of the query */ char *p = strchr(c->querybuf,'\n'); @@ -1877,11 +2148,6 @@ again: sdsupdatelen(query); /* Now we can split the query in arguments */ - if (sdslen(query) == 0) { - /* Ignore empty query */ - sdsfree(query); - return; - } argv = sdssplitlen(query,sdslen(query)," ",1,&argc); sdsfree(query); @@ -1897,13 +2163,19 @@ again: } } zfree(argv); - /* Execute the command. If the client is still valid - * after processCommand() return and there is something - * on the query buffer try to process the next command. */ - if (c->argc && processCommand(c) && sdslen(c->querybuf)) goto again; + if (c->argc) { + /* Execute the command. If the client is still valid + * after processCommand() return and there is something + * on the query buffer try to process the next command. */ + if (processCommand(c) && sdslen(c->querybuf)) goto again; + } else { + /* Nothing to process, argc == 0. Just process the query + * buffer if it's not empty or return to the caller */ + if (sdslen(c->querybuf)) goto again; + } return; } else if (sdslen(c->querybuf) >= REDIS_REQUEST_MAX_SIZE) { - redisLog(REDIS_DEBUG, "Client protocol error"); + redisLog(REDIS_VERBOSE, "Client protocol error"); freeClient(c); return; } @@ -1940,12 +2212,12 @@ static void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mas if (errno == EAGAIN) { nread = 0; } else { - redisLog(REDIS_DEBUG, "Reading from client: %s",strerror(errno)); + redisLog(REDIS_VERBOSE, "Reading from client: %s",strerror(errno)); freeClient(c); return; } } else if (nread == 0) { - redisLog(REDIS_DEBUG, "Client closed connection"); + redisLog(REDIS_VERBOSE, "Client closed connection"); freeClient(c); return; } @@ -1993,12 +2265,17 @@ static redisClient *createClient(int fd) { c->reply = listCreate(); listSetFreeMethod(c->reply,decrRefCount); listSetDupMethod(c->reply,dupClientReplyValue); + c->blockingkeys = NULL; + c->blockingkeysnum = 0; + c->io_keys = listCreate(); + listSetFreeMethod(c->io_keys,decrRefCount); if (aeCreateFileEvent(server.el, c->fd, AE_READABLE, readQueryFromClient, c) == AE_ERR) { freeClient(c); return NULL; } listAddNodeTail(server.clients,c); + initClientMultiState(c); return c; } @@ -2008,6 +2285,11 @@ static void addReply(redisClient *c, robj *obj) { c->replstate == REDIS_REPL_ONLINE) && aeCreateFileEvent(server.el, c->fd, AE_WRITABLE, sendReplyToClient, c) == AE_ERR) return; + + if (server.vm_enabled && obj->storage != REDIS_VM_MEMORY) { + obj = dupStringObject(obj); + obj->refcount = 0; /* getDecodedObject() will increment the refcount */ + } listAddNodeTail(c->reply,getDecodedObject(obj)); } @@ -2033,6 +2315,7 @@ static void addReplyBulkLen(redisClient *c, robj *obj) { } else { long n = (long)obj->ptr; + /* Compute how many bytes will take this integer as a radix 10 string */ len = 1; if (n < 0) { len++; @@ -2055,10 +2338,10 @@ static void acceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { cfd = anetAccept(server.neterr, fd, cip, &cport); if (cfd == AE_ERR) { - redisLog(REDIS_DEBUG,"Accepting client connection: %s", server.neterr); + redisLog(REDIS_VERBOSE,"Accepting client connection: %s", server.neterr); return; } - redisLog(REDIS_DEBUG,"Accepted %s:%d", cip, cport); + redisLog(REDIS_VERBOSE,"Accepted %s:%d", cip, cport); if ((c = createClient(cfd)) == NULL) { redisLog(REDIS_WARNING,"Error allocating resoures for the client"); close(cfd); /* May be already closed, just ingore errors */ @@ -2091,12 +2374,20 @@ static robj *createObject(int type, void *ptr) { o = listNodeValue(head); listDelNode(server.objfreelist,head); } else { - o = zmalloc(sizeof(*o)); + if (server.vm_enabled) { + o = zmalloc(sizeof(*o)); + } else { + o = zmalloc(sizeof(*o)-sizeof(struct redisObjectVM)); + } } o->type = type; o->encoding = REDIS_ENCODING_RAW; o->ptr = ptr; o->refcount = 1; + if (server.vm_enabled) { + o->vm.atime = server.unixtime; + o->storage = REDIS_VM_MEMORY; + } return o; } @@ -2104,6 +2395,11 @@ static robj *createStringObject(char *ptr, size_t len) { return createObject(REDIS_STRING,sdsnewlen(ptr,len)); } +static robj *dupStringObject(robj *o) { + assert(o->encoding == REDIS_ENCODING_RAW); + return createStringObject(o->ptr,sdslen(o->ptr)); +} + static robj *createListObject(void) { list *l = listCreate(); @@ -2151,21 +2447,34 @@ static void freeHashObject(robj *o) { } static void incrRefCount(robj *o) { + redisAssert(!server.vm_enabled || o->storage == REDIS_VM_MEMORY); o->refcount++; -#ifdef DEBUG_REFCOUNT - if (o->type == REDIS_STRING) - printf("Increment '%s'(%p), now is: %d\n",o->ptr,o,o->refcount); -#endif } static void decrRefCount(void *obj) { robj *o = obj; -#ifdef DEBUG_REFCOUNT - if (o->type == REDIS_STRING) - printf("Decrement '%s'(%p), now is: %d\n",o->ptr,o,o->refcount-1); -#endif + /* Object is swapped out, or in the process of being loaded. */ + if (server.vm_enabled && + (o->storage == REDIS_VM_SWAPPED || o->storage == REDIS_VM_LOADING)) + { + if (o->storage == REDIS_VM_SWAPPED || o->storage == REDIS_VM_LOADING) { + redisAssert(o->refcount == 1); + } + if (o->storage == REDIS_VM_LOADING) vmCancelThreadedIOJob(obj); + redisAssert(o->type == REDIS_STRING); + freeStringObject(o); + vmMarkPagesFree(o->vm.page,o->vm.usedpages); + if (listLength(server.objfreelist) > REDIS_OBJFREELIST_MAX || + !listAddNodeHead(server.objfreelist,o)) + zfree(o); + server.vm_stats_swapped_objects--; + return; + } + /* Object is in memory, or in the process of being swapped out. */ if (--(o->refcount) == 0) { + if (server.vm_enabled && o->storage == REDIS_VM_SWAPPING) + vmCancelThreadedIOJob(obj); switch(o->type) { case REDIS_STRING: freeStringObject(o); break; case REDIS_LIST: freeListObject(o); break; @@ -2182,7 +2491,31 @@ static void decrRefCount(void *obj) { static robj *lookupKey(redisDb *db, robj *key) { dictEntry *de = dictFind(db->dict,key); - return de ? dictGetEntryVal(de) : NULL; + if (de) { + robj *key = dictGetEntryKey(de); + robj *val = dictGetEntryVal(de); + + if (server.vm_enabled) { + if (key->storage == REDIS_VM_MEMORY || + key->storage == REDIS_VM_SWAPPING) + { + /* If we were swapping the object out, stop it, this key + * was requested. */ + if (key->storage == REDIS_VM_SWAPPING) + vmCancelThreadedIOJob(key); + /* Update the access time of the key for the aging algorithm. */ + key->vm.atime = server.unixtime; + } else { + /* Our value was swapped on disk. Bring it at home. */ + redisAssert(val == NULL); + val = vmLoadObject(key); + dictGetEntryVal(de) = val; + } + } + return val; + } else { + return NULL; + } } static robj *lookupKeyRead(redisDb *db, robj *key) { @@ -2365,7 +2698,7 @@ static size_t stringObjectLen(robj *o) { } } -/*============================ DB saving/loading ============================ */ +/*============================ RDB saving/loading =========================== */ static int rdbSaveType(FILE *fp, unsigned char type) { if (fwrite(&type,1,1,fp) == 0) return -1; @@ -2486,7 +2819,7 @@ static int rdbSaveStringObjectRaw(FILE *fp, robj *obj) { /* Try LZF compression - under 20 bytes it's unable to compress even * aaaaaaaaaaaaaaaaaa so skip it */ - if (len > 20) { + if (server.rdbcompression && len > 20) { int retval; retval = rdbSaveLzfStringObject(fp,obj); @@ -2505,9 +2838,23 @@ static int rdbSaveStringObjectRaw(FILE *fp, robj *obj) { static int rdbSaveStringObject(FILE *fp, robj *obj) { int retval; - obj = getDecodedObject(obj); - retval = rdbSaveStringObjectRaw(fp,obj); - decrRefCount(obj); + if (obj->storage == REDIS_VM_MEMORY && + obj->encoding != REDIS_ENCODING_RAW) + { + obj = getDecodedObject(obj); + retval = rdbSaveStringObjectRaw(fp,obj); + decrRefCount(obj); + } else { + /* This is a fast path when we are sure the object is not encoded. + * Note that's any *faster* actually as we needed to add the conditional + * but because this may happen in a background process we don't want + * to touch the object fields with incr/decrRefCount in order to + * preveny copy on write of pages. + * + * Also incrRefCount() will have a failing assert() if we try to call + * it against an object with storage != REDIS_VM_MEMORY. */ + retval = rdbSaveStringObjectRaw(fp,obj); + } return retval; } @@ -2538,6 +2885,75 @@ static int rdbSaveDoubleValue(FILE *fp, double val) { return 0; } +/* Save a Redis object. */ +static int rdbSaveObject(FILE *fp, robj *o) { + if (o->type == REDIS_STRING) { + /* Save a string value */ + if (rdbSaveStringObject(fp,o) == -1) return -1; + } else if (o->type == REDIS_LIST) { + /* Save a list value */ + list *list = o->ptr; + listNode *ln; + + listRewind(list); + if (rdbSaveLen(fp,listLength(list)) == -1) return -1; + while((ln = listYield(list))) { + robj *eleobj = listNodeValue(ln); + + if (rdbSaveStringObject(fp,eleobj) == -1) return -1; + } + } else if (o->type == REDIS_SET) { + /* Save a set value */ + dict *set = o->ptr; + dictIterator *di = dictGetIterator(set); + dictEntry *de; + + if (rdbSaveLen(fp,dictSize(set)) == -1) return -1; + while((de = dictNext(di)) != NULL) { + robj *eleobj = dictGetEntryKey(de); + + if (rdbSaveStringObject(fp,eleobj) == -1) return -1; + } + dictReleaseIterator(di); + } else if (o->type == REDIS_ZSET) { + /* Save a set value */ + zset *zs = o->ptr; + dictIterator *di = dictGetIterator(zs->dict); + dictEntry *de; + + if (rdbSaveLen(fp,dictSize(zs->dict)) == -1) return -1; + while((de = dictNext(di)) != NULL) { + robj *eleobj = dictGetEntryKey(de); + double *score = dictGetEntryVal(de); + + if (rdbSaveStringObject(fp,eleobj) == -1) return -1; + if (rdbSaveDoubleValue(fp,*score) == -1) return -1; + } + dictReleaseIterator(di); + } else { + redisAssert(0 != 0); + } + return 0; +} + +/* Return the length the object will have on disk if saved with + * the rdbSaveObject() function. Currently we use a trick to get + * this length with very little changes to the code. In the future + * we could switch to a faster solution. */ +static off_t rdbSavedObjectLen(robj *o, FILE *fp) { + if (fp == NULL) fp = server.devnull; + rewind(fp); + assert(rdbSaveObject(fp,o) != 1); + return ftello(fp); +} + +/* Return the number of pages required to save this object in the swap file */ +static off_t rdbSavedObjectPages(robj *o, FILE *fp) { + off_t bytes = rdbSavedObjectLen(o,fp); + + return (bytes+(server.vm_page_size-1))/server.vm_page_size; +} + /* Save the DB on disk. Return REDIS_ERR on error, REDIS_OK on success */ static int rdbSave(char *filename) { dictIterator *di = NULL; @@ -2581,54 +2997,25 @@ static int rdbSave(char *filename) { if (rdbSaveType(fp,REDIS_EXPIRETIME) == -1) goto werr; if (rdbSaveTime(fp,expiretime) == -1) goto werr; } - /* Save the key and associated value */ - if (rdbSaveType(fp,o->type) == -1) goto werr; - if (rdbSaveStringObject(fp,key) == -1) goto werr; - if (o->type == REDIS_STRING) { - /* Save a string value */ - if (rdbSaveStringObject(fp,o) == -1) goto werr; - } else if (o->type == REDIS_LIST) { - /* Save a list value */ - list *list = o->ptr; - listNode *ln; - - listRewind(list); - if (rdbSaveLen(fp,listLength(list)) == -1) goto werr; - while((ln = listYield(list))) { - robj *eleobj = listNodeValue(ln); - - if (rdbSaveStringObject(fp,eleobj) == -1) goto werr; - } - } else if (o->type == REDIS_SET) { - /* Save a set value */ - dict *set = o->ptr; - dictIterator *di = dictGetIterator(set); - dictEntry *de; - - if (rdbSaveLen(fp,dictSize(set)) == -1) goto werr; - while((de = dictNext(di)) != NULL) { - robj *eleobj = dictGetEntryKey(de); - - if (rdbSaveStringObject(fp,eleobj) == -1) goto werr; - } - dictReleaseIterator(di); - } else if (o->type == REDIS_ZSET) { - /* Save a set value */ - zset *zs = o->ptr; - dictIterator *di = dictGetIterator(zs->dict); - dictEntry *de; - - if (rdbSaveLen(fp,dictSize(zs->dict)) == -1) goto werr; - while((de = dictNext(di)) != NULL) { - robj *eleobj = dictGetEntryKey(de); - double *score = dictGetEntryVal(de); - - if (rdbSaveStringObject(fp,eleobj) == -1) goto werr; - if (rdbSaveDoubleValue(fp,*score) == -1) goto werr; - } - dictReleaseIterator(di); + /* Save the key and associated value. This requires special + * handling if the value is swapped out. */ + if (!server.vm_enabled || key->storage == REDIS_VM_MEMORY || + key->storage == REDIS_VM_SWAPPING) { + /* Save type, key, value */ + if (rdbSaveType(fp,o->type) == -1) goto werr; + if (rdbSaveStringObject(fp,key) == -1) goto werr; + if (rdbSaveObject(fp,o) == -1) goto werr; } else { - redisAssert(0 != 0); + /* REDIS_VM_SWAPPED or REDIS_VM_LOADING */ + robj *po; + /* Get a preview of the object in memory */ + po = vmPreviewObject(key); + /* Save type, key, value */ + if (rdbSaveType(fp,key->vtype) == -1) goto werr; + if (rdbSaveStringObject(fp,key) == -1) goto werr; + if (rdbSaveObject(fp,po) == -1) goto werr; + /* Remove the loaded object from memory */ + decrRefCount(po); } } dictReleaseIterator(di); @@ -2711,35 +3098,29 @@ static time_t rdbLoadTime(FILE *fp) { * * isencoded is set to 1 if the readed length is not actually a length but * an "encoding type", check the above comments for more info */ -static uint32_t rdbLoadLen(FILE *fp, int rdbver, int *isencoded) { +static uint32_t rdbLoadLen(FILE *fp, int *isencoded) { unsigned char buf[2]; uint32_t len; + int type; if (isencoded) *isencoded = 0; - if (rdbver == 0) { + if (fread(buf,1,1,fp) == 0) return REDIS_RDB_LENERR; + type = (buf[0]&0xC0)>>6; + if (type == REDIS_RDB_6BITLEN) { + /* Read a 6 bit len */ + return buf[0]&0x3F; + } else if (type == REDIS_RDB_ENCVAL) { + /* Read a 6 bit len encoding type */ + if (isencoded) *isencoded = 1; + return buf[0]&0x3F; + } else if (type == REDIS_RDB_14BITLEN) { + /* Read a 14 bit len */ + if (fread(buf+1,1,1,fp) == 0) return REDIS_RDB_LENERR; + return ((buf[0]&0x3F)<<8)|buf[1]; + } else { + /* Read a 32 bit len */ if (fread(&len,4,1,fp) == 0) return REDIS_RDB_LENERR; return ntohl(len); - } else { - int type; - - if (fread(buf,1,1,fp) == 0) return REDIS_RDB_LENERR; - type = (buf[0]&0xC0)>>6; - if (type == REDIS_RDB_6BITLEN) { - /* Read a 6 bit len */ - return buf[0]&0x3F; - } else if (type == REDIS_RDB_ENCVAL) { - /* Read a 6 bit len encoding type */ - if (isencoded) *isencoded = 1; - return buf[0]&0x3F; - } else if (type == REDIS_RDB_14BITLEN) { - /* Read a 14 bit len */ - if (fread(buf+1,1,1,fp) == 0) return REDIS_RDB_LENERR; - return ((buf[0]&0x3F)<<8)|buf[1]; - } else { - /* Read a 32 bit len */ - if (fread(&len,4,1,fp) == 0) return REDIS_RDB_LENERR; - return ntohl(len); - } } } @@ -2767,13 +3148,13 @@ static robj *rdbLoadIntegerObject(FILE *fp, int enctype) { return createObject(REDIS_STRING,sdscatprintf(sdsempty(),"%lld",val)); } -static robj *rdbLoadLzfStringObject(FILE*fp, int rdbver) { +static robj *rdbLoadLzfStringObject(FILE*fp) { unsigned int len, clen; unsigned char *c = NULL; sds val = NULL; - if ((clen = rdbLoadLen(fp,rdbver,NULL)) == REDIS_RDB_LENERR) return NULL; - if ((len = rdbLoadLen(fp,rdbver,NULL)) == REDIS_RDB_LENERR) return NULL; + if ((clen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; + if ((len = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; if ((c = zmalloc(clen)) == NULL) goto err; if ((val = sdsnewlen(NULL,len)) == NULL) goto err; if (fread(c,clen,1,fp) == 0) goto err; @@ -2786,12 +3167,12 @@ err: return NULL; } -static robj *rdbLoadStringObject(FILE*fp, int rdbver) { +static robj *rdbLoadStringObject(FILE*fp) { int isencoded; uint32_t len; sds val; - len = rdbLoadLen(fp,rdbver,&isencoded); + len = rdbLoadLen(fp,&isencoded); if (isencoded) { switch(len) { case REDIS_RDB_ENC_INT8: @@ -2799,7 +3180,7 @@ static robj *rdbLoadStringObject(FILE*fp, int rdbver) { case REDIS_RDB_ENC_INT32: return tryObjectSharing(rdbLoadIntegerObject(fp,len)); case REDIS_RDB_ENC_LZF: - return tryObjectSharing(rdbLoadLzfStringObject(fp,rdbver)); + return tryObjectSharing(rdbLoadLzfStringObject(fp)); default: redisAssert(0!=0); } @@ -2832,6 +3213,59 @@ static int rdbLoadDoubleValue(FILE *fp, double *val) { } } +/* Load a Redis object of the specified type from the specified file. + * On success a newly allocated object is returned, otherwise NULL. */ +static robj *rdbLoadObject(int type, FILE *fp) { + robj *o; + + if (type == REDIS_STRING) { + /* Read string value */ + if ((o = rdbLoadStringObject(fp)) == NULL) return NULL; + tryObjectEncoding(o); + } else if (type == REDIS_LIST || type == REDIS_SET) { + /* Read list/set value */ + uint32_t listlen; + + if ((listlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; + o = (type == REDIS_LIST) ? createListObject() : createSetObject(); + /* Load every single element of the list/set */ + while(listlen--) { + robj *ele; + + if ((ele = rdbLoadStringObject(fp)) == NULL) return NULL; + tryObjectEncoding(ele); + if (type == REDIS_LIST) { + listAddNodeTail((list*)o->ptr,ele); + } else { + dictAdd((dict*)o->ptr,ele,NULL); + } + } + } else if (type == REDIS_ZSET) { + /* Read list/set value */ + uint32_t zsetlen; + zset *zs; + + if ((zsetlen = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) return NULL; + o = createZsetObject(); + zs = o->ptr; + /* Load every single element of the list/set */ + while(zsetlen--) { + robj *ele; + double *score = zmalloc(sizeof(double)); + + if ((ele = rdbLoadStringObject(fp)) == NULL) return NULL; + tryObjectEncoding(ele); + if (rdbLoadDoubleValue(fp,score) == -1) return NULL; + dictAdd(zs->dict,ele,score); + zslInsert(zs->zsl,*score,ele); + incrRefCount(ele); /* added to skiplist */ + } + } else { + redisAssert(0 != 0); + } + return o; +} + static int rdbLoad(char *filename) { FILE *fp; robj *keyobj = NULL; @@ -2841,6 +3275,7 @@ static int rdbLoad(char *filename) { redisDb *db = server.db+0; char buf[1024]; time_t expiretime = -1, now = time(NULL); + long long loadedkeys = 0; fp = fopen(filename,"r"); if (!fp) return REDIS_ERR; @@ -2852,7 +3287,7 @@ static int rdbLoad(char *filename) { return REDIS_ERR; } rdbver = atoi(buf+5); - if (rdbver > 1) { + if (rdbver != 1) { fclose(fp); redisLog(REDIS_WARNING,"Can't handle RDB format version %d",rdbver); return REDIS_ERR; @@ -2870,7 +3305,7 @@ static int rdbLoad(char *filename) { if (type == REDIS_EOF) break; /* Handle SELECT DB opcode as a special case */ if (type == REDIS_SELECTDB) { - if ((dbid = rdbLoadLen(fp,rdbver,NULL)) == REDIS_RDB_LENERR) + if ((dbid = rdbLoadLen(fp,NULL)) == REDIS_RDB_LENERR) goto eoferr; if (dbid >= (unsigned)server.dbnum) { redisLog(REDIS_WARNING,"FATAL: Data file was created with a Redis server configured to handle more than %d databases. Exiting\n", server.dbnum); @@ -2881,55 +3316,9 @@ static int rdbLoad(char *filename) { continue; } /* Read key */ - if ((keyobj = rdbLoadStringObject(fp,rdbver)) == NULL) goto eoferr; - - if (type == REDIS_STRING) { - /* Read string value */ - if ((o = rdbLoadStringObject(fp,rdbver)) == NULL) goto eoferr; - tryObjectEncoding(o); - } else if (type == REDIS_LIST || type == REDIS_SET) { - /* Read list/set value */ - uint32_t listlen; - - if ((listlen = rdbLoadLen(fp,rdbver,NULL)) == REDIS_RDB_LENERR) - goto eoferr; - o = (type == REDIS_LIST) ? createListObject() : createSetObject(); - /* Load every single element of the list/set */ - while(listlen--) { - robj *ele; - - if ((ele = rdbLoadStringObject(fp,rdbver)) == NULL) goto eoferr; - tryObjectEncoding(ele); - if (type == REDIS_LIST) { - listAddNodeTail((list*)o->ptr,ele); - } else { - dictAdd((dict*)o->ptr,ele,NULL); - } - } - } else if (type == REDIS_ZSET) { - /* Read list/set value */ - uint32_t zsetlen; - zset *zs; - - if ((zsetlen = rdbLoadLen(fp,rdbver,NULL)) == REDIS_RDB_LENERR) - goto eoferr; - o = createZsetObject(); - zs = o->ptr; - /* Load every single element of the list/set */ - while(zsetlen--) { - robj *ele; - double *score = zmalloc(sizeof(double)); - - if ((ele = rdbLoadStringObject(fp,rdbver)) == NULL) goto eoferr; - tryObjectEncoding(ele); - if (rdbLoadDoubleValue(fp,score) == -1) goto eoferr; - dictAdd(zs->dict,ele,score); - zslInsert(zs->zsl,*score,ele); - incrRefCount(ele); /* added to skiplist */ - } - } else { - redisAssert(0 != 0); - } + if ((keyobj = rdbLoadStringObject(fp)) == NULL) goto eoferr; + /* Read value */ + if ((o = rdbLoadObject(type,fp)) == NULL) goto eoferr; /* Add the new object in the hash table */ retval = dictAdd(d,keyobj,o); if (retval == DICT_ERR) { @@ -2944,6 +3333,13 @@ static int rdbLoad(char *filename) { expiretime = -1; } keyobj = o = NULL; + /* Handle swapping while loading big datasets when VM is on */ + loadedkeys++; + if (server.vm_enabled && (loadedkeys % 5000) == 0) { + while (zmalloc_used_memory() > server.vm_max_memory) { + if (vmSwapOneObjectBlocking() == REDIS_ERR) break; + } + } } fclose(fp); return REDIS_OK; @@ -2986,6 +3382,12 @@ static void setGenericCommand(redisClient *c, int nx) { retval = dictAdd(c->db->dict,c->argv[1],c->argv[2]); if (retval == DICT_ERR) { if (!nx) { + /* If the key is about a swapped value, we want a new key object + * to overwrite the old. So we delete the old key in the database. + * This will also make sure that swap pages about the old object + * will be marked as free. */ + if (deleteIfSwapped(c->db,c->argv[1])) + incrRefCount(c->argv[1]); dictReplace(c->db->dict,c->argv[1],c->argv[2]); incrRefCount(c->argv[2]); } else { @@ -3009,24 +3411,31 @@ static void setnxCommand(redisClient *c) { setGenericCommand(c,1); } -static void getCommand(redisClient *c) { +static int getGenericCommand(redisClient *c) { robj *o = lookupKeyRead(c->db,c->argv[1]); if (o == NULL) { addReply(c,shared.nullbulk); + return REDIS_OK; } else { if (o->type != REDIS_STRING) { addReply(c,shared.wrongtypeerr); + return REDIS_ERR; } else { addReplyBulkLen(c,o); addReply(c,o); addReply(c,shared.crlf); + return REDIS_OK; } } } +static void getCommand(redisClient *c) { + getGenericCommand(c); +} + static void getsetCommand(redisClient *c) { - getCommand(c); + if (getGenericCommand(c) == REDIS_ERR) return; if (dictAdd(c->db->dict,c->argv[1],c->argv[2]) == DICT_ERR) { dictReplace(c->db->dict,c->argv[1],c->argv[2]); } else { @@ -3295,7 +3704,8 @@ static void bgsaveCommand(redisClient *c) { return; } if (rdbSaveBackground(server.dbfilename) == REDIS_OK) { - addReply(c,shared.ok); + char *status = "+Background saving started\r\n"; + addReplySds(c,sdsnew(status)); } else { addReply(c,shared.err); } @@ -3311,20 +3721,26 @@ static void shutdownCommand(redisClient *c) { kill(server.bgsavechildpid,SIGKILL); rdbRemoveTempFile(server.bgsavechildpid); } - /* SYNC SAVE */ - if (rdbSave(server.dbfilename) == REDIS_OK) { - if (server.daemonize) - unlink(server.pidfile); - redisLog(REDIS_WARNING,"%zu bytes used at exit",zmalloc_used_memory()); - redisLog(REDIS_WARNING,"Server exit now, bye bye..."); - exit(1); + if (server.appendonly) { + /* Append only file: fsync() the AOF and exit */ + fsync(server.appendfd); + exit(0); } else { - /* Ooops.. error saving! The best we can do is to continue operating. - * Note that if there was a background saving process, in the next - * cron() Redis will be notified that the background saving aborted, - * handling special stuff like slaves pending for synchronization... */ - redisLog(REDIS_WARNING,"Error trying to save the DB, can't exit"); - addReplySds(c,sdsnew("-ERR can't quit, problems saving the DB\r\n")); + /* Snapshotting. Perform a SYNC SAVE and exit */ + if (rdbSave(server.dbfilename) == REDIS_OK) { + if (server.daemonize) + unlink(server.pidfile); + redisLog(REDIS_WARNING,"%zu bytes used at exit",zmalloc_used_memory()); + redisLog(REDIS_WARNING,"Server exit now, bye bye..."); + exit(0); + } else { + /* Ooops.. error saving! The best we can do is to continue operating. + * Note that if there was a background saving process, in the next + * cron() Redis will be notified that the background saving aborted, + * handling special stuff like slaves pending for synchronization... */ + redisLog(REDIS_WARNING,"Error trying to save the DB, can't exit"); + addReplySds(c,sdsnew("-ERR can't quit, problems saving the DB\r\n")); + } } } @@ -3418,6 +3834,10 @@ static void pushGenericCommand(redisClient *c, int where) { lobj = lookupKeyWrite(c->db,c->argv[1]); if (lobj == NULL) { + if (handleClientsWaitingListPush(c,c->argv[1],c->argv[2])) { + addReply(c,shared.ok); + return; + } lobj = createListObject(); list = lobj->ptr; if (where == REDIS_HEAD) { @@ -3433,6 +3853,10 @@ static void pushGenericCommand(redisClient *c, int where) { addReply(c,shared.wrongtypeerr); return; } + if (handleClientsWaitingListPush(c,c->argv[1],c->argv[2])) { + addReply(c,shared.ok); + return; + } list = lobj->ptr; if (where == REDIS_HEAD) { listAddNodeHead(list,c->argv[2]); @@ -3622,7 +4046,7 @@ static void ltrimCommand(redisClient *c) { o = lookupKeyWrite(c->db,c->argv[1]); if (o == NULL) { - addReply(c,shared.nokeyerr); + addReply(c,shared.ok); } else { if (o->type != REDIS_LIST) { addReply(c,shared.wrongtypeerr); @@ -3737,20 +4161,24 @@ static void rpoplpushcommand(redisClient *c) { robj *ele = listNodeValue(ln); list *dstlist; - if (dobj == NULL) { - - /* Create the list if the key does not exist */ - dobj = createListObject(); - dictAdd(c->db->dict,c->argv[2],dobj); - incrRefCount(c->argv[2]); - } else if (dobj->type != REDIS_LIST) { + if (dobj && dobj->type != REDIS_LIST) { addReply(c,shared.wrongtypeerr); return; } - /* Add the element to the target list */ - dstlist = dobj->ptr; - listAddNodeHead(dstlist,ele); - incrRefCount(ele); + + /* Add the element to the target list (unless it's directly + * passed to some BLPOP-ing client */ + if (!handleClientsWaitingListPush(c,c->argv[2],ele)) { + if (dobj == NULL) { + /* Create the list if the key does not exist */ + dobj = createListObject(); + dictAdd(c->db->dict,c->argv[2],dobj); + incrRefCount(c->argv[2]); + } + dstlist = dobj->ptr; + listAddNodeHead(dstlist,ele); + incrRefCount(ele); + } /* Send the element to the client as reply as well */ addReplyBulkLen(c,ele); @@ -3959,8 +4387,9 @@ static void sinterGenericCommand(redisClient *c, robj **setskeys, unsigned long if (!setobj) { zfree(dv); if (dstkey) { - deleteKey(c->db,dstkey); - addReply(c,shared.ok); + if (deleteKey(c->db,dstkey)) + server.dirty++; + addReply(c,shared.czero); } else { addReply(c,shared.nullmultibulk); } @@ -4515,6 +4944,14 @@ static void zrangeGenericCommand(redisClient *c, int reverse) { robj *o; int start = atoi(c->argv[2]->ptr); int end = atoi(c->argv[3]->ptr); + int withscores = 0; + + if (c->argc == 5 && !strcasecmp(c->argv[4]->ptr,"withscores")) { + withscores = 1; + } else if (c->argc >= 5) { + addReply(c,shared.syntaxerr); + return; + } o = lookupKeyRead(c->db,c->argv[1]); if (o == NULL) { @@ -4557,12 +4994,15 @@ static void zrangeGenericCommand(redisClient *c, int reverse) { ln = ln->forward[0]; } - addReplySds(c,sdscatprintf(sdsempty(),"*%d\r\n",rangelen)); + addReplySds(c,sdscatprintf(sdsempty(),"*%d\r\n", + withscores ? (rangelen*2) : rangelen)); for (j = 0; j < rangelen; j++) { ele = ln->obj; addReplyBulkLen(c,ele); addReply(c,ele); addReply(c,shared.crlf); + if (withscores) + addReplyDouble(c,ln->score); ln = reverse ? ln->backward : ln->forward[0]; } } @@ -4823,7 +5263,7 @@ static void sortCommand(redisClient *c) { /* Lookup the key to sort. It must be of the right types */ sortval = lookupKeyRead(c->db,c->argv[1]); if (sortval == NULL) { - addReply(c,shared.nokeyerr); + addReply(c,shared.nullmultibulk); return; } if (sortval->type != REDIS_SET && sortval->type != REDIS_LIST && @@ -5064,6 +5504,27 @@ static void sortCommand(redisClient *c) { zfree(vector); } +/* Convert an amount of bytes into a human readable string in the form + * of 100B, 2G, 100M, 4K, and so forth. */ +static void bytesToHuman(char *s, unsigned long long n) { + double d; + + if (n < 1024) { + /* Bytes */ + sprintf(s,"%lluB",n); + return; + } else if (n < (1024*1024)) { + d = (double)n/(1024); + sprintf(s,"%.2fK",d); + } else if (n < (1024LL*1024*1024)) { + d = (double)n/(1024*1024); + sprintf(s,"%.2fM",d); + } else if (n < (1024LL*1024*1024*1024)) { + d = (double)n/(1024LL*1024*1024); + sprintf(s,"%.2fM",d); + } +} + /* Create the string returned by the INFO command. This is decoupled * by the INFO command itself as we need to report the same information * on memory corruption problems. */ @@ -5071,35 +5532,47 @@ static sds genRedisInfoString(void) { sds info; time_t uptime = time(NULL)-server.stat_starttime; int j; - + char hmem[64]; + + bytesToHuman(hmem,server.usedmemory); info = sdscatprintf(sdsempty(), "redis_version:%s\r\n" "arch_bits:%s\r\n" "multiplexing_api:%s\r\n" + "process_id:%ld\r\n" "uptime_in_seconds:%ld\r\n" "uptime_in_days:%ld\r\n" "connected_clients:%d\r\n" "connected_slaves:%d\r\n" + "blocked_clients:%d\r\n" "used_memory:%zu\r\n" + "used_memory_human:%s\r\n" "changes_since_last_save:%lld\r\n" "bgsave_in_progress:%d\r\n" "last_save_time:%ld\r\n" + "bgrewriteaof_in_progress:%d\r\n" "total_connections_received:%lld\r\n" "total_commands_processed:%lld\r\n" + "vm_enabled:%d\r\n" "role:%s\r\n" ,REDIS_VERSION, (sizeof(long) == 8) ? "64" : "32", aeGetApiName(), + (long) getpid(), uptime, uptime/(3600*24), listLength(server.clients)-listLength(server.slaves), listLength(server.slaves), + server.blockedclients, server.usedmemory, + hmem, server.dirty, server.bgsavechildpid != -1, server.lastsave, + server.bgrewritechildpid != -1, server.stat_numconnections, server.stat_numcommands, + server.vm_enabled != 0, server.masterhost == NULL ? "master" : "slave" ); if (server.masterhost) { @@ -5115,6 +5588,32 @@ static sds genRedisInfoString(void) { server.master ? ((int)(time(NULL)-server.master->lastinteraction)) : -1 ); } + if (server.vm_enabled) { + info = sdscatprintf(info, + "vm_conf_max_memory:%llu\r\n" + "vm_conf_page_size:%llu\r\n" + "vm_conf_pages:%llu\r\n" + "vm_stats_used_pages:%llu\r\n" + "vm_stats_swapped_objects:%llu\r\n" + "vm_stats_swappin_count:%llu\r\n" + "vm_stats_swappout_count:%llu\r\n" + "vm_stats_io_newjobs_len:%lu\r\n" + "vm_stats_io_processing_len:%lu\r\n" + "vm_stats_io_processed_len:%lu\r\n" + "vm_stats_io_waiting_clients:%lu\r\n" + ,(unsigned long long) server.vm_max_memory, + (unsigned long long) server.vm_page_size, + (unsigned long long) server.vm_pages, + (unsigned long long) server.vm_stats_used_pages, + (unsigned long long) server.vm_stats_swapped_objects, + (unsigned long long) server.vm_stats_swapins, + (unsigned long long) server.vm_stats_swapouts, + (unsigned long) listLength(server.io_newjobs), + (unsigned long) listLength(server.io_processing), + (unsigned long) listLength(server.io_processed), + (unsigned long) listLength(server.io_clients) + ); + } for (j = 0; j < server.dbnum; j++) { long long keys, vkeys; @@ -5250,18 +5749,288 @@ static void ttlCommand(redisClient *c) { addReplySds(c,sdscatprintf(sdsempty(),":%d\r\n",ttl)); } -/* =============================== Replication ============================= */ +/* ================================ MULTI/EXEC ============================== */ -static int syncWrite(int fd, char *ptr, ssize_t size, int timeout) { - ssize_t nwritten, ret = size; - time_t start = time(NULL); +/* Client state initialization for MULTI/EXEC */ +static void initClientMultiState(redisClient *c) { + c->mstate.commands = NULL; + c->mstate.count = 0; +} - timeout++; - while(size) { - if (aeWait(fd,AE_WRITABLE,1000) & AE_WRITABLE) { - nwritten = write(fd,ptr,size); - if (nwritten == -1) return -1; - ptr += nwritten; +/* Release all the resources associated with MULTI/EXEC state */ +static void freeClientMultiState(redisClient *c) { + int j; + + for (j = 0; j < c->mstate.count; j++) { + int i; + multiCmd *mc = c->mstate.commands+j; + + for (i = 0; i < mc->argc; i++) + decrRefCount(mc->argv[i]); + zfree(mc->argv); + } + zfree(c->mstate.commands); +} + +/* Add a new command into the MULTI commands queue */ +static void queueMultiCommand(redisClient *c, struct redisCommand *cmd) { + multiCmd *mc; + int j; + + c->mstate.commands = zrealloc(c->mstate.commands, + sizeof(multiCmd)*(c->mstate.count+1)); + mc = c->mstate.commands+c->mstate.count; + mc->cmd = cmd; + mc->argc = c->argc; + mc->argv = zmalloc(sizeof(robj*)*c->argc); + memcpy(mc->argv,c->argv,sizeof(robj*)*c->argc); + for (j = 0; j < c->argc; j++) + incrRefCount(mc->argv[j]); + c->mstate.count++; +} + +static void multiCommand(redisClient *c) { + c->flags |= REDIS_MULTI; + addReply(c,shared.ok); +} + +static void execCommand(redisClient *c) { + int j; + robj **orig_argv; + int orig_argc; + + if (!(c->flags & REDIS_MULTI)) { + addReplySds(c,sdsnew("-ERR EXEC without MULTI\r\n")); + return; + } + + orig_argv = c->argv; + orig_argc = c->argc; + addReplySds(c,sdscatprintf(sdsempty(),"*%d\r\n",c->mstate.count)); + for (j = 0; j < c->mstate.count; j++) { + c->argc = c->mstate.commands[j].argc; + c->argv = c->mstate.commands[j].argv; + call(c,c->mstate.commands[j].cmd); + } + c->argv = orig_argv; + c->argc = orig_argc; + freeClientMultiState(c); + initClientMultiState(c); + c->flags &= (~REDIS_MULTI); +} + +/* =========================== Blocking Operations ========================= */ + +/* Currently Redis blocking operations support is limited to list POP ops, + * so the current implementation is not fully generic, but it is also not + * completely specific so it will not require a rewrite to support new + * kind of blocking operations in the future. + * + * Still it's important to note that list blocking operations can be already + * used as a notification mechanism in order to implement other blocking + * operations at application level, so there must be a very strong evidence + * of usefulness and generality before new blocking operations are implemented. + * + * This is how the current blocking POP works, we use BLPOP as example: + * - If the user calls BLPOP and the key exists and contains a non empty list + * then LPOP is called instead. So BLPOP is semantically the same as LPOP + * if there is not to block. + * - If instead BLPOP is called and the key does not exists or the list is + * empty we need to block. In order to do so we remove the notification for + * new data to read in the client socket (so that we'll not serve new + * requests if the blocking request is not served). Also we put the client + * in a dictionary (db->blockingkeys) mapping keys to a list of clients + * blocking for this keys. + * - If a PUSH operation against a key with blocked clients waiting is + * performed, we serve the first in the list: basically instead to push + * the new element inside the list we return it to the (first / oldest) + * blocking client, unblock the client, and remove it form the list. + * + * The above comment and the source code should be enough in order to understand + * the implementation and modify / fix it later. + */ + +/* Set a client in blocking mode for the specified key, with the specified + * timeout */ +static void blockForKeys(redisClient *c, robj **keys, int numkeys, time_t timeout) { + dictEntry *de; + list *l; + int j; + + c->blockingkeys = zmalloc(sizeof(robj*)*numkeys); + c->blockingkeysnum = numkeys; + c->blockingto = timeout; + for (j = 0; j < numkeys; j++) { + /* Add the key in the client structure, to map clients -> keys */ + c->blockingkeys[j] = keys[j]; + incrRefCount(keys[j]); + + /* And in the other "side", to map keys -> clients */ + de = dictFind(c->db->blockingkeys,keys[j]); + if (de == NULL) { + int retval; + + /* For every key we take a list of clients blocked for it */ + l = listCreate(); + retval = dictAdd(c->db->blockingkeys,keys[j],l); + incrRefCount(keys[j]); + assert(retval == DICT_OK); + } else { + l = dictGetEntryVal(de); + } + listAddNodeTail(l,c); + } + /* Mark the client as a blocked client */ + c->flags |= REDIS_BLOCKED; + aeDeleteFileEvent(server.el,c->fd,AE_READABLE); + server.blockedclients++; +} + +/* Unblock a client that's waiting in a blocking operation such as BLPOP */ +static void unblockClient(redisClient *c) { + dictEntry *de; + list *l; + int j; + + assert(c->blockingkeys != NULL); + /* The client may wait for multiple keys, so unblock it for every key. */ + for (j = 0; j < c->blockingkeysnum; j++) { + /* Remove this client from the list of clients waiting for this key. */ + de = dictFind(c->db->blockingkeys,c->blockingkeys[j]); + assert(de != NULL); + l = dictGetEntryVal(de); + listDelNode(l,listSearchKey(l,c)); + /* If the list is empty we need to remove it to avoid wasting memory */ + if (listLength(l) == 0) + dictDelete(c->db->blockingkeys,c->blockingkeys[j]); + decrRefCount(c->blockingkeys[j]); + } + /* Cleanup the client structure */ + zfree(c->blockingkeys); + c->blockingkeys = NULL; + c->flags &= (~REDIS_BLOCKED); + server.blockedclients--; + /* Ok now we are ready to get read events from socket, note that we + * can't trap errors here as it's possible that unblockClients() is + * called from freeClient() itself, and the only thing we can do + * if we failed to register the READABLE event is to kill the client. + * Still the following function should never fail in the real world as + * we are sure the file descriptor is sane, and we exit on out of mem. */ + aeCreateFileEvent(server.el, c->fd, AE_READABLE, readQueryFromClient, c); + /* As a final step we want to process data if there is some command waiting + * in the input buffer. Note that this is safe even if unblockClient() + * gets called from freeClient() because freeClient() will be smart + * enough to call this function *after* c->querybuf was set to NULL. */ + if (c->querybuf && sdslen(c->querybuf) > 0) processInputBuffer(c); +} + +/* This should be called from any function PUSHing into lists. + * 'c' is the "pushing client", 'key' is the key it is pushing data against, + * 'ele' is the element pushed. + * + * If the function returns 0 there was no client waiting for a list push + * against this key. + * + * If the function returns 1 there was a client waiting for a list push + * against this key, the element was passed to this client thus it's not + * needed to actually add it to the list and the caller should return asap. */ +static int handleClientsWaitingListPush(redisClient *c, robj *key, robj *ele) { + struct dictEntry *de; + redisClient *receiver; + list *l; + listNode *ln; + + de = dictFind(c->db->blockingkeys,key); + if (de == NULL) return 0; + l = dictGetEntryVal(de); + ln = listFirst(l); + assert(ln != NULL); + receiver = ln->value; + + addReplySds(receiver,sdsnew("*2\r\n")); + addReplyBulkLen(receiver,key); + addReply(receiver,key); + addReply(receiver,shared.crlf); + addReplyBulkLen(receiver,ele); + addReply(receiver,ele); + addReply(receiver,shared.crlf); + unblockClient(receiver); + return 1; +} + +/* Blocking RPOP/LPOP */ +static void blockingPopGenericCommand(redisClient *c, int where) { + robj *o; + time_t timeout; + int j; + + for (j = 1; j < c->argc-1; j++) { + o = lookupKeyWrite(c->db,c->argv[j]); + if (o != NULL) { + if (o->type != REDIS_LIST) { + addReply(c,shared.wrongtypeerr); + return; + } else { + list *list = o->ptr; + if (listLength(list) != 0) { + /* If the list contains elements fall back to the usual + * non-blocking POP operation */ + robj *argv[2], **orig_argv; + int orig_argc; + + /* We need to alter the command arguments before to call + * popGenericCommand() as the command takes a single key. */ + orig_argv = c->argv; + orig_argc = c->argc; + argv[1] = c->argv[j]; + c->argv = argv; + c->argc = 2; + + /* Also the return value is different, we need to output + * the multi bulk reply header and the key name. The + * "real" command will add the last element (the value) + * for us. If this souds like an hack to you it's just + * because it is... */ + addReplySds(c,sdsnew("*2\r\n")); + addReplyBulkLen(c,argv[1]); + addReply(c,argv[1]); + addReply(c,shared.crlf); + popGenericCommand(c,where); + + /* Fix the client structure with the original stuff */ + c->argv = orig_argv; + c->argc = orig_argc; + return; + } + } + } + } + /* If the list is empty or the key does not exists we must block */ + timeout = strtol(c->argv[c->argc-1]->ptr,NULL,10); + if (timeout > 0) timeout += time(NULL); + blockForKeys(c,c->argv+1,c->argc-2,timeout); +} + +static void blpopCommand(redisClient *c) { + blockingPopGenericCommand(c,REDIS_HEAD); +} + +static void brpopCommand(redisClient *c) { + blockingPopGenericCommand(c,REDIS_TAIL); +} + +/* =============================== Replication ============================= */ + +static int syncWrite(int fd, char *ptr, ssize_t size, int timeout) { + ssize_t nwritten, ret = size; + time_t start = time(NULL); + + timeout++; + while(size) { + if (aeWait(fd,AE_WRITABLE,1000) & AE_WRITABLE) { + nwritten = write(fd,ptr,size); + if (nwritten == -1) return -1; + ptr += nwritten; size -= nwritten; } if ((time(NULL)-start) > timeout) { @@ -5405,7 +6174,7 @@ static void sendBulkToSlave(aeEventLoop *el, int fd, void *privdata, int mask) { return; } if ((nwritten = write(fd,buf,buflen)) == -1) { - redisLog(REDIS_DEBUG,"Write error sending DB to slave: %s", + redisLog(REDIS_VERBOSE,"Write error sending DB to slave: %s", strerror(errno)); freeClient(slave); return; @@ -5580,6 +6349,7 @@ static int syncWithMaster(void) { } server.master = createClient(fd); server.master->flags |= REDIS_MASTER; + server.master->authenticated = 1; server.replstate = REDIS_REPL_CONNECTED; return REDIS_OK; } @@ -5608,6 +6378,18 @@ static void slaveofCommand(redisClient *c) { /* ============================ Maxmemory directive ======================== */ +/* Free one object form the pre-allocated objects free list. This is useful + * under low mem conditions as by default we take 1 million free objects + * allocated. */ +static void freeOneObjectFromFreelist(void) { + robj *o; + + listNode *head = listFirst(server.objfreelist); + o = listNodeValue(head); + listDelNode(server.objfreelist,head); + zfree(o); +} + /* This function gets called when 'maxmemory' is set on the config file to limit * the max memory used by the server, and we are out of memory. * This function will try to, in order: @@ -5622,12 +6404,7 @@ static void slaveofCommand(redisClient *c) { static void freeMemoryIfNeeded(void) { while (server.maxmemory && zmalloc_used_memory() > server.maxmemory) { if (listLength(server.objfreelist)) { - robj *o; - - listNode *head = listFirst(server.objfreelist); - o = listNodeValue(head); - listDelNode(server.objfreelist,head); - zfree(o); + freeOneObjectFromFreelist(); } else { int j, k, freed = 0; @@ -5778,6 +6555,7 @@ int loadAppendOnlyFile(char *filename) { struct redisClient *fakeClient; FILE *fp = fopen(filename,"r"); struct redis_stat sb; + unsigned long long loadedkeys = 0; if (redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0) return REDIS_ERR; @@ -5839,6 +6617,13 @@ int loadAppendOnlyFile(char *filename) { /* Clean up, ready for the next command */ for (j = 0; j < argc; j++) decrRefCount(argv[j]); zfree(argv); + /* Handle swapping while loading big datasets when VM is on */ + loadedkeys++; + if (server.vm_enabled && (loadedkeys % 5000) == 0) { + while (zmalloc_used_memory() > server.vm_max_memory) { + if (vmSwapOneObjectBlocking() == REDIS_ERR) break; + } + } } fclose(fp); freeFakeClient(fakeClient); @@ -5859,15 +6644,21 @@ fmterr: /* Write an object into a file in the bulk format $\r\n\r\n */ static int fwriteBulk(FILE *fp, robj *obj) { char buf[128]; - obj = getDecodedObject(obj); + int decrrc = 0; + + if (obj->storage == REDIS_VM_MEMORY && obj->encoding != REDIS_ENCODING_RAW){ + obj = getDecodedObject(obj); + decrrc = 1; + } snprintf(buf,sizeof(buf),"$%ld\r\n",(long)sdslen(obj->ptr)); if (fwrite(buf,strlen(buf),1,fp) == 0) goto err; - if (fwrite(obj->ptr,sdslen(obj->ptr),1,fp) == 0) goto err; + if (sdslen(obj->ptr) && fwrite(obj->ptr,sdslen(obj->ptr),1,fp) == 0) + goto err; if (fwrite("\r\n",2,1,fp) == 0) goto err; - decrRefCount(obj); + if (decrrc) decrRefCount(obj); return 1; err: - decrRefCount(obj); + if (decrrc) decrRefCount(obj); return 0; } @@ -5928,9 +6719,24 @@ static int rewriteAppendOnlyFile(char *filename) { /* Iterate this DB writing every entry */ while((de = dictNext(di)) != NULL) { - robj *key = dictGetEntryKey(de); - robj *o = dictGetEntryVal(de); - time_t expiretime = getExpire(db,key); + robj *key, *o; + time_t expiretime; + int swapped; + + key = dictGetEntryKey(de); + /* If the value for this key is swapped, load a preview in memory. + * We use a "swapped" flag to remember if we need to free the + * value object instead to just increment the ref count anyway + * in order to avoid copy-on-write of pages if we are forked() */ + if (!server.vm_enabled || key->storage == REDIS_VM_MEMORY || + key->storage == REDIS_VM_SWAPPING) { + o = dictGetEntryVal(de); + swapped = 0; + } else { + o = vmPreviewObject(key); + swapped = 1; + } + expiretime = getExpire(db,key); /* Save the key and associated value */ if (o->type == REDIS_STRING) { @@ -5991,13 +6797,14 @@ static int rewriteAppendOnlyFile(char *filename) { } /* Save the expire time */ if (expiretime != -1) { - char cmd[]="*3\r\n$6\r\nEXPIRE\r\n"; + char cmd[]="*3\r\n$8\r\nEXPIREAT\r\n"; /* If this key is already expired skip it */ if (expiretime < now) continue; if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr; if (fwriteBulk(fp,key) == 0) goto werr; if (fwriteBulkLong(fp,expiretime) == 0) goto werr; } + if (swapped) decrRefCount(o); } dictReleaseIterator(di); } @@ -6020,7 +6827,7 @@ static int rewriteAppendOnlyFile(char *filename) { werr: fclose(fp); unlink(tmpfile); - redisLog(REDIS_WARNING,"Write error writing append only fileon disk: %s", strerror(errno)); + redisLog(REDIS_WARNING,"Write error writing append only file on disk: %s", strerror(errno)); if (di) dictReleaseIterator(di); return REDIS_ERR; } @@ -6079,7 +6886,8 @@ static void bgrewriteaofCommand(redisClient *c) { return; } if (rewriteAppendOnlyFileBackground() == REDIS_OK) { - addReply(c,shared.ok); + char *status = "+Background append only file rewriting started\r\n"; + addReplySds(c,sdsnew(status)); } else { addReply(c,shared.err); } @@ -6092,6 +6900,705 @@ static void aofRemoveTempFile(pid_t childpid) { unlink(tmpfile); } +/* Virtual Memory is composed mainly of two subsystems: + * - Blocking Virutal Memory + * - Threaded Virtual Memory I/O + * The two parts are not fully decoupled, but functions are split among two + * different sections of the source code (delimited by comments) in order to + * make more clear what functionality is about the blocking VM and what about + * the threaded (not blocking) VM. + * + * Redis VM design: + * + * Redis VM is a blocking VM (one that blocks reading swapped values from + * disk into memory when a value swapped out is needed in memory) that is made + * unblocking by trying to examine the command argument vector in order to + * load in background values that will likely be needed in order to exec + * the command. The command is executed only once all the relevant keys + * are loaded into memory. + * + * This basically is almost as simple of a blocking VM, but almost as parallel + * as a fully non-blocking VM. + */ + +/* =================== Virtual Memory - Blocking Side ====================== */ +static void vmInit(void) { + off_t totsize; + int pipefds[2]; + + server.vm_fp = fopen("/tmp/redisvm","w+b"); + if (server.vm_fp == NULL) { + redisLog(REDIS_WARNING,"Impossible to open the swap file. Exiting."); + exit(1); + } + server.vm_fd = fileno(server.vm_fp); + server.vm_next_page = 0; + server.vm_near_pages = 0; + server.vm_stats_used_pages = 0; + server.vm_stats_swapped_objects = 0; + server.vm_stats_swapouts = 0; + server.vm_stats_swapins = 0; + totsize = server.vm_pages*server.vm_page_size; + redisLog(REDIS_NOTICE,"Allocating %lld bytes of swap file",totsize); + if (ftruncate(server.vm_fd,totsize) == -1) { + redisLog(REDIS_WARNING,"Can't ftruncate swap file: %s. Exiting.", + strerror(errno)); + exit(1); + } else { + redisLog(REDIS_NOTICE,"Swap file allocated with success"); + } + server.vm_bitmap = zmalloc((server.vm_pages+7)/8); + redisLog(REDIS_VERBOSE,"Allocated %lld bytes page table for %lld pages", + (long long) (server.vm_pages+7)/8, server.vm_pages); + memset(server.vm_bitmap,0,(server.vm_pages+7)/8); + /* Try to remove the swap file, so the OS will really delete it from the + * file system when Redis exists. */ + unlink("/tmp/redisvm"); + + /* Initialize threaded I/O (used by Virtual Memory) */ + server.io_newjobs = listCreate(); + server.io_processing = listCreate(); + server.io_processed = listCreate(); + server.io_clients = listCreate(); + pthread_mutex_init(&server.io_mutex,NULL); + server.io_active_threads = 0; + if (pipe(pipefds) == -1) { + redisLog(REDIS_WARNING,"Unable to intialized VM: pipe(2): %s. Exiting." + ,strerror(errno)); + exit(1); + } + server.io_ready_pipe_read = pipefds[0]; + server.io_ready_pipe_write = pipefds[1]; + redisAssert(anetNonBlock(NULL,server.io_ready_pipe_read) != ANET_ERR); + /* Listen for events in the threaded I/O pipe */ + if (aeCreateFileEvent(server.el, server.io_ready_pipe_read, AE_READABLE, + vmThreadedIOCompletedJob, NULL) == AE_ERR) + oom("creating file event"); +} + +/* Mark the page as used */ +static void vmMarkPageUsed(off_t page) { + off_t byte = page/8; + int bit = page&7; + server.vm_bitmap[byte] |= 1<= server.vm_pages) { + this -= server.vm_pages; + if (this == 0) { + /* Just overflowed, what we found on tail is no longer + * interesting, as it's no longer contiguous. */ + numfree = 0; + } + } + if (vmFreePage(this)) { + /* This is a free page */ + numfree++; + /* Already got N free pages? Return to the caller, with success */ + if (numfree == n) { + *first = this-(n-1); + server.vm_next_page = this+1; + return REDIS_OK; + } + } else { + /* The current one is not a free page */ + numfree = 0; + } + + /* Fast-forward if the current page is not free and we already + * searched enough near this place. */ + since_jump++; + if (!numfree && since_jump >= REDIS_VM_MAX_RANDOM_JUMP/4) { + offset += random() % REDIS_VM_MAX_RANDOM_JUMP; + since_jump = 0; + /* Note that even if we rewind after the jump, we are don't need + * to make sure numfree is set to zero as we only jump *if* it + * is set to zero. */ + } else { + /* Otherwise just check the next page */ + offset++; + } + } + return REDIS_ERR; +} + +/* Swap the 'val' object relative to 'key' into disk. Store all the information + * needed to later retrieve the object into the key object. + * If we can't find enough contiguous empty pages to swap the object on disk + * REDIS_ERR is returned. */ +static int vmSwapObjectBlocking(robj *key, robj *val) { + off_t pages = rdbSavedObjectPages(val,NULL); + off_t page; + + assert(key->storage == REDIS_VM_MEMORY); + assert(key->refcount == 1); + if (vmFindContiguousPages(&page,pages) == REDIS_ERR) return REDIS_ERR; + if (fseeko(server.vm_fp,page*server.vm_page_size,SEEK_SET) == -1) { + redisLog(REDIS_WARNING, + "Critical VM problem in vmSwapObjectBlocking(): can't seek: %s", + strerror(errno)); + return REDIS_ERR; + } + rdbSaveObject(server.vm_fp,val); + key->vm.page = page; + key->vm.usedpages = pages; + key->storage = REDIS_VM_SWAPPED; + key->vtype = val->type; + decrRefCount(val); /* Deallocate the object from memory. */ + vmMarkPagesUsed(page,pages); + redisLog(REDIS_DEBUG,"VM: object %s swapped out at %lld (%lld pages)", + (unsigned char*) key->ptr, + (unsigned long long) page, (unsigned long long) pages); + server.vm_stats_swapped_objects++; + server.vm_stats_swapouts++; + fflush(server.vm_fp); + return REDIS_OK; +} + +/* Load the value object relative to the 'key' object from swap to memory. + * The newly allocated object is returned. + * + * If preview is true the unserialized object is returned to the caller but + * no changes are made to the key object, nor the pages are marked as freed */ +static robj *vmGenericLoadObject(robj *key, int preview) { + robj *val; + + redisAssert(key->storage == REDIS_VM_SWAPPED); + if (fseeko(server.vm_fp,key->vm.page*server.vm_page_size,SEEK_SET) == -1) { + redisLog(REDIS_WARNING, + "Unrecoverable VM problem in vmLoadObject(): can't seek: %s", + strerror(errno)); + exit(1); + } + val = rdbLoadObject(key->vtype,server.vm_fp); + if (val == NULL) { + redisLog(REDIS_WARNING, "Unrecoverable VM problem in vmLoadObject(): can't load object from swap file: %s", strerror(errno)); + exit(1); + } + if (!preview) { + key->storage = REDIS_VM_MEMORY; + key->vm.atime = server.unixtime; + vmMarkPagesFree(key->vm.page,key->vm.usedpages); + redisLog(REDIS_DEBUG, "VM: object %s loaded from disk", + (unsigned char*) key->ptr); + server.vm_stats_swapped_objects--; + } else { + redisLog(REDIS_DEBUG, "VM: object %s previewed from disk", + (unsigned char*) key->ptr); + } + server.vm_stats_swapins++; + return val; +} + +/* Plain object loading, from swap to memory */ +static robj *vmLoadObject(robj *key) { + /* If we are loading the object in background, stop it, we + * need to load this object synchronously ASAP. */ + if (key->storage == REDIS_VM_LOADING) + vmCancelThreadedIOJob(key); + return vmGenericLoadObject(key,0); +} + +/* Just load the value on disk, without to modify the key. + * This is useful when we want to perform some operation on the value + * without to really bring it from swap to memory, like while saving the + * dataset or rewriting the append only log. */ +static robj *vmPreviewObject(robj *key) { + return vmGenericLoadObject(key,1); +} + +/* How a good candidate is this object for swapping? + * The better candidate it is, the greater the returned value. + * + * Currently we try to perform a fast estimation of the object size in + * memory, and combine it with aging informations. + * + * Basically swappability = idle-time * log(estimated size) + * + * Bigger objects are preferred over smaller objects, but not + * proportionally, this is why we use the logarithm. This algorithm is + * just a first try and will probably be tuned later. */ +static double computeObjectSwappability(robj *o) { + time_t age = server.unixtime - o->vm.atime; + long asize = 0; + list *l; + dict *d; + struct dictEntry *de; + int z; + + if (age <= 0) return 0; + switch(o->type) { + case REDIS_STRING: + if (o->encoding != REDIS_ENCODING_RAW) { + asize = sizeof(*o); + } else { + asize = sdslen(o->ptr)+sizeof(*o)+sizeof(long)*2; + } + break; + case REDIS_LIST: + l = o->ptr; + listNode *ln = listFirst(l); + + asize = sizeof(list); + if (ln) { + robj *ele = ln->value; + long elesize; + + elesize = (ele->encoding == REDIS_ENCODING_RAW) ? + (sizeof(*o)+sdslen(ele->ptr)) : + sizeof(*o); + asize += (sizeof(listNode)+elesize)*listLength(l); + } + break; + case REDIS_SET: + case REDIS_ZSET: + z = (o->type == REDIS_ZSET); + d = z ? ((zset*)o->ptr)->dict : o->ptr; + + asize = sizeof(dict)+(sizeof(struct dictEntry*)*dictSlots(d)); + if (z) asize += sizeof(zset)-sizeof(dict); + if (dictSize(d)) { + long elesize; + robj *ele; + + de = dictGetRandomKey(d); + ele = dictGetEntryKey(de); + elesize = (ele->encoding == REDIS_ENCODING_RAW) ? + (sizeof(*o)+sdslen(ele->ptr)) : + sizeof(*o); + asize += (sizeof(struct dictEntry)+elesize)*dictSize(d); + if (z) asize += sizeof(zskiplistNode)*dictSize(d); + } + break; + } + return (double)asize*log(1+asize); +} + +/* Try to swap an object that's a good candidate for swapping. + * Returns REDIS_OK if the object was swapped, REDIS_ERR if it's not possible + * to swap any object at all. + * + * If 'usethreaded' is true, Redis will try to swap the object in background + * using I/O threads. */ +static int vmSwapOneObject(int usethreads) { + int j, i; + struct dictEntry *best = NULL; + double best_swappability = 0; + redisDb *best_db = NULL; + robj *key, *val; + + for (j = 0; j < server.dbnum; j++) { + redisDb *db = server.db+j; + int maxtries = 1000; + + if (dictSize(db->dict) == 0) continue; + for (i = 0; i < 5; i++) { + dictEntry *de; + double swappability; + + if (maxtries) maxtries--; + de = dictGetRandomKey(db->dict); + key = dictGetEntryKey(de); + val = dictGetEntryVal(de); + if (key->storage != REDIS_VM_MEMORY) { + if (maxtries) i--; /* don't count this try */ + continue; + } + swappability = computeObjectSwappability(val); + if (!best || swappability > best_swappability) { + best = de; + best_swappability = swappability; + best_db = db; + } + } + } + if (best == NULL) { + redisLog(REDIS_DEBUG,"No swappable key found!"); + return REDIS_ERR; + } + key = dictGetEntryKey(best); + val = dictGetEntryVal(best); + + redisLog(REDIS_DEBUG,"Key with best swappability: %s, %f", + key->ptr, best_swappability); + + /* Unshare the key if needed */ + if (key->refcount > 1) { + robj *newkey = dupStringObject(key); + decrRefCount(key); + key = dictGetEntryKey(best) = newkey; + } + /* Swap it */ + if (usethreads) { + vmSwapObjectThreaded(key,val,best_db); + return REDIS_OK; + } else { + if (vmSwapObjectBlocking(key,val) == REDIS_OK) { + dictGetEntryVal(best) = NULL; + return REDIS_OK; + } else { + return REDIS_ERR; + } + } +} + +static int vmSwapOneObjectBlocking() { + return vmSwapOneObject(0); +} + +static int vmSwapOneObjectThreaded() { + return vmSwapOneObject(1); +} + +/* Return true if it's safe to swap out objects in a given moment. + * Basically we don't want to swap objects out while there is a BGSAVE + * or a BGAEOREWRITE running in backgroud. */ +static int vmCanSwapOut(void) { + return (server.bgsavechildpid == -1 && server.bgrewritechildpid == -1); +} + +/* Delete a key if swapped. Returns 1 if the key was found, was swapped + * and was deleted. Otherwise 0 is returned. */ +static int deleteIfSwapped(redisDb *db, robj *key) { + dictEntry *de; + robj *foundkey; + + if ((de = dictFind(db->dict,key)) == NULL) return 0; + foundkey = dictGetEntryKey(de); + if (foundkey->storage == REDIS_VM_MEMORY) return 0; + deleteKey(db,key); + return 1; +} + +/* =================== Virtual Memory - Threaded I/O ======================= */ + +static void freeIOJob(iojob *j) { + if (j->type == REDIS_IOJOB_PREPARE_SWAP || + j->type == REDIS_IOJOB_DO_SWAP) + decrRefCount(j->val); + decrRefCount(j->key); + zfree(j); +} + +/* Every time a thread finished a Job, it writes a byte into the write side + * of an unix pipe in order to "awake" the main thread, and this function + * is called. */ +static void vmThreadedIOCompletedJob(aeEventLoop *el, int fd, void *privdata, + int mask) +{ + char buf[1]; + int retval; + REDIS_NOTUSED(el); + REDIS_NOTUSED(mask); + REDIS_NOTUSED(privdata); + + /* For every byte we read in the read side of the pipe, there is one + * I/O job completed to process. */ + while((retval = read(fd,buf,1)) == 1) { + iojob *j; + listNode *ln; + robj *key; + struct dictEntry *de; + + redisLog(REDIS_DEBUG,"Processing I/O completed job"); + assert(listLength(server.io_processed) != 0); + + /* Get the processed element (the oldest one) */ + lockThreadedIO(); + ln = listFirst(server.io_processed); + j = ln->value; + listDelNode(server.io_processed,ln); + unlockThreadedIO(); + /* If this job is marked as canceled, just ignore it */ + if (j->canceled) { + freeIOJob(j); + continue; + } + /* Post process it in the main thread, as there are things we + * can do just here to avoid race conditions and/or invasive locks */ + redisLog(REDIS_DEBUG,"Job type: %d, key at %p (%s) refcount: %d\n", j->type, (void*)j->key, (char*)j->key->ptr, j->key->refcount); + if (j->key->refcount <= 0) { + printf("Ooops ref count is <= 0!\n"); + exit(1); + } + de = dictFind(j->db->dict,j->key); + assert(de != NULL); + key = dictGetEntryKey(de); + if (j->type == REDIS_IOJOB_LOAD) { + /* Key loaded, bring it at home */ + key->storage = REDIS_VM_MEMORY; + key->vm.atime = server.unixtime; + vmMarkPagesFree(key->vm.page,key->vm.usedpages); + redisLog(REDIS_DEBUG, "VM: object %s loaded from disk (threaded)", + (unsigned char*) key->ptr); + server.vm_stats_swapped_objects--; + server.vm_stats_swapins++; + freeIOJob(j); + } else if (j->type == REDIS_IOJOB_PREPARE_SWAP) { + /* Now we know the amount of pages required to swap this object. + * Let's find some space for it, and queue this task again + * rebranded as REDIS_IOJOB_DO_SWAP. */ + if (vmFindContiguousPages(&j->page,j->pages) == REDIS_ERR) { + /* Ooops... no space! */ + freeIOJob(j); + } else { + j->type = REDIS_IOJOB_DO_SWAP; + lockThreadedIO(); + queueIOJob(j); + unlockThreadedIO(); + } + } else if (j->type == REDIS_IOJOB_DO_SWAP) { + robj *val; + + /* Key swapped. We can finally free some memory. */ + val = dictGetEntryVal(de); + key->vm.page = j->page; + key->vm.usedpages = j->pages; + key->storage = REDIS_VM_SWAPPED; + key->vtype = j->val->type; + decrRefCount(val); /* Deallocate the object from memory. */ + dictGetEntryVal(de) = NULL; + vmMarkPagesUsed(j->page,j->pages); + redisLog(REDIS_DEBUG, + "VM: object %s swapped out at %lld (%lld pages) (threaded)", + (unsigned char*) key->ptr, + (unsigned long long) j->page, (unsigned long long) j->pages); + server.vm_stats_swapped_objects++; + server.vm_stats_swapouts++; + freeIOJob(j); + /* Put a few more swap requests in queue if we are still + * out of memory */ + if (zmalloc_used_memory() > server.vm_max_memory) { + int more = 1; + while(more) { + lockThreadedIO(); + more = listLength(server.io_newjobs) < + (unsigned) server.vm_max_threads; + unlockThreadedIO(); + /* Don't waste CPU time if swappable objects are rare. */ + if (vmSwapOneObjectThreaded() == REDIS_ERR) break; + } + } + } + } + if (retval < 0 && errno != EAGAIN) { + redisLog(REDIS_WARNING, + "WARNING: read(2) error in vmThreadedIOCompletedJob() %s", + strerror(errno)); + } +} + +static void lockThreadedIO(void) { + pthread_mutex_lock(&server.io_mutex); +} + +static void unlockThreadedIO(void) { + pthread_mutex_unlock(&server.io_mutex); +} + +/* Remove the specified object from the threaded I/O queue if still not + * processed, otherwise make sure to flag it as canceled. */ +static void vmCancelThreadedIOJob(robj *o) { + list *lists[3] = { + server.io_newjobs, server.io_processing, server.io_processed + }; + int i; + + assert(o->storage == REDIS_VM_LOADING || o->storage == REDIS_VM_SWAPPING); + lockThreadedIO(); + /* Search for a matching key in one of the queues */ + for (i = 0; i < 3; i++) { + listNode *ln; + + listRewind(lists[i]); + while ((ln = listYield(lists[i])) != NULL) { + iojob *job = ln->value; + + if (compareStringObjects(job->key,o) == 0) { + switch(i) { + case 0: /* io_newjobs */ + /* If the job was not yet processed the best thing to do + * is to remove it from the queue at all */ + decrRefCount(job->key); + if (job->type == REDIS_IOJOB_PREPARE_SWAP || + job->type == REDIS_IOJOB_DO_SWAP) + decrRefCount(job->val); + listDelNode(lists[i],ln); + zfree(job); + break; + case 1: /* io_processing */ + case 2: /* io_processed */ + job->canceled = 1; + break; + } + if (o->storage == REDIS_VM_LOADING) + o->storage = REDIS_VM_SWAPPED; + else if (o->storage == REDIS_VM_SWAPPING) + o->storage = REDIS_VM_MEMORY; + unlockThreadedIO(); + return; + } + } + } + unlockThreadedIO(); + assert(1 != 1); /* We should never reach this */ +} + +static void *IOThreadEntryPoint(void *arg) { + iojob *j; + listNode *ln; + REDIS_NOTUSED(arg); + + pthread_detach(pthread_self()); + while(1) { + /* Get a new job to process */ + lockThreadedIO(); + if (listLength(server.io_newjobs) == 0) { + /* No new jobs in queue, exit. */ + printf("Thread %lld exiting, nothing to do\n", + (long long) pthread_self()); + server.io_active_threads--; + unlockThreadedIO(); + return NULL; + } + ln = listFirst(server.io_newjobs); + j = ln->value; + listDelNode(server.io_newjobs,ln); + /* Add the job in the processing queue */ + j->thread = pthread_self(); + listAddNodeTail(server.io_processing,j); + ln = listLast(server.io_processing); /* We use ln later to remove it */ + unlockThreadedIO(); + printf("Thread %lld got a new job: %p about key '%s'\n", + (long long) pthread_self(), (void*)j, (char*)j->key->ptr); + + /* Process the Job */ + if (j->type == REDIS_IOJOB_LOAD) { + } else if (j->type == REDIS_IOJOB_PREPARE_SWAP) { + FILE *fp = fopen("/dev/null","w+"); + j->pages = rdbSavedObjectPages(j->val,fp); + fclose(fp); + } else if (j->type == REDIS_IOJOB_DO_SWAP) { + } + + /* Done: insert the job into the processed queue */ + printf("Thread %lld completed the job: %p\n", + (long long) pthread_self(), (void*)j); + lockThreadedIO(); + listDelNode(server.io_processing,ln); + listAddNodeTail(server.io_processed,j); + unlockThreadedIO(); + + /* Signal the main thread there is new stuff to process */ + assert(write(server.io_ready_pipe_write,"x",1) == 1); + } + return NULL; /* never reached */ +} + +static void spawnIOThread(void) { + pthread_t thread; + + pthread_create(&thread,NULL,IOThreadEntryPoint,NULL); + server.io_active_threads++; +} + +/* This function must be called while with threaded IO locked */ +static void queueIOJob(iojob *j) { + listAddNodeTail(server.io_newjobs,j); + if (server.io_active_threads < server.vm_max_threads) + spawnIOThread(); +} + +static int vmSwapObjectThreaded(robj *key, robj *val, redisDb *db) { + iojob *j; + + assert(key->storage == REDIS_VM_MEMORY); + assert(key->refcount == 1); + + j = zmalloc(sizeof(*j)); + j->type = REDIS_IOJOB_PREPARE_SWAP; + j->db = db; + j->key = dupStringObject(key); + j->val = val; + incrRefCount(val); + j->canceled = 0; + j->thread = (pthread_t) -1; + key->storage = REDIS_VM_SWAPPING; + + lockThreadedIO(); + queueIOJob(j); + unlockThreadedIO(); + return REDIS_OK; +} + /* ================================= Debugging ============================== */ static void debugCommand(redisClient *c) { @@ -6109,6 +7616,14 @@ static void debugCommand(redisClient *c) { } redisLog(REDIS_WARNING,"DB reloaded by DEBUG RELOAD"); addReply(c,shared.ok); + } else if (!strcasecmp(c->argv[1]->ptr,"loadaof")) { + emptyDb(); + if (loadAppendOnlyFile(server.appendfilename) != REDIS_OK) { + addReply(c,shared.err); + return; + } + redisLog(REDIS_WARNING,"Append Only File loaded by DEBUG LOADAOF"); + addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"object") && c->argc == 3) { dictEntry *de = dictFind(c->db->dict,c->argv[2]); robj *key, *val; @@ -6119,13 +7634,52 @@ static void debugCommand(redisClient *c) { } key = dictGetEntryKey(de); val = dictGetEntryVal(de); - addReplySds(c,sdscatprintf(sdsempty(), - "+Key at:%p refcount:%d, value at:%p refcount:%d encoding:%d\r\n", + if (server.vm_enabled && (key->storage == REDIS_VM_MEMORY || + key->storage == REDIS_VM_SWAPPING)) { + addReplySds(c,sdscatprintf(sdsempty(), + "+Key at:%p refcount:%d, value at:%p refcount:%d " + "encoding:%d serializedlength:%lld\r\n", (void*)key, key->refcount, (void*)val, val->refcount, - val->encoding)); + val->encoding, rdbSavedObjectLen(val,NULL))); + } else { + addReplySds(c,sdscatprintf(sdsempty(), + "+Key at:%p refcount:%d, value swapped at: page %llu " + "using %llu pages\r\n", + (void*)key, key->refcount, (unsigned long long) key->vm.page, + (unsigned long long) key->vm.usedpages)); + } + } else if (!strcasecmp(c->argv[1]->ptr,"swapout") && c->argc == 3) { + dictEntry *de = dictFind(c->db->dict,c->argv[2]); + robj *key, *val; + + if (!server.vm_enabled) { + addReplySds(c,sdsnew("-ERR Virtual Memory is disabled\r\n")); + return; + } + if (!de) { + addReply(c,shared.nokeyerr); + return; + } + key = dictGetEntryKey(de); + val = dictGetEntryVal(de); + /* If the key is shared we want to create a copy */ + if (key->refcount > 1) { + robj *newkey = dupStringObject(key); + decrRefCount(key); + key = dictGetEntryKey(de) = newkey; + } + /* Swap it */ + if (key->storage != REDIS_VM_MEMORY) { + addReplySds(c,sdsnew("-ERR This key is not in memory\r\n")); + } else if (vmSwapObjectBlocking(key,val) == REDIS_OK) { + dictGetEntryVal(de) = NULL; + addReply(c,shared.ok); + } else { + addReply(c,shared.err); + } } else { addReplySds(c,sdsnew( - "-ERR Syntax error, try DEBUG [SEGFAULT|OBJECT |RELOAD]\r\n")); + "-ERR Syntax error, try DEBUG [SEGFAULT|OBJECT |SWAPOUT |RELOAD]\r\n")); } } @@ -6211,8 +7765,6 @@ int main(int argc, char **argv) { if (rdbLoad(server.dbfilename) == REDIS_OK) redisLog(REDIS_NOTICE,"DB loaded from disk"); } - if (aeCreateFileEvent(server.el, server.fd, AE_READABLE, - acceptHandler, NULL) == AE_ERR) oom("creating file event"); redisLog(REDIS_NOTICE,"The server is now ready to accept connections on port %d", server.port); aeMain(server.el); aeDeleteEventLoop(server.el); @@ -6241,8 +7793,8 @@ static void *getMcontextEip(ucontext_t *uc) { #else return (void*) uc->uc_mcontext->__ss.__eip; #endif -#elif defined(__i386__) || defined(__X86_64__) /* Linux x86 */ - return (void*) uc->uc_mcontext.gregs[REG_EIP]; +#elif defined(__i386__) || defined(__X86_64__) || defined(__x86_64__) + return (void*) uc->uc_mcontext.gregs[REG_EIP]; /* Linux 32/64 bit */ #elif defined(__ia64__) /* Linux IA64 */ return (void*) uc->uc_mcontext.sc_ip; #else @@ -6283,7 +7835,7 @@ static void segvHandler(int sig, siginfo_t *info, void *secret) { redisLog(REDIS_WARNING,"%d redis-server %p %s + %d", i, trace[i], fn, (unsigned int)offset); } } - // free(messages); Don't call free() with possibly corrupted memory. + /* free(messages); Don't call free() with possibly corrupted memory. */ exit(0); }