X-Git-Url: https://git.saurik.com/redis.git/blobdiff_plain/221782ccc69b4c56608942f3fe9e47773a32866e..f9c6f39b2b0039cf29df6597d41c124048c825cd:/src/redis.h diff --git a/src/redis.h b/src/redis.h index 3fd13e2e..e9a31b8c 100644 --- a/src/redis.h +++ b/src/redis.h @@ -19,18 +19,19 @@ #include #include #include +#include -#include "ae.h" /* Event driven programming library */ -#include "sds.h" /* Dynamic safe strings */ -#include "dict.h" /* Hash tables */ -#include "adlist.h" /* Linked lists */ +#include "ae.h" /* Event driven programming library */ +#include "sds.h" /* Dynamic safe strings */ +#include "dict.h" /* Hash tables */ +#include "adlist.h" /* Linked lists */ #include "zmalloc.h" /* total memory usage aware version of malloc/free */ -#include "anet.h" /* Networking the easy way */ -#include "zipmap.h" /* Compact string -> string data structure */ +#include "anet.h" /* Networking the easy way */ +#include "zipmap.h" /* Compact string -> string data structure */ #include "ziplist.h" /* Compact list data structure */ -#include "intset.h" /* Compact integer set structure */ -#include "version.h" -#include "util.h" +#include "intset.h" /* Compact integer set structure */ +#include "version.h" /* Version macro */ +#include "util.h" /* Misc functions useful in many places */ /* Error codes */ #define REDIS_OK 0 @@ -41,7 +42,6 @@ #define REDIS_MAXIDLETIME (60*5) /* default client timeout */ #define REDIS_IOBUF_LEN 1024 #define REDIS_LOADBUF_LEN 1024 -#define REDIS_STATIC_ARGS 8 #define REDIS_DEFAULT_DBNUM 16 #define REDIS_CONFIGLINE_MAX 1024 #define REDIS_MAX_SYNC_TIME 60 /* Slave can't take more to sync */ @@ -51,6 +51,10 @@ #define REDIS_SHARED_INTEGERS 10000 #define REDIS_REPLY_CHUNK_BYTES (5*1500) /* 5 TCP packets with default MTU */ #define REDIS_MAX_LOGMSG_LEN 1024 /* Default maximum length of syslog messages */ +#define REDIS_AUTO_AOFREWRITE_PERC 100 +#define REDIS_AUTO_AOFREWRITE_MIN_SIZE (1024*1024) +#define REDIS_SLOWLOG_LOG_SLOWER_THAN 10000 +#define REDIS_SLOWLOG_MAX_LEN 64 /* Hash table parameters */ #define REDIS_HT_MINFILL 10 /* Minimal hash table fill 10% */ @@ -85,18 +89,37 @@ #define REDIS_ENCODING_INTSET 6 /* Encoded as intset */ #define REDIS_ENCODING_SKIPLIST 7 /* Encoded as skiplist */ -/* Scheduled IO opeations flags. */ -#define REDIS_IO_LOAD 1 -#define REDIS_IO_SAVE 2 -#define REDIS_IO_LOADINPROG 4 -#define REDIS_IO_SAVEINPROG 8 +/* Object types only used for dumping to disk */ +#define REDIS_EXPIRETIME 253 +#define REDIS_SELECTDB 254 +#define REDIS_EOF 255 -/* Generic IO flags */ -#define REDIS_IO_ONLYLOADS 1 -#define REDIS_IO_ASAP 2 - -#define REDIS_MAX_COMPLETED_JOBS_PROCESSED 1 -#define REDIS_THREAD_STACK_SIZE (1024*1024*4) +/* Defines related to the dump file format. To store 32 bits lengths for short + * keys requires a lot of space, so we check the most significant 2 bits of + * the first byte to interpreter the length: + * + * 00|000000 => if the two MSB are 00 the len is the 6 bits of this byte + * 01|000000 00000000 => 01, the len is 14 byes, 6 bits + 8 bits of next byte + * 10|000000 [32 bit integer] => if it's 01, a full 32 bit len will follow + * 11|000000 this means: specially encoded object will follow. The six bits + * number specify the kind of object that follows. + * See the REDIS_RDB_ENC_* defines. + * + * Lenghts up to 63 are stored using a single byte, most DB keys, and may + * values, will fit inside. */ +#define REDIS_RDB_6BITLEN 0 +#define REDIS_RDB_14BITLEN 1 +#define REDIS_RDB_32BITLEN 2 +#define REDIS_RDB_ENCVAL 3 +#define REDIS_RDB_LENERR UINT_MAX + +/* When a length of a string object stored on disk has the first two bits + * set, the remaining two bits specify a special encoding for the object + * accordingly to the following defines: */ +#define REDIS_RDB_ENC_INT8 0 /* 8 bit signed integer */ +#define REDIS_RDB_ENC_INT16 1 /* 16 bit signed integer */ +#define REDIS_RDB_ENC_INT32 2 /* 32 bit signed integer */ +#define REDIS_RDB_ENC_LZF 3 /* string compressed with FASTLZ */ /* Client flags */ #define REDIS_SLAVE 1 /* This client is a slave server */ @@ -104,21 +127,25 @@ #define REDIS_MONITOR 4 /* This client is a slave monitor, see MONITOR */ #define REDIS_MULTI 8 /* This client is in a MULTI context */ #define REDIS_BLOCKED 16 /* The client is waiting in a blocking operation */ -#define REDIS_IO_WAIT 32 /* The client is waiting for Virtual Memory I/O */ #define REDIS_DIRTY_CAS 64 /* Watched keys modified. EXEC will fail. */ #define REDIS_CLOSE_AFTER_REPLY 128 /* Close after writing entire reply. */ #define REDIS_UNBLOCKED 256 /* This client was unblocked and is stored in server.unblocked_clients */ +#define REDIS_LUA_CLIENT 512 /* This is a non connected client used by Lua */ /* Client request types */ #define REDIS_REQ_INLINE 1 #define REDIS_REQ_MULTIBULK 2 /* Slave replication state - slave side */ -#define REDIS_REPL_NONE 0 /* No active replication */ -#define REDIS_REPL_CONNECT 1 /* Must connect to master */ -#define REDIS_REPL_TRANSFER 2 /* Receiving .rdb from master */ -#define REDIS_REPL_CONNECTED 3 /* Connected to master */ +#define REDIS_REPL_NONE 0 /* No active replication */ +#define REDIS_REPL_CONNECT 1 /* Must connect to master */ +#define REDIS_REPL_CONNECTING 2 /* Connecting to master */ +#define REDIS_REPL_TRANSFER 3 /* Receiving .rdb from master */ +#define REDIS_REPL_CONNECTED 4 /* Connected to master */ + +/* Synchronous read timeout - slave side */ +#define REDIS_REPL_SYNCIO_TIMEOUT 5 /* Slave replication state - from the point of view of master * Note that in SEND_BULK and ONLINE state the slave receives new updates @@ -179,11 +206,8 @@ #define REDIS_MAXMEMORY_ALLKEYS_RANDOM 4 #define REDIS_MAXMEMORY_NO_EVICTION 5 -/* Diskstore background saving thread states */ -#define REDIS_BGSAVE_THREAD_UNACTIVE 0 -#define REDIS_BGSAVE_THREAD_ACTIVE 1 -#define REDIS_BGSAVE_THREAD_DONE_OK 2 -#define REDIS_BGSAVE_THREAD_DONE_ERR 3 +/* Scripting */ +#define REDIS_LUA_TIME_LIMIT 60000 /* milliseconds */ /* We can print the stacktrace, so our assert is defined this way: */ #define redisAssert(_e) ((_e)?(void)0 : (_redisAssert(#_e,__FILE__,__LINE__),_exit(1))) @@ -249,9 +273,6 @@ typedef struct redisDb { dict *dict; /* The keyspace for this DB */ dict *expires; /* Timeout of keys with a timeout set */ dict *blocking_keys; /* Keys with clients waiting for data (BLPOP) */ - dict *io_keys; /* Keys with clients waiting for DS I/O */ - dict *io_negcache; /* Negative caching for disk store */ - dict *io_queued; /* Queued IO operations hash table */ dict *watched_keys; /* WATCHED keys for MULTI/EXEC CAS */ int id; } redisDb; @@ -287,6 +308,7 @@ typedef struct redisClient { sds querybuf; int argc; robj **argv; + struct redisCommand *cmd; int reqtype; int multibulklen; /* number of multi bulk arguments left to read */ long bulklen; /* length of bulk argument in multi bulk request */ @@ -322,7 +344,7 @@ struct sharedObjectsStruct { robj *crlf, *ok, *err, *emptybulk, *czero, *cone, *cnegone, *pong, *space, *colon, *nullbulk, *nullmultibulk, *queued, *emptymultibulk, *wrongtypeerr, *nokeyerr, *syntaxerr, *sameobjecterr, - *outofrangeerr, *loadingerr, *plus, + *outofrangeerr, *noscripterr, *loadingerr, *plus, *select0, *select1, *select2, *select3, *select4, *select5, *select6, *select7, *select8, *select9, *messagebulk, *pmessagebulk, *subscribebulk, *unsubscribebulk, *mbulk3, @@ -473,7 +495,6 @@ typedef struct { struct redisServer { /* General */ - pthread_t mainthread; redisDb *db; dict *commands; /* Command table hahs table */ aeEventLoop *el; @@ -505,6 +526,11 @@ struct redisServer { long long stat_keyspace_hits; /* number of successful lookups of keys */ long long stat_keyspace_misses; /* number of failed lookups of keys */ size_t stat_peak_memory; /* max used memory record */ + long long stat_fork_time; /* time needed to perform latets fork() */ + list *slowlog; + long long slowlog_entry_id; + long long slowlog_log_slower_than; + unsigned long slowlog_max_len; /* Configuration */ int verbosity; int maxidletime; @@ -513,6 +539,11 @@ struct redisServer { int appendonly; int appendfsync; int no_appendfsync_on_rewrite; + int auto_aofrewrite_perc; /* Rewrite AOF if % growth is > M and... */ + off_t auto_aofrewrite_min_size; /* the AOF file is at least N bytes. */ + off_t auto_aofrewrite_base_size;/* AOF size on latest startup or rewrite. */ + off_t appendonly_current_size; /* AOF current size. */ + int aofrewrite_scheduled; /* Rewrite once BGSAVE terminates. */ int shutdown_asap; int activerehashing; char *requirepass; @@ -522,12 +553,10 @@ struct redisServer { time_t lastfsync; int appendfd; int appendseldb; + time_t aof_flush_postponed_start; char *pidfile; pid_t bgsavechildpid; pid_t bgrewritechildpid; - int bgsavethread_state; - pthread_mutex_t bgsavethread_mutex; - pthread_t bgsavethread; sds bgrewritebuf; /* buffer taken by parent during oppend only rewrite */ sds aofbuf; /* AOF buffer, written before entering the event loop */ struct saveparam *saveparams; @@ -547,6 +576,7 @@ struct redisServer { char *masterhost; int masterport; redisClient *master; /* client that is master for this slave */ + int repl_syncio_timeout; /* timeout for synchronous I/O calls */ int replstate; /* replication status if the instance is a slave */ off_t repl_transfer_left; /* bytes left reading .rdb */ int repl_transfer_s; /* slave -> master SYNC socket */ @@ -554,6 +584,7 @@ struct redisServer { char *repl_transfer_tmpfile; /* slave-> master SYNC temp file name */ time_t repl_transfer_lastio; /* unix time of the latest read, for timeout */ int repl_serve_stale_data; /* Serve stale data when link is down? */ + time_t repl_down_since; /* unix time at which link with master went down */ /* Limits */ unsigned int maxclients; unsigned long long maxmemory; @@ -561,19 +592,12 @@ struct redisServer { int maxmemory_samples; /* Blocked clients */ unsigned int bpop_blocked_clients; - unsigned int cache_blocked_clients; list *unblocked_clients; /* list of clients to unblock before next loop */ - list *cache_io_queue; /* IO operations queue */ - int cache_flush_delay; /* seconds to wait before flushing keys */ /* Sort parameters - qsort_r() is only available under BSD so we * have to take this state global, in order to pass it to sortCompare() */ int sort_desc; int sort_alpha; int sort_bypattern; - /* Virtual memory configuration */ - int ds_enabled; /* backend disk in redis.conf */ - char *ds_path; /* location of the disk store on disk */ - unsigned long long cache_max_memory; /* Zip structure config */ size_t hash_max_zipmap_entries; size_t hash_max_zipmap_value; @@ -583,30 +607,6 @@ struct redisServer { size_t zset_max_ziplist_entries; size_t zset_max_ziplist_value; time_t unixtime; /* Unix time sampled every second. */ - /* Virtual memory I/O threads stuff */ - /* An I/O thread process an element taken from the io_jobs queue and - * put the result of the operation in the io_done list. While the - * job is being processed, it's put on io_processing queue. */ - list *io_newjobs; /* List of VM I/O jobs yet to be processed */ - list *io_processing; /* List of VM I/O jobs being processed */ - list *io_processed; /* List of VM I/O jobs already processed */ - list *io_ready_clients; /* Clients ready to be unblocked. All keys loaded */ - pthread_mutex_t io_mutex; /* lock to access io_jobs/io_done/io_thread_job */ - pthread_cond_t io_condvar; /* I/O threads conditional variable */ - pthread_attr_t io_threads_attr; /* attributes for threads creation */ - int io_active_threads; /* Number of running I/O threads */ - int vm_max_threads; /* Max number of I/O threads running at the same time */ - /* Our main thread is blocked on the event loop, locking for sockets ready - * to be read or written, so when a threaded I/O operation is ready to be - * processed by the main thread, the I/O thread will use a unix pipe to - * awake the main thread. The followings are the two pipe FDs. */ - int io_ready_pipe_read; - int io_ready_pipe_write; - /* Virtual memory stats */ - unsigned long long vm_stats_used_pages; - unsigned long long vm_stats_swapped_objects; - unsigned long long vm_stats_swapouts; - unsigned long long vm_stats_swapins; /* Pubsub */ dict *pubsub_channels; /* Map channels to list of subscribed clients */ list *pubsub_patterns; /* A list of pubsub_patterns */ @@ -616,6 +616,12 @@ struct redisServer { /* Cluster */ int cluster_enabled; clusterState cluster; + /* Scripting */ + lua_State *lua; /* The Lua interpreter. We use just one for all clients */ + redisClient *lua_client; /* The "fake client" to query Redis from Lua */ + dict *lua_scripts; /* A dictionary of SHA1 -> Lua scripts */ + long long lua_time_limit; + long long lua_time_start; }; typedef struct pubsubPattern { @@ -631,7 +637,7 @@ struct redisCommand { int arity; int flags; /* Use a function to determine keys arguments in a command line. - * Used both for diskstore preloading and Redis Cluster. */ + * Used for Redis Cluster redirect. */ redisGetKeysProc *getkeys_proc; /* What keys should be loaded in background when calling this command? */ int firstkey; /* The first argument that's a key (0 = no keys) */ @@ -658,27 +664,6 @@ typedef struct _redisSortOperation { robj *pattern; } redisSortOperation; -/* DIsk store threaded I/O request message */ -#define REDIS_IOJOB_LOAD 0 -#define REDIS_IOJOB_SAVE 1 - -typedef struct iojob { - int type; /* Request type, REDIS_IOJOB_* */ - redisDb *db;/* Redis database */ - robj *key; /* This I/O request is about this key */ - robj *val; /* the value to swap for REDIS_IOJOB_SAVE, otherwise this - * field is populated by the I/O thread for REDIS_IOJOB_LOAD. */ - time_t expire; /* Expire time for this key on REDIS_IOJOB_LOAD */ -} iojob; - -/* IO operations scheduled -- check dscache.c for more info */ -typedef struct ioop { - int type; - redisDb *db; - robj *key; - time_t ctime; /* This is the creation time of the entry. */ -} ioop; - /* Structure to hold list iteration abstraction. */ typedef struct { robj *subject; @@ -729,6 +714,7 @@ extern struct sharedObjectsStruct shared; extern dictType setDictType; extern dictType zsetDictType; extern dictType clusterNodesDictType; +extern dictType dbDictType; extern double R_Zero, R_PosInf, R_NegInf, R_Nan; dictType hashDictType; @@ -768,6 +754,8 @@ void addReplyMultiBulkLen(redisClient *c, long length); void *dupClientReplyValue(void *o); void getClientsMaxBuffers(unsigned long *longest_output_list, unsigned long *biggest_input_buffer); +void rewriteClientCommandVector(redisClient *c, int argc, ...); +void rewriteClientCommandArgument(redisClient *c, int i, robj *newval); #ifdef __GNUC__ void addReplyErrorFormat(redisClient *c, const char *fmt, ...) @@ -800,13 +788,14 @@ void popGenericCommand(redisClient *c, int where); void unwatchAllKeys(redisClient *c); void initClientMultiState(redisClient *c); void freeClientMultiState(redisClient *c); -void queueMultiCommand(redisClient *c, struct redisCommand *cmd); +void queueMultiCommand(redisClient *c); void touchWatchedKey(redisDb *db, robj *key); void touchWatchedKeysOnFlush(int dbid); /* Redis object implementation */ void decrRefCount(void *o); void incrRefCount(robj *o); +robj *resetRefCount(robj *obj); void freeStringObject(robj *o); void freeListObject(robj *o); void freeSetObject(robj *o); @@ -841,16 +830,10 @@ unsigned long estimateObjectIdleTime(robj *o); int syncWrite(int fd, char *ptr, ssize_t size, int timeout); int syncRead(int fd, char *ptr, ssize_t size, int timeout); int syncReadLine(int fd, char *ptr, ssize_t size, int timeout); -int fwriteBulkString(FILE *fp, char *s, unsigned long len); -int fwriteBulkDouble(FILE *fp, double d); -int fwriteBulkLongLong(FILE *fp, long long l); -int fwriteBulkObject(FILE *fp, robj *obj); -int fwriteBulkCount(FILE *fp, char prefix, int count); /* Replication */ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc); void replicationFeedMonitors(list *monitors, int dictid, robj **argv, int argc); -int syncWithMaster(void); void updateSlavesWaitingBgsave(int bgsaveerr); void replicationCron(void); @@ -863,7 +846,7 @@ void stopLoading(void); #include "rdb.h" /* AOF persistence */ -void flushAppendOnlyFile(void); +void flushAppendOnlyFile(int force); void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int argc); void aofRemoveTempFile(pid_t childpid); int rewriteAppendOnlyFileBackground(void); @@ -898,9 +881,10 @@ int processCommand(redisClient *c); void setupSignalHandlers(void); struct redisCommand *lookupCommand(sds name); struct redisCommand *lookupCommandByCString(char *s); -void call(redisClient *c, struct redisCommand *cmd); +void call(redisClient *c); int prepareForShutdown(); void redisLog(int level, const char *fmt, ...); +void redisLogRaw(int level, const char *msg); void usage(); void updateDictResizePolicy(void); int htNeedsResize(dict *dict); @@ -908,40 +892,6 @@ void oom(const char *msg); void populateCommandTable(void); void resetCommandTableStats(void); -/* Disk store */ -int dsOpen(void); -int dsClose(void); -int dsSet(redisDb *db, robj *key, robj *val, time_t expire); -robj *dsGet(redisDb *db, robj *key, time_t *expire); -int dsDel(redisDb *db, robj *key); -int dsExists(redisDb *db, robj *key); -void dsFlushDb(int dbid); -int dsRdbSaveBackground(char *filename); -int dsRdbSave(char *filename); - -/* Disk Store Cache */ -void dsInit(void); -void vmThreadedIOCompletedJob(aeEventLoop *el, int fd, void *privdata, int mask); -void lockThreadedIO(void); -void unlockThreadedIO(void); -void freeIOJob(iojob *j); -void queueIOJob(iojob *j); -void waitEmptyIOJobsQueue(void); -void processAllPendingIOJobs(void); -int blockClientOnSwappedKeys(redisClient *c, struct redisCommand *cmd); -int dontWaitForSwappedKey(redisClient *c, robj *key); -void handleClientsBlockedOnSwappedKey(redisDb *db, robj *key); -int cacheFreeOneEntry(void); -void cacheScheduleIOAddFlag(redisDb *db, robj *key, long flag); -void cacheScheduleIODelFlag(redisDb *db, robj *key, long flag); -int cacheScheduleIOGetFlags(redisDb *db, robj *key); -void cacheScheduleIO(redisDb *db, robj *key, int type); -void cacheCron(void); -int cacheKeyMayExist(redisDb *db, robj *key); -void cacheSetKeyMayExist(redisDb *db, robj *key); -void cacheSetKeyDoesNotExist(redisDb *db, robj *key); -void cacheForcePointInTime(void); - /* Set data type */ robj *setTypeCreate(robj *value); int setTypeAdd(robj *subject, robj *value); @@ -994,8 +944,9 @@ robj *lookupKeyRead(redisDb *db, robj *key); robj *lookupKeyWrite(redisDb *db, robj *key); robj *lookupKeyReadOrReply(redisClient *c, robj *key, robj *reply); robj *lookupKeyWriteOrReply(redisClient *c, robj *key, robj *reply); -int dbAdd(redisDb *db, robj *key, robj *val); -int dbReplace(redisDb *db, robj *key, robj *val); +void dbAdd(redisDb *db, robj *key, robj *val); +void dbOverwrite(redisDb *db, robj *key, robj *val); +void setKey(redisDb *db, robj *key, robj *val); int dbExists(redisDb *db, robj *key); robj *dbRandomKey(redisDb *db); int dbDelete(redisDb *db, robj *key); @@ -1023,6 +974,9 @@ int clusterAddNode(clusterNode *node); void clusterCron(void); clusterNode *getNodeByQuery(redisClient *c, struct redisCommand *cmd, robj **argv, int argc, int *hashslot, int *ask); +/* Scripting */ +void scriptingInit(void); + /* Git SHA1 */ char *redisGitSHA1(void); char *redisGitDirty(void); @@ -1151,6 +1105,8 @@ void migrateCommand(redisClient *c); void dumpCommand(redisClient *c); void objectCommand(redisClient *c); void clientCommand(redisClient *c); +void evalCommand(redisClient *c); +void evalShaCommand(redisClient *c); #if defined(__GNUC__) void *calloc(size_t count, size_t size) __attribute__ ((deprecated));