server.io_ready_clients = listCreate();
pthread_mutex_init(&server.io_mutex,NULL);
pthread_cond_init(&server.io_condvar,NULL);
+ pthread_mutex_init(&server.bgsavethread_mutex,NULL);
server.io_active_threads = 0;
if (pipe(pipefds) == -1) {
redisLog(REDIS_WARNING,"Unable to intialized DS: pipe(2): %s. Exiting."
* are swappable objects */
int maxtries = 100;
- if (dictSize(db->dict) == 0) continue;
- for (i = 0; i < 5; i++) {
+ for (i = 0; i < 5 && dictSize(db->dict); i++) {
dictEntry *de;
double swappability;
robj keyobj;
/* Every time a thread finished a Job, it writes a byte into the write side
* of an unix pipe in order to "awake" the main thread, and this function
- * is called. */
+ * is called.
+ *
+ * If privdata == NULL the function will try to put more jobs in the queue
+ * of IO jobs to process as more room is made. privdata is equal to NULL
+ * when the function is called from the event loop, so we want to push
+ * more IO jobs in the queue. Instead when the function is called by
+ * other functions that want to create a write-barrier to avoid race
+ * conditions we don't push new jobs in the queue. */
void vmThreadedIOCompletedJob(aeEventLoop *el, int fd, void *privdata,
int mask)
{
int retval, processed = 0, toprocess = -1;
REDIS_NOTUSED(el);
REDIS_NOTUSED(mask);
- REDIS_NOTUSED(privdata);
/* For every byte we read in the read side of the pipe, there is one
* I/O job completed to process. */
freeIOJob(j);
}
processed++;
+ if (privdata == NULL) cacheScheduleIOPushJobs(0);
if (processed == toprocess) return;
}
if (retval < 0 && errno != EAGAIN) {
iojob *j;
listNode *ln;
REDIS_NOTUSED(arg);
+ long long start;
pthread_detach(pthread_self());
lockThreadedIO();
/* Get a new job to process */
if (listLength(server.io_newjobs) == 0) {
/* Wait for more work to do */
+ redisLog(REDIS_DEBUG,"[T] wait for signal");
pthread_cond_wait(&server.io_condvar,&server.io_mutex);
+ redisLog(REDIS_DEBUG,"[T] signal received");
continue;
}
- redisLog(REDIS_DEBUG,"%ld IO jobs to process",
+ start = ustime();
+ redisLog(REDIS_DEBUG,"[T] %ld IO jobs to process",
listLength(server.io_newjobs));
ln = listFirst(server.io_newjobs);
j = ln->value;
ln = listLast(server.io_processing); /* We use ln later to remove it */
unlockThreadedIO();
- redisLog(REDIS_DEBUG,"Thread %ld: new job type %s: %p about key '%s'",
+ redisLog(REDIS_DEBUG,"[T] %ld: new job type %s: %p about key '%s'",
(long) pthread_self(),
(j->type == REDIS_IOJOB_LOAD) ? "load" : "save",
(void*)j, (char*)j->key->ptr);
if (j->val) j->expire = expire;
} else if (j->type == REDIS_IOJOB_SAVE) {
if (j->val) {
- dsSet(j->db,j->key,j->val);
+ dsSet(j->db,j->key,j->val,j->expire);
} else {
dsDel(j->db,j->key);
}
}
/* Done: insert the job into the processed queue */
- redisLog(REDIS_DEBUG,"Thread %ld completed the job: %p (key %s)",
+ redisLog(REDIS_DEBUG,"[T] %ld completed the job: %p (key %s)",
(long) pthread_self(), (void*)j, (char*)j->key->ptr);
+ redisLog(REDIS_DEBUG,"[T] lock IO");
lockThreadedIO();
+ redisLog(REDIS_DEBUG,"[T] IO locked");
listDelNode(server.io_processing,ln);
listAddNodeTail(server.io_processed,j);
/* Signal the main thread there is new stuff to process */
redisAssert(write(server.io_ready_pipe_write,"x",1) == 1);
+ redisLog(REDIS_DEBUG,"TIME (%c): %lld\n", j->type == REDIS_IOJOB_LOAD ? 'L' : 'S', ustime()-start);
}
/* never reached, but that's the full pattern... */
unlockThreadedIO();
while(max == -1 || max > 0) {
int io_processed_len;
+ redisLog(REDIS_DEBUG,"[P] lock IO");
lockThreadedIO();
+ redisLog(REDIS_DEBUG,"Waiting IO jobs processing: new:%d proessing:%d processed:%d",listLength(server.io_newjobs),listLength(server.io_processing),listLength(server.io_processed));
+
if (listLength(server.io_newjobs) == 0 &&
listLength(server.io_processing) == 0)
{
/* There is nothing more to process */
+ redisLog(REDIS_DEBUG,"[P] Nothing to process, unlock IO, return");
unlockThreadedIO();
break;
}
-#if 0
+#if 1
/* If there are new jobs we need to signal the thread to
* process the next one. FIXME: drop this if useless. */
- redisLog(REDIS_DEBUG,"waitEmptyIOJobsQueue: new %d, processing %d",
+ redisLog(REDIS_DEBUG,"[P] waitEmptyIOJobsQueue: new %d, processing %d, processed %d",
listLength(server.io_newjobs),
- listLength(server.io_processing));
+ listLength(server.io_processing),
+ listLength(server.io_processed));
if (listLength(server.io_newjobs)) {
+ redisLog(REDIS_DEBUG,"[P] There are new jobs, signal");
pthread_cond_signal(&server.io_condvar);
}
#endif
/* Check if we can process some finished job */
io_processed_len = listLength(server.io_processed);
+ redisLog(REDIS_DEBUG,"[P] Unblock IO");
unlockThreadedIO();
+ redisLog(REDIS_DEBUG,"[P] Wait");
+ usleep(10000);
if (io_processed_len) {
vmThreadedIOCompletedJob(NULL,server.io_ready_pipe_read,
(void*)0xdeadbeef,0);
processAllPendingIOJobs();
}
-void cacheCreateIOJob(int type, redisDb *db, robj *key, robj *val) {
+void cacheCreateIOJob(int type, redisDb *db, robj *key, robj *val, time_t expire) {
iojob *j;
j = zmalloc(sizeof(*j));
incrRefCount(key);
j->val = val;
if (val) incrRefCount(val);
+ j->expire = expire;
lockThreadedIO();
queueIOJob(j);
* scheduled completion time, but just do the operation ASAP. This is useful
* when we need to reclaim memory from the IO queue.
*/
-#define MAX_IO_JOBS_QUEUE 100
+#define MAX_IO_JOBS_QUEUE 10
int cacheScheduleIOPushJobs(int flags) {
time_t now = time(NULL);
listNode *ln;
int jobs, topush = 0, pushed = 0;
+ /* Don't push new jobs if there is a threaded BGSAVE in progress. */
+ if (server.bgsavethread != (pthread_t) -1) return 0;
+
/* Sync stuff on disk, but only if we have less
* than MAX_IO_JOBS_QUEUE IO jobs. */
lockThreadedIO();
if (op->type != REDIS_IO_LOAD && flags & REDIS_IO_ONLYLOADS) break;
- if (!(flags & REDIS_IO_ASAP) &&
+ /* Don't execute SAVE before the scheduled time for completion */
+ if (op->type == REDIS_IO_SAVE && !(flags & REDIS_IO_ASAP) &&
(now - op->ctime) < server.cache_flush_delay) break;
/* Don't add a SAVE job in the IO thread queue if there is already
op->type == REDIS_IO_LOAD ? "load" : "save", op->key->ptr);
if (op->type == REDIS_IO_LOAD) {
- cacheCreateIOJob(REDIS_IOJOB_LOAD,op->db,op->key,NULL);
+ cacheCreateIOJob(REDIS_IOJOB_LOAD,op->db,op->key,NULL,0);
} else {
+ time_t expire = -1;
+
/* Lookup the key, in order to put the current value in the IO
* Job. Otherwise if the key does not exists we schedule a disk
* store delete operation, setting the value to NULL. */
de = dictFind(op->db->dict,op->key->ptr);
if (de) {
val = dictGetEntryVal(de);
+ expire = getExpire(op->db,op->key);
} else {
/* Setting the value to NULL tells the IO thread to delete
* the key on disk. */
val = NULL;
}
- cacheCreateIOJob(REDIS_IOJOB_SAVE,op->db,op->key,val);
+ cacheCreateIOJob(REDIS_IOJOB_SAVE,op->db,op->key,val,expire);
}
/* Mark the operation as in progress. */
cacheScheduleIODelFlag(op->db,op->key,op->type);
projects / redis.git / blobdiff