Process async client checks like client timeouts and BLPOP timeouts incrementally...

[redis.git] / src / networking.c

diff --git a/src/networking.c b/src/networking.c
index 412f77dd09faf3f4deaea4517aadf73018f6c9bf..ae77d11b231cd1376e0e4b81df3ff48f04827b9a 100644 (file)
--- a/src/networking.c
+++ b/src/networking.c
@@ -3,6 +3,14 @@
 
 static void setProtocolError(redisClient *c, int pos);
 
 
 static void setProtocolError(redisClient *c, int pos);
 

+/* To evaluate the output buffer size of a client we need to get size of
+ * allocated objects, however we can't used zmalloc_size() directly on sds
+ * strings because of the trick they use to work (the header is before the
+ * returned pointer), so we use this helper function. */
+size_t zmalloc_size_sds(sds s) {
+    return zmalloc_size(s-sizeof(struct sdshdr));
+}
+

 void *dupClientReplyValue(void *o) {
     incrRefCount((robj*)o);
     return o;
 void *dupClientReplyValue(void *o) {
     incrRefCount((robj*)o);
     return o;


@@ -14,7 +22,6 @@ int listMatchObjects(void *a, void *b) {
 
 redisClient *createClient(int fd) {
     redisClient *c = zmalloc(sizeof(redisClient));
 
 redisClient *createClient(int fd) {
     redisClient *c = zmalloc(sizeof(redisClient));

-    c->bufpos = 0;

 
     /* passing -1 as fd it is possible to create a non connected client.
      * This is useful since all the Redis commands needs to be executed
 
     /* passing -1 as fd it is possible to create a non connected client.
      * This is useful since all the Redis commands needs to be executed


@@ -34,6 +41,7 @@ redisClient *createClient(int fd) {
 
     selectDb(c,0);
     c->fd = fd;
 
     selectDb(c,0);
     c->fd = fd;

+    c->bufpos = 0;

     c->querybuf = sdsempty();
     c->reqtype = 0;
     c->argc = 0;
     c->querybuf = sdsempty();
     c->reqtype = 0;
     c->argc = 0;


@@ -43,7 +51,7 @@ redisClient *createClient(int fd) {
     c->bulklen = -1;
     c->sentlen = 0;
     c->flags = 0;
     c->bulklen = -1;
     c->sentlen = 0;
     c->flags = 0;

-    c->lastinteraction = time(NULL);
+    c->ctime = c->lastinteraction = time(NULL);

     c->authenticated = 0;
     c->replstate = REDIS_REPL_NONE;
     c->reply = listCreate();
     c->authenticated = 0;
     c->replstate = REDIS_REPL_NONE;
     c->reply = listCreate();


@@ -137,6 +145,7 @@ void _addReplyObjectToList(redisClient *c, robj *o) {
     if (listLength(c->reply) == 0) {
         incrRefCount(o);
         listAddNodeTail(c->reply,o);
     if (listLength(c->reply) == 0) {
         incrRefCount(o);
         listAddNodeTail(c->reply,o);

+        c->reply_bytes += zmalloc_size_sds(o->ptr);

     } else {
         tail = listNodeValue(listLast(c->reply));
 
     } else {
         tail = listNodeValue(listLast(c->reply));
 


@@ -144,14 +153,16 @@ void _addReplyObjectToList(redisClient *c, robj *o) {
         if (tail->ptr != NULL &&
             sdslen(tail->ptr)+sdslen(o->ptr) <= REDIS_REPLY_CHUNK_BYTES)
         {
         if (tail->ptr != NULL &&
             sdslen(tail->ptr)+sdslen(o->ptr) <= REDIS_REPLY_CHUNK_BYTES)
         {

+            c->reply_bytes -= zmalloc_size_sds(tail->ptr);

             tail = dupLastObjectIfNeeded(c->reply);
             tail->ptr = sdscatlen(tail->ptr,o->ptr,sdslen(o->ptr));
             tail = dupLastObjectIfNeeded(c->reply);
             tail->ptr = sdscatlen(tail->ptr,o->ptr,sdslen(o->ptr));

+            c->reply_bytes += zmalloc_size_sds(tail->ptr);

         } else {
             incrRefCount(o);
             listAddNodeTail(c->reply,o);
         } else {
             incrRefCount(o);
             listAddNodeTail(c->reply,o);

+            c->reply_bytes += zmalloc_size_sds(o->ptr);

         }
     }
         }
     }

-    c->reply_bytes += sdslen(o->ptr);

     asyncCloseClientOnOutputBufferLimitReached(c);
 }
 
     asyncCloseClientOnOutputBufferLimitReached(c);
 }
 


@@ -165,9 +176,9 @@ void _addReplySdsToList(redisClient *c, sds s) {
         return;
     }
 
         return;
     }
 

-    c->reply_bytes += sdslen(s);

     if (listLength(c->reply) == 0) {
         listAddNodeTail(c->reply,createObject(REDIS_STRING,s));
     if (listLength(c->reply) == 0) {
         listAddNodeTail(c->reply,createObject(REDIS_STRING,s));

+        c->reply_bytes += zmalloc_size_sds(s);

     } else {
         tail = listNodeValue(listLast(c->reply));
 
     } else {
         tail = listNodeValue(listLast(c->reply));
 


@@ -175,11 +186,14 @@ void _addReplySdsToList(redisClient *c, sds s) {
         if (tail->ptr != NULL &&
             sdslen(tail->ptr)+sdslen(s) <= REDIS_REPLY_CHUNK_BYTES)
         {
         if (tail->ptr != NULL &&
             sdslen(tail->ptr)+sdslen(s) <= REDIS_REPLY_CHUNK_BYTES)
         {

+            c->reply_bytes -= zmalloc_size_sds(tail->ptr);

             tail = dupLastObjectIfNeeded(c->reply);
             tail->ptr = sdscatlen(tail->ptr,s,sdslen(s));
             tail = dupLastObjectIfNeeded(c->reply);
             tail->ptr = sdscatlen(tail->ptr,s,sdslen(s));

+            c->reply_bytes += zmalloc_size_sds(tail->ptr);

             sdsfree(s);
         } else {
             listAddNodeTail(c->reply,createObject(REDIS_STRING,s));
             sdsfree(s);
         } else {
             listAddNodeTail(c->reply,createObject(REDIS_STRING,s));

+            c->reply_bytes += zmalloc_size_sds(s);

         }
     }
     asyncCloseClientOnOutputBufferLimitReached(c);
         }
     }
     asyncCloseClientOnOutputBufferLimitReached(c);


@@ -191,7 +205,10 @@ void _addReplyStringToList(redisClient *c, char *s, size_t len) {
     if (c->flags & REDIS_CLOSE_AFTER_REPLY) return;
 
     if (listLength(c->reply) == 0) {
     if (c->flags & REDIS_CLOSE_AFTER_REPLY) return;
 
     if (listLength(c->reply) == 0) {

-        listAddNodeTail(c->reply,createStringObject(s,len));
+        robj *o = createStringObject(s,len);
+
+        listAddNodeTail(c->reply,o);
+        c->reply_bytes += zmalloc_size_sds(o->ptr);

     } else {
         tail = listNodeValue(listLast(c->reply));
 
     } else {
         tail = listNodeValue(listLast(c->reply));
 


@@ -199,13 +216,17 @@ void _addReplyStringToList(redisClient *c, char *s, size_t len) {
         if (tail->ptr != NULL &&
             sdslen(tail->ptr)+len <= REDIS_REPLY_CHUNK_BYTES)
         {
         if (tail->ptr != NULL &&
             sdslen(tail->ptr)+len <= REDIS_REPLY_CHUNK_BYTES)
         {

+            c->reply_bytes -= zmalloc_size_sds(tail->ptr);

             tail = dupLastObjectIfNeeded(c->reply);
             tail->ptr = sdscatlen(tail->ptr,s,len);
             tail = dupLastObjectIfNeeded(c->reply);
             tail->ptr = sdscatlen(tail->ptr,s,len);

+            c->reply_bytes += zmalloc_size_sds(tail->ptr);

         } else {
         } else {

-            listAddNodeTail(c->reply,createStringObject(s,len));
+            robj *o = createStringObject(s,len);
+
+            listAddNodeTail(c->reply,o);
+            c->reply_bytes += zmalloc_size_sds(o->ptr);

         }
     }
         }
     }

-    c->reply_bytes += len;

     asyncCloseClientOnOutputBufferLimitReached(c);
 }
 
     asyncCloseClientOnOutputBufferLimitReached(c);
 }
 


@@ -336,7 +357,7 @@ void setDeferredMultiBulkLength(redisClient *c, void *node, long length) {
 
     len = listNodeValue(ln);
     len->ptr = sdscatprintf(sdsempty(),"*%ld\r\n",length);
 
     len = listNodeValue(ln);
     len->ptr = sdscatprintf(sdsempty(),"*%ld\r\n",length);

-    c->reply_bytes += sdslen(len->ptr);
+    c->reply_bytes += zmalloc_size_sds(len->ptr);

     if (ln->next != NULL) {
         next = listNodeValue(ln->next);
 
     if (ln->next != NULL) {
         next = listNodeValue(ln->next);
 


@@ -638,6 +659,7 @@ void freeClientsInAsyncFreeQueue(void) {
 void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) {
     redisClient *c = privdata;
     int nwritten = 0, totwritten = 0, objlen;
 void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) {
     redisClient *c = privdata;
     int nwritten = 0, totwritten = 0, objlen;

+    size_t objmem;

     robj *o;
     REDIS_NOTUSED(el);
     REDIS_NOTUSED(mask);
     robj *o;
     REDIS_NOTUSED(el);
     REDIS_NOTUSED(mask);


@@ -663,6 +685,7 @@ void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) {
         } else {
             o = listNodeValue(listFirst(c->reply));
             objlen = sdslen(o->ptr);
         } else {
             o = listNodeValue(listFirst(c->reply));
             objlen = sdslen(o->ptr);

+            objmem = zmalloc_size_sds(o->ptr);

 
             if (objlen == 0) {
                 listDelNode(c->reply,listFirst(c->reply));
 
             if (objlen == 0) {
                 listDelNode(c->reply,listFirst(c->reply));


@@ -683,7 +706,7 @@ void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) {
             if (c->sentlen == objlen) {
                 listDelNode(c->reply,listFirst(c->reply));
                 c->sentlen = 0;
             if (c->sentlen == objlen) {
                 listDelNode(c->reply,listFirst(c->reply));
                 c->sentlen = 0;

-                c->reply_bytes -= objlen;
+                c->reply_bytes -= objmem;

             }
         }
         /* Note that we avoid to send more than REDIS_MAX_WRITE_PER_EVENT
             }
         }
         /* Note that we avoid to send more than REDIS_MAX_WRITE_PER_EVENT


@@ -728,34 +751,6 @@ void resetClient(redisClient *c) {
     if (!(c->flags & REDIS_MULTI)) c->flags &= (~REDIS_ASKING);
 }
 
     if (!(c->flags & REDIS_MULTI)) c->flags &= (~REDIS_ASKING);
 }
 

-void closeTimedoutClients(void) {
-    redisClient *c;
-    listNode *ln;
-    time_t now = time(NULL);
-    listIter li;
-
-    listRewind(server.clients,&li);
-    while ((ln = listNext(&li)) != NULL) {
-        c = listNodeValue(ln);
-        if (server.maxidletime &&
-            !(c->flags & REDIS_SLAVE) &&    /* no timeout for slaves */
-            !(c->flags & REDIS_MASTER) &&   /* no timeout for masters */
-            !(c->flags & REDIS_BLOCKED) &&  /* no timeout for BLPOP */
-            dictSize(c->pubsub_channels) == 0 && /* no timeout for pubsub */
-            listLength(c->pubsub_patterns) == 0 &&
-            (now - c->lastinteraction > server.maxidletime))
-        {
-            redisLog(REDIS_VERBOSE,"Closing idle client");
-            freeClient(c);
-        } else if (c->flags & REDIS_BLOCKED) {
-            if (c->bpop.timeout != 0 && c->bpop.timeout < now) {
-                addReply(c,shared.nullmultibulk);
-                unblockClientWaitingData(c);
-            }
-        }
-    }
-}
-

 int processInlineBuffer(redisClient *c) {
     char *newline = strstr(c->querybuf,"\r\n");
     int argc, j;
 int processInlineBuffer(redisClient *c) {
     char *newline = strstr(c->querybuf,"\r\n");
     int argc, j;


@@ -1064,11 +1059,7 @@ sds getClientInfoString(redisClient *client) {
     time_t now = time(NULL);
     int emask;
 
     time_t now = time(NULL);
     int emask;
 

-    if (anetPeerToString(client->fd,ip,&port) == -1) {
-        ip[0] = '?';
-        ip[1] = '\0';
-        port = 0;
-    }
+    anetPeerToString(client->fd,ip,&port);

     p = flags;
     if (client->flags & REDIS_SLAVE) {
         if (client->flags & REDIS_MONITOR)
     p = flags;
     if (client->flags & REDIS_SLAVE) {
         if (client->flags & REDIS_MONITOR)


@@ -1092,14 +1083,16 @@ sds getClientInfoString(redisClient *client) {
     if (emask & AE_WRITABLE) *p++ = 'w';
     *p = '\0';
     return sdscatprintf(sdsempty(),
     if (emask & AE_WRITABLE) *p++ = 'w';
     *p = '\0';
     return sdscatprintf(sdsempty(),

-        "addr=%s:%d fd=%d idle=%ld flags=%s db=%d sub=%d psub=%d qbuf=%lu obl=%lu oll=%lu omem=%lu events=%s cmd=%s",
+        "addr=%s:%d fd=%d age=%ld idle=%ld flags=%s db=%d sub=%d psub=%d qbuf=%lu qbuf-free=%lu obl=%lu oll=%lu omem=%lu events=%s cmd=%s",

         ip,port,client->fd,
         ip,port,client->fd,

+        (long)(now - client->ctime),

         (long)(now - client->lastinteraction),
         flags,
         client->db->id,
         (int) dictSize(client->pubsub_channels),
         (int) listLength(client->pubsub_patterns),
         (unsigned long) sdslen(client->querybuf),
         (long)(now - client->lastinteraction),
         flags,
         client->db->id,
         (int) dictSize(client->pubsub_channels),
         (int) listLength(client->pubsub_patterns),
         (unsigned long) sdslen(client->querybuf),

+        (unsigned long) sdsavail(client->querybuf),

         (unsigned long) client->bufpos,
         (unsigned long) listLength(client->reply),
         getClientOutputBufferMemoryUsage(client),
         (unsigned long) client->bufpos,
         (unsigned long) listLength(client->reply),
         getClientOutputBufferMemoryUsage(client),


@@ -1222,7 +1215,7 @@ void rewriteClientCommandArgument(redisClient *c, int i, robj *newval) {
  * the caller wishes. The main usage of this function currently is
  * enforcing the client output length limits. */
 unsigned long getClientOutputBufferMemoryUsage(redisClient *c) {
  * the caller wishes. The main usage of this function currently is
  * enforcing the client output length limits. */
 unsigned long getClientOutputBufferMemoryUsage(redisClient *c) {

-    unsigned long list_item_size = sizeof(listNode);
+    unsigned long list_item_size = sizeof(listNode)+sizeof(robj);

 
     return c->reply_bytes + (list_item_size*listLength(c->reply));
 }
 
     return c->reply_bytes + (list_item_size*listLength(c->reply));
 }


@@ -1306,7 +1299,7 @@ int checkClientOutputBufferLimits(redisClient *c) {
  * called from contexts where the client can't be freed safely, i.e. from the
  * lower level functions pushing data inside the client output buffers. */
 void asyncCloseClientOnOutputBufferLimitReached(redisClient *c) {
  * called from contexts where the client can't be freed safely, i.e. from the
  * lower level functions pushing data inside the client output buffers. */
 void asyncCloseClientOnOutputBufferLimitReached(redisClient *c) {

-    if (c->flags & REDIS_CLOSE_ASAP) return;
+    if (c->reply_bytes == 0 || c->flags & REDIS_CLOSE_ASAP) return;

     if (checkClientOutputBufferLimits(c)) {
         sds client = getClientInfoString(c);
 
     if (checkClientOutputBufferLimits(c)) {
         sds client = getClientInfoString(c);
 


@@ -1315,3 +1308,24 @@ void asyncCloseClientOnOutputBufferLimitReached(redisClient *c) {
         sdsfree(client);
     }
 }
         sdsfree(client);
     }
 }

+
+/* Helper function used by freeMemoryIfNeeded() in order to flush slaves
+ * output buffers without returning control to the event loop. */
+void flushSlavesOutputBuffers(void) {
+    listIter li;
+    listNode *ln;
+
+    listRewind(server.slaves,&li);
+    while((ln = listNext(&li))) {
+        redisClient *slave = listNodeValue(ln);
+        int events;
+
+        events = aeGetFileEvents(server.el,slave->fd);
+        if (events & AE_WRITABLE &&
+            slave->replstate == REDIS_REPL_ONLINE &&
+            listLength(slave->reply))
+        {
+            sendReplyToClient(server.el,slave->fd,slave,0);
+        }
+    }
+}