src/replication.c

   1 #include "redis.h"
   2
   3 #include <sys/time.h>
   4 #include <unistd.h>
   5 #include <fcntl.h>
   6 #include <sys/stat.h>
   7
   8 /* ---------------------------------- MASTER -------------------------------- */
   9
  10 void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) {
  11     listNode *ln;
  12     listIter li;
  13     int j;
  14
  15     listRewind(slaves,&li);
  16     while((ln = listNext(&li))) {
  17         redisClient *slave = ln->value;
  18
  19         /* Don't feed slaves that are still waiting for BGSAVE to start */
  20         if (slave->replstate == REDIS_REPL_WAIT_BGSAVE_START) continue;
  21
  22         /* Feed slaves that are waiting for the initial SYNC (so these commands
  23          * are queued in the output buffer until the intial SYNC completes),
  24          * or are already in sync with the master. */
  25         if (slave->slaveseldb != dictid) {
  26             robj *selectcmd;
  27
  28             switch(dictid) {
  29             case 0: selectcmd = shared.select0; break;
  30             case 1: selectcmd = shared.select1; break;
  31             case 2: selectcmd = shared.select2; break;
  32             case 3: selectcmd = shared.select3; break;
  33             case 4: selectcmd = shared.select4; break;
  34             case 5: selectcmd = shared.select5; break;
  35             case 6: selectcmd = shared.select6; break;
  36             case 7: selectcmd = shared.select7; break;
  37             case 8: selectcmd = shared.select8; break;
  38             case 9: selectcmd = shared.select9; break;
  39             default:
  40                 selectcmd = createObject(REDIS_STRING,
  41                     sdscatprintf(sdsempty(),"select %d\r\n",dictid));
  42                 selectcmd->refcount = 0;
  43                 break;
  44             }
  45             addReply(slave,selectcmd);
  46             slave->slaveseldb = dictid;
  47         }
  48         addReplyMultiBulkLen(slave,argc);
  49         for (j = 0; j < argc; j++) addReplyBulk(slave,argv[j]);
  50     }
  51 }
  52
  53 void replicationFeedMonitors(list *monitors, int dictid, robj **argv, int argc) {
  54     listNode *ln;
  55     listIter li;
  56     int j;
  57     sds cmdrepr = sdsnew("+");
  58     robj *cmdobj;
  59     struct timeval tv;
  60
  61     gettimeofday(&tv,NULL);
  62     cmdrepr = sdscatprintf(cmdrepr,"%ld.%06ld ",(long)tv.tv_sec,(long)tv.tv_usec);
  63     if (dictid != 0) cmdrepr = sdscatprintf(cmdrepr,"(db %d) ", dictid);
  64
  65     for (j = 0; j < argc; j++) {
  66         if (argv[j]->encoding == REDIS_ENCODING_INT) {
  67             cmdrepr = sdscatprintf(cmdrepr, "\"%ld\"", (long)argv[j]->ptr);
  68         } else {
  69             cmdrepr = sdscatrepr(cmdrepr,(char*)argv[j]->ptr,
  70                         sdslen(argv[j]->ptr));
  71         }
  72         if (j != argc-1)
  73             cmdrepr = sdscatlen(cmdrepr," ",1);
  74     }
  75     cmdrepr = sdscatlen(cmdrepr,"\r\n",2);
  76     cmdobj = createObject(REDIS_STRING,cmdrepr);
  77
  78     listRewind(monitors,&li);
  79     while((ln = listNext(&li))) {
  80         redisClient *monitor = ln->value;
  81         addReply(monitor,cmdobj);
  82     }
  83     decrRefCount(cmdobj);
  84 }
  85
  86 void syncCommand(redisClient *c) {
  87     /* ignore SYNC if aleady slave or in monitor mode */
  88     if (c->flags & REDIS_SLAVE) return;
  89
  90     /* Refuse SYNC requests if we are a slave but the link with our master
  91      * is not ok... */
  92     if (server.masterhost && server.repl_state != REDIS_REPL_CONNECTED) {
  93         addReplyError(c,"Can't SYNC while not connected with my master");
  94         return;
  95     }
  96
  97     /* SYNC can't be issued when the server has pending data to send to
  98      * the client about already issued commands. We need a fresh reply
  99      * buffer registering the differences between the BGSAVE and the current
 100      * dataset, so that we can copy to other slaves if needed. */
 101     if (listLength(c->reply) != 0) {
 102         addReplyError(c,"SYNC is invalid with pending input");
 103         return;
 104     }
 105
 106     redisLog(REDIS_NOTICE,"Slave ask for synchronization");
 107     /* Here we need to check if there is a background saving operation
 108      * in progress, or if it is required to start one */
 109     if (server.rdb_child_pid != -1) {
 110         /* Ok a background save is in progress. Let's check if it is a good
 111          * one for replication, i.e. if there is another slave that is
 112          * registering differences since the server forked to save */
 113         redisClient *slave;
 114         listNode *ln;
 115         listIter li;
 116
 117         listRewind(server.slaves,&li);
 118         while((ln = listNext(&li))) {
 119             slave = ln->value;
 120             if (slave->replstate == REDIS_REPL_WAIT_BGSAVE_END) break;
 121         }
 122         if (ln) {
 123             /* Perfect, the server is already registering differences for
 124              * another slave. Set the right state, and copy the buffer. */
 125             copyClientOutputBuffer(c,slave);
 126             c->replstate = REDIS_REPL_WAIT_BGSAVE_END;
 127             redisLog(REDIS_NOTICE,"Waiting for end of BGSAVE for SYNC");
 128         } else {
 129             /* No way, we need to wait for the next BGSAVE in order to
 130              * register differences */
 131             c->replstate = REDIS_REPL_WAIT_BGSAVE_START;
 132             redisLog(REDIS_NOTICE,"Waiting for next BGSAVE for SYNC");
 133         }
 134     } else {
 135         /* Ok we don't have a BGSAVE in progress, let's start one */
 136         redisLog(REDIS_NOTICE,"Starting BGSAVE for SYNC");
 137         if (rdbSaveBackground(server.rdb_filename) != REDIS_OK) {
 138             redisLog(REDIS_NOTICE,"Replication failed, can't BGSAVE");
 139             addReplyError(c,"Unable to perform background save");
 140             return;
 141         }
 142         c->replstate = REDIS_REPL_WAIT_BGSAVE_END;
 143     }
 144     c->repldbfd = -1;
 145     c->flags |= REDIS_SLAVE;
 146     c->slaveseldb = 0;
 147     listAddNodeTail(server.slaves,c);
 148     return;
 149 }
 150
 151 void sendBulkToSlave(aeEventLoop *el, int fd, void *privdata, int mask) {
 152     redisClient *slave = privdata;
 153     REDIS_NOTUSED(el);
 154     REDIS_NOTUSED(mask);
 155     char buf[REDIS_IOBUF_LEN];
 156     ssize_t nwritten, buflen;
 157
 158     if (slave->repldboff == 0) {
 159         /* Write the bulk write count before to transfer the DB. In theory here
 160          * we don't know how much room there is in the output buffer of the
 161          * socket, but in pratice SO_SNDLOWAT (the minimum count for output
 162          * operations) will never be smaller than the few bytes we need. */
 163         sds bulkcount;
 164
 165         bulkcount = sdscatprintf(sdsempty(),"$%lld\r\n",(unsigned long long)
 166             slave->repldbsize);
 167         if (write(fd,bulkcount,sdslen(bulkcount)) != (signed)sdslen(bulkcount))
 168         {
 169             sdsfree(bulkcount);
 170             freeClient(slave);
 171             return;
 172         }
 173         sdsfree(bulkcount);
 174     }
 175     lseek(slave->repldbfd,slave->repldboff,SEEK_SET);
 176     buflen = read(slave->repldbfd,buf,REDIS_IOBUF_LEN);
 177     if (buflen <= 0) {
 178         redisLog(REDIS_WARNING,"Read error sending DB to slave: %s",
 179             (buflen == 0) ? "premature EOF" : strerror(errno));
 180         freeClient(slave);
 181         return;
 182     }
 183     if ((nwritten = write(fd,buf,buflen)) == -1) {
 184         redisLog(REDIS_VERBOSE,"Write error sending DB to slave: %s",
 185             strerror(errno));
 186         freeClient(slave);
 187         return;
 188     }
 189     slave->repldboff += nwritten;
 190     if (slave->repldboff == slave->repldbsize) {
 191         close(slave->repldbfd);
 192         slave->repldbfd = -1;
 193         aeDeleteFileEvent(server.el,slave->fd,AE_WRITABLE);
 194         slave->replstate = REDIS_REPL_ONLINE;
 195         if (aeCreateFileEvent(server.el, slave->fd, AE_WRITABLE,
 196             sendReplyToClient, slave) == AE_ERR) {
 197             freeClient(slave);
 198             return;
 199         }
 200         addReplySds(slave,sdsempty());
 201         redisLog(REDIS_NOTICE,"Synchronization with slave succeeded");
 202     }
 203 }
 204
 205 /* This function is called at the end of every backgrond saving.
 206  * The argument bgsaveerr is REDIS_OK if the background saving succeeded
 207  * otherwise REDIS_ERR is passed to the function.
 208  *
 209  * The goal of this function is to handle slaves waiting for a successful
 210  * background saving in order to perform non-blocking synchronization. */
 211 void updateSlavesWaitingBgsave(int bgsaveerr) {
 212     listNode *ln;
 213     int startbgsave = 0;
 214     listIter li;
 215
 216     listRewind(server.slaves,&li);
 217     while((ln = listNext(&li))) {
 218         redisClient *slave = ln->value;
 219
 220         if (slave->replstate == REDIS_REPL_WAIT_BGSAVE_START) {
 221             startbgsave = 1;
 222             slave->replstate = REDIS_REPL_WAIT_BGSAVE_END;
 223         } else if (slave->replstate == REDIS_REPL_WAIT_BGSAVE_END) {
 224             struct redis_stat buf;
 225
 226             if (bgsaveerr != REDIS_OK) {
 227                 freeClient(slave);
 228                 redisLog(REDIS_WARNING,"SYNC failed. BGSAVE child returned an error");
 229                 continue;
 230             }
 231             if ((slave->repldbfd = open(server.rdb_filename,O_RDONLY)) == -1 ||
 232                 redis_fstat(slave->repldbfd,&buf) == -1) {
 233                 freeClient(slave);
 234                 redisLog(REDIS_WARNING,"SYNC failed. Can't open/stat DB after BGSAVE: %s", strerror(errno));
 235                 continue;
 236             }
 237             slave->repldboff = 0;
 238             slave->repldbsize = buf.st_size;
 239             slave->replstate = REDIS_REPL_SEND_BULK;
 240             aeDeleteFileEvent(server.el,slave->fd,AE_WRITABLE);
 241             if (aeCreateFileEvent(server.el, slave->fd, AE_WRITABLE, sendBulkToSlave, slave) == AE_ERR) {
 242                 freeClient(slave);
 243                 continue;
 244             }
 245         }
 246     }
 247     if (startbgsave) {
 248         if (rdbSaveBackground(server.rdb_filename) != REDIS_OK) {
 249             listIter li;
 250
 251             listRewind(server.slaves,&li);
 252             redisLog(REDIS_WARNING,"SYNC failed. BGSAVE failed");
 253             while((ln = listNext(&li))) {
 254                 redisClient *slave = ln->value;
 255
 256                 if (slave->replstate == REDIS_REPL_WAIT_BGSAVE_START)
 257                     freeClient(slave);
 258             }
 259         }
 260     }
 261 }
 262
 263 /* ----------------------------------- SLAVE -------------------------------- */
 264
 265 /* Abort the async download of the bulk dataset while SYNC-ing with master */
 266 void replicationAbortSyncTransfer(void) {
 267     redisAssert(server.repl_state == REDIS_REPL_TRANSFER);
 268
 269     aeDeleteFileEvent(server.el,server.repl_transfer_s,AE_READABLE);
 270     close(server.repl_transfer_s);
 271     close(server.repl_transfer_fd);
 272     unlink(server.repl_transfer_tmpfile);
 273     zfree(server.repl_transfer_tmpfile);
 274     server.repl_state = REDIS_REPL_CONNECT;
 275 }
 276
 277 /* Asynchronously read the SYNC payload we receive from a master */
 278 void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) {
 279     char buf[4096];
 280     ssize_t nread, readlen;
 281     REDIS_NOTUSED(el);
 282     REDIS_NOTUSED(privdata);
 283     REDIS_NOTUSED(mask);
 284
 285     /* If repl_transfer_left == -1 we still have to read the bulk length
 286      * from the master reply. */
 287     if (server.repl_transfer_left == -1) {
 288         if (syncReadLine(fd,buf,1024,server.repl_syncio_timeout) == -1) {
 289             redisLog(REDIS_WARNING,
 290                 "I/O error reading bulk count from MASTER: %s",
 291                 strerror(errno));
 292             goto error;
 293         }
 294
 295         if (buf[0] == '-') {
 296             redisLog(REDIS_WARNING,
 297                 "MASTER aborted replication with an error: %s",
 298                 buf+1);
 299             goto error;
 300         } else if (buf[0] == '\0') {
 301             /* At this stage just a newline works as a PING in order to take
 302              * the connection live. So we refresh our last interaction
 303              * timestamp. */
 304             server.repl_transfer_lastio = time(NULL);
 305             return;
 306         } else if (buf[0] != '$') {
 307             redisLog(REDIS_WARNING,"Bad protocol from MASTER, the first byte is not '$', are you sure the host and port are right?");
 308             goto error;
 309         }
 310         server.repl_transfer_left = strtol(buf+1,NULL,10);
 311         redisLog(REDIS_NOTICE,
 312             "MASTER <-> SLAVE sync: receiving %ld bytes from master",
 313             server.repl_transfer_left);
 314         return;
 315     }
 316
 317     /* Read bulk data */
 318     readlen = (server.repl_transfer_left < (signed)sizeof(buf)) ?
 319         server.repl_transfer_left : (signed)sizeof(buf);
 320     nread = read(fd,buf,readlen);
 321     if (nread <= 0) {
 322         redisLog(REDIS_WARNING,"I/O error trying to sync with MASTER: %s",
 323             (nread == -1) ? strerror(errno) : "connection lost");
 324         replicationAbortSyncTransfer();
 325         return;
 326     }
 327     server.repl_transfer_lastio = time(NULL);
 328     if (write(server.repl_transfer_fd,buf,nread) != nread) {
 329         redisLog(REDIS_WARNING,"Write error or short write writing to the DB dump file needed for MASTER <-> SLAVE synchrnonization: %s", strerror(errno));
 330         goto error;
 331     }
 332     server.repl_transfer_left -= nread;
 333     /* Check if the transfer is now complete */
 334     if (server.repl_transfer_left == 0) {
 335         if (rename(server.repl_transfer_tmpfile,server.rdb_filename) == -1) {
 336             redisLog(REDIS_WARNING,"Failed trying to rename the temp DB into dump.rdb in MASTER <-> SLAVE synchronization: %s", strerror(errno));
 337             replicationAbortSyncTransfer();
 338             return;
 339         }
 340         redisLog(REDIS_NOTICE, "MASTER <-> SLAVE sync: Loading DB in memory");
 341         emptyDb();
 342         /* Before loading the DB into memory we need to delete the readable
 343          * handler, otherwise it will get called recursively since
 344          * rdbLoad() will call the event loop to process events from time to
 345          * time for non blocking loading. */
 346         aeDeleteFileEvent(server.el,server.repl_transfer_s,AE_READABLE);
 347         if (rdbLoad(server.rdb_filename) != REDIS_OK) {
 348             redisLog(REDIS_WARNING,"Failed trying to load the MASTER synchronization DB from disk");
 349             replicationAbortSyncTransfer();
 350             return;
 351         }
 352         /* Final setup of the connected slave <- master link */
 353         zfree(server.repl_transfer_tmpfile);
 354         close(server.repl_transfer_fd);
 355         server.master = createClient(server.repl_transfer_s);
 356         server.master->flags |= REDIS_MASTER;
 357         server.master->authenticated = 1;
 358         server.repl_state = REDIS_REPL_CONNECTED;
 359         redisLog(REDIS_NOTICE, "MASTER <-> SLAVE sync: Finished with success");
 360         /* Restart the AOF subsystem now that we finished the sync. This
 361          * will trigger an AOF rewrite, and when done will start appending
 362          * to the new file. */
 363         if (server.aof_state != REDIS_AOF_OFF) {
 364             int retry = 10;
 365
 366             stopAppendOnly();
 367             while (retry-- && startAppendOnly() == REDIS_ERR) {
 368                 redisLog(REDIS_WARNING,"Failed enabling the AOF after successful master synchrnization! Trying it again in one second.");
 369                 sleep(1);
 370             }
 371             if (!retry) {
 372                 redisLog(REDIS_WARNING,"FATAL: this slave instance finished the synchronization with its master, but the AOF can't be turned on. Exiting now.");
 373                 exit(1);
 374             }
 375         }
 376     }
 377
 378     return;
 379
 380 error:
 381     replicationAbortSyncTransfer();
 382     return;
 383 }
 384
 385 void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) {
 386     char buf[1024], tmpfile[256];
 387     int dfd, maxtries = 5;
 388     REDIS_NOTUSED(el);
 389     REDIS_NOTUSED(privdata);
 390     REDIS_NOTUSED(mask);
 391
 392     /* If this event fired after the user turned the instance into a master
 393      * with SLAVEOF NO ONE we must just return ASAP. */
 394     if (server.repl_state == REDIS_REPL_NONE) {
 395         close(fd);
 396         return;
 397     }
 398
 399     redisLog(REDIS_NOTICE,"Non blocking connect for SYNC fired the event.");
 400     /* This event should only be triggered once since it is used to have a
 401      * non-blocking connect(2) to the master. It has been triggered when this
 402      * function is called, so we can delete it. */
 403     aeDeleteFileEvent(server.el,fd,AE_READABLE|AE_WRITABLE);
 404
 405     /* AUTH with the master if required. */
 406     if(server.masterauth) {
 407         char authcmd[1024];
 408         size_t authlen;
 409
 410         authlen = snprintf(authcmd,sizeof(authcmd),"AUTH %s\r\n",server.masterauth);
 411         if (syncWrite(fd,authcmd,authlen,server.repl_syncio_timeout) == -1) {
 412             redisLog(REDIS_WARNING,"Unable to AUTH to MASTER: %s",
 413                 strerror(errno));
 414             goto error;
 415         }
 416         /* Read the AUTH result.  */
 417         if (syncReadLine(fd,buf,1024,server.repl_syncio_timeout) == -1) {
 418             redisLog(REDIS_WARNING,"I/O error reading auth result from MASTER: %s",
 419                 strerror(errno));
 420             goto error;
 421         }
 422         if (buf[0] != '+') {
 423             redisLog(REDIS_WARNING,"Cannot AUTH to MASTER, is the masterauth password correct?");
 424             goto error;
 425         }
 426     }
 427
 428     /* Issue the SYNC command */
 429     if (syncWrite(fd,"SYNC \r\n",7,server.repl_syncio_timeout) == -1) {
 430         redisLog(REDIS_WARNING,"I/O error writing to MASTER: %s",
 431             strerror(errno));
 432         goto error;
 433     }
 434
 435     /* Prepare a suitable temp file for bulk transfer */
 436     while(maxtries--) {
 437         snprintf(tmpfile,256,
 438             "temp-%d.%ld.rdb",(int)time(NULL),(long int)getpid());
 439         dfd = open(tmpfile,O_CREAT|O_WRONLY|O_EXCL,0644);
 440         if (dfd != -1) break;
 441         sleep(1);
 442     }
 443     if (dfd == -1) {
 444         redisLog(REDIS_WARNING,"Opening the temp file needed for MASTER <-> SLAVE synchronization: %s",strerror(errno));
 445         goto error;
 446     }
 447
 448     /* Setup the non blocking download of the bulk file. */
 449     if (aeCreateFileEvent(server.el,fd, AE_READABLE,readSyncBulkPayload,NULL)
 450             == AE_ERR)
 451     {
 452         redisLog(REDIS_WARNING,"Can't create readable event for SYNC");
 453         goto error;
 454     }
 455
 456     server.repl_state = REDIS_REPL_TRANSFER;
 457     server.repl_transfer_left = -1;
 458     server.repl_transfer_fd = dfd;
 459     server.repl_transfer_lastio = time(NULL);
 460     server.repl_transfer_tmpfile = zstrdup(tmpfile);
 461     return;
 462
 463 error:
 464     server.repl_state = REDIS_REPL_CONNECT;
 465     close(fd);
 466     return;
 467 }
 468
 469 int connectWithMaster(void) {
 470     int fd;
 471
 472     fd = anetTcpNonBlockConnect(NULL,server.masterhost,server.masterport);
 473     if (fd == -1) {
 474         redisLog(REDIS_WARNING,"Unable to connect to MASTER: %s",
 475             strerror(errno));
 476         return REDIS_ERR;
 477     }
 478
 479     if (aeCreateFileEvent(server.el,fd,AE_READABLE|AE_WRITABLE,syncWithMaster,NULL) ==
 480             AE_ERR)
 481     {
 482         close(fd);
 483         redisLog(REDIS_WARNING,"Can't create readable event for SYNC");
 484         return REDIS_ERR;
 485     }
 486
 487     server.repl_transfer_lastio = time(NULL);
 488     server.repl_transfer_s = fd;
 489     server.repl_state = REDIS_REPL_CONNECTING;
 490     return REDIS_OK;
 491 }
 492
 493 /* This function can be called when a non blocking connection is currently
 494  * in progress to undo it. */
 495 void undoConnectWithMaster(void) {
 496     int fd = server.repl_transfer_s;
 497
 498     redisAssert(server.repl_state == REDIS_REPL_CONNECTING);
 499     aeDeleteFileEvent(server.el,fd,AE_READABLE|AE_WRITABLE);
 500     close(fd);
 501     server.repl_transfer_s = -1;
 502     server.repl_state = REDIS_REPL_CONNECT;
 503 }
 504
 505 void slaveofCommand(redisClient *c) {
 506     if (!strcasecmp(c->argv[1]->ptr,"no") &&
 507         !strcasecmp(c->argv[2]->ptr,"one")) {
 508         if (server.masterhost) {
 509             sdsfree(server.masterhost);
 510             server.masterhost = NULL;
 511             if (server.master) freeClient(server.master);
 512             if (server.repl_state == REDIS_REPL_TRANSFER)
 513                 replicationAbortSyncTransfer();
 514             else if (server.repl_state == REDIS_REPL_CONNECTING)
 515                 undoConnectWithMaster();
 516             server.repl_state = REDIS_REPL_NONE;
 517             redisLog(REDIS_NOTICE,"MASTER MODE enabled (user request)");
 518         }
 519     } else {
 520         sdsfree(server.masterhost);
 521         server.masterhost = sdsdup(c->argv[1]->ptr);
 522         server.masterport = atoi(c->argv[2]->ptr);
 523         if (server.master) freeClient(server.master);
 524         if (server.repl_state == REDIS_REPL_TRANSFER)
 525             replicationAbortSyncTransfer();
 526         server.repl_state = REDIS_REPL_CONNECT;
 527         redisLog(REDIS_NOTICE,"SLAVE OF %s:%d enabled (user request)",
 528             server.masterhost, server.masterport);
 529     }
 530     addReply(c,shared.ok);
 531 }
 532
 533 /* --------------------------- REPLICATION CRON  ---------------------------- */
 534
 535 void replicationCron(void) {
 536     /* Non blocking connection timeout? */
 537     if (server.masterhost && server.repl_state == REDIS_REPL_CONNECTING &&
 538         (time(NULL)-server.repl_transfer_lastio) > server.repl_timeout)
 539     {
 540         redisLog(REDIS_WARNING,"Timeout connecting to the MASTER...");
 541         undoConnectWithMaster();
 542     }
 543
 544     /* Bulk transfer I/O timeout? */
 545     if (server.masterhost && server.repl_state == REDIS_REPL_TRANSFER &&
 546         (time(NULL)-server.repl_transfer_lastio) > server.repl_timeout)
 547     {
 548         redisLog(REDIS_WARNING,"Timeout receiving bulk data from MASTER...");
 549         replicationAbortSyncTransfer();
 550     }
 551
 552     /* Timed out master when we are an already connected slave? */
 553     if (server.masterhost && server.repl_state == REDIS_REPL_CONNECTED &&
 554         (time(NULL)-server.master->lastinteraction) > server.repl_timeout)
 555     {
 556         redisLog(REDIS_WARNING,"MASTER time out: no data nor PING received...");
 557         freeClient(server.master);
 558     }
 559
 560     /* Check if we should connect to a MASTER */
 561     if (server.repl_state == REDIS_REPL_CONNECT) {
 562         redisLog(REDIS_NOTICE,"Connecting to MASTER...");
 563         if (connectWithMaster() == REDIS_OK) {
 564             redisLog(REDIS_NOTICE,"MASTER <-> SLAVE sync started");
 565         }
 566     }
 567
 568     /* If we have attached slaves, PING them from time to time.
 569      * So slaves can implement an explicit timeout to masters, and will
 570      * be able to detect a link disconnection even if the TCP connection
 571      * will not actually go down. */
 572     if (!(server.cronloops % (server.repl_ping_slave_period*10))) {
 573         listIter li;
 574         listNode *ln;
 575
 576         listRewind(server.slaves,&li);
 577         while((ln = listNext(&li))) {
 578             redisClient *slave = ln->value;
 579
 580             /* Don't ping slaves that are in the middle of a bulk transfer
 581              * with the master for first synchronization. */
 582             if (slave->replstate == REDIS_REPL_SEND_BULK) continue;
 583             if (slave->replstate == REDIS_REPL_ONLINE) {
 584                 /* If the slave is online send a normal ping */
 585                 addReplySds(slave,sdsnew("PING\r\n"));
 586             } else {
 587                 /* Otherwise we are in the pre-synchronization stage.
 588                  * Just a newline will do the work of refreshing the
 589                  * connection last interaction time, and at the same time
 590                  * we'll be sure that being a single char there are no
 591                  * short-write problems. */
 592                 if (write(slave->fd, "\n", 1) == -1) {
 593                     /* Don't worry, it's just a ping. */
 594                 }
 595             }
 596         }
 597     }
 598 }