methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.47 2001/02/20 07:03:18 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apt-pkg/fileutl.h>
  29 #include <apt-pkg/acquire-method.h>
  30 #include <apt-pkg/error.h>
  31 #include <apt-pkg/md5.h>
  32
  33 #include <sys/stat.h>
  34 #include <sys/time.h>
  35 #include <utime.h>
  36 #include <unistd.h>
  37 #include <signal.h>
  38 #include <stdio.h>
  39 #include <errno.h>
  40
  41 // Internet stuff
  42 #include <netdb.h>
  43
  44 #include "connect.h"
  45 #include "rfc2553emu.h"
  46 #include "http.h"
  47
  48                                                                         /*}}}*/
  49
  50 string HttpMethod::FailFile;
  51 int HttpMethod::FailFd = -1;
  52 time_t HttpMethod::FailTime = 0;
  53 unsigned long PipelineDepth = 10;
  54 unsigned long TimeOut = 120;
  55 bool Debug = false;
  56
  57 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  58 // ---------------------------------------------------------------------
  59 /* */
  60 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  61 {
  62    Buf = new unsigned char[Size];
  63    Reset();
  64 }
  65                                                                         /*}}}*/
  66 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  67 // ---------------------------------------------------------------------
  68 /* */
  69 void CircleBuf::Reset()
  70 {
  71    InP = 0;
  72    OutP = 0;
  73    StrPos = 0;
  74    MaxGet = (unsigned int)-1;
  75    OutQueue = string();
  76    if (MD5 != 0)
  77    {
  78       delete MD5;
  79       MD5 = new MD5Summation;
  80    }
  81 };
  82                                                                         /*}}}*/
  83 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  84 // ---------------------------------------------------------------------
  85 /* This fills up the buffer with as much data as is in the FD, assuming it
  86    is non-blocking.. */
  87 bool CircleBuf::Read(int Fd)
  88 {
  89    while (1)
  90    {
  91       // Woops, buffer is full
  92       if (InP - OutP == Size)
  93          return true;
  94
  95       // Write the buffer segment
  96       int Res;
  97       Res = read(Fd,Buf + (InP%Size),LeftRead());
  98
  99       if (Res == 0)
 100          return false;
 101       if (Res < 0)
 102       {
 103          if (errno == EAGAIN)
 104             return true;
 105          return false;
 106       }
 107
 108       if (InP == 0)
 109          gettimeofday(&Start,0);
 110       InP += Res;
 111    }
 112 }
 113                                                                         /*}}}*/
 114 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 115 // ---------------------------------------------------------------------
 116 /* This will hold the string in and fill the buffer with it as it empties */
 117 bool CircleBuf::Read(string Data)
 118 {
 119    OutQueue += Data;
 120    FillOut();
 121    return true;
 122 }
 123                                                                         /*}}}*/
 124 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 125 // ---------------------------------------------------------------------
 126 /* */
 127 void CircleBuf::FillOut()
 128 {
 129    if (OutQueue.empty() == true)
 130       return;
 131    while (1)
 132    {
 133       // Woops, buffer is full
 134       if (InP - OutP == Size)
 135          return;
 136
 137       // Write the buffer segment
 138       unsigned long Sz = LeftRead();
 139       if (OutQueue.length() - StrPos < Sz)
 140          Sz = OutQueue.length() - StrPos;
 141       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 142
 143       // Advance
 144       StrPos += Sz;
 145       InP += Sz;
 146       if (OutQueue.length() == StrPos)
 147       {
 148          StrPos = 0;
 149          OutQueue = "";
 150          return;
 151       }
 152    }
 153 }
 154                                                                         /*}}}*/
 155 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 156 // ---------------------------------------------------------------------
 157 /* This empties the buffer into the FD. */
 158 bool CircleBuf::Write(int Fd)
 159 {
 160    while (1)
 161    {
 162       FillOut();
 163
 164       // Woops, buffer is empty
 165       if (OutP == InP)
 166          return true;
 167
 168       if (OutP == MaxGet)
 169          return true;
 170
 171       // Write the buffer segment
 172       int Res;
 173       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 174
 175       if (Res == 0)
 176          return false;
 177       if (Res < 0)
 178       {
 179          if (errno == EAGAIN)
 180             return true;
 181
 182          return false;
 183       }
 184
 185       if (MD5 != 0)
 186          MD5->Add(Buf + (OutP%Size),Res);
 187
 188       OutP += Res;
 189    }
 190 }
 191                                                                         /*}}}*/
 192 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 193 // ---------------------------------------------------------------------
 194 /* This copies till the first empty line */
 195 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 196 {
 197    // We cheat and assume it is unneeded to have more than one buffer load
 198    for (unsigned long I = OutP; I < InP; I++)
 199    {
 200       if (Buf[I%Size] != '\n')
 201          continue;
 202       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 203
 204       if (Single == false)
 205       {
 206          if (Buf[I%Size] != '\n')
 207             continue;
 208          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 209       }
 210
 211       if (I > InP)
 212          I = InP;
 213
 214       Data = "";
 215       while (OutP < I)
 216       {
 217          unsigned long Sz = LeftWrite();
 218          if (Sz == 0)
 219             return false;
 220          if (I - OutP < LeftWrite())
 221             Sz = I - OutP;
 222          Data += string((char *)(Buf + (OutP%Size)),Sz);
 223          OutP += Sz;
 224       }
 225       return true;
 226    }
 227    return false;
 228 }
 229                                                                         /*}}}*/
 230 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 231 // ---------------------------------------------------------------------
 232 /* */
 233 void CircleBuf::Stats()
 234 {
 235    if (InP == 0)
 236       return;
 237
 238    struct timeval Stop;
 239    gettimeofday(&Stop,0);
 240 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 241              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 242    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 243 }
 244                                                                         /*}}}*/
 245
 246 // ServerState::ServerState - Constructor                               /*{{{*/
 247 // ---------------------------------------------------------------------
 248 /* */
 249 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 250                         In(64*1024), Out(4*1024),
 251                         ServerName(Srv)
 252 {
 253    Reset();
 254 }
 255                                                                         /*}}}*/
 256 // ServerState::Open - Open a connection to the server                  /*{{{*/
 257 // ---------------------------------------------------------------------
 258 /* This opens a connection to the server. */
 259 bool ServerState::Open()
 260 {
 261    // Use the already open connection if possible.
 262    if (ServerFd != -1)
 263       return true;
 264
 265    Close();
 266    In.Reset();
 267    Out.Reset();
 268    Persistent = true;
 269
 270    // Determine the proxy setting
 271    if (getenv("http_proxy") == 0)
 272    {
 273       string DefProxy = _config->Find("Acquire::http::Proxy");
 274       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 275       if (SpecificProxy.empty() == false)
 276       {
 277          if (SpecificProxy == "DIRECT")
 278             Proxy = "";
 279          else
 280             Proxy = SpecificProxy;
 281       }
 282       else
 283          Proxy = DefProxy;
 284    }
 285    else
 286       Proxy = getenv("http_proxy");
 287
 288    // Parse no_proxy, a , separated list of hosts
 289    if (getenv("no_proxy") != 0)
 290    {
 291       const char *Start = getenv("no_proxy");
 292       for (const char *Cur = Start; true ; Cur++)
 293       {
 294          if (*Cur != ',' && *Cur != 0)
 295             continue;
 296          if (stringcasecmp(ServerName.Host.begin(),ServerName.Host.end(),
 297                            Start,Cur) == 0)
 298          {
 299             Proxy = "";
 300             break;
 301          }
 302
 303          Start = Cur + 1;
 304          if (*Cur == 0)
 305             break;
 306       }
 307    }
 308
 309    // Determine what host and port to use based on the proxy settings
 310    int Port = 0;
 311    string Host;
 312    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 313    {
 314       if (ServerName.Port != 0)
 315          Port = ServerName.Port;
 316       Host = ServerName.Host;
 317    }
 318    else
 319    {
 320       if (Proxy.Port != 0)
 321          Port = Proxy.Port;
 322       Host = Proxy.Host;
 323    }
 324
 325    // Connect to the remote server
 326    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 327       return false;
 328
 329    return true;
 330 }
 331                                                                         /*}}}*/
 332 // ServerState::Close - Close a connection to the server                /*{{{*/
 333 // ---------------------------------------------------------------------
 334 /* */
 335 bool ServerState::Close()
 336 {
 337    close(ServerFd);
 338    ServerFd = -1;
 339    return true;
 340 }
 341                                                                         /*}}}*/
 342 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 343 // ---------------------------------------------------------------------
 344 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 345    parse error occured */
 346 int ServerState::RunHeaders()
 347 {
 348    State = Header;
 349
 350    Owner->Status("Waiting for file");
 351
 352    Major = 0;
 353    Minor = 0;
 354    Result = 0;
 355    Size = 0;
 356    StartPos = 0;
 357    Encoding = Closes;
 358    HaveContent = false;
 359    time(&Date);
 360
 361    do
 362    {
 363       string Data;
 364       if (In.WriteTillEl(Data) == false)
 365          continue;
 366
 367       if (Debug == true)
 368          clog << Data;
 369
 370       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 371       {
 372          string::const_iterator J = I;
 373          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 374          if (HeaderLine(string(I,J-I)) == false)
 375             return 2;
 376          I = J;
 377       }
 378
 379       // 100 Continue is a Nop...
 380       if (Result == 100)
 381          continue;
 382
 383       // Tidy up the connection persistance state.
 384       if (Encoding == Closes && HaveContent == true)
 385          Persistent = false;
 386
 387       return 0;
 388    }
 389    while (Owner->Go(false,this) == true);
 390
 391    return 1;
 392 }
 393                                                                         /*}}}*/
 394 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 395 // ---------------------------------------------------------------------
 396 /* */
 397 bool ServerState::RunData()
 398 {
 399    State = Data;
 400
 401    // Chunked transfer encoding is fun..
 402    if (Encoding == Chunked)
 403    {
 404       while (1)
 405       {
 406          // Grab the block size
 407          bool Last = true;
 408          string Data;
 409          In.Limit(-1);
 410          do
 411          {
 412             if (In.WriteTillEl(Data,true) == true)
 413                break;
 414          }
 415          while ((Last = Owner->Go(false,this)) == true);
 416
 417          if (Last == false)
 418             return false;
 419
 420          // See if we are done
 421          unsigned long Len = strtol(Data.c_str(),0,16);
 422          if (Len == 0)
 423          {
 424             In.Limit(-1);
 425
 426             // We have to remove the entity trailer
 427             Last = true;
 428             do
 429             {
 430                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 431                   break;
 432             }
 433             while ((Last = Owner->Go(false,this)) == true);
 434             if (Last == false)
 435                return false;
 436             return !_error->PendingError();
 437          }
 438
 439          // Transfer the block
 440          In.Limit(Len);
 441          while (Owner->Go(true,this) == true)
 442             if (In.IsLimit() == true)
 443                break;
 444
 445          // Error
 446          if (In.IsLimit() == false)
 447             return false;
 448
 449          // The server sends an extra new line before the next block specifier..
 450          In.Limit(-1);
 451          Last = true;
 452          do
 453          {
 454             if (In.WriteTillEl(Data,true) == true)
 455                break;
 456          }
 457          while ((Last = Owner->Go(false,this)) == true);
 458          if (Last == false)
 459             return false;
 460       }
 461    }
 462    else
 463    {
 464       /* Closes encoding is used when the server did not specify a size, the
 465          loss of the connection means we are done */
 466       if (Encoding == Closes)
 467          In.Limit(-1);
 468       else
 469          In.Limit(Size - StartPos);
 470
 471       // Just transfer the whole block.
 472       do
 473       {
 474          if (In.IsLimit() == false)
 475             continue;
 476
 477          In.Limit(-1);
 478          return !_error->PendingError();
 479       }
 480       while (Owner->Go(true,this) == true);
 481    }
 482
 483    return Owner->Flush(this) && !_error->PendingError();
 484 }
 485                                                                         /*}}}*/
 486 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 487 // ---------------------------------------------------------------------
 488 /* */
 489 bool ServerState::HeaderLine(string Line)
 490 {
 491    if (Line.empty() == true)
 492       return true;
 493
 494    // The http server might be trying to do something evil.
 495    if (Line.length() >= MAXLEN)
 496       return _error->Error("Got a single header line over %u chars",MAXLEN);
 497
 498    string::size_type Pos = Line.find(' ');
 499    if (Pos == string::npos || Pos+1 > Line.length())
 500    {
 501       // Blah, some servers use "connection:closes", evil.
 502       Pos = Line.find(':');
 503       if (Pos == string::npos || Pos + 2 > Line.length())
 504          return _error->Error("Bad header line");
 505       Pos++;
 506    }
 507
 508    // Parse off any trailing spaces between the : and the next word.
 509    string::size_type Pos2 = Pos;
 510    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 511       Pos2++;
 512
 513    string Tag = string(Line,0,Pos);
 514    string Val = string(Line,Pos2);
 515
 516    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 517    {
 518       // Evil servers return no version
 519       if (Line[4] == '/')
 520       {
 521          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 522                     &Result,Code) != 4)
 523             return _error->Error("The http server sent an invalid reply header");
 524       }
 525       else
 526       {
 527          Major = 0;
 528          Minor = 9;
 529          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 530             return _error->Error("The http server sent an invalid reply header");
 531       }
 532
 533       /* Check the HTTP response header to get the default persistance
 534          state. */
 535       if (Major < 1)
 536          Persistent = false;
 537       else
 538       {
 539          if (Major == 1 && Minor <= 0)
 540             Persistent = false;
 541          else
 542             Persistent = true;
 543       }
 544
 545       return true;
 546    }
 547
 548    if (stringcasecmp(Tag,"Content-Length:") == 0)
 549    {
 550       if (Encoding == Closes)
 551          Encoding = Stream;
 552       HaveContent = true;
 553
 554       // The length is already set from the Content-Range header
 555       if (StartPos != 0)
 556          return true;
 557
 558       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 559          return _error->Error("The http server sent an invalid Content-Length header");
 560       return true;
 561    }
 562
 563    if (stringcasecmp(Tag,"Content-Type:") == 0)
 564    {
 565       HaveContent = true;
 566       return true;
 567    }
 568
 569    if (stringcasecmp(Tag,"Content-Range:") == 0)
 570    {
 571       HaveContent = true;
 572
 573       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 574          return _error->Error("The http server sent an invalid Content-Range header");
 575       if ((unsigned)StartPos > Size)
 576          return _error->Error("This http server has broken range support");
 577       return true;
 578    }
 579
 580    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 581    {
 582       HaveContent = true;
 583       if (stringcasecmp(Val,"chunked") == 0)
 584          Encoding = Chunked;
 585       return true;
 586    }
 587
 588    if (stringcasecmp(Tag,"Connection:") == 0)
 589    {
 590       if (stringcasecmp(Val,"close") == 0)
 591          Persistent = false;
 592       if (stringcasecmp(Val,"keep-alive") == 0)
 593          Persistent = true;
 594       return true;
 595    }
 596
 597    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 598    {
 599       if (StrToTime(Val,Date) == false)
 600          return _error->Error("Unknown date format");
 601       return true;
 602    }
 603
 604    return true;
 605 }
 606                                                                         /*}}}*/
 607
 608 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 609 // ---------------------------------------------------------------------
 610 /* This places the http request in the outbound buffer */
 611 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 612 {
 613    URI Uri = Itm->Uri;
 614
 615    // The HTTP server expects a hostname with a trailing :port
 616    char Buf[1000];
 617    string ProperHost = Uri.Host;
 618    if (Uri.Port != 0)
 619    {
 620       sprintf(Buf,":%u",Uri.Port);
 621       ProperHost += Buf;
 622    }
 623
 624    // Just in case.
 625    if (Itm->Uri.length() >= sizeof(Buf))
 626        abort();
 627
 628    /* Build the request. We include a keep-alive header only for non-proxy
 629       requests. This is to tweak old http/1.0 servers that do support keep-alive
 630       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 631       will glitch HTTP/1.0 proxies because they do not filter it out and
 632       pass it on, HTTP/1.1 says the connection should default to keep alive
 633       and we expect the proxy to do this */
 634    if (Proxy.empty() == true)
 635       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 636               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 637    else
 638    {
 639       /* Generate a cache control header if necessary. We place a max
 640          cache age on index files, optionally set a no-cache directive
 641          and a no-store directive for archives. */
 642       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 643               Itm->Uri.c_str(),ProperHost.c_str());
 644       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 645          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 646       else
 647       {
 648          if (Itm->IndexFile == true)
 649             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 650                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 651          else
 652          {
 653             if (_config->FindB("Acquire::http::No-Store",false) == true)
 654                strcat(Buf,"Cache-Control: no-store\r\n");
 655          }
 656       }
 657    }
 658
 659    string Req = Buf;
 660
 661    // Check for a partial file
 662    struct stat SBuf;
 663    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 664    {
 665       // In this case we send an if-range query with a range header
 666       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 667               TimeRFC1123(SBuf.st_mtime).c_str());
 668       Req += Buf;
 669    }
 670    else
 671    {
 672       if (Itm->LastModified != 0)
 673       {
 674          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 675          Req += Buf;
 676       }
 677    }
 678
 679    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 680       Req += string("Proxy-Authorization: Basic ") +
 681           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 682
 683    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 684       Req += string("Authorization: Basic ") +
 685           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 686
 687    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 688
 689    if (Debug == true)
 690       cerr << Req << endl;
 691
 692    Out.Read(Req);
 693 }
 694                                                                         /*}}}*/
 695 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 696 // ---------------------------------------------------------------------
 697 /* This runs the select loop over the server FDs, Output file FDs and
 698    stdin. */
 699 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 700 {
 701    // Server has closed the connection
 702    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 703                                ToFile == false))
 704       return false;
 705
 706    fd_set rfds,wfds;
 707    FD_ZERO(&rfds);
 708    FD_ZERO(&wfds);
 709
 710    /* Add the server. We only send more requests if the connection will
 711       be persisting */
 712    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 713        && Srv->Persistent == true)
 714       FD_SET(Srv->ServerFd,&wfds);
 715    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 716       FD_SET(Srv->ServerFd,&rfds);
 717
 718    // Add the file
 719    int FileFD = -1;
 720    if (File != 0)
 721       FileFD = File->Fd();
 722
 723    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 724       FD_SET(FileFD,&wfds);
 725
 726    // Add stdin
 727    FD_SET(STDIN_FILENO,&rfds);
 728
 729    // Figure out the max fd
 730    int MaxFd = FileFD;
 731    if (MaxFd < Srv->ServerFd)
 732       MaxFd = Srv->ServerFd;
 733
 734    // Select
 735    struct timeval tv;
 736    tv.tv_sec = TimeOut;
 737    tv.tv_usec = 0;
 738    int Res = 0;
 739    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 740       return _error->Errno("select","Select failed");
 741
 742    if (Res == 0)
 743    {
 744       _error->Error("Connection timed out");
 745       return ServerDie(Srv);
 746    }
 747
 748    // Handle server IO
 749    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 750    {
 751       errno = 0;
 752       if (Srv->In.Read(Srv->ServerFd) == false)
 753          return ServerDie(Srv);
 754    }
 755
 756    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 757    {
 758       errno = 0;
 759       if (Srv->Out.Write(Srv->ServerFd) == false)
 760          return ServerDie(Srv);
 761    }
 762
 763    // Send data to the file
 764    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 765    {
 766       if (Srv->In.Write(FileFD) == false)
 767          return _error->Errno("write","Error writing to output file");
 768    }
 769
 770    // Handle commands from APT
 771    if (FD_ISSET(STDIN_FILENO,&rfds))
 772    {
 773       if (Run(true) != -1)
 774          exit(100);
 775    }
 776
 777    return true;
 778 }
 779                                                                         /*}}}*/
 780 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 781 // ---------------------------------------------------------------------
 782 /* This takes the current input buffer from the Server FD and writes it
 783    into the file */
 784 bool HttpMethod::Flush(ServerState *Srv)
 785 {
 786    if (File != 0)
 787    {
 788       SetNonBlock(File->Fd(),false);
 789       if (Srv->In.WriteSpace() == false)
 790          return true;
 791
 792       while (Srv->In.WriteSpace() == true)
 793       {
 794          if (Srv->In.Write(File->Fd()) == false)
 795             return _error->Errno("write","Error writing to file");
 796          if (Srv->In.IsLimit() == true)
 797             return true;
 798       }
 799
 800       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 801          return true;
 802    }
 803    return false;
 804 }
 805                                                                         /*}}}*/
 806 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 807 // ---------------------------------------------------------------------
 808 /* */
 809 bool HttpMethod::ServerDie(ServerState *Srv)
 810 {
 811    unsigned int LErrno = errno;
 812
 813    // Dump the buffer to the file
 814    if (Srv->State == ServerState::Data)
 815    {
 816       SetNonBlock(File->Fd(),false);
 817       while (Srv->In.WriteSpace() == true)
 818       {
 819          if (Srv->In.Write(File->Fd()) == false)
 820             return _error->Errno("write","Error writing to the file");
 821
 822          // Done
 823          if (Srv->In.IsLimit() == true)
 824             return true;
 825       }
 826    }
 827
 828    // See if this is because the server finished the data stream
 829    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 830        Srv->Encoding != ServerState::Closes)
 831    {
 832       Srv->Close();
 833       if (LErrno == 0)
 834          return _error->Error("Error reading from server Remote end closed connection");
 835       errno = LErrno;
 836       return _error->Errno("read","Error reading from server");
 837    }
 838    else
 839    {
 840       Srv->In.Limit(-1);
 841
 842       // Nothing left in the buffer
 843       if (Srv->In.WriteSpace() == false)
 844          return false;
 845
 846       // We may have got multiple responses back in one packet..
 847       Srv->Close();
 848       return true;
 849    }
 850
 851    return false;
 852 }
 853                                                                         /*}}}*/
 854 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 855 // ---------------------------------------------------------------------
 856 /* We look at the header data we got back from the server and decide what
 857    to do. Returns
 858      0 - File is open,
 859      1 - IMS hit
 860      3 - Unrecoverable error
 861      4 - Error with error content page
 862      5 - Unrecoverable non-server error (close the connection) */
 863 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 864 {
 865    // Not Modified
 866    if (Srv->Result == 304)
 867    {
 868       unlink(Queue->DestFile.c_str());
 869       Res.IMSHit = true;
 870       Res.LastModified = Queue->LastModified;
 871       return 1;
 872    }
 873
 874    /* We have a reply we dont handle. This should indicate a perm server
 875       failure */
 876    if (Srv->Result < 200 || Srv->Result >= 300)
 877    {
 878       _error->Error("%u %s",Srv->Result,Srv->Code);
 879       if (Srv->HaveContent == true)
 880          return 4;
 881       return 3;
 882    }
 883
 884    // This is some sort of 2xx 'data follows' reply
 885    Res.LastModified = Srv->Date;
 886    Res.Size = Srv->Size;
 887
 888    // Open the file
 889    delete File;
 890    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 891    if (_error->PendingError() == true)
 892       return 5;
 893
 894    FailFile = Queue->DestFile;
 895    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 896    FailFd = File->Fd();
 897    FailTime = Srv->Date;
 898
 899    // Set the expected size
 900    if (Srv->StartPos >= 0)
 901    {
 902       Res.ResumePoint = Srv->StartPos;
 903       ftruncate(File->Fd(),Srv->StartPos);
 904    }
 905
 906    // Set the start point
 907    lseek(File->Fd(),0,SEEK_END);
 908
 909    delete Srv->In.MD5;
 910    Srv->In.MD5 = new MD5Summation;
 911
 912    // Fill the MD5 Hash if the file is non-empty (resume)
 913    if (Srv->StartPos > 0)
 914    {
 915       lseek(File->Fd(),0,SEEK_SET);
 916       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 917       {
 918          _error->Errno("read","Problem hashing file");
 919          return 5;
 920       }
 921       lseek(File->Fd(),0,SEEK_END);
 922    }
 923
 924    SetNonBlock(File->Fd(),true);
 925    return 0;
 926 }
 927                                                                         /*}}}*/
 928 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 929 // ---------------------------------------------------------------------
 930 /* This closes and timestamps the open file. This is neccessary to get
 931    resume behavoir on user abort */
 932 void HttpMethod::SigTerm(int)
 933 {
 934    if (FailFd == -1)
 935       _exit(100);
 936    close(FailFd);
 937
 938    // Timestamp
 939    struct utimbuf UBuf;
 940    UBuf.actime = FailTime;
 941    UBuf.modtime = FailTime;
 942    utime(FailFile.c_str(),&UBuf);
 943
 944    _exit(100);
 945 }
 946                                                                         /*}}}*/
 947 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 948 // ---------------------------------------------------------------------
 949 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 950    depth. */
 951 bool HttpMethod::Fetch(FetchItem *)
 952 {
 953    if (Server == 0)
 954       return true;
 955
 956    // Queue the requests
 957    int Depth = -1;
 958    bool Tail = false;
 959    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 960         I = I->Next, Depth++)
 961    {
 962       // If pipelining is disabled, we only queue 1 request
 963       if (Server->Pipeline == false && Depth >= 0)
 964          break;
 965
 966       // Make sure we stick with the same server
 967       if (Server->Comp(I->Uri) == false)
 968          break;
 969       if (QueueBack == I)
 970          Tail = true;
 971       if (Tail == true)
 972       {
 973          QueueBack = I->Next;
 974          SendReq(I,Server->Out);
 975          continue;
 976       }
 977    }
 978
 979    return true;
 980 };
 981                                                                         /*}}}*/
 982 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 983 // ---------------------------------------------------------------------
 984 /* We stash the desired pipeline depth */
 985 bool HttpMethod::Configuration(string Message)
 986 {
 987    if (pkgAcqMethod::Configuration(Message) == false)
 988       return false;
 989
 990    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 991    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 992                                   PipelineDepth);
 993    Debug = _config->FindB("Debug::Acquire::http",false);
 994
 995    return true;
 996 }
 997                                                                         /*}}}*/
 998 // HttpMethod::Loop - Main loop                                         /*{{{*/
 999 // ---------------------------------------------------------------------
1000 /* */
1001 int HttpMethod::Loop()
1002 {
1003    signal(SIGTERM,SigTerm);
1004    signal(SIGINT,SigTerm);
1005
1006    Server = 0;
1007
1008    int FailCounter = 0;
1009    while (1)
1010    {
1011       // We have no commands, wait for some to arrive
1012       if (Queue == 0)
1013       {
1014          if (WaitFd(STDIN_FILENO) == false)
1015             return 0;
1016       }
1017
1018       /* Run messages, we can accept 0 (no message) if we didn't
1019          do a WaitFd above.. Otherwise the FD is closed. */
1020       int Result = Run(true);
1021       if (Result != -1 && (Result != 0 || Queue == 0))
1022          return 100;
1023
1024       if (Queue == 0)
1025          continue;
1026
1027       // Connect to the server
1028       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1029       {
1030          delete Server;
1031          Server = new ServerState(Queue->Uri,this);
1032       }
1033
1034       /* If the server has explicitly said this is the last connection
1035          then we pre-emptively shut down the pipeline and tear down
1036          the connection. This will speed up HTTP/1.0 servers a tad
1037          since we don't have to wait for the close sequence to
1038          complete */
1039       if (Server->Persistent == false)
1040          Server->Close();
1041
1042       // Reset the pipeline
1043       if (Server->ServerFd == -1)
1044          QueueBack = Queue;
1045
1046       // Connnect to the host
1047       if (Server->Open() == false)
1048       {
1049          Fail(true);
1050          delete Server;
1051          Server = 0;
1052          continue;
1053       }
1054
1055       // Fill the pipeline.
1056       Fetch(0);
1057
1058       // Fetch the next URL header data from the server.
1059       switch (Server->RunHeaders())
1060       {
1061          case 0:
1062          break;
1063
1064          // The header data is bad
1065          case 2:
1066          {
1067             _error->Error("Bad header Data");
1068             Fail(true);
1069             RotateDNS();
1070             continue;
1071          }
1072
1073          // The server closed a connection during the header get..
1074          default:
1075          case 1:
1076          {
1077             FailCounter++;
1078             _error->Discard();
1079             Server->Close();
1080             Server->Pipeline = false;
1081
1082             if (FailCounter >= 2)
1083             {
1084                Fail("Connection failed",true);
1085                FailCounter = 0;
1086             }
1087
1088             RotateDNS();
1089             continue;
1090          }
1091       };
1092
1093       // Decide what to do.
1094       FetchResult Res;
1095       Res.Filename = Queue->DestFile;
1096       switch (DealWithHeaders(Res,Server))
1097       {
1098          // Ok, the file is Open
1099          case 0:
1100          {
1101             URIStart(Res);
1102
1103             // Run the data
1104             bool Result =  Server->RunData();
1105
1106             /* If the server is sending back sizeless responses then fill in
1107                the size now */
1108             if (Res.Size == 0)
1109                Res.Size = File->Size();
1110
1111             // Close the file, destroy the FD object and timestamp it
1112             FailFd = -1;
1113             delete File;
1114             File = 0;
1115
1116             // Timestamp
1117             struct utimbuf UBuf;
1118             time(&UBuf.actime);
1119             UBuf.actime = Server->Date;
1120             UBuf.modtime = Server->Date;
1121             utime(Queue->DestFile.c_str(),&UBuf);
1122
1123             // Send status to APT
1124             if (Result == true)
1125             {
1126                Res.MD5Sum = Server->In.MD5->Result();
1127                URIDone(Res);
1128             }
1129             else
1130                Fail(true);
1131
1132             break;
1133          }
1134
1135          // IMS hit
1136          case 1:
1137          {
1138             URIDone(Res);
1139             break;
1140          }
1141
1142          // Hard server error, not found or something
1143          case 3:
1144          {
1145             Fail();
1146             break;
1147          }
1148
1149          // Hard internal error, kill the connection and fail
1150          case 5:
1151          {
1152             Fail();
1153             RotateDNS();
1154             Server->Close();
1155             break;
1156          }
1157
1158          // We need to flush the data, the header is like a 404 w/ error text
1159          case 4:
1160          {
1161             Fail();
1162
1163             // Send to content to dev/null
1164             File = new FileFd("/dev/null",FileFd::WriteExists);
1165             Server->RunData();
1166             delete File;
1167             File = 0;
1168             break;
1169          }
1170
1171          default:
1172          Fail("Internal error");
1173          break;
1174       }
1175
1176       FailCounter = 0;
1177    }
1178
1179    return 0;
1180 }
1181                                                                         /*}}}*/
1182
1183 int main()
1184 {
1185    HttpMethod Mth;
1186
1187    return Mth.Loop();
1188 }