methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apt-pkg/fileutl.h>
  29 #include <apt-pkg/acquire-method.h>
  30 #include <apt-pkg/error.h>
  31 #include <apt-pkg/hashes.h>
  32
  33 #include <sys/stat.h>
  34 #include <sys/time.h>
  35 #include <utime.h>
  36 #include <unistd.h>
  37 #include <signal.h>
  38 #include <stdio.h>
  39 #include <errno.h>
  40 #include <string.h>
  41 #include <iostream>
  42 #include <apti18n.h>
  43
  44 // Internet stuff
  45 #include <netdb.h>
  46
  47 #include "connect.h"
  48 #include "rfc2553emu.h"
  49 #include "http.h"
  50
  51                                                                         /*}}}*/
  52 using namespace std;
  53
  54 string HttpMethod::FailFile;
  55 int HttpMethod::FailFd = -1;
  56 time_t HttpMethod::FailTime = 0;
  57 unsigned long PipelineDepth = 10;
  58 unsigned long TimeOut = 120;
  59 bool Debug = false;
  60
  61 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  62 // ---------------------------------------------------------------------
  63 /* */
  64 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
  65 {
  66    Buf = new unsigned char[Size];
  67    Reset();
  68 }
  69                                                                         /*}}}*/
  70 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  71 // ---------------------------------------------------------------------
  72 /* */
  73 void CircleBuf::Reset()
  74 {
  75    InP = 0;
  76    OutP = 0;
  77    StrPos = 0;
  78    MaxGet = (unsigned int)-1;
  79    OutQueue = string();
  80    if (Hash != 0)
  81    {
  82       delete Hash;
  83       Hash = new Hashes;
  84    }
  85 };
  86                                                                         /*}}}*/
  87 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  88 // ---------------------------------------------------------------------
  89 /* This fills up the buffer with as much data as is in the FD, assuming it
  90    is non-blocking.. */
  91 bool CircleBuf::Read(int Fd)
  92 {
  93    while (1)
  94    {
  95       // Woops, buffer is full
  96       if (InP - OutP == Size)
  97          return true;
  98
  99       // Write the buffer segment
 100       int Res;
 101       Res = read(Fd,Buf + (InP%Size),LeftRead());
 102
 103       if (Res == 0)
 104          return false;
 105       if (Res < 0)
 106       {
 107          if (errno == EAGAIN)
 108             return true;
 109          return false;
 110       }
 111
 112       if (InP == 0)
 113          gettimeofday(&Start,0);
 114       InP += Res;
 115    }
 116 }
 117                                                                         /*}}}*/
 118 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 119 // ---------------------------------------------------------------------
 120 /* This will hold the string in and fill the buffer with it as it empties */
 121 bool CircleBuf::Read(string Data)
 122 {
 123    OutQueue += Data;
 124    FillOut();
 125    return true;
 126 }
 127                                                                         /*}}}*/
 128 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 129 // ---------------------------------------------------------------------
 130 /* */
 131 void CircleBuf::FillOut()
 132 {
 133    if (OutQueue.empty() == true)
 134       return;
 135    while (1)
 136    {
 137       // Woops, buffer is full
 138       if (InP - OutP == Size)
 139          return;
 140
 141       // Write the buffer segment
 142       unsigned long Sz = LeftRead();
 143       if (OutQueue.length() - StrPos < Sz)
 144          Sz = OutQueue.length() - StrPos;
 145       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 146
 147       // Advance
 148       StrPos += Sz;
 149       InP += Sz;
 150       if (OutQueue.length() == StrPos)
 151       {
 152          StrPos = 0;
 153          OutQueue = "";
 154          return;
 155       }
 156    }
 157 }
 158                                                                         /*}}}*/
 159 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 160 // ---------------------------------------------------------------------
 161 /* This empties the buffer into the FD. */
 162 bool CircleBuf::Write(int Fd)
 163 {
 164    while (1)
 165    {
 166       FillOut();
 167
 168       // Woops, buffer is empty
 169       if (OutP == InP)
 170          return true;
 171
 172       if (OutP == MaxGet)
 173          return true;
 174
 175       // Write the buffer segment
 176       int Res;
 177       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 178
 179       if (Res == 0)
 180          return false;
 181       if (Res < 0)
 182       {
 183          if (errno == EAGAIN)
 184             return true;
 185
 186          return false;
 187       }
 188
 189       if (Hash != 0)
 190          Hash->Add(Buf + (OutP%Size),Res);
 191
 192       OutP += Res;
 193    }
 194 }
 195                                                                         /*}}}*/
 196 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 197 // ---------------------------------------------------------------------
 198 /* This copies till the first empty line */
 199 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 200 {
 201    // We cheat and assume it is unneeded to have more than one buffer load
 202    for (unsigned long I = OutP; I < InP; I++)
 203    {
 204       if (Buf[I%Size] != '\n')
 205          continue;
 206       ++I;
 207       if (I < InP  && Buf[I%Size] == '\r')
 208          ++I;
 209
 210       if (Single == false)
 211       {
 212          if (Buf[I%Size] != '\n')
 213             continue;
 214          ++I;
 215          if (I < InP  && Buf[I%Size] == '\r')
 216             ++I;
 217       }
 218
 219       if (I > InP)
 220          I = InP;
 221
 222       Data = "";
 223       while (OutP < I)
 224       {
 225          unsigned long Sz = LeftWrite();
 226          if (Sz == 0)
 227             return false;
 228          if (I - OutP < LeftWrite())
 229             Sz = I - OutP;
 230          Data += string((char *)(Buf + (OutP%Size)),Sz);
 231          OutP += Sz;
 232       }
 233       return true;
 234    }
 235    return false;
 236 }
 237                                                                         /*}}}*/
 238 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 239 // ---------------------------------------------------------------------
 240 /* */
 241 void CircleBuf::Stats()
 242 {
 243    if (InP == 0)
 244       return;
 245
 246    struct timeval Stop;
 247    gettimeofday(&Stop,0);
 248 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 249              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 250    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 251 }
 252                                                                         /*}}}*/
 253
 254 // ServerState::ServerState - Constructor                               /*{{{*/
 255 // ---------------------------------------------------------------------
 256 /* */
 257 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 258                         In(64*1024), Out(4*1024),
 259                         ServerName(Srv)
 260 {
 261    Reset();
 262 }
 263                                                                         /*}}}*/
 264 // ServerState::Open - Open a connection to the server                  /*{{{*/
 265 // ---------------------------------------------------------------------
 266 /* This opens a connection to the server. */
 267 bool ServerState::Open()
 268 {
 269    // Use the already open connection if possible.
 270    if (ServerFd != -1)
 271       return true;
 272
 273    Close();
 274    In.Reset();
 275    Out.Reset();
 276    Persistent = true;
 277
 278    // Determine the proxy setting
 279    if (getenv("http_proxy") == 0)
 280    {
 281       string DefProxy = _config->Find("Acquire::http::Proxy");
 282       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 283       if (SpecificProxy.empty() == false)
 284       {
 285          if (SpecificProxy == "DIRECT")
 286             Proxy = "";
 287          else
 288             Proxy = SpecificProxy;
 289       }
 290       else
 291          Proxy = DefProxy;
 292    }
 293    else
 294       Proxy = getenv("http_proxy");
 295
 296    // Parse no_proxy, a , separated list of domains
 297    if (getenv("no_proxy") != 0)
 298    {
 299       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 300          Proxy = "";
 301    }
 302
 303    // Determine what host and port to use based on the proxy settings
 304    int Port = 0;
 305    string Host;
 306    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 307    {
 308       if (ServerName.Port != 0)
 309          Port = ServerName.Port;
 310       Host = ServerName.Host;
 311    }
 312    else
 313    {
 314       if (Proxy.Port != 0)
 315          Port = Proxy.Port;
 316       Host = Proxy.Host;
 317    }
 318
 319    // Connect to the remote server
 320    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 321       return false;
 322
 323    return true;
 324 }
 325                                                                         /*}}}*/
 326 // ServerState::Close - Close a connection to the server                /*{{{*/
 327 // ---------------------------------------------------------------------
 328 /* */
 329 bool ServerState::Close()
 330 {
 331    close(ServerFd);
 332    ServerFd = -1;
 333    return true;
 334 }
 335                                                                         /*}}}*/
 336 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 337 // ---------------------------------------------------------------------
 338 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 339    parse error occured */
 340 int ServerState::RunHeaders()
 341 {
 342    State = Header;
 343
 344    Owner->Status(_("Waiting for headers"));
 345
 346    Major = 0;
 347    Minor = 0;
 348    Result = 0;
 349    Size = 0;
 350    StartPos = 0;
 351    Encoding = Closes;
 352    HaveContent = false;
 353    time(&Date);
 354
 355    do
 356    {
 357       string Data;
 358       if (In.WriteTillEl(Data) == false)
 359          continue;
 360
 361       if (Debug == true)
 362          clog << Data;
 363
 364       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 365       {
 366          string::const_iterator J = I;
 367          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 368          if (HeaderLine(string(I,J)) == false)
 369             return 2;
 370          I = J;
 371       }
 372
 373       // 100 Continue is a Nop...
 374       if (Result == 100)
 375          continue;
 376
 377       // Tidy up the connection persistance state.
 378       if (Encoding == Closes && HaveContent == true)
 379          Persistent = false;
 380
 381       return 0;
 382    }
 383    while (Owner->Go(false,this) == true);
 384
 385    return 1;
 386 }
 387                                                                         /*}}}*/
 388 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 389 // ---------------------------------------------------------------------
 390 /* */
 391 bool ServerState::RunData()
 392 {
 393    State = Data;
 394
 395    // Chunked transfer encoding is fun..
 396    if (Encoding == Chunked)
 397    {
 398       while (1)
 399       {
 400          // Grab the block size
 401          bool Last = true;
 402          string Data;
 403          In.Limit(-1);
 404          do
 405          {
 406             if (In.WriteTillEl(Data,true) == true)
 407                break;
 408          }
 409          while ((Last = Owner->Go(false,this)) == true);
 410
 411          if (Last == false)
 412             return false;
 413
 414          // See if we are done
 415          unsigned long Len = strtol(Data.c_str(),0,16);
 416          if (Len == 0)
 417          {
 418             In.Limit(-1);
 419
 420             // We have to remove the entity trailer
 421             Last = true;
 422             do
 423             {
 424                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 425                   break;
 426             }
 427             while ((Last = Owner->Go(false,this)) == true);
 428             if (Last == false)
 429                return false;
 430             return !_error->PendingError();
 431          }
 432
 433          // Transfer the block
 434          In.Limit(Len);
 435          while (Owner->Go(true,this) == true)
 436             if (In.IsLimit() == true)
 437                break;
 438
 439          // Error
 440          if (In.IsLimit() == false)
 441             return false;
 442
 443          // The server sends an extra new line before the next block specifier..
 444          In.Limit(-1);
 445          Last = true;
 446          do
 447          {
 448             if (In.WriteTillEl(Data,true) == true)
 449                break;
 450          }
 451          while ((Last = Owner->Go(false,this)) == true);
 452          if (Last == false)
 453             return false;
 454       }
 455    }
 456    else
 457    {
 458       /* Closes encoding is used when the server did not specify a size, the
 459          loss of the connection means we are done */
 460       if (Encoding == Closes)
 461          In.Limit(-1);
 462       else
 463          In.Limit(Size - StartPos);
 464
 465       // Just transfer the whole block.
 466       do
 467       {
 468          if (In.IsLimit() == false)
 469             continue;
 470
 471          In.Limit(-1);
 472          return !_error->PendingError();
 473       }
 474       while (Owner->Go(true,this) == true);
 475    }
 476
 477    return Owner->Flush(this) && !_error->PendingError();
 478 }
 479                                                                         /*}}}*/
 480 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 481 // ---------------------------------------------------------------------
 482 /* */
 483 bool ServerState::HeaderLine(string Line)
 484 {
 485    if (Line.empty() == true)
 486       return true;
 487
 488    // The http server might be trying to do something evil.
 489    if (Line.length() >= MAXLEN)
 490       return _error->Error(_("Got a single header line over %u chars"),MAXLEN);
 491
 492    string::size_type Pos = Line.find(' ');
 493    if (Pos == string::npos || Pos+1 > Line.length())
 494    {
 495       // Blah, some servers use "connection:closes", evil.
 496       Pos = Line.find(':');
 497       if (Pos == string::npos || Pos + 2 > Line.length())
 498          return _error->Error(_("Bad header line"));
 499       Pos++;
 500    }
 501
 502    // Parse off any trailing spaces between the : and the next word.
 503    string::size_type Pos2 = Pos;
 504    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 505       Pos2++;
 506
 507    string Tag = string(Line,0,Pos);
 508    string Val = string(Line,Pos2);
 509
 510    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 511    {
 512       // Evil servers return no version
 513       if (Line[4] == '/')
 514       {
 515          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 516                     &Result,Code) != 4)
 517             return _error->Error(_("The http server sent an invalid reply header"));
 518       }
 519       else
 520       {
 521          Major = 0;
 522          Minor = 9;
 523          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 524             return _error->Error(_("The http server sent an invalid reply header"));
 525       }
 526
 527       /* Check the HTTP response header to get the default persistance
 528          state. */
 529       if (Major < 1)
 530          Persistent = false;
 531       else
 532       {
 533          if (Major == 1 && Minor <= 0)
 534             Persistent = false;
 535          else
 536             Persistent = true;
 537       }
 538
 539       return true;
 540    }
 541
 542    if (stringcasecmp(Tag,"Content-Length:") == 0)
 543    {
 544       if (Encoding == Closes)
 545          Encoding = Stream;
 546       HaveContent = true;
 547
 548       // The length is already set from the Content-Range header
 549       if (StartPos != 0)
 550          return true;
 551
 552       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 553          return _error->Error(_("The http server sent an invalid Content-Length header"));
 554       return true;
 555    }
 556
 557    if (stringcasecmp(Tag,"Content-Type:") == 0)
 558    {
 559       HaveContent = true;
 560       return true;
 561    }
 562
 563    if (stringcasecmp(Tag,"Content-Range:") == 0)
 564    {
 565       HaveContent = true;
 566
 567       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 568          return _error->Error(_("The http server sent an invalid Content-Range header"));
 569       if ((unsigned)StartPos > Size)
 570          return _error->Error(_("This http server has broken range support"));
 571       return true;
 572    }
 573
 574    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 575    {
 576       HaveContent = true;
 577       if (stringcasecmp(Val,"chunked") == 0)
 578          Encoding = Chunked;
 579       return true;
 580    }
 581
 582    if (stringcasecmp(Tag,"Connection:") == 0)
 583    {
 584       if (stringcasecmp(Val,"close") == 0)
 585          Persistent = false;
 586       if (stringcasecmp(Val,"keep-alive") == 0)
 587          Persistent = true;
 588       return true;
 589    }
 590
 591    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 592    {
 593       if (StrToTime(Val,Date) == false)
 594          return _error->Error(_("Unknown date format"));
 595       return true;
 596    }
 597
 598    return true;
 599 }
 600                                                                         /*}}}*/
 601
 602 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 603 // ---------------------------------------------------------------------
 604 /* This places the http request in the outbound buffer */
 605 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 606 {
 607    URI Uri = Itm->Uri;
 608
 609    // The HTTP server expects a hostname with a trailing :port
 610    char Buf[1000];
 611    string ProperHost = Uri.Host;
 612    if (Uri.Port != 0)
 613    {
 614       sprintf(Buf,":%u",Uri.Port);
 615       ProperHost += Buf;
 616    }
 617
 618    // Just in case.
 619    if (Itm->Uri.length() >= sizeof(Buf))
 620        abort();
 621
 622    /* Build the request. We include a keep-alive header only for non-proxy
 623       requests. This is to tweak old http/1.0 servers that do support keep-alive
 624       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 625       will glitch HTTP/1.0 proxies because they do not filter it out and
 626       pass it on, HTTP/1.1 says the connection should default to keep alive
 627       and we expect the proxy to do this */
 628    if (Proxy.empty() == true)
 629       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 630               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 631    else
 632    {
 633       /* Generate a cache control header if necessary. We place a max
 634          cache age on index files, optionally set a no-cache directive
 635          and a no-store directive for archives. */
 636       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 637               Itm->Uri.c_str(),ProperHost.c_str());
 638       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 639          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 640       else
 641       {
 642          if (Itm->IndexFile == true)
 643             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 644                     _config->FindI("Acquire::http::Max-Age",0));
 645          else
 646          {
 647             if (_config->FindB("Acquire::http::No-Store",false) == true)
 648                strcat(Buf,"Cache-Control: no-store\r\n");
 649          }
 650       }
 651    }
 652
 653    string Req = Buf;
 654
 655    // Check for a partial file
 656    struct stat SBuf;
 657    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 658    {
 659       // In this case we send an if-range query with a range header
 660       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 661               TimeRFC1123(SBuf.st_mtime).c_str());
 662       Req += Buf;
 663    }
 664    else
 665    {
 666       if (Itm->LastModified != 0)
 667       {
 668          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 669          Req += Buf;
 670       }
 671    }
 672
 673    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 674       Req += string("Proxy-Authorization: Basic ") +
 675           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 676
 677    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 678       Req += string("Authorization: Basic ") +
 679           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 680
 681    Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
 682
 683    if (Debug == true)
 684       cerr << Req << endl;
 685
 686    Out.Read(Req);
 687 }
 688                                                                         /*}}}*/
 689 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 690 // ---------------------------------------------------------------------
 691 /* This runs the select loop over the server FDs, Output file FDs and
 692    stdin. */
 693 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 694 {
 695    // Server has closed the connection
 696    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 697                                ToFile == false))
 698       return false;
 699
 700    fd_set rfds,wfds;
 701    FD_ZERO(&rfds);
 702    FD_ZERO(&wfds);
 703
 704    /* Add the server. We only send more requests if the connection will
 705       be persisting */
 706    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 707        && Srv->Persistent == true)
 708       FD_SET(Srv->ServerFd,&wfds);
 709    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 710       FD_SET(Srv->ServerFd,&rfds);
 711
 712    // Add the file
 713    int FileFD = -1;
 714    if (File != 0)
 715       FileFD = File->Fd();
 716
 717    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 718       FD_SET(FileFD,&wfds);
 719
 720    // Add stdin
 721    FD_SET(STDIN_FILENO,&rfds);
 722
 723    // Figure out the max fd
 724    int MaxFd = FileFD;
 725    if (MaxFd < Srv->ServerFd)
 726       MaxFd = Srv->ServerFd;
 727
 728    // Select
 729    struct timeval tv;
 730    tv.tv_sec = TimeOut;
 731    tv.tv_usec = 0;
 732    int Res = 0;
 733    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 734    {
 735       if (errno == EINTR)
 736          return true;
 737       return _error->Errno("select",_("Select failed"));
 738    }
 739
 740    if (Res == 0)
 741    {
 742       _error->Error(_("Connection timed out"));
 743       return ServerDie(Srv);
 744    }
 745
 746    // Handle server IO
 747    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 748    {
 749       errno = 0;
 750       if (Srv->In.Read(Srv->ServerFd) == false)
 751          return ServerDie(Srv);
 752    }
 753
 754    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 755    {
 756       errno = 0;
 757       if (Srv->Out.Write(Srv->ServerFd) == false)
 758          return ServerDie(Srv);
 759    }
 760
 761    // Send data to the file
 762    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 763    {
 764       if (Srv->In.Write(FileFD) == false)
 765          return _error->Errno("write",_("Error writing to output file"));
 766    }
 767
 768    // Handle commands from APT
 769    if (FD_ISSET(STDIN_FILENO,&rfds))
 770    {
 771       if (Run(true) != -1)
 772          exit(100);
 773    }
 774
 775    return true;
 776 }
 777                                                                         /*}}}*/
 778 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 779 // ---------------------------------------------------------------------
 780 /* This takes the current input buffer from the Server FD and writes it
 781    into the file */
 782 bool HttpMethod::Flush(ServerState *Srv)
 783 {
 784    if (File != 0)
 785    {
 786       SetNonBlock(File->Fd(),false);
 787       if (Srv->In.WriteSpace() == false)
 788          return true;
 789
 790       while (Srv->In.WriteSpace() == true)
 791       {
 792          if (Srv->In.Write(File->Fd()) == false)
 793             return _error->Errno("write",_("Error writing to file"));
 794          if (Srv->In.IsLimit() == true)
 795             return true;
 796       }
 797
 798       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 799          return true;
 800    }
 801    return false;
 802 }
 803                                                                         /*}}}*/
 804 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 805 // ---------------------------------------------------------------------
 806 /* */
 807 bool HttpMethod::ServerDie(ServerState *Srv)
 808 {
 809    unsigned int LErrno = errno;
 810
 811    // Dump the buffer to the file
 812    if (Srv->State == ServerState::Data)
 813    {
 814       SetNonBlock(File->Fd(),false);
 815       while (Srv->In.WriteSpace() == true)
 816       {
 817          if (Srv->In.Write(File->Fd()) == false)
 818             return _error->Errno("write",_("Error writing to the file"));
 819
 820          // Done
 821          if (Srv->In.IsLimit() == true)
 822             return true;
 823       }
 824    }
 825
 826    // See if this is because the server finished the data stream
 827    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 828        Srv->Encoding != ServerState::Closes)
 829    {
 830       Srv->Close();
 831       if (LErrno == 0)
 832          return _error->Error(_("Error reading from server Remote end closed connection"));
 833       errno = LErrno;
 834       return _error->Errno("read",_("Error reading from server"));
 835    }
 836    else
 837    {
 838       Srv->In.Limit(-1);
 839
 840       // Nothing left in the buffer
 841       if (Srv->In.WriteSpace() == false)
 842          return false;
 843
 844       // We may have got multiple responses back in one packet..
 845       Srv->Close();
 846       return true;
 847    }
 848
 849    return false;
 850 }
 851                                                                         /*}}}*/
 852 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 853 // ---------------------------------------------------------------------
 854 /* We look at the header data we got back from the server and decide what
 855    to do. Returns
 856      0 - File is open,
 857      1 - IMS hit
 858      3 - Unrecoverable error
 859      4 - Error with error content page
 860      5 - Unrecoverable non-server error (close the connection) */
 861 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 862 {
 863    // Not Modified
 864    if (Srv->Result == 304)
 865    {
 866       unlink(Queue->DestFile.c_str());
 867       Res.IMSHit = true;
 868       Res.LastModified = Queue->LastModified;
 869       return 1;
 870    }
 871
 872    /* We have a reply we dont handle. This should indicate a perm server
 873       failure */
 874    if (Srv->Result < 200 || Srv->Result >= 300)
 875    {
 876       _error->Error("%u %s",Srv->Result,Srv->Code);
 877       if (Srv->HaveContent == true)
 878          return 4;
 879       return 3;
 880    }
 881
 882    // This is some sort of 2xx 'data follows' reply
 883    Res.LastModified = Srv->Date;
 884    Res.Size = Srv->Size;
 885
 886    // Open the file
 887    delete File;
 888    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 889    if (_error->PendingError() == true)
 890       return 5;
 891
 892    FailFile = Queue->DestFile;
 893    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 894    FailFd = File->Fd();
 895    FailTime = Srv->Date;
 896
 897    // Set the expected size
 898    if (Srv->StartPos >= 0)
 899    {
 900       Res.ResumePoint = Srv->StartPos;
 901       ftruncate(File->Fd(),Srv->StartPos);
 902    }
 903
 904    // Set the start point
 905    lseek(File->Fd(),0,SEEK_END);
 906
 907    delete Srv->In.Hash;
 908    Srv->In.Hash = new Hashes;
 909
 910    // Fill the Hash if the file is non-empty (resume)
 911    if (Srv->StartPos > 0)
 912    {
 913       lseek(File->Fd(),0,SEEK_SET);
 914       if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
 915       {
 916          _error->Errno("read",_("Problem hashing file"));
 917          return 5;
 918       }
 919       lseek(File->Fd(),0,SEEK_END);
 920    }
 921
 922    SetNonBlock(File->Fd(),true);
 923    return 0;
 924 }
 925                                                                         /*}}}*/
 926 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 927 // ---------------------------------------------------------------------
 928 /* This closes and timestamps the open file. This is neccessary to get
 929    resume behavoir on user abort */
 930 void HttpMethod::SigTerm(int)
 931 {
 932    if (FailFd == -1)
 933       _exit(100);
 934    close(FailFd);
 935
 936    // Timestamp
 937    struct utimbuf UBuf;
 938    UBuf.actime = FailTime;
 939    UBuf.modtime = FailTime;
 940    utime(FailFile.c_str(),&UBuf);
 941
 942    _exit(100);
 943 }
 944                                                                         /*}}}*/
 945 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 946 // ---------------------------------------------------------------------
 947 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 948    depth. */
 949 bool HttpMethod::Fetch(FetchItem *)
 950 {
 951    if (Server == 0)
 952       return true;
 953
 954    // Queue the requests
 955    int Depth = -1;
 956    bool Tail = false;
 957    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 958         I = I->Next, Depth++)
 959    {
 960       // If pipelining is disabled, we only queue 1 request
 961       if (Server->Pipeline == false && Depth >= 0)
 962          break;
 963
 964       // Make sure we stick with the same server
 965       if (Server->Comp(I->Uri) == false)
 966          break;
 967       if (QueueBack == I)
 968          Tail = true;
 969       if (Tail == true)
 970       {
 971          QueueBack = I->Next;
 972          SendReq(I,Server->Out);
 973          continue;
 974       }
 975    }
 976
 977    return true;
 978 };
 979                                                                         /*}}}*/
 980 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 981 // ---------------------------------------------------------------------
 982 /* We stash the desired pipeline depth */
 983 bool HttpMethod::Configuration(string Message)
 984 {
 985    if (pkgAcqMethod::Configuration(Message) == false)
 986       return false;
 987
 988    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 989    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 990                                   PipelineDepth);
 991    Debug = _config->FindB("Debug::Acquire::http",false);
 992
 993    return true;
 994 }
 995                                                                         /*}}}*/
 996 // HttpMethod::Loop - Main loop                                         /*{{{*/
 997 // ---------------------------------------------------------------------
 998 /* */
 999 int HttpMethod::Loop()
1000 {
1001    signal(SIGTERM,SigTerm);
1002    signal(SIGINT,SigTerm);
1003
1004    Server = 0;
1005
1006    int FailCounter = 0;
1007    while (1)
1008    {
1009       // We have no commands, wait for some to arrive
1010       if (Queue == 0)
1011       {
1012          if (WaitFd(STDIN_FILENO) == false)
1013             return 0;
1014       }
1015
1016       /* Run messages, we can accept 0 (no message) if we didn't
1017          do a WaitFd above.. Otherwise the FD is closed. */
1018       int Result = Run(true);
1019       if (Result != -1 && (Result != 0 || Queue == 0))
1020          return 100;
1021
1022       if (Queue == 0)
1023          continue;
1024
1025       // Connect to the server
1026       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1027       {
1028          delete Server;
1029          Server = new ServerState(Queue->Uri,this);
1030       }
1031
1032       /* If the server has explicitly said this is the last connection
1033          then we pre-emptively shut down the pipeline and tear down
1034          the connection. This will speed up HTTP/1.0 servers a tad
1035          since we don't have to wait for the close sequence to
1036          complete */
1037       if (Server->Persistent == false)
1038          Server->Close();
1039
1040       // Reset the pipeline
1041       if (Server->ServerFd == -1)
1042          QueueBack = Queue;
1043
1044       // Connnect to the host
1045       if (Server->Open() == false)
1046       {
1047          Fail(true);
1048          delete Server;
1049          Server = 0;
1050          continue;
1051       }
1052
1053       // Fill the pipeline.
1054       Fetch(0);
1055
1056       // Fetch the next URL header data from the server.
1057       switch (Server->RunHeaders())
1058       {
1059          case 0:
1060          break;
1061
1062          // The header data is bad
1063          case 2:
1064          {
1065             _error->Error(_("Bad header Data"));
1066             Fail(true);
1067             RotateDNS();
1068             continue;
1069          }
1070
1071          // The server closed a connection during the header get..
1072          default:
1073          case 1:
1074          {
1075             FailCounter++;
1076             _error->Discard();
1077             Server->Close();
1078             Server->Pipeline = false;
1079
1080             if (FailCounter >= 2)
1081             {
1082                Fail(_("Connection failed"),true);
1083                FailCounter = 0;
1084             }
1085
1086             RotateDNS();
1087             continue;
1088          }
1089       };
1090
1091       // Decide what to do.
1092       FetchResult Res;
1093       Res.Filename = Queue->DestFile;
1094       switch (DealWithHeaders(Res,Server))
1095       {
1096          // Ok, the file is Open
1097          case 0:
1098          {
1099             URIStart(Res);
1100
1101             // Run the data
1102             bool Result =  Server->RunData();
1103
1104             /* If the server is sending back sizeless responses then fill in
1105                the size now */
1106             if (Res.Size == 0)
1107                Res.Size = File->Size();
1108
1109             // Close the file, destroy the FD object and timestamp it
1110             FailFd = -1;
1111             delete File;
1112             File = 0;
1113
1114             // Timestamp
1115             struct utimbuf UBuf;
1116             time(&UBuf.actime);
1117             UBuf.actime = Server->Date;
1118             UBuf.modtime = Server->Date;
1119             utime(Queue->DestFile.c_str(),&UBuf);
1120
1121             // Send status to APT
1122             if (Result == true)
1123             {
1124                Res.TakeHashes(*Server->In.Hash);
1125                URIDone(Res);
1126             }
1127             else
1128                Fail(true);
1129
1130             break;
1131          }
1132
1133          // IMS hit
1134          case 1:
1135          {
1136             URIDone(Res);
1137             break;
1138          }
1139
1140          // Hard server error, not found or something
1141          case 3:
1142          {
1143             Fail();
1144             break;
1145          }
1146
1147          // Hard internal error, kill the connection and fail
1148          case 5:
1149          {
1150             delete File;
1151             File = 0;
1152
1153             Fail();
1154             RotateDNS();
1155             Server->Close();
1156             break;
1157          }
1158
1159          // We need to flush the data, the header is like a 404 w/ error text
1160          case 4:
1161          {
1162             Fail();
1163
1164             // Send to content to dev/null
1165             File = new FileFd("/dev/null",FileFd::WriteExists);
1166             Server->RunData();
1167             delete File;
1168             File = 0;
1169             break;
1170          }
1171
1172          default:
1173          Fail(_("Internal error"));
1174          break;
1175       }
1176
1177       FailCounter = 0;
1178    }
1179
1180    return 0;
1181 }
1182                                                                         /*}}}*/
1183
1184 int main()
1185 {
1186    setlocale(LC_ALL, "");
1187
1188    HttpMethod Mth;
1189
1190    return Mth.Loop();
1191 }
1192
1193