methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apt-pkg/fileutl.h>
  29 #include <apt-pkg/acquire-method.h>
  30 #include <apt-pkg/error.h>
  31 #include <apt-pkg/hashes.h>
  32
  33 #include <sys/stat.h>
  34 #include <sys/time.h>
  35 #include <utime.h>
  36 #include <unistd.h>
  37 #include <signal.h>
  38 #include <stdio.h>
  39 #include <errno.h>
  40 #include <string.h>
  41 #include <iostream>
  42 #include <apti18n.h>
  43
  44 // Internet stuff
  45 #include <netdb.h>
  46
  47 #include "connect.h"
  48 #include "rfc2553emu.h"
  49 #include "http.h"
  50
  51                                                                         /*}}}*/
  52 using namespace std;
  53
  54 string HttpMethod::FailFile;
  55 int HttpMethod::FailFd = -1;
  56 time_t HttpMethod::FailTime = 0;
  57 unsigned long PipelineDepth = 10;
  58 unsigned long TimeOut = 120;
  59 bool Debug = false;
  60
  61
  62 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  63 // ---------------------------------------------------------------------
  64 /* */
  65 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
  66 {
  67    Buf = new unsigned char[Size];
  68    Reset();
  69 }
  70                                                                         /*}}}*/
  71 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  72 // ---------------------------------------------------------------------
  73 /* */
  74 void CircleBuf::Reset()
  75 {
  76    InP = 0;
  77    OutP = 0;
  78    StrPos = 0;
  79    MaxGet = (unsigned int)-1;
  80    OutQueue = string();
  81    if (Hash != 0)
  82    {
  83       delete Hash;
  84       Hash = new Hashes;
  85    }
  86 };
  87                                                                         /*}}}*/
  88 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  89 // ---------------------------------------------------------------------
  90 /* This fills up the buffer with as much data as is in the FD, assuming it
  91    is non-blocking.. */
  92 bool CircleBuf::Read(int Fd)
  93 {
  94    while (1)
  95    {
  96       // Woops, buffer is full
  97       if (InP - OutP == Size)
  98          return true;
  99
 100       // Write the buffer segment
 101       int Res;
 102       Res = read(Fd,Buf + (InP%Size),LeftRead());
 103
 104       if (Res == 0)
 105          return false;
 106       if (Res < 0)
 107       {
 108          if (errno == EAGAIN)
 109             return true;
 110          return false;
 111       }
 112
 113       if (InP == 0)
 114          gettimeofday(&Start,0);
 115       InP += Res;
 116    }
 117 }
 118                                                                         /*}}}*/
 119 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 120 // ---------------------------------------------------------------------
 121 /* This will hold the string in and fill the buffer with it as it empties */
 122 bool CircleBuf::Read(string Data)
 123 {
 124    OutQueue += Data;
 125    FillOut();
 126    return true;
 127 }
 128                                                                         /*}}}*/
 129 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 130 // ---------------------------------------------------------------------
 131 /* */
 132 void CircleBuf::FillOut()
 133 {
 134    if (OutQueue.empty() == true)
 135       return;
 136    while (1)
 137    {
 138       // Woops, buffer is full
 139       if (InP - OutP == Size)
 140          return;
 141
 142       // Write the buffer segment
 143       unsigned long Sz = LeftRead();
 144       if (OutQueue.length() - StrPos < Sz)
 145          Sz = OutQueue.length() - StrPos;
 146       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 147
 148       // Advance
 149       StrPos += Sz;
 150       InP += Sz;
 151       if (OutQueue.length() == StrPos)
 152       {
 153          StrPos = 0;
 154          OutQueue = "";
 155          return;
 156       }
 157    }
 158 }
 159                                                                         /*}}}*/
 160 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 161 // ---------------------------------------------------------------------
 162 /* This empties the buffer into the FD. */
 163 bool CircleBuf::Write(int Fd)
 164 {
 165    while (1)
 166    {
 167       FillOut();
 168
 169       // Woops, buffer is empty
 170       if (OutP == InP)
 171          return true;
 172
 173       if (OutP == MaxGet)
 174          return true;
 175
 176       // Write the buffer segment
 177       int Res;
 178       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 179
 180       if (Res == 0)
 181          return false;
 182       if (Res < 0)
 183       {
 184          if (errno == EAGAIN)
 185             return true;
 186
 187          return false;
 188       }
 189
 190       if (Hash != 0)
 191          Hash->Add(Buf + (OutP%Size),Res);
 192
 193       OutP += Res;
 194    }
 195 }
 196                                                                         /*}}}*/
 197 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 198 // ---------------------------------------------------------------------
 199 /* This copies till the first empty line */
 200 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 201 {
 202    // We cheat and assume it is unneeded to have more than one buffer load
 203    for (unsigned long I = OutP; I < InP; I++)
 204    {
 205       if (Buf[I%Size] != '\n')
 206          continue;
 207       ++I;
 208
 209       if (Single == false)
 210       {
 211          if (I < InP  && Buf[I%Size] == '\r')
 212             ++I;
 213          if (I >= InP || Buf[I%Size] != '\n')
 214             continue;
 215          ++I;
 216       }
 217
 218       Data = "";
 219       while (OutP < I)
 220       {
 221          unsigned long Sz = LeftWrite();
 222          if (Sz == 0)
 223             return false;
 224          if (I - OutP < Sz)
 225             Sz = I - OutP;
 226          Data += string((char *)(Buf + (OutP%Size)),Sz);
 227          OutP += Sz;
 228       }
 229       return true;
 230    }
 231    return false;
 232 }
 233                                                                         /*}}}*/
 234 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 235 // ---------------------------------------------------------------------
 236 /* */
 237 void CircleBuf::Stats()
 238 {
 239    if (InP == 0)
 240       return;
 241
 242    struct timeval Stop;
 243    gettimeofday(&Stop,0);
 244 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 245              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 246    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 247 }
 248                                                                         /*}}}*/
 249
 250 // ServerState::ServerState - Constructor                               /*{{{*/
 251 // ---------------------------------------------------------------------
 252 /* */
 253 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 254                         In(64*1024), Out(4*1024),
 255                         ServerName(Srv)
 256 {
 257    Reset();
 258 }
 259                                                                         /*}}}*/
 260 // ServerState::Open - Open a connection to the server                  /*{{{*/
 261 // ---------------------------------------------------------------------
 262 /* This opens a connection to the server. */
 263 bool ServerState::Open()
 264 {
 265    // Use the already open connection if possible.
 266    if (ServerFd != -1)
 267       return true;
 268
 269    Close();
 270    In.Reset();
 271    Out.Reset();
 272    Persistent = true;
 273
 274    // Determine the proxy setting
 275    if (getenv("http_proxy") == 0)
 276    {
 277       string DefProxy = _config->Find("Acquire::http::Proxy");
 278       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 279       if (SpecificProxy.empty() == false)
 280       {
 281          if (SpecificProxy == "DIRECT")
 282             Proxy = "";
 283          else
 284             Proxy = SpecificProxy;
 285       }
 286       else
 287          Proxy = DefProxy;
 288    }
 289    else
 290       Proxy = getenv("http_proxy");
 291
 292    // Parse no_proxy, a , separated list of domains
 293    if (getenv("no_proxy") != 0)
 294    {
 295       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 296          Proxy = "";
 297    }
 298
 299    // Determine what host and port to use based on the proxy settings
 300    int Port = 0;
 301    string Host;
 302    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 303    {
 304       if (ServerName.Port != 0)
 305          Port = ServerName.Port;
 306       Host = ServerName.Host;
 307    }
 308    else
 309    {
 310       if (Proxy.Port != 0)
 311          Port = Proxy.Port;
 312       Host = Proxy.Host;
 313    }
 314
 315    // Connect to the remote server
 316    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 317       return false;
 318
 319    return true;
 320 }
 321                                                                         /*}}}*/
 322 // ServerState::Close - Close a connection to the server                /*{{{*/
 323 // ---------------------------------------------------------------------
 324 /* */
 325 bool ServerState::Close()
 326 {
 327    close(ServerFd);
 328    ServerFd = -1;
 329    return true;
 330 }
 331                                                                         /*}}}*/
 332 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 333 // ---------------------------------------------------------------------
 334 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 335    parse error occured */
 336 int ServerState::RunHeaders()
 337 {
 338    State = Header;
 339
 340    Owner->Status(_("Waiting for headers"));
 341
 342    Major = 0;
 343    Minor = 0;
 344    Result = 0;
 345    Size = 0;
 346    StartPos = 0;
 347    Encoding = Closes;
 348    HaveContent = false;
 349    time(&Date);
 350
 351    do
 352    {
 353       string Data;
 354       if (In.WriteTillEl(Data) == false)
 355          continue;
 356
 357       if (Debug == true)
 358          clog << Data;
 359
 360       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 361       {
 362          string::const_iterator J = I;
 363          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 364          if (HeaderLine(string(I,J)) == false)
 365             return 2;
 366          I = J;
 367       }
 368
 369       // 100 Continue is a Nop...
 370       if (Result == 100)
 371          continue;
 372
 373       // Tidy up the connection persistance state.
 374       if (Encoding == Closes && HaveContent == true)
 375          Persistent = false;
 376
 377       return 0;
 378    }
 379    while (Owner->Go(false,this) == true);
 380
 381    return 1;
 382 }
 383                                                                         /*}}}*/
 384 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 385 // ---------------------------------------------------------------------
 386 /* */
 387 bool ServerState::RunData()
 388 {
 389    State = Data;
 390
 391    // Chunked transfer encoding is fun..
 392    if (Encoding == Chunked)
 393    {
 394       while (1)
 395       {
 396          // Grab the block size
 397          bool Last = true;
 398          string Data;
 399          In.Limit(-1);
 400          do
 401          {
 402             if (In.WriteTillEl(Data,true) == true)
 403                break;
 404          }
 405          while ((Last = Owner->Go(false,this)) == true);
 406
 407          if (Last == false)
 408             return false;
 409
 410          // See if we are done
 411          unsigned long Len = strtol(Data.c_str(),0,16);
 412          if (Len == 0)
 413          {
 414             In.Limit(-1);
 415
 416             // We have to remove the entity trailer
 417             Last = true;
 418             do
 419             {
 420                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 421                   break;
 422             }
 423             while ((Last = Owner->Go(false,this)) == true);
 424             if (Last == false)
 425                return false;
 426             return !_error->PendingError();
 427          }
 428
 429          // Transfer the block
 430          In.Limit(Len);
 431          while (Owner->Go(true,this) == true)
 432             if (In.IsLimit() == true)
 433                break;
 434
 435          // Error
 436          if (In.IsLimit() == false)
 437             return false;
 438
 439          // The server sends an extra new line before the next block specifier..
 440          In.Limit(-1);
 441          Last = true;
 442          do
 443          {
 444             if (In.WriteTillEl(Data,true) == true)
 445                break;
 446          }
 447          while ((Last = Owner->Go(false,this)) == true);
 448          if (Last == false)
 449             return false;
 450       }
 451    }
 452    else
 453    {
 454       /* Closes encoding is used when the server did not specify a size, the
 455          loss of the connection means we are done */
 456       if (Encoding == Closes)
 457          In.Limit(-1);
 458       else
 459          In.Limit(Size - StartPos);
 460
 461       // Just transfer the whole block.
 462       do
 463       {
 464          if (In.IsLimit() == false)
 465             continue;
 466
 467          In.Limit(-1);
 468          return !_error->PendingError();
 469       }
 470       while (Owner->Go(true,this) == true);
 471    }
 472
 473    return Owner->Flush(this) && !_error->PendingError();
 474 }
 475                                                                         /*}}}*/
 476 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 477 // ---------------------------------------------------------------------
 478 /* */
 479 bool ServerState::HeaderLine(string Line)
 480 {
 481    if (Line.empty() == true)
 482       return true;
 483
 484    // The http server might be trying to do something evil.
 485    if (Line.length() >= MAXLEN)
 486       return _error->Error(_("Got a single header line over %u chars"),MAXLEN);
 487
 488    string::size_type Pos = Line.find(' ');
 489    if (Pos == string::npos || Pos+1 > Line.length())
 490    {
 491       // Blah, some servers use "connection:closes", evil.
 492       Pos = Line.find(':');
 493       if (Pos == string::npos || Pos + 2 > Line.length())
 494          return _error->Error(_("Bad header line"));
 495       Pos++;
 496    }
 497
 498    // Parse off any trailing spaces between the : and the next word.
 499    string::size_type Pos2 = Pos;
 500    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 501       Pos2++;
 502
 503    string Tag = string(Line,0,Pos);
 504    string Val = string(Line,Pos2);
 505
 506    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 507    {
 508       // Evil servers return no version
 509       if (Line[4] == '/')
 510       {
 511          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 512                     &Result,Code) != 4)
 513             return _error->Error(_("The HTTP server sent an invalid reply header"));
 514       }
 515       else
 516       {
 517          Major = 0;
 518          Minor = 9;
 519          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 520             return _error->Error(_("The HTTP server sent an invalid reply header"));
 521       }
 522
 523       /* Check the HTTP response header to get the default persistance
 524          state. */
 525       if (Major < 1)
 526          Persistent = false;
 527       else
 528       {
 529          if (Major == 1 && Minor <= 0)
 530             Persistent = false;
 531          else
 532             Persistent = true;
 533       }
 534
 535       return true;
 536    }
 537
 538    if (stringcasecmp(Tag,"Content-Length:") == 0)
 539    {
 540       if (Encoding == Closes)
 541          Encoding = Stream;
 542       HaveContent = true;
 543
 544       // The length is already set from the Content-Range header
 545       if (StartPos != 0)
 546          return true;
 547
 548       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 549          return _error->Error(_("The HTTP server sent an invalid Content-Length header"));
 550       return true;
 551    }
 552
 553    if (stringcasecmp(Tag,"Content-Type:") == 0)
 554    {
 555       HaveContent = true;
 556       return true;
 557    }
 558
 559    if (stringcasecmp(Tag,"Content-Range:") == 0)
 560    {
 561       HaveContent = true;
 562
 563       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 564          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 565       if ((unsigned)StartPos > Size)
 566          return _error->Error(_("This HTTP server has broken range support"));
 567       return true;
 568    }
 569
 570    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 571    {
 572       HaveContent = true;
 573       if (stringcasecmp(Val,"chunked") == 0)
 574          Encoding = Chunked;
 575       return true;
 576    }
 577
 578    if (stringcasecmp(Tag,"Connection:") == 0)
 579    {
 580       if (stringcasecmp(Val,"close") == 0)
 581          Persistent = false;
 582       if (stringcasecmp(Val,"keep-alive") == 0)
 583          Persistent = true;
 584       return true;
 585    }
 586
 587    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 588    {
 589       if (StrToTime(Val,Date) == false)
 590          return _error->Error(_("Unknown date format"));
 591       return true;
 592    }
 593
 594    return true;
 595 }
 596                                                                         /*}}}*/
 597
 598 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 599 // ---------------------------------------------------------------------
 600 /* This places the http request in the outbound buffer */
 601 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 602 {
 603    URI Uri = Itm->Uri;
 604
 605    // The HTTP server expects a hostname with a trailing :port
 606    char Buf[1000];
 607    string ProperHost = Uri.Host;
 608    if (Uri.Port != 0)
 609    {
 610       sprintf(Buf,":%u",Uri.Port);
 611       ProperHost += Buf;
 612    }
 613
 614    // Just in case.
 615    if (Itm->Uri.length() >= sizeof(Buf))
 616        abort();
 617
 618    /* Build the request. We include a keep-alive header only for non-proxy
 619       requests. This is to tweak old http/1.0 servers that do support keep-alive
 620       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 621       will glitch HTTP/1.0 proxies because they do not filter it out and
 622       pass it on, HTTP/1.1 says the connection should default to keep alive
 623       and we expect the proxy to do this */
 624    if (Proxy.empty() == true)
 625       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 626               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 627    else
 628    {
 629       /* Generate a cache control header if necessary. We place a max
 630          cache age on index files, optionally set a no-cache directive
 631          and a no-store directive for archives. */
 632       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 633               Itm->Uri.c_str(),ProperHost.c_str());
 634       // only generate a cache control header if we actually want to
 635       // use a cache
 636       if (_config->FindB("Acquire::http::No-Cache",false) == false)
 637       {
 638          if (Itm->IndexFile == true)
 639             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 640                     _config->FindI("Acquire::http::Max-Age",0));
 641          else
 642          {
 643             if (_config->FindB("Acquire::http::No-Store",false) == true)
 644                strcat(Buf,"Cache-Control: no-store\r\n");
 645          }
 646       }
 647    }
 648    // generate a no-cache header if needed
 649    if (_config->FindB("Acquire::http::No-Cache",false) == true)
 650       strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 651
 652
 653    string Req = Buf;
 654
 655    // Check for a partial file
 656    struct stat SBuf;
 657    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 658    {
 659       // In this case we send an if-range query with a range header
 660       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 661               TimeRFC1123(SBuf.st_mtime).c_str());
 662       Req += Buf;
 663    }
 664    else
 665    {
 666       if (Itm->LastModified != 0)
 667       {
 668          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 669          Req += Buf;
 670       }
 671    }
 672
 673    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 674       Req += string("Proxy-Authorization: Basic ") +
 675           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 676
 677    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 678       Req += string("Authorization: Basic ") +
 679           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 680
 681    Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
 682
 683    if (Debug == true)
 684       cerr << Req << endl;
 685
 686    Out.Read(Req);
 687 }
 688                                                                         /*}}}*/
 689 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 690 // ---------------------------------------------------------------------
 691 /* This runs the select loop over the server FDs, Output file FDs and
 692    stdin. */
 693 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 694 {
 695    // Server has closed the connection
 696    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 697                                ToFile == false))
 698       return false;
 699
 700    fd_set rfds,wfds;
 701    FD_ZERO(&rfds);
 702    FD_ZERO(&wfds);
 703
 704    /* Add the server. We only send more requests if the connection will
 705       be persisting */
 706    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 707        && Srv->Persistent == true)
 708       FD_SET(Srv->ServerFd,&wfds);
 709    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 710       FD_SET(Srv->ServerFd,&rfds);
 711
 712    // Add the file
 713    int FileFD = -1;
 714    if (File != 0)
 715       FileFD = File->Fd();
 716
 717    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 718       FD_SET(FileFD,&wfds);
 719
 720    // Add stdin
 721    FD_SET(STDIN_FILENO,&rfds);
 722
 723    // Figure out the max fd
 724    int MaxFd = FileFD;
 725    if (MaxFd < Srv->ServerFd)
 726       MaxFd = Srv->ServerFd;
 727
 728    // Select
 729    struct timeval tv;
 730    tv.tv_sec = TimeOut;
 731    tv.tv_usec = 0;
 732    int Res = 0;
 733    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 734    {
 735       if (errno == EINTR)
 736          return true;
 737       return _error->Errno("select",_("Select failed"));
 738    }
 739
 740    if (Res == 0)
 741    {
 742       _error->Error(_("Connection timed out"));
 743       return ServerDie(Srv);
 744    }
 745
 746    // Handle server IO
 747    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 748    {
 749       errno = 0;
 750       if (Srv->In.Read(Srv->ServerFd) == false)
 751          return ServerDie(Srv);
 752    }
 753
 754    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 755    {
 756       errno = 0;
 757       if (Srv->Out.Write(Srv->ServerFd) == false)
 758          return ServerDie(Srv);
 759    }
 760
 761    // Send data to the file
 762    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 763    {
 764       if (Srv->In.Write(FileFD) == false)
 765          return _error->Errno("write",_("Error writing to output file"));
 766    }
 767
 768    // Handle commands from APT
 769    if (FD_ISSET(STDIN_FILENO,&rfds))
 770    {
 771       if (Run(true) != -1)
 772          exit(100);
 773    }
 774
 775    return true;
 776 }
 777                                                                         /*}}}*/
 778 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 779 // ---------------------------------------------------------------------
 780 /* This takes the current input buffer from the Server FD and writes it
 781    into the file */
 782 bool HttpMethod::Flush(ServerState *Srv)
 783 {
 784    if (File != 0)
 785    {
 786       SetNonBlock(File->Fd(),false);
 787       if (Srv->In.WriteSpace() == false)
 788          return true;
 789
 790       while (Srv->In.WriteSpace() == true)
 791       {
 792          if (Srv->In.Write(File->Fd()) == false)
 793             return _error->Errno("write",_("Error writing to file"));
 794          if (Srv->In.IsLimit() == true)
 795             return true;
 796       }
 797
 798       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 799          return true;
 800    }
 801    return false;
 802 }
 803                                                                         /*}}}*/
 804 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 805 // ---------------------------------------------------------------------
 806 /* */
 807 bool HttpMethod::ServerDie(ServerState *Srv)
 808 {
 809    unsigned int LErrno = errno;
 810
 811    // Dump the buffer to the file
 812    if (Srv->State == ServerState::Data)
 813    {
 814       SetNonBlock(File->Fd(),false);
 815       while (Srv->In.WriteSpace() == true)
 816       {
 817          if (Srv->In.Write(File->Fd()) == false)
 818             return _error->Errno("write",_("Error writing to the file"));
 819
 820          // Done
 821          if (Srv->In.IsLimit() == true)
 822             return true;
 823       }
 824    }
 825
 826    // See if this is because the server finished the data stream
 827    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 828        Srv->Encoding != ServerState::Closes)
 829    {
 830       Srv->Close();
 831       if (LErrno == 0)
 832          return _error->Error(_("Error reading from server. Remote end closed connection"));
 833       errno = LErrno;
 834       return _error->Errno("read",_("Error reading from server"));
 835    }
 836    else
 837    {
 838       Srv->In.Limit(-1);
 839
 840       // Nothing left in the buffer
 841       if (Srv->In.WriteSpace() == false)
 842          return false;
 843
 844       // We may have got multiple responses back in one packet..
 845       Srv->Close();
 846       return true;
 847    }
 848
 849    return false;
 850 }
 851                                                                         /*}}}*/
 852 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 853 // ---------------------------------------------------------------------
 854 /* We look at the header data we got back from the server and decide what
 855    to do. Returns
 856      0 - File is open,
 857      1 - IMS hit
 858      3 - Unrecoverable error
 859      4 - Error with error content page
 860      5 - Unrecoverable non-server error (close the connection) */
 861 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 862 {
 863    // Not Modified
 864    if (Srv->Result == 304)
 865    {
 866       unlink(Queue->DestFile.c_str());
 867       Res.IMSHit = true;
 868       Res.LastModified = Queue->LastModified;
 869       return 1;
 870    }
 871
 872    /* We have a reply we dont handle. This should indicate a perm server
 873       failure */
 874    if (Srv->Result < 200 || Srv->Result >= 300)
 875    {
 876       _error->Error("%u %s",Srv->Result,Srv->Code);
 877       if (Srv->HaveContent == true)
 878          return 4;
 879       return 3;
 880    }
 881
 882    // This is some sort of 2xx 'data follows' reply
 883    Res.LastModified = Srv->Date;
 884    Res.Size = Srv->Size;
 885
 886    // Open the file
 887    delete File;
 888    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 889    if (_error->PendingError() == true)
 890       return 5;
 891
 892    FailFile = Queue->DestFile;
 893    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 894    FailFd = File->Fd();
 895    FailTime = Srv->Date;
 896
 897    // Set the expected size
 898    if (Srv->StartPos >= 0)
 899    {
 900       Res.ResumePoint = Srv->StartPos;
 901       ftruncate(File->Fd(),Srv->StartPos);
 902    }
 903
 904    // Set the start point
 905    lseek(File->Fd(),0,SEEK_END);
 906
 907    delete Srv->In.Hash;
 908    Srv->In.Hash = new Hashes;
 909
 910    // Fill the Hash if the file is non-empty (resume)
 911    if (Srv->StartPos > 0)
 912    {
 913       lseek(File->Fd(),0,SEEK_SET);
 914       if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
 915       {
 916          _error->Errno("read",_("Problem hashing file"));
 917          return 5;
 918       }
 919       lseek(File->Fd(),0,SEEK_END);
 920    }
 921
 922    SetNonBlock(File->Fd(),true);
 923    return 0;
 924 }
 925                                                                         /*}}}*/
 926 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 927 // ---------------------------------------------------------------------
 928 /* This closes and timestamps the open file. This is neccessary to get
 929    resume behavoir on user abort */
 930 void HttpMethod::SigTerm(int)
 931 {
 932    if (FailFd == -1)
 933       _exit(100);
 934    close(FailFd);
 935
 936    // Timestamp
 937    struct utimbuf UBuf;
 938    UBuf.actime = FailTime;
 939    UBuf.modtime = FailTime;
 940    utime(FailFile.c_str(),&UBuf);
 941
 942    _exit(100);
 943 }
 944                                                                         /*}}}*/
 945 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 946 // ---------------------------------------------------------------------
 947 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 948    depth. */
 949 bool HttpMethod::Fetch(FetchItem *)
 950 {
 951    if (Server == 0)
 952       return true;
 953
 954    // Queue the requests
 955    int Depth = -1;
 956    bool Tail = false;
 957    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 958         I = I->Next, Depth++)
 959    {
 960       // If pipelining is disabled, we only queue 1 request
 961       if (Server->Pipeline == false && Depth >= 0)
 962          break;
 963
 964       // Make sure we stick with the same server
 965       if (Server->Comp(I->Uri) == false)
 966          break;
 967       if (QueueBack == I)
 968          Tail = true;
 969       if (Tail == true)
 970       {
 971          QueueBack = I->Next;
 972          SendReq(I,Server->Out);
 973          continue;
 974       }
 975    }
 976
 977    return true;
 978 };
 979                                                                         /*}}}*/
 980 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 981 // ---------------------------------------------------------------------
 982 /* We stash the desired pipeline depth */
 983 bool HttpMethod::Configuration(string Message)
 984 {
 985    if (pkgAcqMethod::Configuration(Message) == false)
 986       return false;
 987
 988    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 989    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 990                                   PipelineDepth);
 991    Debug = _config->FindB("Debug::Acquire::http",false);
 992
 993    return true;
 994 }
 995                                                                         /*}}}*/
 996 // HttpMethod::Loop - Main loop                                         /*{{{*/
 997 // ---------------------------------------------------------------------
 998 /* */
 999 int HttpMethod::Loop()
1000 {
1001    signal(SIGTERM,SigTerm);
1002    signal(SIGINT,SigTerm);
1003
1004    Server = 0;
1005
1006    int FailCounter = 0;
1007    while (1)
1008    {
1009       // We have no commands, wait for some to arrive
1010       if (Queue == 0)
1011       {
1012          if (WaitFd(STDIN_FILENO) == false)
1013             return 0;
1014       }
1015
1016       /* Run messages, we can accept 0 (no message) if we didn't
1017          do a WaitFd above.. Otherwise the FD is closed. */
1018       int Result = Run(true);
1019       if (Result != -1 && (Result != 0 || Queue == 0))
1020          return 100;
1021
1022       if (Queue == 0)
1023          continue;
1024
1025       // Connect to the server
1026       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1027       {
1028          delete Server;
1029          Server = new ServerState(Queue->Uri,this);
1030       }
1031
1032       /* If the server has explicitly said this is the last connection
1033          then we pre-emptively shut down the pipeline and tear down
1034          the connection. This will speed up HTTP/1.0 servers a tad
1035          since we don't have to wait for the close sequence to
1036          complete */
1037       if (Server->Persistent == false)
1038          Server->Close();
1039
1040       // Reset the pipeline
1041       if (Server->ServerFd == -1)
1042          QueueBack = Queue;
1043
1044       // Connnect to the host
1045       if (Server->Open() == false)
1046       {
1047          Fail(true);
1048          delete Server;
1049          Server = 0;
1050          continue;
1051       }
1052
1053       // Fill the pipeline.
1054       Fetch(0);
1055
1056       // Fetch the next URL header data from the server.
1057       switch (Server->RunHeaders())
1058       {
1059          case 0:
1060          break;
1061
1062          // The header data is bad
1063          case 2:
1064          {
1065             _error->Error(_("Bad header data"));
1066             Fail(true);
1067             RotateDNS();
1068             continue;
1069          }
1070
1071          // The server closed a connection during the header get..
1072          default:
1073          case 1:
1074          {
1075             FailCounter++;
1076             _error->Discard();
1077             Server->Close();
1078             Server->Pipeline = false;
1079
1080             if (FailCounter >= 2)
1081             {
1082                Fail(_("Connection failed"),true);
1083                FailCounter = 0;
1084             }
1085
1086             RotateDNS();
1087             continue;
1088          }
1089       };
1090
1091       // Decide what to do.
1092       FetchResult Res;
1093       Res.Filename = Queue->DestFile;
1094       switch (DealWithHeaders(Res,Server))
1095       {
1096          // Ok, the file is Open
1097          case 0:
1098          {
1099             URIStart(Res);
1100
1101             // Run the data
1102             bool Result =  Server->RunData();
1103
1104             /* If the server is sending back sizeless responses then fill in
1105                the size now */
1106             if (Res.Size == 0)
1107                Res.Size = File->Size();
1108
1109             // Close the file, destroy the FD object and timestamp it
1110             FailFd = -1;
1111             delete File;
1112             File = 0;
1113
1114             // Timestamp
1115             struct utimbuf UBuf;
1116             time(&UBuf.actime);
1117             UBuf.actime = Server->Date;
1118             UBuf.modtime = Server->Date;
1119             utime(Queue->DestFile.c_str(),&UBuf);
1120
1121             // Send status to APT
1122             if (Result == true)
1123             {
1124                Res.TakeHashes(*Server->In.Hash);
1125                URIDone(Res);
1126             }
1127             else
1128                Fail(true);
1129
1130             break;
1131          }
1132
1133          // IMS hit
1134          case 1:
1135          {
1136             URIDone(Res);
1137             break;
1138          }
1139
1140          // Hard server error, not found or something
1141          case 3:
1142          {
1143             Fail();
1144             break;
1145          }
1146
1147          // Hard internal error, kill the connection and fail
1148          case 5:
1149          {
1150             delete File;
1151             File = 0;
1152
1153             Fail();
1154             RotateDNS();
1155             Server->Close();
1156             break;
1157          }
1158
1159          // We need to flush the data, the header is like a 404 w/ error text
1160          case 4:
1161          {
1162             Fail();
1163
1164             // Send to content to dev/null
1165             File = new FileFd("/dev/null",FileFd::WriteExists);
1166             Server->RunData();
1167             delete File;
1168             File = 0;
1169             break;
1170          }
1171
1172          default:
1173          Fail(_("Internal error"));
1174          break;
1175       }
1176
1177       FailCounter = 0;
1178    }
1179
1180    return 0;
1181 }
1182                                                                         /*}}}*/
1183
1184 int main()
1185 {
1186    setlocale(LC_ALL, "");
1187
1188    HttpMethod Mth;
1189
1190    return Mth.Loop();
1191 }
1192
1193