methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apt-pkg/fileutl.h>
  29 #include <apt-pkg/acquire-method.h>
  30 #include <apt-pkg/error.h>
  31 #include <apt-pkg/hashes.h>
  32
  33 #include <sys/stat.h>
  34 #include <sys/time.h>
  35 #include <utime.h>
  36 #include <unistd.h>
  37 #include <signal.h>
  38 #include <stdio.h>
  39 #include <errno.h>
  40 #include <string.h>
  41 #include <iostream>
  42 #include <apti18n.h>
  43
  44 // Internet stuff
  45 #include <netdb.h>
  46
  47 #include "connect.h"
  48 #include "rfc2553emu.h"
  49 #include "http.h"
  50
  51                                                                         /*}}}*/
  52 using namespace std;
  53
  54 string HttpMethod::FailFile;
  55 int HttpMethod::FailFd = -1;
  56 time_t HttpMethod::FailTime = 0;
  57 unsigned long PipelineDepth = 10;
  58 unsigned long TimeOut = 120;
  59 bool Debug = false;
  60
  61 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  62 // ---------------------------------------------------------------------
  63 /* */
  64 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
  65 {
  66    Buf = new unsigned char[Size];
  67    Reset();
  68 }
  69                                                                         /*}}}*/
  70 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  71 // ---------------------------------------------------------------------
  72 /* */
  73 void CircleBuf::Reset()
  74 {
  75    InP = 0;
  76    OutP = 0;
  77    StrPos = 0;
  78    MaxGet = (unsigned int)-1;
  79    OutQueue = string();
  80    if (Hash != 0)
  81    {
  82       delete Hash;
  83       Hash = new Hashes;
  84    }
  85 };
  86                                                                         /*}}}*/
  87 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  88 // ---------------------------------------------------------------------
  89 /* This fills up the buffer with as much data as is in the FD, assuming it
  90    is non-blocking.. */
  91 bool CircleBuf::Read(int Fd)
  92 {
  93    while (1)
  94    {
  95       // Woops, buffer is full
  96       if (InP - OutP == Size)
  97          return true;
  98
  99       // Write the buffer segment
 100       int Res;
 101       Res = read(Fd,Buf + (InP%Size),LeftRead());
 102
 103       if (Res == 0)
 104          return false;
 105       if (Res < 0)
 106       {
 107          if (errno == EAGAIN)
 108             return true;
 109          return false;
 110       }
 111
 112       if (InP == 0)
 113          gettimeofday(&Start,0);
 114       InP += Res;
 115    }
 116 }
 117                                                                         /*}}}*/
 118 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 119 // ---------------------------------------------------------------------
 120 /* This will hold the string in and fill the buffer with it as it empties */
 121 bool CircleBuf::Read(string Data)
 122 {
 123    OutQueue += Data;
 124    FillOut();
 125    return true;
 126 }
 127                                                                         /*}}}*/
 128 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 129 // ---------------------------------------------------------------------
 130 /* */
 131 void CircleBuf::FillOut()
 132 {
 133    if (OutQueue.empty() == true)
 134       return;
 135    while (1)
 136    {
 137       // Woops, buffer is full
 138       if (InP - OutP == Size)
 139          return;
 140
 141       // Write the buffer segment
 142       unsigned long Sz = LeftRead();
 143       if (OutQueue.length() - StrPos < Sz)
 144          Sz = OutQueue.length() - StrPos;
 145       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 146
 147       // Advance
 148       StrPos += Sz;
 149       InP += Sz;
 150       if (OutQueue.length() == StrPos)
 151       {
 152          StrPos = 0;
 153          OutQueue = "";
 154          return;
 155       }
 156    }
 157 }
 158                                                                         /*}}}*/
 159 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 160 // ---------------------------------------------------------------------
 161 /* This empties the buffer into the FD. */
 162 bool CircleBuf::Write(int Fd)
 163 {
 164    while (1)
 165    {
 166       FillOut();
 167
 168       // Woops, buffer is empty
 169       if (OutP == InP)
 170          return true;
 171
 172       if (OutP == MaxGet)
 173          return true;
 174
 175       // Write the buffer segment
 176       int Res;
 177       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 178
 179       if (Res == 0)
 180          return false;
 181       if (Res < 0)
 182       {
 183          if (errno == EAGAIN)
 184             return true;
 185
 186          return false;
 187       }
 188
 189       if (Hash != 0)
 190          Hash->Add(Buf + (OutP%Size),Res);
 191
 192       OutP += Res;
 193    }
 194 }
 195                                                                         /*}}}*/
 196 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 197 // ---------------------------------------------------------------------
 198 /* This copies till the first empty line */
 199 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 200 {
 201    // We cheat and assume it is unneeded to have more than one buffer load
 202    for (unsigned long I = OutP; I < InP; I++)
 203    {
 204       if (Buf[I%Size] != '\n')
 205          continue;
 206       ++I;
 207       if (I < InP  && Buf[I%Size] == '\r')
 208          ++I;
 209
 210       if (Single == false)
 211       {
 212          if (Buf[I%Size] != '\n')
 213             continue;
 214          ++I;
 215          if (I < InP  && Buf[I%Size] == '\r')
 216             ++I;
 217       }
 218
 219       if (I > InP)
 220          I = InP;
 221
 222       Data = "";
 223       while (OutP < I)
 224       {
 225          unsigned long Sz = LeftWrite();
 226          if (Sz == 0)
 227             return false;
 228          if (I - OutP < LeftWrite())
 229             Sz = I - OutP;
 230          Data += string((char *)(Buf + (OutP%Size)),Sz);
 231          OutP += Sz;
 232       }
 233       return true;
 234    }
 235    return false;
 236 }
 237                                                                         /*}}}*/
 238 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 239 // ---------------------------------------------------------------------
 240 /* */
 241 void CircleBuf::Stats()
 242 {
 243    if (InP == 0)
 244       return;
 245
 246    struct timeval Stop;
 247    gettimeofday(&Stop,0);
 248 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 249              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 250    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 251 }
 252                                                                         /*}}}*/
 253
 254 // ServerState::ServerState - Constructor                               /*{{{*/
 255 // ---------------------------------------------------------------------
 256 /* */
 257 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 258                         In(64*1024), Out(4*1024),
 259                         ServerName(Srv)
 260 {
 261    Reset();
 262 }
 263                                                                         /*}}}*/
 264 // ServerState::Open - Open a connection to the server                  /*{{{*/
 265 // ---------------------------------------------------------------------
 266 /* This opens a connection to the server. */
 267 bool ServerState::Open()
 268 {
 269    // Use the already open connection if possible.
 270    if (ServerFd != -1)
 271       return true;
 272
 273    Close();
 274    In.Reset();
 275    Out.Reset();
 276    Persistent = true;
 277
 278    // Determine the proxy setting
 279    if (getenv("http_proxy") == 0)
 280    {
 281       string DefProxy = _config->Find("Acquire::http::Proxy");
 282       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 283       if (SpecificProxy.empty() == false)
 284       {
 285          if (SpecificProxy == "DIRECT")
 286             Proxy = "";
 287          else
 288             Proxy = SpecificProxy;
 289       }
 290       else
 291          Proxy = DefProxy;
 292    }
 293    else
 294       Proxy = getenv("http_proxy");
 295
 296    // Parse no_proxy, a , separated list of domains
 297    if (getenv("no_proxy") != 0)
 298    {
 299       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 300          Proxy = "";
 301    }
 302
 303    // Determine what host and port to use based on the proxy settings
 304    int Port = 0;
 305    string Host;
 306    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 307    {
 308       if (ServerName.Port != 0)
 309          Port = ServerName.Port;
 310       Host = ServerName.Host;
 311    }
 312    else
 313    {
 314       if (Proxy.Port != 0)
 315          Port = Proxy.Port;
 316       Host = Proxy.Host;
 317    }
 318
 319    // Connect to the remote server
 320    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 321       return false;
 322
 323    return true;
 324 }
 325                                                                         /*}}}*/
 326 // ServerState::Close - Close a connection to the server                /*{{{*/
 327 // ---------------------------------------------------------------------
 328 /* */
 329 bool ServerState::Close()
 330 {
 331    close(ServerFd);
 332    ServerFd = -1;
 333    return true;
 334 }
 335                                                                         /*}}}*/
 336 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 337 // ---------------------------------------------------------------------
 338 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 339    parse error occured */
 340 int ServerState::RunHeaders()
 341 {
 342    State = Header;
 343
 344    Owner->Status(_("Waiting for headers"));
 345
 346    Major = 0;
 347    Minor = 0;
 348    Result = 0;
 349    Size = 0;
 350    StartPos = 0;
 351    Encoding = Closes;
 352    HaveContent = false;
 353    time(&Date);
 354
 355    do
 356    {
 357       string Data;
 358       if (In.WriteTillEl(Data) == false)
 359          continue;
 360
 361       if (Debug == true)
 362          clog << Data;
 363
 364       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 365       {
 366          string::const_iterator J = I;
 367          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 368          if (HeaderLine(string(I,J)) == false)
 369             return 2;
 370          I = J;
 371       }
 372
 373       // 100 Continue is a Nop...
 374       if (Result == 100)
 375          continue;
 376
 377       // Tidy up the connection persistance state.
 378       if (Encoding == Closes && HaveContent == true)
 379          Persistent = false;
 380
 381       return 0;
 382    }
 383    while (Owner->Go(false,this) == true);
 384
 385    return 1;
 386 }
 387                                                                         /*}}}*/
 388 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 389 // ---------------------------------------------------------------------
 390 /* */
 391 bool ServerState::RunData()
 392 {
 393    State = Data;
 394
 395    // Chunked transfer encoding is fun..
 396    if (Encoding == Chunked)
 397    {
 398       while (1)
 399       {
 400          // Grab the block size
 401          bool Last = true;
 402          string Data;
 403          In.Limit(-1);
 404          do
 405          {
 406             if (In.WriteTillEl(Data,true) == true)
 407                break;
 408          }
 409          while ((Last = Owner->Go(false,this)) == true);
 410
 411          if (Last == false)
 412             return false;
 413
 414          // See if we are done
 415          unsigned long Len = strtol(Data.c_str(),0,16);
 416          if (Len == 0)
 417          {
 418             In.Limit(-1);
 419
 420             // We have to remove the entity trailer
 421             Last = true;
 422             do
 423             {
 424                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 425                   break;
 426             }
 427             while ((Last = Owner->Go(false,this)) == true);
 428             if (Last == false)
 429                return false;
 430             return !_error->PendingError();
 431          }
 432
 433          // Transfer the block
 434          In.Limit(Len);
 435          while (Owner->Go(true,this) == true)
 436             if (In.IsLimit() == true)
 437                break;
 438
 439          // Error
 440          if (In.IsLimit() == false)
 441             return false;
 442
 443          // The server sends an extra new line before the next block specifier..
 444          In.Limit(-1);
 445          Last = true;
 446          do
 447          {
 448             if (In.WriteTillEl(Data,true) == true)
 449                break;
 450          }
 451          while ((Last = Owner->Go(false,this)) == true);
 452          if (Last == false)
 453             return false;
 454       }
 455    }
 456    else
 457    {
 458       /* Closes encoding is used when the server did not specify a size, the
 459          loss of the connection means we are done */
 460       if (Encoding == Closes)
 461          In.Limit(-1);
 462       else
 463          In.Limit(Size - StartPos);
 464
 465       // Just transfer the whole block.
 466       do
 467       {
 468          if (In.IsLimit() == false)
 469             continue;
 470
 471          In.Limit(-1);
 472          return !_error->PendingError();
 473       }
 474       while (Owner->Go(true,this) == true);
 475    }
 476
 477    return Owner->Flush(this) && !_error->PendingError();
 478 }
 479                                                                         /*}}}*/
 480 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 481 // ---------------------------------------------------------------------
 482 /* */
 483 bool ServerState::HeaderLine(string Line)
 484 {
 485    if (Line.empty() == true)
 486       return true;
 487
 488    // The http server might be trying to do something evil.
 489    if (Line.length() >= MAXLEN)
 490       return _error->Error(_("Got a single header line over %u chars"),MAXLEN);
 491
 492    string::size_type Pos = Line.find(' ');
 493    if (Pos == string::npos || Pos+1 > Line.length())
 494    {
 495       // Blah, some servers use "connection:closes", evil.
 496       Pos = Line.find(':');
 497       if (Pos == string::npos || Pos + 2 > Line.length())
 498          return _error->Error(_("Bad header line"));
 499       Pos++;
 500    }
 501
 502    // Parse off any trailing spaces between the : and the next word.
 503    string::size_type Pos2 = Pos;
 504    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 505       Pos2++;
 506
 507    string Tag = string(Line,0,Pos);
 508    string Val = string(Line,Pos2);
 509
 510    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 511    {
 512       // Evil servers return no version
 513       if (Line[4] == '/')
 514       {
 515          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 516                     &Result,Code) != 4)
 517             return _error->Error(_("The HTTP server sent an invalid reply header"));
 518       }
 519       else
 520       {
 521          Major = 0;
 522          Minor = 9;
 523          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 524             return _error->Error(_("The HTTP server sent an invalid reply header"));
 525       }
 526
 527       /* Check the HTTP response header to get the default persistance
 528          state. */
 529       if (Major < 1)
 530          Persistent = false;
 531       else
 532       {
 533          if (Major == 1 && Minor <= 0)
 534             Persistent = false;
 535          else
 536             Persistent = true;
 537       }
 538
 539       return true;
 540    }
 541
 542    if (stringcasecmp(Tag,"Content-Length:") == 0)
 543    {
 544       if (Encoding == Closes)
 545          Encoding = Stream;
 546       HaveContent = true;
 547
 548       // The length is already set from the Content-Range header
 549       if (StartPos != 0)
 550          return true;
 551
 552       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 553          return _error->Error(_("The HTTP server sent an invalid Content-Length header"));
 554       return true;
 555    }
 556
 557    if (stringcasecmp(Tag,"Content-Type:") == 0)
 558    {
 559       HaveContent = true;
 560       return true;
 561    }
 562
 563    if (stringcasecmp(Tag,"Content-Range:") == 0)
 564    {
 565       HaveContent = true;
 566
 567       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 568          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 569       if ((unsigned)StartPos > Size)
 570          return _error->Error(_("This HTTP server has broken range support"));
 571       return true;
 572    }
 573
 574    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 575    {
 576       HaveContent = true;
 577       if (stringcasecmp(Val,"chunked") == 0)
 578          Encoding = Chunked;
 579       return true;
 580    }
 581
 582    if (stringcasecmp(Tag,"Connection:") == 0)
 583    {
 584       if (stringcasecmp(Val,"close") == 0)
 585          Persistent = false;
 586       if (stringcasecmp(Val,"keep-alive") == 0)
 587          Persistent = true;
 588       return true;
 589    }
 590
 591    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 592    {
 593       if (StrToTime(Val,Date) == false)
 594          return _error->Error(_("Unknown date format"));
 595       return true;
 596    }
 597
 598    return true;
 599 }
 600                                                                         /*}}}*/
 601
 602 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 603 // ---------------------------------------------------------------------
 604 /* This places the http request in the outbound buffer */
 605 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 606 {
 607    URI Uri = Itm->Uri;
 608
 609    // The HTTP server expects a hostname with a trailing :port
 610    char Buf[1000];
 611    string ProperHost = Uri.Host;
 612    if (Uri.Port != 0)
 613    {
 614       sprintf(Buf,":%u",Uri.Port);
 615       ProperHost += Buf;
 616    }
 617
 618    // Just in case.
 619    if (Itm->Uri.length() >= sizeof(Buf))
 620        abort();
 621
 622    /* Build the request. We include a keep-alive header only for non-proxy
 623       requests. This is to tweak old http/1.0 servers that do support keep-alive
 624       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 625       will glitch HTTP/1.0 proxies because they do not filter it out and
 626       pass it on, HTTP/1.1 says the connection should default to keep alive
 627       and we expect the proxy to do this */
 628    if (Proxy.empty() == true)
 629       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 630               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 631    else
 632    {
 633       /* Generate a cache control header if necessary. We place a max
 634          cache age on index files, optionally set a no-cache directive
 635          and a no-store directive for archives. */
 636       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 637               Itm->Uri.c_str(),ProperHost.c_str());
 638       // only generate a cache control header if we actually want to
 639       // use a cache
 640       if (_config->FindB("Acquire::http::No-Cache",false) == false)
 641       {
 642          if (Itm->IndexFile == true)
 643             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 644                     _config->FindI("Acquire::http::Max-Age",0));
 645          else
 646          {
 647             if (_config->FindB("Acquire::http::No-Store",false) == true)
 648                strcat(Buf,"Cache-Control: no-store\r\n");
 649          }
 650       }
 651    }
 652    // generate a no-cache header if needed
 653    if (_config->FindB("Acquire::http::No-Cache",false) == true)
 654       strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 655
 656
 657    string Req = Buf;
 658
 659    // Check for a partial file
 660    struct stat SBuf;
 661    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 662    {
 663       // In this case we send an if-range query with a range header
 664       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 665               TimeRFC1123(SBuf.st_mtime).c_str());
 666       Req += Buf;
 667    }
 668    else
 669    {
 670       if (Itm->LastModified != 0)
 671       {
 672          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 673          Req += Buf;
 674       }
 675    }
 676
 677    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 678       Req += string("Proxy-Authorization: Basic ") +
 679           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 680
 681    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 682       Req += string("Authorization: Basic ") +
 683           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 684
 685    Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
 686
 687    if (Debug == true)
 688       cerr << Req << endl;
 689
 690    Out.Read(Req);
 691 }
 692                                                                         /*}}}*/
 693 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 694 // ---------------------------------------------------------------------
 695 /* This runs the select loop over the server FDs, Output file FDs and
 696    stdin. */
 697 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 698 {
 699    // Server has closed the connection
 700    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 701                                ToFile == false))
 702       return false;
 703
 704    fd_set rfds,wfds;
 705    FD_ZERO(&rfds);
 706    FD_ZERO(&wfds);
 707
 708    /* Add the server. We only send more requests if the connection will
 709       be persisting */
 710    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 711        && Srv->Persistent == true)
 712       FD_SET(Srv->ServerFd,&wfds);
 713    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 714       FD_SET(Srv->ServerFd,&rfds);
 715
 716    // Add the file
 717    int FileFD = -1;
 718    if (File != 0)
 719       FileFD = File->Fd();
 720
 721    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 722       FD_SET(FileFD,&wfds);
 723
 724    // Add stdin
 725    FD_SET(STDIN_FILENO,&rfds);
 726
 727    // Figure out the max fd
 728    int MaxFd = FileFD;
 729    if (MaxFd < Srv->ServerFd)
 730       MaxFd = Srv->ServerFd;
 731
 732    // Select
 733    struct timeval tv;
 734    tv.tv_sec = TimeOut;
 735    tv.tv_usec = 0;
 736    int Res = 0;
 737    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 738    {
 739       if (errno == EINTR)
 740          return true;
 741       return _error->Errno("select",_("Select failed"));
 742    }
 743
 744    if (Res == 0)
 745    {
 746       _error->Error(_("Connection timed out"));
 747       return ServerDie(Srv);
 748    }
 749
 750    // Handle server IO
 751    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 752    {
 753       errno = 0;
 754       if (Srv->In.Read(Srv->ServerFd) == false)
 755          return ServerDie(Srv);
 756    }
 757
 758    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 759    {
 760       errno = 0;
 761       if (Srv->Out.Write(Srv->ServerFd) == false)
 762          return ServerDie(Srv);
 763    }
 764
 765    // Send data to the file
 766    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 767    {
 768       if (Srv->In.Write(FileFD) == false)
 769          return _error->Errno("write",_("Error writing to output file"));
 770    }
 771
 772    // Handle commands from APT
 773    if (FD_ISSET(STDIN_FILENO,&rfds))
 774    {
 775       if (Run(true) != -1)
 776          exit(100);
 777    }
 778
 779    return true;
 780 }
 781                                                                         /*}}}*/
 782 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 783 // ---------------------------------------------------------------------
 784 /* This takes the current input buffer from the Server FD and writes it
 785    into the file */
 786 bool HttpMethod::Flush(ServerState *Srv)
 787 {
 788    if (File != 0)
 789    {
 790       SetNonBlock(File->Fd(),false);
 791       if (Srv->In.WriteSpace() == false)
 792          return true;
 793
 794       while (Srv->In.WriteSpace() == true)
 795       {
 796          if (Srv->In.Write(File->Fd()) == false)
 797             return _error->Errno("write",_("Error writing to file"));
 798          if (Srv->In.IsLimit() == true)
 799             return true;
 800       }
 801
 802       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 803          return true;
 804    }
 805    return false;
 806 }
 807                                                                         /*}}}*/
 808 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 809 // ---------------------------------------------------------------------
 810 /* */
 811 bool HttpMethod::ServerDie(ServerState *Srv)
 812 {
 813    unsigned int LErrno = errno;
 814
 815    // Dump the buffer to the file
 816    if (Srv->State == ServerState::Data)
 817    {
 818       SetNonBlock(File->Fd(),false);
 819       while (Srv->In.WriteSpace() == true)
 820       {
 821          if (Srv->In.Write(File->Fd()) == false)
 822             return _error->Errno("write",_("Error writing to the file"));
 823
 824          // Done
 825          if (Srv->In.IsLimit() == true)
 826             return true;
 827       }
 828    }
 829
 830    // See if this is because the server finished the data stream
 831    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 832        Srv->Encoding != ServerState::Closes)
 833    {
 834       Srv->Close();
 835       if (LErrno == 0)
 836          return _error->Error(_("Error reading from server. Remote end closed connection"));
 837       errno = LErrno;
 838       return _error->Errno("read",_("Error reading from server"));
 839    }
 840    else
 841    {
 842       Srv->In.Limit(-1);
 843
 844       // Nothing left in the buffer
 845       if (Srv->In.WriteSpace() == false)
 846          return false;
 847
 848       // We may have got multiple responses back in one packet..
 849       Srv->Close();
 850       return true;
 851    }
 852
 853    return false;
 854 }
 855                                                                         /*}}}*/
 856 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 857 // ---------------------------------------------------------------------
 858 /* We look at the header data we got back from the server and decide what
 859    to do. Returns
 860      0 - File is open,
 861      1 - IMS hit
 862      3 - Unrecoverable error
 863      4 - Error with error content page
 864      5 - Unrecoverable non-server error (close the connection) */
 865 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 866 {
 867    // Not Modified
 868    if (Srv->Result == 304)
 869    {
 870       unlink(Queue->DestFile.c_str());
 871       Res.IMSHit = true;
 872       Res.LastModified = Queue->LastModified;
 873       return 1;
 874    }
 875
 876    /* We have a reply we dont handle. This should indicate a perm server
 877       failure */
 878    if (Srv->Result < 200 || Srv->Result >= 300)
 879    {
 880       _error->Error("%u %s",Srv->Result,Srv->Code);
 881       if (Srv->HaveContent == true)
 882          return 4;
 883       return 3;
 884    }
 885
 886    // This is some sort of 2xx 'data follows' reply
 887    Res.LastModified = Srv->Date;
 888    Res.Size = Srv->Size;
 889
 890    // Open the file
 891    delete File;
 892    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 893    if (_error->PendingError() == true)
 894       return 5;
 895
 896    FailFile = Queue->DestFile;
 897    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 898    FailFd = File->Fd();
 899    FailTime = Srv->Date;
 900
 901    // Set the expected size
 902    if (Srv->StartPos >= 0)
 903    {
 904       Res.ResumePoint = Srv->StartPos;
 905       ftruncate(File->Fd(),Srv->StartPos);
 906    }
 907
 908    // Set the start point
 909    lseek(File->Fd(),0,SEEK_END);
 910
 911    delete Srv->In.Hash;
 912    Srv->In.Hash = new Hashes;
 913
 914    // Fill the Hash if the file is non-empty (resume)
 915    if (Srv->StartPos > 0)
 916    {
 917       lseek(File->Fd(),0,SEEK_SET);
 918       if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
 919       {
 920          _error->Errno("read",_("Problem hashing file"));
 921          return 5;
 922       }
 923       lseek(File->Fd(),0,SEEK_END);
 924    }
 925
 926    SetNonBlock(File->Fd(),true);
 927    return 0;
 928 }
 929                                                                         /*}}}*/
 930 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 931 // ---------------------------------------------------------------------
 932 /* This closes and timestamps the open file. This is neccessary to get
 933    resume behavoir on user abort */
 934 void HttpMethod::SigTerm(int)
 935 {
 936    if (FailFd == -1)
 937       _exit(100);
 938    close(FailFd);
 939
 940    // Timestamp
 941    struct utimbuf UBuf;
 942    UBuf.actime = FailTime;
 943    UBuf.modtime = FailTime;
 944    utime(FailFile.c_str(),&UBuf);
 945
 946    _exit(100);
 947 }
 948                                                                         /*}}}*/
 949 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 950 // ---------------------------------------------------------------------
 951 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 952    depth. */
 953 bool HttpMethod::Fetch(FetchItem *)
 954 {
 955    if (Server == 0)
 956       return true;
 957
 958    // Queue the requests
 959    int Depth = -1;
 960    bool Tail = false;
 961    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 962         I = I->Next, Depth++)
 963    {
 964       // If pipelining is disabled, we only queue 1 request
 965       if (Server->Pipeline == false && Depth >= 0)
 966          break;
 967
 968       // Make sure we stick with the same server
 969       if (Server->Comp(I->Uri) == false)
 970          break;
 971       if (QueueBack == I)
 972          Tail = true;
 973       if (Tail == true)
 974       {
 975          QueueBack = I->Next;
 976          SendReq(I,Server->Out);
 977          continue;
 978       }
 979    }
 980
 981    return true;
 982 };
 983                                                                         /*}}}*/
 984 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 985 // ---------------------------------------------------------------------
 986 /* We stash the desired pipeline depth */
 987 bool HttpMethod::Configuration(string Message)
 988 {
 989    if (pkgAcqMethod::Configuration(Message) == false)
 990       return false;
 991
 992    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 993    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 994                                   PipelineDepth);
 995    Debug = _config->FindB("Debug::Acquire::http",false);
 996
 997    return true;
 998 }
 999                                                                         /*}}}*/
1000 // HttpMethod::Loop - Main loop                                         /*{{{*/
1001 // ---------------------------------------------------------------------
1002 /* */
1003 int HttpMethod::Loop()
1004 {
1005    signal(SIGTERM,SigTerm);
1006    signal(SIGINT,SigTerm);
1007
1008    Server = 0;
1009
1010    int FailCounter = 0;
1011    while (1)
1012    {
1013       // We have no commands, wait for some to arrive
1014       if (Queue == 0)
1015       {
1016          if (WaitFd(STDIN_FILENO) == false)
1017             return 0;
1018       }
1019
1020       /* Run messages, we can accept 0 (no message) if we didn't
1021          do a WaitFd above.. Otherwise the FD is closed. */
1022       int Result = Run(true);
1023       if (Result != -1 && (Result != 0 || Queue == 0))
1024          return 100;
1025
1026       if (Queue == 0)
1027          continue;
1028
1029       // Connect to the server
1030       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1031       {
1032          delete Server;
1033          Server = new ServerState(Queue->Uri,this);
1034       }
1035
1036       /* If the server has explicitly said this is the last connection
1037          then we pre-emptively shut down the pipeline and tear down
1038          the connection. This will speed up HTTP/1.0 servers a tad
1039          since we don't have to wait for the close sequence to
1040          complete */
1041       if (Server->Persistent == false)
1042          Server->Close();
1043
1044       // Reset the pipeline
1045       if (Server->ServerFd == -1)
1046          QueueBack = Queue;
1047
1048       // Connnect to the host
1049       if (Server->Open() == false)
1050       {
1051          Fail(true);
1052          delete Server;
1053          Server = 0;
1054          continue;
1055       }
1056
1057       // Fill the pipeline.
1058       Fetch(0);
1059
1060       // Fetch the next URL header data from the server.
1061       switch (Server->RunHeaders())
1062       {
1063          case 0:
1064          break;
1065
1066          // The header data is bad
1067          case 2:
1068          {
1069             _error->Error(_("Bad header data"));
1070             Fail(true);
1071             RotateDNS();
1072             continue;
1073          }
1074
1075          // The server closed a connection during the header get..
1076          default:
1077          case 1:
1078          {
1079             FailCounter++;
1080             _error->Discard();
1081             Server->Close();
1082             Server->Pipeline = false;
1083
1084             if (FailCounter >= 2)
1085             {
1086                Fail(_("Connection failed"),true);
1087                FailCounter = 0;
1088             }
1089
1090             RotateDNS();
1091             continue;
1092          }
1093       };
1094
1095       // Decide what to do.
1096       FetchResult Res;
1097       Res.Filename = Queue->DestFile;
1098       switch (DealWithHeaders(Res,Server))
1099       {
1100          // Ok, the file is Open
1101          case 0:
1102          {
1103             URIStart(Res);
1104
1105             // Run the data
1106             bool Result =  Server->RunData();
1107
1108             /* If the server is sending back sizeless responses then fill in
1109                the size now */
1110             if (Res.Size == 0)
1111                Res.Size = File->Size();
1112
1113             // Close the file, destroy the FD object and timestamp it
1114             FailFd = -1;
1115             delete File;
1116             File = 0;
1117
1118             // Timestamp
1119             struct utimbuf UBuf;
1120             time(&UBuf.actime);
1121             UBuf.actime = Server->Date;
1122             UBuf.modtime = Server->Date;
1123             utime(Queue->DestFile.c_str(),&UBuf);
1124
1125             // Send status to APT
1126             if (Result == true)
1127             {
1128                Res.TakeHashes(*Server->In.Hash);
1129                URIDone(Res);
1130             }
1131             else
1132                Fail(true);
1133
1134             break;
1135          }
1136
1137          // IMS hit
1138          case 1:
1139          {
1140             URIDone(Res);
1141             break;
1142          }
1143
1144          // Hard server error, not found or something
1145          case 3:
1146          {
1147             Fail();
1148             break;
1149          }
1150
1151          // Hard internal error, kill the connection and fail
1152          case 5:
1153          {
1154             delete File;
1155             File = 0;
1156
1157             Fail();
1158             RotateDNS();
1159             Server->Close();
1160             break;
1161          }
1162
1163          // We need to flush the data, the header is like a 404 w/ error text
1164          case 4:
1165          {
1166             Fail();
1167
1168             // Send to content to dev/null
1169             File = new FileFd("/dev/null",FileFd::WriteExists);
1170             Server->RunData();
1171             delete File;
1172             File = 0;
1173             break;
1174          }
1175
1176          default:
1177          Fail(_("Internal error"));
1178          break;
1179       }
1180
1181       FailCounter = 0;
1182    }
1183
1184    return 0;
1185 }
1186                                                                         /*}}}*/
1187
1188 int main()
1189 {
1190    setlocale(LC_ALL, "");
1191
1192    HttpMethod Mth;
1193
1194    return Mth.Loop();
1195 }
1196
1197