methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.56 2003/02/12 15:33:36 doogie Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apti18n.h>
  29 #include <apt-pkg/fileutl.h>
  30 #include <apt-pkg/acquire-method.h>
  31 #include <apt-pkg/error.h>
  32 #include <apt-pkg/hashes.h>
  33
  34 #include <sys/stat.h>
  35 #include <sys/time.h>
  36 #include <utime.h>
  37 #include <unistd.h>
  38 #include <signal.h>
  39 #include <stdio.h>
  40 #include <errno.h>
  41 #include <string.h>
  42 #include <iostream>
  43
  44 // Internet stuff
  45 #include <netdb.h>
  46
  47 #include "connect.h"
  48 #include "rfc2553emu.h"
  49 #include "http.h"
  50
  51                                                                         /*}}}*/
  52 using namespace std;
  53
  54 string HttpMethod::FailFile;
  55 int HttpMethod::FailFd = -1;
  56 time_t HttpMethod::FailTime = 0;
  57 unsigned long PipelineDepth = 10;
  58 unsigned long TimeOut = 120;
  59 bool Debug = false;
  60
  61 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  62 // ---------------------------------------------------------------------
  63 /* */
  64 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
  65 {
  66    Buf = new unsigned char[Size];
  67    Reset();
  68 }
  69                                                                         /*}}}*/
  70 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  71 // ---------------------------------------------------------------------
  72 /* */
  73 void CircleBuf::Reset()
  74 {
  75    InP = 0;
  76    OutP = 0;
  77    StrPos = 0;
  78    MaxGet = (unsigned int)-1;
  79    OutQueue = string();
  80    if (Hash != 0)
  81    {
  82       delete Hash;
  83       Hash = new Hashes;
  84    }
  85 };
  86                                                                         /*}}}*/
  87 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  88 // ---------------------------------------------------------------------
  89 /* This fills up the buffer with as much data as is in the FD, assuming it
  90    is non-blocking.. */
  91 bool CircleBuf::Read(int Fd)
  92 {
  93    while (1)
  94    {
  95       // Woops, buffer is full
  96       if (InP - OutP == Size)
  97          return true;
  98
  99       // Write the buffer segment
 100       int Res;
 101       Res = read(Fd,Buf + (InP%Size),LeftRead());
 102
 103       if (Res == 0)
 104          return false;
 105       if (Res < 0)
 106       {
 107          if (errno == EAGAIN)
 108             return true;
 109          return false;
 110       }
 111
 112       if (InP == 0)
 113          gettimeofday(&Start,0);
 114       InP += Res;
 115    }
 116 }
 117                                                                         /*}}}*/
 118 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 119 // ---------------------------------------------------------------------
 120 /* This will hold the string in and fill the buffer with it as it empties */
 121 bool CircleBuf::Read(string Data)
 122 {
 123    OutQueue += Data;
 124    FillOut();
 125    return true;
 126 }
 127                                                                         /*}}}*/
 128 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 129 // ---------------------------------------------------------------------
 130 /* */
 131 void CircleBuf::FillOut()
 132 {
 133    if (OutQueue.empty() == true)
 134       return;
 135    while (1)
 136    {
 137       // Woops, buffer is full
 138       if (InP - OutP == Size)
 139          return;
 140
 141       // Write the buffer segment
 142       unsigned long Sz = LeftRead();
 143       if (OutQueue.length() - StrPos < Sz)
 144          Sz = OutQueue.length() - StrPos;
 145       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 146
 147       // Advance
 148       StrPos += Sz;
 149       InP += Sz;
 150       if (OutQueue.length() == StrPos)
 151       {
 152          StrPos = 0;
 153          OutQueue = "";
 154          return;
 155       }
 156    }
 157 }
 158                                                                         /*}}}*/
 159 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 160 // ---------------------------------------------------------------------
 161 /* This empties the buffer into the FD. */
 162 bool CircleBuf::Write(int Fd)
 163 {
 164    while (1)
 165    {
 166       FillOut();
 167
 168       // Woops, buffer is empty
 169       if (OutP == InP)
 170          return true;
 171
 172       if (OutP == MaxGet)
 173          return true;
 174
 175       // Write the buffer segment
 176       int Res;
 177       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 178
 179       if (Res == 0)
 180          return false;
 181       if (Res < 0)
 182       {
 183          if (errno == EAGAIN)
 184             return true;
 185
 186          return false;
 187       }
 188
 189       if (Hash != 0)
 190          Hash->Add(Buf + (OutP%Size),Res);
 191
 192       OutP += Res;
 193    }
 194 }
 195                                                                         /*}}}*/
 196 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 197 // ---------------------------------------------------------------------
 198 /* This copies till the first empty line */
 199 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 200 {
 201    // We cheat and assume it is unneeded to have more than one buffer load
 202    for (unsigned long I = OutP; I < InP; I++)
 203    {
 204       if (Buf[I%Size] != '\n')
 205          continue;
 206       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 207
 208       if (Single == false)
 209       {
 210          if (Buf[I%Size] != '\n')
 211             continue;
 212          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 213       }
 214
 215       if (I > InP)
 216          I = InP;
 217
 218       Data = "";
 219       while (OutP < I)
 220       {
 221          unsigned long Sz = LeftWrite();
 222          if (Sz == 0)
 223             return false;
 224          if (I - OutP < LeftWrite())
 225             Sz = I - OutP;
 226          Data += string((char *)(Buf + (OutP%Size)),Sz);
 227          OutP += Sz;
 228       }
 229       return true;
 230    }
 231    return false;
 232 }
 233                                                                         /*}}}*/
 234 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 235 // ---------------------------------------------------------------------
 236 /* */
 237 void CircleBuf::Stats()
 238 {
 239    if (InP == 0)
 240       return;
 241
 242    struct timeval Stop;
 243    gettimeofday(&Stop,0);
 244 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 245              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 246    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 247 }
 248                                                                         /*}}}*/
 249
 250 // ServerState::ServerState - Constructor                               /*{{{*/
 251 // ---------------------------------------------------------------------
 252 /* */
 253 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 254                         In(64*1024), Out(4*1024),
 255                         ServerName(Srv)
 256 {
 257    Reset();
 258 }
 259                                                                         /*}}}*/
 260 // ServerState::Open - Open a connection to the server                  /*{{{*/
 261 // ---------------------------------------------------------------------
 262 /* This opens a connection to the server. */
 263 bool ServerState::Open()
 264 {
 265    // Use the already open connection if possible.
 266    if (ServerFd != -1)
 267       return true;
 268
 269    Close();
 270    In.Reset();
 271    Out.Reset();
 272    Persistent = true;
 273
 274    // Determine the proxy setting
 275    if (getenv("http_proxy") == 0)
 276    {
 277       string DefProxy = _config->Find("Acquire::http::Proxy");
 278       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 279       if (SpecificProxy.empty() == false)
 280       {
 281          if (SpecificProxy == "DIRECT")
 282             Proxy = "";
 283          else
 284             Proxy = SpecificProxy;
 285       }
 286       else
 287          Proxy = DefProxy;
 288    }
 289    else
 290       Proxy = getenv("http_proxy");
 291
 292    // Parse no_proxy, a , separated list of domains
 293    if (getenv("no_proxy") != 0)
 294    {
 295       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 296          Proxy = "";
 297    }
 298
 299    // Determine what host and port to use based on the proxy settings
 300    int Port = 0;
 301    string Host;
 302    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 303    {
 304       if (ServerName.Port != 0)
 305          Port = ServerName.Port;
 306       Host = ServerName.Host;
 307    }
 308    else
 309    {
 310       if (Proxy.Port != 0)
 311          Port = Proxy.Port;
 312       Host = Proxy.Host;
 313    }
 314
 315    // Connect to the remote server
 316    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 317       return false;
 318
 319    return true;
 320 }
 321                                                                         /*}}}*/
 322 // ServerState::Close - Close a connection to the server                /*{{{*/
 323 // ---------------------------------------------------------------------
 324 /* */
 325 bool ServerState::Close()
 326 {
 327    close(ServerFd);
 328    ServerFd = -1;
 329    return true;
 330 }
 331                                                                         /*}}}*/
 332 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 333 // ---------------------------------------------------------------------
 334 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 335    parse error occured */
 336 int ServerState::RunHeaders()
 337 {
 338    State = Header;
 339
 340    Owner->Status(_("Waiting for headers"));
 341
 342    Major = 0;
 343    Minor = 0;
 344    Result = 0;
 345    Size = 0;
 346    StartPos = 0;
 347    Encoding = Closes;
 348    HaveContent = false;
 349    time(&Date);
 350
 351    do
 352    {
 353       string Data;
 354       if (In.WriteTillEl(Data) == false)
 355          continue;
 356
 357       if (Debug == true)
 358          clog << Data;
 359
 360       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 361       {
 362          string::const_iterator J = I;
 363          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 364          if (HeaderLine(string(I,J)) == false)
 365             return 2;
 366          I = J;
 367       }
 368
 369       // 100 Continue is a Nop...
 370       if (Result == 100)
 371          continue;
 372
 373       // Tidy up the connection persistance state.
 374       if (Encoding == Closes && HaveContent == true)
 375          Persistent = false;
 376
 377       return 0;
 378    }
 379    while (Owner->Go(false,this) == true);
 380
 381    return 1;
 382 }
 383                                                                         /*}}}*/
 384 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 385 // ---------------------------------------------------------------------
 386 /* */
 387 bool ServerState::RunData()
 388 {
 389    State = Data;
 390
 391    // Chunked transfer encoding is fun..
 392    if (Encoding == Chunked)
 393    {
 394       while (1)
 395       {
 396          // Grab the block size
 397          bool Last = true;
 398          string Data;
 399          In.Limit(-1);
 400          do
 401          {
 402             if (In.WriteTillEl(Data,true) == true)
 403                break;
 404          }
 405          while ((Last = Owner->Go(false,this)) == true);
 406
 407          if (Last == false)
 408             return false;
 409
 410          // See if we are done
 411          unsigned long Len = strtol(Data.c_str(),0,16);
 412          if (Len == 0)
 413          {
 414             In.Limit(-1);
 415
 416             // We have to remove the entity trailer
 417             Last = true;
 418             do
 419             {
 420                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 421                   break;
 422             }
 423             while ((Last = Owner->Go(false,this)) == true);
 424             if (Last == false)
 425                return false;
 426             return !_error->PendingError();
 427          }
 428
 429          // Transfer the block
 430          In.Limit(Len);
 431          while (Owner->Go(true,this) == true)
 432             if (In.IsLimit() == true)
 433                break;
 434
 435          // Error
 436          if (In.IsLimit() == false)
 437             return false;
 438
 439          // The server sends an extra new line before the next block specifier..
 440          In.Limit(-1);
 441          Last = true;
 442          do
 443          {
 444             if (In.WriteTillEl(Data,true) == true)
 445                break;
 446          }
 447          while ((Last = Owner->Go(false,this)) == true);
 448          if (Last == false)
 449             return false;
 450       }
 451    }
 452    else
 453    {
 454       /* Closes encoding is used when the server did not specify a size, the
 455          loss of the connection means we are done */
 456       if (Encoding == Closes)
 457          In.Limit(-1);
 458       else
 459          In.Limit(Size - StartPos);
 460
 461       // Just transfer the whole block.
 462       do
 463       {
 464          if (In.IsLimit() == false)
 465             continue;
 466
 467          In.Limit(-1);
 468          return !_error->PendingError();
 469       }
 470       while (Owner->Go(true,this) == true);
 471    }
 472
 473    return Owner->Flush(this) && !_error->PendingError();
 474 }
 475                                                                         /*}}}*/
 476 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 477 // ---------------------------------------------------------------------
 478 /* */
 479 bool ServerState::HeaderLine(string Line)
 480 {
 481    if (Line.empty() == true)
 482       return true;
 483
 484    // The http server might be trying to do something evil.
 485    if (Line.length() >= MAXLEN)
 486       return _error->Error(_("Got a single header line over %u chars"),MAXLEN);
 487
 488    string::size_type Pos = Line.find(' ');
 489    if (Pos == string::npos || Pos+1 > Line.length())
 490    {
 491       // Blah, some servers use "connection:closes", evil.
 492       Pos = Line.find(':');
 493       if (Pos == string::npos || Pos + 2 > Line.length())
 494          return _error->Error(_("Bad header line"));
 495       Pos++;
 496    }
 497
 498    // Parse off any trailing spaces between the : and the next word.
 499    string::size_type Pos2 = Pos;
 500    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 501       Pos2++;
 502
 503    string Tag = string(Line,0,Pos);
 504    string Val = string(Line,Pos2);
 505
 506    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 507    {
 508       // Evil servers return no version
 509       if (Line[4] == '/')
 510       {
 511          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 512                     &Result,Code) != 4)
 513             return _error->Error(_("The http server sent an invalid reply header"));
 514       }
 515       else
 516       {
 517          Major = 0;
 518          Minor = 9;
 519          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 520             return _error->Error(_("The http server sent an invalid reply header"));
 521       }
 522
 523       /* Check the HTTP response header to get the default persistance
 524          state. */
 525       if (Major < 1)
 526          Persistent = false;
 527       else
 528       {
 529          if (Major == 1 && Minor <= 0)
 530             Persistent = false;
 531          else
 532             Persistent = true;
 533       }
 534
 535       return true;
 536    }
 537
 538    if (stringcasecmp(Tag,"Content-Length:") == 0)
 539    {
 540       if (Encoding == Closes)
 541          Encoding = Stream;
 542       HaveContent = true;
 543
 544       // The length is already set from the Content-Range header
 545       if (StartPos != 0)
 546          return true;
 547
 548       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 549          return _error->Error(_("The http server sent an invalid Content-Length header"));
 550       return true;
 551    }
 552
 553    if (stringcasecmp(Tag,"Content-Type:") == 0)
 554    {
 555       HaveContent = true;
 556       return true;
 557    }
 558
 559    if (stringcasecmp(Tag,"Content-Range:") == 0)
 560    {
 561       HaveContent = true;
 562
 563       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 564          return _error->Error(_("The http server sent an invalid Content-Range header"));
 565       if ((unsigned)StartPos > Size)
 566          return _error->Error(_("This http server has broken range support"));
 567       return true;
 568    }
 569
 570    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 571    {
 572       HaveContent = true;
 573       if (stringcasecmp(Val,"chunked") == 0)
 574          Encoding = Chunked;
 575       return true;
 576    }
 577
 578    if (stringcasecmp(Tag,"Connection:") == 0)
 579    {
 580       if (stringcasecmp(Val,"close") == 0)
 581          Persistent = false;
 582       if (stringcasecmp(Val,"keep-alive") == 0)
 583          Persistent = true;
 584       return true;
 585    }
 586
 587    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 588    {
 589       if (StrToTime(Val,Date) == false)
 590          return _error->Error(_("Unknown date format"));
 591       return true;
 592    }
 593
 594    return true;
 595 }
 596                                                                         /*}}}*/
 597
 598 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 599 // ---------------------------------------------------------------------
 600 /* This places the http request in the outbound buffer */
 601 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 602 {
 603    URI Uri = Itm->Uri;
 604
 605    // The HTTP server expects a hostname with a trailing :port
 606    char Buf[1000];
 607    string ProperHost = Uri.Host;
 608    if (Uri.Port != 0)
 609    {
 610       sprintf(Buf,":%u",Uri.Port);
 611       ProperHost += Buf;
 612    }
 613
 614    // Just in case.
 615    if (Itm->Uri.length() >= sizeof(Buf))
 616        abort();
 617
 618    /* Build the request. We include a keep-alive header only for non-proxy
 619       requests. This is to tweak old http/1.0 servers that do support keep-alive
 620       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 621       will glitch HTTP/1.0 proxies because they do not filter it out and
 622       pass it on, HTTP/1.1 says the connection should default to keep alive
 623       and we expect the proxy to do this */
 624    if (Proxy.empty() == true)
 625       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 626               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 627    else
 628    {
 629       /* Generate a cache control header if necessary. We place a max
 630          cache age on index files, optionally set a no-cache directive
 631          and a no-store directive for archives. */
 632       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 633               Itm->Uri.c_str(),ProperHost.c_str());
 634       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 635          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 636       else
 637       {
 638          if (Itm->IndexFile == true)
 639             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 640                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 641          else
 642          {
 643             if (_config->FindB("Acquire::http::No-Store",false) == true)
 644                strcat(Buf,"Cache-Control: no-store\r\n");
 645          }
 646       }
 647    }
 648
 649    string Req = Buf;
 650
 651    // Check for a partial file
 652    struct stat SBuf;
 653    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 654    {
 655       // In this case we send an if-range query with a range header
 656       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 657               TimeRFC1123(SBuf.st_mtime).c_str());
 658       Req += Buf;
 659    }
 660    else
 661    {
 662       if (Itm->LastModified != 0)
 663       {
 664          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 665          Req += Buf;
 666       }
 667    }
 668
 669    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 670       Req += string("Proxy-Authorization: Basic ") +
 671           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 672
 673    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 674       Req += string("Authorization: Basic ") +
 675           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 676
 677    Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
 678
 679    if (Debug == true)
 680       cerr << Req << endl;
 681
 682    Out.Read(Req);
 683 }
 684                                                                         /*}}}*/
 685 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 686 // ---------------------------------------------------------------------
 687 /* This runs the select loop over the server FDs, Output file FDs and
 688    stdin. */
 689 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 690 {
 691    // Server has closed the connection
 692    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 693                                ToFile == false))
 694       return false;
 695
 696    fd_set rfds,wfds;
 697    FD_ZERO(&rfds);
 698    FD_ZERO(&wfds);
 699
 700    /* Add the server. We only send more requests if the connection will
 701       be persisting */
 702    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 703        && Srv->Persistent == true)
 704       FD_SET(Srv->ServerFd,&wfds);
 705    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 706       FD_SET(Srv->ServerFd,&rfds);
 707
 708    // Add the file
 709    int FileFD = -1;
 710    if (File != 0)
 711       FileFD = File->Fd();
 712
 713    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 714       FD_SET(FileFD,&wfds);
 715
 716    // Add stdin
 717    FD_SET(STDIN_FILENO,&rfds);
 718
 719    // Figure out the max fd
 720    int MaxFd = FileFD;
 721    if (MaxFd < Srv->ServerFd)
 722       MaxFd = Srv->ServerFd;
 723
 724    // Select
 725    struct timeval tv;
 726    tv.tv_sec = TimeOut;
 727    tv.tv_usec = 0;
 728    int Res = 0;
 729    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 730    {
 731       if (errno == EINTR)
 732          return true;
 733       return _error->Errno("select",_("Select failed"));
 734    }
 735
 736    if (Res == 0)
 737    {
 738       _error->Error(_("Connection timed out"));
 739       return ServerDie(Srv);
 740    }
 741
 742    // Handle server IO
 743    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 744    {
 745       errno = 0;
 746       if (Srv->In.Read(Srv->ServerFd) == false)
 747          return ServerDie(Srv);
 748    }
 749
 750    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 751    {
 752       errno = 0;
 753       if (Srv->Out.Write(Srv->ServerFd) == false)
 754          return ServerDie(Srv);
 755    }
 756
 757    // Send data to the file
 758    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 759    {
 760       if (Srv->In.Write(FileFD) == false)
 761          return _error->Errno("write",_("Error writing to output file"));
 762    }
 763
 764    // Handle commands from APT
 765    if (FD_ISSET(STDIN_FILENO,&rfds))
 766    {
 767       if (Run(true) != -1)
 768          exit(100);
 769    }
 770
 771    return true;
 772 }
 773                                                                         /*}}}*/
 774 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 775 // ---------------------------------------------------------------------
 776 /* This takes the current input buffer from the Server FD and writes it
 777    into the file */
 778 bool HttpMethod::Flush(ServerState *Srv)
 779 {
 780    if (File != 0)
 781    {
 782       SetNonBlock(File->Fd(),false);
 783       if (Srv->In.WriteSpace() == false)
 784          return true;
 785
 786       while (Srv->In.WriteSpace() == true)
 787       {
 788          if (Srv->In.Write(File->Fd()) == false)
 789             return _error->Errno("write",_("Error writing to file"));
 790          if (Srv->In.IsLimit() == true)
 791             return true;
 792       }
 793
 794       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 795          return true;
 796    }
 797    return false;
 798 }
 799                                                                         /*}}}*/
 800 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 801 // ---------------------------------------------------------------------
 802 /* */
 803 bool HttpMethod::ServerDie(ServerState *Srv)
 804 {
 805    unsigned int LErrno = errno;
 806
 807    // Dump the buffer to the file
 808    if (Srv->State == ServerState::Data)
 809    {
 810       SetNonBlock(File->Fd(),false);
 811       while (Srv->In.WriteSpace() == true)
 812       {
 813          if (Srv->In.Write(File->Fd()) == false)
 814             return _error->Errno("write",_("Error writing to the file"));
 815
 816          // Done
 817          if (Srv->In.IsLimit() == true)
 818             return true;
 819       }
 820    }
 821
 822    // See if this is because the server finished the data stream
 823    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 824        Srv->Encoding != ServerState::Closes)
 825    {
 826       Srv->Close();
 827       if (LErrno == 0)
 828          return _error->Error(_("Error reading from server Remote end closed connection"));
 829       errno = LErrno;
 830       return _error->Errno("read",_("Error reading from server"));
 831    }
 832    else
 833    {
 834       Srv->In.Limit(-1);
 835
 836       // Nothing left in the buffer
 837       if (Srv->In.WriteSpace() == false)
 838          return false;
 839
 840       // We may have got multiple responses back in one packet..
 841       Srv->Close();
 842       return true;
 843    }
 844
 845    return false;
 846 }
 847                                                                         /*}}}*/
 848 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 849 // ---------------------------------------------------------------------
 850 /* We look at the header data we got back from the server and decide what
 851    to do. Returns
 852      0 - File is open,
 853      1 - IMS hit
 854      3 - Unrecoverable error
 855      4 - Error with error content page
 856      5 - Unrecoverable non-server error (close the connection) */
 857 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 858 {
 859    // Not Modified
 860    if (Srv->Result == 304)
 861    {
 862       unlink(Queue->DestFile.c_str());
 863       Res.IMSHit = true;
 864       Res.LastModified = Queue->LastModified;
 865       return 1;
 866    }
 867
 868    /* We have a reply we dont handle. This should indicate a perm server
 869       failure */
 870    if (Srv->Result < 200 || Srv->Result >= 300)
 871    {
 872       _error->Error("%u %s",Srv->Result,Srv->Code);
 873       if (Srv->HaveContent == true)
 874          return 4;
 875       return 3;
 876    }
 877
 878    // This is some sort of 2xx 'data follows' reply
 879    Res.LastModified = Srv->Date;
 880    Res.Size = Srv->Size;
 881
 882    // Open the file
 883    delete File;
 884    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 885    if (_error->PendingError() == true)
 886       return 5;
 887
 888    FailFile = Queue->DestFile;
 889    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 890    FailFd = File->Fd();
 891    FailTime = Srv->Date;
 892
 893    // Set the expected size
 894    if (Srv->StartPos >= 0)
 895    {
 896       Res.ResumePoint = Srv->StartPos;
 897       ftruncate(File->Fd(),Srv->StartPos);
 898    }
 899
 900    // Set the start point
 901    lseek(File->Fd(),0,SEEK_END);
 902
 903    delete Srv->In.Hash;
 904    Srv->In.Hash = new Hashes;
 905
 906    // Fill the Hash if the file is non-empty (resume)
 907    if (Srv->StartPos > 0)
 908    {
 909       lseek(File->Fd(),0,SEEK_SET);
 910       if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
 911       {
 912          _error->Errno("read",_("Problem hashing file"));
 913          return 5;
 914       }
 915       lseek(File->Fd(),0,SEEK_END);
 916    }
 917
 918    SetNonBlock(File->Fd(),true);
 919    return 0;
 920 }
 921                                                                         /*}}}*/
 922 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 923 // ---------------------------------------------------------------------
 924 /* This closes and timestamps the open file. This is neccessary to get
 925    resume behavoir on user abort */
 926 void HttpMethod::SigTerm(int)
 927 {
 928    if (FailFd == -1)
 929       _exit(100);
 930    close(FailFd);
 931
 932    // Timestamp
 933    struct utimbuf UBuf;
 934    UBuf.actime = FailTime;
 935    UBuf.modtime = FailTime;
 936    utime(FailFile.c_str(),&UBuf);
 937
 938    _exit(100);
 939 }
 940                                                                         /*}}}*/
 941 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 942 // ---------------------------------------------------------------------
 943 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 944    depth. */
 945 bool HttpMethod::Fetch(FetchItem *)
 946 {
 947    if (Server == 0)
 948       return true;
 949
 950    // Queue the requests
 951    int Depth = -1;
 952    bool Tail = false;
 953    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 954         I = I->Next, Depth++)
 955    {
 956       // If pipelining is disabled, we only queue 1 request
 957       if (Server->Pipeline == false && Depth >= 0)
 958          break;
 959
 960       // Make sure we stick with the same server
 961       if (Server->Comp(I->Uri) == false)
 962          break;
 963       if (QueueBack == I)
 964          Tail = true;
 965       if (Tail == true)
 966       {
 967          QueueBack = I->Next;
 968          SendReq(I,Server->Out);
 969          continue;
 970       }
 971    }
 972
 973    return true;
 974 };
 975                                                                         /*}}}*/
 976 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 977 // ---------------------------------------------------------------------
 978 /* We stash the desired pipeline depth */
 979 bool HttpMethod::Configuration(string Message)
 980 {
 981    if (pkgAcqMethod::Configuration(Message) == false)
 982       return false;
 983
 984    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 985    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 986                                   PipelineDepth);
 987    Debug = _config->FindB("Debug::Acquire::http",false);
 988
 989    return true;
 990 }
 991                                                                         /*}}}*/
 992 // HttpMethod::Loop - Main loop                                         /*{{{*/
 993 // ---------------------------------------------------------------------
 994 /* */
 995 int HttpMethod::Loop()
 996 {
 997    signal(SIGTERM,SigTerm);
 998    signal(SIGINT,SigTerm);
 999
1000    Server = 0;
1001
1002    int FailCounter = 0;
1003    while (1)
1004    {
1005       // We have no commands, wait for some to arrive
1006       if (Queue == 0)
1007       {
1008          if (WaitFd(STDIN_FILENO) == false)
1009             return 0;
1010       }
1011
1012       /* Run messages, we can accept 0 (no message) if we didn't
1013          do a WaitFd above.. Otherwise the FD is closed. */
1014       int Result = Run(true);
1015       if (Result != -1 && (Result != 0 || Queue == 0))
1016          return 100;
1017
1018       if (Queue == 0)
1019          continue;
1020
1021       // Connect to the server
1022       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1023       {
1024          delete Server;
1025          Server = new ServerState(Queue->Uri,this);
1026       }
1027
1028       /* If the server has explicitly said this is the last connection
1029          then we pre-emptively shut down the pipeline and tear down
1030          the connection. This will speed up HTTP/1.0 servers a tad
1031          since we don't have to wait for the close sequence to
1032          complete */
1033       if (Server->Persistent == false)
1034          Server->Close();
1035
1036       // Reset the pipeline
1037       if (Server->ServerFd == -1)
1038          QueueBack = Queue;
1039
1040       // Connnect to the host
1041       if (Server->Open() == false)
1042       {
1043          Fail(true);
1044          delete Server;
1045          Server = 0;
1046          continue;
1047       }
1048
1049       // Fill the pipeline.
1050       Fetch(0);
1051
1052       // Fetch the next URL header data from the server.
1053       switch (Server->RunHeaders())
1054       {
1055          case 0:
1056          break;
1057
1058          // The header data is bad
1059          case 2:
1060          {
1061             _error->Error(_("Bad header Data"));
1062             Fail(true);
1063             RotateDNS();
1064             continue;
1065          }
1066
1067          // The server closed a connection during the header get..
1068          default:
1069          case 1:
1070          {
1071             FailCounter++;
1072             _error->Discard();
1073             Server->Close();
1074             Server->Pipeline = false;
1075
1076             if (FailCounter >= 2)
1077             {
1078                Fail(_("Connection failed"),true);
1079                FailCounter = 0;
1080             }
1081
1082             RotateDNS();
1083             continue;
1084          }
1085       };
1086
1087       // Decide what to do.
1088       FetchResult Res;
1089       Res.Filename = Queue->DestFile;
1090       switch (DealWithHeaders(Res,Server))
1091       {
1092          // Ok, the file is Open
1093          case 0:
1094          {
1095             URIStart(Res);
1096
1097             // Run the data
1098             bool Result =  Server->RunData();
1099
1100             /* If the server is sending back sizeless responses then fill in
1101                the size now */
1102             if (Res.Size == 0)
1103                Res.Size = File->Size();
1104
1105             // Close the file, destroy the FD object and timestamp it
1106             FailFd = -1;
1107             delete File;
1108             File = 0;
1109
1110             // Timestamp
1111             struct utimbuf UBuf;
1112             time(&UBuf.actime);
1113             UBuf.actime = Server->Date;
1114             UBuf.modtime = Server->Date;
1115             utime(Queue->DestFile.c_str(),&UBuf);
1116
1117             // Send status to APT
1118             if (Result == true)
1119             {
1120                Res.TakeHashes(*Server->In.Hash);
1121                URIDone(Res);
1122             }
1123             else
1124                Fail(true);
1125
1126             break;
1127          }
1128
1129          // IMS hit
1130          case 1:
1131          {
1132             URIDone(Res);
1133             break;
1134          }
1135
1136          // Hard server error, not found or something
1137          case 3:
1138          {
1139             Fail();
1140             break;
1141          }
1142
1143          // Hard internal error, kill the connection and fail
1144          case 5:
1145          {
1146             delete File;
1147             File = 0;
1148
1149             Fail();
1150             RotateDNS();
1151             Server->Close();
1152             break;
1153          }
1154
1155          // We need to flush the data, the header is like a 404 w/ error text
1156          case 4:
1157          {
1158             Fail();
1159
1160             // Send to content to dev/null
1161             File = new FileFd("/dev/null",FileFd::WriteExists);
1162             Server->RunData();
1163             delete File;
1164             File = 0;
1165             break;
1166          }
1167
1168          default:
1169          Fail(_("Internal error"));
1170          break;
1171       }
1172
1173       FailCounter = 0;
1174    }
1175
1176    return 0;
1177 }
1178                                                                         /*}}}*/
1179
1180 int main()
1181 {
1182    HttpMethod Mth;
1183
1184    return Mth.Loop();
1185 }
1186
1187