methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apt-pkg/fileutl.h>
  29 #include <apt-pkg/acquire-method.h>
  30 #include <apt-pkg/error.h>
  31 #include <apt-pkg/hashes.h>
  32
  33 #include <sys/stat.h>
  34 #include <sys/time.h>
  35 #include <utime.h>
  36 #include <unistd.h>
  37 #include <signal.h>
  38 #include <stdio.h>
  39 #include <errno.h>
  40 #include <string.h>
  41 #include <iostream>
  42 #include <apti18n.h>
  43
  44 // Internet stuff
  45 #include <netdb.h>
  46
  47 #include "connect.h"
  48 #include "rfc2553emu.h"
  49 #include "http.h"
  50
  51                                                                         /*}}}*/
  52 using namespace std;
  53
  54 string HttpMethod::FailFile;
  55 int HttpMethod::FailFd = -1;
  56 time_t HttpMethod::FailTime = 0;
  57 unsigned long PipelineDepth = 10;
  58 unsigned long TimeOut = 120;
  59 bool Debug = false;
  60
  61 unsigned long CircleBuf::BwReadLimit=0;
  62 unsigned long CircleBuf::BwTickReadData=0;
  63 struct timeval CircleBuf::BwReadTick={0,0};
  64 const unsigned int CircleBuf::BW_HZ=10;
  65
  66 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  67 // ---------------------------------------------------------------------
  68 /* */
  69 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
  70 {
  71    Buf = new unsigned char[Size];
  72    Reset();
  73
  74    CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
  75 }
  76                                                                         /*}}}*/
  77 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  78 // ---------------------------------------------------------------------
  79 /* */
  80 void CircleBuf::Reset()
  81 {
  82    InP = 0;
  83    OutP = 0;
  84    StrPos = 0;
  85    MaxGet = (unsigned int)-1;
  86    OutQueue = string();
  87    if (Hash != 0)
  88    {
  89       delete Hash;
  90       Hash = new Hashes;
  91    }
  92 };
  93                                                                         /*}}}*/
  94 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  95 // ---------------------------------------------------------------------
  96 /* This fills up the buffer with as much data as is in the FD, assuming it
  97    is non-blocking.. */
  98 bool CircleBuf::Read(int Fd)
  99 {
 100    unsigned long BwReadMax;
 101
 102    while (1)
 103    {
 104       // Woops, buffer is full
 105       if (InP - OutP == Size)
 106          return true;
 107
 108       // what's left to read in this tick
 109       BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
 110
 111       if(CircleBuf::BwReadLimit) {
 112          struct timeval now;
 113          gettimeofday(&now,0);
 114
 115          unsigned long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
 116             now.tv_usec-CircleBuf::BwReadTick.tv_usec;
 117          if(d > 1000000/BW_HZ) {
 118             CircleBuf::BwReadTick = now;
 119             CircleBuf::BwTickReadData = 0;
 120          }
 121
 122          if(CircleBuf::BwTickReadData >= BwReadMax) {
 123             usleep(1000000/BW_HZ);
 124             return true;
 125          }
 126       }
 127
 128       // Write the buffer segment
 129       int Res;
 130       if(CircleBuf::BwReadLimit) {
 131          Res = read(Fd,Buf + (InP%Size),
 132                     BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
 133       } else
 134          Res = read(Fd,Buf + (InP%Size),LeftRead());
 135
 136       if(Res > 0 && BwReadLimit > 0)
 137          CircleBuf::BwTickReadData += Res;
 138
 139       if (Res == 0)
 140          return false;
 141       if (Res < 0)
 142       {
 143          if (errno == EAGAIN)
 144             return true;
 145          return false;
 146       }
 147
 148       if (InP == 0)
 149          gettimeofday(&Start,0);
 150       InP += Res;
 151    }
 152 }
 153                                                                         /*}}}*/
 154 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 155 // ---------------------------------------------------------------------
 156 /* This will hold the string in and fill the buffer with it as it empties */
 157 bool CircleBuf::Read(string Data)
 158 {
 159    OutQueue += Data;
 160    FillOut();
 161    return true;
 162 }
 163                                                                         /*}}}*/
 164 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 165 // ---------------------------------------------------------------------
 166 /* */
 167 void CircleBuf::FillOut()
 168 {
 169    if (OutQueue.empty() == true)
 170       return;
 171    while (1)
 172    {
 173       // Woops, buffer is full
 174       if (InP - OutP == Size)
 175          return;
 176
 177       // Write the buffer segment
 178       unsigned long Sz = LeftRead();
 179       if (OutQueue.length() - StrPos < Sz)
 180          Sz = OutQueue.length() - StrPos;
 181       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 182
 183       // Advance
 184       StrPos += Sz;
 185       InP += Sz;
 186       if (OutQueue.length() == StrPos)
 187       {
 188          StrPos = 0;
 189          OutQueue = "";
 190          return;
 191       }
 192    }
 193 }
 194                                                                         /*}}}*/
 195 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 196 // ---------------------------------------------------------------------
 197 /* This empties the buffer into the FD. */
 198 bool CircleBuf::Write(int Fd)
 199 {
 200    while (1)
 201    {
 202       FillOut();
 203
 204       // Woops, buffer is empty
 205       if (OutP == InP)
 206          return true;
 207
 208       if (OutP == MaxGet)
 209          return true;
 210
 211       // Write the buffer segment
 212       int Res;
 213       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 214
 215       if (Res == 0)
 216          return false;
 217       if (Res < 0)
 218       {
 219          if (errno == EAGAIN)
 220             return true;
 221
 222          return false;
 223       }
 224
 225       if (Hash != 0)
 226          Hash->Add(Buf + (OutP%Size),Res);
 227
 228       OutP += Res;
 229    }
 230 }
 231                                                                         /*}}}*/
 232 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 233 // ---------------------------------------------------------------------
 234 /* This copies till the first empty line */
 235 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 236 {
 237    // We cheat and assume it is unneeded to have more than one buffer load
 238    for (unsigned long I = OutP; I < InP; I++)
 239    {
 240       if (Buf[I%Size] != '\n')
 241          continue;
 242       ++I;
 243
 244       if (Single == false)
 245       {
 246          if (I < InP  && Buf[I%Size] == '\r')
 247             ++I;
 248          if (I >= InP || Buf[I%Size] != '\n')
 249             continue;
 250          ++I;
 251       }
 252
 253       Data = "";
 254       while (OutP < I)
 255       {
 256          unsigned long Sz = LeftWrite();
 257          if (Sz == 0)
 258             return false;
 259          if (I - OutP < Sz)
 260             Sz = I - OutP;
 261          Data += string((char *)(Buf + (OutP%Size)),Sz);
 262          OutP += Sz;
 263       }
 264       return true;
 265    }
 266    return false;
 267 }
 268                                                                         /*}}}*/
 269 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 270 // ---------------------------------------------------------------------
 271 /* */
 272 void CircleBuf::Stats()
 273 {
 274    if (InP == 0)
 275       return;
 276
 277    struct timeval Stop;
 278    gettimeofday(&Stop,0);
 279 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 280              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 281    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 282 }
 283                                                                         /*}}}*/
 284
 285 // ServerState::ServerState - Constructor                               /*{{{*/
 286 // ---------------------------------------------------------------------
 287 /* */
 288 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 289                         In(64*1024), Out(4*1024),
 290                         ServerName(Srv)
 291 {
 292    Reset();
 293 }
 294                                                                         /*}}}*/
 295 // ServerState::Open - Open a connection to the server                  /*{{{*/
 296 // ---------------------------------------------------------------------
 297 /* This opens a connection to the server. */
 298 bool ServerState::Open()
 299 {
 300    // Use the already open connection if possible.
 301    if (ServerFd != -1)
 302       return true;
 303
 304    Close();
 305    In.Reset();
 306    Out.Reset();
 307    Persistent = true;
 308
 309    // Determine the proxy setting
 310    if (getenv("http_proxy") == 0)
 311    {
 312       string DefProxy = _config->Find("Acquire::http::Proxy");
 313       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 314       if (SpecificProxy.empty() == false)
 315       {
 316          if (SpecificProxy == "DIRECT")
 317             Proxy = "";
 318          else
 319             Proxy = SpecificProxy;
 320       }
 321       else
 322          Proxy = DefProxy;
 323    }
 324    else
 325       Proxy = getenv("http_proxy");
 326
 327    // Parse no_proxy, a , separated list of domains
 328    if (getenv("no_proxy") != 0)
 329    {
 330       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 331          Proxy = "";
 332    }
 333
 334    // Determine what host and port to use based on the proxy settings
 335    int Port = 0;
 336    string Host;
 337    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 338    {
 339       if (ServerName.Port != 0)
 340          Port = ServerName.Port;
 341       Host = ServerName.Host;
 342    }
 343    else
 344    {
 345       if (Proxy.Port != 0)
 346          Port = Proxy.Port;
 347       Host = Proxy.Host;
 348    }
 349
 350    // Connect to the remote server
 351    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 352       return false;
 353
 354    return true;
 355 }
 356                                                                         /*}}}*/
 357 // ServerState::Close - Close a connection to the server                /*{{{*/
 358 // ---------------------------------------------------------------------
 359 /* */
 360 bool ServerState::Close()
 361 {
 362    close(ServerFd);
 363    ServerFd = -1;
 364    return true;
 365 }
 366                                                                         /*}}}*/
 367 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 368 // ---------------------------------------------------------------------
 369 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 370    parse error occured */
 371 int ServerState::RunHeaders()
 372 {
 373    State = Header;
 374
 375    Owner->Status(_("Waiting for headers"));
 376
 377    Major = 0;
 378    Minor = 0;
 379    Result = 0;
 380    Size = 0;
 381    StartPos = 0;
 382    Encoding = Closes;
 383    HaveContent = false;
 384    time(&Date);
 385
 386    do
 387    {
 388       string Data;
 389       if (In.WriteTillEl(Data) == false)
 390          continue;
 391
 392       if (Debug == true)
 393          clog << Data;
 394
 395       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 396       {
 397          string::const_iterator J = I;
 398          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 399          if (HeaderLine(string(I,J)) == false)
 400             return 2;
 401          I = J;
 402       }
 403
 404       // 100 Continue is a Nop...
 405       if (Result == 100)
 406          continue;
 407
 408       // Tidy up the connection persistance state.
 409       if (Encoding == Closes && HaveContent == true)
 410          Persistent = false;
 411
 412       return 0;
 413    }
 414    while (Owner->Go(false,this) == true);
 415
 416    return 1;
 417 }
 418                                                                         /*}}}*/
 419 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 420 // ---------------------------------------------------------------------
 421 /* */
 422 bool ServerState::RunData()
 423 {
 424    State = Data;
 425
 426    // Chunked transfer encoding is fun..
 427    if (Encoding == Chunked)
 428    {
 429       while (1)
 430       {
 431          // Grab the block size
 432          bool Last = true;
 433          string Data;
 434          In.Limit(-1);
 435          do
 436          {
 437             if (In.WriteTillEl(Data,true) == true)
 438                break;
 439          }
 440          while ((Last = Owner->Go(false,this)) == true);
 441
 442          if (Last == false)
 443             return false;
 444
 445          // See if we are done
 446          unsigned long Len = strtol(Data.c_str(),0,16);
 447          if (Len == 0)
 448          {
 449             In.Limit(-1);
 450
 451             // We have to remove the entity trailer
 452             Last = true;
 453             do
 454             {
 455                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 456                   break;
 457             }
 458             while ((Last = Owner->Go(false,this)) == true);
 459             if (Last == false)
 460                return false;
 461             return !_error->PendingError();
 462          }
 463
 464          // Transfer the block
 465          In.Limit(Len);
 466          while (Owner->Go(true,this) == true)
 467             if (In.IsLimit() == true)
 468                break;
 469
 470          // Error
 471          if (In.IsLimit() == false)
 472             return false;
 473
 474          // The server sends an extra new line before the next block specifier..
 475          In.Limit(-1);
 476          Last = true;
 477          do
 478          {
 479             if (In.WriteTillEl(Data,true) == true)
 480                break;
 481          }
 482          while ((Last = Owner->Go(false,this)) == true);
 483          if (Last == false)
 484             return false;
 485       }
 486    }
 487    else
 488    {
 489       /* Closes encoding is used when the server did not specify a size, the
 490          loss of the connection means we are done */
 491       if (Encoding == Closes)
 492          In.Limit(-1);
 493       else
 494          In.Limit(Size - StartPos);
 495
 496       // Just transfer the whole block.
 497       do
 498       {
 499          if (In.IsLimit() == false)
 500             continue;
 501
 502          In.Limit(-1);
 503          return !_error->PendingError();
 504       }
 505       while (Owner->Go(true,this) == true);
 506    }
 507
 508    return Owner->Flush(this) && !_error->PendingError();
 509 }
 510                                                                         /*}}}*/
 511 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 512 // ---------------------------------------------------------------------
 513 /* */
 514 bool ServerState::HeaderLine(string Line)
 515 {
 516    if (Line.empty() == true)
 517       return true;
 518
 519    // The http server might be trying to do something evil.
 520    if (Line.length() >= MAXLEN)
 521       return _error->Error(_("Got a single header line over %u chars"),MAXLEN);
 522
 523    string::size_type Pos = Line.find(' ');
 524    if (Pos == string::npos || Pos+1 > Line.length())
 525    {
 526       // Blah, some servers use "connection:closes", evil.
 527       Pos = Line.find(':');
 528       if (Pos == string::npos || Pos + 2 > Line.length())
 529          return _error->Error(_("Bad header line"));
 530       Pos++;
 531    }
 532
 533    // Parse off any trailing spaces between the : and the next word.
 534    string::size_type Pos2 = Pos;
 535    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 536       Pos2++;
 537
 538    string Tag = string(Line,0,Pos);
 539    string Val = string(Line,Pos2);
 540
 541    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 542    {
 543       // Evil servers return no version
 544       if (Line[4] == '/')
 545       {
 546          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 547                     &Result,Code) != 4)
 548             return _error->Error(_("The HTTP server sent an invalid reply header"));
 549       }
 550       else
 551       {
 552          Major = 0;
 553          Minor = 9;
 554          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 555             return _error->Error(_("The HTTP server sent an invalid reply header"));
 556       }
 557
 558       /* Check the HTTP response header to get the default persistance
 559          state. */
 560       if (Major < 1)
 561          Persistent = false;
 562       else
 563       {
 564          if (Major == 1 && Minor <= 0)
 565             Persistent = false;
 566          else
 567             Persistent = true;
 568       }
 569
 570       return true;
 571    }
 572
 573    if (stringcasecmp(Tag,"Content-Length:") == 0)
 574    {
 575       if (Encoding == Closes)
 576          Encoding = Stream;
 577       HaveContent = true;
 578
 579       // The length is already set from the Content-Range header
 580       if (StartPos != 0)
 581          return true;
 582
 583       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 584          return _error->Error(_("The HTTP server sent an invalid Content-Length header"));
 585       return true;
 586    }
 587
 588    if (stringcasecmp(Tag,"Content-Type:") == 0)
 589    {
 590       HaveContent = true;
 591       return true;
 592    }
 593
 594    if (stringcasecmp(Tag,"Content-Range:") == 0)
 595    {
 596       HaveContent = true;
 597
 598       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 599          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 600       if ((unsigned)StartPos > Size)
 601          return _error->Error(_("This HTTP server has broken range support"));
 602       return true;
 603    }
 604
 605    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 606    {
 607       HaveContent = true;
 608       if (stringcasecmp(Val,"chunked") == 0)
 609          Encoding = Chunked;
 610       return true;
 611    }
 612
 613    if (stringcasecmp(Tag,"Connection:") == 0)
 614    {
 615       if (stringcasecmp(Val,"close") == 0)
 616          Persistent = false;
 617       if (stringcasecmp(Val,"keep-alive") == 0)
 618          Persistent = true;
 619       return true;
 620    }
 621
 622    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 623    {
 624       if (StrToTime(Val,Date) == false)
 625          return _error->Error(_("Unknown date format"));
 626       return true;
 627    }
 628
 629    return true;
 630 }
 631                                                                         /*}}}*/
 632
 633 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 634 // ---------------------------------------------------------------------
 635 /* This places the http request in the outbound buffer */
 636 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 637 {
 638    URI Uri = Itm->Uri;
 639
 640    // The HTTP server expects a hostname with a trailing :port
 641    char Buf[1000];
 642    string ProperHost = Uri.Host;
 643    if (Uri.Port != 0)
 644    {
 645       sprintf(Buf,":%u",Uri.Port);
 646       ProperHost += Buf;
 647    }
 648
 649    // Just in case.
 650    if (Itm->Uri.length() >= sizeof(Buf))
 651        abort();
 652
 653    /* Build the request. We include a keep-alive header only for non-proxy
 654       requests. This is to tweak old http/1.0 servers that do support keep-alive
 655       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 656       will glitch HTTP/1.0 proxies because they do not filter it out and
 657       pass it on, HTTP/1.1 says the connection should default to keep alive
 658       and we expect the proxy to do this */
 659    if (Proxy.empty() == true || Proxy.Host.empty())
 660       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 661               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 662    else
 663    {
 664       /* Generate a cache control header if necessary. We place a max
 665          cache age on index files, optionally set a no-cache directive
 666          and a no-store directive for archives. */
 667       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 668               Itm->Uri.c_str(),ProperHost.c_str());
 669       // only generate a cache control header if we actually want to
 670       // use a cache
 671       if (_config->FindB("Acquire::http::No-Cache",false) == false)
 672       {
 673          if (Itm->IndexFile == true)
 674             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 675                     _config->FindI("Acquire::http::Max-Age",0));
 676          else
 677          {
 678             if (_config->FindB("Acquire::http::No-Store",false) == true)
 679                strcat(Buf,"Cache-Control: no-store\r\n");
 680          }
 681       }
 682    }
 683    // generate a no-cache header if needed
 684    if (_config->FindB("Acquire::http::No-Cache",false) == true)
 685       strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 686
 687
 688    string Req = Buf;
 689
 690    // Check for a partial file
 691    struct stat SBuf;
 692    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 693    {
 694       // In this case we send an if-range query with a range header
 695       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 696               TimeRFC1123(SBuf.st_mtime).c_str());
 697       Req += Buf;
 698    }
 699    else
 700    {
 701       if (Itm->LastModified != 0)
 702       {
 703          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 704          Req += Buf;
 705       }
 706    }
 707
 708    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 709       Req += string("Proxy-Authorization: Basic ") +
 710           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 711
 712    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 713       Req += string("Authorization: Basic ") +
 714           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 715
 716    Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
 717
 718    if (Debug == true)
 719       cerr << Req << endl;
 720
 721    Out.Read(Req);
 722 }
 723                                                                         /*}}}*/
 724 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 725 // ---------------------------------------------------------------------
 726 /* This runs the select loop over the server FDs, Output file FDs and
 727    stdin. */
 728 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 729 {
 730    // Server has closed the connection
 731    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 732                                ToFile == false))
 733       return false;
 734
 735    fd_set rfds,wfds;
 736    FD_ZERO(&rfds);
 737    FD_ZERO(&wfds);
 738
 739    /* Add the server. We only send more requests if the connection will
 740       be persisting */
 741    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 742        && Srv->Persistent == true)
 743       FD_SET(Srv->ServerFd,&wfds);
 744    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 745       FD_SET(Srv->ServerFd,&rfds);
 746
 747    // Add the file
 748    int FileFD = -1;
 749    if (File != 0)
 750       FileFD = File->Fd();
 751
 752    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 753       FD_SET(FileFD,&wfds);
 754
 755    // Add stdin
 756    FD_SET(STDIN_FILENO,&rfds);
 757
 758    // Figure out the max fd
 759    int MaxFd = FileFD;
 760    if (MaxFd < Srv->ServerFd)
 761       MaxFd = Srv->ServerFd;
 762
 763    // Select
 764    struct timeval tv;
 765    tv.tv_sec = TimeOut;
 766    tv.tv_usec = 0;
 767    int Res = 0;
 768    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 769    {
 770       if (errno == EINTR)
 771          return true;
 772       return _error->Errno("select",_("Select failed"));
 773    }
 774
 775    if (Res == 0)
 776    {
 777       _error->Error(_("Connection timed out"));
 778       return ServerDie(Srv);
 779    }
 780
 781    // Handle server IO
 782    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 783    {
 784       errno = 0;
 785       if (Srv->In.Read(Srv->ServerFd) == false)
 786          return ServerDie(Srv);
 787    }
 788
 789    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 790    {
 791       errno = 0;
 792       if (Srv->Out.Write(Srv->ServerFd) == false)
 793          return ServerDie(Srv);
 794    }
 795
 796    // Send data to the file
 797    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 798    {
 799       if (Srv->In.Write(FileFD) == false)
 800          return _error->Errno("write",_("Error writing to output file"));
 801    }
 802
 803    // Handle commands from APT
 804    if (FD_ISSET(STDIN_FILENO,&rfds))
 805    {
 806       if (Run(true) != -1)
 807          exit(100);
 808    }
 809
 810    return true;
 811 }
 812                                                                         /*}}}*/
 813 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 814 // ---------------------------------------------------------------------
 815 /* This takes the current input buffer from the Server FD and writes it
 816    into the file */
 817 bool HttpMethod::Flush(ServerState *Srv)
 818 {
 819    if (File != 0)
 820    {
 821       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 822       // can't be set
 823       if (File->Name() != "/dev/null")
 824          SetNonBlock(File->Fd(),false);
 825       if (Srv->In.WriteSpace() == false)
 826          return true;
 827
 828       while (Srv->In.WriteSpace() == true)
 829       {
 830          if (Srv->In.Write(File->Fd()) == false)
 831             return _error->Errno("write",_("Error writing to file"));
 832          if (Srv->In.IsLimit() == true)
 833             return true;
 834       }
 835
 836       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 837          return true;
 838    }
 839    return false;
 840 }
 841                                                                         /*}}}*/
 842 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 843 // ---------------------------------------------------------------------
 844 /* */
 845 bool HttpMethod::ServerDie(ServerState *Srv)
 846 {
 847    unsigned int LErrno = errno;
 848
 849    // Dump the buffer to the file
 850    if (Srv->State == ServerState::Data)
 851    {
 852       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 853       // can't be set
 854       if (File->Name() != "/dev/null")
 855          SetNonBlock(File->Fd(),false);
 856       while (Srv->In.WriteSpace() == true)
 857       {
 858          if (Srv->In.Write(File->Fd()) == false)
 859             return _error->Errno("write",_("Error writing to the file"));
 860
 861          // Done
 862          if (Srv->In.IsLimit() == true)
 863             return true;
 864       }
 865    }
 866
 867    // See if this is because the server finished the data stream
 868    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 869        Srv->Encoding != ServerState::Closes)
 870    {
 871       Srv->Close();
 872       if (LErrno == 0)
 873          return _error->Error(_("Error reading from server. Remote end closed connection"));
 874       errno = LErrno;
 875       return _error->Errno("read",_("Error reading from server"));
 876    }
 877    else
 878    {
 879       Srv->In.Limit(-1);
 880
 881       // Nothing left in the buffer
 882       if (Srv->In.WriteSpace() == false)
 883          return false;
 884
 885       // We may have got multiple responses back in one packet..
 886       Srv->Close();
 887       return true;
 888    }
 889
 890    return false;
 891 }
 892                                                                         /*}}}*/
 893 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 894 // ---------------------------------------------------------------------
 895 /* We look at the header data we got back from the server and decide what
 896    to do. Returns
 897      0 - File is open,
 898      1 - IMS hit
 899      3 - Unrecoverable error
 900      4 - Error with error content page
 901      5 - Unrecoverable non-server error (close the connection) */
 902 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 903 {
 904    // Not Modified
 905    if (Srv->Result == 304)
 906    {
 907       unlink(Queue->DestFile.c_str());
 908       Res.IMSHit = true;
 909       Res.LastModified = Queue->LastModified;
 910       return 1;
 911    }
 912
 913    /* We have a reply we dont handle. This should indicate a perm server
 914       failure */
 915    if (Srv->Result < 200 || Srv->Result >= 300)
 916    {
 917       _error->Error("%u %s",Srv->Result,Srv->Code);
 918       if (Srv->HaveContent == true)
 919          return 4;
 920       return 3;
 921    }
 922
 923    // This is some sort of 2xx 'data follows' reply
 924    Res.LastModified = Srv->Date;
 925    Res.Size = Srv->Size;
 926
 927    // Open the file
 928    delete File;
 929    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 930    if (_error->PendingError() == true)
 931       return 5;
 932
 933    FailFile = Queue->DestFile;
 934    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 935    FailFd = File->Fd();
 936    FailTime = Srv->Date;
 937
 938    // Set the expected size
 939    if (Srv->StartPos >= 0)
 940    {
 941       Res.ResumePoint = Srv->StartPos;
 942       ftruncate(File->Fd(),Srv->StartPos);
 943    }
 944
 945    // Set the start point
 946    lseek(File->Fd(),0,SEEK_END);
 947
 948    delete Srv->In.Hash;
 949    Srv->In.Hash = new Hashes;
 950
 951    // Fill the Hash if the file is non-empty (resume)
 952    if (Srv->StartPos > 0)
 953    {
 954       lseek(File->Fd(),0,SEEK_SET);
 955       if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
 956       {
 957          _error->Errno("read",_("Problem hashing file"));
 958          return 5;
 959       }
 960       lseek(File->Fd(),0,SEEK_END);
 961    }
 962
 963    SetNonBlock(File->Fd(),true);
 964    return 0;
 965 }
 966                                                                         /*}}}*/
 967 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 968 // ---------------------------------------------------------------------
 969 /* This closes and timestamps the open file. This is neccessary to get
 970    resume behavoir on user abort */
 971 void HttpMethod::SigTerm(int)
 972 {
 973    if (FailFd == -1)
 974       _exit(100);
 975    close(FailFd);
 976
 977    // Timestamp
 978    struct utimbuf UBuf;
 979    UBuf.actime = FailTime;
 980    UBuf.modtime = FailTime;
 981    utime(FailFile.c_str(),&UBuf);
 982
 983    _exit(100);
 984 }
 985                                                                         /*}}}*/
 986 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 987 // ---------------------------------------------------------------------
 988 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 989    depth. */
 990 bool HttpMethod::Fetch(FetchItem *)
 991 {
 992    if (Server == 0)
 993       return true;
 994
 995    // Queue the requests
 996    int Depth = -1;
 997    bool Tail = false;
 998    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 999         I = I->Next, Depth++)
1000    {
1001       // If pipelining is disabled, we only queue 1 request
1002       if (Server->Pipeline == false && Depth >= 0)
1003          break;
1004
1005       // Make sure we stick with the same server
1006       if (Server->Comp(I->Uri) == false)
1007          break;
1008       if (QueueBack == I)
1009          Tail = true;
1010       if (Tail == true)
1011       {
1012          QueueBack = I->Next;
1013          SendReq(I,Server->Out);
1014          continue;
1015       }
1016    }
1017
1018    return true;
1019 };
1020                                                                         /*}}}*/
1021 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
1022 // ---------------------------------------------------------------------
1023 /* We stash the desired pipeline depth */
1024 bool HttpMethod::Configuration(string Message)
1025 {
1026    if (pkgAcqMethod::Configuration(Message) == false)
1027       return false;
1028
1029    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
1030    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
1031                                   PipelineDepth);
1032    Debug = _config->FindB("Debug::Acquire::http",false);
1033
1034    return true;
1035 }
1036                                                                         /*}}}*/
1037 // HttpMethod::Loop - Main loop                                         /*{{{*/
1038 // ---------------------------------------------------------------------
1039 /* */
1040 int HttpMethod::Loop()
1041 {
1042    signal(SIGTERM,SigTerm);
1043    signal(SIGINT,SigTerm);
1044
1045    Server = 0;
1046
1047    int FailCounter = 0;
1048    while (1)
1049    {
1050       // We have no commands, wait for some to arrive
1051       if (Queue == 0)
1052       {
1053          if (WaitFd(STDIN_FILENO) == false)
1054             return 0;
1055       }
1056
1057       /* Run messages, we can accept 0 (no message) if we didn't
1058          do a WaitFd above.. Otherwise the FD is closed. */
1059       int Result = Run(true);
1060       if (Result != -1 && (Result != 0 || Queue == 0))
1061          return 100;
1062
1063       if (Queue == 0)
1064          continue;
1065
1066       // Connect to the server
1067       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1068       {
1069          delete Server;
1070          Server = new ServerState(Queue->Uri,this);
1071       }
1072
1073       /* If the server has explicitly said this is the last connection
1074          then we pre-emptively shut down the pipeline and tear down
1075          the connection. This will speed up HTTP/1.0 servers a tad
1076          since we don't have to wait for the close sequence to
1077          complete */
1078       if (Server->Persistent == false)
1079          Server->Close();
1080
1081       // Reset the pipeline
1082       if (Server->ServerFd == -1)
1083          QueueBack = Queue;
1084
1085       // Connnect to the host
1086       if (Server->Open() == false)
1087       {
1088          Fail(true);
1089          delete Server;
1090          Server = 0;
1091          continue;
1092       }
1093
1094       // Fill the pipeline.
1095       Fetch(0);
1096
1097       // Fetch the next URL header data from the server.
1098       switch (Server->RunHeaders())
1099       {
1100          case 0:
1101          break;
1102
1103          // The header data is bad
1104          case 2:
1105          {
1106             _error->Error(_("Bad header data"));
1107             Fail(true);
1108             RotateDNS();
1109             continue;
1110          }
1111
1112          // The server closed a connection during the header get..
1113          default:
1114          case 1:
1115          {
1116             FailCounter++;
1117             _error->Discard();
1118             Server->Close();
1119             Server->Pipeline = false;
1120
1121             if (FailCounter >= 2)
1122             {
1123                Fail(_("Connection failed"),true);
1124                FailCounter = 0;
1125             }
1126
1127             RotateDNS();
1128             continue;
1129          }
1130       };
1131
1132       // Decide what to do.
1133       FetchResult Res;
1134       Res.Filename = Queue->DestFile;
1135       switch (DealWithHeaders(Res,Server))
1136       {
1137          // Ok, the file is Open
1138          case 0:
1139          {
1140             URIStart(Res);
1141
1142             // Run the data
1143             bool Result =  Server->RunData();
1144
1145             /* If the server is sending back sizeless responses then fill in
1146                the size now */
1147             if (Res.Size == 0)
1148                Res.Size = File->Size();
1149
1150             // Close the file, destroy the FD object and timestamp it
1151             FailFd = -1;
1152             delete File;
1153             File = 0;
1154
1155             // Timestamp
1156             struct utimbuf UBuf;
1157             time(&UBuf.actime);
1158             UBuf.actime = Server->Date;
1159             UBuf.modtime = Server->Date;
1160             utime(Queue->DestFile.c_str(),&UBuf);
1161
1162             // Send status to APT
1163             if (Result == true)
1164             {
1165                Res.TakeHashes(*Server->In.Hash);
1166                URIDone(Res);
1167             }
1168             else
1169                Fail(true);
1170
1171             break;
1172          }
1173
1174          // IMS hit
1175          case 1:
1176          {
1177             URIDone(Res);
1178             break;
1179          }
1180
1181          // Hard server error, not found or something
1182          case 3:
1183          {
1184             Fail();
1185             break;
1186          }
1187
1188          // Hard internal error, kill the connection and fail
1189          case 5:
1190          {
1191             delete File;
1192             File = 0;
1193
1194             Fail();
1195             RotateDNS();
1196             Server->Close();
1197             break;
1198          }
1199
1200          // We need to flush the data, the header is like a 404 w/ error text
1201          case 4:
1202          {
1203             Fail();
1204
1205             // Send to content to dev/null
1206             File = new FileFd("/dev/null",FileFd::WriteExists);
1207             Server->RunData();
1208             delete File;
1209             File = 0;
1210             break;
1211          }
1212
1213          default:
1214          Fail(_("Internal error"));
1215          break;
1216       }
1217
1218       FailCounter = 0;
1219    }
1220
1221    return 0;
1222 }
1223                                                                         /*}}}*/
1224
1225 int main()
1226 {
1227    setlocale(LC_ALL, "");
1228
1229    HttpMethod Mth;
1230
1231    return Mth.Loop();
1232 }
1233
1234