methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apt-pkg/fileutl.h>
  29 #include <apt-pkg/acquire-method.h>
  30 #include <apt-pkg/error.h>
  31 #include <apt-pkg/hashes.h>
  32
  33 #include <sys/stat.h>
  34 #include <sys/time.h>
  35 #include <utime.h>
  36 #include <unistd.h>
  37 #include <signal.h>
  38 #include <stdio.h>
  39 #include <errno.h>
  40 #include <string.h>
  41 #include <iostream>
  42 #include <apti18n.h>
  43
  44 // Internet stuff
  45 #include <netdb.h>
  46
  47 #include "connect.h"
  48 #include "rfc2553emu.h"
  49 #include "http.h"
  50
  51                                                                         /*}}}*/
  52 using namespace std;
  53
  54 string HttpMethod::FailFile;
  55 int HttpMethod::FailFd = -1;
  56 time_t HttpMethod::FailTime = 0;
  57 unsigned long PipelineDepth = 10;
  58 unsigned long TimeOut = 120;
  59 bool Debug = false;
  60
  61
  62 unsigned long CircleBuf::BwReadLimit=0;
  63 unsigned long CircleBuf::BwTickReadData=0;
  64 struct timeval CircleBuf::BwReadTick={0,0};
  65 const unsigned int CircleBuf::BW_HZ=10;
  66
  67 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  68 // ---------------------------------------------------------------------
  69 /* */
  70 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
  71 {
  72    Buf = new unsigned char[Size];
  73    Reset();
  74
  75    CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
  76 }
  77                                                                         /*}}}*/
  78 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  79 // ---------------------------------------------------------------------
  80 /* */
  81 void CircleBuf::Reset()
  82 {
  83    InP = 0;
  84    OutP = 0;
  85    StrPos = 0;
  86    MaxGet = (unsigned int)-1;
  87    OutQueue = string();
  88    if (Hash != 0)
  89    {
  90       delete Hash;
  91       Hash = new Hashes;
  92    }
  93 };
  94                                                                         /*}}}*/
  95 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  96 // ---------------------------------------------------------------------
  97 /* This fills up the buffer with as much data as is in the FD, assuming it
  98    is non-blocking.. */
  99 bool CircleBuf::Read(int Fd)
 100 {
 101    unsigned long BwReadMax;
 102
 103    while (1)
 104    {
 105       // Woops, buffer is full
 106       if (InP - OutP == Size)
 107          return true;
 108
 109       // what's left to read in this tick
 110       BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
 111
 112       if(CircleBuf::BwReadLimit) {
 113          struct timeval now;
 114          gettimeofday(&now,0);
 115
 116          unsigned long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
 117             now.tv_usec-CircleBuf::BwReadTick.tv_usec;
 118          if(d > 1000000/BW_HZ) {
 119             CircleBuf::BwReadTick = now;
 120             CircleBuf::BwTickReadData = 0;
 121          }
 122
 123          if(CircleBuf::BwTickReadData >= BwReadMax) {
 124             usleep(1000000/BW_HZ);
 125             return true;
 126          }
 127       }
 128
 129       // Write the buffer segment
 130       int Res;
 131       if(CircleBuf::BwReadLimit) {
 132          Res = read(Fd,Buf + (InP%Size),
 133                     BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
 134       } else
 135          Res = read(Fd,Buf + (InP%Size),LeftRead());
 136
 137       if(Res > 0 && BwReadLimit > 0)
 138          CircleBuf::BwTickReadData += Res;
 139
 140       if (Res == 0)
 141          return false;
 142       if (Res < 0)
 143       {
 144          if (errno == EAGAIN)
 145             return true;
 146          return false;
 147       }
 148
 149       if (InP == 0)
 150          gettimeofday(&Start,0);
 151       InP += Res;
 152    }
 153 }
 154                                                                         /*}}}*/
 155 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 156 // ---------------------------------------------------------------------
 157 /* This will hold the string in and fill the buffer with it as it empties */
 158 bool CircleBuf::Read(string Data)
 159 {
 160    OutQueue += Data;
 161    FillOut();
 162    return true;
 163 }
 164                                                                         /*}}}*/
 165 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 166 // ---------------------------------------------------------------------
 167 /* */
 168 void CircleBuf::FillOut()
 169 {
 170    if (OutQueue.empty() == true)
 171       return;
 172    while (1)
 173    {
 174       // Woops, buffer is full
 175       if (InP - OutP == Size)
 176          return;
 177
 178       // Write the buffer segment
 179       unsigned long Sz = LeftRead();
 180       if (OutQueue.length() - StrPos < Sz)
 181          Sz = OutQueue.length() - StrPos;
 182       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 183
 184       // Advance
 185       StrPos += Sz;
 186       InP += Sz;
 187       if (OutQueue.length() == StrPos)
 188       {
 189          StrPos = 0;
 190          OutQueue = "";
 191          return;
 192       }
 193    }
 194 }
 195                                                                         /*}}}*/
 196 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 197 // ---------------------------------------------------------------------
 198 /* This empties the buffer into the FD. */
 199 bool CircleBuf::Write(int Fd)
 200 {
 201    while (1)
 202    {
 203       FillOut();
 204
 205       // Woops, buffer is empty
 206       if (OutP == InP)
 207          return true;
 208
 209       if (OutP == MaxGet)
 210          return true;
 211
 212       // Write the buffer segment
 213       int Res;
 214       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 215
 216       if (Res == 0)
 217          return false;
 218       if (Res < 0)
 219       {
 220          if (errno == EAGAIN)
 221             return true;
 222
 223          return false;
 224       }
 225
 226       if (Hash != 0)
 227          Hash->Add(Buf + (OutP%Size),Res);
 228
 229       OutP += Res;
 230    }
 231 }
 232                                                                         /*}}}*/
 233 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 234 // ---------------------------------------------------------------------
 235 /* This copies till the first empty line */
 236 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 237 {
 238    // We cheat and assume it is unneeded to have more than one buffer load
 239    for (unsigned long I = OutP; I < InP; I++)
 240    {
 241       if (Buf[I%Size] != '\n')
 242          continue;
 243       ++I;
 244       if (I < InP  && Buf[I%Size] == '\r')
 245          ++I;
 246
 247       if (Single == false)
 248       {
 249          if (Buf[I%Size] != '\n')
 250             continue;
 251          ++I;
 252          if (I < InP  && Buf[I%Size] == '\r')
 253             ++I;
 254       }
 255
 256       if (I > InP)
 257          I = InP;
 258
 259       Data = "";
 260       while (OutP < I)
 261       {
 262          unsigned long Sz = LeftWrite();
 263          if (Sz == 0)
 264             return false;
 265          if (I - OutP < LeftWrite())
 266             Sz = I - OutP;
 267          Data += string((char *)(Buf + (OutP%Size)),Sz);
 268          OutP += Sz;
 269       }
 270       return true;
 271    }
 272    return false;
 273 }
 274                                                                         /*}}}*/
 275 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 276 // ---------------------------------------------------------------------
 277 /* */
 278 void CircleBuf::Stats()
 279 {
 280    if (InP == 0)
 281       return;
 282
 283    struct timeval Stop;
 284    gettimeofday(&Stop,0);
 285 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 286              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 287    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 288 }
 289                                                                         /*}}}*/
 290
 291 // ServerState::ServerState - Constructor                               /*{{{*/
 292 // ---------------------------------------------------------------------
 293 /* */
 294 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 295                         In(64*1024), Out(4*1024),
 296                         ServerName(Srv)
 297 {
 298    Reset();
 299 }
 300                                                                         /*}}}*/
 301 // ServerState::Open - Open a connection to the server                  /*{{{*/
 302 // ---------------------------------------------------------------------
 303 /* This opens a connection to the server. */
 304 bool ServerState::Open()
 305 {
 306    // Use the already open connection if possible.
 307    if (ServerFd != -1)
 308       return true;
 309
 310    Close();
 311    In.Reset();
 312    Out.Reset();
 313    Persistent = true;
 314
 315    // Determine the proxy setting
 316    if (getenv("http_proxy") == 0)
 317    {
 318       string DefProxy = _config->Find("Acquire::http::Proxy");
 319       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 320       if (SpecificProxy.empty() == false)
 321       {
 322          if (SpecificProxy == "DIRECT")
 323             Proxy = "";
 324          else
 325             Proxy = SpecificProxy;
 326       }
 327       else
 328          Proxy = DefProxy;
 329    }
 330    else
 331       Proxy = getenv("http_proxy");
 332
 333    // Parse no_proxy, a , separated list of domains
 334    if (getenv("no_proxy") != 0)
 335    {
 336       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 337          Proxy = "";
 338    }
 339
 340    // Determine what host and port to use based on the proxy settings
 341    int Port = 0;
 342    string Host;
 343    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 344    {
 345       if (ServerName.Port != 0)
 346          Port = ServerName.Port;
 347       Host = ServerName.Host;
 348    }
 349    else
 350    {
 351       if (Proxy.Port != 0)
 352          Port = Proxy.Port;
 353       Host = Proxy.Host;
 354    }
 355
 356    // Connect to the remote server
 357    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 358       return false;
 359
 360    return true;
 361 }
 362                                                                         /*}}}*/
 363 // ServerState::Close - Close a connection to the server                /*{{{*/
 364 // ---------------------------------------------------------------------
 365 /* */
 366 bool ServerState::Close()
 367 {
 368    close(ServerFd);
 369    ServerFd = -1;
 370    return true;
 371 }
 372                                                                         /*}}}*/
 373 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 374 // ---------------------------------------------------------------------
 375 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 376    parse error occured */
 377 int ServerState::RunHeaders()
 378 {
 379    State = Header;
 380
 381    Owner->Status(_("Waiting for headers"));
 382
 383    Major = 0;
 384    Minor = 0;
 385    Result = 0;
 386    Size = 0;
 387    StartPos = 0;
 388    Encoding = Closes;
 389    HaveContent = false;
 390    time(&Date);
 391
 392    do
 393    {
 394       string Data;
 395       if (In.WriteTillEl(Data) == false)
 396          continue;
 397
 398       if (Debug == true)
 399          clog << Data;
 400
 401       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 402       {
 403          string::const_iterator J = I;
 404          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 405          if (HeaderLine(string(I,J)) == false)
 406             return 2;
 407          I = J;
 408       }
 409
 410       // 100 Continue is a Nop...
 411       if (Result == 100)
 412          continue;
 413
 414       // Tidy up the connection persistance state.
 415       if (Encoding == Closes && HaveContent == true)
 416          Persistent = false;
 417
 418       return 0;
 419    }
 420    while (Owner->Go(false,this) == true);
 421
 422    return 1;
 423 }
 424                                                                         /*}}}*/
 425 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 426 // ---------------------------------------------------------------------
 427 /* */
 428 bool ServerState::RunData()
 429 {
 430    State = Data;
 431
 432    // Chunked transfer encoding is fun..
 433    if (Encoding == Chunked)
 434    {
 435       while (1)
 436       {
 437          // Grab the block size
 438          bool Last = true;
 439          string Data;
 440          In.Limit(-1);
 441          do
 442          {
 443             if (In.WriteTillEl(Data,true) == true)
 444                break;
 445          }
 446          while ((Last = Owner->Go(false,this)) == true);
 447
 448          if (Last == false)
 449             return false;
 450
 451          // See if we are done
 452          unsigned long Len = strtol(Data.c_str(),0,16);
 453          if (Len == 0)
 454          {
 455             In.Limit(-1);
 456
 457             // We have to remove the entity trailer
 458             Last = true;
 459             do
 460             {
 461                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 462                   break;
 463             }
 464             while ((Last = Owner->Go(false,this)) == true);
 465             if (Last == false)
 466                return false;
 467             return !_error->PendingError();
 468          }
 469
 470          // Transfer the block
 471          In.Limit(Len);
 472          while (Owner->Go(true,this) == true)
 473             if (In.IsLimit() == true)
 474                break;
 475
 476          // Error
 477          if (In.IsLimit() == false)
 478             return false;
 479
 480          // The server sends an extra new line before the next block specifier..
 481          In.Limit(-1);
 482          Last = true;
 483          do
 484          {
 485             if (In.WriteTillEl(Data,true) == true)
 486                break;
 487          }
 488          while ((Last = Owner->Go(false,this)) == true);
 489          if (Last == false)
 490             return false;
 491       }
 492    }
 493    else
 494    {
 495       /* Closes encoding is used when the server did not specify a size, the
 496          loss of the connection means we are done */
 497       if (Encoding == Closes)
 498          In.Limit(-1);
 499       else
 500          In.Limit(Size - StartPos);
 501
 502       // Just transfer the whole block.
 503       do
 504       {
 505          if (In.IsLimit() == false)
 506             continue;
 507
 508          In.Limit(-1);
 509          return !_error->PendingError();
 510       }
 511       while (Owner->Go(true,this) == true);
 512    }
 513
 514    return Owner->Flush(this) && !_error->PendingError();
 515 }
 516                                                                         /*}}}*/
 517 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 518 // ---------------------------------------------------------------------
 519 /* */
 520 bool ServerState::HeaderLine(string Line)
 521 {
 522    if (Line.empty() == true)
 523       return true;
 524
 525    // The http server might be trying to do something evil.
 526    if (Line.length() >= MAXLEN)
 527       return _error->Error(_("Got a single header line over %u chars"),MAXLEN);
 528
 529    string::size_type Pos = Line.find(' ');
 530    if (Pos == string::npos || Pos+1 > Line.length())
 531    {
 532       // Blah, some servers use "connection:closes", evil.
 533       Pos = Line.find(':');
 534       if (Pos == string::npos || Pos + 2 > Line.length())
 535          return _error->Error(_("Bad header line"));
 536       Pos++;
 537    }
 538
 539    // Parse off any trailing spaces between the : and the next word.
 540    string::size_type Pos2 = Pos;
 541    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 542       Pos2++;
 543
 544    string Tag = string(Line,0,Pos);
 545    string Val = string(Line,Pos2);
 546
 547    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 548    {
 549       // Evil servers return no version
 550       if (Line[4] == '/')
 551       {
 552          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 553                     &Result,Code) != 4)
 554             return _error->Error(_("The HTTP server sent an invalid reply header"));
 555       }
 556       else
 557       {
 558          Major = 0;
 559          Minor = 9;
 560          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 561             return _error->Error(_("The HTTP server sent an invalid reply header"));
 562       }
 563
 564       /* Check the HTTP response header to get the default persistance
 565          state. */
 566       if (Major < 1)
 567          Persistent = false;
 568       else
 569       {
 570          if (Major == 1 && Minor <= 0)
 571             Persistent = false;
 572          else
 573             Persistent = true;
 574       }
 575
 576       return true;
 577    }
 578
 579    if (stringcasecmp(Tag,"Content-Length:") == 0)
 580    {
 581       if (Encoding == Closes)
 582          Encoding = Stream;
 583       HaveContent = true;
 584
 585       // The length is already set from the Content-Range header
 586       if (StartPos != 0)
 587          return true;
 588
 589       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 590          return _error->Error(_("The HTTP server sent an invalid Content-Length header"));
 591       return true;
 592    }
 593
 594    if (stringcasecmp(Tag,"Content-Type:") == 0)
 595    {
 596       HaveContent = true;
 597       return true;
 598    }
 599
 600    if (stringcasecmp(Tag,"Content-Range:") == 0)
 601    {
 602       HaveContent = true;
 603
 604       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 605          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 606       if ((unsigned)StartPos > Size)
 607          return _error->Error(_("This HTTP server has broken range support"));
 608       return true;
 609    }
 610
 611    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 612    {
 613       HaveContent = true;
 614       if (stringcasecmp(Val,"chunked") == 0)
 615          Encoding = Chunked;
 616       return true;
 617    }
 618
 619    if (stringcasecmp(Tag,"Connection:") == 0)
 620    {
 621       if (stringcasecmp(Val,"close") == 0)
 622          Persistent = false;
 623       if (stringcasecmp(Val,"keep-alive") == 0)
 624          Persistent = true;
 625       return true;
 626    }
 627
 628    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 629    {
 630       if (StrToTime(Val,Date) == false)
 631          return _error->Error(_("Unknown date format"));
 632       return true;
 633    }
 634
 635    return true;
 636 }
 637                                                                         /*}}}*/
 638
 639 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 640 // ---------------------------------------------------------------------
 641 /* This places the http request in the outbound buffer */
 642 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 643 {
 644    URI Uri = Itm->Uri;
 645
 646    // The HTTP server expects a hostname with a trailing :port
 647    char Buf[1000];
 648    string ProperHost = Uri.Host;
 649    if (Uri.Port != 0)
 650    {
 651       sprintf(Buf,":%u",Uri.Port);
 652       ProperHost += Buf;
 653    }
 654
 655    // Just in case.
 656    if (Itm->Uri.length() >= sizeof(Buf))
 657        abort();
 658
 659    /* Build the request. We include a keep-alive header only for non-proxy
 660       requests. This is to tweak old http/1.0 servers that do support keep-alive
 661       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 662       will glitch HTTP/1.0 proxies because they do not filter it out and
 663       pass it on, HTTP/1.1 says the connection should default to keep alive
 664       and we expect the proxy to do this */
 665    if (Proxy.empty() == true)
 666       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 667               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 668    else
 669    {
 670       /* Generate a cache control header if necessary. We place a max
 671          cache age on index files, optionally set a no-cache directive
 672          and a no-store directive for archives. */
 673       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 674               Itm->Uri.c_str(),ProperHost.c_str());
 675       // only generate a cache control header if we actually want to
 676       // use a cache
 677       if (_config->FindB("Acquire::http::No-Cache",false) == false)
 678       {
 679          if (Itm->IndexFile == true)
 680             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 681                     _config->FindI("Acquire::http::Max-Age",0));
 682          else
 683          {
 684             if (_config->FindB("Acquire::http::No-Store",false) == true)
 685                strcat(Buf,"Cache-Control: no-store\r\n");
 686          }
 687       }
 688    }
 689    // generate a no-cache header if needed
 690    if (_config->FindB("Acquire::http::No-Cache",false) == true)
 691       strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 692
 693
 694    string Req = Buf;
 695
 696    // Check for a partial file
 697    struct stat SBuf;
 698    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 699    {
 700       // In this case we send an if-range query with a range header
 701       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 702               TimeRFC1123(SBuf.st_mtime).c_str());
 703       Req += Buf;
 704    }
 705    else
 706    {
 707       if (Itm->LastModified != 0)
 708       {
 709          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 710          Req += Buf;
 711       }
 712    }
 713
 714    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 715       Req += string("Proxy-Authorization: Basic ") +
 716           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 717
 718    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 719       Req += string("Authorization: Basic ") +
 720           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 721
 722    Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
 723
 724    if (Debug == true)
 725       cerr << Req << endl;
 726
 727    Out.Read(Req);
 728 }
 729                                                                         /*}}}*/
 730 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 731 // ---------------------------------------------------------------------
 732 /* This runs the select loop over the server FDs, Output file FDs and
 733    stdin. */
 734 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 735 {
 736    // Server has closed the connection
 737    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 738                                ToFile == false))
 739       return false;
 740
 741    fd_set rfds,wfds;
 742    FD_ZERO(&rfds);
 743    FD_ZERO(&wfds);
 744
 745    /* Add the server. We only send more requests if the connection will
 746       be persisting */
 747    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 748        && Srv->Persistent == true)
 749       FD_SET(Srv->ServerFd,&wfds);
 750    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 751       FD_SET(Srv->ServerFd,&rfds);
 752
 753    // Add the file
 754    int FileFD = -1;
 755    if (File != 0)
 756       FileFD = File->Fd();
 757
 758    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 759       FD_SET(FileFD,&wfds);
 760
 761    // Add stdin
 762    FD_SET(STDIN_FILENO,&rfds);
 763
 764    // Figure out the max fd
 765    int MaxFd = FileFD;
 766    if (MaxFd < Srv->ServerFd)
 767       MaxFd = Srv->ServerFd;
 768
 769    // Select
 770    struct timeval tv;
 771    tv.tv_sec = TimeOut;
 772    tv.tv_usec = 0;
 773    int Res = 0;
 774    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 775    {
 776       if (errno == EINTR)
 777          return true;
 778       return _error->Errno("select",_("Select failed"));
 779    }
 780
 781    if (Res == 0)
 782    {
 783       _error->Error(_("Connection timed out"));
 784       return ServerDie(Srv);
 785    }
 786
 787    // Handle server IO
 788    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 789    {
 790       errno = 0;
 791       if (Srv->In.Read(Srv->ServerFd) == false)
 792          return ServerDie(Srv);
 793    }
 794
 795    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 796    {
 797       errno = 0;
 798       if (Srv->Out.Write(Srv->ServerFd) == false)
 799          return ServerDie(Srv);
 800    }
 801
 802    // Send data to the file
 803    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 804    {
 805       if (Srv->In.Write(FileFD) == false)
 806          return _error->Errno("write",_("Error writing to output file"));
 807    }
 808
 809    // Handle commands from APT
 810    if (FD_ISSET(STDIN_FILENO,&rfds))
 811    {
 812       if (Run(true) != -1)
 813          exit(100);
 814    }
 815
 816    return true;
 817 }
 818                                                                         /*}}}*/
 819 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 820 // ---------------------------------------------------------------------
 821 /* This takes the current input buffer from the Server FD and writes it
 822    into the file */
 823 bool HttpMethod::Flush(ServerState *Srv)
 824 {
 825    if (File != 0)
 826    {
 827       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 828       // can't be set
 829       if (File->Name() != "/dev/null")
 830          SetNonBlock(File->Fd(),false);
 831       if (Srv->In.WriteSpace() == false)
 832          return true;
 833
 834       while (Srv->In.WriteSpace() == true)
 835       {
 836          if (Srv->In.Write(File->Fd()) == false)
 837             return _error->Errno("write",_("Error writing to file"));
 838          if (Srv->In.IsLimit() == true)
 839             return true;
 840       }
 841
 842       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 843          return true;
 844    }
 845    return false;
 846 }
 847                                                                         /*}}}*/
 848 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 849 // ---------------------------------------------------------------------
 850 /* */
 851 bool HttpMethod::ServerDie(ServerState *Srv)
 852 {
 853    unsigned int LErrno = errno;
 854
 855    // Dump the buffer to the file
 856    if (Srv->State == ServerState::Data)
 857    {
 858       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 859       // can't be set
 860       if (File->Name() != "/dev/null")
 861          SetNonBlock(File->Fd(),false);
 862       while (Srv->In.WriteSpace() == true)
 863       {
 864          if (Srv->In.Write(File->Fd()) == false)
 865             return _error->Errno("write",_("Error writing to the file"));
 866
 867          // Done
 868          if (Srv->In.IsLimit() == true)
 869             return true;
 870       }
 871    }
 872
 873    // See if this is because the server finished the data stream
 874    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 875        Srv->Encoding != ServerState::Closes)
 876    {
 877       Srv->Close();
 878       if (LErrno == 0)
 879          return _error->Error(_("Error reading from server. Remote end closed connection"));
 880       errno = LErrno;
 881       return _error->Errno("read",_("Error reading from server"));
 882    }
 883    else
 884    {
 885       Srv->In.Limit(-1);
 886
 887       // Nothing left in the buffer
 888       if (Srv->In.WriteSpace() == false)
 889          return false;
 890
 891       // We may have got multiple responses back in one packet..
 892       Srv->Close();
 893       return true;
 894    }
 895
 896    return false;
 897 }
 898                                                                         /*}}}*/
 899 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 900 // ---------------------------------------------------------------------
 901 /* We look at the header data we got back from the server and decide what
 902    to do. Returns
 903      0 - File is open,
 904      1 - IMS hit
 905      3 - Unrecoverable error
 906      4 - Error with error content page
 907      5 - Unrecoverable non-server error (close the connection) */
 908 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 909 {
 910    // Not Modified
 911    if (Srv->Result == 304)
 912    {
 913       unlink(Queue->DestFile.c_str());
 914       Res.IMSHit = true;
 915       Res.LastModified = Queue->LastModified;
 916       return 1;
 917    }
 918
 919    /* We have a reply we dont handle. This should indicate a perm server
 920       failure */
 921    if (Srv->Result < 200 || Srv->Result >= 300)
 922    {
 923       _error->Error("%u %s",Srv->Result,Srv->Code);
 924       if (Srv->HaveContent == true)
 925          return 4;
 926       return 3;
 927    }
 928
 929    // This is some sort of 2xx 'data follows' reply
 930    Res.LastModified = Srv->Date;
 931    Res.Size = Srv->Size;
 932
 933    // Open the file
 934    delete File;
 935    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 936    if (_error->PendingError() == true)
 937       return 5;
 938
 939    FailFile = Queue->DestFile;
 940    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 941    FailFd = File->Fd();
 942    FailTime = Srv->Date;
 943
 944    // Set the expected size
 945    if (Srv->StartPos >= 0)
 946    {
 947       Res.ResumePoint = Srv->StartPos;
 948       ftruncate(File->Fd(),Srv->StartPos);
 949    }
 950
 951    // Set the start point
 952    lseek(File->Fd(),0,SEEK_END);
 953
 954    delete Srv->In.Hash;
 955    Srv->In.Hash = new Hashes;
 956
 957    // Fill the Hash if the file is non-empty (resume)
 958    if (Srv->StartPos > 0)
 959    {
 960       lseek(File->Fd(),0,SEEK_SET);
 961       if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
 962       {
 963          _error->Errno("read",_("Problem hashing file"));
 964          return 5;
 965       }
 966       lseek(File->Fd(),0,SEEK_END);
 967    }
 968
 969    SetNonBlock(File->Fd(),true);
 970    return 0;
 971 }
 972                                                                         /*}}}*/
 973 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 974 // ---------------------------------------------------------------------
 975 /* This closes and timestamps the open file. This is neccessary to get
 976    resume behavoir on user abort */
 977 void HttpMethod::SigTerm(int)
 978 {
 979    if (FailFd == -1)
 980       _exit(100);
 981    close(FailFd);
 982
 983    // Timestamp
 984    struct utimbuf UBuf;
 985    UBuf.actime = FailTime;
 986    UBuf.modtime = FailTime;
 987    utime(FailFile.c_str(),&UBuf);
 988
 989    _exit(100);
 990 }
 991                                                                         /*}}}*/
 992 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 993 // ---------------------------------------------------------------------
 994 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 995    depth. */
 996 bool HttpMethod::Fetch(FetchItem *)
 997 {
 998    if (Server == 0)
 999       return true;
1000
1001    // Queue the requests
1002    int Depth = -1;
1003    bool Tail = false;
1004    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
1005         I = I->Next, Depth++)
1006    {
1007       // If pipelining is disabled, we only queue 1 request
1008       if (Server->Pipeline == false && Depth >= 0)
1009          break;
1010
1011       // Make sure we stick with the same server
1012       if (Server->Comp(I->Uri) == false)
1013          break;
1014       if (QueueBack == I)
1015          Tail = true;
1016       if (Tail == true)
1017       {
1018          QueueBack = I->Next;
1019          SendReq(I,Server->Out);
1020          continue;
1021       }
1022    }
1023
1024    return true;
1025 };
1026                                                                         /*}}}*/
1027 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
1028 // ---------------------------------------------------------------------
1029 /* We stash the desired pipeline depth */
1030 bool HttpMethod::Configuration(string Message)
1031 {
1032    if (pkgAcqMethod::Configuration(Message) == false)
1033       return false;
1034
1035    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
1036    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
1037                                   PipelineDepth);
1038    Debug = _config->FindB("Debug::Acquire::http",false);
1039
1040    return true;
1041 }
1042                                                                         /*}}}*/
1043 // HttpMethod::Loop - Main loop                                         /*{{{*/
1044 // ---------------------------------------------------------------------
1045 /* */
1046 int HttpMethod::Loop()
1047 {
1048    signal(SIGTERM,SigTerm);
1049    signal(SIGINT,SigTerm);
1050
1051    Server = 0;
1052
1053    int FailCounter = 0;
1054    while (1)
1055    {
1056       // We have no commands, wait for some to arrive
1057       if (Queue == 0)
1058       {
1059          if (WaitFd(STDIN_FILENO) == false)
1060             return 0;
1061       }
1062
1063       /* Run messages, we can accept 0 (no message) if we didn't
1064          do a WaitFd above.. Otherwise the FD is closed. */
1065       int Result = Run(true);
1066       if (Result != -1 && (Result != 0 || Queue == 0))
1067          return 100;
1068
1069       if (Queue == 0)
1070          continue;
1071
1072       // Connect to the server
1073       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1074       {
1075          delete Server;
1076          Server = new ServerState(Queue->Uri,this);
1077       }
1078
1079       /* If the server has explicitly said this is the last connection
1080          then we pre-emptively shut down the pipeline and tear down
1081          the connection. This will speed up HTTP/1.0 servers a tad
1082          since we don't have to wait for the close sequence to
1083          complete */
1084       if (Server->Persistent == false)
1085          Server->Close();
1086
1087       // Reset the pipeline
1088       if (Server->ServerFd == -1)
1089          QueueBack = Queue;
1090
1091       // Connnect to the host
1092       if (Server->Open() == false)
1093       {
1094          Fail(true);
1095          delete Server;
1096          Server = 0;
1097          continue;
1098       }
1099
1100       // Fill the pipeline.
1101       Fetch(0);
1102
1103       // Fetch the next URL header data from the server.
1104       switch (Server->RunHeaders())
1105       {
1106          case 0:
1107          break;
1108
1109          // The header data is bad
1110          case 2:
1111          {
1112             _error->Error(_("Bad header data"));
1113             Fail(true);
1114             RotateDNS();
1115             continue;
1116          }
1117
1118          // The server closed a connection during the header get..
1119          default:
1120          case 1:
1121          {
1122             FailCounter++;
1123             _error->Discard();
1124             Server->Close();
1125             Server->Pipeline = false;
1126
1127             if (FailCounter >= 2)
1128             {
1129                Fail(_("Connection failed"),true);
1130                FailCounter = 0;
1131             }
1132
1133             RotateDNS();
1134             continue;
1135          }
1136       };
1137
1138       // Decide what to do.
1139       FetchResult Res;
1140       Res.Filename = Queue->DestFile;
1141       switch (DealWithHeaders(Res,Server))
1142       {
1143          // Ok, the file is Open
1144          case 0:
1145          {
1146             URIStart(Res);
1147
1148             // Run the data
1149             bool Result =  Server->RunData();
1150
1151             /* If the server is sending back sizeless responses then fill in
1152                the size now */
1153             if (Res.Size == 0)
1154                Res.Size = File->Size();
1155
1156             // Close the file, destroy the FD object and timestamp it
1157             FailFd = -1;
1158             delete File;
1159             File = 0;
1160
1161             // Timestamp
1162             struct utimbuf UBuf;
1163             time(&UBuf.actime);
1164             UBuf.actime = Server->Date;
1165             UBuf.modtime = Server->Date;
1166             utime(Queue->DestFile.c_str(),&UBuf);
1167
1168             // Send status to APT
1169             if (Result == true)
1170             {
1171                Res.TakeHashes(*Server->In.Hash);
1172                URIDone(Res);
1173             }
1174             else
1175                Fail(true);
1176
1177             break;
1178          }
1179
1180          // IMS hit
1181          case 1:
1182          {
1183             URIDone(Res);
1184             break;
1185          }
1186
1187          // Hard server error, not found or something
1188          case 3:
1189          {
1190             Fail();
1191             break;
1192          }
1193
1194          // Hard internal error, kill the connection and fail
1195          case 5:
1196          {
1197             delete File;
1198             File = 0;
1199
1200             Fail();
1201             RotateDNS();
1202             Server->Close();
1203             break;
1204          }
1205
1206          // We need to flush the data, the header is like a 404 w/ error text
1207          case 4:
1208          {
1209             Fail();
1210
1211             // Send to content to dev/null
1212             File = new FileFd("/dev/null",FileFd::WriteExists);
1213             Server->RunData();
1214             delete File;
1215             File = 0;
1216             break;
1217          }
1218
1219          default:
1220          Fail(_("Internal error"));
1221          break;
1222       }
1223
1224       FailCounter = 0;
1225    }
1226
1227    return 0;
1228 }
1229                                                                         /*}}}*/
1230
1231 int main()
1232 {
1233    setlocale(LC_ALL, "");
1234
1235    HttpMethod Mth;
1236
1237    return Mth.Loop();
1238 }
1239
1240