methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apt-pkg/fileutl.h>
  29 #include <apt-pkg/acquire-method.h>
  30 #include <apt-pkg/error.h>
  31 #include <apt-pkg/hashes.h>
  32
  33 #include <sys/stat.h>
  34 #include <sys/time.h>
  35 #include <utime.h>
  36 #include <unistd.h>
  37 #include <signal.h>
  38 #include <stdio.h>
  39 #include <errno.h>
  40 #include <string.h>
  41 #include <iostream>
  42 #include <apti18n.h>
  43
  44 // Internet stuff
  45 #include <netdb.h>
  46
  47 #include "connect.h"
  48 #include "rfc2553emu.h"
  49 #include "http.h"
  50
  51                                                                         /*}}}*/
  52 using namespace std;
  53
  54 string HttpMethod::FailFile;
  55 int HttpMethod::FailFd = -1;
  56 time_t HttpMethod::FailTime = 0;
  57 unsigned long PipelineDepth = 10;
  58 unsigned long TimeOut = 120;
  59 bool Debug = false;
  60
  61
  62 unsigned long CircleBuf::BwReadLimit=0;
  63 unsigned long CircleBuf::BwTickReadData=0;
  64 struct timeval CircleBuf::BwReadTick={0,0};
  65 const unsigned int CircleBuf::BW_HZ=10;
  66
  67 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  68 // ---------------------------------------------------------------------
  69 /* */
  70 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
  71 {
  72    Buf = new unsigned char[Size];
  73    Reset();
  74
  75    CircleBuf::BwReadLimit = _config->FindI("Acquire::http::DlLimit",0)*1024;
  76 }
  77                                                                         /*}}}*/
  78 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  79 // ---------------------------------------------------------------------
  80 /* */
  81 void CircleBuf::Reset()
  82 {
  83    InP = 0;
  84    OutP = 0;
  85    StrPos = 0;
  86    MaxGet = (unsigned int)-1;
  87    OutQueue = string();
  88    if (Hash != 0)
  89    {
  90       delete Hash;
  91       Hash = new Hashes;
  92    }
  93 };
  94                                                                         /*}}}*/
  95 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  96 // ---------------------------------------------------------------------
  97 /* This fills up the buffer with as much data as is in the FD, assuming it
  98    is non-blocking.. */
  99 bool CircleBuf::Read(int Fd)
 100 {
 101    unsigned long BwReadMax;
 102
 103    while (1)
 104    {
 105       // Woops, buffer is full
 106       if (InP - OutP == Size)
 107          return true;
 108
 109       // what's left to read in this tick
 110       BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
 111
 112       if(CircleBuf::BwReadLimit) {
 113          struct timeval now;
 114          gettimeofday(&now,0);
 115
 116          unsigned long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
 117             now.tv_usec-CircleBuf::BwReadTick.tv_usec;
 118          if(d > 1000000/BW_HZ) {
 119             CircleBuf::BwReadTick = now;
 120             CircleBuf::BwTickReadData = 0;
 121          }
 122
 123          if(CircleBuf::BwTickReadData >= BwReadMax) {
 124             usleep(1000000/BW_HZ);
 125             return true;
 126          }
 127       }
 128
 129       // Write the buffer segment
 130       int Res;
 131       if(CircleBuf::BwReadLimit) {
 132          Res = read(Fd,Buf + (InP%Size),
 133                     BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
 134       } else
 135          Res = read(Fd,Buf + (InP%Size),LeftRead());
 136
 137       if(Res > 0 && BwReadLimit > 0)
 138          CircleBuf::BwTickReadData += Res;
 139
 140       if (Res == 0)
 141          return false;
 142       if (Res < 0)
 143       {
 144          if (errno == EAGAIN)
 145             return true;
 146          return false;
 147       }
 148
 149       if (InP == 0)
 150          gettimeofday(&Start,0);
 151       InP += Res;
 152    }
 153 }
 154                                                                         /*}}}*/
 155 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 156 // ---------------------------------------------------------------------
 157 /* This will hold the string in and fill the buffer with it as it empties */
 158 bool CircleBuf::Read(string Data)
 159 {
 160    OutQueue += Data;
 161    FillOut();
 162    return true;
 163 }
 164                                                                         /*}}}*/
 165 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 166 // ---------------------------------------------------------------------
 167 /* */
 168 void CircleBuf::FillOut()
 169 {
 170    if (OutQueue.empty() == true)
 171       return;
 172    while (1)
 173    {
 174       // Woops, buffer is full
 175       if (InP - OutP == Size)
 176          return;
 177
 178       // Write the buffer segment
 179       unsigned long Sz = LeftRead();
 180       if (OutQueue.length() - StrPos < Sz)
 181          Sz = OutQueue.length() - StrPos;
 182       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 183
 184       // Advance
 185       StrPos += Sz;
 186       InP += Sz;
 187       if (OutQueue.length() == StrPos)
 188       {
 189          StrPos = 0;
 190          OutQueue = "";
 191          return;
 192       }
 193    }
 194 }
 195                                                                         /*}}}*/
 196 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 197 // ---------------------------------------------------------------------
 198 /* This empties the buffer into the FD. */
 199 bool CircleBuf::Write(int Fd)
 200 {
 201    while (1)
 202    {
 203       FillOut();
 204
 205       // Woops, buffer is empty
 206       if (OutP == InP)
 207          return true;
 208
 209       if (OutP == MaxGet)
 210          return true;
 211
 212       // Write the buffer segment
 213       int Res;
 214       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 215
 216       if (Res == 0)
 217          return false;
 218       if (Res < 0)
 219       {
 220          if (errno == EAGAIN)
 221             return true;
 222
 223          return false;
 224       }
 225
 226       if (Hash != 0)
 227          Hash->Add(Buf + (OutP%Size),Res);
 228
 229       OutP += Res;
 230    }
 231 }
 232                                                                         /*}}}*/
 233 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 234 // ---------------------------------------------------------------------
 235 /* This copies till the first empty line */
 236 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 237 {
 238    // We cheat and assume it is unneeded to have more than one buffer load
 239    for (unsigned long I = OutP; I < InP; I++)
 240    {
 241       if (Buf[I%Size] != '\n')
 242          continue;
 243       ++I;
 244       if (I < InP  && Buf[I%Size] == '\r')
 245          ++I;
 246
 247       if (Single == false)
 248       {
 249          if (Buf[I%Size] != '\n')
 250             continue;
 251          ++I;
 252          if (I < InP  && Buf[I%Size] == '\r')
 253             ++I;
 254       }
 255
 256       if (I > InP)
 257          I = InP;
 258
 259       Data = "";
 260       while (OutP < I)
 261       {
 262          unsigned long Sz = LeftWrite();
 263          if (Sz == 0)
 264             return false;
 265          if (I - OutP < LeftWrite())
 266             Sz = I - OutP;
 267          Data += string((char *)(Buf + (OutP%Size)),Sz);
 268          OutP += Sz;
 269       }
 270       return true;
 271    }
 272    return false;
 273 }
 274                                                                         /*}}}*/
 275 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 276 // ---------------------------------------------------------------------
 277 /* */
 278 void CircleBuf::Stats()
 279 {
 280    if (InP == 0)
 281       return;
 282
 283    struct timeval Stop;
 284    gettimeofday(&Stop,0);
 285 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 286              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 287    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 288 }
 289                                                                         /*}}}*/
 290
 291 // ServerState::ServerState - Constructor                               /*{{{*/
 292 // ---------------------------------------------------------------------
 293 /* */
 294 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 295                         In(64*1024), Out(4*1024),
 296                         ServerName(Srv)
 297 {
 298    Reset();
 299 }
 300                                                                         /*}}}*/
 301 // ServerState::Open - Open a connection to the server                  /*{{{*/
 302 // ---------------------------------------------------------------------
 303 /* This opens a connection to the server. */
 304 bool ServerState::Open()
 305 {
 306    // Use the already open connection if possible.
 307    if (ServerFd != -1)
 308       return true;
 309
 310    Close();
 311    In.Reset();
 312    Out.Reset();
 313    Persistent = true;
 314
 315    // Determine the proxy setting
 316    if (getenv("http_proxy") == 0)
 317    {
 318       string DefProxy = _config->Find("Acquire::http::Proxy");
 319       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 320       if (SpecificProxy.empty() == false)
 321       {
 322          if (SpecificProxy == "DIRECT")
 323             Proxy = "";
 324          else
 325             Proxy = SpecificProxy;
 326       }
 327       else
 328          Proxy = DefProxy;
 329    }
 330    else
 331       Proxy = getenv("http_proxy");
 332
 333    // Parse no_proxy, a , separated list of domains
 334    if (getenv("no_proxy") != 0)
 335    {
 336       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 337          Proxy = "";
 338    }
 339
 340    // Determine what host and port to use based on the proxy settings
 341    int Port = 0;
 342    string Host;
 343    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 344    {
 345       if (ServerName.Port != 0)
 346          Port = ServerName.Port;
 347       Host = ServerName.Host;
 348    }
 349    else
 350    {
 351       if (Proxy.Port != 0)
 352          Port = Proxy.Port;
 353       Host = Proxy.Host;
 354    }
 355
 356    // Connect to the remote server
 357    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 358       return false;
 359
 360    return true;
 361 }
 362                                                                         /*}}}*/
 363 // ServerState::Close - Close a connection to the server                /*{{{*/
 364 // ---------------------------------------------------------------------
 365 /* */
 366 bool ServerState::Close()
 367 {
 368    close(ServerFd);
 369    ServerFd = -1;
 370    return true;
 371 }
 372                                                                         /*}}}*/
 373 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 374 // ---------------------------------------------------------------------
 375 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 376    parse error occured */
 377 int ServerState::RunHeaders()
 378 {
 379    State = Header;
 380
 381    Owner->Status(_("Waiting for headers"));
 382
 383    Major = 0;
 384    Minor = 0;
 385    Result = 0;
 386    Size = 0;
 387    StartPos = 0;
 388    Encoding = Closes;
 389    HaveContent = false;
 390    time(&Date);
 391
 392    do
 393    {
 394       string Data;
 395       if (In.WriteTillEl(Data) == false)
 396          continue;
 397
 398       if (Debug == true)
 399          clog << Data;
 400
 401       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 402       {
 403          string::const_iterator J = I;
 404          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 405          if (HeaderLine(string(I,J)) == false)
 406             return 2;
 407          I = J;
 408       }
 409
 410       // 100 Continue is a Nop...
 411       if (Result == 100)
 412          continue;
 413
 414       // Tidy up the connection persistance state.
 415       if (Encoding == Closes && HaveContent == true)
 416          Persistent = false;
 417
 418       return 0;
 419    }
 420    while (Owner->Go(false,this) == true);
 421
 422    return 1;
 423 }
 424                                                                         /*}}}*/
 425 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 426 // ---------------------------------------------------------------------
 427 /* */
 428 bool ServerState::RunData()
 429 {
 430    State = Data;
 431
 432    // Chunked transfer encoding is fun..
 433    if (Encoding == Chunked)
 434    {
 435       while (1)
 436       {
 437          // Grab the block size
 438          bool Last = true;
 439          string Data;
 440          In.Limit(-1);
 441          do
 442          {
 443             if (In.WriteTillEl(Data,true) == true)
 444                break;
 445          }
 446          while ((Last = Owner->Go(false,this)) == true);
 447
 448          if (Last == false)
 449             return false;
 450
 451          // See if we are done
 452          unsigned long Len = strtol(Data.c_str(),0,16);
 453          if (Len == 0)
 454          {
 455             In.Limit(-1);
 456
 457             // We have to remove the entity trailer
 458             Last = true;
 459             do
 460             {
 461                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 462                   break;
 463             }
 464             while ((Last = Owner->Go(false,this)) == true);
 465             if (Last == false)
 466                return false;
 467             return !_error->PendingError();
 468          }
 469
 470          // Transfer the block
 471          In.Limit(Len);
 472          while (Owner->Go(true,this) == true)
 473             if (In.IsLimit() == true)
 474                break;
 475
 476          // Error
 477          if (In.IsLimit() == false)
 478             return false;
 479
 480          // The server sends an extra new line before the next block specifier..
 481          In.Limit(-1);
 482          Last = true;
 483          do
 484          {
 485             if (In.WriteTillEl(Data,true) == true)
 486                break;
 487          }
 488          while ((Last = Owner->Go(false,this)) == true);
 489          if (Last == false)
 490             return false;
 491       }
 492    }
 493    else
 494    {
 495       /* Closes encoding is used when the server did not specify a size, the
 496          loss of the connection means we are done */
 497       if (Encoding == Closes)
 498          In.Limit(-1);
 499       else
 500          In.Limit(Size - StartPos);
 501
 502       // Just transfer the whole block.
 503       do
 504       {
 505          if (In.IsLimit() == false)
 506             continue;
 507
 508          In.Limit(-1);
 509          return !_error->PendingError();
 510       }
 511       while (Owner->Go(true,this) == true);
 512    }
 513
 514    return Owner->Flush(this) && !_error->PendingError();
 515 }
 516                                                                         /*}}}*/
 517 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 518 // ---------------------------------------------------------------------
 519 /* */
 520 bool ServerState::HeaderLine(string Line)
 521 {
 522    if (Line.empty() == true)
 523       return true;
 524
 525    // The http server might be trying to do something evil.
 526    if (Line.length() >= MAXLEN)
 527       return _error->Error(_("Got a single header line over %u chars"),MAXLEN);
 528
 529    string::size_type Pos = Line.find(' ');
 530    if (Pos == string::npos || Pos+1 > Line.length())
 531    {
 532       // Blah, some servers use "connection:closes", evil.
 533       Pos = Line.find(':');
 534       if (Pos == string::npos || Pos + 2 > Line.length())
 535          return _error->Error(_("Bad header line"));
 536       Pos++;
 537    }
 538
 539    // Parse off any trailing spaces between the : and the next word.
 540    string::size_type Pos2 = Pos;
 541    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 542       Pos2++;
 543
 544    string Tag = string(Line,0,Pos);
 545    string Val = string(Line,Pos2);
 546
 547    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 548    {
 549       // Evil servers return no version
 550       if (Line[4] == '/')
 551       {
 552          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 553                     &Result,Code) != 4)
 554             return _error->Error(_("The HTTP server sent an invalid reply header"));
 555       }
 556       else
 557       {
 558          Major = 0;
 559          Minor = 9;
 560          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 561             return _error->Error(_("The HTTP server sent an invalid reply header"));
 562       }
 563
 564       /* Check the HTTP response header to get the default persistance
 565          state. */
 566       if (Major < 1)
 567          Persistent = false;
 568       else
 569       {
 570          if (Major == 1 && Minor <= 0)
 571             Persistent = false;
 572          else
 573             Persistent = true;
 574       }
 575
 576       return true;
 577    }
 578
 579    if (stringcasecmp(Tag,"Content-Length:") == 0)
 580    {
 581       if (Encoding == Closes)
 582          Encoding = Stream;
 583       HaveContent = true;
 584
 585       // The length is already set from the Content-Range header
 586       if (StartPos != 0)
 587          return true;
 588
 589       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 590          return _error->Error(_("The HTTP server sent an invalid Content-Length header"));
 591       return true;
 592    }
 593
 594    if (stringcasecmp(Tag,"Content-Type:") == 0)
 595    {
 596       HaveContent = true;
 597       return true;
 598    }
 599
 600    if (stringcasecmp(Tag,"Content-Range:") == 0)
 601    {
 602       HaveContent = true;
 603
 604       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 605          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 606       if ((unsigned)StartPos > Size)
 607          return _error->Error(_("This HTTP server has broken range support"));
 608       return true;
 609    }
 610
 611    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 612    {
 613       HaveContent = true;
 614       if (stringcasecmp(Val,"chunked") == 0)
 615          Encoding = Chunked;
 616       return true;
 617    }
 618
 619    if (stringcasecmp(Tag,"Connection:") == 0)
 620    {
 621       if (stringcasecmp(Val,"close") == 0)
 622          Persistent = false;
 623       if (stringcasecmp(Val,"keep-alive") == 0)
 624          Persistent = true;
 625       return true;
 626    }
 627
 628    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 629    {
 630       if (StrToTime(Val,Date) == false)
 631          return _error->Error(_("Unknown date format"));
 632       return true;
 633    }
 634
 635    return true;
 636 }
 637                                                                         /*}}}*/
 638
 639 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 640 // ---------------------------------------------------------------------
 641 /* This places the http request in the outbound buffer */
 642 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 643 {
 644    URI Uri = Itm->Uri;
 645
 646    // The HTTP server expects a hostname with a trailing :port
 647    char Buf[1000];
 648    string ProperHost = Uri.Host;
 649    if (Uri.Port != 0)
 650    {
 651       sprintf(Buf,":%u",Uri.Port);
 652       ProperHost += Buf;
 653    }
 654
 655    // Just in case.
 656    if (Itm->Uri.length() >= sizeof(Buf))
 657        abort();
 658
 659    /* Build the request. We include a keep-alive header only for non-proxy
 660       requests. This is to tweak old http/1.0 servers that do support keep-alive
 661       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 662       will glitch HTTP/1.0 proxies because they do not filter it out and
 663       pass it on, HTTP/1.1 says the connection should default to keep alive
 664       and we expect the proxy to do this */
 665    if (Proxy.empty() == true)
 666       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 667               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 668    else
 669    {
 670       /* Generate a cache control header if necessary. We place a max
 671          cache age on index files, optionally set a no-cache directive
 672          and a no-store directive for archives. */
 673       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 674               Itm->Uri.c_str(),ProperHost.c_str());
 675       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 676          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 677       else
 678       {
 679          if (Itm->IndexFile == true)
 680             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 681                     _config->FindI("Acquire::http::Max-Age",0));
 682          else
 683          {
 684             if (_config->FindB("Acquire::http::No-Store",false) == true)
 685                strcat(Buf,"Cache-Control: no-store\r\n");
 686          }
 687       }
 688    }
 689
 690    string Req = Buf;
 691
 692    // Check for a partial file
 693    struct stat SBuf;
 694    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 695    {
 696       // In this case we send an if-range query with a range header
 697       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 698               TimeRFC1123(SBuf.st_mtime).c_str());
 699       Req += Buf;
 700    }
 701    else
 702    {
 703       if (Itm->LastModified != 0)
 704       {
 705          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 706          Req += Buf;
 707       }
 708    }
 709
 710    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 711       Req += string("Proxy-Authorization: Basic ") +
 712           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 713
 714    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 715       Req += string("Authorization: Basic ") +
 716           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 717
 718    Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
 719
 720    if (Debug == true)
 721       cerr << Req << endl;
 722
 723    Out.Read(Req);
 724 }
 725                                                                         /*}}}*/
 726 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 727 // ---------------------------------------------------------------------
 728 /* This runs the select loop over the server FDs, Output file FDs and
 729    stdin. */
 730 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 731 {
 732    // Server has closed the connection
 733    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 734                                ToFile == false))
 735       return false;
 736
 737    fd_set rfds,wfds;
 738    FD_ZERO(&rfds);
 739    FD_ZERO(&wfds);
 740
 741    /* Add the server. We only send more requests if the connection will
 742       be persisting */
 743    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 744        && Srv->Persistent == true)
 745       FD_SET(Srv->ServerFd,&wfds);
 746    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 747       FD_SET(Srv->ServerFd,&rfds);
 748
 749    // Add the file
 750    int FileFD = -1;
 751    if (File != 0)
 752       FileFD = File->Fd();
 753
 754    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 755       FD_SET(FileFD,&wfds);
 756
 757    // Add stdin
 758    FD_SET(STDIN_FILENO,&rfds);
 759
 760    // Figure out the max fd
 761    int MaxFd = FileFD;
 762    if (MaxFd < Srv->ServerFd)
 763       MaxFd = Srv->ServerFd;
 764
 765    // Select
 766    struct timeval tv;
 767    tv.tv_sec = TimeOut;
 768    tv.tv_usec = 0;
 769    int Res = 0;
 770    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 771    {
 772       if (errno == EINTR)
 773          return true;
 774       return _error->Errno("select",_("Select failed"));
 775    }
 776
 777    if (Res == 0)
 778    {
 779       _error->Error(_("Connection timed out"));
 780       return ServerDie(Srv);
 781    }
 782
 783    // Handle server IO
 784    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 785    {
 786       errno = 0;
 787       if (Srv->In.Read(Srv->ServerFd) == false)
 788          return ServerDie(Srv);
 789    }
 790
 791    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 792    {
 793       errno = 0;
 794       if (Srv->Out.Write(Srv->ServerFd) == false)
 795          return ServerDie(Srv);
 796    }
 797
 798    // Send data to the file
 799    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 800    {
 801       if (Srv->In.Write(FileFD) == false)
 802          return _error->Errno("write",_("Error writing to output file"));
 803    }
 804
 805    // Handle commands from APT
 806    if (FD_ISSET(STDIN_FILENO,&rfds))
 807    {
 808       if (Run(true) != -1)
 809          exit(100);
 810    }
 811
 812    return true;
 813 }
 814                                                                         /*}}}*/
 815 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 816 // ---------------------------------------------------------------------
 817 /* This takes the current input buffer from the Server FD and writes it
 818    into the file */
 819 bool HttpMethod::Flush(ServerState *Srv)
 820 {
 821    if (File != 0)
 822    {
 823       SetNonBlock(File->Fd(),false);
 824       if (Srv->In.WriteSpace() == false)
 825          return true;
 826
 827       while (Srv->In.WriteSpace() == true)
 828       {
 829          if (Srv->In.Write(File->Fd()) == false)
 830             return _error->Errno("write",_("Error writing to file"));
 831          if (Srv->In.IsLimit() == true)
 832             return true;
 833       }
 834
 835       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 836          return true;
 837    }
 838    return false;
 839 }
 840                                                                         /*}}}*/
 841 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 842 // ---------------------------------------------------------------------
 843 /* */
 844 bool HttpMethod::ServerDie(ServerState *Srv)
 845 {
 846    unsigned int LErrno = errno;
 847
 848    // Dump the buffer to the file
 849    if (Srv->State == ServerState::Data)
 850    {
 851       SetNonBlock(File->Fd(),false);
 852       while (Srv->In.WriteSpace() == true)
 853       {
 854          if (Srv->In.Write(File->Fd()) == false)
 855             return _error->Errno("write",_("Error writing to the file"));
 856
 857          // Done
 858          if (Srv->In.IsLimit() == true)
 859             return true;
 860       }
 861    }
 862
 863    // See if this is because the server finished the data stream
 864    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 865        Srv->Encoding != ServerState::Closes)
 866    {
 867       Srv->Close();
 868       if (LErrno == 0)
 869          return _error->Error(_("Error reading from server. Remote end closed connection"));
 870       errno = LErrno;
 871       return _error->Errno("read",_("Error reading from server"));
 872    }
 873    else
 874    {
 875       Srv->In.Limit(-1);
 876
 877       // Nothing left in the buffer
 878       if (Srv->In.WriteSpace() == false)
 879          return false;
 880
 881       // We may have got multiple responses back in one packet..
 882       Srv->Close();
 883       return true;
 884    }
 885
 886    return false;
 887 }
 888                                                                         /*}}}*/
 889 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 890 // ---------------------------------------------------------------------
 891 /* We look at the header data we got back from the server and decide what
 892    to do. Returns
 893      0 - File is open,
 894      1 - IMS hit
 895      3 - Unrecoverable error
 896      4 - Error with error content page
 897      5 - Unrecoverable non-server error (close the connection) */
 898 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 899 {
 900    // Not Modified
 901    if (Srv->Result == 304)
 902    {
 903       unlink(Queue->DestFile.c_str());
 904       Res.IMSHit = true;
 905       Res.LastModified = Queue->LastModified;
 906       return 1;
 907    }
 908
 909    /* We have a reply we dont handle. This should indicate a perm server
 910       failure */
 911    if (Srv->Result < 200 || Srv->Result >= 300)
 912    {
 913       _error->Error("%u %s",Srv->Result,Srv->Code);
 914       if (Srv->HaveContent == true)
 915          return 4;
 916       return 3;
 917    }
 918
 919    // This is some sort of 2xx 'data follows' reply
 920    Res.LastModified = Srv->Date;
 921    Res.Size = Srv->Size;
 922
 923    // Open the file
 924    delete File;
 925    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 926    if (_error->PendingError() == true)
 927       return 5;
 928
 929    FailFile = Queue->DestFile;
 930    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 931    FailFd = File->Fd();
 932    FailTime = Srv->Date;
 933
 934    // Set the expected size
 935    if (Srv->StartPos >= 0)
 936    {
 937       Res.ResumePoint = Srv->StartPos;
 938       ftruncate(File->Fd(),Srv->StartPos);
 939    }
 940
 941    // Set the start point
 942    lseek(File->Fd(),0,SEEK_END);
 943
 944    delete Srv->In.Hash;
 945    Srv->In.Hash = new Hashes;
 946
 947    // Fill the Hash if the file is non-empty (resume)
 948    if (Srv->StartPos > 0)
 949    {
 950       lseek(File->Fd(),0,SEEK_SET);
 951       if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
 952       {
 953          _error->Errno("read",_("Problem hashing file"));
 954          return 5;
 955       }
 956       lseek(File->Fd(),0,SEEK_END);
 957    }
 958
 959    SetNonBlock(File->Fd(),true);
 960    return 0;
 961 }
 962                                                                         /*}}}*/
 963 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 964 // ---------------------------------------------------------------------
 965 /* This closes and timestamps the open file. This is neccessary to get
 966    resume behavoir on user abort */
 967 void HttpMethod::SigTerm(int)
 968 {
 969    if (FailFd == -1)
 970       _exit(100);
 971    close(FailFd);
 972
 973    // Timestamp
 974    struct utimbuf UBuf;
 975    UBuf.actime = FailTime;
 976    UBuf.modtime = FailTime;
 977    utime(FailFile.c_str(),&UBuf);
 978
 979    _exit(100);
 980 }
 981                                                                         /*}}}*/
 982 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 983 // ---------------------------------------------------------------------
 984 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 985    depth. */
 986 bool HttpMethod::Fetch(FetchItem *)
 987 {
 988    if (Server == 0)
 989       return true;
 990
 991    // Queue the requests
 992    int Depth = -1;
 993    bool Tail = false;
 994    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 995         I = I->Next, Depth++)
 996    {
 997       // If pipelining is disabled, we only queue 1 request
 998       if (Server->Pipeline == false && Depth >= 0)
 999          break;
1000
1001       // Make sure we stick with the same server
1002       if (Server->Comp(I->Uri) == false)
1003          break;
1004       if (QueueBack == I)
1005          Tail = true;
1006       if (Tail == true)
1007       {
1008          QueueBack = I->Next;
1009          SendReq(I,Server->Out);
1010          continue;
1011       }
1012    }
1013
1014    return true;
1015 };
1016                                                                         /*}}}*/
1017 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
1018 // ---------------------------------------------------------------------
1019 /* We stash the desired pipeline depth */
1020 bool HttpMethod::Configuration(string Message)
1021 {
1022    if (pkgAcqMethod::Configuration(Message) == false)
1023       return false;
1024
1025    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
1026    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
1027                                   PipelineDepth);
1028    Debug = _config->FindB("Debug::Acquire::http",false);
1029
1030    return true;
1031 }
1032                                                                         /*}}}*/
1033 // HttpMethod::Loop - Main loop                                         /*{{{*/
1034 // ---------------------------------------------------------------------
1035 /* */
1036 int HttpMethod::Loop()
1037 {
1038    signal(SIGTERM,SigTerm);
1039    signal(SIGINT,SigTerm);
1040
1041    Server = 0;
1042
1043    int FailCounter = 0;
1044    while (1)
1045    {
1046       // We have no commands, wait for some to arrive
1047       if (Queue == 0)
1048       {
1049          if (WaitFd(STDIN_FILENO) == false)
1050             return 0;
1051       }
1052
1053       /* Run messages, we can accept 0 (no message) if we didn't
1054          do a WaitFd above.. Otherwise the FD is closed. */
1055       int Result = Run(true);
1056       if (Result != -1 && (Result != 0 || Queue == 0))
1057          return 100;
1058
1059       if (Queue == 0)
1060          continue;
1061
1062       // Connect to the server
1063       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1064       {
1065          delete Server;
1066          Server = new ServerState(Queue->Uri,this);
1067       }
1068
1069       /* If the server has explicitly said this is the last connection
1070          then we pre-emptively shut down the pipeline and tear down
1071          the connection. This will speed up HTTP/1.0 servers a tad
1072          since we don't have to wait for the close sequence to
1073          complete */
1074       if (Server->Persistent == false)
1075          Server->Close();
1076
1077       // Reset the pipeline
1078       if (Server->ServerFd == -1)
1079          QueueBack = Queue;
1080
1081       // Connnect to the host
1082       if (Server->Open() == false)
1083       {
1084          Fail(true);
1085          delete Server;
1086          Server = 0;
1087          continue;
1088       }
1089
1090       // Fill the pipeline.
1091       Fetch(0);
1092
1093       // Fetch the next URL header data from the server.
1094       switch (Server->RunHeaders())
1095       {
1096          case 0:
1097          break;
1098
1099          // The header data is bad
1100          case 2:
1101          {
1102             _error->Error(_("Bad header data"));
1103             Fail(true);
1104             RotateDNS();
1105             continue;
1106          }
1107
1108          // The server closed a connection during the header get..
1109          default:
1110          case 1:
1111          {
1112             FailCounter++;
1113             _error->Discard();
1114             Server->Close();
1115             Server->Pipeline = false;
1116
1117             if (FailCounter >= 2)
1118             {
1119                Fail(_("Connection failed"),true);
1120                FailCounter = 0;
1121             }
1122
1123             RotateDNS();
1124             continue;
1125          }
1126       };
1127
1128       // Decide what to do.
1129       FetchResult Res;
1130       Res.Filename = Queue->DestFile;
1131       switch (DealWithHeaders(Res,Server))
1132       {
1133          // Ok, the file is Open
1134          case 0:
1135          {
1136             URIStart(Res);
1137
1138             // Run the data
1139             bool Result =  Server->RunData();
1140
1141             /* If the server is sending back sizeless responses then fill in
1142                the size now */
1143             if (Res.Size == 0)
1144                Res.Size = File->Size();
1145
1146             // Close the file, destroy the FD object and timestamp it
1147             FailFd = -1;
1148             delete File;
1149             File = 0;
1150
1151             // Timestamp
1152             struct utimbuf UBuf;
1153             time(&UBuf.actime);
1154             UBuf.actime = Server->Date;
1155             UBuf.modtime = Server->Date;
1156             utime(Queue->DestFile.c_str(),&UBuf);
1157
1158             // Send status to APT
1159             if (Result == true)
1160             {
1161                Res.TakeHashes(*Server->In.Hash);
1162                URIDone(Res);
1163             }
1164             else
1165                Fail(true);
1166
1167             break;
1168          }
1169
1170          // IMS hit
1171          case 1:
1172          {
1173             URIDone(Res);
1174             break;
1175          }
1176
1177          // Hard server error, not found or something
1178          case 3:
1179          {
1180             Fail();
1181             break;
1182          }
1183
1184          // Hard internal error, kill the connection and fail
1185          case 5:
1186          {
1187             delete File;
1188             File = 0;
1189
1190             Fail();
1191             RotateDNS();
1192             Server->Close();
1193             break;
1194          }
1195
1196          // We need to flush the data, the header is like a 404 w/ error text
1197          case 4:
1198          {
1199             Fail();
1200
1201             // Send to content to dev/null
1202             File = new FileFd("/dev/null",FileFd::WriteExists);
1203             Server->RunData();
1204             delete File;
1205             File = 0;
1206             break;
1207          }
1208
1209          default:
1210          Fail(_("Internal error"));
1211          break;
1212       }
1213
1214       FailCounter = 0;
1215    }
1216
1217    return 0;
1218 }
1219                                                                         /*}}}*/
1220
1221 int main()
1222 {
1223    setlocale(LC_ALL, "");
1224
1225    HttpMethod Mth;
1226
1227    return Mth.Loop();
1228 }
1229
1230