methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.50 2001/03/06 07:15:29 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apt-pkg/fileutl.h>
  29 #include <apt-pkg/acquire-method.h>
  30 #include <apt-pkg/error.h>
  31 #include <apt-pkg/hashes.h>
  32
  33 #include <sys/stat.h>
  34 #include <sys/time.h>
  35 #include <utime.h>
  36 #include <unistd.h>
  37 #include <signal.h>
  38 #include <stdio.h>
  39 #include <errno.h>
  40
  41 // Internet stuff
  42 #include <netdb.h>
  43
  44 #include "connect.h"
  45 #include "rfc2553emu.h"
  46 #include "http.h"
  47
  48                                                                         /*}}}*/
  49
  50 string HttpMethod::FailFile;
  51 int HttpMethod::FailFd = -1;
  52 time_t HttpMethod::FailTime = 0;
  53 unsigned long PipelineDepth = 10;
  54 unsigned long TimeOut = 120;
  55 bool Debug = false;
  56
  57 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  58 // ---------------------------------------------------------------------
  59 /* */
  60 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
  61 {
  62    Buf = new unsigned char[Size];
  63    Reset();
  64 }
  65                                                                         /*}}}*/
  66 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  67 // ---------------------------------------------------------------------
  68 /* */
  69 void CircleBuf::Reset()
  70 {
  71    InP = 0;
  72    OutP = 0;
  73    StrPos = 0;
  74    MaxGet = (unsigned int)-1;
  75    OutQueue = string();
  76    if (Hash != 0)
  77    {
  78       delete Hash;
  79       Hash = new Hashes;
  80    }
  81 };
  82                                                                         /*}}}*/
  83 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  84 // ---------------------------------------------------------------------
  85 /* This fills up the buffer with as much data as is in the FD, assuming it
  86    is non-blocking.. */
  87 bool CircleBuf::Read(int Fd)
  88 {
  89    while (1)
  90    {
  91       // Woops, buffer is full
  92       if (InP - OutP == Size)
  93          return true;
  94
  95       // Write the buffer segment
  96       int Res;
  97       Res = read(Fd,Buf + (InP%Size),LeftRead());
  98
  99       if (Res == 0)
 100          return false;
 101       if (Res < 0)
 102       {
 103          if (errno == EAGAIN)
 104             return true;
 105          return false;
 106       }
 107
 108       if (InP == 0)
 109          gettimeofday(&Start,0);
 110       InP += Res;
 111    }
 112 }
 113                                                                         /*}}}*/
 114 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 115 // ---------------------------------------------------------------------
 116 /* This will hold the string in and fill the buffer with it as it empties */
 117 bool CircleBuf::Read(string Data)
 118 {
 119    OutQueue += Data;
 120    FillOut();
 121    return true;
 122 }
 123                                                                         /*}}}*/
 124 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 125 // ---------------------------------------------------------------------
 126 /* */
 127 void CircleBuf::FillOut()
 128 {
 129    if (OutQueue.empty() == true)
 130       return;
 131    while (1)
 132    {
 133       // Woops, buffer is full
 134       if (InP - OutP == Size)
 135          return;
 136
 137       // Write the buffer segment
 138       unsigned long Sz = LeftRead();
 139       if (OutQueue.length() - StrPos < Sz)
 140          Sz = OutQueue.length() - StrPos;
 141       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 142
 143       // Advance
 144       StrPos += Sz;
 145       InP += Sz;
 146       if (OutQueue.length() == StrPos)
 147       {
 148          StrPos = 0;
 149          OutQueue = "";
 150          return;
 151       }
 152    }
 153 }
 154                                                                         /*}}}*/
 155 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 156 // ---------------------------------------------------------------------
 157 /* This empties the buffer into the FD. */
 158 bool CircleBuf::Write(int Fd)
 159 {
 160    while (1)
 161    {
 162       FillOut();
 163
 164       // Woops, buffer is empty
 165       if (OutP == InP)
 166          return true;
 167
 168       if (OutP == MaxGet)
 169          return true;
 170
 171       // Write the buffer segment
 172       int Res;
 173       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 174
 175       if (Res == 0)
 176          return false;
 177       if (Res < 0)
 178       {
 179          if (errno == EAGAIN)
 180             return true;
 181
 182          return false;
 183       }
 184
 185       if (Hash != 0)
 186          Hash->Add(Buf + (OutP%Size),Res);
 187
 188       OutP += Res;
 189    }
 190 }
 191                                                                         /*}}}*/
 192 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 193 // ---------------------------------------------------------------------
 194 /* This copies till the first empty line */
 195 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 196 {
 197    // We cheat and assume it is unneeded to have more than one buffer load
 198    for (unsigned long I = OutP; I < InP; I++)
 199    {
 200       if (Buf[I%Size] != '\n')
 201          continue;
 202       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 203
 204       if (Single == false)
 205       {
 206          if (Buf[I%Size] != '\n')
 207             continue;
 208          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 209       }
 210
 211       if (I > InP)
 212          I = InP;
 213
 214       Data = "";
 215       while (OutP < I)
 216       {
 217          unsigned long Sz = LeftWrite();
 218          if (Sz == 0)
 219             return false;
 220          if (I - OutP < LeftWrite())
 221             Sz = I - OutP;
 222          Data += string((char *)(Buf + (OutP%Size)),Sz);
 223          OutP += Sz;
 224       }
 225       return true;
 226    }
 227    return false;
 228 }
 229                                                                         /*}}}*/
 230 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 231 // ---------------------------------------------------------------------
 232 /* */
 233 void CircleBuf::Stats()
 234 {
 235    if (InP == 0)
 236       return;
 237
 238    struct timeval Stop;
 239    gettimeofday(&Stop,0);
 240 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 241              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 242    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 243 }
 244                                                                         /*}}}*/
 245
 246 // ServerState::ServerState - Constructor                               /*{{{*/
 247 // ---------------------------------------------------------------------
 248 /* */
 249 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 250                         In(64*1024), Out(4*1024),
 251                         ServerName(Srv)
 252 {
 253    Reset();
 254 }
 255                                                                         /*}}}*/
 256 // ServerState::Open - Open a connection to the server                  /*{{{*/
 257 // ---------------------------------------------------------------------
 258 /* This opens a connection to the server. */
 259 bool ServerState::Open()
 260 {
 261    // Use the already open connection if possible.
 262    if (ServerFd != -1)
 263       return true;
 264
 265    Close();
 266    In.Reset();
 267    Out.Reset();
 268    Persistent = true;
 269
 270    // Determine the proxy setting
 271    if (getenv("http_proxy") == 0)
 272    {
 273       string DefProxy = _config->Find("Acquire::http::Proxy");
 274       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 275       if (SpecificProxy.empty() == false)
 276       {
 277          if (SpecificProxy == "DIRECT")
 278             Proxy = "";
 279          else
 280             Proxy = SpecificProxy;
 281       }
 282       else
 283          Proxy = DefProxy;
 284    }
 285    else
 286       Proxy = getenv("http_proxy");
 287
 288    // Parse no_proxy, a , separated list of domains
 289    if (getenv("no_proxy") != 0)
 290    {
 291       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 292          Proxy = "";
 293    }
 294
 295    // Determine what host and port to use based on the proxy settings
 296    int Port = 0;
 297    string Host;
 298    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 299    {
 300       if (ServerName.Port != 0)
 301          Port = ServerName.Port;
 302       Host = ServerName.Host;
 303    }
 304    else
 305    {
 306       if (Proxy.Port != 0)
 307          Port = Proxy.Port;
 308       Host = Proxy.Host;
 309    }
 310
 311    // Connect to the remote server
 312    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 313       return false;
 314
 315    return true;
 316 }
 317                                                                         /*}}}*/
 318 // ServerState::Close - Close a connection to the server                /*{{{*/
 319 // ---------------------------------------------------------------------
 320 /* */
 321 bool ServerState::Close()
 322 {
 323    close(ServerFd);
 324    ServerFd = -1;
 325    return true;
 326 }
 327                                                                         /*}}}*/
 328 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 329 // ---------------------------------------------------------------------
 330 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 331    parse error occured */
 332 int ServerState::RunHeaders()
 333 {
 334    State = Header;
 335
 336    Owner->Status("Waiting for file");
 337
 338    Major = 0;
 339    Minor = 0;
 340    Result = 0;
 341    Size = 0;
 342    StartPos = 0;
 343    Encoding = Closes;
 344    HaveContent = false;
 345    time(&Date);
 346
 347    do
 348    {
 349       string Data;
 350       if (In.WriteTillEl(Data) == false)
 351          continue;
 352
 353       if (Debug == true)
 354          clog << Data;
 355
 356       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 357       {
 358          string::const_iterator J = I;
 359          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 360          if (HeaderLine(string(I,J-I)) == false)
 361             return 2;
 362          I = J;
 363       }
 364
 365       // 100 Continue is a Nop...
 366       if (Result == 100)
 367          continue;
 368
 369       // Tidy up the connection persistance state.
 370       if (Encoding == Closes && HaveContent == true)
 371          Persistent = false;
 372
 373       return 0;
 374    }
 375    while (Owner->Go(false,this) == true);
 376
 377    return 1;
 378 }
 379                                                                         /*}}}*/
 380 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 381 // ---------------------------------------------------------------------
 382 /* */
 383 bool ServerState::RunData()
 384 {
 385    State = Data;
 386
 387    // Chunked transfer encoding is fun..
 388    if (Encoding == Chunked)
 389    {
 390       while (1)
 391       {
 392          // Grab the block size
 393          bool Last = true;
 394          string Data;
 395          In.Limit(-1);
 396          do
 397          {
 398             if (In.WriteTillEl(Data,true) == true)
 399                break;
 400          }
 401          while ((Last = Owner->Go(false,this)) == true);
 402
 403          if (Last == false)
 404             return false;
 405
 406          // See if we are done
 407          unsigned long Len = strtol(Data.c_str(),0,16);
 408          if (Len == 0)
 409          {
 410             In.Limit(-1);
 411
 412             // We have to remove the entity trailer
 413             Last = true;
 414             do
 415             {
 416                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 417                   break;
 418             }
 419             while ((Last = Owner->Go(false,this)) == true);
 420             if (Last == false)
 421                return false;
 422             return !_error->PendingError();
 423          }
 424
 425          // Transfer the block
 426          In.Limit(Len);
 427          while (Owner->Go(true,this) == true)
 428             if (In.IsLimit() == true)
 429                break;
 430
 431          // Error
 432          if (In.IsLimit() == false)
 433             return false;
 434
 435          // The server sends an extra new line before the next block specifier..
 436          In.Limit(-1);
 437          Last = true;
 438          do
 439          {
 440             if (In.WriteTillEl(Data,true) == true)
 441                break;
 442          }
 443          while ((Last = Owner->Go(false,this)) == true);
 444          if (Last == false)
 445             return false;
 446       }
 447    }
 448    else
 449    {
 450       /* Closes encoding is used when the server did not specify a size, the
 451          loss of the connection means we are done */
 452       if (Encoding == Closes)
 453          In.Limit(-1);
 454       else
 455          In.Limit(Size - StartPos);
 456
 457       // Just transfer the whole block.
 458       do
 459       {
 460          if (In.IsLimit() == false)
 461             continue;
 462
 463          In.Limit(-1);
 464          return !_error->PendingError();
 465       }
 466       while (Owner->Go(true,this) == true);
 467    }
 468
 469    return Owner->Flush(this) && !_error->PendingError();
 470 }
 471                                                                         /*}}}*/
 472 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 473 // ---------------------------------------------------------------------
 474 /* */
 475 bool ServerState::HeaderLine(string Line)
 476 {
 477    if (Line.empty() == true)
 478       return true;
 479
 480    // The http server might be trying to do something evil.
 481    if (Line.length() >= MAXLEN)
 482       return _error->Error("Got a single header line over %u chars",MAXLEN);
 483
 484    string::size_type Pos = Line.find(' ');
 485    if (Pos == string::npos || Pos+1 > Line.length())
 486    {
 487       // Blah, some servers use "connection:closes", evil.
 488       Pos = Line.find(':');
 489       if (Pos == string::npos || Pos + 2 > Line.length())
 490          return _error->Error("Bad header line");
 491       Pos++;
 492    }
 493
 494    // Parse off any trailing spaces between the : and the next word.
 495    string::size_type Pos2 = Pos;
 496    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 497       Pos2++;
 498
 499    string Tag = string(Line,0,Pos);
 500    string Val = string(Line,Pos2);
 501
 502    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 503    {
 504       // Evil servers return no version
 505       if (Line[4] == '/')
 506       {
 507          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 508                     &Result,Code) != 4)
 509             return _error->Error("The http server sent an invalid reply header");
 510       }
 511       else
 512       {
 513          Major = 0;
 514          Minor = 9;
 515          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 516             return _error->Error("The http server sent an invalid reply header");
 517       }
 518
 519       /* Check the HTTP response header to get the default persistance
 520          state. */
 521       if (Major < 1)
 522          Persistent = false;
 523       else
 524       {
 525          if (Major == 1 && Minor <= 0)
 526             Persistent = false;
 527          else
 528             Persistent = true;
 529       }
 530
 531       return true;
 532    }
 533
 534    if (stringcasecmp(Tag,"Content-Length:") == 0)
 535    {
 536       if (Encoding == Closes)
 537          Encoding = Stream;
 538       HaveContent = true;
 539
 540       // The length is already set from the Content-Range header
 541       if (StartPos != 0)
 542          return true;
 543
 544       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 545          return _error->Error("The http server sent an invalid Content-Length header");
 546       return true;
 547    }
 548
 549    if (stringcasecmp(Tag,"Content-Type:") == 0)
 550    {
 551       HaveContent = true;
 552       return true;
 553    }
 554
 555    if (stringcasecmp(Tag,"Content-Range:") == 0)
 556    {
 557       HaveContent = true;
 558
 559       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 560          return _error->Error("The http server sent an invalid Content-Range header");
 561       if ((unsigned)StartPos > Size)
 562          return _error->Error("This http server has broken range support");
 563       return true;
 564    }
 565
 566    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 567    {
 568       HaveContent = true;
 569       if (stringcasecmp(Val,"chunked") == 0)
 570          Encoding = Chunked;
 571       return true;
 572    }
 573
 574    if (stringcasecmp(Tag,"Connection:") == 0)
 575    {
 576       if (stringcasecmp(Val,"close") == 0)
 577          Persistent = false;
 578       if (stringcasecmp(Val,"keep-alive") == 0)
 579          Persistent = true;
 580       return true;
 581    }
 582
 583    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 584    {
 585       if (StrToTime(Val,Date) == false)
 586          return _error->Error("Unknown date format");
 587       return true;
 588    }
 589
 590    return true;
 591 }
 592                                                                         /*}}}*/
 593
 594 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 595 // ---------------------------------------------------------------------
 596 /* This places the http request in the outbound buffer */
 597 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 598 {
 599    URI Uri = Itm->Uri;
 600
 601    // The HTTP server expects a hostname with a trailing :port
 602    char Buf[1000];
 603    string ProperHost = Uri.Host;
 604    if (Uri.Port != 0)
 605    {
 606       sprintf(Buf,":%u",Uri.Port);
 607       ProperHost += Buf;
 608    }
 609
 610    // Just in case.
 611    if (Itm->Uri.length() >= sizeof(Buf))
 612        abort();
 613
 614    /* Build the request. We include a keep-alive header only for non-proxy
 615       requests. This is to tweak old http/1.0 servers that do support keep-alive
 616       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 617       will glitch HTTP/1.0 proxies because they do not filter it out and
 618       pass it on, HTTP/1.1 says the connection should default to keep alive
 619       and we expect the proxy to do this */
 620    if (Proxy.empty() == true)
 621       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 622               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 623    else
 624    {
 625       /* Generate a cache control header if necessary. We place a max
 626          cache age on index files, optionally set a no-cache directive
 627          and a no-store directive for archives. */
 628       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 629               Itm->Uri.c_str(),ProperHost.c_str());
 630       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 631          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 632       else
 633       {
 634          if (Itm->IndexFile == true)
 635             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 636                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 637          else
 638          {
 639             if (_config->FindB("Acquire::http::No-Store",false) == true)
 640                strcat(Buf,"Cache-Control: no-store\r\n");
 641          }
 642       }
 643    }
 644
 645    string Req = Buf;
 646
 647    // Check for a partial file
 648    struct stat SBuf;
 649    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 650    {
 651       // In this case we send an if-range query with a range header
 652       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 653               TimeRFC1123(SBuf.st_mtime).c_str());
 654       Req += Buf;
 655    }
 656    else
 657    {
 658       if (Itm->LastModified != 0)
 659       {
 660          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 661          Req += Buf;
 662       }
 663    }
 664
 665    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 666       Req += string("Proxy-Authorization: Basic ") +
 667           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 668
 669    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 670       Req += string("Authorization: Basic ") +
 671           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 672
 673    Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
 674
 675    if (Debug == true)
 676       cerr << Req << endl;
 677
 678    Out.Read(Req);
 679 }
 680                                                                         /*}}}*/
 681 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 682 // ---------------------------------------------------------------------
 683 /* This runs the select loop over the server FDs, Output file FDs and
 684    stdin. */
 685 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 686 {
 687    // Server has closed the connection
 688    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 689                                ToFile == false))
 690       return false;
 691
 692    fd_set rfds,wfds;
 693    FD_ZERO(&rfds);
 694    FD_ZERO(&wfds);
 695
 696    /* Add the server. We only send more requests if the connection will
 697       be persisting */
 698    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 699        && Srv->Persistent == true)
 700       FD_SET(Srv->ServerFd,&wfds);
 701    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 702       FD_SET(Srv->ServerFd,&rfds);
 703
 704    // Add the file
 705    int FileFD = -1;
 706    if (File != 0)
 707       FileFD = File->Fd();
 708
 709    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 710       FD_SET(FileFD,&wfds);
 711
 712    // Add stdin
 713    FD_SET(STDIN_FILENO,&rfds);
 714
 715    // Figure out the max fd
 716    int MaxFd = FileFD;
 717    if (MaxFd < Srv->ServerFd)
 718       MaxFd = Srv->ServerFd;
 719
 720    // Select
 721    struct timeval tv;
 722    tv.tv_sec = TimeOut;
 723    tv.tv_usec = 0;
 724    int Res = 0;
 725    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 726       return _error->Errno("select","Select failed");
 727
 728    if (Res == 0)
 729    {
 730       _error->Error("Connection timed out");
 731       return ServerDie(Srv);
 732    }
 733
 734    // Handle server IO
 735    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 736    {
 737       errno = 0;
 738       if (Srv->In.Read(Srv->ServerFd) == false)
 739          return ServerDie(Srv);
 740    }
 741
 742    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 743    {
 744       errno = 0;
 745       if (Srv->Out.Write(Srv->ServerFd) == false)
 746          return ServerDie(Srv);
 747    }
 748
 749    // Send data to the file
 750    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 751    {
 752       if (Srv->In.Write(FileFD) == false)
 753          return _error->Errno("write","Error writing to output file");
 754    }
 755
 756    // Handle commands from APT
 757    if (FD_ISSET(STDIN_FILENO,&rfds))
 758    {
 759       if (Run(true) != -1)
 760          exit(100);
 761    }
 762
 763    return true;
 764 }
 765                                                                         /*}}}*/
 766 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 767 // ---------------------------------------------------------------------
 768 /* This takes the current input buffer from the Server FD and writes it
 769    into the file */
 770 bool HttpMethod::Flush(ServerState *Srv)
 771 {
 772    if (File != 0)
 773    {
 774       SetNonBlock(File->Fd(),false);
 775       if (Srv->In.WriteSpace() == false)
 776          return true;
 777
 778       while (Srv->In.WriteSpace() == true)
 779       {
 780          if (Srv->In.Write(File->Fd()) == false)
 781             return _error->Errno("write","Error writing to file");
 782          if (Srv->In.IsLimit() == true)
 783             return true;
 784       }
 785
 786       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 787          return true;
 788    }
 789    return false;
 790 }
 791                                                                         /*}}}*/
 792 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 793 // ---------------------------------------------------------------------
 794 /* */
 795 bool HttpMethod::ServerDie(ServerState *Srv)
 796 {
 797    unsigned int LErrno = errno;
 798
 799    // Dump the buffer to the file
 800    if (Srv->State == ServerState::Data)
 801    {
 802       SetNonBlock(File->Fd(),false);
 803       while (Srv->In.WriteSpace() == true)
 804       {
 805          if (Srv->In.Write(File->Fd()) == false)
 806             return _error->Errno("write","Error writing to the file");
 807
 808          // Done
 809          if (Srv->In.IsLimit() == true)
 810             return true;
 811       }
 812    }
 813
 814    // See if this is because the server finished the data stream
 815    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 816        Srv->Encoding != ServerState::Closes)
 817    {
 818       Srv->Close();
 819       if (LErrno == 0)
 820          return _error->Error("Error reading from server Remote end closed connection");
 821       errno = LErrno;
 822       return _error->Errno("read","Error reading from server");
 823    }
 824    else
 825    {
 826       Srv->In.Limit(-1);
 827
 828       // Nothing left in the buffer
 829       if (Srv->In.WriteSpace() == false)
 830          return false;
 831
 832       // We may have got multiple responses back in one packet..
 833       Srv->Close();
 834       return true;
 835    }
 836
 837    return false;
 838 }
 839                                                                         /*}}}*/
 840 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 841 // ---------------------------------------------------------------------
 842 /* We look at the header data we got back from the server and decide what
 843    to do. Returns
 844      0 - File is open,
 845      1 - IMS hit
 846      3 - Unrecoverable error
 847      4 - Error with error content page
 848      5 - Unrecoverable non-server error (close the connection) */
 849 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 850 {
 851    // Not Modified
 852    if (Srv->Result == 304)
 853    {
 854       unlink(Queue->DestFile.c_str());
 855       Res.IMSHit = true;
 856       Res.LastModified = Queue->LastModified;
 857       return 1;
 858    }
 859
 860    /* We have a reply we dont handle. This should indicate a perm server
 861       failure */
 862    if (Srv->Result < 200 || Srv->Result >= 300)
 863    {
 864       _error->Error("%u %s",Srv->Result,Srv->Code);
 865       if (Srv->HaveContent == true)
 866          return 4;
 867       return 3;
 868    }
 869
 870    // This is some sort of 2xx 'data follows' reply
 871    Res.LastModified = Srv->Date;
 872    Res.Size = Srv->Size;
 873
 874    // Open the file
 875    delete File;
 876    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 877    if (_error->PendingError() == true)
 878       return 5;
 879
 880    FailFile = Queue->DestFile;
 881    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 882    FailFd = File->Fd();
 883    FailTime = Srv->Date;
 884
 885    // Set the expected size
 886    if (Srv->StartPos >= 0)
 887    {
 888       Res.ResumePoint = Srv->StartPos;
 889       ftruncate(File->Fd(),Srv->StartPos);
 890    }
 891
 892    // Set the start point
 893    lseek(File->Fd(),0,SEEK_END);
 894
 895    delete Srv->In.Hash;
 896    Srv->In.Hash = new Hashes;
 897
 898    // Fill the Hash if the file is non-empty (resume)
 899    if (Srv->StartPos > 0)
 900    {
 901       lseek(File->Fd(),0,SEEK_SET);
 902       if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
 903       {
 904          _error->Errno("read","Problem hashing file");
 905          return 5;
 906       }
 907       lseek(File->Fd(),0,SEEK_END);
 908    }
 909
 910    SetNonBlock(File->Fd(),true);
 911    return 0;
 912 }
 913                                                                         /*}}}*/
 914 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 915 // ---------------------------------------------------------------------
 916 /* This closes and timestamps the open file. This is neccessary to get
 917    resume behavoir on user abort */
 918 void HttpMethod::SigTerm(int)
 919 {
 920    if (FailFd == -1)
 921       _exit(100);
 922    close(FailFd);
 923
 924    // Timestamp
 925    struct utimbuf UBuf;
 926    UBuf.actime = FailTime;
 927    UBuf.modtime = FailTime;
 928    utime(FailFile.c_str(),&UBuf);
 929
 930    _exit(100);
 931 }
 932                                                                         /*}}}*/
 933 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 934 // ---------------------------------------------------------------------
 935 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 936    depth. */
 937 bool HttpMethod::Fetch(FetchItem *)
 938 {
 939    if (Server == 0)
 940       return true;
 941
 942    // Queue the requests
 943    int Depth = -1;
 944    bool Tail = false;
 945    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 946         I = I->Next, Depth++)
 947    {
 948       // If pipelining is disabled, we only queue 1 request
 949       if (Server->Pipeline == false && Depth >= 0)
 950          break;
 951
 952       // Make sure we stick with the same server
 953       if (Server->Comp(I->Uri) == false)
 954          break;
 955       if (QueueBack == I)
 956          Tail = true;
 957       if (Tail == true)
 958       {
 959          QueueBack = I->Next;
 960          SendReq(I,Server->Out);
 961          continue;
 962       }
 963    }
 964
 965    return true;
 966 };
 967                                                                         /*}}}*/
 968 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 969 // ---------------------------------------------------------------------
 970 /* We stash the desired pipeline depth */
 971 bool HttpMethod::Configuration(string Message)
 972 {
 973    if (pkgAcqMethod::Configuration(Message) == false)
 974       return false;
 975
 976    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 977    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 978                                   PipelineDepth);
 979    Debug = _config->FindB("Debug::Acquire::http",false);
 980
 981    return true;
 982 }
 983                                                                         /*}}}*/
 984 // HttpMethod::Loop - Main loop                                         /*{{{*/
 985 // ---------------------------------------------------------------------
 986 /* */
 987 int HttpMethod::Loop()
 988 {
 989    signal(SIGTERM,SigTerm);
 990    signal(SIGINT,SigTerm);
 991
 992    Server = 0;
 993
 994    int FailCounter = 0;
 995    while (1)
 996    {
 997       // We have no commands, wait for some to arrive
 998       if (Queue == 0)
 999       {
1000          if (WaitFd(STDIN_FILENO) == false)
1001             return 0;
1002       }
1003
1004       /* Run messages, we can accept 0 (no message) if we didn't
1005          do a WaitFd above.. Otherwise the FD is closed. */
1006       int Result = Run(true);
1007       if (Result != -1 && (Result != 0 || Queue == 0))
1008          return 100;
1009
1010       if (Queue == 0)
1011          continue;
1012
1013       // Connect to the server
1014       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1015       {
1016          delete Server;
1017          Server = new ServerState(Queue->Uri,this);
1018       }
1019
1020       /* If the server has explicitly said this is the last connection
1021          then we pre-emptively shut down the pipeline and tear down
1022          the connection. This will speed up HTTP/1.0 servers a tad
1023          since we don't have to wait for the close sequence to
1024          complete */
1025       if (Server->Persistent == false)
1026          Server->Close();
1027
1028       // Reset the pipeline
1029       if (Server->ServerFd == -1)
1030          QueueBack = Queue;
1031
1032       // Connnect to the host
1033       if (Server->Open() == false)
1034       {
1035          Fail(true);
1036          delete Server;
1037          Server = 0;
1038          continue;
1039       }
1040
1041       // Fill the pipeline.
1042       Fetch(0);
1043
1044       // Fetch the next URL header data from the server.
1045       switch (Server->RunHeaders())
1046       {
1047          case 0:
1048          break;
1049
1050          // The header data is bad
1051          case 2:
1052          {
1053             _error->Error("Bad header Data");
1054             Fail(true);
1055             RotateDNS();
1056             continue;
1057          }
1058
1059          // The server closed a connection during the header get..
1060          default:
1061          case 1:
1062          {
1063             FailCounter++;
1064             _error->Discard();
1065             Server->Close();
1066             Server->Pipeline = false;
1067
1068             if (FailCounter >= 2)
1069             {
1070                Fail("Connection failed",true);
1071                FailCounter = 0;
1072             }
1073
1074             RotateDNS();
1075             continue;
1076          }
1077       };
1078
1079       // Decide what to do.
1080       FetchResult Res;
1081       Res.Filename = Queue->DestFile;
1082       switch (DealWithHeaders(Res,Server))
1083       {
1084          // Ok, the file is Open
1085          case 0:
1086          {
1087             URIStart(Res);
1088
1089             // Run the data
1090             bool Result =  Server->RunData();
1091
1092             /* If the server is sending back sizeless responses then fill in
1093                the size now */
1094             if (Res.Size == 0)
1095                Res.Size = File->Size();
1096
1097             // Close the file, destroy the FD object and timestamp it
1098             FailFd = -1;
1099             delete File;
1100             File = 0;
1101
1102             // Timestamp
1103             struct utimbuf UBuf;
1104             time(&UBuf.actime);
1105             UBuf.actime = Server->Date;
1106             UBuf.modtime = Server->Date;
1107             utime(Queue->DestFile.c_str(),&UBuf);
1108
1109             // Send status to APT
1110             if (Result == true)
1111             {
1112                Res.MD5Sum = Server->In.Hash->MD5.Result();
1113                URIDone(Res);
1114             }
1115             else
1116                Fail(true);
1117
1118             break;
1119          }
1120
1121          // IMS hit
1122          case 1:
1123          {
1124             URIDone(Res);
1125             break;
1126          }
1127
1128          // Hard server error, not found or something
1129          case 3:
1130          {
1131             Fail();
1132             break;
1133          }
1134
1135          // Hard internal error, kill the connection and fail
1136          case 5:
1137          {
1138             Fail();
1139             RotateDNS();
1140             Server->Close();
1141             break;
1142          }
1143
1144          // We need to flush the data, the header is like a 404 w/ error text
1145          case 4:
1146          {
1147             Fail();
1148
1149             // Send to content to dev/null
1150             File = new FileFd("/dev/null",FileFd::WriteExists);
1151             Server->RunData();
1152             delete File;
1153             File = 0;
1154             break;
1155          }
1156
1157          default:
1158          Fail("Internal error");
1159          break;
1160       }
1161
1162       FailCounter = 0;
1163    }
1164
1165    return 0;
1166 }
1167                                                                         /*}}}*/
1168
1169 int main()
1170 {
1171    HttpMethod Mth;
1172
1173    return Mth.Loop();
1174 }