methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.44 2000/01/30 08:16:43 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41 #include <errno.h>
  42
  43 // Internet stuff
  44 #include <netdb.h>
  45
  46 #include "connect.h"
  47 #include "rfc2553emu.h"
  48 #include "http.h"
  49
  50                                                                         /*}}}*/
  51
  52 string HttpMethod::FailFile;
  53 int HttpMethod::FailFd = -1;
  54 time_t HttpMethod::FailTime = 0;
  55 unsigned long PipelineDepth = 10;
  56 unsigned long TimeOut = 120;
  57 bool Debug = false;
  58
  59 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  60 // ---------------------------------------------------------------------
  61 /* */
  62 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  63 {
  64    Buf = new unsigned char[Size];
  65    Reset();
  66 }
  67                                                                         /*}}}*/
  68 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  69 // ---------------------------------------------------------------------
  70 /* */
  71 void CircleBuf::Reset()
  72 {
  73    InP = 0;
  74    OutP = 0;
  75    StrPos = 0;
  76    MaxGet = (unsigned int)-1;
  77    OutQueue = string();
  78    if (MD5 != 0)
  79    {
  80       delete MD5;
  81       MD5 = new MD5Summation;
  82    }
  83 };
  84                                                                         /*}}}*/
  85 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* This fills up the buffer with as much data as is in the FD, assuming it
  88    is non-blocking.. */
  89 bool CircleBuf::Read(int Fd)
  90 {
  91    while (1)
  92    {
  93       // Woops, buffer is full
  94       if (InP - OutP == Size)
  95          return true;
  96
  97       // Write the buffer segment
  98       int Res;
  99       Res = read(Fd,Buf + (InP%Size),LeftRead());
 100
 101       if (Res == 0)
 102          return false;
 103       if (Res < 0)
 104       {
 105          if (errno == EAGAIN)
 106             return true;
 107          return false;
 108       }
 109
 110       if (InP == 0)
 111          gettimeofday(&Start,0);
 112       InP += Res;
 113    }
 114 }
 115                                                                         /*}}}*/
 116 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 117 // ---------------------------------------------------------------------
 118 /* This will hold the string in and fill the buffer with it as it empties */
 119 bool CircleBuf::Read(string Data)
 120 {
 121    OutQueue += Data;
 122    FillOut();
 123    return true;
 124 }
 125                                                                         /*}}}*/
 126 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 127 // ---------------------------------------------------------------------
 128 /* */
 129 void CircleBuf::FillOut()
 130 {
 131    if (OutQueue.empty() == true)
 132       return;
 133    while (1)
 134    {
 135       // Woops, buffer is full
 136       if (InP - OutP == Size)
 137          return;
 138
 139       // Write the buffer segment
 140       unsigned long Sz = LeftRead();
 141       if (OutQueue.length() - StrPos < Sz)
 142          Sz = OutQueue.length() - StrPos;
 143       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 144
 145       // Advance
 146       StrPos += Sz;
 147       InP += Sz;
 148       if (OutQueue.length() == StrPos)
 149       {
 150          StrPos = 0;
 151          OutQueue = "";
 152          return;
 153       }
 154    }
 155 }
 156                                                                         /*}}}*/
 157 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 158 // ---------------------------------------------------------------------
 159 /* This empties the buffer into the FD. */
 160 bool CircleBuf::Write(int Fd)
 161 {
 162    while (1)
 163    {
 164       FillOut();
 165
 166       // Woops, buffer is empty
 167       if (OutP == InP)
 168          return true;
 169
 170       if (OutP == MaxGet)
 171          return true;
 172
 173       // Write the buffer segment
 174       int Res;
 175       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 176
 177       if (Res == 0)
 178          return false;
 179       if (Res < 0)
 180       {
 181          if (errno == EAGAIN)
 182             return true;
 183
 184          return false;
 185       }
 186
 187       if (MD5 != 0)
 188          MD5->Add(Buf + (OutP%Size),Res);
 189
 190       OutP += Res;
 191    }
 192 }
 193                                                                         /*}}}*/
 194 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 195 // ---------------------------------------------------------------------
 196 /* This copies till the first empty line */
 197 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 198 {
 199    // We cheat and assume it is unneeded to have more than one buffer load
 200    for (unsigned long I = OutP; I < InP; I++)
 201    {
 202       if (Buf[I%Size] != '\n')
 203          continue;
 204       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 205
 206       if (Single == false)
 207       {
 208          if (Buf[I%Size] != '\n')
 209             continue;
 210          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 211       }
 212
 213       if (I > InP)
 214          I = InP;
 215
 216       Data = "";
 217       while (OutP < I)
 218       {
 219          unsigned long Sz = LeftWrite();
 220          if (Sz == 0)
 221             return false;
 222          if (I - OutP < LeftWrite())
 223             Sz = I - OutP;
 224          Data += string((char *)(Buf + (OutP%Size)),Sz);
 225          OutP += Sz;
 226       }
 227       return true;
 228    }
 229    return false;
 230 }
 231                                                                         /*}}}*/
 232 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 233 // ---------------------------------------------------------------------
 234 /* */
 235 void CircleBuf::Stats()
 236 {
 237    if (InP == 0)
 238       return;
 239
 240    struct timeval Stop;
 241    gettimeofday(&Stop,0);
 242 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 243              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 244    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 245 }
 246                                                                         /*}}}*/
 247
 248 // ServerState::ServerState - Constructor                               /*{{{*/
 249 // ---------------------------------------------------------------------
 250 /* */
 251 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 252                         In(64*1024), Out(4*1024),
 253                         ServerName(Srv)
 254 {
 255    Reset();
 256 }
 257                                                                         /*}}}*/
 258 // ServerState::Open - Open a connection to the server                  /*{{{*/
 259 // ---------------------------------------------------------------------
 260 /* This opens a connection to the server. */
 261 string LastHost;
 262 int LastPort = 0;
 263 struct addrinfo *LastHostAddr = 0;
 264 bool ServerState::Open()
 265 {
 266    // Use the already open connection if possible.
 267    if (ServerFd != -1)
 268       return true;
 269
 270    Close();
 271    In.Reset();
 272    Out.Reset();
 273
 274    // Determine the proxy setting
 275    if (getenv("http_proxy") == 0)
 276    {
 277       string DefProxy = _config->Find("Acquire::http::Proxy");
 278       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 279       if (SpecificProxy.empty() == false)
 280       {
 281          if (SpecificProxy == "DIRECT")
 282             Proxy = "";
 283          else
 284             Proxy = SpecificProxy;
 285       }
 286       else
 287          Proxy = DefProxy;
 288    }
 289    else
 290       Proxy = getenv("http_proxy");
 291
 292    // Parse no_proxy, a , seperated list of hosts
 293    if (getenv("no_proxy") != 0)
 294    {
 295       const char *Start = getenv("no_proxy");
 296       for (const char *Cur = Start; true ; Cur++)
 297       {
 298          if (*Cur != ',' && *Cur != 0)
 299             continue;
 300          if (stringcasecmp(ServerName.Host.begin(),ServerName.Host.end(),
 301                            Start,Cur) == 0)
 302          {
 303             Proxy = "";
 304             break;
 305          }
 306
 307          Start = Cur + 1;
 308          if (*Cur == 0)
 309             break;
 310       }
 311    }
 312
 313    // Determine what host and port to use based on the proxy settings
 314    int Port = 0;
 315    string Host;
 316    if (Proxy.empty() == true)
 317    {
 318       if (ServerName.Port != 0)
 319          Port = ServerName.Port;
 320       Host = ServerName.Host;
 321    }
 322    else
 323    {
 324       if (Proxy.Port != 0)
 325          Port = Proxy.Port;
 326       Host = Proxy.Host;
 327    }
 328
 329    // Connect to the remote server
 330    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 331       return false;
 332
 333    return true;
 334 }
 335                                                                         /*}}}*/
 336 // ServerState::Close - Close a connection to the server                /*{{{*/
 337 // ---------------------------------------------------------------------
 338 /* */
 339 bool ServerState::Close()
 340 {
 341    close(ServerFd);
 342    ServerFd = -1;
 343    return true;
 344 }
 345                                                                         /*}}}*/
 346 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 347 // ---------------------------------------------------------------------
 348 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 349    parse error occured */
 350 int ServerState::RunHeaders()
 351 {
 352    State = Header;
 353
 354    Owner->Status("Waiting for file");
 355
 356    Major = 0;
 357    Minor = 0;
 358    Result = 0;
 359    Size = 0;
 360    StartPos = 0;
 361    Encoding = Closes;
 362    HaveContent = false;
 363    time(&Date);
 364
 365    do
 366    {
 367       string Data;
 368       if (In.WriteTillEl(Data) == false)
 369          continue;
 370
 371       if (Debug == true)
 372          clog << Data;
 373
 374       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 375       {
 376          string::const_iterator J = I;
 377          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 378          if (HeaderLine(string(I,J-I)) == false)
 379             return 2;
 380          I = J;
 381       }
 382       return 0;
 383    }
 384    while (Owner->Go(false,this) == true);
 385
 386    return 1;
 387 }
 388                                                                         /*}}}*/
 389 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 390 // ---------------------------------------------------------------------
 391 /* */
 392 bool ServerState::RunData()
 393 {
 394    State = Data;
 395
 396    // Chunked transfer encoding is fun..
 397    if (Encoding == Chunked)
 398    {
 399       while (1)
 400       {
 401          // Grab the block size
 402          bool Last = true;
 403          string Data;
 404          In.Limit(-1);
 405          do
 406          {
 407             if (In.WriteTillEl(Data,true) == true)
 408                break;
 409          }
 410          while ((Last = Owner->Go(false,this)) == true);
 411
 412          if (Last == false)
 413             return false;
 414
 415          // See if we are done
 416          unsigned long Len = strtol(Data.c_str(),0,16);
 417          if (Len == 0)
 418          {
 419             In.Limit(-1);
 420
 421             // We have to remove the entity trailer
 422             Last = true;
 423             do
 424             {
 425                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 426                   break;
 427             }
 428             while ((Last = Owner->Go(false,this)) == true);
 429             if (Last == false)
 430                return false;
 431             return !_error->PendingError();
 432          }
 433
 434          // Transfer the block
 435          In.Limit(Len);
 436          while (Owner->Go(true,this) == true)
 437             if (In.IsLimit() == true)
 438                break;
 439
 440          // Error
 441          if (In.IsLimit() == false)
 442             return false;
 443
 444          // The server sends an extra new line before the next block specifier..
 445          In.Limit(-1);
 446          Last = true;
 447          do
 448          {
 449             if (In.WriteTillEl(Data,true) == true)
 450                break;
 451          }
 452          while ((Last = Owner->Go(false,this)) == true);
 453          if (Last == false)
 454             return false;
 455       }
 456    }
 457    else
 458    {
 459       /* Closes encoding is used when the server did not specify a size, the
 460          loss of the connection means we are done */
 461       if (Encoding == Closes)
 462          In.Limit(-1);
 463       else
 464          In.Limit(Size - StartPos);
 465
 466       // Just transfer the whole block.
 467       do
 468       {
 469          if (In.IsLimit() == false)
 470             continue;
 471
 472          In.Limit(-1);
 473          return !_error->PendingError();
 474       }
 475       while (Owner->Go(true,this) == true);
 476    }
 477
 478    return Owner->Flush(this) && !_error->PendingError();
 479 }
 480                                                                         /*}}}*/
 481 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 482 // ---------------------------------------------------------------------
 483 /* */
 484 bool ServerState::HeaderLine(string Line)
 485 {
 486    if (Line.empty() == true)
 487       return true;
 488
 489    // The http server might be trying to do something evil.
 490    if (Line.length() >= MAXLEN)
 491       return _error->Error("Got a single header line over %u chars",MAXLEN);
 492
 493    string::size_type Pos = Line.find(' ');
 494    if (Pos == string::npos || Pos+1 > Line.length())
 495    {
 496       // Blah, some servers use "connection:closes", evil.
 497       Pos = Line.find(':');
 498       if (Pos == string::npos || Pos + 2 > Line.length())
 499          return _error->Error("Bad header line");
 500       Pos++;
 501    }
 502
 503    // Parse off any trailing spaces between the : and the next word.
 504    string::size_type Pos2 = Pos;
 505    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 506       Pos2++;
 507
 508    string Tag = string(Line,0,Pos);
 509    string Val = string(Line,Pos2);
 510
 511    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 512    {
 513       // Evil servers return no version
 514       if (Line[4] == '/')
 515       {
 516          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 517                     &Result,Code) != 4)
 518             return _error->Error("The http server sent an invalid reply header");
 519       }
 520       else
 521       {
 522          Major = 0;
 523          Minor = 9;
 524          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 525             return _error->Error("The http server sent an invalid reply header");
 526       }
 527
 528       return true;
 529    }
 530
 531    if (stringcasecmp(Tag,"Content-Length:") == 0)
 532    {
 533       if (Encoding == Closes)
 534          Encoding = Stream;
 535       HaveContent = true;
 536
 537       // The length is already set from the Content-Range header
 538       if (StartPos != 0)
 539          return true;
 540
 541       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 542          return _error->Error("The http server sent an invalid Content-Length header");
 543       return true;
 544    }
 545
 546    if (stringcasecmp(Tag,"Content-Type:") == 0)
 547    {
 548       HaveContent = true;
 549       return true;
 550    }
 551
 552    if (stringcasecmp(Tag,"Content-Range:") == 0)
 553    {
 554       HaveContent = true;
 555
 556       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 557          return _error->Error("The http server sent an invalid Content-Range header");
 558       if ((unsigned)StartPos > Size)
 559          return _error->Error("This http server has broken range support");
 560       return true;
 561    }
 562
 563    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 564    {
 565       HaveContent = true;
 566       if (stringcasecmp(Val,"chunked") == 0)
 567          Encoding = Chunked;
 568
 569       return true;
 570    }
 571
 572    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 573    {
 574       if (StrToTime(Val,Date) == false)
 575          return _error->Error("Unknown date format");
 576       return true;
 577    }
 578
 579    return true;
 580 }
 581                                                                         /*}}}*/
 582
 583 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 584 // ---------------------------------------------------------------------
 585 /* This places the http request in the outbound buffer */
 586 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 587 {
 588    URI Uri = Itm->Uri;
 589
 590    // The HTTP server expects a hostname with a trailing :port
 591    char Buf[1000];
 592    string ProperHost = Uri.Host;
 593    if (Uri.Port != 0)
 594    {
 595       sprintf(Buf,":%u",Uri.Port);
 596       ProperHost += Buf;
 597    }
 598
 599    // Just in case.
 600    if (Itm->Uri.length() >= sizeof(Buf))
 601        abort();
 602
 603    /* Build the request. We include a keep-alive header only for non-proxy
 604       requests. This is to tweak old http/1.0 servers that do support keep-alive
 605       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 606       will glitch HTTP/1.0 proxies because they do not filter it out and
 607       pass it on, HTTP/1.1 says the connection should default to keep alive
 608       and we expect the proxy to do this */
 609    if (Proxy.empty() == true)
 610       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 611               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 612    else
 613    {
 614       /* Generate a cache control header if necessary. We place a max
 615          cache age on index files, optionally set a no-cache directive
 616          and a no-store directive for archives. */
 617       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 618               Itm->Uri.c_str(),ProperHost.c_str());
 619       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 620          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 621       else
 622       {
 623          if (Itm->IndexFile == true)
 624             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 625                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 626          else
 627          {
 628             if (_config->FindB("Acquire::http::No-Store",false) == true)
 629                strcat(Buf,"Cache-Control: no-store\r\n");
 630          }
 631       }
 632    }
 633
 634    string Req = Buf;
 635
 636    // Check for a partial file
 637    struct stat SBuf;
 638    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 639    {
 640       // In this case we send an if-range query with a range header
 641       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 642               TimeRFC1123(SBuf.st_mtime).c_str());
 643       Req += Buf;
 644    }
 645    else
 646    {
 647       if (Itm->LastModified != 0)
 648       {
 649          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 650          Req += Buf;
 651       }
 652    }
 653
 654    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 655       Req += string("Proxy-Authorization: Basic ") +
 656           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 657
 658    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 659
 660    if (Debug == true)
 661       cerr << Req << endl;
 662
 663    Out.Read(Req);
 664 }
 665                                                                         /*}}}*/
 666 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 667 // ---------------------------------------------------------------------
 668 /* This runs the select loop over the server FDs, Output file FDs and
 669    stdin. */
 670 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 671 {
 672    // Server has closed the connection
 673    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 674                                ToFile == false))
 675       return false;
 676
 677    fd_set rfds,wfds;
 678    FD_ZERO(&rfds);
 679    FD_ZERO(&wfds);
 680
 681    // Add the server
 682    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 683       FD_SET(Srv->ServerFd,&wfds);
 684    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 685       FD_SET(Srv->ServerFd,&rfds);
 686
 687    // Add the file
 688    int FileFD = -1;
 689    if (File != 0)
 690       FileFD = File->Fd();
 691
 692    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 693       FD_SET(FileFD,&wfds);
 694
 695    // Add stdin
 696    FD_SET(STDIN_FILENO,&rfds);
 697
 698    // Figure out the max fd
 699    int MaxFd = FileFD;
 700    if (MaxFd < Srv->ServerFd)
 701       MaxFd = Srv->ServerFd;
 702
 703    // Select
 704    struct timeval tv;
 705    tv.tv_sec = TimeOut;
 706    tv.tv_usec = 0;
 707    int Res = 0;
 708    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 709       return _error->Errno("select","Select failed");
 710
 711    if (Res == 0)
 712    {
 713       _error->Error("Connection timed out");
 714       return ServerDie(Srv);
 715    }
 716
 717    // Handle server IO
 718    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 719    {
 720       errno = 0;
 721       if (Srv->In.Read(Srv->ServerFd) == false)
 722          return ServerDie(Srv);
 723    }
 724
 725    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 726    {
 727       errno = 0;
 728       if (Srv->Out.Write(Srv->ServerFd) == false)
 729          return ServerDie(Srv);
 730    }
 731
 732    // Send data to the file
 733    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 734    {
 735       if (Srv->In.Write(FileFD) == false)
 736          return _error->Errno("write","Error writing to output file");
 737    }
 738
 739    // Handle commands from APT
 740    if (FD_ISSET(STDIN_FILENO,&rfds))
 741    {
 742       if (Run(true) != -1)
 743          exit(100);
 744    }
 745
 746    return true;
 747 }
 748                                                                         /*}}}*/
 749 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 750 // ---------------------------------------------------------------------
 751 /* This takes the current input buffer from the Server FD and writes it
 752    into the file */
 753 bool HttpMethod::Flush(ServerState *Srv)
 754 {
 755    if (File != 0)
 756    {
 757       SetNonBlock(File->Fd(),false);
 758       if (Srv->In.WriteSpace() == false)
 759          return true;
 760
 761       while (Srv->In.WriteSpace() == true)
 762       {
 763          if (Srv->In.Write(File->Fd()) == false)
 764             return _error->Errno("write","Error writing to file");
 765          if (Srv->In.IsLimit() == true)
 766             return true;
 767       }
 768
 769       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 770          return true;
 771    }
 772    return false;
 773 }
 774                                                                         /*}}}*/
 775 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 776 // ---------------------------------------------------------------------
 777 /* */
 778 bool HttpMethod::ServerDie(ServerState *Srv)
 779 {
 780    unsigned int LErrno = errno;
 781
 782    // Dump the buffer to the file
 783    if (Srv->State == ServerState::Data)
 784    {
 785       SetNonBlock(File->Fd(),false);
 786       while (Srv->In.WriteSpace() == true)
 787       {
 788          if (Srv->In.Write(File->Fd()) == false)
 789             return _error->Errno("write","Error writing to the file");
 790
 791          // Done
 792          if (Srv->In.IsLimit() == true)
 793             return true;
 794       }
 795    }
 796
 797    // See if this is because the server finished the data stream
 798    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 799        Srv->Encoding != ServerState::Closes)
 800    {
 801       Srv->Close();
 802       if (LErrno == 0)
 803          return _error->Error("Error reading from server Remote end closed connection");
 804       errno = LErrno;
 805       return _error->Errno("read","Error reading from server");
 806    }
 807    else
 808    {
 809       Srv->In.Limit(-1);
 810
 811       // Nothing left in the buffer
 812       if (Srv->In.WriteSpace() == false)
 813          return false;
 814
 815       // We may have got multiple responses back in one packet..
 816       Srv->Close();
 817       return true;
 818    }
 819
 820    return false;
 821 }
 822                                                                         /*}}}*/
 823 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 824 // ---------------------------------------------------------------------
 825 /* We look at the header data we got back from the server and decide what
 826    to do. Returns
 827      0 - File is open,
 828      1 - IMS hit
 829      3 - Unrecoverable error
 830      4 - Error with error content page
 831      5 - Unrecoverable non-server error (close the connection) */
 832 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 833 {
 834    // Not Modified
 835    if (Srv->Result == 304)
 836    {
 837       unlink(Queue->DestFile.c_str());
 838       Res.IMSHit = true;
 839       Res.LastModified = Queue->LastModified;
 840       return 1;
 841    }
 842
 843    /* We have a reply we dont handle. This should indicate a perm server
 844       failure */
 845    if (Srv->Result < 200 || Srv->Result >= 300)
 846    {
 847       _error->Error("%u %s",Srv->Result,Srv->Code);
 848       if (Srv->HaveContent == true)
 849          return 4;
 850       return 3;
 851    }
 852
 853    // This is some sort of 2xx 'data follows' reply
 854    Res.LastModified = Srv->Date;
 855    Res.Size = Srv->Size;
 856
 857    // Open the file
 858    delete File;
 859    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 860    if (_error->PendingError() == true)
 861       return 5;
 862
 863    FailFile = Queue->DestFile;
 864    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 865    FailFd = File->Fd();
 866    FailTime = Srv->Date;
 867
 868    // Set the expected size
 869    if (Srv->StartPos >= 0)
 870    {
 871       Res.ResumePoint = Srv->StartPos;
 872       ftruncate(File->Fd(),Srv->StartPos);
 873    }
 874
 875    // Set the start point
 876    lseek(File->Fd(),0,SEEK_END);
 877
 878    delete Srv->In.MD5;
 879    Srv->In.MD5 = new MD5Summation;
 880
 881    // Fill the MD5 Hash if the file is non-empty (resume)
 882    if (Srv->StartPos > 0)
 883    {
 884       lseek(File->Fd(),0,SEEK_SET);
 885       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 886       {
 887          _error->Errno("read","Problem hashing file");
 888          return 5;
 889       }
 890       lseek(File->Fd(),0,SEEK_END);
 891    }
 892
 893    SetNonBlock(File->Fd(),true);
 894    return 0;
 895 }
 896                                                                         /*}}}*/
 897 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 898 // ---------------------------------------------------------------------
 899 /* This closes and timestamps the open file. This is neccessary to get
 900    resume behavoir on user abort */
 901 void HttpMethod::SigTerm(int)
 902 {
 903    if (FailFd == -1)
 904       _exit(100);
 905    close(FailFd);
 906
 907    // Timestamp
 908    struct utimbuf UBuf;
 909    UBuf.actime = FailTime;
 910    UBuf.modtime = FailTime;
 911    utime(FailFile.c_str(),&UBuf);
 912
 913    _exit(100);
 914 }
 915                                                                         /*}}}*/
 916 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 917 // ---------------------------------------------------------------------
 918 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 919    depth. */
 920 bool HttpMethod::Fetch(FetchItem *)
 921 {
 922    if (Server == 0)
 923       return true;
 924
 925    // Queue the requests
 926    int Depth = -1;
 927    bool Tail = false;
 928    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 929         I = I->Next, Depth++)
 930    {
 931       // If pipelining is disabled, we only queue 1 request
 932       if (Server->Pipeline == false && Depth >= 0)
 933          break;
 934
 935       // Make sure we stick with the same server
 936       if (Server->Comp(I->Uri) == false)
 937          break;
 938       if (QueueBack == I)
 939          Tail = true;
 940       if (Tail == true)
 941       {
 942          QueueBack = I->Next;
 943          SendReq(I,Server->Out);
 944          continue;
 945       }
 946    }
 947
 948    return true;
 949 };
 950                                                                         /*}}}*/
 951 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 952 // ---------------------------------------------------------------------
 953 /* We stash the desired pipeline depth */
 954 bool HttpMethod::Configuration(string Message)
 955 {
 956    if (pkgAcqMethod::Configuration(Message) == false)
 957       return false;
 958
 959    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 960    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 961                                   PipelineDepth);
 962    Debug = _config->FindB("Debug::Acquire::http",false);
 963
 964    return true;
 965 }
 966                                                                         /*}}}*/
 967 // HttpMethod::Loop - Main loop                                         /*{{{*/
 968 // ---------------------------------------------------------------------
 969 /* */
 970 int HttpMethod::Loop()
 971 {
 972    signal(SIGTERM,SigTerm);
 973    signal(SIGINT,SigTerm);
 974
 975    Server = 0;
 976
 977    int FailCounter = 0;
 978    while (1)
 979    {
 980       // We have no commands, wait for some to arrive
 981       if (Queue == 0)
 982       {
 983          if (WaitFd(STDIN_FILENO) == false)
 984             return 0;
 985       }
 986
 987       /* Run messages, we can accept 0 (no message) if we didn't
 988          do a WaitFd above.. Otherwise the FD is closed. */
 989       int Result = Run(true);
 990       if (Result != -1 && (Result != 0 || Queue == 0))
 991          return 100;
 992
 993       if (Queue == 0)
 994          continue;
 995
 996       // Connect to the server
 997       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 998       {
 999          delete Server;
1000          Server = new ServerState(Queue->Uri,this);
1001       }
1002
1003       // Reset the pipeline
1004       if (Server->ServerFd == -1)
1005          QueueBack = Queue;
1006
1007       // Connnect to the host
1008       if (Server->Open() == false)
1009       {
1010          Fail(true);
1011          delete Server;
1012          Server = 0;
1013          continue;
1014       }
1015
1016       // Fill the pipeline.
1017       Fetch(0);
1018
1019       // Fetch the next URL header data from the server.
1020       switch (Server->RunHeaders())
1021       {
1022          case 0:
1023          break;
1024
1025          // The header data is bad
1026          case 2:
1027          {
1028             _error->Error("Bad header Data");
1029             Fail(true);
1030             continue;
1031          }
1032
1033          // The server closed a connection during the header get..
1034          default:
1035          case 1:
1036          {
1037             FailCounter++;
1038             _error->Discard();
1039             Server->Close();
1040             Server->Pipeline = false;
1041
1042             if (FailCounter >= 2)
1043             {
1044                Fail("Connection failed",true);
1045                FailCounter = 0;
1046             }
1047
1048             continue;
1049          }
1050       };
1051
1052       // Decide what to do.
1053       FetchResult Res;
1054       Res.Filename = Queue->DestFile;
1055       switch (DealWithHeaders(Res,Server))
1056       {
1057          // Ok, the file is Open
1058          case 0:
1059          {
1060             URIStart(Res);
1061
1062             // Run the data
1063             bool Result =  Server->RunData();
1064
1065             // Close the file, destroy the FD object and timestamp it
1066             FailFd = -1;
1067             delete File;
1068             File = 0;
1069
1070             // Timestamp
1071             struct utimbuf UBuf;
1072             time(&UBuf.actime);
1073             UBuf.actime = Server->Date;
1074             UBuf.modtime = Server->Date;
1075             utime(Queue->DestFile.c_str(),&UBuf);
1076
1077             // Send status to APT
1078             if (Result == true)
1079             {
1080                Res.MD5Sum = Server->In.MD5->Result();
1081                URIDone(Res);
1082             }
1083             else
1084                Fail(true);
1085
1086             break;
1087          }
1088
1089          // IMS hit
1090          case 1:
1091          {
1092             URIDone(Res);
1093             break;
1094          }
1095
1096          // Hard server error, not found or something
1097          case 3:
1098          {
1099             Fail();
1100             break;
1101          }
1102
1103          // Hard internal error, kill the connection and fail
1104          case 5:
1105          {
1106             Fail();
1107             Server->Close();
1108             break;
1109          }
1110
1111          // We need to flush the data, the header is like a 404 w/ error text
1112          case 4:
1113          {
1114             Fail();
1115
1116             // Send to content to dev/null
1117             File = new FileFd("/dev/null",FileFd::WriteExists);
1118             Server->RunData();
1119             delete File;
1120             File = 0;
1121             break;
1122          }
1123
1124          default:
1125          Fail("Internal error");
1126          break;
1127       }
1128
1129       FailCounter = 0;
1130    }
1131
1132    return 0;
1133 }
1134                                                                         /*}}}*/
1135
1136 int main()
1137 {
1138    HttpMethod Mth;
1139
1140    return Mth.Loop();
1141 }