methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.26 1999/02/15 00:26:55 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41 #include <errno.h>
  42
  43 // Internet stuff
  44 #include <netinet/in.h>
  45 #include <sys/socket.h>
  46 #include <arpa/inet.h>
  47 #include <netdb.h>
  48
  49 #include "http.h"
  50                                                                         /*}}}*/
  51
  52 string HttpMethod::FailFile;
  53 int HttpMethod::FailFd = -1;
  54 time_t HttpMethod::FailTime = 0;
  55 unsigned long PipelineDepth = 10;
  56 unsigned long TimeOut = 120;
  57
  58 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  59 // ---------------------------------------------------------------------
  60 /* */
  61 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  62 {
  63    Buf = new unsigned char[Size];
  64    Reset();
  65 }
  66                                                                         /*}}}*/
  67 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  68 // ---------------------------------------------------------------------
  69 /* */
  70 void CircleBuf::Reset()
  71 {
  72    InP = 0;
  73    OutP = 0;
  74    StrPos = 0;
  75    MaxGet = (unsigned int)-1;
  76    OutQueue = string();
  77    if (MD5 != 0)
  78    {
  79       delete MD5;
  80       MD5 = new MD5Summation;
  81    }
  82 };
  83                                                                         /*}}}*/
  84 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  85 // ---------------------------------------------------------------------
  86 /* This fills up the buffer with as much data as is in the FD, assuming it
  87    is non-blocking.. */
  88 bool CircleBuf::Read(int Fd)
  89 {
  90    while (1)
  91    {
  92       // Woops, buffer is full
  93       if (InP - OutP == Size)
  94          return true;
  95
  96       // Write the buffer segment
  97       int Res;
  98       Res = read(Fd,Buf + (InP%Size),LeftRead());
  99
 100       if (Res == 0)
 101          return false;
 102       if (Res < 0)
 103       {
 104          if (errno == EAGAIN)
 105             return true;
 106          return false;
 107       }
 108
 109       if (InP == 0)
 110          gettimeofday(&Start,0);
 111       InP += Res;
 112    }
 113 }
 114                                                                         /*}}}*/
 115 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 116 // ---------------------------------------------------------------------
 117 /* This will hold the string in and fill the buffer with it as it empties */
 118 bool CircleBuf::Read(string Data)
 119 {
 120    OutQueue += Data;
 121    FillOut();
 122    return true;
 123 }
 124                                                                         /*}}}*/
 125 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* */
 128 void CircleBuf::FillOut()
 129 {
 130    if (OutQueue.empty() == true)
 131       return;
 132    while (1)
 133    {
 134       // Woops, buffer is full
 135       if (InP - OutP == Size)
 136          return;
 137
 138       // Write the buffer segment
 139       unsigned long Sz = LeftRead();
 140       if (OutQueue.length() - StrPos < Sz)
 141          Sz = OutQueue.length() - StrPos;
 142       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 143
 144       // Advance
 145       StrPos += Sz;
 146       InP += Sz;
 147       if (OutQueue.length() == StrPos)
 148       {
 149          StrPos = 0;
 150          OutQueue = "";
 151          return;
 152       }
 153    }
 154 }
 155                                                                         /*}}}*/
 156 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 157 // ---------------------------------------------------------------------
 158 /* This empties the buffer into the FD. */
 159 bool CircleBuf::Write(int Fd)
 160 {
 161    while (1)
 162    {
 163       FillOut();
 164
 165       // Woops, buffer is empty
 166       if (OutP == InP)
 167          return true;
 168
 169       if (OutP == MaxGet)
 170          return true;
 171
 172       // Write the buffer segment
 173       int Res;
 174       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 175
 176       if (Res == 0)
 177          return false;
 178       if (Res < 0)
 179       {
 180          if (errno == EAGAIN)
 181             return true;
 182
 183          return false;
 184       }
 185
 186       if (MD5 != 0)
 187          MD5->Add(Buf + (OutP%Size),Res);
 188
 189       OutP += Res;
 190    }
 191 }
 192                                                                         /*}}}*/
 193 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 194 // ---------------------------------------------------------------------
 195 /* This copies till the first empty line */
 196 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 197 {
 198    // We cheat and assume it is unneeded to have more than one buffer load
 199    for (unsigned long I = OutP; I < InP; I++)
 200    {
 201       if (Buf[I%Size] != '\n')
 202          continue;
 203       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 204
 205       if (Single == false)
 206       {
 207          if (Buf[I%Size] != '\n')
 208             continue;
 209          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 210       }
 211
 212       if (I > InP)
 213          I = InP;
 214
 215       Data = "";
 216       while (OutP < I)
 217       {
 218          unsigned long Sz = LeftWrite();
 219          if (Sz == 0)
 220             return false;
 221          if (I - OutP < LeftWrite())
 222             Sz = I - OutP;
 223          Data += string((char *)(Buf + (OutP%Size)),Sz);
 224          OutP += Sz;
 225       }
 226       return true;
 227    }
 228    return false;
 229 }
 230                                                                         /*}}}*/
 231 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 232 // ---------------------------------------------------------------------
 233 /* */
 234 void CircleBuf::Stats()
 235 {
 236    if (InP == 0)
 237       return;
 238
 239    struct timeval Stop;
 240    gettimeofday(&Stop,0);
 241 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 242              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 243    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 244 }
 245                                                                         /*}}}*/
 246
 247 // ServerState::ServerState - Constructor                               /*{{{*/
 248 // ---------------------------------------------------------------------
 249 /* */
 250 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 251                         In(64*1024), Out(4*1024),
 252                         ServerName(Srv)
 253 {
 254    Reset();
 255 }
 256                                                                         /*}}}*/
 257 // ServerState::Open - Open a connection to the server                  /*{{{*/
 258 // ---------------------------------------------------------------------
 259 /* This opens a connection to the server. */
 260 string LastHost;
 261 in_addr LastHostA;
 262 bool ServerState::Open()
 263 {
 264    // Use the already open connection if possible.
 265    if (ServerFd != -1)
 266       return true;
 267
 268    Close();
 269    In.Reset();
 270    Out.Reset();
 271
 272    // Determine the proxy setting
 273    if (getenv("http_proxy") == 0)
 274    {
 275       string DefProxy = _config->Find("Acquire::http::Proxy");
 276       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 277       if (SpecificProxy.empty() == false)
 278       {
 279          if (SpecificProxy == "DIRECT")
 280             Proxy = "";
 281          else
 282             Proxy = SpecificProxy;
 283       }
 284       else
 285          Proxy = DefProxy;
 286    }
 287    else
 288       Proxy = getenv("http_proxy");
 289
 290    // Determine what host and port to use based on the proxy settings
 291    int Port = 80;
 292    string Host;
 293    if (Proxy.empty() == true)
 294    {
 295       if (ServerName.Port != 0)
 296          Port = ServerName.Port;
 297       Host = ServerName.Host;
 298    }
 299    else
 300    {
 301       if (Proxy.Port != 0)
 302          Port = Proxy.Port;
 303       Host = Proxy.Host;
 304    }
 305
 306    /* We used a cached address record.. Yes this is against the spec but
 307       the way we have setup our rotating dns suggests that this is more
 308       sensible */
 309    if (LastHost != Host)
 310    {
 311       Owner->Status("Connecting to %s",Host.c_str());
 312
 313       // Lookup the host
 314       hostent *Addr = gethostbyname(Host.c_str());
 315       if (Addr == 0 || Addr->h_addr_list[0] == 0)
 316          return _error->Error("Could not resolve '%s'",Host.c_str());
 317       LastHost = Host;
 318       LastHostA = *(in_addr *)(Addr->h_addr_list[0]);
 319    }
 320
 321    Owner->Status("Connecting to %s (%s)",Host.c_str(),inet_ntoa(LastHostA));
 322
 323    // Get a socket
 324    if ((ServerFd = socket(AF_INET,SOCK_STREAM,0)) < 0)
 325       return _error->Errno("socket","Could not create a socket");
 326
 327    // Connect to the server
 328    struct sockaddr_in server;
 329    server.sin_family = AF_INET;
 330    server.sin_port = htons(Port);
 331    server.sin_addr = LastHostA;
 332    SetNonBlock(ServerFd,true);
 333    if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0 &&
 334        errno != EINPROGRESS)
 335       return _error->Errno("socket","Could not create a socket");
 336
 337    /* This implements a timeout for connect by opening the connection
 338       nonblocking */
 339    fd_set wfds;
 340    FD_ZERO(&wfds);
 341    FD_SET(ServerFd,&wfds);
 342    struct timeval tv;
 343    tv.tv_sec = TimeOut;
 344    tv.tv_usec = 0;
 345    int Res = 0;
 346    if ((Res = select(ServerFd+1,0,&wfds,0,&tv)) < 0)
 347       return _error->Errno("select","Select failed");
 348    if (Res == 0)
 349       return _error->Error("Could not connect, connection timed out");
 350    unsigned int Err,Len=sizeof(Err);
 351    if (getsockopt(ServerFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 352       return _error->Errno("getsockopt","Failed");
 353    if (Err != 0)
 354       return _error->Error("Could not connect.");
 355
 356    return true;
 357 }
 358                                                                         /*}}}*/
 359 // ServerState::Close - Close a connection to the server                /*{{{*/
 360 // ---------------------------------------------------------------------
 361 /* */
 362 bool ServerState::Close()
 363 {
 364    close(ServerFd);
 365    ServerFd = -1;
 366    return true;
 367 }
 368                                                                         /*}}}*/
 369 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 370 // ---------------------------------------------------------------------
 371 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 372    parse error occured */
 373 int ServerState::RunHeaders()
 374 {
 375    State = Header;
 376
 377    Owner->Status("Waiting for file");
 378
 379    Major = 0;
 380    Minor = 0;
 381    Result = 0;
 382    Size = 0;
 383    StartPos = 0;
 384    Encoding = Closes;
 385    HaveContent = false;
 386    time(&Date);
 387
 388    do
 389    {
 390       string Data;
 391       if (In.WriteTillEl(Data) == false)
 392          continue;
 393
 394       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 395       {
 396          string::const_iterator J = I;
 397          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 398          if (HeaderLine(string(I,J-I)) == false)
 399             return 2;
 400          I = J;
 401       }
 402       return 0;
 403    }
 404    while (Owner->Go(false,this) == true);
 405
 406    return 1;
 407 }
 408                                                                         /*}}}*/
 409 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 410 // ---------------------------------------------------------------------
 411 /* */
 412 bool ServerState::RunData()
 413 {
 414    State = Data;
 415
 416    // Chunked transfer encoding is fun..
 417    if (Encoding == Chunked)
 418    {
 419       while (1)
 420       {
 421          // Grab the block size
 422          bool Last = true;
 423          string Data;
 424          In.Limit(-1);
 425          do
 426          {
 427             if (In.WriteTillEl(Data,true) == true)
 428                break;
 429          }
 430          while ((Last = Owner->Go(false,this)) == true);
 431
 432          if (Last == false)
 433             return false;
 434
 435          // See if we are done
 436          unsigned long Len = strtol(Data.c_str(),0,16);
 437          if (Len == 0)
 438          {
 439             In.Limit(-1);
 440
 441             // We have to remove the entity trailer
 442             Last = true;
 443             do
 444             {
 445                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 446                   break;
 447             }
 448             while ((Last = Owner->Go(false,this)) == true);
 449             if (Last == false)
 450                return false;
 451             return !_error->PendingError();
 452          }
 453
 454          // Transfer the block
 455          In.Limit(Len);
 456          while (Owner->Go(true,this) == true)
 457             if (In.IsLimit() == true)
 458                break;
 459
 460          // Error
 461          if (In.IsLimit() == false)
 462             return false;
 463
 464          // The server sends an extra new line before the next block specifier..
 465          In.Limit(-1);
 466          Last = true;
 467          do
 468          {
 469             if (In.WriteTillEl(Data,true) == true)
 470                break;
 471          }
 472          while ((Last = Owner->Go(false,this)) == true);
 473          if (Last == false)
 474             return false;
 475       }
 476    }
 477    else
 478    {
 479       /* Closes encoding is used when the server did not specify a size, the
 480          loss of the connection means we are done */
 481       if (Encoding == Closes)
 482          In.Limit(-1);
 483       else
 484          In.Limit(Size - StartPos);
 485
 486       // Just transfer the whole block.
 487       do
 488       {
 489          if (In.IsLimit() == false)
 490             continue;
 491
 492          In.Limit(-1);
 493          return !_error->PendingError();
 494       }
 495       while (Owner->Go(true,this) == true);
 496    }
 497
 498    return Owner->Flush(this) && !_error->PendingError();
 499 }
 500                                                                         /*}}}*/
 501 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 502 // ---------------------------------------------------------------------
 503 /* */
 504 bool ServerState::HeaderLine(string Line)
 505 {
 506    if (Line.empty() == true)
 507       return true;
 508
 509    // The http server might be trying to do something evil.
 510    if (Line.length() >= MAXLEN)
 511       return _error->Error("Got a single header line over %u chars",MAXLEN);
 512
 513    string::size_type Pos = Line.find(' ');
 514    if (Pos == string::npos || Pos+1 > Line.length())
 515       return _error->Error("Bad header line");
 516
 517    string Tag = string(Line,0,Pos);
 518    string Val = string(Line,Pos+1);
 519
 520    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 521    {
 522       // Evil servers return no version
 523       if (Line[4] == '/')
 524       {
 525          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 526                     &Result,Code) != 4)
 527             return _error->Error("The http server sent an invalid reply header");
 528       }
 529       else
 530       {
 531          Major = 0;
 532          Minor = 9;
 533          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 534             return _error->Error("The http server sent an invalid reply header");
 535       }
 536
 537       return true;
 538    }
 539
 540    if (stringcasecmp(Tag,"Content-Length:") == 0)
 541    {
 542       if (Encoding == Closes)
 543          Encoding = Stream;
 544       HaveContent = true;
 545
 546       // The length is already set from the Content-Range header
 547       if (StartPos != 0)
 548          return true;
 549
 550       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 551          return _error->Error("The http server sent an invalid Content-Length header");
 552       return true;
 553    }
 554
 555    if (stringcasecmp(Tag,"Content-Type:") == 0)
 556    {
 557       HaveContent = true;
 558       return true;
 559    }
 560
 561    if (stringcasecmp(Tag,"Content-Range:") == 0)
 562    {
 563       HaveContent = true;
 564
 565       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 566          return _error->Error("The http server sent an invalid Content-Range header");
 567       if ((unsigned)StartPos > Size)
 568          return _error->Error("This http server has broken range support");
 569       return true;
 570    }
 571
 572    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 573    {
 574       HaveContent = true;
 575       if (stringcasecmp(Val,"chunked") == 0)
 576          Encoding = Chunked;
 577
 578       return true;
 579    }
 580
 581    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 582    {
 583       if (StrToTime(Val,Date) == false)
 584          return _error->Error("Unknown date format");
 585       return true;
 586    }
 587
 588    return true;
 589 }
 590                                                                         /*}}}*/
 591
 592 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 593 // ---------------------------------------------------------------------
 594 /* This places the http request in the outbound buffer */
 595 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 596 {
 597    URI Uri = Itm->Uri;
 598
 599    // The HTTP server expects a hostname with a trailing :port
 600    char Buf[1000];
 601    string ProperHost = Uri.Host;
 602    if (Uri.Port != 0)
 603    {
 604       sprintf(Buf,":%u",Uri.Port);
 605       ProperHost += Buf;
 606    }
 607
 608    // Just in case.
 609    if (Itm->Uri.length() >= sizeof(Buf))
 610        abort();
 611
 612    /* Build the request. We include a keep-alive header only for non-proxy
 613       requests. This is to tweak old http/1.0 servers that do support keep-alive
 614       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 615       will glitch HTTP/1.0 proxies because they do not filter it out and
 616       pass it on, HTTP/1.1 says the connection should default to keep alive
 617       and we expect the proxy to do this */
 618    if (Proxy.empty() == true)
 619       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 620               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 621    else
 622    {
 623       /* Generate a cache control header if necessary. We place a max
 624          cache age on index files, optionally set a no-cache directive
 625          and a no-store directive for archives. */
 626       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 627               Itm->Uri.c_str(),ProperHost.c_str());
 628       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 629          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 630       else
 631       {
 632          if (Itm->IndexFile == true)
 633             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 634                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 635          else
 636          {
 637             if (_config->FindB("Acquire::http::No-Store",false) == true)
 638                strcat(Buf,"Cache-Control: no-store\r\n");
 639          }
 640       }
 641    }
 642
 643    string Req = Buf;
 644
 645    // Check for a partial file
 646    struct stat SBuf;
 647    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 648    {
 649       // In this case we send an if-range query with a range header
 650       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 651               TimeRFC1123(SBuf.st_mtime).c_str());
 652       Req += Buf;
 653    }
 654    else
 655    {
 656       if (Itm->LastModified != 0)
 657       {
 658          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 659          Req += Buf;
 660       }
 661    }
 662
 663    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 664       Req += string("Proxy-Authorization: Basic ") +
 665           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 666
 667    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 668 //   cerr << Req << endl;
 669
 670    Out.Read(Req);
 671 }
 672                                                                         /*}}}*/
 673 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 674 // ---------------------------------------------------------------------
 675 /* This runs the select loop over the server FDs, Output file FDs and
 676    stdin. */
 677 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 678 {
 679    // Server has closed the connection
 680    if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
 681       return false;
 682
 683    fd_set rfds,wfds,efds;
 684    FD_ZERO(&rfds);
 685    FD_ZERO(&wfds);
 686    FD_ZERO(&efds);
 687
 688    // Add the server
 689    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 690       FD_SET(Srv->ServerFd,&wfds);
 691    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 692       FD_SET(Srv->ServerFd,&rfds);
 693
 694    // Add the file
 695    int FileFD = -1;
 696    if (File != 0)
 697       FileFD = File->Fd();
 698
 699    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 700       FD_SET(FileFD,&wfds);
 701
 702    // Add stdin
 703    FD_SET(STDIN_FILENO,&rfds);
 704
 705    // Error Set
 706    if (FileFD != -1)
 707       FD_SET(FileFD,&efds);
 708    if (Srv->ServerFd != -1)
 709       FD_SET(Srv->ServerFd,&efds);
 710
 711    // Figure out the max fd
 712    int MaxFd = FileFD;
 713    if (MaxFd < Srv->ServerFd)
 714       MaxFd = Srv->ServerFd;
 715
 716    // Select
 717    struct timeval tv;
 718    tv.tv_sec = TimeOut;
 719    tv.tv_usec = 0;
 720    int Res = 0;
 721    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 722       return _error->Errno("select","Select failed");
 723
 724    if (Res == 0)
 725    {
 726       _error->Error("Connection timed out");
 727       return ServerDie(Srv);
 728    }
 729
 730    // Some kind of exception (error) on the sockets, die
 731    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 732        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 733       return _error->Error("Socket Exception");
 734
 735    // Handle server IO
 736    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 737    {
 738       errno = 0;
 739       if (Srv->In.Read(Srv->ServerFd) == false)
 740          return ServerDie(Srv);
 741    }
 742
 743    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 744    {
 745       errno = 0;
 746       if (Srv->Out.Write(Srv->ServerFd) == false)
 747          return ServerDie(Srv);
 748    }
 749
 750    // Send data to the file
 751    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 752    {
 753       if (Srv->In.Write(FileFD) == false)
 754          return _error->Errno("write","Error writing to output file");
 755    }
 756
 757    // Handle commands from APT
 758    if (FD_ISSET(STDIN_FILENO,&rfds))
 759    {
 760       if (Run(true) != 0)
 761          exit(100);
 762    }
 763
 764    return true;
 765 }
 766                                                                         /*}}}*/
 767 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 768 // ---------------------------------------------------------------------
 769 /* This takes the current input buffer from the Server FD and writes it
 770    into the file */
 771 bool HttpMethod::Flush(ServerState *Srv)
 772 {
 773    if (File != 0)
 774    {
 775       SetNonBlock(File->Fd(),false);
 776       if (Srv->In.WriteSpace() == false)
 777          return true;
 778
 779       while (Srv->In.WriteSpace() == true)
 780       {
 781          if (Srv->In.Write(File->Fd()) == false)
 782             return _error->Errno("write","Error writing to file");
 783          if (Srv->In.IsLimit() == true)
 784             return true;
 785       }
 786
 787       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 788          return true;
 789    }
 790    return false;
 791 }
 792                                                                         /*}}}*/
 793 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 794 // ---------------------------------------------------------------------
 795 /* */
 796 bool HttpMethod::ServerDie(ServerState *Srv)
 797 {
 798    // Dump the buffer to the file
 799    if (Srv->State == ServerState::Data)
 800    {
 801       SetNonBlock(File->Fd(),false);
 802       while (Srv->In.WriteSpace() == true)
 803       {
 804          if (Srv->In.Write(File->Fd()) == false)
 805             return _error->Errno("write","Error writing to the file");
 806
 807          // Done
 808          if (Srv->In.IsLimit() == true)
 809             return true;
 810       }
 811    }
 812
 813    // See if this is because the server finished the data stream
 814    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 815        Srv->Encoding != ServerState::Closes)
 816    {
 817       Srv->Close();
 818       if (errno == 0)
 819          return _error->Error("Error reading from server Remote end closed connection");
 820       return _error->Errno("read","Error reading from server");
 821    }
 822    else
 823    {
 824       Srv->In.Limit(-1);
 825
 826       // Nothing left in the buffer
 827       if (Srv->In.WriteSpace() == false)
 828          return false;
 829
 830       // We may have got multiple responses back in one packet..
 831       Srv->Close();
 832       return true;
 833    }
 834
 835    return false;
 836 }
 837                                                                         /*}}}*/
 838 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 839 // ---------------------------------------------------------------------
 840 /* We look at the header data we got back from the server and decide what
 841    to do. Returns
 842      0 - File is open,
 843      1 - IMS hit
 844      3 - Unrecoverable error
 845      4 - Error with error content page
 846      5 - Unrecoverable non-server error (close the connection) */
 847 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 848 {
 849    // Not Modified
 850    if (Srv->Result == 304)
 851    {
 852       unlink(Queue->DestFile.c_str());
 853       Res.IMSHit = true;
 854       Res.LastModified = Queue->LastModified;
 855       return 1;
 856    }
 857
 858    /* We have a reply we dont handle. This should indicate a perm server
 859       failure */
 860    if (Srv->Result < 200 || Srv->Result >= 300)
 861    {
 862       _error->Error("%u %s",Srv->Result,Srv->Code);
 863       if (Srv->HaveContent == true)
 864          return 4;
 865       return 3;
 866    }
 867
 868    // This is some sort of 2xx 'data follows' reply
 869    Res.LastModified = Srv->Date;
 870    Res.Size = Srv->Size;
 871
 872    // Open the file
 873    delete File;
 874    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 875    if (_error->PendingError() == true)
 876       return 5;
 877
 878    FailFile = Queue->DestFile;
 879    FailFile.c_str();   // Make sure we don't do a malloc in the signal handler
 880    FailFd = File->Fd();
 881    FailTime = Srv->Date;
 882
 883    // Set the expected size
 884    if (Srv->StartPos >= 0)
 885    {
 886       Res.ResumePoint = Srv->StartPos;
 887       ftruncate(File->Fd(),Srv->StartPos);
 888    }
 889
 890    // Set the start point
 891    lseek(File->Fd(),0,SEEK_END);
 892
 893    delete Srv->In.MD5;
 894    Srv->In.MD5 = new MD5Summation;
 895
 896    // Fill the MD5 Hash if the file is non-empty (resume)
 897    if (Srv->StartPos > 0)
 898    {
 899       lseek(File->Fd(),0,SEEK_SET);
 900       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 901       {
 902          _error->Errno("read","Problem hashing file");
 903          return 5;
 904       }
 905       lseek(File->Fd(),0,SEEK_END);
 906    }
 907
 908    SetNonBlock(File->Fd(),true);
 909    return 0;
 910 }
 911                                                                         /*}}}*/
 912 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 913 // ---------------------------------------------------------------------
 914 /* This closes and timestamps the open file. This is neccessary to get
 915    resume behavoir on user abort */
 916 void HttpMethod::SigTerm(int)
 917 {
 918    if (FailFd == -1)
 919       exit(100);
 920    close(FailFd);
 921
 922    // Timestamp
 923    struct utimbuf UBuf;
 924    time(&UBuf.actime);
 925    UBuf.actime = FailTime;
 926    UBuf.modtime = FailTime;
 927    utime(FailFile.c_str(),&UBuf);
 928
 929    exit(100);
 930 }
 931                                                                         /*}}}*/
 932 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 933 // ---------------------------------------------------------------------
 934 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 935    depth. */
 936 bool HttpMethod::Fetch(FetchItem *)
 937 {
 938    if (Server == 0)
 939       return true;
 940
 941    // Queue the requests
 942    int Depth = -1;
 943    bool Tail = false;
 944    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++)
 945    {
 946       // Make sure we stick with the same server
 947       if (Server->Comp(I->Uri) == false)
 948          break;
 949       if (QueueBack == I)
 950          Tail = true;
 951       if (Tail == true)
 952       {
 953          QueueBack = I->Next;
 954          SendReq(I,Server->Out);
 955          continue;
 956       }
 957    }
 958
 959    return true;
 960 };
 961                                                                         /*}}}*/
 962 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 963 // ---------------------------------------------------------------------
 964 /* We stash the desired pipeline depth */
 965 bool HttpMethod::Configuration(string Message)
 966 {
 967    if (pkgAcqMethod::Configuration(Message) == false)
 968       return false;
 969
 970    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 971    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 972                                   PipelineDepth);
 973
 974    return true;
 975 }
 976                                                                         /*}}}*/
 977 // HttpMethod::Loop - Main loop                                         /*{{{*/
 978 // ---------------------------------------------------------------------
 979 /* */
 980 int HttpMethod::Loop()
 981 {
 982    signal(SIGTERM,SigTerm);
 983    signal(SIGINT,SigTerm);
 984
 985    Server = 0;
 986
 987    int FailCounter = 0;
 988    while (1)
 989    {
 990       if (FailCounter >= 2)
 991       {
 992          Fail("Massive Server Brain Damage",true);
 993          FailCounter = 0;
 994       }
 995
 996       // We have no commands, wait for some to arrive
 997       if (Queue == 0)
 998       {
 999          if (WaitFd(STDIN_FILENO) == false)
1000             return 0;
1001       }
1002
1003       // Run messages
1004       if (Run(true) != 0)
1005          return 100;
1006
1007       if (Queue == 0)
1008          continue;
1009
1010       // Connect to the server
1011       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1012       {
1013          delete Server;
1014          Server = new ServerState(Queue->Uri,this);
1015       }
1016
1017       // Reset the pipeline
1018       if (Server->ServerFd == -1)
1019          QueueBack = Queue;
1020
1021       // Connnect to the host
1022       if (Server->Open() == false)
1023       {
1024          Fail(true);
1025          delete Server;
1026          Server = 0;
1027          continue;
1028       }
1029
1030       // Fill the pipeline.
1031       Fetch(0);
1032
1033       // Fetch the next URL header data from the server.
1034       switch (Server->RunHeaders())
1035       {
1036          case 0:
1037          break;
1038
1039          // The header data is bad
1040          case 2:
1041          {
1042             _error->Error("Bad header Data");
1043             Fail(true);
1044             continue;
1045          }
1046
1047          // The server closed a connection during the header get..
1048          default:
1049          case 1:
1050          {
1051             FailCounter++;
1052             _error->Discard();
1053             Server->Close();
1054             continue;
1055          }
1056       };
1057
1058       // Decide what to do.
1059       FetchResult Res;
1060       Res.Filename = Queue->DestFile;
1061       switch (DealWithHeaders(Res,Server))
1062       {
1063          // Ok, the file is Open
1064          case 0:
1065          {
1066             URIStart(Res);
1067
1068             // Run the data
1069             bool Result =  Server->RunData();
1070
1071             // Close the file, destroy the FD object and timestamp it
1072             FailFd = -1;
1073             delete File;
1074             File = 0;
1075
1076             // Timestamp
1077             struct utimbuf UBuf;
1078             time(&UBuf.actime);
1079             UBuf.actime = Server->Date;
1080             UBuf.modtime = Server->Date;
1081             utime(Queue->DestFile.c_str(),&UBuf);
1082
1083             // Send status to APT
1084             if (Result == true)
1085             {
1086                Res.MD5Sum = Server->In.MD5->Result();
1087                URIDone(Res);
1088             }
1089             else
1090                Fail();
1091
1092             break;
1093          }
1094
1095          // IMS hit
1096          case 1:
1097          {
1098             URIDone(Res);
1099             break;
1100          }
1101
1102          // Hard server error, not found or something
1103          case 3:
1104          {
1105             Fail();
1106             break;
1107          }
1108
1109          // Hard internal error, kill the connection and fail
1110          case 5:
1111          {
1112             Fail();
1113             Server->Close();
1114             break;
1115          }
1116
1117          // We need to flush the data, the header is like a 404 w/ error text
1118          case 4:
1119          {
1120             Fail();
1121
1122             // Send to content to dev/null
1123             File = new FileFd("/dev/null",FileFd::WriteExists);
1124             Server->RunData();
1125             delete File;
1126             File = 0;
1127             break;
1128          }
1129
1130          default:
1131          Fail("Internal error");
1132          break;
1133       }
1134
1135       FailCounter = 0;
1136    }
1137
1138    return 0;
1139 }
1140                                                                         /*}}}*/
1141
1142 int main()
1143 {
1144    HttpMethod Mth;
1145
1146    return Mth.Loop();
1147 }