methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.28 1999/03/15 06:01:00 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41 #include <errno.h>
  42
  43 // Internet stuff
  44 #include <netinet/in.h>
  45 #include <sys/socket.h>
  46 #include <arpa/inet.h>
  47 #include <netdb.h>
  48
  49 #include "http.h"
  50                                                                         /*}}}*/
  51
  52 string HttpMethod::FailFile;
  53 int HttpMethod::FailFd = -1;
  54 time_t HttpMethod::FailTime = 0;
  55 unsigned long PipelineDepth = 10;
  56 unsigned long TimeOut = 120;
  57
  58 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  59 // ---------------------------------------------------------------------
  60 /* */
  61 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  62 {
  63    Buf = new unsigned char[Size];
  64    Reset();
  65 }
  66                                                                         /*}}}*/
  67 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  68 // ---------------------------------------------------------------------
  69 /* */
  70 void CircleBuf::Reset()
  71 {
  72    InP = 0;
  73    OutP = 0;
  74    StrPos = 0;
  75    MaxGet = (unsigned int)-1;
  76    OutQueue = string();
  77    if (MD5 != 0)
  78    {
  79       delete MD5;
  80       MD5 = new MD5Summation;
  81    }
  82 };
  83                                                                         /*}}}*/
  84 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  85 // ---------------------------------------------------------------------
  86 /* This fills up the buffer with as much data as is in the FD, assuming it
  87    is non-blocking.. */
  88 bool CircleBuf::Read(int Fd)
  89 {
  90    while (1)
  91    {
  92       // Woops, buffer is full
  93       if (InP - OutP == Size)
  94          return true;
  95
  96       // Write the buffer segment
  97       int Res;
  98       Res = read(Fd,Buf + (InP%Size),LeftRead());
  99
 100       if (Res == 0)
 101          return false;
 102       if (Res < 0)
 103       {
 104          if (errno == EAGAIN)
 105             return true;
 106          return false;
 107       }
 108
 109       if (InP == 0)
 110          gettimeofday(&Start,0);
 111       InP += Res;
 112    }
 113 }
 114                                                                         /*}}}*/
 115 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 116 // ---------------------------------------------------------------------
 117 /* This will hold the string in and fill the buffer with it as it empties */
 118 bool CircleBuf::Read(string Data)
 119 {
 120    OutQueue += Data;
 121    FillOut();
 122    return true;
 123 }
 124                                                                         /*}}}*/
 125 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* */
 128 void CircleBuf::FillOut()
 129 {
 130    if (OutQueue.empty() == true)
 131       return;
 132    while (1)
 133    {
 134       // Woops, buffer is full
 135       if (InP - OutP == Size)
 136          return;
 137
 138       // Write the buffer segment
 139       unsigned long Sz = LeftRead();
 140       if (OutQueue.length() - StrPos < Sz)
 141          Sz = OutQueue.length() - StrPos;
 142       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 143
 144       // Advance
 145       StrPos += Sz;
 146       InP += Sz;
 147       if (OutQueue.length() == StrPos)
 148       {
 149          StrPos = 0;
 150          OutQueue = "";
 151          return;
 152       }
 153    }
 154 }
 155                                                                         /*}}}*/
 156 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 157 // ---------------------------------------------------------------------
 158 /* This empties the buffer into the FD. */
 159 bool CircleBuf::Write(int Fd)
 160 {
 161    while (1)
 162    {
 163       FillOut();
 164
 165       // Woops, buffer is empty
 166       if (OutP == InP)
 167          return true;
 168
 169       if (OutP == MaxGet)
 170          return true;
 171
 172       // Write the buffer segment
 173       int Res;
 174       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 175
 176       if (Res == 0)
 177          return false;
 178       if (Res < 0)
 179       {
 180          if (errno == EAGAIN)
 181             return true;
 182
 183          return false;
 184       }
 185
 186       if (MD5 != 0)
 187          MD5->Add(Buf + (OutP%Size),Res);
 188
 189       OutP += Res;
 190    }
 191 }
 192                                                                         /*}}}*/
 193 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 194 // ---------------------------------------------------------------------
 195 /* This copies till the first empty line */
 196 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 197 {
 198    // We cheat and assume it is unneeded to have more than one buffer load
 199    for (unsigned long I = OutP; I < InP; I++)
 200    {
 201       if (Buf[I%Size] != '\n')
 202          continue;
 203       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 204
 205       if (Single == false)
 206       {
 207          if (Buf[I%Size] != '\n')
 208             continue;
 209          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 210       }
 211
 212       if (I > InP)
 213          I = InP;
 214
 215       Data = "";
 216       while (OutP < I)
 217       {
 218          unsigned long Sz = LeftWrite();
 219          if (Sz == 0)
 220             return false;
 221          if (I - OutP < LeftWrite())
 222             Sz = I - OutP;
 223          Data += string((char *)(Buf + (OutP%Size)),Sz);
 224          OutP += Sz;
 225       }
 226       return true;
 227    }
 228    return false;
 229 }
 230                                                                         /*}}}*/
 231 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 232 // ---------------------------------------------------------------------
 233 /* */
 234 void CircleBuf::Stats()
 235 {
 236    if (InP == 0)
 237       return;
 238
 239    struct timeval Stop;
 240    gettimeofday(&Stop,0);
 241 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 242              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 243    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 244 }
 245                                                                         /*}}}*/
 246
 247 // ServerState::ServerState - Constructor                               /*{{{*/
 248 // ---------------------------------------------------------------------
 249 /* */
 250 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 251                         In(64*1024), Out(4*1024),
 252                         ServerName(Srv)
 253 {
 254    Reset();
 255 }
 256                                                                         /*}}}*/
 257 // ServerState::Open - Open a connection to the server                  /*{{{*/
 258 // ---------------------------------------------------------------------
 259 /* This opens a connection to the server. */
 260 string LastHost;
 261 in_addr LastHostA;
 262 bool ServerState::Open()
 263 {
 264    // Use the already open connection if possible.
 265    if (ServerFd != -1)
 266       return true;
 267
 268    Close();
 269    In.Reset();
 270    Out.Reset();
 271
 272    // Determine the proxy setting
 273    if (getenv("http_proxy") == 0)
 274    {
 275       string DefProxy = _config->Find("Acquire::http::Proxy");
 276       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 277       if (SpecificProxy.empty() == false)
 278       {
 279          if (SpecificProxy == "DIRECT")
 280             Proxy = "";
 281          else
 282             Proxy = SpecificProxy;
 283       }
 284       else
 285          Proxy = DefProxy;
 286    }
 287    else
 288       Proxy = getenv("http_proxy");
 289
 290    // Determine what host and port to use based on the proxy settings
 291    int Port = 80;
 292    string Host;
 293    if (Proxy.empty() == true)
 294    {
 295       if (ServerName.Port != 0)
 296          Port = ServerName.Port;
 297       Host = ServerName.Host;
 298    }
 299    else
 300    {
 301       if (Proxy.Port != 0)
 302          Port = Proxy.Port;
 303       Host = Proxy.Host;
 304    }
 305
 306    /* We used a cached address record.. Yes this is against the spec but
 307       the way we have setup our rotating dns suggests that this is more
 308       sensible */
 309    if (LastHost != Host)
 310    {
 311       Owner->Status("Connecting to %s",Host.c_str());
 312
 313       // Lookup the host
 314       hostent *Addr = gethostbyname(Host.c_str());
 315       if (Addr == 0 || Addr->h_addr_list[0] == 0)
 316          return _error->Error("Could not resolve '%s'",Host.c_str());
 317       LastHost = Host;
 318       LastHostA = *(in_addr *)(Addr->h_addr_list[0]);
 319    }
 320
 321    Owner->Status("Connecting to %s (%s)",Host.c_str(),inet_ntoa(LastHostA));
 322
 323    // Get a socket
 324    if ((ServerFd = socket(AF_INET,SOCK_STREAM,0)) < 0)
 325       return _error->Errno("socket","Could not create a socket");
 326
 327    // Connect to the server
 328    struct sockaddr_in server;
 329    server.sin_family = AF_INET;
 330    server.sin_port = htons(Port);
 331    server.sin_addr = LastHostA;
 332    SetNonBlock(ServerFd,true);
 333    if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0 &&
 334        errno != EINPROGRESS)
 335       return _error->Errno("socket","Could not create a socket");
 336
 337    /* This implements a timeout for connect by opening the connection
 338       nonblocking */
 339    if (WaitFd(ServerFd,true,TimeOut) == false)
 340       return _error->Error("Could not connect, connection timed out");
 341    unsigned int Err;
 342    unsigned int Len = sizeof(Err);
 343    if (getsockopt(ServerFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 344       return _error->Errno("getsockopt","Failed");
 345    if (Err != 0)
 346       return _error->Error("Could not connect.");
 347
 348    return true;
 349 }
 350                                                                         /*}}}*/
 351 // ServerState::Close - Close a connection to the server                /*{{{*/
 352 // ---------------------------------------------------------------------
 353 /* */
 354 bool ServerState::Close()
 355 {
 356    close(ServerFd);
 357    ServerFd = -1;
 358    return true;
 359 }
 360                                                                         /*}}}*/
 361 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 362 // ---------------------------------------------------------------------
 363 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 364    parse error occured */
 365 int ServerState::RunHeaders()
 366 {
 367    State = Header;
 368
 369    Owner->Status("Waiting for file");
 370
 371    Major = 0;
 372    Minor = 0;
 373    Result = 0;
 374    Size = 0;
 375    StartPos = 0;
 376    Encoding = Closes;
 377    HaveContent = false;
 378    time(&Date);
 379
 380    do
 381    {
 382       string Data;
 383       if (In.WriteTillEl(Data) == false)
 384          continue;
 385
 386       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 387       {
 388          string::const_iterator J = I;
 389          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 390          if (HeaderLine(string(I,J-I)) == false)
 391             return 2;
 392          I = J;
 393       }
 394       return 0;
 395    }
 396    while (Owner->Go(false,this) == true);
 397
 398    return 1;
 399 }
 400                                                                         /*}}}*/
 401 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 402 // ---------------------------------------------------------------------
 403 /* */
 404 bool ServerState::RunData()
 405 {
 406    State = Data;
 407
 408    // Chunked transfer encoding is fun..
 409    if (Encoding == Chunked)
 410    {
 411       while (1)
 412       {
 413          // Grab the block size
 414          bool Last = true;
 415          string Data;
 416          In.Limit(-1);
 417          do
 418          {
 419             if (In.WriteTillEl(Data,true) == true)
 420                break;
 421          }
 422          while ((Last = Owner->Go(false,this)) == true);
 423
 424          if (Last == false)
 425             return false;
 426
 427          // See if we are done
 428          unsigned long Len = strtol(Data.c_str(),0,16);
 429          if (Len == 0)
 430          {
 431             In.Limit(-1);
 432
 433             // We have to remove the entity trailer
 434             Last = true;
 435             do
 436             {
 437                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 438                   break;
 439             }
 440             while ((Last = Owner->Go(false,this)) == true);
 441             if (Last == false)
 442                return false;
 443             return !_error->PendingError();
 444          }
 445
 446          // Transfer the block
 447          In.Limit(Len);
 448          while (Owner->Go(true,this) == true)
 449             if (In.IsLimit() == true)
 450                break;
 451
 452          // Error
 453          if (In.IsLimit() == false)
 454             return false;
 455
 456          // The server sends an extra new line before the next block specifier..
 457          In.Limit(-1);
 458          Last = true;
 459          do
 460          {
 461             if (In.WriteTillEl(Data,true) == true)
 462                break;
 463          }
 464          while ((Last = Owner->Go(false,this)) == true);
 465          if (Last == false)
 466             return false;
 467       }
 468    }
 469    else
 470    {
 471       /* Closes encoding is used when the server did not specify a size, the
 472          loss of the connection means we are done */
 473       if (Encoding == Closes)
 474          In.Limit(-1);
 475       else
 476          In.Limit(Size - StartPos);
 477
 478       // Just transfer the whole block.
 479       do
 480       {
 481          if (In.IsLimit() == false)
 482             continue;
 483
 484          In.Limit(-1);
 485          return !_error->PendingError();
 486       }
 487       while (Owner->Go(true,this) == true);
 488    }
 489
 490    return Owner->Flush(this) && !_error->PendingError();
 491 }
 492                                                                         /*}}}*/
 493 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 494 // ---------------------------------------------------------------------
 495 /* */
 496 bool ServerState::HeaderLine(string Line)
 497 {
 498    if (Line.empty() == true)
 499       return true;
 500
 501    // The http server might be trying to do something evil.
 502    if (Line.length() >= MAXLEN)
 503       return _error->Error("Got a single header line over %u chars",MAXLEN);
 504
 505    string::size_type Pos = Line.find(' ');
 506    if (Pos == string::npos || Pos+1 > Line.length())
 507       return _error->Error("Bad header line");
 508
 509    string Tag = string(Line,0,Pos);
 510    string Val = string(Line,Pos+1);
 511
 512    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 513    {
 514       // Evil servers return no version
 515       if (Line[4] == '/')
 516       {
 517          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 518                     &Result,Code) != 4)
 519             return _error->Error("The http server sent an invalid reply header");
 520       }
 521       else
 522       {
 523          Major = 0;
 524          Minor = 9;
 525          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 526             return _error->Error("The http server sent an invalid reply header");
 527       }
 528
 529       return true;
 530    }
 531
 532    if (stringcasecmp(Tag,"Content-Length:") == 0)
 533    {
 534       if (Encoding == Closes)
 535          Encoding = Stream;
 536       HaveContent = true;
 537
 538       // The length is already set from the Content-Range header
 539       if (StartPos != 0)
 540          return true;
 541
 542       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 543          return _error->Error("The http server sent an invalid Content-Length header");
 544       return true;
 545    }
 546
 547    if (stringcasecmp(Tag,"Content-Type:") == 0)
 548    {
 549       HaveContent = true;
 550       return true;
 551    }
 552
 553    if (stringcasecmp(Tag,"Content-Range:") == 0)
 554    {
 555       HaveContent = true;
 556
 557       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 558          return _error->Error("The http server sent an invalid Content-Range header");
 559       if ((unsigned)StartPos > Size)
 560          return _error->Error("This http server has broken range support");
 561       return true;
 562    }
 563
 564    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 565    {
 566       HaveContent = true;
 567       if (stringcasecmp(Val,"chunked") == 0)
 568          Encoding = Chunked;
 569
 570       return true;
 571    }
 572
 573    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 574    {
 575       if (StrToTime(Val,Date) == false)
 576          return _error->Error("Unknown date format");
 577       return true;
 578    }
 579
 580    return true;
 581 }
 582                                                                         /*}}}*/
 583
 584 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 585 // ---------------------------------------------------------------------
 586 /* This places the http request in the outbound buffer */
 587 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 588 {
 589    URI Uri = Itm->Uri;
 590
 591    // The HTTP server expects a hostname with a trailing :port
 592    char Buf[1000];
 593    string ProperHost = Uri.Host;
 594    if (Uri.Port != 0)
 595    {
 596       sprintf(Buf,":%u",Uri.Port);
 597       ProperHost += Buf;
 598    }
 599
 600    // Just in case.
 601    if (Itm->Uri.length() >= sizeof(Buf))
 602        abort();
 603
 604    /* Build the request. We include a keep-alive header only for non-proxy
 605       requests. This is to tweak old http/1.0 servers that do support keep-alive
 606       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 607       will glitch HTTP/1.0 proxies because they do not filter it out and
 608       pass it on, HTTP/1.1 says the connection should default to keep alive
 609       and we expect the proxy to do this */
 610    if (Proxy.empty() == true)
 611       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 612               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 613    else
 614    {
 615       /* Generate a cache control header if necessary. We place a max
 616          cache age on index files, optionally set a no-cache directive
 617          and a no-store directive for archives. */
 618       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 619               Itm->Uri.c_str(),ProperHost.c_str());
 620       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 621          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 622       else
 623       {
 624          if (Itm->IndexFile == true)
 625             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 626                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 627          else
 628          {
 629             if (_config->FindB("Acquire::http::No-Store",false) == true)
 630                strcat(Buf,"Cache-Control: no-store\r\n");
 631          }
 632       }
 633    }
 634
 635    string Req = Buf;
 636
 637    // Check for a partial file
 638    struct stat SBuf;
 639    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 640    {
 641       // In this case we send an if-range query with a range header
 642       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 643               TimeRFC1123(SBuf.st_mtime).c_str());
 644       Req += Buf;
 645    }
 646    else
 647    {
 648       if (Itm->LastModified != 0)
 649       {
 650          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 651          Req += Buf;
 652       }
 653    }
 654
 655    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 656       Req += string("Proxy-Authorization: Basic ") +
 657           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 658
 659    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 660 //   cerr << Req << endl;
 661
 662    Out.Read(Req);
 663 }
 664                                                                         /*}}}*/
 665 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 666 // ---------------------------------------------------------------------
 667 /* This runs the select loop over the server FDs, Output file FDs and
 668    stdin. */
 669 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 670 {
 671    // Server has closed the connection
 672    if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
 673       return false;
 674
 675    fd_set rfds,wfds,efds;
 676    FD_ZERO(&rfds);
 677    FD_ZERO(&wfds);
 678    FD_ZERO(&efds);
 679
 680    // Add the server
 681    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 682       FD_SET(Srv->ServerFd,&wfds);
 683    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 684       FD_SET(Srv->ServerFd,&rfds);
 685
 686    // Add the file
 687    int FileFD = -1;
 688    if (File != 0)
 689       FileFD = File->Fd();
 690
 691    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 692       FD_SET(FileFD,&wfds);
 693
 694    // Add stdin
 695    FD_SET(STDIN_FILENO,&rfds);
 696
 697    // Error Set
 698    if (FileFD != -1)
 699       FD_SET(FileFD,&efds);
 700    if (Srv->ServerFd != -1)
 701       FD_SET(Srv->ServerFd,&efds);
 702
 703    // Figure out the max fd
 704    int MaxFd = FileFD;
 705    if (MaxFd < Srv->ServerFd)
 706       MaxFd = Srv->ServerFd;
 707
 708    // Select
 709    struct timeval tv;
 710    tv.tv_sec = TimeOut;
 711    tv.tv_usec = 0;
 712    int Res = 0;
 713    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 714       return _error->Errno("select","Select failed");
 715
 716    if (Res == 0)
 717    {
 718       _error->Error("Connection timed out");
 719       return ServerDie(Srv);
 720    }
 721
 722    // Some kind of exception (error) on the sockets, die
 723    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 724        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 725       return _error->Error("Socket Exception");
 726
 727    // Handle server IO
 728    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 729    {
 730       errno = 0;
 731       if (Srv->In.Read(Srv->ServerFd) == false)
 732          return ServerDie(Srv);
 733    }
 734
 735    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 736    {
 737       errno = 0;
 738       if (Srv->Out.Write(Srv->ServerFd) == false)
 739          return ServerDie(Srv);
 740    }
 741
 742    // Send data to the file
 743    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 744    {
 745       if (Srv->In.Write(FileFD) == false)
 746          return _error->Errno("write","Error writing to output file");
 747    }
 748
 749    // Handle commands from APT
 750    if (FD_ISSET(STDIN_FILENO,&rfds))
 751    {
 752       if (Run(true) != 0)
 753          exit(100);
 754    }
 755
 756    return true;
 757 }
 758                                                                         /*}}}*/
 759 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 760 // ---------------------------------------------------------------------
 761 /* This takes the current input buffer from the Server FD and writes it
 762    into the file */
 763 bool HttpMethod::Flush(ServerState *Srv)
 764 {
 765    if (File != 0)
 766    {
 767       SetNonBlock(File->Fd(),false);
 768       if (Srv->In.WriteSpace() == false)
 769          return true;
 770
 771       while (Srv->In.WriteSpace() == true)
 772       {
 773          if (Srv->In.Write(File->Fd()) == false)
 774             return _error->Errno("write","Error writing to file");
 775          if (Srv->In.IsLimit() == true)
 776             return true;
 777       }
 778
 779       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 780          return true;
 781    }
 782    return false;
 783 }
 784                                                                         /*}}}*/
 785 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 786 // ---------------------------------------------------------------------
 787 /* */
 788 bool HttpMethod::ServerDie(ServerState *Srv)
 789 {
 790    unsigned int LErrno = errno;
 791
 792    // Dump the buffer to the file
 793    if (Srv->State == ServerState::Data)
 794    {
 795       SetNonBlock(File->Fd(),false);
 796       while (Srv->In.WriteSpace() == true)
 797       {
 798          if (Srv->In.Write(File->Fd()) == false)
 799             return _error->Errno("write","Error writing to the file");
 800
 801          // Done
 802          if (Srv->In.IsLimit() == true)
 803             return true;
 804       }
 805    }
 806
 807    // See if this is because the server finished the data stream
 808    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 809        Srv->Encoding != ServerState::Closes)
 810    {
 811       Srv->Close();
 812       if (LErrno == 0)
 813          return _error->Error("Error reading from server Remote end closed connection");
 814       errno = LErrno;
 815       return _error->Errno("read","Error reading from server");
 816    }
 817    else
 818    {
 819       Srv->In.Limit(-1);
 820
 821       // Nothing left in the buffer
 822       if (Srv->In.WriteSpace() == false)
 823          return false;
 824
 825       // We may have got multiple responses back in one packet..
 826       Srv->Close();
 827       return true;
 828    }
 829
 830    return false;
 831 }
 832                                                                         /*}}}*/
 833 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 834 // ---------------------------------------------------------------------
 835 /* We look at the header data we got back from the server and decide what
 836    to do. Returns
 837      0 - File is open,
 838      1 - IMS hit
 839      3 - Unrecoverable error
 840      4 - Error with error content page
 841      5 - Unrecoverable non-server error (close the connection) */
 842 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 843 {
 844    // Not Modified
 845    if (Srv->Result == 304)
 846    {
 847       unlink(Queue->DestFile.c_str());
 848       Res.IMSHit = true;
 849       Res.LastModified = Queue->LastModified;
 850       return 1;
 851    }
 852
 853    /* We have a reply we dont handle. This should indicate a perm server
 854       failure */
 855    if (Srv->Result < 200 || Srv->Result >= 300)
 856    {
 857       _error->Error("%u %s",Srv->Result,Srv->Code);
 858       if (Srv->HaveContent == true)
 859          return 4;
 860       return 3;
 861    }
 862
 863    // This is some sort of 2xx 'data follows' reply
 864    Res.LastModified = Srv->Date;
 865    Res.Size = Srv->Size;
 866
 867    // Open the file
 868    delete File;
 869    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 870    if (_error->PendingError() == true)
 871       return 5;
 872
 873    FailFile = Queue->DestFile;
 874    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 875    FailFd = File->Fd();
 876    FailTime = Srv->Date;
 877
 878    // Set the expected size
 879    if (Srv->StartPos >= 0)
 880    {
 881       Res.ResumePoint = Srv->StartPos;
 882       ftruncate(File->Fd(),Srv->StartPos);
 883    }
 884
 885    // Set the start point
 886    lseek(File->Fd(),0,SEEK_END);
 887
 888    delete Srv->In.MD5;
 889    Srv->In.MD5 = new MD5Summation;
 890
 891    // Fill the MD5 Hash if the file is non-empty (resume)
 892    if (Srv->StartPos > 0)
 893    {
 894       lseek(File->Fd(),0,SEEK_SET);
 895       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 896       {
 897          _error->Errno("read","Problem hashing file");
 898          return 5;
 899       }
 900       lseek(File->Fd(),0,SEEK_END);
 901    }
 902
 903    SetNonBlock(File->Fd(),true);
 904    return 0;
 905 }
 906                                                                         /*}}}*/
 907 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 908 // ---------------------------------------------------------------------
 909 /* This closes and timestamps the open file. This is neccessary to get
 910    resume behavoir on user abort */
 911 void HttpMethod::SigTerm(int)
 912 {
 913    if (FailFd == -1)
 914       exit(100);
 915    close(FailFd);
 916
 917    // Timestamp
 918    struct utimbuf UBuf;
 919    time(&UBuf.actime);
 920    UBuf.actime = FailTime;
 921    UBuf.modtime = FailTime;
 922    utime(FailFile.c_str(),&UBuf);
 923
 924    exit(100);
 925 }
 926                                                                         /*}}}*/
 927 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 928 // ---------------------------------------------------------------------
 929 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 930    depth. */
 931 bool HttpMethod::Fetch(FetchItem *)
 932 {
 933    if (Server == 0)
 934       return true;
 935
 936    // Queue the requests
 937    int Depth = -1;
 938    bool Tail = false;
 939    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++)
 940    {
 941       // Make sure we stick with the same server
 942       if (Server->Comp(I->Uri) == false)
 943          break;
 944       if (QueueBack == I)
 945          Tail = true;
 946       if (Tail == true)
 947       {
 948          QueueBack = I->Next;
 949          SendReq(I,Server->Out);
 950          continue;
 951       }
 952    }
 953
 954    return true;
 955 };
 956                                                                         /*}}}*/
 957 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 958 // ---------------------------------------------------------------------
 959 /* We stash the desired pipeline depth */
 960 bool HttpMethod::Configuration(string Message)
 961 {
 962    if (pkgAcqMethod::Configuration(Message) == false)
 963       return false;
 964
 965    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 966    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 967                                   PipelineDepth);
 968
 969    return true;
 970 }
 971                                                                         /*}}}*/
 972 // HttpMethod::Loop - Main loop                                         /*{{{*/
 973 // ---------------------------------------------------------------------
 974 /* */
 975 int HttpMethod::Loop()
 976 {
 977    signal(SIGTERM,SigTerm);
 978    signal(SIGINT,SigTerm);
 979
 980    Server = 0;
 981
 982    int FailCounter = 0;
 983    while (1)
 984    {
 985       // We have no commands, wait for some to arrive
 986       if (Queue == 0)
 987       {
 988          if (WaitFd(STDIN_FILENO) == false)
 989             return 0;
 990       }
 991
 992       // Run messages
 993       if (Run(true) != 0)
 994          return 100;
 995
 996       if (Queue == 0)
 997          continue;
 998
 999       // Connect to the server
1000       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1001       {
1002          delete Server;
1003          Server = new ServerState(Queue->Uri,this);
1004       }
1005
1006       // Reset the pipeline
1007       if (Server->ServerFd == -1)
1008          QueueBack = Queue;
1009
1010       // Connnect to the host
1011       if (Server->Open() == false)
1012       {
1013          Fail(true);
1014          delete Server;
1015          Server = 0;
1016          continue;
1017       }
1018
1019       // Fill the pipeline.
1020       Fetch(0);
1021
1022       // Fetch the next URL header data from the server.
1023       switch (Server->RunHeaders())
1024       {
1025          case 0:
1026          break;
1027
1028          // The header data is bad
1029          case 2:
1030          {
1031             _error->Error("Bad header Data");
1032             Fail(true);
1033             continue;
1034          }
1035
1036          // The server closed a connection during the header get..
1037          default:
1038          case 1:
1039          {
1040             FailCounter++;
1041             _error->Discard();
1042             Server->Close();
1043
1044             if (FailCounter >= 2)
1045             {
1046                Fail("Connection timed out",true);
1047                FailCounter = 0;
1048             }
1049
1050             continue;
1051          }
1052       };
1053
1054       // Decide what to do.
1055       FetchResult Res;
1056       Res.Filename = Queue->DestFile;
1057       switch (DealWithHeaders(Res,Server))
1058       {
1059          // Ok, the file is Open
1060          case 0:
1061          {
1062             URIStart(Res);
1063
1064             // Run the data
1065             bool Result =  Server->RunData();
1066
1067             // Close the file, destroy the FD object and timestamp it
1068             FailFd = -1;
1069             delete File;
1070             File = 0;
1071
1072             // Timestamp
1073             struct utimbuf UBuf;
1074             time(&UBuf.actime);
1075             UBuf.actime = Server->Date;
1076             UBuf.modtime = Server->Date;
1077             utime(Queue->DestFile.c_str(),&UBuf);
1078
1079             // Send status to APT
1080             if (Result == true)
1081             {
1082                Res.MD5Sum = Server->In.MD5->Result();
1083                URIDone(Res);
1084             }
1085             else
1086                Fail(true);
1087
1088             break;
1089          }
1090
1091          // IMS hit
1092          case 1:
1093          {
1094             URIDone(Res);
1095             break;
1096          }
1097
1098          // Hard server error, not found or something
1099          case 3:
1100          {
1101             Fail();
1102             break;
1103          }
1104
1105          // Hard internal error, kill the connection and fail
1106          case 5:
1107          {
1108             Fail();
1109             Server->Close();
1110             break;
1111          }
1112
1113          // We need to flush the data, the header is like a 404 w/ error text
1114          case 4:
1115          {
1116             Fail();
1117
1118             // Send to content to dev/null
1119             File = new FileFd("/dev/null",FileFd::WriteExists);
1120             Server->RunData();
1121             delete File;
1122             File = 0;
1123             break;
1124          }
1125
1126          default:
1127          Fail("Internal error");
1128          break;
1129       }
1130
1131       FailCounter = 0;
1132    }
1133
1134    return 0;
1135 }
1136                                                                         /*}}}*/
1137
1138 int main()
1139 {
1140    HttpMethod Mth;
1141
1142    return Mth.Loop();
1143 }