methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.29 1999/03/15 07:20:41 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41 #include <errno.h>
  42
  43 // Internet stuff
  44 #include <netinet/in.h>
  45 #include <sys/socket.h>
  46 #include <arpa/inet.h>
  47 #include <netdb.h>
  48
  49 #include "http.h"
  50                                                                         /*}}}*/
  51
  52 string HttpMethod::FailFile;
  53 int HttpMethod::FailFd = -1;
  54 time_t HttpMethod::FailTime = 0;
  55 unsigned long PipelineDepth = 10;
  56 unsigned long TimeOut = 120;
  57
  58 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  59 // ---------------------------------------------------------------------
  60 /* */
  61 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  62 {
  63    Buf = new unsigned char[Size];
  64    Reset();
  65 }
  66                                                                         /*}}}*/
  67 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  68 // ---------------------------------------------------------------------
  69 /* */
  70 void CircleBuf::Reset()
  71 {
  72    InP = 0;
  73    OutP = 0;
  74    StrPos = 0;
  75    MaxGet = (unsigned int)-1;
  76    OutQueue = string();
  77    if (MD5 != 0)
  78    {
  79       delete MD5;
  80       MD5 = new MD5Summation;
  81    }
  82 };
  83                                                                         /*}}}*/
  84 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  85 // ---------------------------------------------------------------------
  86 /* This fills up the buffer with as much data as is in the FD, assuming it
  87    is non-blocking.. */
  88 bool CircleBuf::Read(int Fd)
  89 {
  90    while (1)
  91    {
  92       // Woops, buffer is full
  93       if (InP - OutP == Size)
  94          return true;
  95
  96       // Write the buffer segment
  97       int Res;
  98       Res = read(Fd,Buf + (InP%Size),LeftRead());
  99
 100       if (Res == 0)
 101          return false;
 102       if (Res < 0)
 103       {
 104          if (errno == EAGAIN)
 105             return true;
 106          return false;
 107       }
 108
 109       if (InP == 0)
 110          gettimeofday(&Start,0);
 111       InP += Res;
 112    }
 113 }
 114                                                                         /*}}}*/
 115 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 116 // ---------------------------------------------------------------------
 117 /* This will hold the string in and fill the buffer with it as it empties */
 118 bool CircleBuf::Read(string Data)
 119 {
 120    OutQueue += Data;
 121    FillOut();
 122    return true;
 123 }
 124                                                                         /*}}}*/
 125 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* */
 128 void CircleBuf::FillOut()
 129 {
 130    if (OutQueue.empty() == true)
 131       return;
 132    while (1)
 133    {
 134       // Woops, buffer is full
 135       if (InP - OutP == Size)
 136          return;
 137
 138       // Write the buffer segment
 139       unsigned long Sz = LeftRead();
 140       if (OutQueue.length() - StrPos < Sz)
 141          Sz = OutQueue.length() - StrPos;
 142       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 143
 144       // Advance
 145       StrPos += Sz;
 146       InP += Sz;
 147       if (OutQueue.length() == StrPos)
 148       {
 149          StrPos = 0;
 150          OutQueue = "";
 151          return;
 152       }
 153    }
 154 }
 155                                                                         /*}}}*/
 156 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 157 // ---------------------------------------------------------------------
 158 /* This empties the buffer into the FD. */
 159 bool CircleBuf::Write(int Fd)
 160 {
 161    while (1)
 162    {
 163       FillOut();
 164
 165       // Woops, buffer is empty
 166       if (OutP == InP)
 167          return true;
 168
 169       if (OutP == MaxGet)
 170          return true;
 171
 172       // Write the buffer segment
 173       int Res;
 174       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 175
 176       if (Res == 0)
 177          return false;
 178       if (Res < 0)
 179       {
 180          if (errno == EAGAIN)
 181             return true;
 182
 183          return false;
 184       }
 185
 186       if (MD5 != 0)
 187          MD5->Add(Buf + (OutP%Size),Res);
 188
 189       OutP += Res;
 190    }
 191 }
 192                                                                         /*}}}*/
 193 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 194 // ---------------------------------------------------------------------
 195 /* This copies till the first empty line */
 196 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 197 {
 198    // We cheat and assume it is unneeded to have more than one buffer load
 199    for (unsigned long I = OutP; I < InP; I++)
 200    {
 201       if (Buf[I%Size] != '\n')
 202          continue;
 203       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 204
 205       if (Single == false)
 206       {
 207          if (Buf[I%Size] != '\n')
 208             continue;
 209          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 210       }
 211
 212       if (I > InP)
 213          I = InP;
 214
 215       Data = "";
 216       while (OutP < I)
 217       {
 218          unsigned long Sz = LeftWrite();
 219          if (Sz == 0)
 220             return false;
 221          if (I - OutP < LeftWrite())
 222             Sz = I - OutP;
 223          Data += string((char *)(Buf + (OutP%Size)),Sz);
 224          OutP += Sz;
 225       }
 226       return true;
 227    }
 228    return false;
 229 }
 230                                                                         /*}}}*/
 231 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 232 // ---------------------------------------------------------------------
 233 /* */
 234 void CircleBuf::Stats()
 235 {
 236    if (InP == 0)
 237       return;
 238
 239    struct timeval Stop;
 240    gettimeofday(&Stop,0);
 241 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 242              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 243    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 244 }
 245                                                                         /*}}}*/
 246
 247 // ServerState::ServerState - Constructor                               /*{{{*/
 248 // ---------------------------------------------------------------------
 249 /* */
 250 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 251                         In(64*1024), Out(4*1024),
 252                         ServerName(Srv)
 253 {
 254    Reset();
 255 }
 256                                                                         /*}}}*/
 257 // ServerState::Open - Open a connection to the server                  /*{{{*/
 258 // ---------------------------------------------------------------------
 259 /* This opens a connection to the server. */
 260 string LastHost;
 261 in_addr LastHostA;
 262 bool ServerState::Open()
 263 {
 264    // Use the already open connection if possible.
 265    if (ServerFd != -1)
 266       return true;
 267
 268    Close();
 269    In.Reset();
 270    Out.Reset();
 271
 272    // Determine the proxy setting
 273    if (getenv("http_proxy") == 0)
 274    {
 275       string DefProxy = _config->Find("Acquire::http::Proxy");
 276       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 277       if (SpecificProxy.empty() == false)
 278       {
 279          if (SpecificProxy == "DIRECT")
 280             Proxy = "";
 281          else
 282             Proxy = SpecificProxy;
 283       }
 284       else
 285          Proxy = DefProxy;
 286    }
 287    else
 288       Proxy = getenv("http_proxy");
 289
 290    // Determine what host and port to use based on the proxy settings
 291    int Port = 80;
 292    string Host;
 293    if (Proxy.empty() == true)
 294    {
 295       if (ServerName.Port != 0)
 296          Port = ServerName.Port;
 297       Host = ServerName.Host;
 298    }
 299    else
 300    {
 301       if (Proxy.Port != 0)
 302          Port = Proxy.Port;
 303       Host = Proxy.Host;
 304    }
 305
 306    /* We used a cached address record.. Yes this is against the spec but
 307       the way we have setup our rotating dns suggests that this is more
 308       sensible */
 309    if (LastHost != Host)
 310    {
 311       Owner->Status("Connecting to %s",Host.c_str());
 312
 313       // Lookup the host
 314       hostent *Addr = gethostbyname(Host.c_str());
 315       if (Addr == 0 || Addr->h_addr_list[0] == 0)
 316          return _error->Error("Could not resolve '%s'",Host.c_str());
 317       LastHost = Host;
 318       LastHostA = *(in_addr *)(Addr->h_addr_list[0]);
 319    }
 320
 321    Owner->Status("Connecting to %s (%s)",Host.c_str(),inet_ntoa(LastHostA));
 322
 323    // Get a socket
 324    if ((ServerFd = socket(AF_INET,SOCK_STREAM,0)) < 0)
 325       return _error->Errno("socket","Could not create a socket");
 326
 327    // Connect to the server
 328    struct sockaddr_in server;
 329    server.sin_family = AF_INET;
 330    server.sin_port = htons(Port);
 331    server.sin_addr = LastHostA;
 332    SetNonBlock(ServerFd,true);
 333    if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0 &&
 334        errno != EINPROGRESS)
 335       return _error->Errno("socket","Could not create a socket");
 336
 337    /* This implements a timeout for connect by opening the connection
 338       nonblocking */
 339    if (WaitFd(ServerFd,true,TimeOut) == false)
 340       return _error->Error("Could not connect, connection timed out");
 341    unsigned int Err;
 342    unsigned int Len = sizeof(Err);
 343    if (getsockopt(ServerFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 344       return _error->Errno("getsockopt","Failed");
 345    if (Err != 0)
 346       return _error->Error("Could not connect.");
 347
 348    return true;
 349 }
 350                                                                         /*}}}*/
 351 // ServerState::Close - Close a connection to the server                /*{{{*/
 352 // ---------------------------------------------------------------------
 353 /* */
 354 bool ServerState::Close()
 355 {
 356    close(ServerFd);
 357    ServerFd = -1;
 358    return true;
 359 }
 360                                                                         /*}}}*/
 361 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 362 // ---------------------------------------------------------------------
 363 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 364    parse error occured */
 365 int ServerState::RunHeaders()
 366 {
 367    State = Header;
 368
 369    Owner->Status("Waiting for file");
 370
 371    Major = 0;
 372    Minor = 0;
 373    Result = 0;
 374    Size = 0;
 375    StartPos = 0;
 376    Encoding = Closes;
 377    HaveContent = false;
 378    time(&Date);
 379
 380    do
 381    {
 382       string Data;
 383       if (In.WriteTillEl(Data) == false)
 384          continue;
 385
 386       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 387       {
 388          string::const_iterator J = I;
 389          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 390          if (HeaderLine(string(I,J-I)) == false)
 391             return 2;
 392          I = J;
 393       }
 394       return 0;
 395    }
 396    while (Owner->Go(false,this) == true);
 397
 398    return 1;
 399 }
 400                                                                         /*}}}*/
 401 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 402 // ---------------------------------------------------------------------
 403 /* */
 404 bool ServerState::RunData()
 405 {
 406    State = Data;
 407
 408    // Chunked transfer encoding is fun..
 409    if (Encoding == Chunked)
 410    {
 411       while (1)
 412       {
 413          // Grab the block size
 414          bool Last = true;
 415          string Data;
 416          In.Limit(-1);
 417          do
 418          {
 419             if (In.WriteTillEl(Data,true) == true)
 420                break;
 421          }
 422          while ((Last = Owner->Go(false,this)) == true);
 423
 424          if (Last == false)
 425             return false;
 426
 427          // See if we are done
 428          unsigned long Len = strtol(Data.c_str(),0,16);
 429          if (Len == 0)
 430          {
 431             In.Limit(-1);
 432
 433             // We have to remove the entity trailer
 434             Last = true;
 435             do
 436             {
 437                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 438                   break;
 439             }
 440             while ((Last = Owner->Go(false,this)) == true);
 441             if (Last == false)
 442                return false;
 443             return !_error->PendingError();
 444          }
 445
 446          // Transfer the block
 447          In.Limit(Len);
 448          while (Owner->Go(true,this) == true)
 449             if (In.IsLimit() == true)
 450                break;
 451
 452          // Error
 453          if (In.IsLimit() == false)
 454             return false;
 455
 456          // The server sends an extra new line before the next block specifier..
 457          In.Limit(-1);
 458          Last = true;
 459          do
 460          {
 461             if (In.WriteTillEl(Data,true) == true)
 462                break;
 463          }
 464          while ((Last = Owner->Go(false,this)) == true);
 465          if (Last == false)
 466             return false;
 467       }
 468    }
 469    else
 470    {
 471       /* Closes encoding is used when the server did not specify a size, the
 472          loss of the connection means we are done */
 473       if (Encoding == Closes)
 474          In.Limit(-1);
 475       else
 476          In.Limit(Size - StartPos);
 477
 478       // Just transfer the whole block.
 479       do
 480       {
 481          if (In.IsLimit() == false)
 482             continue;
 483
 484          In.Limit(-1);
 485          return !_error->PendingError();
 486       }
 487       while (Owner->Go(true,this) == true);
 488    }
 489
 490    return Owner->Flush(this) && !_error->PendingError();
 491 }
 492                                                                         /*}}}*/
 493 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 494 // ---------------------------------------------------------------------
 495 /* */
 496 bool ServerState::HeaderLine(string Line)
 497 {
 498    if (Line.empty() == true)
 499       return true;
 500
 501    // The http server might be trying to do something evil.
 502    if (Line.length() >= MAXLEN)
 503       return _error->Error("Got a single header line over %u chars",MAXLEN);
 504
 505    string::size_type Pos = Line.find(' ');
 506    if (Pos == string::npos || Pos+1 > Line.length())
 507       return _error->Error("Bad header line");
 508
 509    string Tag = string(Line,0,Pos);
 510    string Val = string(Line,Pos+1);
 511
 512    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 513    {
 514       // Evil servers return no version
 515       if (Line[4] == '/')
 516       {
 517          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 518                     &Result,Code) != 4)
 519             return _error->Error("The http server sent an invalid reply header");
 520       }
 521       else
 522       {
 523          Major = 0;
 524          Minor = 9;
 525          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 526             return _error->Error("The http server sent an invalid reply header");
 527       }
 528
 529       return true;
 530    }
 531
 532    if (stringcasecmp(Tag,"Content-Length:") == 0)
 533    {
 534       if (Encoding == Closes)
 535          Encoding = Stream;
 536       HaveContent = true;
 537
 538       // The length is already set from the Content-Range header
 539       if (StartPos != 0)
 540          return true;
 541
 542       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 543          return _error->Error("The http server sent an invalid Content-Length header");
 544       return true;
 545    }
 546
 547    if (stringcasecmp(Tag,"Content-Type:") == 0)
 548    {
 549       HaveContent = true;
 550       return true;
 551    }
 552
 553    if (stringcasecmp(Tag,"Content-Range:") == 0)
 554    {
 555       HaveContent = true;
 556
 557       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 558          return _error->Error("The http server sent an invalid Content-Range header");
 559       if ((unsigned)StartPos > Size)
 560          return _error->Error("This http server has broken range support");
 561       return true;
 562    }
 563
 564    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 565    {
 566       HaveContent = true;
 567       if (stringcasecmp(Val,"chunked") == 0)
 568          Encoding = Chunked;
 569
 570       return true;
 571    }
 572
 573    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 574    {
 575       if (StrToTime(Val,Date) == false)
 576          return _error->Error("Unknown date format");
 577       return true;
 578    }
 579
 580    return true;
 581 }
 582                                                                         /*}}}*/
 583
 584 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 585 // ---------------------------------------------------------------------
 586 /* This places the http request in the outbound buffer */
 587 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 588 {
 589    URI Uri = Itm->Uri;
 590
 591    // The HTTP server expects a hostname with a trailing :port
 592    char Buf[1000];
 593    string ProperHost = Uri.Host;
 594    if (Uri.Port != 0)
 595    {
 596       sprintf(Buf,":%u",Uri.Port);
 597       ProperHost += Buf;
 598    }
 599
 600    // Just in case.
 601    if (Itm->Uri.length() >= sizeof(Buf))
 602        abort();
 603
 604    /* Build the request. We include a keep-alive header only for non-proxy
 605       requests. This is to tweak old http/1.0 servers that do support keep-alive
 606       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 607       will glitch HTTP/1.0 proxies because they do not filter it out and
 608       pass it on, HTTP/1.1 says the connection should default to keep alive
 609       and we expect the proxy to do this */
 610    if (Proxy.empty() == true)
 611       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 612               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 613    else
 614    {
 615       /* Generate a cache control header if necessary. We place a max
 616          cache age on index files, optionally set a no-cache directive
 617          and a no-store directive for archives. */
 618       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 619               Itm->Uri.c_str(),ProperHost.c_str());
 620       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 621          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 622       else
 623       {
 624          if (Itm->IndexFile == true)
 625             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 626                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 627          else
 628          {
 629             if (_config->FindB("Acquire::http::No-Store",false) == true)
 630                strcat(Buf,"Cache-Control: no-store\r\n");
 631          }
 632       }
 633    }
 634
 635    string Req = Buf;
 636
 637    // Check for a partial file
 638    struct stat SBuf;
 639    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 640    {
 641       // In this case we send an if-range query with a range header
 642       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 643               TimeRFC1123(SBuf.st_mtime).c_str());
 644       Req += Buf;
 645    }
 646    else
 647    {
 648       if (Itm->LastModified != 0)
 649       {
 650          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 651          Req += Buf;
 652       }
 653    }
 654
 655    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 656       Req += string("Proxy-Authorization: Basic ") +
 657           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 658
 659    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 660 //   cerr << Req << endl;
 661
 662    Out.Read(Req);
 663 }
 664                                                                         /*}}}*/
 665 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 666 // ---------------------------------------------------------------------
 667 /* This runs the select loop over the server FDs, Output file FDs and
 668    stdin. */
 669 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 670 {
 671    // Server has closed the connection
 672    if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
 673       return false;
 674
 675    fd_set rfds,wfds,efds;
 676    FD_ZERO(&rfds);
 677    FD_ZERO(&wfds);
 678    FD_ZERO(&efds);
 679
 680    // Add the server
 681    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 682       FD_SET(Srv->ServerFd,&wfds);
 683    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 684       FD_SET(Srv->ServerFd,&rfds);
 685
 686    // Add the file
 687    int FileFD = -1;
 688    if (File != 0)
 689       FileFD = File->Fd();
 690
 691    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 692       FD_SET(FileFD,&wfds);
 693
 694    // Add stdin
 695    FD_SET(STDIN_FILENO,&rfds);
 696
 697    // Error Set
 698    if (FileFD != -1)
 699       FD_SET(FileFD,&efds);
 700    if (Srv->ServerFd != -1)
 701       FD_SET(Srv->ServerFd,&efds);
 702
 703    // Figure out the max fd
 704    int MaxFd = FileFD;
 705    if (MaxFd < Srv->ServerFd)
 706       MaxFd = Srv->ServerFd;
 707
 708    // Select
 709    struct timeval tv;
 710    tv.tv_sec = TimeOut;
 711    tv.tv_usec = 0;
 712    int Res = 0;
 713    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 714       return _error->Errno("select","Select failed");
 715
 716    if (Res == 0)
 717    {
 718       _error->Error("Connection timed out");
 719       return ServerDie(Srv);
 720    }
 721
 722    // Some kind of exception (error) on the sockets, die
 723    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 724        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 725       return _error->Error("Socket Exception");
 726
 727    // Handle server IO
 728    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 729    {
 730       errno = 0;
 731       if (Srv->In.Read(Srv->ServerFd) == false)
 732          return ServerDie(Srv);
 733    }
 734
 735    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 736    {
 737       errno = 0;
 738       if (Srv->Out.Write(Srv->ServerFd) == false)
 739          return ServerDie(Srv);
 740    }
 741
 742    // Send data to the file
 743    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 744    {
 745       if (Srv->In.Write(FileFD) == false)
 746          return _error->Errno("write","Error writing to output file");
 747    }
 748
 749    // Handle commands from APT
 750    if (FD_ISSET(STDIN_FILENO,&rfds))
 751    {
 752       if (Run(true) != 0)
 753          exit(100);
 754    }
 755
 756    return true;
 757 }
 758                                                                         /*}}}*/
 759 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 760 // ---------------------------------------------------------------------
 761 /* This takes the current input buffer from the Server FD and writes it
 762    into the file */
 763 bool HttpMethod::Flush(ServerState *Srv)
 764 {
 765    if (File != 0)
 766    {
 767       SetNonBlock(File->Fd(),false);
 768       if (Srv->In.WriteSpace() == false)
 769          return true;
 770
 771       while (Srv->In.WriteSpace() == true)
 772       {
 773          if (Srv->In.Write(File->Fd()) == false)
 774             return _error->Errno("write","Error writing to file");
 775          if (Srv->In.IsLimit() == true)
 776             return true;
 777       }
 778
 779       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 780          return true;
 781    }
 782    return false;
 783 }
 784                                                                         /*}}}*/
 785 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 786 // ---------------------------------------------------------------------
 787 /* */
 788 bool HttpMethod::ServerDie(ServerState *Srv)
 789 {
 790    unsigned int LErrno = errno;
 791
 792    // Dump the buffer to the file
 793    if (Srv->State == ServerState::Data)
 794    {
 795       SetNonBlock(File->Fd(),false);
 796       while (Srv->In.WriteSpace() == true)
 797       {
 798          if (Srv->In.Write(File->Fd()) == false)
 799             return _error->Errno("write","Error writing to the file");
 800
 801          // Done
 802          if (Srv->In.IsLimit() == true)
 803             return true;
 804       }
 805    }
 806
 807    // See if this is because the server finished the data stream
 808    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 809        Srv->Encoding != ServerState::Closes)
 810    {
 811       Srv->Close();
 812       if (LErrno == 0)
 813          return _error->Error("Error reading from server Remote end closed connection");
 814       errno = LErrno;
 815       return _error->Errno("read","Error reading from server");
 816    }
 817    else
 818    {
 819       Srv->In.Limit(-1);
 820
 821       // Nothing left in the buffer
 822       if (Srv->In.WriteSpace() == false)
 823          return false;
 824
 825       // We may have got multiple responses back in one packet..
 826       Srv->Close();
 827       return true;
 828    }
 829
 830    return false;
 831 }
 832                                                                         /*}}}*/
 833 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 834 // ---------------------------------------------------------------------
 835 /* We look at the header data we got back from the server and decide what
 836    to do. Returns
 837      0 - File is open,
 838      1 - IMS hit
 839      3 - Unrecoverable error
 840      4 - Error with error content page
 841      5 - Unrecoverable non-server error (close the connection) */
 842 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 843 {
 844    // Not Modified
 845    if (Srv->Result == 304)
 846    {
 847       unlink(Queue->DestFile.c_str());
 848       Res.IMSHit = true;
 849       Res.LastModified = Queue->LastModified;
 850       return 1;
 851    }
 852
 853    /* We have a reply we dont handle. This should indicate a perm server
 854       failure */
 855    if (Srv->Result < 200 || Srv->Result >= 300)
 856    {
 857       _error->Error("%u %s",Srv->Result,Srv->Code);
 858       if (Srv->HaveContent == true)
 859          return 4;
 860       return 3;
 861    }
 862
 863    // This is some sort of 2xx 'data follows' reply
 864    Res.LastModified = Srv->Date;
 865    Res.Size = Srv->Size;
 866
 867    // Open the file
 868    delete File;
 869    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 870    if (_error->PendingError() == true)
 871       return 5;
 872
 873    FailFile = Queue->DestFile;
 874    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 875    FailFd = File->Fd();
 876    FailTime = Srv->Date;
 877
 878    // Set the expected size
 879    if (Srv->StartPos >= 0)
 880    {
 881       Res.ResumePoint = Srv->StartPos;
 882       ftruncate(File->Fd(),Srv->StartPos);
 883    }
 884
 885    // Set the start point
 886    lseek(File->Fd(),0,SEEK_END);
 887
 888    delete Srv->In.MD5;
 889    Srv->In.MD5 = new MD5Summation;
 890
 891    // Fill the MD5 Hash if the file is non-empty (resume)
 892    if (Srv->StartPos > 0)
 893    {
 894       lseek(File->Fd(),0,SEEK_SET);
 895       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 896       {
 897          _error->Errno("read","Problem hashing file");
 898          return 5;
 899       }
 900       lseek(File->Fd(),0,SEEK_END);
 901    }
 902
 903    SetNonBlock(File->Fd(),true);
 904    return 0;
 905 }
 906                                                                         /*}}}*/
 907 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 908 // ---------------------------------------------------------------------
 909 /* This closes and timestamps the open file. This is neccessary to get
 910    resume behavoir on user abort */
 911 void HttpMethod::SigTerm(int)
 912 {
 913    if (FailFd == -1)
 914       exit(100);
 915    close(FailFd);
 916
 917    // Timestamp
 918    struct utimbuf UBuf;
 919    UBuf.actime = FailTime;
 920    UBuf.modtime = FailTime;
 921    utime(FailFile.c_str(),&UBuf);
 922
 923    exit(100);
 924 }
 925                                                                         /*}}}*/
 926 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 927 // ---------------------------------------------------------------------
 928 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 929    depth. */
 930 bool HttpMethod::Fetch(FetchItem *)
 931 {
 932    if (Server == 0)
 933       return true;
 934
 935    // Queue the requests
 936    int Depth = -1;
 937    bool Tail = false;
 938    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++)
 939    {
 940       // Make sure we stick with the same server
 941       if (Server->Comp(I->Uri) == false)
 942          break;
 943       if (QueueBack == I)
 944          Tail = true;
 945       if (Tail == true)
 946       {
 947          QueueBack = I->Next;
 948          SendReq(I,Server->Out);
 949          continue;
 950       }
 951    }
 952
 953    return true;
 954 };
 955                                                                         /*}}}*/
 956 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 957 // ---------------------------------------------------------------------
 958 /* We stash the desired pipeline depth */
 959 bool HttpMethod::Configuration(string Message)
 960 {
 961    if (pkgAcqMethod::Configuration(Message) == false)
 962       return false;
 963
 964    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 965    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 966                                   PipelineDepth);
 967
 968    return true;
 969 }
 970                                                                         /*}}}*/
 971 // HttpMethod::Loop - Main loop                                         /*{{{*/
 972 // ---------------------------------------------------------------------
 973 /* */
 974 int HttpMethod::Loop()
 975 {
 976    signal(SIGTERM,SigTerm);
 977    signal(SIGINT,SigTerm);
 978
 979    Server = 0;
 980
 981    int FailCounter = 0;
 982    while (1)
 983    {
 984       // We have no commands, wait for some to arrive
 985       if (Queue == 0)
 986       {
 987          if (WaitFd(STDIN_FILENO) == false)
 988             return 0;
 989       }
 990
 991       // Run messages
 992       if (Run(true) != 0)
 993          return 100;
 994
 995       if (Queue == 0)
 996          continue;
 997
 998       // Connect to the server
 999       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1000       {
1001          delete Server;
1002          Server = new ServerState(Queue->Uri,this);
1003       }
1004
1005       // Reset the pipeline
1006       if (Server->ServerFd == -1)
1007          QueueBack = Queue;
1008
1009       // Connnect to the host
1010       if (Server->Open() == false)
1011       {
1012          Fail(true);
1013          delete Server;
1014          Server = 0;
1015          continue;
1016       }
1017
1018       // Fill the pipeline.
1019       Fetch(0);
1020
1021       // Fetch the next URL header data from the server.
1022       switch (Server->RunHeaders())
1023       {
1024          case 0:
1025          break;
1026
1027          // The header data is bad
1028          case 2:
1029          {
1030             _error->Error("Bad header Data");
1031             Fail(true);
1032             continue;
1033          }
1034
1035          // The server closed a connection during the header get..
1036          default:
1037          case 1:
1038          {
1039             FailCounter++;
1040             _error->Discard();
1041             Server->Close();
1042
1043             if (FailCounter >= 2)
1044             {
1045                Fail("Connection timed out",true);
1046                FailCounter = 0;
1047             }
1048
1049             continue;
1050          }
1051       };
1052
1053       // Decide what to do.
1054       FetchResult Res;
1055       Res.Filename = Queue->DestFile;
1056       switch (DealWithHeaders(Res,Server))
1057       {
1058          // Ok, the file is Open
1059          case 0:
1060          {
1061             URIStart(Res);
1062
1063             // Run the data
1064             bool Result =  Server->RunData();
1065
1066             // Close the file, destroy the FD object and timestamp it
1067             FailFd = -1;
1068             delete File;
1069             File = 0;
1070
1071             // Timestamp
1072             struct utimbuf UBuf;
1073             time(&UBuf.actime);
1074             UBuf.actime = Server->Date;
1075             UBuf.modtime = Server->Date;
1076             utime(Queue->DestFile.c_str(),&UBuf);
1077
1078             // Send status to APT
1079             if (Result == true)
1080             {
1081                Res.MD5Sum = Server->In.MD5->Result();
1082                URIDone(Res);
1083             }
1084             else
1085                Fail(true);
1086
1087             break;
1088          }
1089
1090          // IMS hit
1091          case 1:
1092          {
1093             URIDone(Res);
1094             break;
1095          }
1096
1097          // Hard server error, not found or something
1098          case 3:
1099          {
1100             Fail();
1101             break;
1102          }
1103
1104          // Hard internal error, kill the connection and fail
1105          case 5:
1106          {
1107             Fail();
1108             Server->Close();
1109             break;
1110          }
1111
1112          // We need to flush the data, the header is like a 404 w/ error text
1113          case 4:
1114          {
1115             Fail();
1116
1117             // Send to content to dev/null
1118             File = new FileFd("/dev/null",FileFd::WriteExists);
1119             Server->RunData();
1120             delete File;
1121             File = 0;
1122             break;
1123          }
1124
1125          default:
1126          Fail("Internal error");
1127          break;
1128       }
1129
1130       FailCounter = 0;
1131    }
1132
1133    return 0;
1134 }
1135                                                                         /*}}}*/
1136
1137 int main()
1138 {
1139    HttpMethod Mth;
1140
1141    return Mth.Loop();
1142 }