methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.30 1999/04/04 02:02:04 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41 #include <errno.h>
  42
  43 // Internet stuff
  44 #include <netinet/in.h>
  45 #include <sys/socket.h>
  46 #include <arpa/inet.h>
  47 #include <netdb.h>
  48
  49 #include "http.h"
  50                                                                         /*}}}*/
  51
  52 string HttpMethod::FailFile;
  53 int HttpMethod::FailFd = -1;
  54 time_t HttpMethod::FailTime = 0;
  55 unsigned long PipelineDepth = 10;
  56 unsigned long TimeOut = 120;
  57
  58 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  59 // ---------------------------------------------------------------------
  60 /* */
  61 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  62 {
  63    Buf = new unsigned char[Size];
  64    Reset();
  65 }
  66                                                                         /*}}}*/
  67 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  68 // ---------------------------------------------------------------------
  69 /* */
  70 void CircleBuf::Reset()
  71 {
  72    InP = 0;
  73    OutP = 0;
  74    StrPos = 0;
  75    MaxGet = (unsigned int)-1;
  76    OutQueue = string();
  77    if (MD5 != 0)
  78    {
  79       delete MD5;
  80       MD5 = new MD5Summation;
  81    }
  82 };
  83                                                                         /*}}}*/
  84 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  85 // ---------------------------------------------------------------------
  86 /* This fills up the buffer with as much data as is in the FD, assuming it
  87    is non-blocking.. */
  88 bool CircleBuf::Read(int Fd)
  89 {
  90    while (1)
  91    {
  92       // Woops, buffer is full
  93       if (InP - OutP == Size)
  94          return true;
  95
  96       // Write the buffer segment
  97       int Res;
  98       Res = read(Fd,Buf + (InP%Size),LeftRead());
  99
 100       if (Res == 0)
 101          return false;
 102       if (Res < 0)
 103       {
 104          if (errno == EAGAIN)
 105             return true;
 106          return false;
 107       }
 108
 109       if (InP == 0)
 110          gettimeofday(&Start,0);
 111       InP += Res;
 112    }
 113 }
 114                                                                         /*}}}*/
 115 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 116 // ---------------------------------------------------------------------
 117 /* This will hold the string in and fill the buffer with it as it empties */
 118 bool CircleBuf::Read(string Data)
 119 {
 120    OutQueue += Data;
 121    FillOut();
 122    return true;
 123 }
 124                                                                         /*}}}*/
 125 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* */
 128 void CircleBuf::FillOut()
 129 {
 130    if (OutQueue.empty() == true)
 131       return;
 132    while (1)
 133    {
 134       // Woops, buffer is full
 135       if (InP - OutP == Size)
 136          return;
 137
 138       // Write the buffer segment
 139       unsigned long Sz = LeftRead();
 140       if (OutQueue.length() - StrPos < Sz)
 141          Sz = OutQueue.length() - StrPos;
 142       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 143
 144       // Advance
 145       StrPos += Sz;
 146       InP += Sz;
 147       if (OutQueue.length() == StrPos)
 148       {
 149          StrPos = 0;
 150          OutQueue = "";
 151          return;
 152       }
 153    }
 154 }
 155                                                                         /*}}}*/
 156 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 157 // ---------------------------------------------------------------------
 158 /* This empties the buffer into the FD. */
 159 bool CircleBuf::Write(int Fd)
 160 {
 161    while (1)
 162    {
 163       FillOut();
 164
 165       // Woops, buffer is empty
 166       if (OutP == InP)
 167          return true;
 168
 169       if (OutP == MaxGet)
 170          return true;
 171
 172       // Write the buffer segment
 173       int Res;
 174       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 175
 176       if (Res == 0)
 177          return false;
 178       if (Res < 0)
 179       {
 180          if (errno == EAGAIN)
 181             return true;
 182
 183          return false;
 184       }
 185
 186       if (MD5 != 0)
 187          MD5->Add(Buf + (OutP%Size),Res);
 188
 189       OutP += Res;
 190    }
 191 }
 192                                                                         /*}}}*/
 193 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 194 // ---------------------------------------------------------------------
 195 /* This copies till the first empty line */
 196 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 197 {
 198    // We cheat and assume it is unneeded to have more than one buffer load
 199    for (unsigned long I = OutP; I < InP; I++)
 200    {
 201       if (Buf[I%Size] != '\n')
 202          continue;
 203       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 204
 205       if (Single == false)
 206       {
 207          if (Buf[I%Size] != '\n')
 208             continue;
 209          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 210       }
 211
 212       if (I > InP)
 213          I = InP;
 214
 215       Data = "";
 216       while (OutP < I)
 217       {
 218          unsigned long Sz = LeftWrite();
 219          if (Sz == 0)
 220             return false;
 221          if (I - OutP < LeftWrite())
 222             Sz = I - OutP;
 223          Data += string((char *)(Buf + (OutP%Size)),Sz);
 224          OutP += Sz;
 225       }
 226       return true;
 227    }
 228    return false;
 229 }
 230                                                                         /*}}}*/
 231 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 232 // ---------------------------------------------------------------------
 233 /* */
 234 void CircleBuf::Stats()
 235 {
 236    if (InP == 0)
 237       return;
 238
 239    struct timeval Stop;
 240    gettimeofday(&Stop,0);
 241 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 242              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 243    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 244 }
 245                                                                         /*}}}*/
 246
 247 // ServerState::ServerState - Constructor                               /*{{{*/
 248 // ---------------------------------------------------------------------
 249 /* */
 250 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 251                         In(64*1024), Out(4*1024),
 252                         ServerName(Srv)
 253 {
 254    Reset();
 255 }
 256                                                                         /*}}}*/
 257 // ServerState::Open - Open a connection to the server                  /*{{{*/
 258 // ---------------------------------------------------------------------
 259 /* This opens a connection to the server. */
 260 string LastHost;
 261 in_addr LastHostA;
 262 bool ServerState::Open()
 263 {
 264    // Use the already open connection if possible.
 265    if (ServerFd != -1)
 266       return true;
 267
 268    Close();
 269    In.Reset();
 270    Out.Reset();
 271
 272    // Determine the proxy setting
 273    if (getenv("http_proxy") == 0)
 274    {
 275       string DefProxy = _config->Find("Acquire::http::Proxy");
 276       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 277       if (SpecificProxy.empty() == false)
 278       {
 279          if (SpecificProxy == "DIRECT")
 280             Proxy = "";
 281          else
 282             Proxy = SpecificProxy;
 283       }
 284       else
 285          Proxy = DefProxy;
 286    }
 287    else
 288       Proxy = getenv("http_proxy");
 289
 290    // Determine what host and port to use based on the proxy settings
 291    int Port = 80;
 292    string Host;
 293    if (Proxy.empty() == true)
 294    {
 295       if (ServerName.Port != 0)
 296          Port = ServerName.Port;
 297       Host = ServerName.Host;
 298    }
 299    else
 300    {
 301       if (Proxy.Port != 0)
 302          Port = Proxy.Port;
 303       Host = Proxy.Host;
 304    }
 305
 306    /* We used a cached address record.. Yes this is against the spec but
 307       the way we have setup our rotating dns suggests that this is more
 308       sensible */
 309    if (LastHost != Host)
 310    {
 311       Owner->Status("Connecting to %s",Host.c_str());
 312
 313       // Lookup the host
 314       hostent *Addr = gethostbyname(Host.c_str());
 315       if (Addr == 0 || Addr->h_addr_list[0] == 0)
 316          return _error->Error("Could not resolve '%s'",Host.c_str());
 317       LastHost = Host;
 318       LastHostA = *(in_addr *)(Addr->h_addr_list[0]);
 319    }
 320
 321    Owner->Status("Connecting to %s (%s)",Host.c_str(),inet_ntoa(LastHostA));
 322
 323    // Get a socket
 324    if ((ServerFd = socket(AF_INET,SOCK_STREAM,0)) < 0)
 325       return _error->Errno("socket","Could not create a socket");
 326
 327    // Connect to the server
 328    struct sockaddr_in server;
 329    server.sin_family = AF_INET;
 330    server.sin_port = htons(Port);
 331    server.sin_addr = LastHostA;
 332    SetNonBlock(ServerFd,true);
 333    if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0 &&
 334        errno != EINPROGRESS)
 335       return _error->Errno("socket","Could not create a socket");
 336
 337    /* This implements a timeout for connect by opening the connection
 338       nonblocking */
 339    if (WaitFd(ServerFd,true,TimeOut) == false)
 340       return _error->Error("Could not connect, connection timed out");
 341    unsigned int Err;
 342    unsigned int Len = sizeof(Err);
 343    if (getsockopt(ServerFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 344       return _error->Errno("getsockopt","Failed");
 345    if (Err != 0)
 346       return _error->Error("Could not connect.");
 347
 348    return true;
 349 }
 350                                                                         /*}}}*/
 351 // ServerState::Close - Close a connection to the server                /*{{{*/
 352 // ---------------------------------------------------------------------
 353 /* */
 354 bool ServerState::Close()
 355 {
 356    close(ServerFd);
 357    ServerFd = -1;
 358    return true;
 359 }
 360                                                                         /*}}}*/
 361 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 362 // ---------------------------------------------------------------------
 363 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 364    parse error occured */
 365 int ServerState::RunHeaders()
 366 {
 367    State = Header;
 368
 369    Owner->Status("Waiting for file");
 370
 371    Major = 0;
 372    Minor = 0;
 373    Result = 0;
 374    Size = 0;
 375    StartPos = 0;
 376    Encoding = Closes;
 377    HaveContent = false;
 378    time(&Date);
 379
 380    do
 381    {
 382       string Data;
 383       if (In.WriteTillEl(Data) == false)
 384          continue;
 385
 386       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 387       {
 388          string::const_iterator J = I;
 389          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 390          if (HeaderLine(string(I,J-I)) == false)
 391             return 2;
 392          I = J;
 393       }
 394       return 0;
 395    }
 396    while (Owner->Go(false,this) == true);
 397
 398    return 1;
 399 }
 400                                                                         /*}}}*/
 401 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 402 // ---------------------------------------------------------------------
 403 /* */
 404 bool ServerState::RunData()
 405 {
 406    State = Data;
 407
 408    // Chunked transfer encoding is fun..
 409    if (Encoding == Chunked)
 410    {
 411       while (1)
 412       {
 413          // Grab the block size
 414          bool Last = true;
 415          string Data;
 416          In.Limit(-1);
 417          do
 418          {
 419             if (In.WriteTillEl(Data,true) == true)
 420                break;
 421          }
 422          while ((Last = Owner->Go(false,this)) == true);
 423
 424          if (Last == false)
 425             return false;
 426
 427          // See if we are done
 428          unsigned long Len = strtol(Data.c_str(),0,16);
 429          if (Len == 0)
 430          {
 431             In.Limit(-1);
 432
 433             // We have to remove the entity trailer
 434             Last = true;
 435             do
 436             {
 437                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 438                   break;
 439             }
 440             while ((Last = Owner->Go(false,this)) == true);
 441             if (Last == false)
 442                return false;
 443             return !_error->PendingError();
 444          }
 445
 446          // Transfer the block
 447          In.Limit(Len);
 448          while (Owner->Go(true,this) == true)
 449             if (In.IsLimit() == true)
 450                break;
 451
 452          // Error
 453          if (In.IsLimit() == false)
 454             return false;
 455
 456          // The server sends an extra new line before the next block specifier..
 457          In.Limit(-1);
 458          Last = true;
 459          do
 460          {
 461             if (In.WriteTillEl(Data,true) == true)
 462                break;
 463          }
 464          while ((Last = Owner->Go(false,this)) == true);
 465          if (Last == false)
 466             return false;
 467       }
 468    }
 469    else
 470    {
 471       /* Closes encoding is used when the server did not specify a size, the
 472          loss of the connection means we are done */
 473       if (Encoding == Closes)
 474          In.Limit(-1);
 475       else
 476          In.Limit(Size - StartPos);
 477
 478       // Just transfer the whole block.
 479       do
 480       {
 481          if (In.IsLimit() == false)
 482             continue;
 483
 484          In.Limit(-1);
 485          return !_error->PendingError();
 486       }
 487       while (Owner->Go(true,this) == true);
 488    }
 489
 490    return Owner->Flush(this) && !_error->PendingError();
 491 }
 492                                                                         /*}}}*/
 493 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 494 // ---------------------------------------------------------------------
 495 /* */
 496 bool ServerState::HeaderLine(string Line)
 497 {
 498    if (Line.empty() == true)
 499       return true;
 500
 501    // The http server might be trying to do something evil.
 502    if (Line.length() >= MAXLEN)
 503       return _error->Error("Got a single header line over %u chars",MAXLEN);
 504
 505    string::size_type Pos = Line.find(' ');
 506    if (Pos == string::npos || Pos+1 > Line.length())
 507    {
 508       // Blah, some servers use "connection:closes", evil.
 509       Pos = Line.find(':');
 510       if (Pos == string::npos || Pos + 2 > Line.length())
 511          return _error->Error("Bad header line");
 512       Pos++;
 513    }
 514
 515    // Parse off any trailing spaces between the : and the next word.
 516    string::size_type Pos2 = Pos;
 517    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 518       Pos2++;
 519
 520    string Tag = string(Line,0,Pos);
 521    string Val = string(Line,Pos2);
 522
 523    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 524    {
 525       // Evil servers return no version
 526       if (Line[4] == '/')
 527       {
 528          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 529                     &Result,Code) != 4)
 530             return _error->Error("The http server sent an invalid reply header");
 531       }
 532       else
 533       {
 534          Major = 0;
 535          Minor = 9;
 536          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 537             return _error->Error("The http server sent an invalid reply header");
 538       }
 539
 540       return true;
 541    }
 542
 543    if (stringcasecmp(Tag,"Content-Length:") == 0)
 544    {
 545       if (Encoding == Closes)
 546          Encoding = Stream;
 547       HaveContent = true;
 548
 549       // The length is already set from the Content-Range header
 550       if (StartPos != 0)
 551          return true;
 552
 553       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 554          return _error->Error("The http server sent an invalid Content-Length header");
 555       return true;
 556    }
 557
 558    if (stringcasecmp(Tag,"Content-Type:") == 0)
 559    {
 560       HaveContent = true;
 561       return true;
 562    }
 563
 564    if (stringcasecmp(Tag,"Content-Range:") == 0)
 565    {
 566       HaveContent = true;
 567
 568       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 569          return _error->Error("The http server sent an invalid Content-Range header");
 570       if ((unsigned)StartPos > Size)
 571          return _error->Error("This http server has broken range support");
 572       return true;
 573    }
 574
 575    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 576    {
 577       HaveContent = true;
 578       if (stringcasecmp(Val,"chunked") == 0)
 579          Encoding = Chunked;
 580
 581       return true;
 582    }
 583
 584    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 585    {
 586       if (StrToTime(Val,Date) == false)
 587          return _error->Error("Unknown date format");
 588       return true;
 589    }
 590
 591    return true;
 592 }
 593                                                                         /*}}}*/
 594
 595 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 596 // ---------------------------------------------------------------------
 597 /* This places the http request in the outbound buffer */
 598 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 599 {
 600    URI Uri = Itm->Uri;
 601
 602    // The HTTP server expects a hostname with a trailing :port
 603    char Buf[1000];
 604    string ProperHost = Uri.Host;
 605    if (Uri.Port != 0)
 606    {
 607       sprintf(Buf,":%u",Uri.Port);
 608       ProperHost += Buf;
 609    }
 610
 611    // Just in case.
 612    if (Itm->Uri.length() >= sizeof(Buf))
 613        abort();
 614
 615    /* Build the request. We include a keep-alive header only for non-proxy
 616       requests. This is to tweak old http/1.0 servers that do support keep-alive
 617       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 618       will glitch HTTP/1.0 proxies because they do not filter it out and
 619       pass it on, HTTP/1.1 says the connection should default to keep alive
 620       and we expect the proxy to do this */
 621    if (Proxy.empty() == true)
 622       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 623               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 624    else
 625    {
 626       /* Generate a cache control header if necessary. We place a max
 627          cache age on index files, optionally set a no-cache directive
 628          and a no-store directive for archives. */
 629       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 630               Itm->Uri.c_str(),ProperHost.c_str());
 631       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 632          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 633       else
 634       {
 635          if (Itm->IndexFile == true)
 636             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 637                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 638          else
 639          {
 640             if (_config->FindB("Acquire::http::No-Store",false) == true)
 641                strcat(Buf,"Cache-Control: no-store\r\n");
 642          }
 643       }
 644    }
 645
 646    string Req = Buf;
 647
 648    // Check for a partial file
 649    struct stat SBuf;
 650    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 651    {
 652       // In this case we send an if-range query with a range header
 653       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 654               TimeRFC1123(SBuf.st_mtime).c_str());
 655       Req += Buf;
 656    }
 657    else
 658    {
 659       if (Itm->LastModified != 0)
 660       {
 661          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 662          Req += Buf;
 663       }
 664    }
 665
 666    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 667       Req += string("Proxy-Authorization: Basic ") +
 668           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 669
 670    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 671 //   cerr << Req << endl;
 672
 673    Out.Read(Req);
 674 }
 675                                                                         /*}}}*/
 676 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 677 // ---------------------------------------------------------------------
 678 /* This runs the select loop over the server FDs, Output file FDs and
 679    stdin. */
 680 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 681 {
 682    // Server has closed the connection
 683    if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
 684       return false;
 685
 686    fd_set rfds,wfds,efds;
 687    FD_ZERO(&rfds);
 688    FD_ZERO(&wfds);
 689    FD_ZERO(&efds);
 690
 691    // Add the server
 692    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 693       FD_SET(Srv->ServerFd,&wfds);
 694    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 695       FD_SET(Srv->ServerFd,&rfds);
 696
 697    // Add the file
 698    int FileFD = -1;
 699    if (File != 0)
 700       FileFD = File->Fd();
 701
 702    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 703       FD_SET(FileFD,&wfds);
 704
 705    // Add stdin
 706    FD_SET(STDIN_FILENO,&rfds);
 707
 708    // Error Set
 709    if (FileFD != -1)
 710       FD_SET(FileFD,&efds);
 711    if (Srv->ServerFd != -1)
 712       FD_SET(Srv->ServerFd,&efds);
 713
 714    // Figure out the max fd
 715    int MaxFd = FileFD;
 716    if (MaxFd < Srv->ServerFd)
 717       MaxFd = Srv->ServerFd;
 718
 719    // Select
 720    struct timeval tv;
 721    tv.tv_sec = TimeOut;
 722    tv.tv_usec = 0;
 723    int Res = 0;
 724    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 725       return _error->Errno("select","Select failed");
 726
 727    if (Res == 0)
 728    {
 729       _error->Error("Connection timed out");
 730       return ServerDie(Srv);
 731    }
 732
 733    // Some kind of exception (error) on the sockets, die
 734    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 735        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 736       return _error->Error("Socket Exception");
 737
 738    // Handle server IO
 739    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 740    {
 741       errno = 0;
 742       if (Srv->In.Read(Srv->ServerFd) == false)
 743          return ServerDie(Srv);
 744    }
 745
 746    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 747    {
 748       errno = 0;
 749       if (Srv->Out.Write(Srv->ServerFd) == false)
 750          return ServerDie(Srv);
 751    }
 752
 753    // Send data to the file
 754    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 755    {
 756       if (Srv->In.Write(FileFD) == false)
 757          return _error->Errno("write","Error writing to output file");
 758    }
 759
 760    // Handle commands from APT
 761    if (FD_ISSET(STDIN_FILENO,&rfds))
 762    {
 763       if (Run(true) != 0)
 764          exit(100);
 765    }
 766
 767    return true;
 768 }
 769                                                                         /*}}}*/
 770 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 771 // ---------------------------------------------------------------------
 772 /* This takes the current input buffer from the Server FD and writes it
 773    into the file */
 774 bool HttpMethod::Flush(ServerState *Srv)
 775 {
 776    if (File != 0)
 777    {
 778       SetNonBlock(File->Fd(),false);
 779       if (Srv->In.WriteSpace() == false)
 780          return true;
 781
 782       while (Srv->In.WriteSpace() == true)
 783       {
 784          if (Srv->In.Write(File->Fd()) == false)
 785             return _error->Errno("write","Error writing to file");
 786          if (Srv->In.IsLimit() == true)
 787             return true;
 788       }
 789
 790       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 791          return true;
 792    }
 793    return false;
 794 }
 795                                                                         /*}}}*/
 796 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 797 // ---------------------------------------------------------------------
 798 /* */
 799 bool HttpMethod::ServerDie(ServerState *Srv)
 800 {
 801    unsigned int LErrno = errno;
 802
 803    // Dump the buffer to the file
 804    if (Srv->State == ServerState::Data)
 805    {
 806       SetNonBlock(File->Fd(),false);
 807       while (Srv->In.WriteSpace() == true)
 808       {
 809          if (Srv->In.Write(File->Fd()) == false)
 810             return _error->Errno("write","Error writing to the file");
 811
 812          // Done
 813          if (Srv->In.IsLimit() == true)
 814             return true;
 815       }
 816    }
 817
 818    // See if this is because the server finished the data stream
 819    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 820        Srv->Encoding != ServerState::Closes)
 821    {
 822       Srv->Close();
 823       if (LErrno == 0)
 824          return _error->Error("Error reading from server Remote end closed connection");
 825       errno = LErrno;
 826       return _error->Errno("read","Error reading from server");
 827    }
 828    else
 829    {
 830       Srv->In.Limit(-1);
 831
 832       // Nothing left in the buffer
 833       if (Srv->In.WriteSpace() == false)
 834          return false;
 835
 836       // We may have got multiple responses back in one packet..
 837       Srv->Close();
 838       return true;
 839    }
 840
 841    return false;
 842 }
 843                                                                         /*}}}*/
 844 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 845 // ---------------------------------------------------------------------
 846 /* We look at the header data we got back from the server and decide what
 847    to do. Returns
 848      0 - File is open,
 849      1 - IMS hit
 850      3 - Unrecoverable error
 851      4 - Error with error content page
 852      5 - Unrecoverable non-server error (close the connection) */
 853 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 854 {
 855    // Not Modified
 856    if (Srv->Result == 304)
 857    {
 858       unlink(Queue->DestFile.c_str());
 859       Res.IMSHit = true;
 860       Res.LastModified = Queue->LastModified;
 861       return 1;
 862    }
 863
 864    /* We have a reply we dont handle. This should indicate a perm server
 865       failure */
 866    if (Srv->Result < 200 || Srv->Result >= 300)
 867    {
 868       _error->Error("%u %s",Srv->Result,Srv->Code);
 869       if (Srv->HaveContent == true)
 870          return 4;
 871       return 3;
 872    }
 873
 874    // This is some sort of 2xx 'data follows' reply
 875    Res.LastModified = Srv->Date;
 876    Res.Size = Srv->Size;
 877
 878    // Open the file
 879    delete File;
 880    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 881    if (_error->PendingError() == true)
 882       return 5;
 883
 884    FailFile = Queue->DestFile;
 885    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 886    FailFd = File->Fd();
 887    FailTime = Srv->Date;
 888
 889    // Set the expected size
 890    if (Srv->StartPos >= 0)
 891    {
 892       Res.ResumePoint = Srv->StartPos;
 893       ftruncate(File->Fd(),Srv->StartPos);
 894    }
 895
 896    // Set the start point
 897    lseek(File->Fd(),0,SEEK_END);
 898
 899    delete Srv->In.MD5;
 900    Srv->In.MD5 = new MD5Summation;
 901
 902    // Fill the MD5 Hash if the file is non-empty (resume)
 903    if (Srv->StartPos > 0)
 904    {
 905       lseek(File->Fd(),0,SEEK_SET);
 906       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 907       {
 908          _error->Errno("read","Problem hashing file");
 909          return 5;
 910       }
 911       lseek(File->Fd(),0,SEEK_END);
 912    }
 913
 914    SetNonBlock(File->Fd(),true);
 915    return 0;
 916 }
 917                                                                         /*}}}*/
 918 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 919 // ---------------------------------------------------------------------
 920 /* This closes and timestamps the open file. This is neccessary to get
 921    resume behavoir on user abort */
 922 void HttpMethod::SigTerm(int)
 923 {
 924    if (FailFd == -1)
 925       exit(100);
 926    close(FailFd);
 927
 928    // Timestamp
 929    struct utimbuf UBuf;
 930    UBuf.actime = FailTime;
 931    UBuf.modtime = FailTime;
 932    utime(FailFile.c_str(),&UBuf);
 933
 934    exit(100);
 935 }
 936                                                                         /*}}}*/
 937 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 938 // ---------------------------------------------------------------------
 939 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 940    depth. */
 941 bool HttpMethod::Fetch(FetchItem *)
 942 {
 943    if (Server == 0)
 944       return true;
 945
 946    // Queue the requests
 947    int Depth = -1;
 948    bool Tail = false;
 949    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++)
 950    {
 951       // Make sure we stick with the same server
 952       if (Server->Comp(I->Uri) == false)
 953          break;
 954       if (QueueBack == I)
 955          Tail = true;
 956       if (Tail == true)
 957       {
 958          QueueBack = I->Next;
 959          SendReq(I,Server->Out);
 960          continue;
 961       }
 962    }
 963
 964    return true;
 965 };
 966                                                                         /*}}}*/
 967 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 968 // ---------------------------------------------------------------------
 969 /* We stash the desired pipeline depth */
 970 bool HttpMethod::Configuration(string Message)
 971 {
 972    if (pkgAcqMethod::Configuration(Message) == false)
 973       return false;
 974
 975    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 976    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 977                                   PipelineDepth);
 978
 979    return true;
 980 }
 981                                                                         /*}}}*/
 982 // HttpMethod::Loop - Main loop                                         /*{{{*/
 983 // ---------------------------------------------------------------------
 984 /* */
 985 int HttpMethod::Loop()
 986 {
 987    signal(SIGTERM,SigTerm);
 988    signal(SIGINT,SigTerm);
 989
 990    Server = 0;
 991
 992    int FailCounter = 0;
 993    while (1)
 994    {
 995       // We have no commands, wait for some to arrive
 996       if (Queue == 0)
 997       {
 998          if (WaitFd(STDIN_FILENO) == false)
 999             return 0;
1000       }
1001
1002       // Run messages
1003       if (Run(true) != 0)
1004          return 100;
1005
1006       if (Queue == 0)
1007          continue;
1008
1009       // Connect to the server
1010       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1011       {
1012          delete Server;
1013          Server = new ServerState(Queue->Uri,this);
1014       }
1015
1016       // Reset the pipeline
1017       if (Server->ServerFd == -1)
1018          QueueBack = Queue;
1019
1020       // Connnect to the host
1021       if (Server->Open() == false)
1022       {
1023          Fail(true);
1024          delete Server;
1025          Server = 0;
1026          continue;
1027       }
1028
1029       // Fill the pipeline.
1030       Fetch(0);
1031
1032       // Fetch the next URL header data from the server.
1033       switch (Server->RunHeaders())
1034       {
1035          case 0:
1036          break;
1037
1038          // The header data is bad
1039          case 2:
1040          {
1041             _error->Error("Bad header Data");
1042             Fail(true);
1043             continue;
1044          }
1045
1046          // The server closed a connection during the header get..
1047          default:
1048          case 1:
1049          {
1050             FailCounter++;
1051             _error->Discard();
1052             Server->Close();
1053
1054             if (FailCounter >= 2)
1055             {
1056                Fail("Connection timed out",true);
1057                FailCounter = 0;
1058             }
1059
1060             continue;
1061          }
1062       };
1063
1064       // Decide what to do.
1065       FetchResult Res;
1066       Res.Filename = Queue->DestFile;
1067       switch (DealWithHeaders(Res,Server))
1068       {
1069          // Ok, the file is Open
1070          case 0:
1071          {
1072             URIStart(Res);
1073
1074             // Run the data
1075             bool Result =  Server->RunData();
1076
1077             // Close the file, destroy the FD object and timestamp it
1078             FailFd = -1;
1079             delete File;
1080             File = 0;
1081
1082             // Timestamp
1083             struct utimbuf UBuf;
1084             time(&UBuf.actime);
1085             UBuf.actime = Server->Date;
1086             UBuf.modtime = Server->Date;
1087             utime(Queue->DestFile.c_str(),&UBuf);
1088
1089             // Send status to APT
1090             if (Result == true)
1091             {
1092                Res.MD5Sum = Server->In.MD5->Result();
1093                URIDone(Res);
1094             }
1095             else
1096                Fail(true);
1097
1098             break;
1099          }
1100
1101          // IMS hit
1102          case 1:
1103          {
1104             URIDone(Res);
1105             break;
1106          }
1107
1108          // Hard server error, not found or something
1109          case 3:
1110          {
1111             Fail();
1112             break;
1113          }
1114
1115          // Hard internal error, kill the connection and fail
1116          case 5:
1117          {
1118             Fail();
1119             Server->Close();
1120             break;
1121          }
1122
1123          // We need to flush the data, the header is like a 404 w/ error text
1124          case 4:
1125          {
1126             Fail();
1127
1128             // Send to content to dev/null
1129             File = new FileFd("/dev/null",FileFd::WriteExists);
1130             Server->RunData();
1131             delete File;
1132             File = 0;
1133             break;
1134          }
1135
1136          default:
1137          Fail("Internal error");
1138          break;
1139       }
1140
1141       FailCounter = 0;
1142    }
1143
1144    return 0;
1145 }
1146                                                                         /*}}}*/
1147
1148 int main()
1149 {
1150    HttpMethod Mth;
1151
1152    return Mth.Loop();
1153 }