methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.39 1999/10/18 03:18:40 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41 #include <errno.h>
  42
  43 // Internet stuff
  44 #include <netdb.h>
  45
  46 #include "connect.h"
  47 #include "rfc2553emu.h"
  48 #include "http.h"
  49
  50                                                                         /*}}}*/
  51
  52 string HttpMethod::FailFile;
  53 int HttpMethod::FailFd = -1;
  54 time_t HttpMethod::FailTime = 0;
  55 unsigned long PipelineDepth = 10;
  56 unsigned long TimeOut = 120;
  57 bool Debug = false;
  58
  59 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  60 // ---------------------------------------------------------------------
  61 /* */
  62 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  63 {
  64    Buf = new unsigned char[Size];
  65    Reset();
  66 }
  67                                                                         /*}}}*/
  68 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  69 // ---------------------------------------------------------------------
  70 /* */
  71 void CircleBuf::Reset()
  72 {
  73    InP = 0;
  74    OutP = 0;
  75    StrPos = 0;
  76    MaxGet = (unsigned int)-1;
  77    OutQueue = string();
  78    if (MD5 != 0)
  79    {
  80       delete MD5;
  81       MD5 = new MD5Summation;
  82    }
  83 };
  84                                                                         /*}}}*/
  85 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* This fills up the buffer with as much data as is in the FD, assuming it
  88    is non-blocking.. */
  89 bool CircleBuf::Read(int Fd)
  90 {
  91    while (1)
  92    {
  93       // Woops, buffer is full
  94       if (InP - OutP == Size)
  95          return true;
  96
  97       // Write the buffer segment
  98       int Res;
  99       Res = read(Fd,Buf + (InP%Size),LeftRead());
 100
 101       if (Res == 0)
 102          return false;
 103       if (Res < 0)
 104       {
 105          if (errno == EAGAIN)
 106             return true;
 107          return false;
 108       }
 109
 110       if (InP == 0)
 111          gettimeofday(&Start,0);
 112       InP += Res;
 113    }
 114 }
 115                                                                         /*}}}*/
 116 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 117 // ---------------------------------------------------------------------
 118 /* This will hold the string in and fill the buffer with it as it empties */
 119 bool CircleBuf::Read(string Data)
 120 {
 121    OutQueue += Data;
 122    FillOut();
 123    return true;
 124 }
 125                                                                         /*}}}*/
 126 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 127 // ---------------------------------------------------------------------
 128 /* */
 129 void CircleBuf::FillOut()
 130 {
 131    if (OutQueue.empty() == true)
 132       return;
 133    while (1)
 134    {
 135       // Woops, buffer is full
 136       if (InP - OutP == Size)
 137          return;
 138
 139       // Write the buffer segment
 140       unsigned long Sz = LeftRead();
 141       if (OutQueue.length() - StrPos < Sz)
 142          Sz = OutQueue.length() - StrPos;
 143       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 144
 145       // Advance
 146       StrPos += Sz;
 147       InP += Sz;
 148       if (OutQueue.length() == StrPos)
 149       {
 150          StrPos = 0;
 151          OutQueue = "";
 152          return;
 153       }
 154    }
 155 }
 156                                                                         /*}}}*/
 157 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 158 // ---------------------------------------------------------------------
 159 /* This empties the buffer into the FD. */
 160 bool CircleBuf::Write(int Fd)
 161 {
 162    while (1)
 163    {
 164       FillOut();
 165
 166       // Woops, buffer is empty
 167       if (OutP == InP)
 168          return true;
 169
 170       if (OutP == MaxGet)
 171          return true;
 172
 173       // Write the buffer segment
 174       int Res;
 175       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 176
 177       if (Res == 0)
 178          return false;
 179       if (Res < 0)
 180       {
 181          if (errno == EAGAIN)
 182             return true;
 183
 184          return false;
 185       }
 186
 187       if (MD5 != 0)
 188          MD5->Add(Buf + (OutP%Size),Res);
 189
 190       OutP += Res;
 191    }
 192 }
 193                                                                         /*}}}*/
 194 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 195 // ---------------------------------------------------------------------
 196 /* This copies till the first empty line */
 197 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 198 {
 199    // We cheat and assume it is unneeded to have more than one buffer load
 200    for (unsigned long I = OutP; I < InP; I++)
 201    {
 202       if (Buf[I%Size] != '\n')
 203          continue;
 204       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 205
 206       if (Single == false)
 207       {
 208          if (Buf[I%Size] != '\n')
 209             continue;
 210          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 211       }
 212
 213       if (I > InP)
 214          I = InP;
 215
 216       Data = "";
 217       while (OutP < I)
 218       {
 219          unsigned long Sz = LeftWrite();
 220          if (Sz == 0)
 221             return false;
 222          if (I - OutP < LeftWrite())
 223             Sz = I - OutP;
 224          Data += string((char *)(Buf + (OutP%Size)),Sz);
 225          OutP += Sz;
 226       }
 227       return true;
 228    }
 229    return false;
 230 }
 231                                                                         /*}}}*/
 232 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 233 // ---------------------------------------------------------------------
 234 /* */
 235 void CircleBuf::Stats()
 236 {
 237    if (InP == 0)
 238       return;
 239
 240    struct timeval Stop;
 241    gettimeofday(&Stop,0);
 242 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 243              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 244    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 245 }
 246                                                                         /*}}}*/
 247
 248 // ServerState::ServerState - Constructor                               /*{{{*/
 249 // ---------------------------------------------------------------------
 250 /* */
 251 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 252                         In(64*1024), Out(4*1024),
 253                         ServerName(Srv)
 254 {
 255    Reset();
 256 }
 257                                                                         /*}}}*/
 258 // ServerState::Open - Open a connection to the server                  /*{{{*/
 259 // ---------------------------------------------------------------------
 260 /* This opens a connection to the server. */
 261 string LastHost;
 262 int LastPort = 0;
 263 struct addrinfo *LastHostAddr = 0;
 264 bool ServerState::Open()
 265 {
 266    // Use the already open connection if possible.
 267    if (ServerFd != -1)
 268       return true;
 269
 270    Close();
 271    In.Reset();
 272    Out.Reset();
 273
 274    // Determine the proxy setting
 275    if (getenv("http_proxy") == 0)
 276    {
 277       string DefProxy = _config->Find("Acquire::http::Proxy");
 278       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 279       if (SpecificProxy.empty() == false)
 280       {
 281          if (SpecificProxy == "DIRECT")
 282             Proxy = "";
 283          else
 284             Proxy = SpecificProxy;
 285       }
 286       else
 287          Proxy = DefProxy;
 288    }
 289    else
 290       Proxy = getenv("http_proxy");
 291
 292    // Parse no_proxy, a , seperated list of hosts
 293    if (getenv("no_proxy") != 0)
 294    {
 295       const char *Start = getenv("no_proxy");
 296       for (const char *Cur = Start; true ; Cur++)
 297       {
 298          if (*Cur != ',' && *Cur != 0)
 299             continue;
 300          if (stringcasecmp(ServerName.Host.begin(),ServerName.Host.end(),
 301                            Start,Cur) == 0)
 302          {
 303             Proxy = "";
 304             break;
 305          }
 306
 307          Start = Cur + 1;
 308          if (*Cur == 0)
 309             break;
 310       }
 311    }
 312
 313    // Determine what host and port to use based on the proxy settings
 314    int Port = 0;
 315    string Host;
 316    if (Proxy.empty() == true)
 317    {
 318       if (ServerName.Port != 0)
 319          Port = ServerName.Port;
 320       Host = ServerName.Host;
 321    }
 322    else
 323    {
 324       if (Proxy.Port != 0)
 325          Port = Proxy.Port;
 326       Host = Proxy.Host;
 327    }
 328
 329    // Connect to the remote server
 330    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 331       return false;
 332
 333    return true;
 334 }
 335                                                                         /*}}}*/
 336 // ServerState::Close - Close a connection to the server                /*{{{*/
 337 // ---------------------------------------------------------------------
 338 /* */
 339 bool ServerState::Close()
 340 {
 341    close(ServerFd);
 342    ServerFd = -1;
 343    return true;
 344 }
 345                                                                         /*}}}*/
 346 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 347 // ---------------------------------------------------------------------
 348 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 349    parse error occured */
 350 int ServerState::RunHeaders()
 351 {
 352    State = Header;
 353
 354    Owner->Status("Waiting for file");
 355
 356    Major = 0;
 357    Minor = 0;
 358    Result = 0;
 359    Size = 0;
 360    StartPos = 0;
 361    Encoding = Closes;
 362    HaveContent = false;
 363    time(&Date);
 364
 365    do
 366    {
 367       string Data;
 368       if (In.WriteTillEl(Data) == false)
 369          continue;
 370
 371       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 372       {
 373          string::const_iterator J = I;
 374          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 375          if (HeaderLine(string(I,J-I)) == false)
 376             return 2;
 377          I = J;
 378       }
 379       return 0;
 380    }
 381    while (Owner->Go(false,this) == true);
 382
 383    return 1;
 384 }
 385                                                                         /*}}}*/
 386 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 387 // ---------------------------------------------------------------------
 388 /* */
 389 bool ServerState::RunData()
 390 {
 391    State = Data;
 392
 393    // Chunked transfer encoding is fun..
 394    if (Encoding == Chunked)
 395    {
 396       while (1)
 397       {
 398          // Grab the block size
 399          bool Last = true;
 400          string Data;
 401          In.Limit(-1);
 402          do
 403          {
 404             if (In.WriteTillEl(Data,true) == true)
 405                break;
 406          }
 407          while ((Last = Owner->Go(false,this)) == true);
 408
 409          if (Last == false)
 410             return false;
 411
 412          // See if we are done
 413          unsigned long Len = strtol(Data.c_str(),0,16);
 414          if (Len == 0)
 415          {
 416             In.Limit(-1);
 417
 418             // We have to remove the entity trailer
 419             Last = true;
 420             do
 421             {
 422                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 423                   break;
 424             }
 425             while ((Last = Owner->Go(false,this)) == true);
 426             if (Last == false)
 427                return false;
 428             return !_error->PendingError();
 429          }
 430
 431          // Transfer the block
 432          In.Limit(Len);
 433          while (Owner->Go(true,this) == true)
 434             if (In.IsLimit() == true)
 435                break;
 436
 437          // Error
 438          if (In.IsLimit() == false)
 439             return false;
 440
 441          // The server sends an extra new line before the next block specifier..
 442          In.Limit(-1);
 443          Last = true;
 444          do
 445          {
 446             if (In.WriteTillEl(Data,true) == true)
 447                break;
 448          }
 449          while ((Last = Owner->Go(false,this)) == true);
 450          if (Last == false)
 451             return false;
 452       }
 453    }
 454    else
 455    {
 456       /* Closes encoding is used when the server did not specify a size, the
 457          loss of the connection means we are done */
 458       if (Encoding == Closes)
 459          In.Limit(-1);
 460       else
 461          In.Limit(Size - StartPos);
 462
 463       // Just transfer the whole block.
 464       do
 465       {
 466          if (In.IsLimit() == false)
 467             continue;
 468
 469          In.Limit(-1);
 470          return !_error->PendingError();
 471       }
 472       while (Owner->Go(true,this) == true);
 473    }
 474
 475    return Owner->Flush(this) && !_error->PendingError();
 476 }
 477                                                                         /*}}}*/
 478 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 479 // ---------------------------------------------------------------------
 480 /* */
 481 bool ServerState::HeaderLine(string Line)
 482 {
 483    if (Line.empty() == true)
 484       return true;
 485
 486    // The http server might be trying to do something evil.
 487    if (Line.length() >= MAXLEN)
 488       return _error->Error("Got a single header line over %u chars",MAXLEN);
 489
 490    string::size_type Pos = Line.find(' ');
 491    if (Pos == string::npos || Pos+1 > Line.length())
 492    {
 493       // Blah, some servers use "connection:closes", evil.
 494       Pos = Line.find(':');
 495       if (Pos == string::npos || Pos + 2 > Line.length())
 496          return _error->Error("Bad header line");
 497       Pos++;
 498    }
 499
 500    // Parse off any trailing spaces between the : and the next word.
 501    string::size_type Pos2 = Pos;
 502    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 503       Pos2++;
 504
 505    string Tag = string(Line,0,Pos);
 506    string Val = string(Line,Pos2);
 507
 508    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 509    {
 510       // Evil servers return no version
 511       if (Line[4] == '/')
 512       {
 513          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 514                     &Result,Code) != 4)
 515             return _error->Error("The http server sent an invalid reply header");
 516       }
 517       else
 518       {
 519          Major = 0;
 520          Minor = 9;
 521          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 522             return _error->Error("The http server sent an invalid reply header");
 523       }
 524
 525       return true;
 526    }
 527
 528    if (stringcasecmp(Tag,"Content-Length:") == 0)
 529    {
 530       if (Encoding == Closes)
 531          Encoding = Stream;
 532       HaveContent = true;
 533
 534       // The length is already set from the Content-Range header
 535       if (StartPos != 0)
 536          return true;
 537
 538       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 539          return _error->Error("The http server sent an invalid Content-Length header");
 540       return true;
 541    }
 542
 543    if (stringcasecmp(Tag,"Content-Type:") == 0)
 544    {
 545       HaveContent = true;
 546       return true;
 547    }
 548
 549    if (stringcasecmp(Tag,"Content-Range:") == 0)
 550    {
 551       HaveContent = true;
 552
 553       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 554          return _error->Error("The http server sent an invalid Content-Range header");
 555       if ((unsigned)StartPos > Size)
 556          return _error->Error("This http server has broken range support");
 557       return true;
 558    }
 559
 560    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 561    {
 562       HaveContent = true;
 563       if (stringcasecmp(Val,"chunked") == 0)
 564          Encoding = Chunked;
 565
 566       return true;
 567    }
 568
 569    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 570    {
 571       if (StrToTime(Val,Date) == false)
 572          return _error->Error("Unknown date format");
 573       return true;
 574    }
 575
 576    return true;
 577 }
 578                                                                         /*}}}*/
 579
 580 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 581 // ---------------------------------------------------------------------
 582 /* This places the http request in the outbound buffer */
 583 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 584 {
 585    URI Uri = Itm->Uri;
 586
 587    // The HTTP server expects a hostname with a trailing :port
 588    char Buf[1000];
 589    string ProperHost = Uri.Host;
 590    if (Uri.Port != 0)
 591    {
 592       sprintf(Buf,":%u",Uri.Port);
 593       ProperHost += Buf;
 594    }
 595
 596    // Just in case.
 597    if (Itm->Uri.length() >= sizeof(Buf))
 598        abort();
 599
 600    /* Build the request. We include a keep-alive header only for non-proxy
 601       requests. This is to tweak old http/1.0 servers that do support keep-alive
 602       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 603       will glitch HTTP/1.0 proxies because they do not filter it out and
 604       pass it on, HTTP/1.1 says the connection should default to keep alive
 605       and we expect the proxy to do this */
 606    if (Proxy.empty() == true)
 607       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 608               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 609    else
 610    {
 611       /* Generate a cache control header if necessary. We place a max
 612          cache age on index files, optionally set a no-cache directive
 613          and a no-store directive for archives. */
 614       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 615               Itm->Uri.c_str(),ProperHost.c_str());
 616       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 617          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 618       else
 619       {
 620          if (Itm->IndexFile == true)
 621             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 622                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 623          else
 624          {
 625             if (_config->FindB("Acquire::http::No-Store",false) == true)
 626                strcat(Buf,"Cache-Control: no-store\r\n");
 627          }
 628       }
 629    }
 630
 631    string Req = Buf;
 632
 633    // Check for a partial file
 634    struct stat SBuf;
 635    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 636    {
 637       // In this case we send an if-range query with a range header
 638       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 639               TimeRFC1123(SBuf.st_mtime).c_str());
 640       Req += Buf;
 641    }
 642    else
 643    {
 644       if (Itm->LastModified != 0)
 645       {
 646          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 647          Req += Buf;
 648       }
 649    }
 650
 651    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 652       Req += string("Proxy-Authorization: Basic ") +
 653           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 654
 655    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 656
 657    if (Debug == true)
 658       cerr << Req << endl;
 659
 660    Out.Read(Req);
 661 }
 662                                                                         /*}}}*/
 663 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 664 // ---------------------------------------------------------------------
 665 /* This runs the select loop over the server FDs, Output file FDs and
 666    stdin. */
 667 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 668 {
 669    // Server has closed the connection
 670    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 671                                ToFile == false))
 672       return false;
 673
 674    fd_set rfds,wfds,efds;
 675    FD_ZERO(&rfds);
 676    FD_ZERO(&wfds);
 677    FD_ZERO(&efds);
 678
 679    // Add the server
 680    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 681       FD_SET(Srv->ServerFd,&wfds);
 682    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 683       FD_SET(Srv->ServerFd,&rfds);
 684
 685    // Add the file
 686    int FileFD = -1;
 687    if (File != 0)
 688       FileFD = File->Fd();
 689
 690    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 691       FD_SET(FileFD,&wfds);
 692
 693    // Add stdin
 694    FD_SET(STDIN_FILENO,&rfds);
 695
 696    // Error Set
 697    if (FileFD != -1)
 698       FD_SET(FileFD,&efds);
 699    if (Srv->ServerFd != -1)
 700       FD_SET(Srv->ServerFd,&efds);
 701
 702    // Figure out the max fd
 703    int MaxFd = FileFD;
 704    if (MaxFd < Srv->ServerFd)
 705       MaxFd = Srv->ServerFd;
 706
 707    // Select
 708    struct timeval tv;
 709    tv.tv_sec = TimeOut;
 710    tv.tv_usec = 0;
 711    int Res = 0;
 712    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 713       return _error->Errno("select","Select failed");
 714
 715    if (Res == 0)
 716    {
 717       _error->Error("Connection timed out");
 718       return ServerDie(Srv);
 719    }
 720
 721    // Some kind of exception (error) on the sockets, die
 722    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 723        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 724       return _error->Error("Socket Exception");
 725
 726    // Handle server IO
 727    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 728    {
 729       errno = 0;
 730       if (Srv->In.Read(Srv->ServerFd) == false)
 731          return ServerDie(Srv);
 732    }
 733
 734    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 735    {
 736       errno = 0;
 737       if (Srv->Out.Write(Srv->ServerFd) == false)
 738          return ServerDie(Srv);
 739    }
 740
 741    // Send data to the file
 742    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 743    {
 744       if (Srv->In.Write(FileFD) == false)
 745          return _error->Errno("write","Error writing to output file");
 746    }
 747
 748    // Handle commands from APT
 749    if (FD_ISSET(STDIN_FILENO,&rfds))
 750    {
 751       if (Run(true) != -1)
 752          exit(100);
 753    }
 754
 755    return true;
 756 }
 757                                                                         /*}}}*/
 758 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 759 // ---------------------------------------------------------------------
 760 /* This takes the current input buffer from the Server FD and writes it
 761    into the file */
 762 bool HttpMethod::Flush(ServerState *Srv)
 763 {
 764    if (File != 0)
 765    {
 766       SetNonBlock(File->Fd(),false);
 767       if (Srv->In.WriteSpace() == false)
 768          return true;
 769
 770       while (Srv->In.WriteSpace() == true)
 771       {
 772          if (Srv->In.Write(File->Fd()) == false)
 773             return _error->Errno("write","Error writing to file");
 774          if (Srv->In.IsLimit() == true)
 775             return true;
 776       }
 777
 778       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 779          return true;
 780    }
 781    return false;
 782 }
 783                                                                         /*}}}*/
 784 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 785 // ---------------------------------------------------------------------
 786 /* */
 787 bool HttpMethod::ServerDie(ServerState *Srv)
 788 {
 789    unsigned int LErrno = errno;
 790
 791    // Dump the buffer to the file
 792    if (Srv->State == ServerState::Data)
 793    {
 794       SetNonBlock(File->Fd(),false);
 795       while (Srv->In.WriteSpace() == true)
 796       {
 797          if (Srv->In.Write(File->Fd()) == false)
 798             return _error->Errno("write","Error writing to the file");
 799
 800          // Done
 801          if (Srv->In.IsLimit() == true)
 802             return true;
 803       }
 804    }
 805
 806    // See if this is because the server finished the data stream
 807    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 808        Srv->Encoding != ServerState::Closes)
 809    {
 810       Srv->Close();
 811       if (LErrno == 0)
 812          return _error->Error("Error reading from server Remote end closed connection");
 813       errno = LErrno;
 814       return _error->Errno("read","Error reading from server");
 815    }
 816    else
 817    {
 818       Srv->In.Limit(-1);
 819
 820       // Nothing left in the buffer
 821       if (Srv->In.WriteSpace() == false)
 822          return false;
 823
 824       // We may have got multiple responses back in one packet..
 825       Srv->Close();
 826       return true;
 827    }
 828
 829    return false;
 830 }
 831                                                                         /*}}}*/
 832 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 833 // ---------------------------------------------------------------------
 834 /* We look at the header data we got back from the server and decide what
 835    to do. Returns
 836      0 - File is open,
 837      1 - IMS hit
 838      3 - Unrecoverable error
 839      4 - Error with error content page
 840      5 - Unrecoverable non-server error (close the connection) */
 841 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 842 {
 843    // Not Modified
 844    if (Srv->Result == 304)
 845    {
 846       unlink(Queue->DestFile.c_str());
 847       Res.IMSHit = true;
 848       Res.LastModified = Queue->LastModified;
 849       return 1;
 850    }
 851
 852    /* We have a reply we dont handle. This should indicate a perm server
 853       failure */
 854    if (Srv->Result < 200 || Srv->Result >= 300)
 855    {
 856       _error->Error("%u %s",Srv->Result,Srv->Code);
 857       if (Srv->HaveContent == true)
 858          return 4;
 859       return 3;
 860    }
 861
 862    // This is some sort of 2xx 'data follows' reply
 863    Res.LastModified = Srv->Date;
 864    Res.Size = Srv->Size;
 865
 866    // Open the file
 867    delete File;
 868    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 869    if (_error->PendingError() == true)
 870       return 5;
 871
 872    FailFile = Queue->DestFile;
 873    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 874    FailFd = File->Fd();
 875    FailTime = Srv->Date;
 876
 877    // Set the expected size
 878    if (Srv->StartPos >= 0)
 879    {
 880       Res.ResumePoint = Srv->StartPos;
 881       ftruncate(File->Fd(),Srv->StartPos);
 882    }
 883
 884    // Set the start point
 885    lseek(File->Fd(),0,SEEK_END);
 886
 887    delete Srv->In.MD5;
 888    Srv->In.MD5 = new MD5Summation;
 889
 890    // Fill the MD5 Hash if the file is non-empty (resume)
 891    if (Srv->StartPos > 0)
 892    {
 893       lseek(File->Fd(),0,SEEK_SET);
 894       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 895       {
 896          _error->Errno("read","Problem hashing file");
 897          return 5;
 898       }
 899       lseek(File->Fd(),0,SEEK_END);
 900    }
 901
 902    SetNonBlock(File->Fd(),true);
 903    return 0;
 904 }
 905                                                                         /*}}}*/
 906 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 907 // ---------------------------------------------------------------------
 908 /* This closes and timestamps the open file. This is neccessary to get
 909    resume behavoir on user abort */
 910 void HttpMethod::SigTerm(int)
 911 {
 912    if (FailFd == -1)
 913       exit(100);
 914    close(FailFd);
 915
 916    // Timestamp
 917    struct utimbuf UBuf;
 918    UBuf.actime = FailTime;
 919    UBuf.modtime = FailTime;
 920    utime(FailFile.c_str(),&UBuf);
 921
 922    exit(100);
 923 }
 924                                                                         /*}}}*/
 925 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 926 // ---------------------------------------------------------------------
 927 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 928    depth. */
 929 bool HttpMethod::Fetch(FetchItem *)
 930 {
 931    if (Server == 0)
 932       return true;
 933
 934    // Queue the requests
 935    int Depth = -1;
 936    bool Tail = false;
 937    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++)
 938    {
 939       // Make sure we stick with the same server
 940       if (Server->Comp(I->Uri) == false)
 941          break;
 942       if (QueueBack == I)
 943          Tail = true;
 944       if (Tail == true)
 945       {
 946          QueueBack = I->Next;
 947          SendReq(I,Server->Out);
 948          continue;
 949       }
 950    }
 951
 952    return true;
 953 };
 954                                                                         /*}}}*/
 955 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 956 // ---------------------------------------------------------------------
 957 /* We stash the desired pipeline depth */
 958 bool HttpMethod::Configuration(string Message)
 959 {
 960    if (pkgAcqMethod::Configuration(Message) == false)
 961       return false;
 962
 963    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 964    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 965                                   PipelineDepth);
 966    Debug = _config->FindB("Debug::Acquire::http",false);
 967
 968    return true;
 969 }
 970                                                                         /*}}}*/
 971 // HttpMethod::Loop - Main loop                                         /*{{{*/
 972 // ---------------------------------------------------------------------
 973 /* */
 974 int HttpMethod::Loop()
 975 {
 976    signal(SIGTERM,SigTerm);
 977    signal(SIGINT,SigTerm);
 978
 979    Server = 0;
 980
 981    int FailCounter = 0;
 982    while (1)
 983    {
 984       // We have no commands, wait for some to arrive
 985       if (Queue == 0)
 986       {
 987          if (WaitFd(STDIN_FILENO) == false)
 988             return 0;
 989       }
 990
 991       /* Run messages, we can accept 0 (no message) if we didn't
 992          do a WaitFd above.. Otherwise the FD is closed. */
 993       int Result = Run(true);
 994       if (Result != -1 && (Result != 0 || Queue == 0))
 995          return 100;
 996
 997       if (Queue == 0)
 998          continue;
 999
1000       // Connect to the server
1001       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1002       {
1003          delete Server;
1004          Server = new ServerState(Queue->Uri,this);
1005       }
1006
1007       // Reset the pipeline
1008       if (Server->ServerFd == -1)
1009          QueueBack = Queue;
1010
1011       // Connnect to the host
1012       if (Server->Open() == false)
1013       {
1014          Fail(true);
1015          delete Server;
1016          Server = 0;
1017          continue;
1018       }
1019
1020       // Fill the pipeline.
1021       Fetch(0);
1022
1023       // Fetch the next URL header data from the server.
1024       switch (Server->RunHeaders())
1025       {
1026          case 0:
1027          break;
1028
1029          // The header data is bad
1030          case 2:
1031          {
1032             _error->Error("Bad header Data");
1033             Fail(true);
1034             continue;
1035          }
1036
1037          // The server closed a connection during the header get..
1038          default:
1039          case 1:
1040          {
1041             FailCounter++;
1042             _error->Discard();
1043             Server->Close();
1044
1045             if (FailCounter >= 2)
1046             {
1047                Fail("Connection failed",true);
1048                FailCounter = 0;
1049             }
1050
1051             continue;
1052          }
1053       };
1054
1055       // Decide what to do.
1056       FetchResult Res;
1057       Res.Filename = Queue->DestFile;
1058       switch (DealWithHeaders(Res,Server))
1059       {
1060          // Ok, the file is Open
1061          case 0:
1062          {
1063             URIStart(Res);
1064
1065             // Run the data
1066             bool Result =  Server->RunData();
1067
1068             // Close the file, destroy the FD object and timestamp it
1069             FailFd = -1;
1070             delete File;
1071             File = 0;
1072
1073             // Timestamp
1074             struct utimbuf UBuf;
1075             time(&UBuf.actime);
1076             UBuf.actime = Server->Date;
1077             UBuf.modtime = Server->Date;
1078             utime(Queue->DestFile.c_str(),&UBuf);
1079
1080             // Send status to APT
1081             if (Result == true)
1082             {
1083                Res.MD5Sum = Server->In.MD5->Result();
1084                URIDone(Res);
1085             }
1086             else
1087                Fail(true);
1088
1089             break;
1090          }
1091
1092          // IMS hit
1093          case 1:
1094          {
1095             URIDone(Res);
1096             break;
1097          }
1098
1099          // Hard server error, not found or something
1100          case 3:
1101          {
1102             Fail();
1103             break;
1104          }
1105
1106          // Hard internal error, kill the connection and fail
1107          case 5:
1108          {
1109             Fail();
1110             Server->Close();
1111             break;
1112          }
1113
1114          // We need to flush the data, the header is like a 404 w/ error text
1115          case 4:
1116          {
1117             Fail();
1118
1119             // Send to content to dev/null
1120             File = new FileFd("/dev/null",FileFd::WriteExists);
1121             Server->RunData();
1122             delete File;
1123             File = 0;
1124             break;
1125          }
1126
1127          default:
1128          Fail("Internal error");
1129          break;
1130       }
1131
1132       FailCounter = 0;
1133    }
1134
1135    return 0;
1136 }
1137                                                                         /*}}}*/
1138
1139 int main()
1140 {
1141    HttpMethod Mth;
1142
1143    return Mth.Loop();
1144 }