methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.11 1998/12/07 07:32:03 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41
  42 // Internet stuff
  43 #include <netinet/in.h>
  44 #include <sys/socket.h>
  45 #include <arpa/inet.h>
  46 #include <netdb.h>
  47
  48 #include "http.h"
  49                                                                         /*}}}*/
  50
  51 string HttpMethod::FailFile;
  52 int HttpMethod::FailFd = -1;
  53 time_t HttpMethod::FailTime = 0;
  54 unsigned long PipelineDepth = 5;
  55
  56 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  57 // ---------------------------------------------------------------------
  58 /* */
  59 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  60 {
  61    Buf = new unsigned char[Size];
  62    Reset();
  63 }
  64                                                                         /*}}}*/
  65 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  66 // ---------------------------------------------------------------------
  67 /* */
  68 void CircleBuf::Reset()
  69 {
  70    InP = 0;
  71    OutP = 0;
  72    StrPos = 0;
  73    MaxGet = (unsigned int)-1;
  74    OutQueue = string();
  75    if (MD5 != 0)
  76    {
  77       delete MD5;
  78       MD5 = new MD5Summation;
  79    }
  80 };
  81                                                                         /*}}}*/
  82 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  83 // ---------------------------------------------------------------------
  84 /* This fills up the buffer with as much data as is in the FD, assuming it
  85    is non-blocking.. */
  86 bool CircleBuf::Read(int Fd)
  87 {
  88    while (1)
  89    {
  90       // Woops, buffer is full
  91       if (InP - OutP == Size)
  92          return true;
  93
  94       // Write the buffer segment
  95       int Res;
  96       Res = read(Fd,Buf + (InP%Size),LeftRead());
  97
  98       if (Res == 0)
  99          return false;
 100       if (Res < 0)
 101       {
 102          if (errno == EAGAIN)
 103             return true;
 104          return false;
 105       }
 106
 107       if (InP == 0)
 108          gettimeofday(&Start,0);
 109       InP += Res;
 110    }
 111 }
 112                                                                         /*}}}*/
 113 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 114 // ---------------------------------------------------------------------
 115 /* This will hold the string in and fill the buffer with it as it empties */
 116 bool CircleBuf::Read(string Data)
 117 {
 118    OutQueue += Data;
 119    FillOut();
 120    return true;
 121 }
 122                                                                         /*}}}*/
 123 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 124 // ---------------------------------------------------------------------
 125 /* */
 126 void CircleBuf::FillOut()
 127 {
 128    if (OutQueue.empty() == true)
 129       return;
 130    while (1)
 131    {
 132       // Woops, buffer is full
 133       if (InP - OutP == Size)
 134          return;
 135
 136       // Write the buffer segment
 137       unsigned long Sz = LeftRead();
 138       if (OutQueue.length() - StrPos < Sz)
 139          Sz = OutQueue.length() - StrPos;
 140       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 141
 142       // Advance
 143       StrPos += Sz;
 144       InP += Sz;
 145       if (OutQueue.length() == StrPos)
 146       {
 147          StrPos = 0;
 148          OutQueue = "";
 149          return;
 150       }
 151    }
 152 }
 153                                                                         /*}}}*/
 154 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 155 // ---------------------------------------------------------------------
 156 /* This empties the buffer into the FD. */
 157 bool CircleBuf::Write(int Fd)
 158 {
 159    while (1)
 160    {
 161       FillOut();
 162
 163       // Woops, buffer is empty
 164       if (OutP == InP)
 165          return true;
 166
 167       if (OutP == MaxGet)
 168          return true;
 169
 170       // Write the buffer segment
 171       int Res;
 172       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 173
 174       if (Res == 0)
 175          return false;
 176       if (Res < 0)
 177       {
 178          if (errno == EAGAIN)
 179             return true;
 180
 181          return false;
 182       }
 183
 184       if (MD5 != 0)
 185          MD5->Add(Buf + (OutP%Size),Res);
 186
 187       OutP += Res;
 188    }
 189 }
 190                                                                         /*}}}*/
 191 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 192 // ---------------------------------------------------------------------
 193 /* This copies till the first empty line */
 194 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 195 {
 196    // We cheat and assume it is unneeded to have more than one buffer load
 197    for (unsigned long I = OutP; I < InP; I++)
 198    {
 199       if (Buf[I%Size] != '\n')
 200          continue;
 201       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 202
 203       if (Single == false)
 204       {
 205          if (Buf[I%Size] != '\n')
 206             continue;
 207          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 208       }
 209
 210       if (I > InP)
 211          I = InP;
 212
 213       Data = "";
 214       while (OutP < I)
 215       {
 216          unsigned long Sz = LeftWrite();
 217          if (Sz == 0)
 218             return false;
 219          if (I - OutP < LeftWrite())
 220             Sz = I - OutP;
 221          Data += string((char *)(Buf + (OutP%Size)),Sz);
 222          OutP += Sz;
 223       }
 224       return true;
 225    }
 226    return false;
 227 }
 228                                                                         /*}}}*/
 229 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 230 // ---------------------------------------------------------------------
 231 /* */
 232 void CircleBuf::Stats()
 233 {
 234    if (InP == 0)
 235       return;
 236
 237    struct timeval Stop;
 238    gettimeofday(&Stop,0);
 239 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 240              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 241    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 242 }
 243                                                                         /*}}}*/
 244
 245 // ServerState::ServerState - Constructor                               /*{{{*/
 246 // ---------------------------------------------------------------------
 247 /* */
 248 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 249                         In(64*1024), Out(1*1024),
 250                         ServerName(Srv)
 251 {
 252    Reset();
 253 }
 254                                                                         /*}}}*/
 255 // ServerState::Open - Open a connection to the server                  /*{{{*/
 256 // ---------------------------------------------------------------------
 257 /* This opens a connection to the server. */
 258 string LastHost;
 259 in_addr LastHostA;
 260 bool ServerState::Open()
 261 {
 262    // Use the already open connection if possible.
 263    if (ServerFd != -1)
 264       return true;
 265
 266    Close();
 267    In.Reset();
 268    Out.Reset();
 269
 270    // Determine the proxy setting
 271    if (getenv("http_proxy") == 0)
 272    {
 273       string DefProxy = _config->Find("Acquire::http::Proxy");
 274       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 275       if (SpecificProxy.empty() == false)
 276       {
 277          if (SpecificProxy == "DIRECT")
 278             Proxy = "";
 279          else
 280             Proxy = SpecificProxy;
 281       }
 282       else
 283          Proxy = DefProxy;
 284    }
 285    else
 286       Proxy = getenv("http_proxy");
 287
 288    // Determine what host and port to use based on the proxy settings
 289    int Port = 80;
 290    string Host;
 291    if (Proxy.empty() == true)
 292    {
 293       if (ServerName.Port != 0)
 294          Port = ServerName.Port;
 295       Host = ServerName.Host;
 296    }
 297    else
 298    {
 299       if (Proxy.Port != 0)
 300          Port = Proxy.Port;
 301       Host = Proxy.Host;
 302    }
 303
 304    /* We used a cached address record.. Yes this is against the spec but
 305       the way we have setup our rotating dns suggests that this is more
 306       sensible */
 307    if (LastHost != Host)
 308    {
 309       Owner->Status("Connecting to %s",Host.c_str());
 310
 311       // Lookup the host
 312       hostent *Addr = gethostbyname(Host.c_str());
 313       if (Addr == 0)
 314          return _error->Error("Could not resolve '%s'",Host.c_str());
 315       LastHost = Host;
 316       LastHostA = *(in_addr *)(Addr->h_addr_list[0]);
 317    }
 318
 319    Owner->Status("Connecting to %s (%s)",Host.c_str(),inet_ntoa(LastHostA));
 320
 321    // Get a socket
 322    if ((ServerFd = socket(AF_INET,SOCK_STREAM,0)) < 0)
 323       return _error->Errno("socket","Could not create a socket");
 324
 325    // Connect to the server
 326    struct sockaddr_in server;
 327    server.sin_family = AF_INET;
 328    server.sin_port = htons(Port);
 329    server.sin_addr = LastHostA;
 330    if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0)
 331       return _error->Errno("socket","Could not create a socket");
 332
 333    SetNonBlock(ServerFd,true);
 334    return true;
 335 }
 336                                                                         /*}}}*/
 337 // ServerState::Close - Close a connection to the server                /*{{{*/
 338 // ---------------------------------------------------------------------
 339 /* */
 340 bool ServerState::Close()
 341 {
 342    close(ServerFd);
 343    ServerFd = -1;
 344    return true;
 345 }
 346                                                                         /*}}}*/
 347 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 348 // ---------------------------------------------------------------------
 349 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 350    parse error occured */
 351 int ServerState::RunHeaders()
 352 {
 353    State = Header;
 354
 355    Owner->Status("Waiting for file");
 356
 357    Major = 0;
 358    Minor = 0;
 359    Result = 0;
 360    Size = 0;
 361    StartPos = 0;
 362    Encoding = Closes;
 363    HaveContent = false;
 364    time(&Date);
 365
 366    do
 367    {
 368       string Data;
 369       if (In.WriteTillEl(Data) == false)
 370          continue;
 371
 372       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 373       {
 374          string::const_iterator J = I;
 375          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 376          if (HeaderLine(string(I,J-I)) == false)
 377             return 2;
 378          I = J;
 379       }
 380       return 0;
 381    }
 382    while (Owner->Go(false,this) == true);
 383
 384    return 1;
 385 }
 386                                                                         /*}}}*/
 387 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 388 // ---------------------------------------------------------------------
 389 /* */
 390 bool ServerState::RunData()
 391 {
 392    State = Data;
 393
 394    // Chunked transfer encoding is fun..
 395    if (Encoding == Chunked)
 396    {
 397       while (1)
 398       {
 399          // Grab the block size
 400          bool Last = true;
 401          string Data;
 402          In.Limit(-1);
 403          do
 404          {
 405             if (In.WriteTillEl(Data,true) == true)
 406                break;
 407          }
 408          while ((Last = Owner->Go(false,this)) == true);
 409
 410          if (Last == false)
 411             return false;
 412
 413          // See if we are done
 414          unsigned long Len = strtol(Data.c_str(),0,16);
 415          if (Len == 0)
 416          {
 417             In.Limit(-1);
 418
 419             // We have to remove the entity trailer
 420             Last = true;
 421             do
 422             {
 423                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 424                   break;
 425             }
 426             while ((Last = Owner->Go(false,this)) == true);
 427             if (Last == false)
 428                return false;
 429             return true;
 430          }
 431
 432          // Transfer the block
 433          In.Limit(Len);
 434          while (Owner->Go(true,this) == true)
 435             if (In.IsLimit() == true)
 436                break;
 437
 438          // Error
 439          if (In.IsLimit() == false)
 440             return false;
 441
 442          // The server sends an extra new line before the next block specifier..
 443          In.Limit(-1);
 444          Last = true;
 445          do
 446          {
 447             if (In.WriteTillEl(Data,true) == true)
 448                break;
 449          }
 450          while ((Last = Owner->Go(false,this)) == true);
 451          if (Last == false)
 452             return false;
 453       }
 454    }
 455    else
 456    {
 457       /* Closes encoding is used when the server did not specify a size, the
 458          loss of the connection means we are done */
 459       if (Encoding == Closes)
 460          In.Limit(-1);
 461       else
 462          In.Limit(Size - StartPos);
 463
 464       // Just transfer the whole block.
 465       do
 466       {
 467          if (In.IsLimit() == false)
 468             continue;
 469
 470          In.Limit(-1);
 471          return true;
 472       }
 473       while (Owner->Go(true,this) == true);
 474    }
 475
 476    return Owner->Flush(this);
 477 }
 478                                                                         /*}}}*/
 479 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 480 // ---------------------------------------------------------------------
 481 /* */
 482 bool ServerState::HeaderLine(string Line)
 483 {
 484    if (Line.empty() == true)
 485       return true;
 486
 487    // The http server might be trying to do something evil.
 488    if (Line.length() >= MAXLEN)
 489       return _error->Error("Got a single header line over %u chars",MAXLEN);
 490
 491    string::size_type Pos = Line.find(' ');
 492    if (Pos == string::npos || Pos+1 > Line.length())
 493       return _error->Error("Bad header line");
 494
 495    string Tag = string(Line,0,Pos);
 496    string Val = string(Line,Pos+1);
 497
 498    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 499    {
 500       // Evil servers return no version
 501       if (Line[4] == '/')
 502       {
 503          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 504                     &Result,Code) != 4)
 505             return _error->Error("The http server sent an invalid reply header");
 506       }
 507       else
 508       {
 509          Major = 0;
 510          Minor = 9;
 511          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 512             return _error->Error("The http server sent an invalid reply header");
 513       }
 514
 515       return true;
 516    }
 517
 518    if (stringcasecmp(Tag,"Content-Length:") == 0)
 519    {
 520       if (Encoding == Closes)
 521          Encoding = Stream;
 522       HaveContent = true;
 523
 524       // The length is already set from the Content-Range header
 525       if (StartPos != 0)
 526          return true;
 527
 528       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 529          return _error->Error("The http server sent an invalid Content-Length header");
 530       return true;
 531    }
 532
 533    if (stringcasecmp(Tag,"Content-Type:") == 0)
 534    {
 535       HaveContent = true;
 536       return true;
 537    }
 538
 539    if (stringcasecmp(Tag,"Content-Range:") == 0)
 540    {
 541       HaveContent = true;
 542
 543       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 544          return _error->Error("The http server sent an invalid Content-Range header");
 545       if ((unsigned)StartPos > Size)
 546          return _error->Error("This http server has broken range support");
 547       return true;
 548    }
 549
 550    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 551    {
 552       HaveContent = true;
 553       if (stringcasecmp(Val,"chunked") == 0)
 554          Encoding = Chunked;
 555
 556       return true;
 557    }
 558
 559    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 560    {
 561       if (StrToTime(Val,Date) == false)
 562          return _error->Error("Unknown date format");
 563       return true;
 564    }
 565
 566    return true;
 567 }
 568                                                                         /*}}}*/
 569
 570 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 571 // ---------------------------------------------------------------------
 572 /* This places the http request in the outbound buffer */
 573 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 574 {
 575    URI Uri = Itm->Uri;
 576
 577    // The HTTP server expects a hostname with a trailing :port
 578    char Buf[1000];
 579    string ProperHost = Uri.Host;
 580    if (Uri.Port != 0)
 581    {
 582       sprintf(Buf,":%u",Uri.Port);
 583       ProperHost += Buf;
 584    }
 585
 586    // Just in case.
 587    if (Itm->Uri.length() >= sizeof(Buf))
 588        abort();
 589
 590    /* Build the request. We include a keep-alive header only for non-proxy
 591       requests. This is to tweak old http/1.0 servers that do support keep-alive
 592       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 593       will glitch HTTP/1.0 proxies because they do not filter it out and
 594       pass it on, HTTP/1.1 says the connection should default to keep alive
 595       and we expect the proxy to do this */
 596    if (Proxy.empty() == true)
 597       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 598               Uri.Path.c_str(),ProperHost.c_str());
 599    else
 600    {
 601       /* Generate a cache control header if necessary. We place a max
 602          cache age on index files, optionally set a no-cache directive
 603          and a no-store directive for archives. */
 604       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 605               Itm->Uri.c_str(),ProperHost.c_str());
 606       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 607          strcat(Buf,"Cache-Control: no-cache\r\n");
 608       else
 609       {
 610          if (Itm->IndexFile == true)
 611             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 612                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 613          else
 614          {
 615             if (_config->FindB("Acquire::http::No-Store",false) == true)
 616                strcat(Buf,"Cache-Control: no-store\r\n");
 617          }
 618       }
 619    }
 620
 621    string Req = Buf;
 622
 623    // Check for a partial file
 624    struct stat SBuf;
 625    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 626    {
 627       // In this case we send an if-range query with a range header
 628       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 629               TimeRFC1123(SBuf.st_mtime).c_str());
 630       Req += Buf;
 631    }
 632    else
 633    {
 634       if (Itm->LastModified != 0)
 635       {
 636          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 637          Req += Buf;
 638       }
 639    }
 640
 641 /*   if (ProxyAuth.empty() == false)
 642       Req += string("Proxy-Authorization: Basic ") + Base64Encode(ProxyAuth) + "\r\n";*/
 643
 644    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 645 //   cerr << Req << endl;
 646
 647    Out.Read(Req);
 648 }
 649                                                                         /*}}}*/
 650 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 651 // ---------------------------------------------------------------------
 652 /* This runs the select loop over the server FDs, Output file FDs and
 653    stdin. */
 654 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 655 {
 656    // Server has closed the connection
 657    if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
 658       return false;
 659
 660    fd_set rfds,wfds,efds;
 661    FD_ZERO(&rfds);
 662    FD_ZERO(&wfds);
 663    FD_ZERO(&efds);
 664
 665    // Add the server
 666    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 667       FD_SET(Srv->ServerFd,&wfds);
 668    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 669       FD_SET(Srv->ServerFd,&rfds);
 670
 671    // Add the file
 672    int FileFD = -1;
 673    if (File != 0)
 674       FileFD = File->Fd();
 675
 676    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 677       FD_SET(FileFD,&wfds);
 678
 679    // Add stdin
 680    FD_SET(STDIN_FILENO,&rfds);
 681
 682    // Error Set
 683    if (FileFD != -1)
 684       FD_SET(FileFD,&efds);
 685    if (Srv->ServerFd != -1)
 686       FD_SET(Srv->ServerFd,&efds);
 687
 688    // Figure out the max fd
 689    int MaxFd = FileFD;
 690    if (MaxFd < Srv->ServerFd)
 691       MaxFd = Srv->ServerFd;
 692
 693    // Select
 694    struct timeval tv;
 695    tv.tv_sec = 120;
 696    tv.tv_usec = 0;
 697    int Res = 0;
 698    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 699       return _error->Errno("select","Select failed");
 700
 701    if (Res == 0)
 702    {
 703       _error->Error("Connection timed out");
 704       return ServerDie(Srv);
 705    }
 706
 707    // Some kind of exception (error) on the sockets, die
 708    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 709        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 710       return _error->Error("Socket Exception");
 711
 712    // Handle server IO
 713    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 714    {
 715       errno = 0;
 716       if (Srv->In.Read(Srv->ServerFd) == false)
 717          return ServerDie(Srv);
 718    }
 719
 720    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 721    {
 722       errno = 0;
 723       if (Srv->Out.Write(Srv->ServerFd) == false)
 724          return ServerDie(Srv);
 725    }
 726
 727    // Send data to the file
 728    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 729    {
 730       if (Srv->In.Write(FileFD) == false)
 731          return _error->Errno("write","Error writing to output file");
 732    }
 733
 734    // Handle commands from APT
 735    if (FD_ISSET(STDIN_FILENO,&rfds))
 736    {
 737       if (Run(true) != 0)
 738          exit(100);
 739    }
 740
 741    return true;
 742 }
 743                                                                         /*}}}*/
 744 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 745 // ---------------------------------------------------------------------
 746 /* This takes the current input buffer from the Server FD and writes it
 747    into the file */
 748 bool HttpMethod::Flush(ServerState *Srv)
 749 {
 750    if (File != 0)
 751    {
 752       SetNonBlock(File->Fd(),false);
 753       if (Srv->In.WriteSpace() == false)
 754          return true;
 755
 756       while (Srv->In.WriteSpace() == true)
 757       {
 758          if (Srv->In.Write(File->Fd()) == false)
 759             return _error->Errno("write","Error writing to file");
 760          if (Srv->In.IsLimit() == true)
 761             return true;
 762       }
 763
 764       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 765          return true;
 766    }
 767    return false;
 768 }
 769                                                                         /*}}}*/
 770 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 771 // ---------------------------------------------------------------------
 772 /* */
 773 bool HttpMethod::ServerDie(ServerState *Srv)
 774 {
 775    // Dump the buffer to the file
 776    if (Srv->State == ServerState::Data)
 777    {
 778       SetNonBlock(File->Fd(),false);
 779       while (Srv->In.WriteSpace() == true)
 780       {
 781          if (Srv->In.Write(File->Fd()) == false)
 782             return _error->Errno("write","Error writing to the file");
 783
 784          // Done
 785          if (Srv->In.IsLimit() == true)
 786             return true;
 787       }
 788    }
 789
 790    // See if this is because the server finished the data stream
 791    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 792        Srv->Encoding != ServerState::Closes)
 793    {
 794       if (errno == 0)
 795          return _error->Error("Error reading from server Remote end closed connection");
 796       return _error->Errno("read","Error reading from server");
 797    }
 798    else
 799    {
 800       Srv->In.Limit(-1);
 801
 802       // Nothing left in the buffer
 803       if (Srv->In.WriteSpace() == false)
 804          return false;
 805
 806       // We may have got multiple responses back in one packet..
 807       Srv->Close();
 808       return true;
 809    }
 810
 811    return false;
 812 }
 813                                                                         /*}}}*/
 814 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 815 // ---------------------------------------------------------------------
 816 /* We look at the header data we got back from the server and decide what
 817    to do. Returns
 818      0 - File is open,
 819      1 - IMS hit
 820      3 - Unrecoverable error
 821      4 - Error with error content page
 822      5 - Unrecoverable non-server error (close the connection) */
 823 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 824 {
 825    // Not Modified
 826    if (Srv->Result == 304)
 827    {
 828       unlink(Queue->DestFile.c_str());
 829       Res.IMSHit = true;
 830       Res.LastModified = Queue->LastModified;
 831       return 1;
 832    }
 833
 834    /* We have a reply we dont handle. This should indicate a perm server
 835       failure */
 836    if (Srv->Result < 200 || Srv->Result >= 300)
 837    {
 838       _error->Error("%u %s",Srv->Result,Srv->Code);
 839       if (Srv->HaveContent == true)
 840          return 4;
 841       return 3;
 842    }
 843
 844    // This is some sort of 2xx 'data follows' reply
 845    Res.LastModified = Srv->Date;
 846    Res.Size = Srv->Size;
 847
 848    // Open the file
 849    delete File;
 850    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 851    if (_error->PendingError() == true)
 852       return 5;
 853
 854    FailFile = Queue->DestFile;
 855    FailFd = File->Fd();
 856    FailTime = Srv->Date;
 857
 858    // Set the expected size
 859    if (Srv->StartPos >= 0)
 860    {
 861       Res.ResumePoint = Srv->StartPos;
 862       ftruncate(File->Fd(),Srv->StartPos);
 863    }
 864
 865    // Set the start point
 866    lseek(File->Fd(),0,SEEK_END);
 867
 868    delete Srv->In.MD5;
 869    Srv->In.MD5 = new MD5Summation;
 870
 871    // Fill the MD5 Hash if the file is non-empty (resume)
 872    if (Srv->StartPos > 0)
 873    {
 874       lseek(File->Fd(),0,SEEK_SET);
 875       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 876       {
 877          _error->Errno("read","Problem hashing file");
 878          return 5;
 879       }
 880       lseek(File->Fd(),0,SEEK_END);
 881    }
 882
 883    SetNonBlock(File->Fd(),true);
 884    return 0;
 885 }
 886                                                                         /*}}}*/
 887 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 888 // ---------------------------------------------------------------------
 889 /* This closes and timestamps the open file. This is neccessary to get
 890    resume behavoir on user abort */
 891 void HttpMethod::SigTerm(int)
 892 {
 893    if (FailFd == -1)
 894       exit(100);
 895    close(FailFd);
 896
 897    // Timestamp
 898    struct utimbuf UBuf;
 899    time(&UBuf.actime);
 900    UBuf.actime = FailTime;
 901    UBuf.modtime = FailTime;
 902    utime(FailFile.c_str(),&UBuf);
 903
 904    exit(100);
 905 }
 906                                                                         /*}}}*/
 907 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 908 // ---------------------------------------------------------------------
 909 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 910    depth. */
 911 bool HttpMethod::Fetch(FetchItem *)
 912 {
 913    if (Server == 0)
 914       return true;
 915
 916    // Queue the requests
 917    int Depth = -1;
 918    bool Tail = false;
 919    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++)
 920    {
 921       // Make sure we stick with the same server
 922       if (Server->Comp(I->Uri) == false)
 923          break;
 924
 925       if (QueueBack == I)
 926          Tail = true;
 927       if (Tail == true)
 928       {
 929          Depth++;
 930          QueueBack = I->Next;
 931          SendReq(I,Server->Out);
 932          continue;
 933       }
 934    }
 935
 936    return true;
 937 };
 938                                                                         /*}}}*/
 939 // HttpMethod::Loop - Main loop                                         /*{{{*/
 940 // ---------------------------------------------------------------------
 941 /* */
 942 int HttpMethod::Loop()
 943 {
 944    signal(SIGTERM,SigTerm);
 945    signal(SIGINT,SigTerm);
 946
 947    Server = 0;
 948
 949    int FailCounter = 0;
 950    while (1)
 951    {
 952       if (FailCounter >= 2)
 953       {
 954          Fail("Massive Server Brain Damage");
 955          FailCounter = 0;
 956       }
 957
 958       // We have no commands, wait for some to arrive
 959       if (Queue == 0)
 960       {
 961          if (WaitFd(STDIN_FILENO) == false)
 962             return 0;
 963       }
 964
 965       // Run messages
 966       if (Run(true) != 0)
 967          return 100;
 968
 969       if (Queue == 0)
 970          continue;
 971
 972       // Connect to the server
 973       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 974       {
 975          delete Server;
 976          Server = new ServerState(Queue->Uri,this);
 977       }
 978
 979       // Connnect to the host
 980       if (Server->Open() == false)
 981       {
 982          Fail();
 983          continue;
 984       }
 985
 986       // Fill the pipeline.
 987       Fetch(0);
 988
 989       // Fetch the next URL header data from the server.
 990       switch (Server->RunHeaders())
 991       {
 992          case 0:
 993          break;
 994
 995          // The header data is bad
 996          case 2:
 997          {
 998             _error->Error("Bad header Data");
 999             Fail();
1000             continue;
1001          }
1002
1003          // The server closed a connection during the header get..
1004          default:
1005          case 1:
1006          {
1007             FailCounter++;
1008             _error->DumpErrors();
1009             Server->Close();
1010             continue;
1011          }
1012       };
1013
1014       // Decide what to do.
1015       FetchResult Res;
1016       Res.Filename = Queue->DestFile;
1017       switch (DealWithHeaders(Res,Server))
1018       {
1019          // Ok, the file is Open
1020          case 0:
1021          {
1022             URIStart(Res);
1023
1024             // Run the data
1025             bool Result =  Server->RunData();
1026
1027             // Close the file, destroy the FD object and timestamp it
1028             FailFd = -1;
1029             delete File;
1030             File = 0;
1031
1032             // Timestamp
1033             struct utimbuf UBuf;
1034             time(&UBuf.actime);
1035             UBuf.actime = Server->Date;
1036             UBuf.modtime = Server->Date;
1037             utime(Queue->DestFile.c_str(),&UBuf);
1038
1039             // Send status to APT
1040             if (Result == true)
1041             {
1042                Res.MD5Sum = Server->In.MD5->Result();
1043                URIDone(Res);
1044             }
1045             else
1046                Fail();
1047
1048             break;
1049          }
1050
1051          // IMS hit
1052          case 1:
1053          {
1054             URIDone(Res);
1055             break;
1056          }
1057
1058          // Hard server error, not found or something
1059          case 3:
1060          {
1061             Fail();
1062             break;
1063          }
1064
1065          // Hard internal error, kill the connection and fail
1066          case 5:
1067          {
1068             Fail();
1069             Server->Close();
1070             break;
1071          }
1072
1073          // We need to flush the data, the header is like a 404 w/ error text
1074          case 4:
1075          {
1076             Fail();
1077
1078             // Send to content to dev/null
1079             File = new FileFd("/dev/null",FileFd::WriteExists);
1080             Server->RunData();
1081             delete File;
1082             File = 0;
1083             break;
1084          }
1085
1086          default:
1087          Fail("Internal error");
1088          break;
1089       }
1090
1091       FailCounter = 0;
1092    }
1093
1094    return 0;
1095 }
1096                                                                         /*}}}*/
1097
1098 int main()
1099 {
1100    HttpMethod Mth;
1101
1102    return Mth.Loop();
1103 }