methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.2 1998/11/01 08:07:13 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <stdio.h>
  40
  41 // Internet stuff
  42 #include <netinet/in.h>
  43 #include <sys/socket.h>
  44 #include <arpa/inet.h>
  45 #include <netdb.h>
  46
  47 #include "http.h"
  48                                                                         /*}}}*/
  49
  50 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  51 // ---------------------------------------------------------------------
  52 /* */
  53 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  54 {
  55    Buf = new unsigned char[Size];
  56    Reset();
  57 }
  58                                                                         /*}}}*/
  59 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  60 // ---------------------------------------------------------------------
  61 /* */
  62 void CircleBuf::Reset()
  63 {
  64    InP = 0;
  65    OutP = 0;
  66    StrPos = 0;
  67    MaxGet = (unsigned int)-1;
  68    OutQueue = string();
  69    if (MD5 != 0)
  70    {
  71       delete MD5;
  72       MD5 = new MD5Summation;
  73    }
  74 };
  75                                                                         /*}}}*/
  76 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  77 // ---------------------------------------------------------------------
  78 /* This fills up the buffer with as much data as is in the FD, assuming it
  79    is non-blocking.. */
  80 bool CircleBuf::Read(int Fd)
  81 {
  82    while (1)
  83    {
  84       // Woops, buffer is full
  85       if (InP - OutP == Size)
  86          return true;
  87
  88       // Write the buffer segment
  89       int Res;
  90       Res = read(Fd,Buf + (InP%Size),LeftRead());
  91
  92       if (Res == 0)
  93          return false;
  94       if (Res < 0)
  95       {
  96          if (errno == EAGAIN)
  97             return true;
  98          return false;
  99       }
 100
 101       if (InP == 0)
 102          gettimeofday(&Start,0);
 103       InP += Res;
 104    }
 105 }
 106                                                                         /*}}}*/
 107 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 108 // ---------------------------------------------------------------------
 109 /* This will hold the string in and fill the buffer with it as it empties */
 110 bool CircleBuf::Read(string Data)
 111 {
 112    OutQueue += Data;
 113    FillOut();
 114    return true;
 115 }
 116                                                                         /*}}}*/
 117 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 118 // ---------------------------------------------------------------------
 119 /* */
 120 void CircleBuf::FillOut()
 121 {
 122    if (OutQueue.empty() == true)
 123       return;
 124    while (1)
 125    {
 126       // Woops, buffer is full
 127       if (InP - OutP == Size)
 128          return;
 129
 130       // Write the buffer segment
 131       unsigned long Sz = LeftRead();
 132       if (OutQueue.length() - StrPos < Sz)
 133          Sz = OutQueue.length() - StrPos;
 134       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 135
 136       // Advance
 137       StrPos += Sz;
 138       InP += Sz;
 139       if (OutQueue.length() == StrPos)
 140       {
 141          StrPos = 0;
 142          OutQueue = "";
 143          return;
 144       }
 145    }
 146 }
 147                                                                         /*}}}*/
 148 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 149 // ---------------------------------------------------------------------
 150 /* This empties the buffer into the FD. */
 151 bool CircleBuf::Write(int Fd)
 152 {
 153    while (1)
 154    {
 155       FillOut();
 156
 157       // Woops, buffer is empty
 158       if (OutP == InP)
 159          return true;
 160
 161       if (OutP == MaxGet)
 162          return true;
 163
 164       // Write the buffer segment
 165       int Res;
 166       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 167
 168       if (Res == 0)
 169          return false;
 170       if (Res < 0)
 171       {
 172          if (errno == EAGAIN)
 173             return true;
 174
 175          return false;
 176       }
 177
 178       if (MD5 != 0)
 179          MD5->Add(Buf + (OutP%Size),Res);
 180
 181       OutP += Res;
 182    }
 183 }
 184                                                                         /*}}}*/
 185 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 186 // ---------------------------------------------------------------------
 187 /* This copies till the first empty line */
 188 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 189 {
 190    // We cheat and assume it is unneeded to have more than one buffer load
 191    for (unsigned long I = OutP; I < InP; I++)
 192    {
 193       if (Buf[I%Size] != '\n')
 194          continue;
 195       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 196
 197       if (Single == false)
 198       {
 199          if (Buf[I%Size] != '\n')
 200             continue;
 201          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 202       }
 203
 204       if (I > InP)
 205          I = InP;
 206
 207       Data = "";
 208       while (OutP < I)
 209       {
 210          unsigned long Sz = LeftWrite();
 211          if (Sz == 0)
 212             return false;
 213          if (I - OutP < LeftWrite())
 214             Sz = I - OutP;
 215          Data += string((char *)(Buf + (OutP%Size)),Sz);
 216          OutP += Sz;
 217       }
 218       return true;
 219    }
 220    return false;
 221 }
 222                                                                         /*}}}*/
 223 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 224 // ---------------------------------------------------------------------
 225 /* */
 226 void CircleBuf::Stats()
 227 {
 228    if (InP == 0)
 229       return;
 230
 231    struct timeval Stop;
 232    gettimeofday(&Stop,0);
 233 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 234              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 235    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 236 }
 237                                                                         /*}}}*/
 238
 239 // ServerState::ServerState - Constructor                               /*{{{*/
 240 // ---------------------------------------------------------------------
 241 /* */
 242 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 243                         In(64*1024), Out(1*1024),
 244                         ServerName(Srv)
 245 {
 246    Reset();
 247 }
 248                                                                         /*}}}*/
 249 // ServerState::Open - Open a connection to the server                  /*{{{*/
 250 // ---------------------------------------------------------------------
 251 /* This opens a connection to the server. */
 252 string LastHost;
 253 in_addr LastHostA;
 254 bool ServerState::Open()
 255 {
 256    // Use the already open connection if possible.
 257    if (ServerFd != -1)
 258       return true;
 259
 260    Close();
 261
 262    int Port = 80;
 263    string Host;
 264
 265    if (Proxy.empty() == true)
 266    {
 267       if (ServerName.Port != 0)
 268          Port = ServerName.Port;
 269       Host = ServerName.Host;
 270    }
 271    else
 272    {
 273       if (Proxy.Port != 0)
 274          Port = Proxy.Port;
 275       Host = Proxy.Host;
 276    }
 277
 278    if (LastHost != Host)
 279    {
 280       Owner->Status("Connecting to %s",Host.c_str());
 281
 282       // Lookup the host
 283       hostent *Addr = gethostbyname(Host.c_str());
 284       if (Addr == 0)
 285          return _error->Error("Could not resolve '%s'",Host.c_str());
 286       LastHost = Host;
 287       LastHostA = *(in_addr *)(Addr->h_addr_list[0]);
 288    }
 289
 290    Owner->Status("Connecting to %s (%s)",Host.c_str(),inet_ntoa(LastHostA));
 291
 292    // Get a socket
 293    if ((ServerFd = socket(AF_INET,SOCK_STREAM,0)) < 0)
 294       return _error->Errno("socket","Could not create a socket");
 295
 296    // Connect to the server
 297    struct sockaddr_in server;
 298    server.sin_family = AF_INET;
 299    server.sin_port = htons(Port);
 300    server.sin_addr = LastHostA;
 301    if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0)
 302       return _error->Errno("socket","Could not create a socket");
 303
 304    SetNonBlock(ServerFd,true);
 305    return true;
 306 }
 307                                                                         /*}}}*/
 308 // ServerState::Close - Close a connection to the server                /*{{{*/
 309 // ---------------------------------------------------------------------
 310 /* */
 311 bool ServerState::Close()
 312 {
 313    close(ServerFd);
 314    ServerFd = -1;
 315    In.Reset();
 316    Out.Reset();
 317    return true;
 318 }
 319                                                                         /*}}}*/
 320 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 321 // ---------------------------------------------------------------------
 322 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 323    parse error occured */
 324 int ServerState::RunHeaders()
 325 {
 326    State = Header;
 327
 328    Owner->Status("Waiting for file");
 329
 330    Major = 0;
 331    Minor = 0;
 332    Result = 0;
 333    Size = 0;
 334    StartPos = 0;
 335    Encoding = Closes;
 336    HaveContent = false;
 337    time(&Date);
 338
 339    do
 340    {
 341       string Data;
 342       if (In.WriteTillEl(Data) == false)
 343          continue;
 344
 345       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 346       {
 347          string::const_iterator J = I;
 348          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 349          if (HeaderLine(string(I,J-I)) == false)
 350             return 2;
 351          I = J;
 352       }
 353       return 0;
 354    }
 355    while (Owner->Go(false,this) == true);
 356
 357    return 1;
 358 }
 359                                                                         /*}}}*/
 360 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 361 // ---------------------------------------------------------------------
 362 /* */
 363 bool ServerState::RunData()
 364 {
 365    State = Data;
 366
 367    // Chunked transfer encoding is fun..
 368    if (Encoding == Chunked)
 369    {
 370       while (1)
 371       {
 372          // Grab the block size
 373          bool Last = true;
 374          string Data;
 375          In.Limit(-1);
 376          do
 377          {
 378             if (In.WriteTillEl(Data,true) == true)
 379                break;
 380          }
 381          while ((Last = Owner->Go(false,this)) == true);
 382
 383          if (Last == false)
 384             return false;
 385
 386          // See if we are done
 387          unsigned long Len = strtol(Data.c_str(),0,16);
 388          if (Len == 0)
 389          {
 390             In.Limit(-1);
 391
 392             // We have to remove the entity trailer
 393             Last = true;
 394             do
 395             {
 396                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 397                   break;
 398             }
 399             while ((Last = Owner->Go(false,this)) == true);
 400             if (Last == false)
 401                return false;
 402             return true;
 403          }
 404
 405          // Transfer the block
 406          In.Limit(Len);
 407          while (Owner->Go(true,this) == true)
 408             if (In.IsLimit() == true)
 409                break;
 410
 411          // Error
 412          if (In.IsLimit() == false)
 413             return false;
 414
 415          // The server sends an extra new line before the next block specifier..
 416          In.Limit(-1);
 417          Last = true;
 418          do
 419          {
 420             if (In.WriteTillEl(Data,true) == true)
 421                break;
 422          }
 423          while ((Last = Owner->Go(false,this)) == true);
 424          if (Last == false)
 425             return false;
 426       }
 427    }
 428    else
 429    {
 430       /* Closes encoding is used when the server did not specify a size, the
 431          loss of the connection means we are done */
 432       if (Encoding == Closes)
 433          In.Limit(-1);
 434       else
 435          In.Limit(Size - StartPos);
 436
 437       // Just transfer the whole block.
 438       do
 439       {
 440          if (In.IsLimit() == false)
 441             continue;
 442
 443          In.Limit(-1);
 444          return true;
 445       }
 446       while (Owner->Go(true,this) == true);
 447    }
 448
 449    return Owner->Flush(this);
 450 }
 451                                                                         /*}}}*/
 452 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 453 // ---------------------------------------------------------------------
 454 /* */
 455 bool ServerState::HeaderLine(string Line)
 456 {
 457    if (Line.empty() == true)
 458       return true;
 459
 460    // The http server might be trying to do something evil.
 461    if (Line.length() >= MAXLEN)
 462       return _error->Error("Got a single header line over %u chars",MAXLEN);
 463
 464    string::size_type Pos = Line.find(' ');
 465    if (Pos == string::npos || Pos+1 > Line.length())
 466       return _error->Error("Bad header line");
 467
 468    string Tag = string(Line,0,Pos);
 469    string Val = string(Line,Pos+1);
 470
 471    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 472    {
 473       // Evil servers return no version
 474       if (Line[4] == '/')
 475       {
 476          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 477                     &Result,Code) != 4)
 478             return _error->Error("The http server sent an invalid reply header");
 479       }
 480       else
 481       {
 482          Major = 0;
 483          Minor = 9;
 484          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 485             return _error->Error("The http server sent an invalid reply header");
 486       }
 487
 488       return true;
 489    }
 490
 491    if (stringcasecmp(Tag,"Content-Length:") == 0)
 492    {
 493       if (Encoding == Closes)
 494          Encoding = Stream;
 495       HaveContent = true;
 496
 497       // The length is already set from the Content-Range header
 498       if (StartPos != 0)
 499          return true;
 500
 501       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 502          return _error->Error("The http server sent an invalid Content-Length header");
 503       return true;
 504    }
 505
 506    if (stringcasecmp(Tag,"Content-Type:") == 0)
 507    {
 508       HaveContent = true;
 509       return true;
 510    }
 511
 512    if (stringcasecmp(Tag,"Content-Range:") == 0)
 513    {
 514       HaveContent = true;
 515
 516       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 517          return _error->Error("The http server sent an invalid Content-Range header");
 518       if ((unsigned)StartPos > Size)
 519          return _error->Error("This http server has broken range support");
 520       return true;
 521    }
 522
 523    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 524    {
 525       HaveContent = true;
 526       if (stringcasecmp(Val,"chunked") == 0)
 527          Encoding = Chunked;
 528
 529       return true;
 530    }
 531
 532    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 533    {
 534       if (StrToTime(Val,Date) == false)
 535          return _error->Error("Unknown date format");
 536       return true;
 537    }
 538
 539    return true;
 540 }
 541                                                                         /*}}}*/
 542
 543 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 544 // ---------------------------------------------------------------------
 545 /* This places the http request in the outbound buffer */
 546 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 547 {
 548    URI Uri = Itm->Uri;
 549
 550    // The HTTP server expects a hostname with a trailing :port
 551    char Buf[300];
 552    string ProperHost = Uri.Host;
 553    if (Uri.Port != 0)
 554    {
 555       sprintf(Buf,":%u",Uri.Port);
 556       ProperHost += Buf;
 557    }
 558
 559    // Build the request
 560    if (Proxy.empty() == true)
 561       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 562               Uri.Path.c_str(),ProperHost.c_str());
 563    else
 564       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 565               Itm->Uri.c_str(),ProperHost.c_str());
 566    string Req = Buf;
 567
 568    // Check for a partial file
 569    struct stat SBuf;
 570    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 571    {
 572       // In this case we send an if-range query with a range header
 573       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 574               TimeRFC1123(SBuf.st_mtime).c_str());
 575       Req += Buf;
 576    }
 577    else
 578    {
 579       if (Itm->LastModified != 0)
 580       {
 581          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 582          Req += Buf;
 583       }
 584    }
 585
 586 /*   if (ProxyAuth.empty() == false)
 587       Req += string("Proxy-Authorization: Basic ") + Base64Encode(ProxyAuth) + "\r\n";*/
 588
 589    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 590 //   cout << Req << endl;
 591
 592    Out.Read(Req);
 593 }
 594                                                                         /*}}}*/
 595 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 596 // ---------------------------------------------------------------------
 597 /* This runs the select loop over the server FDs, Output file FDs and
 598    stdin. */
 599 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 600 {
 601    // Server has closed the connection
 602    if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
 603       return false;
 604
 605    fd_set rfds,wfds,efds;
 606    FD_ZERO(&rfds);
 607    FD_ZERO(&wfds);
 608    FD_ZERO(&efds);
 609
 610    // Add the server
 611    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 612       FD_SET(Srv->ServerFd,&wfds);
 613    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 614       FD_SET(Srv->ServerFd,&rfds);
 615
 616    // Add the file
 617    int FileFD = -1;
 618    if (File != 0)
 619       FileFD = File->Fd();
 620
 621    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 622       FD_SET(FileFD,&wfds);
 623
 624    // Add stdin
 625    FD_SET(STDIN_FILENO,&rfds);
 626
 627    // Error Set
 628    if (FileFD != -1)
 629       FD_SET(FileFD,&efds);
 630    if (Srv->ServerFd != -1)
 631       FD_SET(Srv->ServerFd,&efds);
 632
 633    // Figure out the max fd
 634    int MaxFd = FileFD;
 635    if (MaxFd < Srv->ServerFd)
 636       MaxFd = Srv->ServerFd;
 637
 638    // Select
 639    struct timeval tv;
 640    tv.tv_sec = 120;
 641    tv.tv_usec = 0;
 642    int Res = 0;
 643    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 644       return _error->Errno("select","Select failed");
 645
 646    if (Res == 0)
 647    {
 648       _error->Error("Connection timed out");
 649       return ServerDie(Srv);
 650    }
 651
 652    // Some kind of exception (error) on the sockets, die
 653    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 654        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 655       return _error->Error("Socket Exception");
 656
 657    // Handle server IO
 658    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 659    {
 660       errno = 0;
 661       if (Srv->In.Read(Srv->ServerFd) == false)
 662          return ServerDie(Srv);
 663    }
 664
 665    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 666    {
 667       errno = 0;
 668       if (Srv->Out.Write(Srv->ServerFd) == false)
 669          return ServerDie(Srv);
 670    }
 671
 672    // Send data to the file
 673    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 674    {
 675       if (Srv->In.Write(FileFD) == false)
 676          return _error->Errno("write","Error writing to output file");
 677    }
 678
 679    // Handle commands from APT
 680    if (FD_ISSET(STDIN_FILENO,&rfds))
 681    {
 682       if (Run(true) != 0)
 683          exit(100);
 684    }
 685
 686    return true;
 687 }
 688                                                                         /*}}}*/
 689 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 690 // ---------------------------------------------------------------------
 691 /* This takes the current input buffer from the Server FD and writes it
 692    into the file */
 693 bool HttpMethod::Flush(ServerState *Srv)
 694 {
 695    if (File != 0)
 696    {
 697       SetNonBlock(File->Fd(),false);
 698       if (Srv->In.WriteSpace() == false)
 699          return true;
 700
 701       while (Srv->In.WriteSpace() == true)
 702       {
 703          if (Srv->In.Write(File->Fd()) == false)
 704             return _error->Errno("write","Error writing to file");
 705          if (Srv->In.IsLimit() == true)
 706             return true;
 707       }
 708
 709       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 710          return true;
 711    }
 712    return false;
 713 }
 714                                                                         /*}}}*/
 715 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 716 // ---------------------------------------------------------------------
 717 /* */
 718 bool HttpMethod::ServerDie(ServerState *Srv)
 719 {
 720    // Dump the buffer to the file
 721    if (Srv->State == ServerState::Data)
 722    {
 723       SetNonBlock(File->Fd(),false);
 724       while (Srv->In.WriteSpace() == true)
 725       {
 726          if (Srv->In.Write(File->Fd()) == false)
 727             return _error->Errno("write","Error writing to the file");
 728
 729          // Done
 730          if (Srv->In.IsLimit() == true)
 731             return true;
 732       }
 733    }
 734
 735    // See if this is because the server finished the data stream
 736    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 737        Srv->Encoding != ServerState::Closes)
 738    {
 739       if (errno == 0)
 740          return _error->Error("Error reading from server Remote end closed connection");
 741       return _error->Errno("read","Error reading from server");
 742    }
 743    else
 744    {
 745       Srv->In.Limit(-1);
 746
 747       // Nothing left in the buffer
 748       if (Srv->In.WriteSpace() == false)
 749          return false;
 750
 751       // We may have got multiple responses back in one packet..
 752       Srv->Close();
 753       return true;
 754    }
 755
 756    return false;
 757 }
 758                                                                         /*}}}*/
 759 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 760 // ---------------------------------------------------------------------
 761 /* We look at the header data we got back from the server and decide what
 762    to do. Returns
 763      0 - File is open,
 764      1 - IMS hit
 765      3 - Unrecoverable error
 766      4 - Error with error content page */
 767 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 768 {
 769    // Not Modified
 770    if (Srv->Result == 304)
 771    {
 772       unlink(Queue->DestFile.c_str());
 773       Res.IMSHit = true;
 774       Res.LastModified = Queue->LastModified;
 775       return 1;
 776    }
 777
 778    /* We have a reply we dont handle. This should indicate a perm server
 779       failure */
 780    if (Srv->Result < 200 || Srv->Result >= 300)
 781    {
 782       _error->Error("%u %s",Srv->Result,Srv->Code);
 783       if (Srv->HaveContent == true)
 784          return 4;
 785       return 3;
 786    }
 787
 788    // This is some sort of 2xx 'data follows' reply
 789    Res.LastModified = Srv->Date;
 790    Res.Size = Srv->Size;
 791
 792    // Open the file
 793    delete File;
 794    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 795    if (_error->PendingError() == true)
 796       return 3;
 797
 798    // Set the expected size
 799    if (Srv->StartPos >= 0)
 800    {
 801       Res.ResumePoint = Srv->StartPos;
 802       ftruncate(File->Fd(),Srv->StartPos);
 803    }
 804
 805    // Set the start point
 806    lseek(File->Fd(),0,SEEK_END);
 807
 808    delete Srv->In.MD5;
 809    Srv->In.MD5 = new MD5Summation;
 810
 811    // Fill the MD5 Hash if the file is non-empty (resume)
 812    if (Srv->StartPos > 0)
 813    {
 814       lseek(File->Fd(),0,SEEK_SET);
 815       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 816       {
 817          _error->Errno("read","Problem hashing file");
 818          return 3;
 819       }
 820       lseek(File->Fd(),0,SEEK_END);
 821    }
 822
 823    SetNonBlock(File->Fd(),true);
 824    return 0;
 825 }
 826                                                                         /*}}}*/
 827 // HttpMethod::Loop                                                     /*{{{*/
 828 // ---------------------------------------------------------------------
 829 /* */
 830 int HttpMethod::Loop()
 831 {
 832    ServerState *Server = 0;
 833
 834    int FailCounter = 0;
 835    while (1)
 836    {
 837       if (FailCounter >= 2)
 838       {
 839          Fail("Massive Server Brain Damage");
 840          FailCounter = 0;
 841       }
 842
 843       // We have no commands, wait for some to arrive
 844       if (Queue == 0)
 845       {
 846          if (WaitFd(STDIN_FILENO) == false)
 847             return 0;
 848       }
 849
 850       // Run messages
 851       if (Run(true) != 0)
 852          return 100;
 853
 854       if (Queue == 0)
 855          continue;
 856
 857       // Connect to the server
 858       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 859       {
 860          delete Server;
 861          Server = new ServerState(Queue->Uri,this);
 862       }
 863
 864       // Connnect to the host
 865       if (Server->Open() == false)
 866       {
 867          Fail();
 868          continue;
 869       }
 870
 871       // Queue the request
 872       SendReq(Queue,Server->Out);
 873
 874       // Fetch the next URL header data from the server.
 875       switch (Server->RunHeaders())
 876       {
 877          case 0:
 878          break;
 879
 880          // The header data is bad
 881          case 2:
 882          {
 883             _error->Error("Bad header Data");
 884             Fail();
 885             continue;
 886          }
 887
 888          // The server closed a connection during the header get..
 889          default:
 890          case 1:
 891          {
 892             FailCounter++;
 893             _error->DumpErrors();
 894             Server->Close();
 895             continue;
 896          }
 897       };
 898
 899       // Decide what to do.
 900       FetchResult Res;
 901       switch (DealWithHeaders(Res,Server))
 902       {
 903          // Ok, the file is Open
 904          case 0:
 905          {
 906             URIStart(Res);
 907
 908             // Run the data
 909             if (Server->RunData() == false)
 910                Fail();
 911             else
 912             {
 913                Res.MD5Sum = Server->In.MD5->Result();
 914                URIDone(Res);
 915             }
 916
 917             delete File;
 918             File = 0;
 919             break;
 920          }
 921
 922          // IMS hit
 923          case 1:
 924          {
 925             URIDone(Res);
 926             break;
 927          }
 928
 929          // Hard server error, not found or something
 930          case 3:
 931          {
 932             Fail();
 933             break;
 934          }
 935
 936          // We need to flush the data, the header is like a 404 w/ error text
 937          case 4:
 938          {
 939             Fail();
 940
 941             // Send to content to dev/null
 942             File = new FileFd("/dev/null",FileFd::WriteExists);
 943             Server->RunData();
 944             delete File;
 945             File = 0;
 946             break;
 947          }
 948
 949          default:
 950          Fail("Internal error");
 951          break;
 952       }
 953
 954       FailCounter = 0;
 955    }
 956
 957    return 0;
 958 }
 959                                                                         /*}}}*/
 960
 961 int main()
 962 {
 963    HttpMethod Mth;
 964
 965    return Mth.Loop();
 966 }