methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.7 1998/11/23 21:28:43 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41
  42 // Internet stuff
  43 #include <netinet/in.h>
  44 #include <sys/socket.h>
  45 #include <arpa/inet.h>
  46 #include <netdb.h>
  47
  48 #include "http.h"
  49                                                                         /*}}}*/
  50
  51 string HttpMethod::FailFile;
  52 int HttpMethod::FailFd = -1;
  53 time_t HttpMethod::FailTime = 0;
  54
  55 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* */
  58 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  59 {
  60    Buf = new unsigned char[Size];
  61    Reset();
  62 }
  63                                                                         /*}}}*/
  64 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* */
  67 void CircleBuf::Reset()
  68 {
  69    InP = 0;
  70    OutP = 0;
  71    StrPos = 0;
  72    MaxGet = (unsigned int)-1;
  73    OutQueue = string();
  74    if (MD5 != 0)
  75    {
  76       delete MD5;
  77       MD5 = new MD5Summation;
  78    }
  79 };
  80                                                                         /*}}}*/
  81 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  82 // ---------------------------------------------------------------------
  83 /* This fills up the buffer with as much data as is in the FD, assuming it
  84    is non-blocking.. */
  85 bool CircleBuf::Read(int Fd)
  86 {
  87    while (1)
  88    {
  89       // Woops, buffer is full
  90       if (InP - OutP == Size)
  91          return true;
  92
  93       // Write the buffer segment
  94       int Res;
  95       Res = read(Fd,Buf + (InP%Size),LeftRead());
  96
  97       if (Res == 0)
  98          return false;
  99       if (Res < 0)
 100       {
 101          if (errno == EAGAIN)
 102             return true;
 103          return false;
 104       }
 105
 106       if (InP == 0)
 107          gettimeofday(&Start,0);
 108       InP += Res;
 109    }
 110 }
 111                                                                         /*}}}*/
 112 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 113 // ---------------------------------------------------------------------
 114 /* This will hold the string in and fill the buffer with it as it empties */
 115 bool CircleBuf::Read(string Data)
 116 {
 117    OutQueue += Data;
 118    FillOut();
 119    return true;
 120 }
 121                                                                         /*}}}*/
 122 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 123 // ---------------------------------------------------------------------
 124 /* */
 125 void CircleBuf::FillOut()
 126 {
 127    if (OutQueue.empty() == true)
 128       return;
 129    while (1)
 130    {
 131       // Woops, buffer is full
 132       if (InP - OutP == Size)
 133          return;
 134
 135       // Write the buffer segment
 136       unsigned long Sz = LeftRead();
 137       if (OutQueue.length() - StrPos < Sz)
 138          Sz = OutQueue.length() - StrPos;
 139       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 140
 141       // Advance
 142       StrPos += Sz;
 143       InP += Sz;
 144       if (OutQueue.length() == StrPos)
 145       {
 146          StrPos = 0;
 147          OutQueue = "";
 148          return;
 149       }
 150    }
 151 }
 152                                                                         /*}}}*/
 153 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 154 // ---------------------------------------------------------------------
 155 /* This empties the buffer into the FD. */
 156 bool CircleBuf::Write(int Fd)
 157 {
 158    while (1)
 159    {
 160       FillOut();
 161
 162       // Woops, buffer is empty
 163       if (OutP == InP)
 164          return true;
 165
 166       if (OutP == MaxGet)
 167          return true;
 168
 169       // Write the buffer segment
 170       int Res;
 171       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 172
 173       if (Res == 0)
 174          return false;
 175       if (Res < 0)
 176       {
 177          if (errno == EAGAIN)
 178             return true;
 179
 180          return false;
 181       }
 182
 183       if (MD5 != 0)
 184          MD5->Add(Buf + (OutP%Size),Res);
 185
 186       OutP += Res;
 187    }
 188 }
 189                                                                         /*}}}*/
 190 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 191 // ---------------------------------------------------------------------
 192 /* This copies till the first empty line */
 193 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 194 {
 195    // We cheat and assume it is unneeded to have more than one buffer load
 196    for (unsigned long I = OutP; I < InP; I++)
 197    {
 198       if (Buf[I%Size] != '\n')
 199          continue;
 200       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 201
 202       if (Single == false)
 203       {
 204          if (Buf[I%Size] != '\n')
 205             continue;
 206          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 207       }
 208
 209       if (I > InP)
 210          I = InP;
 211
 212       Data = "";
 213       while (OutP < I)
 214       {
 215          unsigned long Sz = LeftWrite();
 216          if (Sz == 0)
 217             return false;
 218          if (I - OutP < LeftWrite())
 219             Sz = I - OutP;
 220          Data += string((char *)(Buf + (OutP%Size)),Sz);
 221          OutP += Sz;
 222       }
 223       return true;
 224    }
 225    return false;
 226 }
 227                                                                         /*}}}*/
 228 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 229 // ---------------------------------------------------------------------
 230 /* */
 231 void CircleBuf::Stats()
 232 {
 233    if (InP == 0)
 234       return;
 235
 236    struct timeval Stop;
 237    gettimeofday(&Stop,0);
 238 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 239              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 240    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 241 }
 242                                                                         /*}}}*/
 243
 244 // ServerState::ServerState - Constructor                               /*{{{*/
 245 // ---------------------------------------------------------------------
 246 /* */
 247 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 248                         In(64*1024), Out(1*1024),
 249                         ServerName(Srv)
 250 {
 251    Reset();
 252 }
 253                                                                         /*}}}*/
 254 // ServerState::Open - Open a connection to the server                  /*{{{*/
 255 // ---------------------------------------------------------------------
 256 /* This opens a connection to the server. */
 257 string LastHost;
 258 in_addr LastHostA;
 259 bool ServerState::Open()
 260 {
 261    // Use the already open connection if possible.
 262    if (ServerFd != -1)
 263       return true;
 264
 265    Close();
 266    In.Reset();
 267    Out.Reset();
 268
 269    // Determine the proxy setting
 270    if (getenv("http_proxy") != 0)
 271    {
 272       string DefProxy = _config->Find("Acquire::http::Proxy");
 273       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 274       if (SpecificProxy.empty() == false)
 275       {
 276          if (SpecificProxy == "DIRECT")
 277             Proxy = "";
 278          else
 279             Proxy = SpecificProxy;
 280       }
 281       else
 282          Proxy = DefProxy;
 283    }
 284    else
 285       Proxy = getenv("http_proxy");
 286
 287    // Determine what host and port to use based on the proxy settings
 288    int Port = 80;
 289    string Host;
 290    if (Proxy.empty() == true)
 291    {
 292       if (ServerName.Port != 0)
 293          Port = ServerName.Port;
 294       Host = ServerName.Host;
 295    }
 296    else
 297    {
 298       if (Proxy.Port != 0)
 299          Port = Proxy.Port;
 300       Host = Proxy.Host;
 301    }
 302
 303    /* We used a cached address record.. Yes this is against the spec but
 304       the way we have setup our rotating dns suggests that this is more
 305       sensible */
 306    if (LastHost != Host)
 307    {
 308       Owner->Status("Connecting to %s",Host.c_str());
 309
 310       // Lookup the host
 311       hostent *Addr = gethostbyname(Host.c_str());
 312       if (Addr == 0)
 313          return _error->Error("Could not resolve '%s'",Host.c_str());
 314       LastHost = Host;
 315       LastHostA = *(in_addr *)(Addr->h_addr_list[0]);
 316    }
 317
 318    Owner->Status("Connecting to %s (%s)",Host.c_str(),inet_ntoa(LastHostA));
 319
 320    // Get a socket
 321    if ((ServerFd = socket(AF_INET,SOCK_STREAM,0)) < 0)
 322       return _error->Errno("socket","Could not create a socket");
 323
 324    // Connect to the server
 325    struct sockaddr_in server;
 326    server.sin_family = AF_INET;
 327    server.sin_port = htons(Port);
 328    server.sin_addr = LastHostA;
 329    if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0)
 330       return _error->Errno("socket","Could not create a socket");
 331
 332    SetNonBlock(ServerFd,true);
 333    return true;
 334 }
 335                                                                         /*}}}*/
 336 // ServerState::Close - Close a connection to the server                /*{{{*/
 337 // ---------------------------------------------------------------------
 338 /* */
 339 bool ServerState::Close()
 340 {
 341    close(ServerFd);
 342    ServerFd = -1;
 343    return true;
 344 }
 345                                                                         /*}}}*/
 346 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 347 // ---------------------------------------------------------------------
 348 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 349    parse error occured */
 350 int ServerState::RunHeaders()
 351 {
 352    State = Header;
 353
 354    Owner->Status("Waiting for file");
 355
 356    Major = 0;
 357    Minor = 0;
 358    Result = 0;
 359    Size = 0;
 360    StartPos = 0;
 361    Encoding = Closes;
 362    HaveContent = false;
 363    time(&Date);
 364
 365    do
 366    {
 367       string Data;
 368       if (In.WriteTillEl(Data) == false)
 369          continue;
 370
 371       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 372       {
 373          string::const_iterator J = I;
 374          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 375          if (HeaderLine(string(I,J-I)) == false)
 376             return 2;
 377          I = J;
 378       }
 379       return 0;
 380    }
 381    while (Owner->Go(false,this) == true);
 382
 383    return 1;
 384 }
 385                                                                         /*}}}*/
 386 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 387 // ---------------------------------------------------------------------
 388 /* */
 389 bool ServerState::RunData()
 390 {
 391    State = Data;
 392
 393    // Chunked transfer encoding is fun..
 394    if (Encoding == Chunked)
 395    {
 396       while (1)
 397       {
 398          // Grab the block size
 399          bool Last = true;
 400          string Data;
 401          In.Limit(-1);
 402          do
 403          {
 404             if (In.WriteTillEl(Data,true) == true)
 405                break;
 406          }
 407          while ((Last = Owner->Go(false,this)) == true);
 408
 409          if (Last == false)
 410             return false;
 411
 412          // See if we are done
 413          unsigned long Len = strtol(Data.c_str(),0,16);
 414          if (Len == 0)
 415          {
 416             In.Limit(-1);
 417
 418             // We have to remove the entity trailer
 419             Last = true;
 420             do
 421             {
 422                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 423                   break;
 424             }
 425             while ((Last = Owner->Go(false,this)) == true);
 426             if (Last == false)
 427                return false;
 428             return true;
 429          }
 430
 431          // Transfer the block
 432          In.Limit(Len);
 433          while (Owner->Go(true,this) == true)
 434             if (In.IsLimit() == true)
 435                break;
 436
 437          // Error
 438          if (In.IsLimit() == false)
 439             return false;
 440
 441          // The server sends an extra new line before the next block specifier..
 442          In.Limit(-1);
 443          Last = true;
 444          do
 445          {
 446             if (In.WriteTillEl(Data,true) == true)
 447                break;
 448          }
 449          while ((Last = Owner->Go(false,this)) == true);
 450          if (Last == false)
 451             return false;
 452       }
 453    }
 454    else
 455    {
 456       /* Closes encoding is used when the server did not specify a size, the
 457          loss of the connection means we are done */
 458       if (Encoding == Closes)
 459          In.Limit(-1);
 460       else
 461          In.Limit(Size - StartPos);
 462
 463       // Just transfer the whole block.
 464       do
 465       {
 466          if (In.IsLimit() == false)
 467             continue;
 468
 469          In.Limit(-1);
 470          return true;
 471       }
 472       while (Owner->Go(true,this) == true);
 473    }
 474
 475    return Owner->Flush(this);
 476 }
 477                                                                         /*}}}*/
 478 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 479 // ---------------------------------------------------------------------
 480 /* */
 481 bool ServerState::HeaderLine(string Line)
 482 {
 483    if (Line.empty() == true)
 484       return true;
 485
 486    // The http server might be trying to do something evil.
 487    if (Line.length() >= MAXLEN)
 488       return _error->Error("Got a single header line over %u chars",MAXLEN);
 489
 490    string::size_type Pos = Line.find(' ');
 491    if (Pos == string::npos || Pos+1 > Line.length())
 492       return _error->Error("Bad header line");
 493
 494    string Tag = string(Line,0,Pos);
 495    string Val = string(Line,Pos+1);
 496
 497    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 498    {
 499       // Evil servers return no version
 500       if (Line[4] == '/')
 501       {
 502          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 503                     &Result,Code) != 4)
 504             return _error->Error("The http server sent an invalid reply header");
 505       }
 506       else
 507       {
 508          Major = 0;
 509          Minor = 9;
 510          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 511             return _error->Error("The http server sent an invalid reply header");
 512       }
 513
 514       return true;
 515    }
 516
 517    if (stringcasecmp(Tag,"Content-Length:") == 0)
 518    {
 519       if (Encoding == Closes)
 520          Encoding = Stream;
 521       HaveContent = true;
 522
 523       // The length is already set from the Content-Range header
 524       if (StartPos != 0)
 525          return true;
 526
 527       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 528          return _error->Error("The http server sent an invalid Content-Length header");
 529       return true;
 530    }
 531
 532    if (stringcasecmp(Tag,"Content-Type:") == 0)
 533    {
 534       HaveContent = true;
 535       return true;
 536    }
 537
 538    if (stringcasecmp(Tag,"Content-Range:") == 0)
 539    {
 540       HaveContent = true;
 541
 542       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 543          return _error->Error("The http server sent an invalid Content-Range header");
 544       if ((unsigned)StartPos > Size)
 545          return _error->Error("This http server has broken range support");
 546       return true;
 547    }
 548
 549    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 550    {
 551       HaveContent = true;
 552       if (stringcasecmp(Val,"chunked") == 0)
 553          Encoding = Chunked;
 554
 555       return true;
 556    }
 557
 558    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 559    {
 560       if (StrToTime(Val,Date) == false)
 561          return _error->Error("Unknown date format");
 562       return true;
 563    }
 564
 565    return true;
 566 }
 567                                                                         /*}}}*/
 568
 569 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 570 // ---------------------------------------------------------------------
 571 /* This places the http request in the outbound buffer */
 572 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 573 {
 574    URI Uri = Itm->Uri;
 575
 576    // The HTTP server expects a hostname with a trailing :port
 577    char Buf[300];
 578    string ProperHost = Uri.Host;
 579    if (Uri.Port != 0)
 580    {
 581       sprintf(Buf,":%u",Uri.Port);
 582       ProperHost += Buf;
 583    }
 584
 585    /* Build the request. We include a keep-alive header only for non-proxy
 586       requests. This is to tweak old http/1.0 servers that do support keep-alive
 587       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 588       will glitch HTTP/1.0 proxies because they do not filter it out and
 589       pass it on, HTTP/1.1 says the connection should default to keep alive
 590       and we expect the proxy to do this */
 591    if (Proxy.empty() == true)
 592       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 593               Uri.Path.c_str(),ProperHost.c_str());
 594    else
 595       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 596               Itm->Uri.c_str(),ProperHost.c_str());
 597    string Req = Buf;
 598
 599    // Check for a partial file
 600    struct stat SBuf;
 601    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 602    {
 603       // In this case we send an if-range query with a range header
 604       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 605               TimeRFC1123(SBuf.st_mtime).c_str());
 606       Req += Buf;
 607    }
 608    else
 609    {
 610       if (Itm->LastModified != 0)
 611       {
 612          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 613          Req += Buf;
 614       }
 615    }
 616
 617 /*   if (ProxyAuth.empty() == false)
 618       Req += string("Proxy-Authorization: Basic ") + Base64Encode(ProxyAuth) + "\r\n";*/
 619
 620    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 621 //   cerr << Req << endl;
 622
 623    Out.Read(Req);
 624 }
 625                                                                         /*}}}*/
 626 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 627 // ---------------------------------------------------------------------
 628 /* This runs the select loop over the server FDs, Output file FDs and
 629    stdin. */
 630 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 631 {
 632    // Server has closed the connection
 633    if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
 634       return false;
 635
 636    fd_set rfds,wfds,efds;
 637    FD_ZERO(&rfds);
 638    FD_ZERO(&wfds);
 639    FD_ZERO(&efds);
 640
 641    // Add the server
 642    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 643       FD_SET(Srv->ServerFd,&wfds);
 644    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 645       FD_SET(Srv->ServerFd,&rfds);
 646
 647    // Add the file
 648    int FileFD = -1;
 649    if (File != 0)
 650       FileFD = File->Fd();
 651
 652    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 653       FD_SET(FileFD,&wfds);
 654
 655    // Add stdin
 656    FD_SET(STDIN_FILENO,&rfds);
 657
 658    // Error Set
 659    if (FileFD != -1)
 660       FD_SET(FileFD,&efds);
 661    if (Srv->ServerFd != -1)
 662       FD_SET(Srv->ServerFd,&efds);
 663
 664    // Figure out the max fd
 665    int MaxFd = FileFD;
 666    if (MaxFd < Srv->ServerFd)
 667       MaxFd = Srv->ServerFd;
 668
 669    // Select
 670    struct timeval tv;
 671    tv.tv_sec = 120;
 672    tv.tv_usec = 0;
 673    int Res = 0;
 674    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 675       return _error->Errno("select","Select failed");
 676
 677    if (Res == 0)
 678    {
 679       _error->Error("Connection timed out");
 680       return ServerDie(Srv);
 681    }
 682
 683    // Some kind of exception (error) on the sockets, die
 684    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 685        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 686       return _error->Error("Socket Exception");
 687
 688    // Handle server IO
 689    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 690    {
 691       errno = 0;
 692       if (Srv->In.Read(Srv->ServerFd) == false)
 693          return ServerDie(Srv);
 694    }
 695
 696    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 697    {
 698       errno = 0;
 699       if (Srv->Out.Write(Srv->ServerFd) == false)
 700          return ServerDie(Srv);
 701    }
 702
 703    // Send data to the file
 704    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 705    {
 706       if (Srv->In.Write(FileFD) == false)
 707          return _error->Errno("write","Error writing to output file");
 708    }
 709
 710    // Handle commands from APT
 711    if (FD_ISSET(STDIN_FILENO,&rfds))
 712    {
 713       if (Run(true) != 0)
 714          exit(100);
 715    }
 716
 717    return true;
 718 }
 719                                                                         /*}}}*/
 720 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 721 // ---------------------------------------------------------------------
 722 /* This takes the current input buffer from the Server FD and writes it
 723    into the file */
 724 bool HttpMethod::Flush(ServerState *Srv)
 725 {
 726    if (File != 0)
 727    {
 728       SetNonBlock(File->Fd(),false);
 729       if (Srv->In.WriteSpace() == false)
 730          return true;
 731
 732       while (Srv->In.WriteSpace() == true)
 733       {
 734          if (Srv->In.Write(File->Fd()) == false)
 735             return _error->Errno("write","Error writing to file");
 736          if (Srv->In.IsLimit() == true)
 737             return true;
 738       }
 739
 740       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 741          return true;
 742    }
 743    return false;
 744 }
 745                                                                         /*}}}*/
 746 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 747 // ---------------------------------------------------------------------
 748 /* */
 749 bool HttpMethod::ServerDie(ServerState *Srv)
 750 {
 751    // Dump the buffer to the file
 752    if (Srv->State == ServerState::Data)
 753    {
 754       SetNonBlock(File->Fd(),false);
 755       while (Srv->In.WriteSpace() == true)
 756       {
 757          if (Srv->In.Write(File->Fd()) == false)
 758             return _error->Errno("write","Error writing to the file");
 759
 760          // Done
 761          if (Srv->In.IsLimit() == true)
 762             return true;
 763       }
 764    }
 765
 766    // See if this is because the server finished the data stream
 767    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 768        Srv->Encoding != ServerState::Closes)
 769    {
 770       if (errno == 0)
 771          return _error->Error("Error reading from server Remote end closed connection");
 772       return _error->Errno("read","Error reading from server");
 773    }
 774    else
 775    {
 776       Srv->In.Limit(-1);
 777
 778       // Nothing left in the buffer
 779       if (Srv->In.WriteSpace() == false)
 780          return false;
 781
 782       // We may have got multiple responses back in one packet..
 783       Srv->Close();
 784       return true;
 785    }
 786
 787    return false;
 788 }
 789                                                                         /*}}}*/
 790 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 791 // ---------------------------------------------------------------------
 792 /* We look at the header data we got back from the server and decide what
 793    to do. Returns
 794      0 - File is open,
 795      1 - IMS hit
 796      3 - Unrecoverable error
 797      4 - Error with error content page
 798      5 - Unrecoverable non-server error (close the connection) */
 799 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 800 {
 801    // Not Modified
 802    if (Srv->Result == 304)
 803    {
 804       unlink(Queue->DestFile.c_str());
 805       Res.IMSHit = true;
 806       Res.LastModified = Queue->LastModified;
 807       return 1;
 808    }
 809
 810    /* We have a reply we dont handle. This should indicate a perm server
 811       failure */
 812    if (Srv->Result < 200 || Srv->Result >= 300)
 813    {
 814       _error->Error("%u %s",Srv->Result,Srv->Code);
 815       if (Srv->HaveContent == true)
 816          return 4;
 817       return 3;
 818    }
 819
 820    // This is some sort of 2xx 'data follows' reply
 821    Res.LastModified = Srv->Date;
 822    Res.Size = Srv->Size;
 823
 824    // Open the file
 825    delete File;
 826    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 827    if (_error->PendingError() == true)
 828       return 5;
 829
 830    FailFile = Queue->DestFile;
 831    FailFd = File->Fd();
 832    FailTime = Srv->Date;
 833
 834    // Set the expected size
 835    if (Srv->StartPos >= 0)
 836    {
 837       Res.ResumePoint = Srv->StartPos;
 838       ftruncate(File->Fd(),Srv->StartPos);
 839    }
 840
 841    // Set the start point
 842    lseek(File->Fd(),0,SEEK_END);
 843
 844    delete Srv->In.MD5;
 845    Srv->In.MD5 = new MD5Summation;
 846
 847    // Fill the MD5 Hash if the file is non-empty (resume)
 848    if (Srv->StartPos > 0)
 849    {
 850       lseek(File->Fd(),0,SEEK_SET);
 851       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 852       {
 853          _error->Errno("read","Problem hashing file");
 854          return 5;
 855       }
 856       lseek(File->Fd(),0,SEEK_END);
 857    }
 858
 859    SetNonBlock(File->Fd(),true);
 860    return 0;
 861 }
 862                                                                         /*}}}*/
 863 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 864 // ---------------------------------------------------------------------
 865 /* This closes and timestamps the open file. This is neccessary to get
 866    resume behavoir on user abort */
 867 void HttpMethod::SigTerm(int)
 868 {
 869    if (FailFd == -1)
 870       exit(100);
 871    close(FailFd);
 872
 873    // Timestamp
 874    struct utimbuf UBuf;
 875    time(&UBuf.actime);
 876    UBuf.actime = FailTime;
 877    UBuf.modtime = FailTime;
 878    utime(FailFile.c_str(),&UBuf);
 879
 880    exit(100);
 881 }
 882                                                                         /*}}}*/
 883 // HttpMethod::Loop - Main loop                                         /*{{{*/
 884 // ---------------------------------------------------------------------
 885 /* */
 886 int HttpMethod::Loop()
 887 {
 888    signal(SIGTERM,SigTerm);
 889    signal(SIGINT,SigTerm);
 890
 891    ServerState *Server = 0;
 892
 893    int FailCounter = 0;
 894    while (1)
 895    {
 896       if (FailCounter >= 2)
 897       {
 898          Fail("Massive Server Brain Damage");
 899          FailCounter = 0;
 900       }
 901
 902       // We have no commands, wait for some to arrive
 903       if (Queue == 0)
 904       {
 905          if (WaitFd(STDIN_FILENO) == false)
 906             return 0;
 907       }
 908
 909       // Run messages
 910       if (Run(true) != 0)
 911          return 100;
 912
 913       if (Queue == 0)
 914          continue;
 915
 916       // Connect to the server
 917       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 918       {
 919          delete Server;
 920          Server = new ServerState(Queue->Uri,this);
 921       }
 922
 923       // Connnect to the host
 924       if (Server->Open() == false)
 925       {
 926          Fail();
 927          continue;
 928       }
 929
 930       // Queue the request
 931       SendReq(Queue,Server->Out);
 932
 933       // Fetch the next URL header data from the server.
 934       switch (Server->RunHeaders())
 935       {
 936          case 0:
 937          break;
 938
 939          // The header data is bad
 940          case 2:
 941          {
 942             _error->Error("Bad header Data");
 943             Fail();
 944             continue;
 945          }
 946
 947          // The server closed a connection during the header get..
 948          default:
 949          case 1:
 950          {
 951             FailCounter++;
 952             _error->DumpErrors();
 953             Server->Close();
 954             continue;
 955          }
 956       };
 957
 958       // Decide what to do.
 959       FetchResult Res;
 960       Res.Filename = Queue->DestFile;
 961       switch (DealWithHeaders(Res,Server))
 962       {
 963          // Ok, the file is Open
 964          case 0:
 965          {
 966             URIStart(Res);
 967
 968             // Run the data
 969             bool Result =  Server->RunData();
 970
 971             // Close the file, destroy the FD object and timestamp it
 972             FailFd = -1;
 973             delete File;
 974             File = 0;
 975
 976             // Timestamp
 977             struct utimbuf UBuf;
 978             time(&UBuf.actime);
 979             UBuf.actime = Server->Date;
 980             UBuf.modtime = Server->Date;
 981             utime(Queue->DestFile.c_str(),&UBuf);
 982
 983             // Send status to APT
 984             if (Result == true)
 985             {
 986                Res.MD5Sum = Server->In.MD5->Result();
 987                URIDone(Res);
 988             }
 989             else
 990                Fail();
 991
 992             break;
 993          }
 994
 995          // IMS hit
 996          case 1:
 997          {
 998             URIDone(Res);
 999             break;
1000          }
1001
1002          // Hard server error, not found or something
1003          case 3:
1004          {
1005             Fail();
1006             break;
1007          }
1008
1009          // Hard internal error, kill the connection and fail
1010          case 5:
1011          {
1012             Fail();
1013             Server->Close();
1014             break;
1015          }
1016
1017          // We need to flush the data, the header is like a 404 w/ error text
1018          case 4:
1019          {
1020             Fail();
1021
1022             // Send to content to dev/null
1023             File = new FileFd("/dev/null",FileFd::WriteExists);
1024             Server->RunData();
1025             delete File;
1026             File = 0;
1027             break;
1028          }
1029
1030          default:
1031          Fail("Internal error");
1032          break;
1033       }
1034
1035       FailCounter = 0;
1036    }
1037
1038    return 0;
1039 }
1040                                                                         /*}}}*/
1041
1042 int main()
1043 {
1044    HttpMethod Mth;
1045
1046    return Mth.Loop();
1047 }