methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.6 1998/11/21 06:09:09 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41
  42 // Internet stuff
  43 #include <netinet/in.h>
  44 #include <sys/socket.h>
  45 #include <arpa/inet.h>
  46 #include <netdb.h>
  47
  48 #include "http.h"
  49                                                                         /*}}}*/
  50
  51 string HttpMethod::FailFile;
  52 int HttpMethod::FailFd = -1;
  53 time_t HttpMethod::FailTime = 0;
  54
  55 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* */
  58 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  59 {
  60    Buf = new unsigned char[Size];
  61    Reset();
  62 }
  63                                                                         /*}}}*/
  64 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* */
  67 void CircleBuf::Reset()
  68 {
  69    InP = 0;
  70    OutP = 0;
  71    StrPos = 0;
  72    MaxGet = (unsigned int)-1;
  73    OutQueue = string();
  74    if (MD5 != 0)
  75    {
  76       delete MD5;
  77       MD5 = new MD5Summation;
  78    }
  79 };
  80                                                                         /*}}}*/
  81 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  82 // ---------------------------------------------------------------------
  83 /* This fills up the buffer with as much data as is in the FD, assuming it
  84    is non-blocking.. */
  85 bool CircleBuf::Read(int Fd)
  86 {
  87    while (1)
  88    {
  89       // Woops, buffer is full
  90       if (InP - OutP == Size)
  91          return true;
  92
  93       // Write the buffer segment
  94       int Res;
  95       Res = read(Fd,Buf + (InP%Size),LeftRead());
  96
  97       if (Res == 0)
  98          return false;
  99       if (Res < 0)
 100       {
 101          if (errno == EAGAIN)
 102             return true;
 103          return false;
 104       }
 105
 106       if (InP == 0)
 107          gettimeofday(&Start,0);
 108       InP += Res;
 109    }
 110 }
 111                                                                         /*}}}*/
 112 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 113 // ---------------------------------------------------------------------
 114 /* This will hold the string in and fill the buffer with it as it empties */
 115 bool CircleBuf::Read(string Data)
 116 {
 117    OutQueue += Data;
 118    FillOut();
 119    return true;
 120 }
 121                                                                         /*}}}*/
 122 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 123 // ---------------------------------------------------------------------
 124 /* */
 125 void CircleBuf::FillOut()
 126 {
 127    if (OutQueue.empty() == true)
 128       return;
 129    while (1)
 130    {
 131       // Woops, buffer is full
 132       if (InP - OutP == Size)
 133          return;
 134
 135       // Write the buffer segment
 136       unsigned long Sz = LeftRead();
 137       if (OutQueue.length() - StrPos < Sz)
 138          Sz = OutQueue.length() - StrPos;
 139       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 140
 141       // Advance
 142       StrPos += Sz;
 143       InP += Sz;
 144       if (OutQueue.length() == StrPos)
 145       {
 146          StrPos = 0;
 147          OutQueue = "";
 148          return;
 149       }
 150    }
 151 }
 152                                                                         /*}}}*/
 153 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 154 // ---------------------------------------------------------------------
 155 /* This empties the buffer into the FD. */
 156 bool CircleBuf::Write(int Fd)
 157 {
 158    while (1)
 159    {
 160       FillOut();
 161
 162       // Woops, buffer is empty
 163       if (OutP == InP)
 164          return true;
 165
 166       if (OutP == MaxGet)
 167          return true;
 168
 169       // Write the buffer segment
 170       int Res;
 171       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 172
 173       if (Res == 0)
 174          return false;
 175       if (Res < 0)
 176       {
 177          if (errno == EAGAIN)
 178             return true;
 179
 180          return false;
 181       }
 182
 183       if (MD5 != 0)
 184          MD5->Add(Buf + (OutP%Size),Res);
 185
 186       OutP += Res;
 187    }
 188 }
 189                                                                         /*}}}*/
 190 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 191 // ---------------------------------------------------------------------
 192 /* This copies till the first empty line */
 193 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 194 {
 195    // We cheat and assume it is unneeded to have more than one buffer load
 196    for (unsigned long I = OutP; I < InP; I++)
 197    {
 198       if (Buf[I%Size] != '\n')
 199          continue;
 200       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 201
 202       if (Single == false)
 203       {
 204          if (Buf[I%Size] != '\n')
 205             continue;
 206          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 207       }
 208
 209       if (I > InP)
 210          I = InP;
 211
 212       Data = "";
 213       while (OutP < I)
 214       {
 215          unsigned long Sz = LeftWrite();
 216          if (Sz == 0)
 217             return false;
 218          if (I - OutP < LeftWrite())
 219             Sz = I - OutP;
 220          Data += string((char *)(Buf + (OutP%Size)),Sz);
 221          OutP += Sz;
 222       }
 223       return true;
 224    }
 225    return false;
 226 }
 227                                                                         /*}}}*/
 228 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 229 // ---------------------------------------------------------------------
 230 /* */
 231 void CircleBuf::Stats()
 232 {
 233    if (InP == 0)
 234       return;
 235
 236    struct timeval Stop;
 237    gettimeofday(&Stop,0);
 238 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 239              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 240    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 241 }
 242                                                                         /*}}}*/
 243
 244 // ServerState::ServerState - Constructor                               /*{{{*/
 245 // ---------------------------------------------------------------------
 246 /* */
 247 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 248                         In(64*1024), Out(1*1024),
 249                         ServerName(Srv)
 250 {
 251    Reset();
 252 }
 253                                                                         /*}}}*/
 254 // ServerState::Open - Open a connection to the server                  /*{{{*/
 255 // ---------------------------------------------------------------------
 256 /* This opens a connection to the server. */
 257 string LastHost;
 258 in_addr LastHostA;
 259 bool ServerState::Open()
 260 {
 261    // Use the already open connection if possible.
 262    if (ServerFd != -1)
 263       return true;
 264
 265    Close();
 266    In.Reset();
 267    Out.Reset();
 268
 269    // Determine the proxy setting
 270    string DefProxy = _config->Find("Acquire::http::Proxy",getenv("http_proxy"));
 271    string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 272    if (SpecificProxy.empty() == false)
 273    {
 274       if (SpecificProxy == "DIRECT")
 275          Proxy = "";
 276       else
 277          Proxy = SpecificProxy;
 278    }
 279    else
 280       Proxy = DefProxy;
 281
 282    // Determine what host and port to use based on the proxy settings
 283    int Port = 80;
 284    string Host;
 285    if (Proxy.empty() == true)
 286    {
 287       if (ServerName.Port != 0)
 288          Port = ServerName.Port;
 289       Host = ServerName.Host;
 290    }
 291    else
 292    {
 293       if (Proxy.Port != 0)
 294          Port = Proxy.Port;
 295       Host = Proxy.Host;
 296    }
 297
 298    /* We used a cached address record.. Yes this is against the spec but
 299       the way we have setup our rotating dns suggests that this is more
 300       sensible */
 301    if (LastHost != Host)
 302    {
 303       Owner->Status("Connecting to %s",Host.c_str());
 304
 305       // Lookup the host
 306       hostent *Addr = gethostbyname(Host.c_str());
 307       if (Addr == 0)
 308          return _error->Error("Could not resolve '%s'",Host.c_str());
 309       LastHost = Host;
 310       LastHostA = *(in_addr *)(Addr->h_addr_list[0]);
 311    }
 312
 313    Owner->Status("Connecting to %s (%s)",Host.c_str(),inet_ntoa(LastHostA));
 314
 315    // Get a socket
 316    if ((ServerFd = socket(AF_INET,SOCK_STREAM,0)) < 0)
 317       return _error->Errno("socket","Could not create a socket");
 318
 319    // Connect to the server
 320    struct sockaddr_in server;
 321    server.sin_family = AF_INET;
 322    server.sin_port = htons(Port);
 323    server.sin_addr = LastHostA;
 324    if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0)
 325       return _error->Errno("socket","Could not create a socket");
 326
 327    SetNonBlock(ServerFd,true);
 328    return true;
 329 }
 330                                                                         /*}}}*/
 331 // ServerState::Close - Close a connection to the server                /*{{{*/
 332 // ---------------------------------------------------------------------
 333 /* */
 334 bool ServerState::Close()
 335 {
 336    close(ServerFd);
 337    ServerFd = -1;
 338    return true;
 339 }
 340                                                                         /*}}}*/
 341 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 342 // ---------------------------------------------------------------------
 343 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 344    parse error occured */
 345 int ServerState::RunHeaders()
 346 {
 347    State = Header;
 348
 349    Owner->Status("Waiting for file");
 350
 351    Major = 0;
 352    Minor = 0;
 353    Result = 0;
 354    Size = 0;
 355    StartPos = 0;
 356    Encoding = Closes;
 357    HaveContent = false;
 358    time(&Date);
 359
 360    do
 361    {
 362       string Data;
 363       if (In.WriteTillEl(Data) == false)
 364          continue;
 365
 366       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 367       {
 368          string::const_iterator J = I;
 369          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 370          if (HeaderLine(string(I,J-I)) == false)
 371             return 2;
 372          I = J;
 373       }
 374       return 0;
 375    }
 376    while (Owner->Go(false,this) == true);
 377
 378    return 1;
 379 }
 380                                                                         /*}}}*/
 381 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 382 // ---------------------------------------------------------------------
 383 /* */
 384 bool ServerState::RunData()
 385 {
 386    State = Data;
 387
 388    // Chunked transfer encoding is fun..
 389    if (Encoding == Chunked)
 390    {
 391       while (1)
 392       {
 393          // Grab the block size
 394          bool Last = true;
 395          string Data;
 396          In.Limit(-1);
 397          do
 398          {
 399             if (In.WriteTillEl(Data,true) == true)
 400                break;
 401          }
 402          while ((Last = Owner->Go(false,this)) == true);
 403
 404          if (Last == false)
 405             return false;
 406
 407          // See if we are done
 408          unsigned long Len = strtol(Data.c_str(),0,16);
 409          if (Len == 0)
 410          {
 411             In.Limit(-1);
 412
 413             // We have to remove the entity trailer
 414             Last = true;
 415             do
 416             {
 417                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 418                   break;
 419             }
 420             while ((Last = Owner->Go(false,this)) == true);
 421             if (Last == false)
 422                return false;
 423             return true;
 424          }
 425
 426          // Transfer the block
 427          In.Limit(Len);
 428          while (Owner->Go(true,this) == true)
 429             if (In.IsLimit() == true)
 430                break;
 431
 432          // Error
 433          if (In.IsLimit() == false)
 434             return false;
 435
 436          // The server sends an extra new line before the next block specifier..
 437          In.Limit(-1);
 438          Last = true;
 439          do
 440          {
 441             if (In.WriteTillEl(Data,true) == true)
 442                break;
 443          }
 444          while ((Last = Owner->Go(false,this)) == true);
 445          if (Last == false)
 446             return false;
 447       }
 448    }
 449    else
 450    {
 451       /* Closes encoding is used when the server did not specify a size, the
 452          loss of the connection means we are done */
 453       if (Encoding == Closes)
 454          In.Limit(-1);
 455       else
 456          In.Limit(Size - StartPos);
 457
 458       // Just transfer the whole block.
 459       do
 460       {
 461          if (In.IsLimit() == false)
 462             continue;
 463
 464          In.Limit(-1);
 465          return true;
 466       }
 467       while (Owner->Go(true,this) == true);
 468    }
 469
 470    return Owner->Flush(this);
 471 }
 472                                                                         /*}}}*/
 473 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 474 // ---------------------------------------------------------------------
 475 /* */
 476 bool ServerState::HeaderLine(string Line)
 477 {
 478    if (Line.empty() == true)
 479       return true;
 480
 481    // The http server might be trying to do something evil.
 482    if (Line.length() >= MAXLEN)
 483       return _error->Error("Got a single header line over %u chars",MAXLEN);
 484
 485    string::size_type Pos = Line.find(' ');
 486    if (Pos == string::npos || Pos+1 > Line.length())
 487       return _error->Error("Bad header line");
 488
 489    string Tag = string(Line,0,Pos);
 490    string Val = string(Line,Pos+1);
 491
 492    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 493    {
 494       // Evil servers return no version
 495       if (Line[4] == '/')
 496       {
 497          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 498                     &Result,Code) != 4)
 499             return _error->Error("The http server sent an invalid reply header");
 500       }
 501       else
 502       {
 503          Major = 0;
 504          Minor = 9;
 505          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 506             return _error->Error("The http server sent an invalid reply header");
 507       }
 508
 509       return true;
 510    }
 511
 512    if (stringcasecmp(Tag,"Content-Length:") == 0)
 513    {
 514       if (Encoding == Closes)
 515          Encoding = Stream;
 516       HaveContent = true;
 517
 518       // The length is already set from the Content-Range header
 519       if (StartPos != 0)
 520          return true;
 521
 522       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 523          return _error->Error("The http server sent an invalid Content-Length header");
 524       return true;
 525    }
 526
 527    if (stringcasecmp(Tag,"Content-Type:") == 0)
 528    {
 529       HaveContent = true;
 530       return true;
 531    }
 532
 533    if (stringcasecmp(Tag,"Content-Range:") == 0)
 534    {
 535       HaveContent = true;
 536
 537       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 538          return _error->Error("The http server sent an invalid Content-Range header");
 539       if ((unsigned)StartPos > Size)
 540          return _error->Error("This http server has broken range support");
 541       return true;
 542    }
 543
 544    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 545    {
 546       HaveContent = true;
 547       if (stringcasecmp(Val,"chunked") == 0)
 548          Encoding = Chunked;
 549
 550       return true;
 551    }
 552
 553    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 554    {
 555       if (StrToTime(Val,Date) == false)
 556          return _error->Error("Unknown date format");
 557       return true;
 558    }
 559
 560    return true;
 561 }
 562                                                                         /*}}}*/
 563
 564 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 565 // ---------------------------------------------------------------------
 566 /* This places the http request in the outbound buffer */
 567 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 568 {
 569    URI Uri = Itm->Uri;
 570
 571    // The HTTP server expects a hostname with a trailing :port
 572    char Buf[300];
 573    string ProperHost = Uri.Host;
 574    if (Uri.Port != 0)
 575    {
 576       sprintf(Buf,":%u",Uri.Port);
 577       ProperHost += Buf;
 578    }
 579
 580    /* Build the request. We include a keep-alive header only for non-proxy
 581       requests. This is to tweak old http/1.0 servers that do support keep-alive
 582       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 583       will glitch HTTP/1.0 proxies because they do not filter it out and
 584       pass it on, HTTP/1.1 says the connection should default to keep alive
 585       and we expect the proxy to do this */
 586    if (Proxy.empty() == true)
 587       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 588               Uri.Path.c_str(),ProperHost.c_str());
 589    else
 590       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 591               Itm->Uri.c_str(),ProperHost.c_str());
 592    string Req = Buf;
 593
 594    // Check for a partial file
 595    struct stat SBuf;
 596    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 597    {
 598       // In this case we send an if-range query with a range header
 599       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 600               TimeRFC1123(SBuf.st_mtime).c_str());
 601       Req += Buf;
 602    }
 603    else
 604    {
 605       if (Itm->LastModified != 0)
 606       {
 607          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 608          Req += Buf;
 609       }
 610    }
 611
 612 /*   if (ProxyAuth.empty() == false)
 613       Req += string("Proxy-Authorization: Basic ") + Base64Encode(ProxyAuth) + "\r\n";*/
 614
 615    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 616 //   cerr << Req << endl;
 617
 618    Out.Read(Req);
 619 }
 620                                                                         /*}}}*/
 621 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 622 // ---------------------------------------------------------------------
 623 /* This runs the select loop over the server FDs, Output file FDs and
 624    stdin. */
 625 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 626 {
 627    // Server has closed the connection
 628    if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
 629       return false;
 630
 631    fd_set rfds,wfds,efds;
 632    FD_ZERO(&rfds);
 633    FD_ZERO(&wfds);
 634    FD_ZERO(&efds);
 635
 636    // Add the server
 637    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 638       FD_SET(Srv->ServerFd,&wfds);
 639    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 640       FD_SET(Srv->ServerFd,&rfds);
 641
 642    // Add the file
 643    int FileFD = -1;
 644    if (File != 0)
 645       FileFD = File->Fd();
 646
 647    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 648       FD_SET(FileFD,&wfds);
 649
 650    // Add stdin
 651    FD_SET(STDIN_FILENO,&rfds);
 652
 653    // Error Set
 654    if (FileFD != -1)
 655       FD_SET(FileFD,&efds);
 656    if (Srv->ServerFd != -1)
 657       FD_SET(Srv->ServerFd,&efds);
 658
 659    // Figure out the max fd
 660    int MaxFd = FileFD;
 661    if (MaxFd < Srv->ServerFd)
 662       MaxFd = Srv->ServerFd;
 663
 664    // Select
 665    struct timeval tv;
 666    tv.tv_sec = 120;
 667    tv.tv_usec = 0;
 668    int Res = 0;
 669    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 670       return _error->Errno("select","Select failed");
 671
 672    if (Res == 0)
 673    {
 674       _error->Error("Connection timed out");
 675       return ServerDie(Srv);
 676    }
 677
 678    // Some kind of exception (error) on the sockets, die
 679    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 680        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 681       return _error->Error("Socket Exception");
 682
 683    // Handle server IO
 684    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 685    {
 686       errno = 0;
 687       if (Srv->In.Read(Srv->ServerFd) == false)
 688          return ServerDie(Srv);
 689    }
 690
 691    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 692    {
 693       errno = 0;
 694       if (Srv->Out.Write(Srv->ServerFd) == false)
 695          return ServerDie(Srv);
 696    }
 697
 698    // Send data to the file
 699    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 700    {
 701       if (Srv->In.Write(FileFD) == false)
 702          return _error->Errno("write","Error writing to output file");
 703    }
 704
 705    // Handle commands from APT
 706    if (FD_ISSET(STDIN_FILENO,&rfds))
 707    {
 708       if (Run(true) != 0)
 709          exit(100);
 710    }
 711
 712    return true;
 713 }
 714                                                                         /*}}}*/
 715 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 716 // ---------------------------------------------------------------------
 717 /* This takes the current input buffer from the Server FD and writes it
 718    into the file */
 719 bool HttpMethod::Flush(ServerState *Srv)
 720 {
 721    if (File != 0)
 722    {
 723       SetNonBlock(File->Fd(),false);
 724       if (Srv->In.WriteSpace() == false)
 725          return true;
 726
 727       while (Srv->In.WriteSpace() == true)
 728       {
 729          if (Srv->In.Write(File->Fd()) == false)
 730             return _error->Errno("write","Error writing to file");
 731          if (Srv->In.IsLimit() == true)
 732             return true;
 733       }
 734
 735       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 736          return true;
 737    }
 738    return false;
 739 }
 740                                                                         /*}}}*/
 741 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 742 // ---------------------------------------------------------------------
 743 /* */
 744 bool HttpMethod::ServerDie(ServerState *Srv)
 745 {
 746    // Dump the buffer to the file
 747    if (Srv->State == ServerState::Data)
 748    {
 749       SetNonBlock(File->Fd(),false);
 750       while (Srv->In.WriteSpace() == true)
 751       {
 752          if (Srv->In.Write(File->Fd()) == false)
 753             return _error->Errno("write","Error writing to the file");
 754
 755          // Done
 756          if (Srv->In.IsLimit() == true)
 757             return true;
 758       }
 759    }
 760
 761    // See if this is because the server finished the data stream
 762    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 763        Srv->Encoding != ServerState::Closes)
 764    {
 765       if (errno == 0)
 766          return _error->Error("Error reading from server Remote end closed connection");
 767       return _error->Errno("read","Error reading from server");
 768    }
 769    else
 770    {
 771       Srv->In.Limit(-1);
 772
 773       // Nothing left in the buffer
 774       if (Srv->In.WriteSpace() == false)
 775          return false;
 776
 777       // We may have got multiple responses back in one packet..
 778       Srv->Close();
 779       return true;
 780    }
 781
 782    return false;
 783 }
 784                                                                         /*}}}*/
 785 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 786 // ---------------------------------------------------------------------
 787 /* We look at the header data we got back from the server and decide what
 788    to do. Returns
 789      0 - File is open,
 790      1 - IMS hit
 791      3 - Unrecoverable error
 792      4 - Error with error content page
 793      5 - Unrecoverable non-server error (close the connection) */
 794 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 795 {
 796    // Not Modified
 797    if (Srv->Result == 304)
 798    {
 799       unlink(Queue->DestFile.c_str());
 800       Res.IMSHit = true;
 801       Res.LastModified = Queue->LastModified;
 802       return 1;
 803    }
 804
 805    /* We have a reply we dont handle. This should indicate a perm server
 806       failure */
 807    if (Srv->Result < 200 || Srv->Result >= 300)
 808    {
 809       _error->Error("%u %s",Srv->Result,Srv->Code);
 810       if (Srv->HaveContent == true)
 811          return 4;
 812       return 3;
 813    }
 814
 815    // This is some sort of 2xx 'data follows' reply
 816    Res.LastModified = Srv->Date;
 817    Res.Size = Srv->Size;
 818
 819    // Open the file
 820    delete File;
 821    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 822    if (_error->PendingError() == true)
 823       return 5;
 824
 825    FailFile = Queue->DestFile;
 826    FailFd = File->Fd();
 827    FailTime = Srv->Date;
 828
 829    // Set the expected size
 830    if (Srv->StartPos >= 0)
 831    {
 832       Res.ResumePoint = Srv->StartPos;
 833       ftruncate(File->Fd(),Srv->StartPos);
 834    }
 835
 836    // Set the start point
 837    lseek(File->Fd(),0,SEEK_END);
 838
 839    delete Srv->In.MD5;
 840    Srv->In.MD5 = new MD5Summation;
 841
 842    // Fill the MD5 Hash if the file is non-empty (resume)
 843    if (Srv->StartPos > 0)
 844    {
 845       lseek(File->Fd(),0,SEEK_SET);
 846       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 847       {
 848          _error->Errno("read","Problem hashing file");
 849          return 5;
 850       }
 851       lseek(File->Fd(),0,SEEK_END);
 852    }
 853
 854    SetNonBlock(File->Fd(),true);
 855    return 0;
 856 }
 857                                                                         /*}}}*/
 858 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 859 // ---------------------------------------------------------------------
 860 /* This closes and timestamps the open file. This is neccessary to get
 861    resume behavoir on user abort */
 862 void HttpMethod::SigTerm(int)
 863 {
 864    if (FailFd == -1)
 865       exit(100);
 866    close(FailFd);
 867
 868    // Timestamp
 869    struct utimbuf UBuf;
 870    time(&UBuf.actime);
 871    UBuf.actime = FailTime;
 872    UBuf.modtime = FailTime;
 873    utime(FailFile.c_str(),&UBuf);
 874
 875    exit(100);
 876 }
 877                                                                         /*}}}*/
 878 // HttpMethod::Loop - Main loop                                         /*{{{*/
 879 // ---------------------------------------------------------------------
 880 /* */
 881 int HttpMethod::Loop()
 882 {
 883    signal(SIGTERM,SigTerm);
 884    signal(SIGINT,SigTerm);
 885
 886    ServerState *Server = 0;
 887
 888    int FailCounter = 0;
 889    while (1)
 890    {
 891       if (FailCounter >= 2)
 892       {
 893          Fail("Massive Server Brain Damage");
 894          FailCounter = 0;
 895       }
 896
 897       // We have no commands, wait for some to arrive
 898       if (Queue == 0)
 899       {
 900          if (WaitFd(STDIN_FILENO) == false)
 901             return 0;
 902       }
 903
 904       // Run messages
 905       if (Run(true) != 0)
 906          return 100;
 907
 908       if (Queue == 0)
 909          continue;
 910
 911       // Connect to the server
 912       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 913       {
 914          delete Server;
 915          Server = new ServerState(Queue->Uri,this);
 916       }
 917
 918       // Connnect to the host
 919       if (Server->Open() == false)
 920       {
 921          Fail();
 922          continue;
 923       }
 924
 925       // Queue the request
 926       SendReq(Queue,Server->Out);
 927
 928       // Fetch the next URL header data from the server.
 929       switch (Server->RunHeaders())
 930       {
 931          case 0:
 932          break;
 933
 934          // The header data is bad
 935          case 2:
 936          {
 937             _error->Error("Bad header Data");
 938             Fail();
 939             continue;
 940          }
 941
 942          // The server closed a connection during the header get..
 943          default:
 944          case 1:
 945          {
 946             FailCounter++;
 947             _error->DumpErrors();
 948             Server->Close();
 949             continue;
 950          }
 951       };
 952
 953       // Decide what to do.
 954       FetchResult Res;
 955       Res.Filename = Queue->DestFile;
 956       switch (DealWithHeaders(Res,Server))
 957       {
 958          // Ok, the file is Open
 959          case 0:
 960          {
 961             URIStart(Res);
 962
 963             // Run the data
 964             bool Result =  Server->RunData();
 965
 966             // Close the file, destroy the FD object and timestamp it
 967             FailFd = -1;
 968             delete File;
 969             File = 0;
 970
 971             // Timestamp
 972             struct utimbuf UBuf;
 973             time(&UBuf.actime);
 974             UBuf.actime = Server->Date;
 975             UBuf.modtime = Server->Date;
 976             utime(Queue->DestFile.c_str(),&UBuf);
 977
 978             // Send status to APT
 979             if (Result == true)
 980             {
 981                Res.MD5Sum = Server->In.MD5->Result();
 982                URIDone(Res);
 983             }
 984             else
 985                Fail();
 986
 987             break;
 988          }
 989
 990          // IMS hit
 991          case 1:
 992          {
 993             URIDone(Res);
 994             break;
 995          }
 996
 997          // Hard server error, not found or something
 998          case 3:
 999          {
1000             Fail();
1001             break;
1002          }
1003
1004          // Hard internal error, kill the connection and fail
1005          case 5:
1006          {
1007             Fail();
1008             Server->Close();
1009             break;
1010          }
1011
1012          // We need to flush the data, the header is like a 404 w/ error text
1013          case 4:
1014          {
1015             Fail();
1016
1017             // Send to content to dev/null
1018             File = new FileFd("/dev/null",FileFd::WriteExists);
1019             Server->RunData();
1020             delete File;
1021             File = 0;
1022             break;
1023          }
1024
1025          default:
1026          Fail("Internal error");
1027          break;
1028       }
1029
1030       FailCounter = 0;
1031    }
1032
1033    return 0;
1034 }
1035                                                                         /*}}}*/
1036
1037 int main()
1038 {
1039    HttpMethod Mth;
1040
1041    return Mth.Loop();
1042 }