methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.3 1998/11/04 07:10:49 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41
  42 // Internet stuff
  43 #include <netinet/in.h>
  44 #include <sys/socket.h>
  45 #include <arpa/inet.h>
  46 #include <netdb.h>
  47
  48 #include "http.h"
  49                                                                         /*}}}*/
  50
  51 string HttpMethod::FailFile;
  52 int HttpMethod::FailFd = -1;
  53 time_t HttpMethod::FailTime = 0;
  54
  55 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* */
  58 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  59 {
  60    Buf = new unsigned char[Size];
  61    Reset();
  62 }
  63                                                                         /*}}}*/
  64 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* */
  67 void CircleBuf::Reset()
  68 {
  69    InP = 0;
  70    OutP = 0;
  71    StrPos = 0;
  72    MaxGet = (unsigned int)-1;
  73    OutQueue = string();
  74    if (MD5 != 0)
  75    {
  76       delete MD5;
  77       MD5 = new MD5Summation;
  78    }
  79 };
  80                                                                         /*}}}*/
  81 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  82 // ---------------------------------------------------------------------
  83 /* This fills up the buffer with as much data as is in the FD, assuming it
  84    is non-blocking.. */
  85 bool CircleBuf::Read(int Fd)
  86 {
  87    while (1)
  88    {
  89       // Woops, buffer is full
  90       if (InP - OutP == Size)
  91          return true;
  92
  93       // Write the buffer segment
  94       int Res;
  95       Res = read(Fd,Buf + (InP%Size),LeftRead());
  96
  97       if (Res == 0)
  98          return false;
  99       if (Res < 0)
 100       {
 101          if (errno == EAGAIN)
 102             return true;
 103          return false;
 104       }
 105
 106       if (InP == 0)
 107          gettimeofday(&Start,0);
 108       InP += Res;
 109    }
 110 }
 111                                                                         /*}}}*/
 112 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 113 // ---------------------------------------------------------------------
 114 /* This will hold the string in and fill the buffer with it as it empties */
 115 bool CircleBuf::Read(string Data)
 116 {
 117    OutQueue += Data;
 118    FillOut();
 119    return true;
 120 }
 121                                                                         /*}}}*/
 122 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 123 // ---------------------------------------------------------------------
 124 /* */
 125 void CircleBuf::FillOut()
 126 {
 127    if (OutQueue.empty() == true)
 128       return;
 129    while (1)
 130    {
 131       // Woops, buffer is full
 132       if (InP - OutP == Size)
 133          return;
 134
 135       // Write the buffer segment
 136       unsigned long Sz = LeftRead();
 137       if (OutQueue.length() - StrPos < Sz)
 138          Sz = OutQueue.length() - StrPos;
 139       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 140
 141       // Advance
 142       StrPos += Sz;
 143       InP += Sz;
 144       if (OutQueue.length() == StrPos)
 145       {
 146          StrPos = 0;
 147          OutQueue = "";
 148          return;
 149       }
 150    }
 151 }
 152                                                                         /*}}}*/
 153 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 154 // ---------------------------------------------------------------------
 155 /* This empties the buffer into the FD. */
 156 bool CircleBuf::Write(int Fd)
 157 {
 158    while (1)
 159    {
 160       FillOut();
 161
 162       // Woops, buffer is empty
 163       if (OutP == InP)
 164          return true;
 165
 166       if (OutP == MaxGet)
 167          return true;
 168
 169       // Write the buffer segment
 170       int Res;
 171       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 172
 173       if (Res == 0)
 174          return false;
 175       if (Res < 0)
 176       {
 177          if (errno == EAGAIN)
 178             return true;
 179
 180          return false;
 181       }
 182
 183       if (MD5 != 0)
 184          MD5->Add(Buf + (OutP%Size),Res);
 185
 186       OutP += Res;
 187    }
 188 }
 189                                                                         /*}}}*/
 190 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 191 // ---------------------------------------------------------------------
 192 /* This copies till the first empty line */
 193 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 194 {
 195    // We cheat and assume it is unneeded to have more than one buffer load
 196    for (unsigned long I = OutP; I < InP; I++)
 197    {
 198       if (Buf[I%Size] != '\n')
 199          continue;
 200       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 201
 202       if (Single == false)
 203       {
 204          if (Buf[I%Size] != '\n')
 205             continue;
 206          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 207       }
 208
 209       if (I > InP)
 210          I = InP;
 211
 212       Data = "";
 213       while (OutP < I)
 214       {
 215          unsigned long Sz = LeftWrite();
 216          if (Sz == 0)
 217             return false;
 218          if (I - OutP < LeftWrite())
 219             Sz = I - OutP;
 220          Data += string((char *)(Buf + (OutP%Size)),Sz);
 221          OutP += Sz;
 222       }
 223       return true;
 224    }
 225    return false;
 226 }
 227                                                                         /*}}}*/
 228 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 229 // ---------------------------------------------------------------------
 230 /* */
 231 void CircleBuf::Stats()
 232 {
 233    if (InP == 0)
 234       return;
 235
 236    struct timeval Stop;
 237    gettimeofday(&Stop,0);
 238 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 239              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 240    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 241 }
 242                                                                         /*}}}*/
 243
 244 // ServerState::ServerState - Constructor                               /*{{{*/
 245 // ---------------------------------------------------------------------
 246 /* */
 247 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 248                         In(64*1024), Out(1*1024),
 249                         ServerName(Srv)
 250 {
 251    Reset();
 252 }
 253                                                                         /*}}}*/
 254 // ServerState::Open - Open a connection to the server                  /*{{{*/
 255 // ---------------------------------------------------------------------
 256 /* This opens a connection to the server. */
 257 string LastHost;
 258 in_addr LastHostA;
 259 bool ServerState::Open()
 260 {
 261    // Use the already open connection if possible.
 262    if (ServerFd != -1)
 263       return true;
 264
 265    Close();
 266    In.Reset();
 267    Out.Reset();
 268
 269    // Determine the proxy setting
 270    string DefProxy = _config->Find("Acquire::http::Proxy",getenv("http_proxy"));
 271    string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 272    if (SpecificProxy.empty() == false)
 273    {
 274       if (SpecificProxy == "DIRECT")
 275          Proxy = "";
 276       else
 277          Proxy = SpecificProxy;
 278    }
 279    else
 280       Proxy = DefProxy;
 281
 282    // Determine what host and port to use based on the proxy settings
 283    int Port = 80;
 284    string Host;
 285    if (Proxy.empty() == true)
 286    {
 287       if (ServerName.Port != 0)
 288          Port = ServerName.Port;
 289       Host = ServerName.Host;
 290    }
 291    else
 292    {
 293       if (Proxy.Port != 0)
 294          Port = Proxy.Port;
 295       Host = Proxy.Host;
 296    }
 297
 298    /* We used a cached address record.. Yes this is against the spec but
 299       the way we have setup our rotating dns suggests that this is more
 300       sensible */
 301    if (LastHost != Host)
 302    {
 303       Owner->Status("Connecting to %s",Host.c_str());
 304
 305       // Lookup the host
 306       hostent *Addr = gethostbyname(Host.c_str());
 307       if (Addr == 0)
 308          return _error->Error("Could not resolve '%s'",Host.c_str());
 309       LastHost = Host;
 310       LastHostA = *(in_addr *)(Addr->h_addr_list[0]);
 311    }
 312
 313    Owner->Status("Connecting to %s (%s)",Host.c_str(),inet_ntoa(LastHostA));
 314
 315    // Get a socket
 316    if ((ServerFd = socket(AF_INET,SOCK_STREAM,0)) < 0)
 317       return _error->Errno("socket","Could not create a socket");
 318
 319    // Connect to the server
 320    struct sockaddr_in server;
 321    server.sin_family = AF_INET;
 322    server.sin_port = htons(Port);
 323    server.sin_addr = LastHostA;
 324    if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0)
 325       return _error->Errno("socket","Could not create a socket");
 326
 327    SetNonBlock(ServerFd,true);
 328    return true;
 329 }
 330                                                                         /*}}}*/
 331 // ServerState::Close - Close a connection to the server                /*{{{*/
 332 // ---------------------------------------------------------------------
 333 /* */
 334 bool ServerState::Close()
 335 {
 336    close(ServerFd);
 337    ServerFd = -1;
 338    return true;
 339 }
 340                                                                         /*}}}*/
 341 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 342 // ---------------------------------------------------------------------
 343 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 344    parse error occured */
 345 int ServerState::RunHeaders()
 346 {
 347    State = Header;
 348
 349    Owner->Status("Waiting for file");
 350
 351    Major = 0;
 352    Minor = 0;
 353    Result = 0;
 354    Size = 0;
 355    StartPos = 0;
 356    Encoding = Closes;
 357    HaveContent = false;
 358    time(&Date);
 359
 360    do
 361    {
 362       string Data;
 363       if (In.WriteTillEl(Data) == false)
 364          continue;
 365
 366       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 367       {
 368          string::const_iterator J = I;
 369          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 370          if (HeaderLine(string(I,J-I)) == false)
 371             return 2;
 372          I = J;
 373       }
 374       return 0;
 375    }
 376    while (Owner->Go(false,this) == true);
 377
 378    return 1;
 379 }
 380                                                                         /*}}}*/
 381 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 382 // ---------------------------------------------------------------------
 383 /* */
 384 bool ServerState::RunData()
 385 {
 386    State = Data;
 387
 388    // Chunked transfer encoding is fun..
 389    if (Encoding == Chunked)
 390    {
 391       while (1)
 392       {
 393          // Grab the block size
 394          bool Last = true;
 395          string Data;
 396          In.Limit(-1);
 397          do
 398          {
 399             if (In.WriteTillEl(Data,true) == true)
 400                break;
 401          }
 402          while ((Last = Owner->Go(false,this)) == true);
 403
 404          if (Last == false)
 405             return false;
 406
 407          // See if we are done
 408          unsigned long Len = strtol(Data.c_str(),0,16);
 409          if (Len == 0)
 410          {
 411             In.Limit(-1);
 412
 413             // We have to remove the entity trailer
 414             Last = true;
 415             do
 416             {
 417                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 418                   break;
 419             }
 420             while ((Last = Owner->Go(false,this)) == true);
 421             if (Last == false)
 422                return false;
 423             return true;
 424          }
 425
 426          // Transfer the block
 427          In.Limit(Len);
 428          while (Owner->Go(true,this) == true)
 429             if (In.IsLimit() == true)
 430                break;
 431
 432          // Error
 433          if (In.IsLimit() == false)
 434             return false;
 435
 436          // The server sends an extra new line before the next block specifier..
 437          In.Limit(-1);
 438          Last = true;
 439          do
 440          {
 441             if (In.WriteTillEl(Data,true) == true)
 442                break;
 443          }
 444          while ((Last = Owner->Go(false,this)) == true);
 445          if (Last == false)
 446             return false;
 447       }
 448    }
 449    else
 450    {
 451       /* Closes encoding is used when the server did not specify a size, the
 452          loss of the connection means we are done */
 453       if (Encoding == Closes)
 454          In.Limit(-1);
 455       else
 456          In.Limit(Size - StartPos);
 457
 458       // Just transfer the whole block.
 459       do
 460       {
 461          if (In.IsLimit() == false)
 462             continue;
 463
 464          In.Limit(-1);
 465          return true;
 466       }
 467       while (Owner->Go(true,this) == true);
 468    }
 469
 470    return Owner->Flush(this);
 471 }
 472                                                                         /*}}}*/
 473 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 474 // ---------------------------------------------------------------------
 475 /* */
 476 bool ServerState::HeaderLine(string Line)
 477 {
 478    if (Line.empty() == true)
 479       return true;
 480
 481    // The http server might be trying to do something evil.
 482    if (Line.length() >= MAXLEN)
 483       return _error->Error("Got a single header line over %u chars",MAXLEN);
 484
 485    string::size_type Pos = Line.find(' ');
 486    if (Pos == string::npos || Pos+1 > Line.length())
 487       return _error->Error("Bad header line");
 488
 489    string Tag = string(Line,0,Pos);
 490    string Val = string(Line,Pos+1);
 491
 492    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 493    {
 494       // Evil servers return no version
 495       if (Line[4] == '/')
 496       {
 497          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 498                     &Result,Code) != 4)
 499             return _error->Error("The http server sent an invalid reply header");
 500       }
 501       else
 502       {
 503          Major = 0;
 504          Minor = 9;
 505          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 506             return _error->Error("The http server sent an invalid reply header");
 507       }
 508
 509       return true;
 510    }
 511
 512    if (stringcasecmp(Tag,"Content-Length:") == 0)
 513    {
 514       if (Encoding == Closes)
 515          Encoding = Stream;
 516       HaveContent = true;
 517
 518       // The length is already set from the Content-Range header
 519       if (StartPos != 0)
 520          return true;
 521
 522       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 523          return _error->Error("The http server sent an invalid Content-Length header");
 524       return true;
 525    }
 526
 527    if (stringcasecmp(Tag,"Content-Type:") == 0)
 528    {
 529       HaveContent = true;
 530       return true;
 531    }
 532
 533    if (stringcasecmp(Tag,"Content-Range:") == 0)
 534    {
 535       HaveContent = true;
 536
 537       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 538          return _error->Error("The http server sent an invalid Content-Range header");
 539       if ((unsigned)StartPos > Size)
 540          return _error->Error("This http server has broken range support");
 541       return true;
 542    }
 543
 544    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 545    {
 546       HaveContent = true;
 547       if (stringcasecmp(Val,"chunked") == 0)
 548          Encoding = Chunked;
 549
 550       return true;
 551    }
 552
 553    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 554    {
 555       if (StrToTime(Val,Date) == false)
 556          return _error->Error("Unknown date format");
 557       return true;
 558    }
 559
 560    return true;
 561 }
 562                                                                         /*}}}*/
 563
 564 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 565 // ---------------------------------------------------------------------
 566 /* This places the http request in the outbound buffer */
 567 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 568 {
 569    URI Uri = Itm->Uri;
 570
 571    // The HTTP server expects a hostname with a trailing :port
 572    char Buf[300];
 573    string ProperHost = Uri.Host;
 574    if (Uri.Port != 0)
 575    {
 576       sprintf(Buf,":%u",Uri.Port);
 577       ProperHost += Buf;
 578    }
 579
 580    /* Build the request. We include a keep-alive header only for non-proxy
 581       requests. This is to tweak old http/1.0 servers that do support keep-alive
 582       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 583       will glitch HTTP/1.0 proxies because they do not filter it out and
 584       pass it on, HTTP/1.1 says the connection should default to keep alive
 585       and we expect the proxy to do this */
 586    if (Proxy.empty() == true)
 587       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 588               Uri.Path.c_str(),ProperHost.c_str());
 589    else
 590       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 591               Itm->Uri.c_str(),ProperHost.c_str());
 592    string Req = Buf;
 593
 594    // Check for a partial file
 595    struct stat SBuf;
 596    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 597    {
 598       // In this case we send an if-range query with a range header
 599       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 600               TimeRFC1123(SBuf.st_mtime).c_str());
 601       Req += Buf;
 602    }
 603    else
 604    {
 605       if (Itm->LastModified != 0)
 606       {
 607          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 608          Req += Buf;
 609       }
 610    }
 611
 612 /*   if (ProxyAuth.empty() == false)
 613       Req += string("Proxy-Authorization: Basic ") + Base64Encode(ProxyAuth) + "\r\n";*/
 614
 615    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 616 //   cout << Req << endl;
 617
 618    Out.Read(Req);
 619 }
 620                                                                         /*}}}*/
 621 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 622 // ---------------------------------------------------------------------
 623 /* This runs the select loop over the server FDs, Output file FDs and
 624    stdin. */
 625 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 626 {
 627    // Server has closed the connection
 628    if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
 629       return false;
 630
 631    fd_set rfds,wfds,efds;
 632    FD_ZERO(&rfds);
 633    FD_ZERO(&wfds);
 634    FD_ZERO(&efds);
 635
 636    // Add the server
 637    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 638       FD_SET(Srv->ServerFd,&wfds);
 639    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 640       FD_SET(Srv->ServerFd,&rfds);
 641
 642    // Add the file
 643    int FileFD = -1;
 644    if (File != 0)
 645       FileFD = File->Fd();
 646
 647    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 648       FD_SET(FileFD,&wfds);
 649
 650    // Add stdin
 651    FD_SET(STDIN_FILENO,&rfds);
 652
 653    // Error Set
 654    if (FileFD != -1)
 655       FD_SET(FileFD,&efds);
 656    if (Srv->ServerFd != -1)
 657       FD_SET(Srv->ServerFd,&efds);
 658
 659    // Figure out the max fd
 660    int MaxFd = FileFD;
 661    if (MaxFd < Srv->ServerFd)
 662       MaxFd = Srv->ServerFd;
 663
 664    // Select
 665    struct timeval tv;
 666    tv.tv_sec = 120;
 667    tv.tv_usec = 0;
 668    int Res = 0;
 669    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 670       return _error->Errno("select","Select failed");
 671
 672    if (Res == 0)
 673    {
 674       _error->Error("Connection timed out");
 675       return ServerDie(Srv);
 676    }
 677
 678    // Some kind of exception (error) on the sockets, die
 679    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 680        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 681       return _error->Error("Socket Exception");
 682
 683    // Handle server IO
 684    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 685    {
 686       errno = 0;
 687       if (Srv->In.Read(Srv->ServerFd) == false)
 688          return ServerDie(Srv);
 689    }
 690
 691    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 692    {
 693       errno = 0;
 694       if (Srv->Out.Write(Srv->ServerFd) == false)
 695          return ServerDie(Srv);
 696    }
 697
 698    // Send data to the file
 699    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 700    {
 701       if (Srv->In.Write(FileFD) == false)
 702          return _error->Errno("write","Error writing to output file");
 703    }
 704
 705    // Handle commands from APT
 706    if (FD_ISSET(STDIN_FILENO,&rfds))
 707    {
 708       if (Run(true) != 0)
 709          exit(100);
 710    }
 711
 712    return true;
 713 }
 714                                                                         /*}}}*/
 715 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 716 // ---------------------------------------------------------------------
 717 /* This takes the current input buffer from the Server FD and writes it
 718    into the file */
 719 bool HttpMethod::Flush(ServerState *Srv)
 720 {
 721    if (File != 0)
 722    {
 723       SetNonBlock(File->Fd(),false);
 724       if (Srv->In.WriteSpace() == false)
 725          return true;
 726
 727       while (Srv->In.WriteSpace() == true)
 728       {
 729          if (Srv->In.Write(File->Fd()) == false)
 730             return _error->Errno("write","Error writing to file");
 731          if (Srv->In.IsLimit() == true)
 732             return true;
 733       }
 734
 735       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 736          return true;
 737    }
 738    return false;
 739 }
 740                                                                         /*}}}*/
 741 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 742 // ---------------------------------------------------------------------
 743 /* */
 744 bool HttpMethod::ServerDie(ServerState *Srv)
 745 {
 746    // Dump the buffer to the file
 747    if (Srv->State == ServerState::Data)
 748    {
 749       SetNonBlock(File->Fd(),false);
 750       while (Srv->In.WriteSpace() == true)
 751       {
 752          if (Srv->In.Write(File->Fd()) == false)
 753             return _error->Errno("write","Error writing to the file");
 754
 755          // Done
 756          if (Srv->In.IsLimit() == true)
 757             return true;
 758       }
 759    }
 760
 761    // See if this is because the server finished the data stream
 762    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 763        Srv->Encoding != ServerState::Closes)
 764    {
 765       if (errno == 0)
 766          return _error->Error("Error reading from server Remote end closed connection");
 767       return _error->Errno("read","Error reading from server");
 768    }
 769    else
 770    {
 771       Srv->In.Limit(-1);
 772
 773       // Nothing left in the buffer
 774       if (Srv->In.WriteSpace() == false)
 775          return false;
 776
 777       // We may have got multiple responses back in one packet..
 778       Srv->Close();
 779       return true;
 780    }
 781
 782    return false;
 783 }
 784                                                                         /*}}}*/
 785 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 786 // ---------------------------------------------------------------------
 787 /* We look at the header data we got back from the server and decide what
 788    to do. Returns
 789      0 - File is open,
 790      1 - IMS hit
 791      3 - Unrecoverable error
 792      4 - Error with error content page */
 793 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 794 {
 795    // Not Modified
 796    if (Srv->Result == 304)
 797    {
 798       unlink(Queue->DestFile.c_str());
 799       Res.IMSHit = true;
 800       Res.LastModified = Queue->LastModified;
 801       return 1;
 802    }
 803
 804    /* We have a reply we dont handle. This should indicate a perm server
 805       failure */
 806    if (Srv->Result < 200 || Srv->Result >= 300)
 807    {
 808       _error->Error("%u %s",Srv->Result,Srv->Code);
 809       if (Srv->HaveContent == true)
 810          return 4;
 811       return 3;
 812    }
 813
 814    // This is some sort of 2xx 'data follows' reply
 815    Res.LastModified = Srv->Date;
 816    Res.Size = Srv->Size;
 817
 818    // Open the file
 819    delete File;
 820    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 821    if (_error->PendingError() == true)
 822       return 3;
 823
 824    FailFile = Queue->DestFile;
 825    FailFd = File->Fd();
 826    FailTime = Srv->Date;
 827
 828    // Set the expected size
 829    if (Srv->StartPos >= 0)
 830    {
 831       Res.ResumePoint = Srv->StartPos;
 832       ftruncate(File->Fd(),Srv->StartPos);
 833    }
 834
 835    // Set the start point
 836    lseek(File->Fd(),0,SEEK_END);
 837
 838    delete Srv->In.MD5;
 839    Srv->In.MD5 = new MD5Summation;
 840
 841    // Fill the MD5 Hash if the file is non-empty (resume)
 842    if (Srv->StartPos > 0)
 843    {
 844       lseek(File->Fd(),0,SEEK_SET);
 845       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 846       {
 847          _error->Errno("read","Problem hashing file");
 848          return 3;
 849       }
 850       lseek(File->Fd(),0,SEEK_END);
 851    }
 852
 853    SetNonBlock(File->Fd(),true);
 854    return 0;
 855 }
 856                                                                         /*}}}*/
 857 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 858 // ---------------------------------------------------------------------
 859 /* This closes and timestamps the open file. This is neccessary to get
 860    resume behavoir on user abort */
 861 void HttpMethod::SigTerm(int)
 862 {
 863    if (FailFd == -1)
 864       exit(100);
 865    close(FailFd);
 866
 867    // Timestamp
 868    struct utimbuf UBuf;
 869    time(&UBuf.actime);
 870    UBuf.actime = FailTime;
 871    UBuf.modtime = FailTime;
 872    utime(FailFile.c_str(),&UBuf);
 873
 874    exit(100);
 875 }
 876                                                                         /*}}}*/
 877 // HttpMethod::Loop - Main loop                                         /*{{{*/
 878 // ---------------------------------------------------------------------
 879 /* */
 880 int HttpMethod::Loop()
 881 {
 882    signal(SIGTERM,SigTerm);
 883    signal(SIGINT,SigTerm);
 884
 885    ServerState *Server = 0;
 886
 887    int FailCounter = 0;
 888    while (1)
 889    {
 890       if (FailCounter >= 2)
 891       {
 892          Fail("Massive Server Brain Damage");
 893          FailCounter = 0;
 894       }
 895
 896       // We have no commands, wait for some to arrive
 897       if (Queue == 0)
 898       {
 899          if (WaitFd(STDIN_FILENO) == false)
 900             return 0;
 901       }
 902
 903       // Run messages
 904       if (Run(true) != 0)
 905          return 100;
 906
 907       if (Queue == 0)
 908          continue;
 909
 910       // Connect to the server
 911       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 912       {
 913          delete Server;
 914          Server = new ServerState(Queue->Uri,this);
 915       }
 916
 917       // Connnect to the host
 918       if (Server->Open() == false)
 919       {
 920          Fail();
 921          continue;
 922       }
 923
 924       // Queue the request
 925       SendReq(Queue,Server->Out);
 926
 927       // Fetch the next URL header data from the server.
 928       switch (Server->RunHeaders())
 929       {
 930          case 0:
 931          break;
 932
 933          // The header data is bad
 934          case 2:
 935          {
 936             _error->Error("Bad header Data");
 937             Fail();
 938             continue;
 939          }
 940
 941          // The server closed a connection during the header get..
 942          default:
 943          case 1:
 944          {
 945             FailCounter++;
 946             _error->DumpErrors();
 947             Server->Close();
 948             continue;
 949          }
 950       };
 951
 952       // Decide what to do.
 953       FetchResult Res;
 954       switch (DealWithHeaders(Res,Server))
 955       {
 956          // Ok, the file is Open
 957          case 0:
 958          {
 959             URIStart(Res);
 960
 961             // Run the data
 962             bool Result =  Server->RunData();
 963
 964             // Close the file, destroy the FD object and timestamp it
 965             FailFd = -1;
 966             delete File;
 967             File = 0;
 968
 969             // Timestamp
 970             struct utimbuf UBuf;
 971             time(&UBuf.actime);
 972             UBuf.actime = Server->Date;
 973             UBuf.modtime = Server->Date;
 974             utime(Queue->DestFile.c_str(),&UBuf);
 975
 976             // Send status to APT
 977             if (Result == true)
 978             {
 979                Res.MD5Sum = Server->In.MD5->Result();
 980                URIDone(Res);
 981             }
 982             else
 983                Fail();
 984
 985             break;
 986          }
 987
 988          // IMS hit
 989          case 1:
 990          {
 991             URIDone(Res);
 992             break;
 993          }
 994
 995          // Hard server error, not found or something
 996          case 3:
 997          {
 998             Fail();
 999             break;
1000          }
1001
1002          // We need to flush the data, the header is like a 404 w/ error text
1003          case 4:
1004          {
1005             Fail();
1006
1007             // Send to content to dev/null
1008             File = new FileFd("/dev/null",FileFd::WriteExists);
1009             Server->RunData();
1010             delete File;
1011             File = 0;
1012             break;
1013          }
1014
1015          default:
1016          Fail("Internal error");
1017          break;
1018       }
1019
1020       FailCounter = 0;
1021    }
1022
1023    return 0;
1024 }
1025                                                                         /*}}}*/
1026
1027 int main()
1028 {
1029    HttpMethod Mth;
1030
1031    return Mth.Loop();
1032 }