methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.38 1999/10/03 21:09:27 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41 #include <errno.h>
  42
  43 // Internet stuff
  44 #include <netdb.h>
  45
  46 #include "connect.h"
  47 #include "rfc2553emu.h"
  48 #include "http.h"
  49
  50                                                                         /*}}}*/
  51
  52 string HttpMethod::FailFile;
  53 int HttpMethod::FailFd = -1;
  54 time_t HttpMethod::FailTime = 0;
  55 unsigned long PipelineDepth = 10;
  56 unsigned long TimeOut = 120;
  57 bool Debug = false;
  58
  59 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  60 // ---------------------------------------------------------------------
  61 /* */
  62 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  63 {
  64    Buf = new unsigned char[Size];
  65    Reset();
  66 }
  67                                                                         /*}}}*/
  68 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  69 // ---------------------------------------------------------------------
  70 /* */
  71 void CircleBuf::Reset()
  72 {
  73    InP = 0;
  74    OutP = 0;
  75    StrPos = 0;
  76    MaxGet = (unsigned int)-1;
  77    OutQueue = string();
  78    if (MD5 != 0)
  79    {
  80       delete MD5;
  81       MD5 = new MD5Summation;
  82    }
  83 };
  84                                                                         /*}}}*/
  85 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* This fills up the buffer with as much data as is in the FD, assuming it
  88    is non-blocking.. */
  89 bool CircleBuf::Read(int Fd)
  90 {
  91    while (1)
  92    {
  93       // Woops, buffer is full
  94       if (InP - OutP == Size)
  95          return true;
  96
  97       // Write the buffer segment
  98       int Res;
  99       Res = read(Fd,Buf + (InP%Size),LeftRead());
 100
 101       if (Res == 0)
 102          return false;
 103       if (Res < 0)
 104       {
 105          if (errno == EAGAIN)
 106             return true;
 107          return false;
 108       }
 109
 110       if (InP == 0)
 111          gettimeofday(&Start,0);
 112       InP += Res;
 113    }
 114 }
 115                                                                         /*}}}*/
 116 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 117 // ---------------------------------------------------------------------
 118 /* This will hold the string in and fill the buffer with it as it empties */
 119 bool CircleBuf::Read(string Data)
 120 {
 121    OutQueue += Data;
 122    FillOut();
 123    return true;
 124 }
 125                                                                         /*}}}*/
 126 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 127 // ---------------------------------------------------------------------
 128 /* */
 129 void CircleBuf::FillOut()
 130 {
 131    if (OutQueue.empty() == true)
 132       return;
 133    while (1)
 134    {
 135       // Woops, buffer is full
 136       if (InP - OutP == Size)
 137          return;
 138
 139       // Write the buffer segment
 140       unsigned long Sz = LeftRead();
 141       if (OutQueue.length() - StrPos < Sz)
 142          Sz = OutQueue.length() - StrPos;
 143       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 144
 145       // Advance
 146       StrPos += Sz;
 147       InP += Sz;
 148       if (OutQueue.length() == StrPos)
 149       {
 150          StrPos = 0;
 151          OutQueue = "";
 152          return;
 153       }
 154    }
 155 }
 156                                                                         /*}}}*/
 157 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 158 // ---------------------------------------------------------------------
 159 /* This empties the buffer into the FD. */
 160 bool CircleBuf::Write(int Fd)
 161 {
 162    while (1)
 163    {
 164       FillOut();
 165
 166       // Woops, buffer is empty
 167       if (OutP == InP)
 168          return true;
 169
 170       if (OutP == MaxGet)
 171          return true;
 172
 173       // Write the buffer segment
 174       int Res;
 175       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 176
 177       if (Res == 0)
 178          return false;
 179       if (Res < 0)
 180       {
 181          if (errno == EAGAIN)
 182             return true;
 183
 184          return false;
 185       }
 186
 187       if (MD5 != 0)
 188          MD5->Add(Buf + (OutP%Size),Res);
 189
 190       OutP += Res;
 191    }
 192 }
 193                                                                         /*}}}*/
 194 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 195 // ---------------------------------------------------------------------
 196 /* This copies till the first empty line */
 197 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 198 {
 199    // We cheat and assume it is unneeded to have more than one buffer load
 200    for (unsigned long I = OutP; I < InP; I++)
 201    {
 202       if (Buf[I%Size] != '\n')
 203          continue;
 204       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 205
 206       if (Single == false)
 207       {
 208          if (Buf[I%Size] != '\n')
 209             continue;
 210          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 211       }
 212
 213       if (I > InP)
 214          I = InP;
 215
 216       Data = "";
 217       while (OutP < I)
 218       {
 219          unsigned long Sz = LeftWrite();
 220          if (Sz == 0)
 221             return false;
 222          if (I - OutP < LeftWrite())
 223             Sz = I - OutP;
 224          Data += string((char *)(Buf + (OutP%Size)),Sz);
 225          OutP += Sz;
 226       }
 227       return true;
 228    }
 229    return false;
 230 }
 231                                                                         /*}}}*/
 232 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 233 // ---------------------------------------------------------------------
 234 /* */
 235 void CircleBuf::Stats()
 236 {
 237    if (InP == 0)
 238       return;
 239
 240    struct timeval Stop;
 241    gettimeofday(&Stop,0);
 242 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 243              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 244    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 245 }
 246                                                                         /*}}}*/
 247
 248 // ServerState::ServerState - Constructor                               /*{{{*/
 249 // ---------------------------------------------------------------------
 250 /* */
 251 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 252                         In(64*1024), Out(4*1024),
 253                         ServerName(Srv)
 254 {
 255    Reset();
 256 }
 257                                                                         /*}}}*/
 258 // ServerState::Open - Open a connection to the server                  /*{{{*/
 259 // ---------------------------------------------------------------------
 260 /* This opens a connection to the server. */
 261 string LastHost;
 262 int LastPort = 0;
 263 struct addrinfo *LastHostAddr = 0;
 264 bool ServerState::Open()
 265 {
 266    // Use the already open connection if possible.
 267    if (ServerFd != -1)
 268       return true;
 269
 270    Close();
 271    In.Reset();
 272    Out.Reset();
 273
 274    // Determine the proxy setting
 275    if (getenv("http_proxy") == 0)
 276    {
 277       string DefProxy = _config->Find("Acquire::http::Proxy");
 278       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 279       if (SpecificProxy.empty() == false)
 280       {
 281          if (SpecificProxy == "DIRECT")
 282             Proxy = "";
 283          else
 284             Proxy = SpecificProxy;
 285       }
 286       else
 287          Proxy = DefProxy;
 288    }
 289    else
 290       Proxy = getenv("http_proxy");
 291
 292    // Determine what host and port to use based on the proxy settings
 293    int Port = 0;
 294    string Host;
 295    if (Proxy.empty() == true)
 296    {
 297       if (ServerName.Port != 0)
 298          Port = ServerName.Port;
 299       Host = ServerName.Host;
 300    }
 301    else
 302    {
 303       if (Proxy.Port != 0)
 304          Port = Proxy.Port;
 305       Host = Proxy.Host;
 306    }
 307
 308    // Connect to the remote server
 309    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 310       return false;
 311
 312    return true;
 313 }
 314                                                                         /*}}}*/
 315 // ServerState::Close - Close a connection to the server                /*{{{*/
 316 // ---------------------------------------------------------------------
 317 /* */
 318 bool ServerState::Close()
 319 {
 320    close(ServerFd);
 321    ServerFd = -1;
 322    return true;
 323 }
 324                                                                         /*}}}*/
 325 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 326 // ---------------------------------------------------------------------
 327 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 328    parse error occured */
 329 int ServerState::RunHeaders()
 330 {
 331    State = Header;
 332
 333    Owner->Status("Waiting for file");
 334
 335    Major = 0;
 336    Minor = 0;
 337    Result = 0;
 338    Size = 0;
 339    StartPos = 0;
 340    Encoding = Closes;
 341    HaveContent = false;
 342    time(&Date);
 343
 344    do
 345    {
 346       string Data;
 347       if (In.WriteTillEl(Data) == false)
 348          continue;
 349
 350       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 351       {
 352          string::const_iterator J = I;
 353          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 354          if (HeaderLine(string(I,J-I)) == false)
 355             return 2;
 356          I = J;
 357       }
 358       return 0;
 359    }
 360    while (Owner->Go(false,this) == true);
 361
 362    return 1;
 363 }
 364                                                                         /*}}}*/
 365 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 366 // ---------------------------------------------------------------------
 367 /* */
 368 bool ServerState::RunData()
 369 {
 370    State = Data;
 371
 372    // Chunked transfer encoding is fun..
 373    if (Encoding == Chunked)
 374    {
 375       while (1)
 376       {
 377          // Grab the block size
 378          bool Last = true;
 379          string Data;
 380          In.Limit(-1);
 381          do
 382          {
 383             if (In.WriteTillEl(Data,true) == true)
 384                break;
 385          }
 386          while ((Last = Owner->Go(false,this)) == true);
 387
 388          if (Last == false)
 389             return false;
 390
 391          // See if we are done
 392          unsigned long Len = strtol(Data.c_str(),0,16);
 393          if (Len == 0)
 394          {
 395             In.Limit(-1);
 396
 397             // We have to remove the entity trailer
 398             Last = true;
 399             do
 400             {
 401                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 402                   break;
 403             }
 404             while ((Last = Owner->Go(false,this)) == true);
 405             if (Last == false)
 406                return false;
 407             return !_error->PendingError();
 408          }
 409
 410          // Transfer the block
 411          In.Limit(Len);
 412          while (Owner->Go(true,this) == true)
 413             if (In.IsLimit() == true)
 414                break;
 415
 416          // Error
 417          if (In.IsLimit() == false)
 418             return false;
 419
 420          // The server sends an extra new line before the next block specifier..
 421          In.Limit(-1);
 422          Last = true;
 423          do
 424          {
 425             if (In.WriteTillEl(Data,true) == true)
 426                break;
 427          }
 428          while ((Last = Owner->Go(false,this)) == true);
 429          if (Last == false)
 430             return false;
 431       }
 432    }
 433    else
 434    {
 435       /* Closes encoding is used when the server did not specify a size, the
 436          loss of the connection means we are done */
 437       if (Encoding == Closes)
 438          In.Limit(-1);
 439       else
 440          In.Limit(Size - StartPos);
 441
 442       // Just transfer the whole block.
 443       do
 444       {
 445          if (In.IsLimit() == false)
 446             continue;
 447
 448          In.Limit(-1);
 449          return !_error->PendingError();
 450       }
 451       while (Owner->Go(true,this) == true);
 452    }
 453
 454    return Owner->Flush(this) && !_error->PendingError();
 455 }
 456                                                                         /*}}}*/
 457 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 458 // ---------------------------------------------------------------------
 459 /* */
 460 bool ServerState::HeaderLine(string Line)
 461 {
 462    if (Line.empty() == true)
 463       return true;
 464
 465    // The http server might be trying to do something evil.
 466    if (Line.length() >= MAXLEN)
 467       return _error->Error("Got a single header line over %u chars",MAXLEN);
 468
 469    string::size_type Pos = Line.find(' ');
 470    if (Pos == string::npos || Pos+1 > Line.length())
 471    {
 472       // Blah, some servers use "connection:closes", evil.
 473       Pos = Line.find(':');
 474       if (Pos == string::npos || Pos + 2 > Line.length())
 475          return _error->Error("Bad header line");
 476       Pos++;
 477    }
 478
 479    // Parse off any trailing spaces between the : and the next word.
 480    string::size_type Pos2 = Pos;
 481    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 482       Pos2++;
 483
 484    string Tag = string(Line,0,Pos);
 485    string Val = string(Line,Pos2);
 486
 487    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 488    {
 489       // Evil servers return no version
 490       if (Line[4] == '/')
 491       {
 492          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 493                     &Result,Code) != 4)
 494             return _error->Error("The http server sent an invalid reply header");
 495       }
 496       else
 497       {
 498          Major = 0;
 499          Minor = 9;
 500          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 501             return _error->Error("The http server sent an invalid reply header");
 502       }
 503
 504       return true;
 505    }
 506
 507    if (stringcasecmp(Tag,"Content-Length:") == 0)
 508    {
 509       if (Encoding == Closes)
 510          Encoding = Stream;
 511       HaveContent = true;
 512
 513       // The length is already set from the Content-Range header
 514       if (StartPos != 0)
 515          return true;
 516
 517       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 518          return _error->Error("The http server sent an invalid Content-Length header");
 519       return true;
 520    }
 521
 522    if (stringcasecmp(Tag,"Content-Type:") == 0)
 523    {
 524       HaveContent = true;
 525       return true;
 526    }
 527
 528    if (stringcasecmp(Tag,"Content-Range:") == 0)
 529    {
 530       HaveContent = true;
 531
 532       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 533          return _error->Error("The http server sent an invalid Content-Range header");
 534       if ((unsigned)StartPos > Size)
 535          return _error->Error("This http server has broken range support");
 536       return true;
 537    }
 538
 539    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 540    {
 541       HaveContent = true;
 542       if (stringcasecmp(Val,"chunked") == 0)
 543          Encoding = Chunked;
 544
 545       return true;
 546    }
 547
 548    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 549    {
 550       if (StrToTime(Val,Date) == false)
 551          return _error->Error("Unknown date format");
 552       return true;
 553    }
 554
 555    return true;
 556 }
 557                                                                         /*}}}*/
 558
 559 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 560 // ---------------------------------------------------------------------
 561 /* This places the http request in the outbound buffer */
 562 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 563 {
 564    URI Uri = Itm->Uri;
 565
 566    // The HTTP server expects a hostname with a trailing :port
 567    char Buf[1000];
 568    string ProperHost = Uri.Host;
 569    if (Uri.Port != 0)
 570    {
 571       sprintf(Buf,":%u",Uri.Port);
 572       ProperHost += Buf;
 573    }
 574
 575    // Just in case.
 576    if (Itm->Uri.length() >= sizeof(Buf))
 577        abort();
 578
 579    /* Build the request. We include a keep-alive header only for non-proxy
 580       requests. This is to tweak old http/1.0 servers that do support keep-alive
 581       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 582       will glitch HTTP/1.0 proxies because they do not filter it out and
 583       pass it on, HTTP/1.1 says the connection should default to keep alive
 584       and we expect the proxy to do this */
 585    if (Proxy.empty() == true)
 586       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 587               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 588    else
 589    {
 590       /* Generate a cache control header if necessary. We place a max
 591          cache age on index files, optionally set a no-cache directive
 592          and a no-store directive for archives. */
 593       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 594               Itm->Uri.c_str(),ProperHost.c_str());
 595       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 596          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 597       else
 598       {
 599          if (Itm->IndexFile == true)
 600             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 601                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 602          else
 603          {
 604             if (_config->FindB("Acquire::http::No-Store",false) == true)
 605                strcat(Buf,"Cache-Control: no-store\r\n");
 606          }
 607       }
 608    }
 609
 610    string Req = Buf;
 611
 612    // Check for a partial file
 613    struct stat SBuf;
 614    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 615    {
 616       // In this case we send an if-range query with a range header
 617       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 618               TimeRFC1123(SBuf.st_mtime).c_str());
 619       Req += Buf;
 620    }
 621    else
 622    {
 623       if (Itm->LastModified != 0)
 624       {
 625          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 626          Req += Buf;
 627       }
 628    }
 629
 630    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 631       Req += string("Proxy-Authorization: Basic ") +
 632           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 633
 634    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 635
 636    if (Debug == true)
 637       cerr << Req << endl;
 638
 639    Out.Read(Req);
 640 }
 641                                                                         /*}}}*/
 642 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 643 // ---------------------------------------------------------------------
 644 /* This runs the select loop over the server FDs, Output file FDs and
 645    stdin. */
 646 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 647 {
 648    // Server has closed the connection
 649    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 650                                ToFile == false))
 651       return false;
 652
 653    fd_set rfds,wfds,efds;
 654    FD_ZERO(&rfds);
 655    FD_ZERO(&wfds);
 656    FD_ZERO(&efds);
 657
 658    // Add the server
 659    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 660       FD_SET(Srv->ServerFd,&wfds);
 661    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 662       FD_SET(Srv->ServerFd,&rfds);
 663
 664    // Add the file
 665    int FileFD = -1;
 666    if (File != 0)
 667       FileFD = File->Fd();
 668
 669    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 670       FD_SET(FileFD,&wfds);
 671
 672    // Add stdin
 673    FD_SET(STDIN_FILENO,&rfds);
 674
 675    // Error Set
 676    if (FileFD != -1)
 677       FD_SET(FileFD,&efds);
 678    if (Srv->ServerFd != -1)
 679       FD_SET(Srv->ServerFd,&efds);
 680
 681    // Figure out the max fd
 682    int MaxFd = FileFD;
 683    if (MaxFd < Srv->ServerFd)
 684       MaxFd = Srv->ServerFd;
 685
 686    // Select
 687    struct timeval tv;
 688    tv.tv_sec = TimeOut;
 689    tv.tv_usec = 0;
 690    int Res = 0;
 691    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 692       return _error->Errno("select","Select failed");
 693
 694    if (Res == 0)
 695    {
 696       _error->Error("Connection timed out");
 697       return ServerDie(Srv);
 698    }
 699
 700    // Some kind of exception (error) on the sockets, die
 701    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 702        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 703       return _error->Error("Socket Exception");
 704
 705    // Handle server IO
 706    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 707    {
 708       errno = 0;
 709       if (Srv->In.Read(Srv->ServerFd) == false)
 710          return ServerDie(Srv);
 711    }
 712
 713    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 714    {
 715       errno = 0;
 716       if (Srv->Out.Write(Srv->ServerFd) == false)
 717          return ServerDie(Srv);
 718    }
 719
 720    // Send data to the file
 721    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 722    {
 723       if (Srv->In.Write(FileFD) == false)
 724          return _error->Errno("write","Error writing to output file");
 725    }
 726
 727    // Handle commands from APT
 728    if (FD_ISSET(STDIN_FILENO,&rfds))
 729    {
 730       if (Run(true) != -1)
 731          exit(100);
 732    }
 733
 734    return true;
 735 }
 736                                                                         /*}}}*/
 737 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 738 // ---------------------------------------------------------------------
 739 /* This takes the current input buffer from the Server FD and writes it
 740    into the file */
 741 bool HttpMethod::Flush(ServerState *Srv)
 742 {
 743    if (File != 0)
 744    {
 745       SetNonBlock(File->Fd(),false);
 746       if (Srv->In.WriteSpace() == false)
 747          return true;
 748
 749       while (Srv->In.WriteSpace() == true)
 750       {
 751          if (Srv->In.Write(File->Fd()) == false)
 752             return _error->Errno("write","Error writing to file");
 753          if (Srv->In.IsLimit() == true)
 754             return true;
 755       }
 756
 757       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 758          return true;
 759    }
 760    return false;
 761 }
 762                                                                         /*}}}*/
 763 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 764 // ---------------------------------------------------------------------
 765 /* */
 766 bool HttpMethod::ServerDie(ServerState *Srv)
 767 {
 768    unsigned int LErrno = errno;
 769
 770    // Dump the buffer to the file
 771    if (Srv->State == ServerState::Data)
 772    {
 773       SetNonBlock(File->Fd(),false);
 774       while (Srv->In.WriteSpace() == true)
 775       {
 776          if (Srv->In.Write(File->Fd()) == false)
 777             return _error->Errno("write","Error writing to the file");
 778
 779          // Done
 780          if (Srv->In.IsLimit() == true)
 781             return true;
 782       }
 783    }
 784
 785    // See if this is because the server finished the data stream
 786    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 787        Srv->Encoding != ServerState::Closes)
 788    {
 789       Srv->Close();
 790       if (LErrno == 0)
 791          return _error->Error("Error reading from server Remote end closed connection");
 792       errno = LErrno;
 793       return _error->Errno("read","Error reading from server");
 794    }
 795    else
 796    {
 797       Srv->In.Limit(-1);
 798
 799       // Nothing left in the buffer
 800       if (Srv->In.WriteSpace() == false)
 801          return false;
 802
 803       // We may have got multiple responses back in one packet..
 804       Srv->Close();
 805       return true;
 806    }
 807
 808    return false;
 809 }
 810                                                                         /*}}}*/
 811 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 812 // ---------------------------------------------------------------------
 813 /* We look at the header data we got back from the server and decide what
 814    to do. Returns
 815      0 - File is open,
 816      1 - IMS hit
 817      3 - Unrecoverable error
 818      4 - Error with error content page
 819      5 - Unrecoverable non-server error (close the connection) */
 820 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 821 {
 822    // Not Modified
 823    if (Srv->Result == 304)
 824    {
 825       unlink(Queue->DestFile.c_str());
 826       Res.IMSHit = true;
 827       Res.LastModified = Queue->LastModified;
 828       return 1;
 829    }
 830
 831    /* We have a reply we dont handle. This should indicate a perm server
 832       failure */
 833    if (Srv->Result < 200 || Srv->Result >= 300)
 834    {
 835       _error->Error("%u %s",Srv->Result,Srv->Code);
 836       if (Srv->HaveContent == true)
 837          return 4;
 838       return 3;
 839    }
 840
 841    // This is some sort of 2xx 'data follows' reply
 842    Res.LastModified = Srv->Date;
 843    Res.Size = Srv->Size;
 844
 845    // Open the file
 846    delete File;
 847    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 848    if (_error->PendingError() == true)
 849       return 5;
 850
 851    FailFile = Queue->DestFile;
 852    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 853    FailFd = File->Fd();
 854    FailTime = Srv->Date;
 855
 856    // Set the expected size
 857    if (Srv->StartPos >= 0)
 858    {
 859       Res.ResumePoint = Srv->StartPos;
 860       ftruncate(File->Fd(),Srv->StartPos);
 861    }
 862
 863    // Set the start point
 864    lseek(File->Fd(),0,SEEK_END);
 865
 866    delete Srv->In.MD5;
 867    Srv->In.MD5 = new MD5Summation;
 868
 869    // Fill the MD5 Hash if the file is non-empty (resume)
 870    if (Srv->StartPos > 0)
 871    {
 872       lseek(File->Fd(),0,SEEK_SET);
 873       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 874       {
 875          _error->Errno("read","Problem hashing file");
 876          return 5;
 877       }
 878       lseek(File->Fd(),0,SEEK_END);
 879    }
 880
 881    SetNonBlock(File->Fd(),true);
 882    return 0;
 883 }
 884                                                                         /*}}}*/
 885 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 886 // ---------------------------------------------------------------------
 887 /* This closes and timestamps the open file. This is neccessary to get
 888    resume behavoir on user abort */
 889 void HttpMethod::SigTerm(int)
 890 {
 891    if (FailFd == -1)
 892       exit(100);
 893    close(FailFd);
 894
 895    // Timestamp
 896    struct utimbuf UBuf;
 897    UBuf.actime = FailTime;
 898    UBuf.modtime = FailTime;
 899    utime(FailFile.c_str(),&UBuf);
 900
 901    exit(100);
 902 }
 903                                                                         /*}}}*/
 904 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 905 // ---------------------------------------------------------------------
 906 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 907    depth. */
 908 bool HttpMethod::Fetch(FetchItem *)
 909 {
 910    if (Server == 0)
 911       return true;
 912
 913    // Queue the requests
 914    int Depth = -1;
 915    bool Tail = false;
 916    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++)
 917    {
 918       // Make sure we stick with the same server
 919       if (Server->Comp(I->Uri) == false)
 920          break;
 921       if (QueueBack == I)
 922          Tail = true;
 923       if (Tail == true)
 924       {
 925          QueueBack = I->Next;
 926          SendReq(I,Server->Out);
 927          continue;
 928       }
 929    }
 930
 931    return true;
 932 };
 933                                                                         /*}}}*/
 934 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 935 // ---------------------------------------------------------------------
 936 /* We stash the desired pipeline depth */
 937 bool HttpMethod::Configuration(string Message)
 938 {
 939    if (pkgAcqMethod::Configuration(Message) == false)
 940       return false;
 941
 942    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 943    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 944                                   PipelineDepth);
 945    Debug = _config->FindB("Debug::Acquire::http",false);
 946
 947    return true;
 948 }
 949                                                                         /*}}}*/
 950 // HttpMethod::Loop - Main loop                                         /*{{{*/
 951 // ---------------------------------------------------------------------
 952 /* */
 953 int HttpMethod::Loop()
 954 {
 955    signal(SIGTERM,SigTerm);
 956    signal(SIGINT,SigTerm);
 957
 958    Server = 0;
 959
 960    int FailCounter = 0;
 961    while (1)
 962    {
 963       // We have no commands, wait for some to arrive
 964       if (Queue == 0)
 965       {
 966          if (WaitFd(STDIN_FILENO) == false)
 967             return 0;
 968       }
 969
 970       /* Run messages, we can accept 0 (no message) if we didn't
 971          do a WaitFd above.. Otherwise the FD is closed. */
 972       int Result = Run(true);
 973       if (Result != -1 && (Result != 0 || Queue == 0))
 974          return 100;
 975
 976       if (Queue == 0)
 977          continue;
 978
 979       // Connect to the server
 980       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 981       {
 982          delete Server;
 983          Server = new ServerState(Queue->Uri,this);
 984       }
 985
 986       // Reset the pipeline
 987       if (Server->ServerFd == -1)
 988          QueueBack = Queue;
 989
 990       // Connnect to the host
 991       if (Server->Open() == false)
 992       {
 993          Fail(true);
 994          delete Server;
 995          Server = 0;
 996          continue;
 997       }
 998
 999       // Fill the pipeline.
1000       Fetch(0);
1001
1002       // Fetch the next URL header data from the server.
1003       switch (Server->RunHeaders())
1004       {
1005          case 0:
1006          break;
1007
1008          // The header data is bad
1009          case 2:
1010          {
1011             _error->Error("Bad header Data");
1012             Fail(true);
1013             continue;
1014          }
1015
1016          // The server closed a connection during the header get..
1017          default:
1018          case 1:
1019          {
1020             FailCounter++;
1021             _error->Discard();
1022             Server->Close();
1023
1024             if (FailCounter >= 2)
1025             {
1026                Fail("Connection failed",true);
1027                FailCounter = 0;
1028             }
1029
1030             continue;
1031          }
1032       };
1033
1034       // Decide what to do.
1035       FetchResult Res;
1036       Res.Filename = Queue->DestFile;
1037       switch (DealWithHeaders(Res,Server))
1038       {
1039          // Ok, the file is Open
1040          case 0:
1041          {
1042             URIStart(Res);
1043
1044             // Run the data
1045             bool Result =  Server->RunData();
1046
1047             // Close the file, destroy the FD object and timestamp it
1048             FailFd = -1;
1049             delete File;
1050             File = 0;
1051
1052             // Timestamp
1053             struct utimbuf UBuf;
1054             time(&UBuf.actime);
1055             UBuf.actime = Server->Date;
1056             UBuf.modtime = Server->Date;
1057             utime(Queue->DestFile.c_str(),&UBuf);
1058
1059             // Send status to APT
1060             if (Result == true)
1061             {
1062                Res.MD5Sum = Server->In.MD5->Result();
1063                URIDone(Res);
1064             }
1065             else
1066                Fail(true);
1067
1068             break;
1069          }
1070
1071          // IMS hit
1072          case 1:
1073          {
1074             URIDone(Res);
1075             break;
1076          }
1077
1078          // Hard server error, not found or something
1079          case 3:
1080          {
1081             Fail();
1082             break;
1083          }
1084
1085          // Hard internal error, kill the connection and fail
1086          case 5:
1087          {
1088             Fail();
1089             Server->Close();
1090             break;
1091          }
1092
1093          // We need to flush the data, the header is like a 404 w/ error text
1094          case 4:
1095          {
1096             Fail();
1097
1098             // Send to content to dev/null
1099             File = new FileFd("/dev/null",FileFd::WriteExists);
1100             Server->RunData();
1101             delete File;
1102             File = 0;
1103             break;
1104          }
1105
1106          default:
1107          Fail("Internal error");
1108          break;
1109       }
1110
1111       FailCounter = 0;
1112    }
1113
1114    return 0;
1115 }
1116                                                                         /*}}}*/
1117
1118 int main()
1119 {
1120    HttpMethod Mth;
1121
1122    return Mth.Loop();
1123 }