methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.31 1999/05/25 05:56:24 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41 #include <errno.h>
  42
  43 // Internet stuff
  44 /*#include <netinet/in.h>
  45 #include <sys/socket.h>
  46 #include <arpa/inet.h>
  47 #include <netdb.h>*/
  48
  49 #include "rfc2553emu.h"
  50 #include "http.h"
  51
  52                                                                         /*}}}*/
  53
  54 string HttpMethod::FailFile;
  55 int HttpMethod::FailFd = -1;
  56 time_t HttpMethod::FailTime = 0;
  57 unsigned long PipelineDepth = 10;
  58 unsigned long TimeOut = 120;
  59
  60 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  61 // ---------------------------------------------------------------------
  62 /* */
  63 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  64 {
  65    Buf = new unsigned char[Size];
  66    Reset();
  67 }
  68                                                                         /*}}}*/
  69 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  70 // ---------------------------------------------------------------------
  71 /* */
  72 void CircleBuf::Reset()
  73 {
  74    InP = 0;
  75    OutP = 0;
  76    StrPos = 0;
  77    MaxGet = (unsigned int)-1;
  78    OutQueue = string();
  79    if (MD5 != 0)
  80    {
  81       delete MD5;
  82       MD5 = new MD5Summation;
  83    }
  84 };
  85                                                                         /*}}}*/
  86 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  87 // ---------------------------------------------------------------------
  88 /* This fills up the buffer with as much data as is in the FD, assuming it
  89    is non-blocking.. */
  90 bool CircleBuf::Read(int Fd)
  91 {
  92    while (1)
  93    {
  94       // Woops, buffer is full
  95       if (InP - OutP == Size)
  96          return true;
  97
  98       // Write the buffer segment
  99       int Res;
 100       Res = read(Fd,Buf + (InP%Size),LeftRead());
 101
 102       if (Res == 0)
 103          return false;
 104       if (Res < 0)
 105       {
 106          if (errno == EAGAIN)
 107             return true;
 108          return false;
 109       }
 110
 111       if (InP == 0)
 112          gettimeofday(&Start,0);
 113       InP += Res;
 114    }
 115 }
 116                                                                         /*}}}*/
 117 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 118 // ---------------------------------------------------------------------
 119 /* This will hold the string in and fill the buffer with it as it empties */
 120 bool CircleBuf::Read(string Data)
 121 {
 122    OutQueue += Data;
 123    FillOut();
 124    return true;
 125 }
 126                                                                         /*}}}*/
 127 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 128 // ---------------------------------------------------------------------
 129 /* */
 130 void CircleBuf::FillOut()
 131 {
 132    if (OutQueue.empty() == true)
 133       return;
 134    while (1)
 135    {
 136       // Woops, buffer is full
 137       if (InP - OutP == Size)
 138          return;
 139
 140       // Write the buffer segment
 141       unsigned long Sz = LeftRead();
 142       if (OutQueue.length() - StrPos < Sz)
 143          Sz = OutQueue.length() - StrPos;
 144       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 145
 146       // Advance
 147       StrPos += Sz;
 148       InP += Sz;
 149       if (OutQueue.length() == StrPos)
 150       {
 151          StrPos = 0;
 152          OutQueue = "";
 153          return;
 154       }
 155    }
 156 }
 157                                                                         /*}}}*/
 158 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 159 // ---------------------------------------------------------------------
 160 /* This empties the buffer into the FD. */
 161 bool CircleBuf::Write(int Fd)
 162 {
 163    while (1)
 164    {
 165       FillOut();
 166
 167       // Woops, buffer is empty
 168       if (OutP == InP)
 169          return true;
 170
 171       if (OutP == MaxGet)
 172          return true;
 173
 174       // Write the buffer segment
 175       int Res;
 176       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 177
 178       if (Res == 0)
 179          return false;
 180       if (Res < 0)
 181       {
 182          if (errno == EAGAIN)
 183             return true;
 184
 185          return false;
 186       }
 187
 188       if (MD5 != 0)
 189          MD5->Add(Buf + (OutP%Size),Res);
 190
 191       OutP += Res;
 192    }
 193 }
 194                                                                         /*}}}*/
 195 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 196 // ---------------------------------------------------------------------
 197 /* This copies till the first empty line */
 198 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 199 {
 200    // We cheat and assume it is unneeded to have more than one buffer load
 201    for (unsigned long I = OutP; I < InP; I++)
 202    {
 203       if (Buf[I%Size] != '\n')
 204          continue;
 205       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 206
 207       if (Single == false)
 208       {
 209          if (Buf[I%Size] != '\n')
 210             continue;
 211          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 212       }
 213
 214       if (I > InP)
 215          I = InP;
 216
 217       Data = "";
 218       while (OutP < I)
 219       {
 220          unsigned long Sz = LeftWrite();
 221          if (Sz == 0)
 222             return false;
 223          if (I - OutP < LeftWrite())
 224             Sz = I - OutP;
 225          Data += string((char *)(Buf + (OutP%Size)),Sz);
 226          OutP += Sz;
 227       }
 228       return true;
 229    }
 230    return false;
 231 }
 232                                                                         /*}}}*/
 233 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 234 // ---------------------------------------------------------------------
 235 /* */
 236 void CircleBuf::Stats()
 237 {
 238    if (InP == 0)
 239       return;
 240
 241    struct timeval Stop;
 242    gettimeofday(&Stop,0);
 243 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 244              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 245    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 246 }
 247                                                                         /*}}}*/
 248
 249 // ServerState::ServerState - Constructor                               /*{{{*/
 250 // ---------------------------------------------------------------------
 251 /* */
 252 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 253                         In(64*1024), Out(4*1024),
 254                         ServerName(Srv)
 255 {
 256    Reset();
 257 }
 258                                                                         /*}}}*/
 259 // ServerState::Open - Open a connection to the server                  /*{{{*/
 260 // ---------------------------------------------------------------------
 261 /* This opens a connection to the server. */
 262 string LastHost;
 263 int LastPort = 0;
 264 struct addrinfo *LastHostAddr = 0;
 265 bool ServerState::Open()
 266 {
 267    // Use the already open connection if possible.
 268    if (ServerFd != -1)
 269       return true;
 270
 271    Close();
 272    In.Reset();
 273    Out.Reset();
 274
 275    // Determine the proxy setting
 276    if (getenv("http_proxy") == 0)
 277    {
 278       string DefProxy = _config->Find("Acquire::http::Proxy");
 279       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 280       if (SpecificProxy.empty() == false)
 281       {
 282          if (SpecificProxy == "DIRECT")
 283             Proxy = "";
 284          else
 285             Proxy = SpecificProxy;
 286       }
 287       else
 288          Proxy = DefProxy;
 289    }
 290    else
 291       Proxy = getenv("http_proxy");
 292
 293    // Determine what host and port to use based on the proxy settings
 294    int Port = 0;
 295    string Host;
 296    if (Proxy.empty() == true)
 297    {
 298       if (ServerName.Port != 0)
 299          Port = ServerName.Port;
 300       Host = ServerName.Host;
 301    }
 302    else
 303    {
 304       if (Proxy.Port != 0)
 305          Port = Proxy.Port;
 306       Host = Proxy.Host;
 307    }
 308
 309    /* We used a cached address record.. Yes this is against the spec but
 310       the way we have setup our rotating dns suggests that this is more
 311       sensible */
 312    if (LastHost != Host || LastPort != Port)
 313    {
 314       Owner->Status("Connecting to %s",Host.c_str());
 315
 316       // Lookup the host
 317       char S[30] = "http";
 318       if (Port != 0)
 319          snprintf(S,sizeof(S),"%u",Port);
 320
 321       // Free the old address structure
 322       if (LastHostAddr != 0)
 323       {
 324          freeaddrinfo(LastHostAddr);
 325          LastHostAddr = 0;
 326       }
 327
 328       // We only understand SOCK_STREAM sockets.
 329       struct addrinfo Hints;
 330       memset(&Hints,0,sizeof(Hints));
 331       Hints.ai_socktype = SOCK_STREAM;
 332
 333       // Resolve both the host and service simultaneously
 334       if (getaddrinfo(Host.c_str(),S,&Hints,&LastHostAddr) != 0 ||
 335           LastHostAddr == 0)
 336          return _error->Error("Could not resolve '%s'",Host.c_str());
 337
 338       LastHost = Host;
 339       LastPort = Port;
 340    }
 341
 342    // Connect to the server
 343    // Get a socket
 344 //   Owner->Status("Connecting to %s (%s)",Host.c_str(),inet_ntoa(LastHostA));
 345    if ((ServerFd = socket(LastHostAddr->ai_family,LastHostAddr->ai_socktype,
 346                           LastHostAddr->ai_protocol)) < 0)
 347       return _error->Errno("socket","Could not create a socket");
 348    SetNonBlock(ServerFd,true);
 349    if (connect(ServerFd,LastHostAddr->ai_addr,LastHostAddr->ai_addrlen) < 0 &&
 350        errno != EINPROGRESS)
 351       return _error->Errno("connect","Connect initiate the connection");
 352
 353    /* This implements a timeout for connect by opening the connection
 354       nonblocking */
 355    if (WaitFd(ServerFd,true,TimeOut) == false)
 356       return _error->Error("Could not connect, connection timed out");
 357    unsigned int Err;
 358    unsigned int Len = sizeof(Err);
 359    if (getsockopt(ServerFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 360       return _error->Errno("getsockopt","Failed");
 361    if (Err != 0)
 362       return _error->Error("Could not connect.");
 363
 364    return true;
 365 }
 366                                                                         /*}}}*/
 367 // ServerState::Close - Close a connection to the server                /*{{{*/
 368 // ---------------------------------------------------------------------
 369 /* */
 370 bool ServerState::Close()
 371 {
 372    close(ServerFd);
 373    ServerFd = -1;
 374    return true;
 375 }
 376                                                                         /*}}}*/
 377 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 378 // ---------------------------------------------------------------------
 379 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 380    parse error occured */
 381 int ServerState::RunHeaders()
 382 {
 383    State = Header;
 384
 385    Owner->Status("Waiting for file");
 386
 387    Major = 0;
 388    Minor = 0;
 389    Result = 0;
 390    Size = 0;
 391    StartPos = 0;
 392    Encoding = Closes;
 393    HaveContent = false;
 394    time(&Date);
 395
 396    do
 397    {
 398       string Data;
 399       if (In.WriteTillEl(Data) == false)
 400          continue;
 401
 402       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 403       {
 404          string::const_iterator J = I;
 405          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 406          if (HeaderLine(string(I,J-I)) == false)
 407             return 2;
 408          I = J;
 409       }
 410       return 0;
 411    }
 412    while (Owner->Go(false,this) == true);
 413
 414    return 1;
 415 }
 416                                                                         /*}}}*/
 417 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 418 // ---------------------------------------------------------------------
 419 /* */
 420 bool ServerState::RunData()
 421 {
 422    State = Data;
 423
 424    // Chunked transfer encoding is fun..
 425    if (Encoding == Chunked)
 426    {
 427       while (1)
 428       {
 429          // Grab the block size
 430          bool Last = true;
 431          string Data;
 432          In.Limit(-1);
 433          do
 434          {
 435             if (In.WriteTillEl(Data,true) == true)
 436                break;
 437          }
 438          while ((Last = Owner->Go(false,this)) == true);
 439
 440          if (Last == false)
 441             return false;
 442
 443          // See if we are done
 444          unsigned long Len = strtol(Data.c_str(),0,16);
 445          if (Len == 0)
 446          {
 447             In.Limit(-1);
 448
 449             // We have to remove the entity trailer
 450             Last = true;
 451             do
 452             {
 453                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 454                   break;
 455             }
 456             while ((Last = Owner->Go(false,this)) == true);
 457             if (Last == false)
 458                return false;
 459             return !_error->PendingError();
 460          }
 461
 462          // Transfer the block
 463          In.Limit(Len);
 464          while (Owner->Go(true,this) == true)
 465             if (In.IsLimit() == true)
 466                break;
 467
 468          // Error
 469          if (In.IsLimit() == false)
 470             return false;
 471
 472          // The server sends an extra new line before the next block specifier..
 473          In.Limit(-1);
 474          Last = true;
 475          do
 476          {
 477             if (In.WriteTillEl(Data,true) == true)
 478                break;
 479          }
 480          while ((Last = Owner->Go(false,this)) == true);
 481          if (Last == false)
 482             return false;
 483       }
 484    }
 485    else
 486    {
 487       /* Closes encoding is used when the server did not specify a size, the
 488          loss of the connection means we are done */
 489       if (Encoding == Closes)
 490          In.Limit(-1);
 491       else
 492          In.Limit(Size - StartPos);
 493
 494       // Just transfer the whole block.
 495       do
 496       {
 497          if (In.IsLimit() == false)
 498             continue;
 499
 500          In.Limit(-1);
 501          return !_error->PendingError();
 502       }
 503       while (Owner->Go(true,this) == true);
 504    }
 505
 506    return Owner->Flush(this) && !_error->PendingError();
 507 }
 508                                                                         /*}}}*/
 509 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 510 // ---------------------------------------------------------------------
 511 /* */
 512 bool ServerState::HeaderLine(string Line)
 513 {
 514    if (Line.empty() == true)
 515       return true;
 516
 517    // The http server might be trying to do something evil.
 518    if (Line.length() >= MAXLEN)
 519       return _error->Error("Got a single header line over %u chars",MAXLEN);
 520
 521    string::size_type Pos = Line.find(' ');
 522    if (Pos == string::npos || Pos+1 > Line.length())
 523    {
 524       // Blah, some servers use "connection:closes", evil.
 525       Pos = Line.find(':');
 526       if (Pos == string::npos || Pos + 2 > Line.length())
 527          return _error->Error("Bad header line");
 528       Pos++;
 529    }
 530
 531    // Parse off any trailing spaces between the : and the next word.
 532    string::size_type Pos2 = Pos;
 533    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 534       Pos2++;
 535
 536    string Tag = string(Line,0,Pos);
 537    string Val = string(Line,Pos2);
 538
 539    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 540    {
 541       // Evil servers return no version
 542       if (Line[4] == '/')
 543       {
 544          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 545                     &Result,Code) != 4)
 546             return _error->Error("The http server sent an invalid reply header");
 547       }
 548       else
 549       {
 550          Major = 0;
 551          Minor = 9;
 552          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 553             return _error->Error("The http server sent an invalid reply header");
 554       }
 555
 556       return true;
 557    }
 558
 559    if (stringcasecmp(Tag,"Content-Length:") == 0)
 560    {
 561       if (Encoding == Closes)
 562          Encoding = Stream;
 563       HaveContent = true;
 564
 565       // The length is already set from the Content-Range header
 566       if (StartPos != 0)
 567          return true;
 568
 569       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 570          return _error->Error("The http server sent an invalid Content-Length header");
 571       return true;
 572    }
 573
 574    if (stringcasecmp(Tag,"Content-Type:") == 0)
 575    {
 576       HaveContent = true;
 577       return true;
 578    }
 579
 580    if (stringcasecmp(Tag,"Content-Range:") == 0)
 581    {
 582       HaveContent = true;
 583
 584       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 585          return _error->Error("The http server sent an invalid Content-Range header");
 586       if ((unsigned)StartPos > Size)
 587          return _error->Error("This http server has broken range support");
 588       return true;
 589    }
 590
 591    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 592    {
 593       HaveContent = true;
 594       if (stringcasecmp(Val,"chunked") == 0)
 595          Encoding = Chunked;
 596
 597       return true;
 598    }
 599
 600    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 601    {
 602       if (StrToTime(Val,Date) == false)
 603          return _error->Error("Unknown date format");
 604       return true;
 605    }
 606
 607    return true;
 608 }
 609                                                                         /*}}}*/
 610
 611 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 612 // ---------------------------------------------------------------------
 613 /* This places the http request in the outbound buffer */
 614 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 615 {
 616    URI Uri = Itm->Uri;
 617
 618    // The HTTP server expects a hostname with a trailing :port
 619    char Buf[1000];
 620    string ProperHost = Uri.Host;
 621    if (Uri.Port != 0)
 622    {
 623       sprintf(Buf,":%u",Uri.Port);
 624       ProperHost += Buf;
 625    }
 626
 627    // Just in case.
 628    if (Itm->Uri.length() >= sizeof(Buf))
 629        abort();
 630
 631    /* Build the request. We include a keep-alive header only for non-proxy
 632       requests. This is to tweak old http/1.0 servers that do support keep-alive
 633       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 634       will glitch HTTP/1.0 proxies because they do not filter it out and
 635       pass it on, HTTP/1.1 says the connection should default to keep alive
 636       and we expect the proxy to do this */
 637    if (Proxy.empty() == true)
 638       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 639               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 640    else
 641    {
 642       /* Generate a cache control header if necessary. We place a max
 643          cache age on index files, optionally set a no-cache directive
 644          and a no-store directive for archives. */
 645       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 646               Itm->Uri.c_str(),ProperHost.c_str());
 647       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 648          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 649       else
 650       {
 651          if (Itm->IndexFile == true)
 652             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 653                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 654          else
 655          {
 656             if (_config->FindB("Acquire::http::No-Store",false) == true)
 657                strcat(Buf,"Cache-Control: no-store\r\n");
 658          }
 659       }
 660    }
 661
 662    string Req = Buf;
 663
 664    // Check for a partial file
 665    struct stat SBuf;
 666    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 667    {
 668       // In this case we send an if-range query with a range header
 669       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 670               TimeRFC1123(SBuf.st_mtime).c_str());
 671       Req += Buf;
 672    }
 673    else
 674    {
 675       if (Itm->LastModified != 0)
 676       {
 677          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 678          Req += Buf;
 679       }
 680    }
 681
 682    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 683       Req += string("Proxy-Authorization: Basic ") +
 684           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 685
 686    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 687 //   cerr << Req << endl;
 688
 689    Out.Read(Req);
 690 }
 691                                                                         /*}}}*/
 692 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 693 // ---------------------------------------------------------------------
 694 /* This runs the select loop over the server FDs, Output file FDs and
 695    stdin. */
 696 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 697 {
 698    // Server has closed the connection
 699    if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
 700       return false;
 701
 702    fd_set rfds,wfds,efds;
 703    FD_ZERO(&rfds);
 704    FD_ZERO(&wfds);
 705    FD_ZERO(&efds);
 706
 707    // Add the server
 708    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 709       FD_SET(Srv->ServerFd,&wfds);
 710    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 711       FD_SET(Srv->ServerFd,&rfds);
 712
 713    // Add the file
 714    int FileFD = -1;
 715    if (File != 0)
 716       FileFD = File->Fd();
 717
 718    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 719       FD_SET(FileFD,&wfds);
 720
 721    // Add stdin
 722    FD_SET(STDIN_FILENO,&rfds);
 723
 724    // Error Set
 725    if (FileFD != -1)
 726       FD_SET(FileFD,&efds);
 727    if (Srv->ServerFd != -1)
 728       FD_SET(Srv->ServerFd,&efds);
 729
 730    // Figure out the max fd
 731    int MaxFd = FileFD;
 732    if (MaxFd < Srv->ServerFd)
 733       MaxFd = Srv->ServerFd;
 734
 735    // Select
 736    struct timeval tv;
 737    tv.tv_sec = TimeOut;
 738    tv.tv_usec = 0;
 739    int Res = 0;
 740    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 741       return _error->Errno("select","Select failed");
 742
 743    if (Res == 0)
 744    {
 745       _error->Error("Connection timed out");
 746       return ServerDie(Srv);
 747    }
 748
 749    // Some kind of exception (error) on the sockets, die
 750    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 751        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 752       return _error->Error("Socket Exception");
 753
 754    // Handle server IO
 755    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 756    {
 757       errno = 0;
 758       if (Srv->In.Read(Srv->ServerFd) == false)
 759          return ServerDie(Srv);
 760    }
 761
 762    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 763    {
 764       errno = 0;
 765       if (Srv->Out.Write(Srv->ServerFd) == false)
 766          return ServerDie(Srv);
 767    }
 768
 769    // Send data to the file
 770    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 771    {
 772       if (Srv->In.Write(FileFD) == false)
 773          return _error->Errno("write","Error writing to output file");
 774    }
 775
 776    // Handle commands from APT
 777    if (FD_ISSET(STDIN_FILENO,&rfds))
 778    {
 779       if (Run(true) != 0)
 780          exit(100);
 781    }
 782
 783    return true;
 784 }
 785                                                                         /*}}}*/
 786 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 787 // ---------------------------------------------------------------------
 788 /* This takes the current input buffer from the Server FD and writes it
 789    into the file */
 790 bool HttpMethod::Flush(ServerState *Srv)
 791 {
 792    if (File != 0)
 793    {
 794       SetNonBlock(File->Fd(),false);
 795       if (Srv->In.WriteSpace() == false)
 796          return true;
 797
 798       while (Srv->In.WriteSpace() == true)
 799       {
 800          if (Srv->In.Write(File->Fd()) == false)
 801             return _error->Errno("write","Error writing to file");
 802          if (Srv->In.IsLimit() == true)
 803             return true;
 804       }
 805
 806       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 807          return true;
 808    }
 809    return false;
 810 }
 811                                                                         /*}}}*/
 812 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 813 // ---------------------------------------------------------------------
 814 /* */
 815 bool HttpMethod::ServerDie(ServerState *Srv)
 816 {
 817    unsigned int LErrno = errno;
 818
 819    // Dump the buffer to the file
 820    if (Srv->State == ServerState::Data)
 821    {
 822       SetNonBlock(File->Fd(),false);
 823       while (Srv->In.WriteSpace() == true)
 824       {
 825          if (Srv->In.Write(File->Fd()) == false)
 826             return _error->Errno("write","Error writing to the file");
 827
 828          // Done
 829          if (Srv->In.IsLimit() == true)
 830             return true;
 831       }
 832    }
 833
 834    // See if this is because the server finished the data stream
 835    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 836        Srv->Encoding != ServerState::Closes)
 837    {
 838       Srv->Close();
 839       if (LErrno == 0)
 840          return _error->Error("Error reading from server Remote end closed connection");
 841       errno = LErrno;
 842       return _error->Errno("read","Error reading from server");
 843    }
 844    else
 845    {
 846       Srv->In.Limit(-1);
 847
 848       // Nothing left in the buffer
 849       if (Srv->In.WriteSpace() == false)
 850          return false;
 851
 852       // We may have got multiple responses back in one packet..
 853       Srv->Close();
 854       return true;
 855    }
 856
 857    return false;
 858 }
 859                                                                         /*}}}*/
 860 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 861 // ---------------------------------------------------------------------
 862 /* We look at the header data we got back from the server and decide what
 863    to do. Returns
 864      0 - File is open,
 865      1 - IMS hit
 866      3 - Unrecoverable error
 867      4 - Error with error content page
 868      5 - Unrecoverable non-server error (close the connection) */
 869 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 870 {
 871    // Not Modified
 872    if (Srv->Result == 304)
 873    {
 874       unlink(Queue->DestFile.c_str());
 875       Res.IMSHit = true;
 876       Res.LastModified = Queue->LastModified;
 877       return 1;
 878    }
 879
 880    /* We have a reply we dont handle. This should indicate a perm server
 881       failure */
 882    if (Srv->Result < 200 || Srv->Result >= 300)
 883    {
 884       _error->Error("%u %s",Srv->Result,Srv->Code);
 885       if (Srv->HaveContent == true)
 886          return 4;
 887       return 3;
 888    }
 889
 890    // This is some sort of 2xx 'data follows' reply
 891    Res.LastModified = Srv->Date;
 892    Res.Size = Srv->Size;
 893
 894    // Open the file
 895    delete File;
 896    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 897    if (_error->PendingError() == true)
 898       return 5;
 899
 900    FailFile = Queue->DestFile;
 901    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 902    FailFd = File->Fd();
 903    FailTime = Srv->Date;
 904
 905    // Set the expected size
 906    if (Srv->StartPos >= 0)
 907    {
 908       Res.ResumePoint = Srv->StartPos;
 909       ftruncate(File->Fd(),Srv->StartPos);
 910    }
 911
 912    // Set the start point
 913    lseek(File->Fd(),0,SEEK_END);
 914
 915    delete Srv->In.MD5;
 916    Srv->In.MD5 = new MD5Summation;
 917
 918    // Fill the MD5 Hash if the file is non-empty (resume)
 919    if (Srv->StartPos > 0)
 920    {
 921       lseek(File->Fd(),0,SEEK_SET);
 922       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 923       {
 924          _error->Errno("read","Problem hashing file");
 925          return 5;
 926       }
 927       lseek(File->Fd(),0,SEEK_END);
 928    }
 929
 930    SetNonBlock(File->Fd(),true);
 931    return 0;
 932 }
 933                                                                         /*}}}*/
 934 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 935 // ---------------------------------------------------------------------
 936 /* This closes and timestamps the open file. This is neccessary to get
 937    resume behavoir on user abort */
 938 void HttpMethod::SigTerm(int)
 939 {
 940    if (FailFd == -1)
 941       exit(100);
 942    close(FailFd);
 943
 944    // Timestamp
 945    struct utimbuf UBuf;
 946    UBuf.actime = FailTime;
 947    UBuf.modtime = FailTime;
 948    utime(FailFile.c_str(),&UBuf);
 949
 950    exit(100);
 951 }
 952                                                                         /*}}}*/
 953 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 954 // ---------------------------------------------------------------------
 955 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 956    depth. */
 957 bool HttpMethod::Fetch(FetchItem *)
 958 {
 959    if (Server == 0)
 960       return true;
 961
 962    // Queue the requests
 963    int Depth = -1;
 964    bool Tail = false;
 965    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++)
 966    {
 967       // Make sure we stick with the same server
 968       if (Server->Comp(I->Uri) == false)
 969          break;
 970       if (QueueBack == I)
 971          Tail = true;
 972       if (Tail == true)
 973       {
 974          QueueBack = I->Next;
 975          SendReq(I,Server->Out);
 976          continue;
 977       }
 978    }
 979
 980    return true;
 981 };
 982                                                                         /*}}}*/
 983 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 984 // ---------------------------------------------------------------------
 985 /* We stash the desired pipeline depth */
 986 bool HttpMethod::Configuration(string Message)
 987 {
 988    if (pkgAcqMethod::Configuration(Message) == false)
 989       return false;
 990
 991    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 992    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 993                                   PipelineDepth);
 994
 995    return true;
 996 }
 997                                                                         /*}}}*/
 998 // HttpMethod::Loop - Main loop                                         /*{{{*/
 999 // ---------------------------------------------------------------------
1000 /* */
1001 int HttpMethod::Loop()
1002 {
1003    signal(SIGTERM,SigTerm);
1004    signal(SIGINT,SigTerm);
1005
1006    Server = 0;
1007
1008    int FailCounter = 0;
1009    while (1)
1010    {
1011       // We have no commands, wait for some to arrive
1012       if (Queue == 0)
1013       {
1014          if (WaitFd(STDIN_FILENO) == false)
1015             return 0;
1016       }
1017
1018       // Run messages
1019       if (Run(true) != 0)
1020          return 100;
1021
1022       if (Queue == 0)
1023          continue;
1024
1025       // Connect to the server
1026       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1027       {
1028          delete Server;
1029          Server = new ServerState(Queue->Uri,this);
1030       }
1031
1032       // Reset the pipeline
1033       if (Server->ServerFd == -1)
1034          QueueBack = Queue;
1035
1036       // Connnect to the host
1037       if (Server->Open() == false)
1038       {
1039          Fail(true);
1040          delete Server;
1041          Server = 0;
1042          continue;
1043       }
1044
1045       // Fill the pipeline.
1046       Fetch(0);
1047
1048       // Fetch the next URL header data from the server.
1049       switch (Server->RunHeaders())
1050       {
1051          case 0:
1052          break;
1053
1054          // The header data is bad
1055          case 2:
1056          {
1057             _error->Error("Bad header Data");
1058             Fail(true);
1059             continue;
1060          }
1061
1062          // The server closed a connection during the header get..
1063          default:
1064          case 1:
1065          {
1066             FailCounter++;
1067             _error->Discard();
1068             Server->Close();
1069
1070             if (FailCounter >= 2)
1071             {
1072                Fail("Connection timed out",true);
1073                FailCounter = 0;
1074             }
1075
1076             continue;
1077          }
1078       };
1079
1080       // Decide what to do.
1081       FetchResult Res;
1082       Res.Filename = Queue->DestFile;
1083       switch (DealWithHeaders(Res,Server))
1084       {
1085          // Ok, the file is Open
1086          case 0:
1087          {
1088             URIStart(Res);
1089
1090             // Run the data
1091             bool Result =  Server->RunData();
1092
1093             // Close the file, destroy the FD object and timestamp it
1094             FailFd = -1;
1095             delete File;
1096             File = 0;
1097
1098             // Timestamp
1099             struct utimbuf UBuf;
1100             time(&UBuf.actime);
1101             UBuf.actime = Server->Date;
1102             UBuf.modtime = Server->Date;
1103             utime(Queue->DestFile.c_str(),&UBuf);
1104
1105             // Send status to APT
1106             if (Result == true)
1107             {
1108                Res.MD5Sum = Server->In.MD5->Result();
1109                URIDone(Res);
1110             }
1111             else
1112                Fail(true);
1113
1114             break;
1115          }
1116
1117          // IMS hit
1118          case 1:
1119          {
1120             URIDone(Res);
1121             break;
1122          }
1123
1124          // Hard server error, not found or something
1125          case 3:
1126          {
1127             Fail();
1128             break;
1129          }
1130
1131          // Hard internal error, kill the connection and fail
1132          case 5:
1133          {
1134             Fail();
1135             Server->Close();
1136             break;
1137          }
1138
1139          // We need to flush the data, the header is like a 404 w/ error text
1140          case 4:
1141          {
1142             Fail();
1143
1144             // Send to content to dev/null
1145             File = new FileFd("/dev/null",FileFd::WriteExists);
1146             Server->RunData();
1147             delete File;
1148             File = 0;
1149             break;
1150          }
1151
1152          default:
1153          Fail("Internal error");
1154          break;
1155       }
1156
1157       FailCounter = 0;
1158    }
1159
1160    return 0;
1161 }
1162                                                                         /*}}}*/
1163
1164 int main()
1165 {
1166    HttpMethod Mth;
1167
1168    return Mth.Loop();
1169 }