methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.21 1999/01/04 00:57:56 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41
  42 // Internet stuff
  43 #include <netinet/in.h>
  44 #include <sys/socket.h>
  45 #include <arpa/inet.h>
  46 #include <netdb.h>
  47
  48 #include "http.h"
  49                                                                         /*}}}*/
  50
  51 string HttpMethod::FailFile;
  52 int HttpMethod::FailFd = -1;
  53 time_t HttpMethod::FailTime = 0;
  54 unsigned long PipelineDepth = 10;
  55 unsigned long TimeOut = 120;
  56
  57 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  58 // ---------------------------------------------------------------------
  59 /* */
  60 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  61 {
  62    Buf = new unsigned char[Size];
  63    Reset();
  64 }
  65                                                                         /*}}}*/
  66 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  67 // ---------------------------------------------------------------------
  68 /* */
  69 void CircleBuf::Reset()
  70 {
  71    InP = 0;
  72    OutP = 0;
  73    StrPos = 0;
  74    MaxGet = (unsigned int)-1;
  75    OutQueue = string();
  76    if (MD5 != 0)
  77    {
  78       delete MD5;
  79       MD5 = new MD5Summation;
  80    }
  81 };
  82                                                                         /*}}}*/
  83 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  84 // ---------------------------------------------------------------------
  85 /* This fills up the buffer with as much data as is in the FD, assuming it
  86    is non-blocking.. */
  87 bool CircleBuf::Read(int Fd)
  88 {
  89    while (1)
  90    {
  91       // Woops, buffer is full
  92       if (InP - OutP == Size)
  93          return true;
  94
  95       // Write the buffer segment
  96       int Res;
  97       Res = read(Fd,Buf + (InP%Size),LeftRead());
  98
  99       if (Res == 0)
 100          return false;
 101       if (Res < 0)
 102       {
 103          if (errno == EAGAIN)
 104             return true;
 105          return false;
 106       }
 107
 108       if (InP == 0)
 109          gettimeofday(&Start,0);
 110       InP += Res;
 111    }
 112 }
 113                                                                         /*}}}*/
 114 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 115 // ---------------------------------------------------------------------
 116 /* This will hold the string in and fill the buffer with it as it empties */
 117 bool CircleBuf::Read(string Data)
 118 {
 119    OutQueue += Data;
 120    FillOut();
 121    return true;
 122 }
 123                                                                         /*}}}*/
 124 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 125 // ---------------------------------------------------------------------
 126 /* */
 127 void CircleBuf::FillOut()
 128 {
 129    if (OutQueue.empty() == true)
 130       return;
 131    while (1)
 132    {
 133       // Woops, buffer is full
 134       if (InP - OutP == Size)
 135          return;
 136
 137       // Write the buffer segment
 138       unsigned long Sz = LeftRead();
 139       if (OutQueue.length() - StrPos < Sz)
 140          Sz = OutQueue.length() - StrPos;
 141       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 142
 143       // Advance
 144       StrPos += Sz;
 145       InP += Sz;
 146       if (OutQueue.length() == StrPos)
 147       {
 148          StrPos = 0;
 149          OutQueue = "";
 150          return;
 151       }
 152    }
 153 }
 154                                                                         /*}}}*/
 155 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 156 // ---------------------------------------------------------------------
 157 /* This empties the buffer into the FD. */
 158 bool CircleBuf::Write(int Fd)
 159 {
 160    while (1)
 161    {
 162       FillOut();
 163
 164       // Woops, buffer is empty
 165       if (OutP == InP)
 166          return true;
 167
 168       if (OutP == MaxGet)
 169          return true;
 170
 171       // Write the buffer segment
 172       int Res;
 173       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 174
 175       if (Res == 0)
 176          return false;
 177       if (Res < 0)
 178       {
 179          if (errno == EAGAIN)
 180             return true;
 181
 182          return false;
 183       }
 184
 185       if (MD5 != 0)
 186          MD5->Add(Buf + (OutP%Size),Res);
 187
 188       OutP += Res;
 189    }
 190 }
 191                                                                         /*}}}*/
 192 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 193 // ---------------------------------------------------------------------
 194 /* This copies till the first empty line */
 195 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 196 {
 197    // We cheat and assume it is unneeded to have more than one buffer load
 198    for (unsigned long I = OutP; I < InP; I++)
 199    {
 200       if (Buf[I%Size] != '\n')
 201          continue;
 202       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 203
 204       if (Single == false)
 205       {
 206          if (Buf[I%Size] != '\n')
 207             continue;
 208          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 209       }
 210
 211       if (I > InP)
 212          I = InP;
 213
 214       Data = "";
 215       while (OutP < I)
 216       {
 217          unsigned long Sz = LeftWrite();
 218          if (Sz == 0)
 219             return false;
 220          if (I - OutP < LeftWrite())
 221             Sz = I - OutP;
 222          Data += string((char *)(Buf + (OutP%Size)),Sz);
 223          OutP += Sz;
 224       }
 225       return true;
 226    }
 227    return false;
 228 }
 229                                                                         /*}}}*/
 230 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 231 // ---------------------------------------------------------------------
 232 /* */
 233 void CircleBuf::Stats()
 234 {
 235    if (InP == 0)
 236       return;
 237
 238    struct timeval Stop;
 239    gettimeofday(&Stop,0);
 240 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 241              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 242    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 243 }
 244                                                                         /*}}}*/
 245
 246 // ServerState::ServerState - Constructor                               /*{{{*/
 247 // ---------------------------------------------------------------------
 248 /* */
 249 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 250                         In(64*1024), Out(4*1024),
 251                         ServerName(Srv)
 252 {
 253    Reset();
 254 }
 255                                                                         /*}}}*/
 256 // ServerState::Open - Open a connection to the server                  /*{{{*/
 257 // ---------------------------------------------------------------------
 258 /* This opens a connection to the server. */
 259 string LastHost;
 260 in_addr LastHostA;
 261 bool ServerState::Open()
 262 {
 263    // Use the already open connection if possible.
 264    if (ServerFd != -1)
 265       return true;
 266
 267    Close();
 268    In.Reset();
 269    Out.Reset();
 270
 271    // Determine the proxy setting
 272    if (getenv("http_proxy") == 0)
 273    {
 274       string DefProxy = _config->Find("Acquire::http::Proxy");
 275       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 276       if (SpecificProxy.empty() == false)
 277       {
 278          if (SpecificProxy == "DIRECT")
 279             Proxy = "";
 280          else
 281             Proxy = SpecificProxy;
 282       }
 283       else
 284          Proxy = DefProxy;
 285    }
 286    else
 287       Proxy = getenv("http_proxy");
 288
 289    // Determine what host and port to use based on the proxy settings
 290    int Port = 80;
 291    string Host;
 292    if (Proxy.empty() == true)
 293    {
 294       if (ServerName.Port != 0)
 295          Port = ServerName.Port;
 296       Host = ServerName.Host;
 297    }
 298    else
 299    {
 300       if (Proxy.Port != 0)
 301          Port = Proxy.Port;
 302       Host = Proxy.Host;
 303    }
 304
 305    /* We used a cached address record.. Yes this is against the spec but
 306       the way we have setup our rotating dns suggests that this is more
 307       sensible */
 308    if (LastHost != Host)
 309    {
 310       Owner->Status("Connecting to %s",Host.c_str());
 311
 312       // Lookup the host
 313       hostent *Addr = gethostbyname(Host.c_str());
 314       if (Addr == 0 || Addr->h_addr_list[0] == 0)
 315          return _error->Error("Could not resolve '%s'",Host.c_str());
 316       LastHost = Host;
 317       LastHostA = *(in_addr *)(Addr->h_addr_list[0]);
 318    }
 319
 320    Owner->Status("Connecting to %s (%s)",Host.c_str(),inet_ntoa(LastHostA));
 321
 322    // Get a socket
 323    if ((ServerFd = socket(AF_INET,SOCK_STREAM,0)) < 0)
 324       return _error->Errno("socket","Could not create a socket");
 325
 326    // Connect to the server
 327    struct sockaddr_in server;
 328    server.sin_family = AF_INET;
 329    server.sin_port = htons(Port);
 330    server.sin_addr = LastHostA;
 331    SetNonBlock(ServerFd,true);
 332    if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0 &&
 333        errno != EINPROGRESS)
 334       return _error->Errno("socket","Could not create a socket");
 335
 336    /* This implements a timeout for connect by opening the connection
 337       nonblocking */
 338    fd_set wfds;
 339    FD_ZERO(&wfds);
 340    FD_SET(ServerFd,&wfds);
 341    struct timeval tv;
 342    tv.tv_sec = TimeOut;
 343    tv.tv_usec = 0;
 344    int Res = 0;
 345    if ((Res = select(ServerFd+1,0,&wfds,0,&tv)) < 0)
 346       return _error->Errno("select","Select failed");
 347    if (Res == 0)
 348       return _error->Error("Could not connect, connection timed out");
 349    unsigned int Err,Len=sizeof(Err);
 350    if (getsockopt(ServerFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 351       return _error->Errno("getsockopt","Failed");
 352    if (Err != 0)
 353       return _error->Error("Could not connect.");
 354
 355    return true;
 356 }
 357                                                                         /*}}}*/
 358 // ServerState::Close - Close a connection to the server                /*{{{*/
 359 // ---------------------------------------------------------------------
 360 /* */
 361 bool ServerState::Close()
 362 {
 363    close(ServerFd);
 364    ServerFd = -1;
 365    return true;
 366 }
 367                                                                         /*}}}*/
 368 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 369 // ---------------------------------------------------------------------
 370 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 371    parse error occured */
 372 int ServerState::RunHeaders()
 373 {
 374    State = Header;
 375
 376    Owner->Status("Waiting for file");
 377
 378    Major = 0;
 379    Minor = 0;
 380    Result = 0;
 381    Size = 0;
 382    StartPos = 0;
 383    Encoding = Closes;
 384    HaveContent = false;
 385    time(&Date);
 386
 387    do
 388    {
 389       string Data;
 390       if (In.WriteTillEl(Data) == false)
 391          continue;
 392
 393       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 394       {
 395          string::const_iterator J = I;
 396          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 397          if (HeaderLine(string(I,J-I)) == false)
 398             return 2;
 399          I = J;
 400       }
 401       return 0;
 402    }
 403    while (Owner->Go(false,this) == true);
 404
 405    return 1;
 406 }
 407                                                                         /*}}}*/
 408 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 409 // ---------------------------------------------------------------------
 410 /* */
 411 bool ServerState::RunData()
 412 {
 413    State = Data;
 414
 415    // Chunked transfer encoding is fun..
 416    if (Encoding == Chunked)
 417    {
 418       while (1)
 419       {
 420          // Grab the block size
 421          bool Last = true;
 422          string Data;
 423          In.Limit(-1);
 424          do
 425          {
 426             if (In.WriteTillEl(Data,true) == true)
 427                break;
 428          }
 429          while ((Last = Owner->Go(false,this)) == true);
 430
 431          if (Last == false)
 432             return false;
 433
 434          // See if we are done
 435          unsigned long Len = strtol(Data.c_str(),0,16);
 436          if (Len == 0)
 437          {
 438             In.Limit(-1);
 439
 440             // We have to remove the entity trailer
 441             Last = true;
 442             do
 443             {
 444                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 445                   break;
 446             }
 447             while ((Last = Owner->Go(false,this)) == true);
 448             if (Last == false)
 449                return false;
 450             return true;
 451          }
 452
 453          // Transfer the block
 454          In.Limit(Len);
 455          while (Owner->Go(true,this) == true)
 456             if (In.IsLimit() == true)
 457                break;
 458
 459          // Error
 460          if (In.IsLimit() == false)
 461             return false;
 462
 463          // The server sends an extra new line before the next block specifier..
 464          In.Limit(-1);
 465          Last = true;
 466          do
 467          {
 468             if (In.WriteTillEl(Data,true) == true)
 469                break;
 470          }
 471          while ((Last = Owner->Go(false,this)) == true);
 472          if (Last == false)
 473             return false;
 474       }
 475    }
 476    else
 477    {
 478       /* Closes encoding is used when the server did not specify a size, the
 479          loss of the connection means we are done */
 480       if (Encoding == Closes)
 481          In.Limit(-1);
 482       else
 483          In.Limit(Size - StartPos);
 484
 485       // Just transfer the whole block.
 486       do
 487       {
 488          if (In.IsLimit() == false)
 489             continue;
 490
 491          In.Limit(-1);
 492          return true;
 493       }
 494       while (Owner->Go(true,this) == true);
 495    }
 496
 497    return Owner->Flush(this);
 498 }
 499                                                                         /*}}}*/
 500 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 501 // ---------------------------------------------------------------------
 502 /* */
 503 bool ServerState::HeaderLine(string Line)
 504 {
 505    if (Line.empty() == true)
 506       return true;
 507
 508    // The http server might be trying to do something evil.
 509    if (Line.length() >= MAXLEN)
 510       return _error->Error("Got a single header line over %u chars",MAXLEN);
 511
 512    string::size_type Pos = Line.find(' ');
 513    if (Pos == string::npos || Pos+1 > Line.length())
 514       return _error->Error("Bad header line");
 515
 516    string Tag = string(Line,0,Pos);
 517    string Val = string(Line,Pos+1);
 518
 519    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 520    {
 521       // Evil servers return no version
 522       if (Line[4] == '/')
 523       {
 524          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 525                     &Result,Code) != 4)
 526             return _error->Error("The http server sent an invalid reply header");
 527       }
 528       else
 529       {
 530          Major = 0;
 531          Minor = 9;
 532          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 533             return _error->Error("The http server sent an invalid reply header");
 534       }
 535
 536       return true;
 537    }
 538
 539    if (stringcasecmp(Tag,"Content-Length:") == 0)
 540    {
 541       if (Encoding == Closes)
 542          Encoding = Stream;
 543       HaveContent = true;
 544
 545       // The length is already set from the Content-Range header
 546       if (StartPos != 0)
 547          return true;
 548
 549       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 550          return _error->Error("The http server sent an invalid Content-Length header");
 551       return true;
 552    }
 553
 554    if (stringcasecmp(Tag,"Content-Type:") == 0)
 555    {
 556       HaveContent = true;
 557       return true;
 558    }
 559
 560    if (stringcasecmp(Tag,"Content-Range:") == 0)
 561    {
 562       HaveContent = true;
 563
 564       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 565          return _error->Error("The http server sent an invalid Content-Range header");
 566       if ((unsigned)StartPos > Size)
 567          return _error->Error("This http server has broken range support");
 568       return true;
 569    }
 570
 571    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 572    {
 573       HaveContent = true;
 574       if (stringcasecmp(Val,"chunked") == 0)
 575          Encoding = Chunked;
 576
 577       return true;
 578    }
 579
 580    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 581    {
 582       if (StrToTime(Val,Date) == false)
 583          return _error->Error("Unknown date format");
 584       return true;
 585    }
 586
 587    return true;
 588 }
 589                                                                         /*}}}*/
 590
 591 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 592 // ---------------------------------------------------------------------
 593 /* This places the http request in the outbound buffer */
 594 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 595 {
 596    URI Uri = Itm->Uri;
 597
 598    // The HTTP server expects a hostname with a trailing :port
 599    char Buf[1000];
 600    string ProperHost = Uri.Host;
 601    if (Uri.Port != 0)
 602    {
 603       sprintf(Buf,":%u",Uri.Port);
 604       ProperHost += Buf;
 605    }
 606
 607    // Just in case.
 608    if (Itm->Uri.length() >= sizeof(Buf))
 609        abort();
 610
 611    /* Build the request. We include a keep-alive header only for non-proxy
 612       requests. This is to tweak old http/1.0 servers that do support keep-alive
 613       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 614       will glitch HTTP/1.0 proxies because they do not filter it out and
 615       pass it on, HTTP/1.1 says the connection should default to keep alive
 616       and we expect the proxy to do this */
 617    if (Proxy.empty() == true)
 618       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 619               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 620    else
 621    {
 622       /* Generate a cache control header if necessary. We place a max
 623          cache age on index files, optionally set a no-cache directive
 624          and a no-store directive for archives. */
 625       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 626               Itm->Uri.c_str(),ProperHost.c_str());
 627       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 628          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 629       else
 630       {
 631          if (Itm->IndexFile == true)
 632             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 633                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 634          else
 635          {
 636             if (_config->FindB("Acquire::http::No-Store",false) == true)
 637                strcat(Buf,"Cache-Control: no-store\r\n");
 638          }
 639       }
 640    }
 641
 642    string Req = Buf;
 643
 644    // Check for a partial file
 645    struct stat SBuf;
 646    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 647    {
 648       // In this case we send an if-range query with a range header
 649       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 650               TimeRFC1123(SBuf.st_mtime).c_str());
 651       Req += Buf;
 652    }
 653    else
 654    {
 655       if (Itm->LastModified != 0)
 656       {
 657          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 658          Req += Buf;
 659       }
 660    }
 661
 662    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 663       Req += string("Proxy-Authorization: Basic ") +
 664           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 665
 666    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 667 //   cerr << Req << endl;
 668
 669    Out.Read(Req);
 670 }
 671                                                                         /*}}}*/
 672 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 673 // ---------------------------------------------------------------------
 674 /* This runs the select loop over the server FDs, Output file FDs and
 675    stdin. */
 676 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 677 {
 678    // Server has closed the connection
 679    if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
 680       return false;
 681
 682    fd_set rfds,wfds,efds;
 683    FD_ZERO(&rfds);
 684    FD_ZERO(&wfds);
 685    FD_ZERO(&efds);
 686
 687    // Add the server
 688    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 689       FD_SET(Srv->ServerFd,&wfds);
 690    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 691       FD_SET(Srv->ServerFd,&rfds);
 692
 693    // Add the file
 694    int FileFD = -1;
 695    if (File != 0)
 696       FileFD = File->Fd();
 697
 698    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 699       FD_SET(FileFD,&wfds);
 700
 701    // Add stdin
 702    FD_SET(STDIN_FILENO,&rfds);
 703
 704    // Error Set
 705    if (FileFD != -1)
 706       FD_SET(FileFD,&efds);
 707    if (Srv->ServerFd != -1)
 708       FD_SET(Srv->ServerFd,&efds);
 709
 710    // Figure out the max fd
 711    int MaxFd = FileFD;
 712    if (MaxFd < Srv->ServerFd)
 713       MaxFd = Srv->ServerFd;
 714
 715    // Select
 716    struct timeval tv;
 717    tv.tv_sec = TimeOut;
 718    tv.tv_usec = 0;
 719    int Res = 0;
 720    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 721       return _error->Errno("select","Select failed");
 722
 723    if (Res == 0)
 724    {
 725       _error->Error("Connection timed out");
 726       return ServerDie(Srv);
 727    }
 728
 729    // Some kind of exception (error) on the sockets, die
 730    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 731        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 732       return _error->Error("Socket Exception");
 733
 734    // Handle server IO
 735    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 736    {
 737       errno = 0;
 738       if (Srv->In.Read(Srv->ServerFd) == false)
 739          return ServerDie(Srv);
 740    }
 741
 742    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 743    {
 744       errno = 0;
 745       if (Srv->Out.Write(Srv->ServerFd) == false)
 746          return ServerDie(Srv);
 747    }
 748
 749    // Send data to the file
 750    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 751    {
 752       if (Srv->In.Write(FileFD) == false)
 753          return _error->Errno("write","Error writing to output file");
 754    }
 755
 756    // Handle commands from APT
 757    if (FD_ISSET(STDIN_FILENO,&rfds))
 758    {
 759       if (Run(true) != 0)
 760          exit(100);
 761    }
 762
 763    return true;
 764 }
 765                                                                         /*}}}*/
 766 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 767 // ---------------------------------------------------------------------
 768 /* This takes the current input buffer from the Server FD and writes it
 769    into the file */
 770 bool HttpMethod::Flush(ServerState *Srv)
 771 {
 772    if (File != 0)
 773    {
 774       SetNonBlock(File->Fd(),false);
 775       if (Srv->In.WriteSpace() == false)
 776          return true;
 777
 778       while (Srv->In.WriteSpace() == true)
 779       {
 780          if (Srv->In.Write(File->Fd()) == false)
 781             return _error->Errno("write","Error writing to file");
 782          if (Srv->In.IsLimit() == true)
 783             return true;
 784       }
 785
 786       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 787          return true;
 788    }
 789    return false;
 790 }
 791                                                                         /*}}}*/
 792 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 793 // ---------------------------------------------------------------------
 794 /* */
 795 bool HttpMethod::ServerDie(ServerState *Srv)
 796 {
 797    // Dump the buffer to the file
 798    if (Srv->State == ServerState::Data)
 799    {
 800       SetNonBlock(File->Fd(),false);
 801       while (Srv->In.WriteSpace() == true)
 802       {
 803          if (Srv->In.Write(File->Fd()) == false)
 804             return _error->Errno("write","Error writing to the file");
 805
 806          // Done
 807          if (Srv->In.IsLimit() == true)
 808             return true;
 809       }
 810    }
 811
 812    // See if this is because the server finished the data stream
 813    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 814        Srv->Encoding != ServerState::Closes)
 815    {
 816       Srv->Close();
 817       if (errno == 0)
 818          return _error->Error("Error reading from server Remote end closed connection");
 819       return _error->Errno("read","Error reading from server");
 820    }
 821    else
 822    {
 823       Srv->In.Limit(-1);
 824
 825       // Nothing left in the buffer
 826       if (Srv->In.WriteSpace() == false)
 827          return false;
 828
 829       // We may have got multiple responses back in one packet..
 830       Srv->Close();
 831       return true;
 832    }
 833
 834    return false;
 835 }
 836                                                                         /*}}}*/
 837 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 838 // ---------------------------------------------------------------------
 839 /* We look at the header data we got back from the server and decide what
 840    to do. Returns
 841      0 - File is open,
 842      1 - IMS hit
 843      3 - Unrecoverable error
 844      4 - Error with error content page
 845      5 - Unrecoverable non-server error (close the connection) */
 846 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 847 {
 848    // Not Modified
 849    if (Srv->Result == 304)
 850    {
 851       unlink(Queue->DestFile.c_str());
 852       Res.IMSHit = true;
 853       Res.LastModified = Queue->LastModified;
 854       return 1;
 855    }
 856
 857    /* We have a reply we dont handle. This should indicate a perm server
 858       failure */
 859    if (Srv->Result < 200 || Srv->Result >= 300)
 860    {
 861       _error->Error("%u %s",Srv->Result,Srv->Code);
 862       if (Srv->HaveContent == true)
 863          return 4;
 864       return 3;
 865    }
 866
 867    // This is some sort of 2xx 'data follows' reply
 868    Res.LastModified = Srv->Date;
 869    Res.Size = Srv->Size;
 870
 871    // Open the file
 872    delete File;
 873    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 874    if (_error->PendingError() == true)
 875       return 5;
 876
 877    FailFile = Queue->DestFile;
 878    FailFd = File->Fd();
 879    FailTime = Srv->Date;
 880
 881    // Set the expected size
 882    if (Srv->StartPos >= 0)
 883    {
 884       Res.ResumePoint = Srv->StartPos;
 885       ftruncate(File->Fd(),Srv->StartPos);
 886    }
 887
 888    // Set the start point
 889    lseek(File->Fd(),0,SEEK_END);
 890
 891    delete Srv->In.MD5;
 892    Srv->In.MD5 = new MD5Summation;
 893
 894    // Fill the MD5 Hash if the file is non-empty (resume)
 895    if (Srv->StartPos > 0)
 896    {
 897       lseek(File->Fd(),0,SEEK_SET);
 898       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 899       {
 900          _error->Errno("read","Problem hashing file");
 901          return 5;
 902       }
 903       lseek(File->Fd(),0,SEEK_END);
 904    }
 905
 906    SetNonBlock(File->Fd(),true);
 907    return 0;
 908 }
 909                                                                         /*}}}*/
 910 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 911 // ---------------------------------------------------------------------
 912 /* This closes and timestamps the open file. This is neccessary to get
 913    resume behavoir on user abort */
 914 void HttpMethod::SigTerm(int)
 915 {
 916    if (FailFd == -1)
 917       exit(100);
 918    close(FailFd);
 919
 920    // Timestamp
 921    struct utimbuf UBuf;
 922    time(&UBuf.actime);
 923    UBuf.actime = FailTime;
 924    UBuf.modtime = FailTime;
 925    utime(FailFile.c_str(),&UBuf);
 926
 927    exit(100);
 928 }
 929                                                                         /*}}}*/
 930 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 931 // ---------------------------------------------------------------------
 932 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 933    depth. */
 934 bool HttpMethod::Fetch(FetchItem *)
 935 {
 936    if (Server == 0)
 937       return true;
 938
 939    // Queue the requests
 940    int Depth = -1;
 941    bool Tail = false;
 942    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++)
 943    {
 944       // Make sure we stick with the same server
 945       if (Server->Comp(I->Uri) == false)
 946          break;
 947       if (QueueBack == I)
 948          Tail = true;
 949       if (Tail == true)
 950       {
 951          QueueBack = I->Next;
 952          SendReq(I,Server->Out);
 953          continue;
 954       }
 955    }
 956
 957    return true;
 958 };
 959                                                                         /*}}}*/
 960 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 961 // ---------------------------------------------------------------------
 962 /* We stash the desired pipeline depth */
 963 bool HttpMethod::Configuration(string Message)
 964 {
 965    if (pkgAcqMethod::Configuration(Message) == false)
 966       return false;
 967
 968    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 969    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 970                                   PipelineDepth);
 971
 972    return true;
 973 }
 974                                                                         /*}}}*/
 975 // HttpMethod::Loop - Main loop                                         /*{{{*/
 976 // ---------------------------------------------------------------------
 977 /* */
 978 int HttpMethod::Loop()
 979 {
 980    signal(SIGTERM,SigTerm);
 981    signal(SIGINT,SigTerm);
 982
 983    Server = 0;
 984
 985    int FailCounter = 0;
 986    while (1)
 987    {
 988       if (FailCounter >= 2)
 989       {
 990          Fail("Massive Server Brain Damage");
 991          FailCounter = 0;
 992       }
 993
 994       // We have no commands, wait for some to arrive
 995       if (Queue == 0)
 996       {
 997          if (WaitFd(STDIN_FILENO) == false)
 998             return 0;
 999       }
1000
1001       // Run messages
1002       if (Run(true) != 0)
1003          return 100;
1004
1005       if (Queue == 0)
1006          continue;
1007
1008       // Connect to the server
1009       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1010       {
1011          delete Server;
1012          Server = new ServerState(Queue->Uri,this);
1013       }
1014
1015       // Reset the pipeline
1016       if (Server->ServerFd == -1)
1017          QueueBack = Queue;
1018
1019       // Connnect to the host
1020       if (Server->Open() == false)
1021       {
1022          Fail();
1023          continue;
1024       }
1025
1026       // Fill the pipeline.
1027       Fetch(0);
1028
1029       // Fetch the next URL header data from the server.
1030       switch (Server->RunHeaders())
1031       {
1032          case 0:
1033          break;
1034
1035          // The header data is bad
1036          case 2:
1037          {
1038             _error->Error("Bad header Data");
1039             Fail();
1040             continue;
1041          }
1042
1043          // The server closed a connection during the header get..
1044          default:
1045          case 1:
1046          {
1047             FailCounter++;
1048             _error->Discard();
1049             Server->Close();
1050             continue;
1051          }
1052       };
1053
1054       // Decide what to do.
1055       FetchResult Res;
1056       Res.Filename = Queue->DestFile;
1057       switch (DealWithHeaders(Res,Server))
1058       {
1059          // Ok, the file is Open
1060          case 0:
1061          {
1062             URIStart(Res);
1063
1064             // Run the data
1065             bool Result =  Server->RunData();
1066
1067             // Close the file, destroy the FD object and timestamp it
1068             FailFd = -1;
1069             delete File;
1070             File = 0;
1071
1072             // Timestamp
1073             struct utimbuf UBuf;
1074             time(&UBuf.actime);
1075             UBuf.actime = Server->Date;
1076             UBuf.modtime = Server->Date;
1077             utime(Queue->DestFile.c_str(),&UBuf);
1078
1079             // Send status to APT
1080             if (Result == true)
1081             {
1082                Res.MD5Sum = Server->In.MD5->Result();
1083                URIDone(Res);
1084             }
1085             else
1086                Fail();
1087
1088             break;
1089          }
1090
1091          // IMS hit
1092          case 1:
1093          {
1094             URIDone(Res);
1095             break;
1096          }
1097
1098          // Hard server error, not found or something
1099          case 3:
1100          {
1101             Fail();
1102             break;
1103          }
1104
1105          // Hard internal error, kill the connection and fail
1106          case 5:
1107          {
1108             Fail();
1109             Server->Close();
1110             break;
1111          }
1112
1113          // We need to flush the data, the header is like a 404 w/ error text
1114          case 4:
1115          {
1116             Fail();
1117
1118             // Send to content to dev/null
1119             File = new FileFd("/dev/null",FileFd::WriteExists);
1120             Server->RunData();
1121             delete File;
1122             File = 0;
1123             break;
1124          }
1125
1126          default:
1127          Fail("Internal error");
1128          break;
1129       }
1130
1131       FailCounter = 0;
1132    }
1133
1134    return 0;
1135 }
1136                                                                         /*}}}*/
1137
1138 int main()
1139 {
1140    HttpMethod Mth;
1141
1142    return Mth.Loop();
1143 }