methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.24 1999/01/30 08:23:49 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41
  42 // Internet stuff
  43 #include <netinet/in.h>
  44 #include <sys/socket.h>
  45 #include <arpa/inet.h>
  46 #include <netdb.h>
  47
  48 #include "http.h"
  49                                                                         /*}}}*/
  50
  51 string HttpMethod::FailFile;
  52 int HttpMethod::FailFd = -1;
  53 time_t HttpMethod::FailTime = 0;
  54 unsigned long PipelineDepth = 10;
  55 unsigned long TimeOut = 120;
  56
  57 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  58 // ---------------------------------------------------------------------
  59 /* */
  60 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  61 {
  62    Buf = new unsigned char[Size];
  63    Reset();
  64 }
  65                                                                         /*}}}*/
  66 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  67 // ---------------------------------------------------------------------
  68 /* */
  69 void CircleBuf::Reset()
  70 {
  71    InP = 0;
  72    OutP = 0;
  73    StrPos = 0;
  74    MaxGet = (unsigned int)-1;
  75    OutQueue = string();
  76    if (MD5 != 0)
  77    {
  78       delete MD5;
  79       MD5 = new MD5Summation;
  80    }
  81 };
  82                                                                         /*}}}*/
  83 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  84 // ---------------------------------------------------------------------
  85 /* This fills up the buffer with as much data as is in the FD, assuming it
  86    is non-blocking.. */
  87 bool CircleBuf::Read(int Fd)
  88 {
  89    while (1)
  90    {
  91       // Woops, buffer is full
  92       if (InP - OutP == Size)
  93          return true;
  94
  95       // Write the buffer segment
  96       int Res;
  97       Res = read(Fd,Buf + (InP%Size),LeftRead());
  98
  99       if (Res == 0)
 100          return false;
 101       if (Res < 0)
 102       {
 103          if (errno == EAGAIN)
 104             return true;
 105          return false;
 106       }
 107
 108       if (InP == 0)
 109          gettimeofday(&Start,0);
 110       InP += Res;
 111    }
 112 }
 113                                                                         /*}}}*/
 114 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 115 // ---------------------------------------------------------------------
 116 /* This will hold the string in and fill the buffer with it as it empties */
 117 bool CircleBuf::Read(string Data)
 118 {
 119    OutQueue += Data;
 120    FillOut();
 121    return true;
 122 }
 123                                                                         /*}}}*/
 124 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 125 // ---------------------------------------------------------------------
 126 /* */
 127 void CircleBuf::FillOut()
 128 {
 129    if (OutQueue.empty() == true)
 130       return;
 131    while (1)
 132    {
 133       // Woops, buffer is full
 134       if (InP - OutP == Size)
 135          return;
 136
 137       // Write the buffer segment
 138       unsigned long Sz = LeftRead();
 139       if (OutQueue.length() - StrPos < Sz)
 140          Sz = OutQueue.length() - StrPos;
 141       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 142
 143       // Advance
 144       StrPos += Sz;
 145       InP += Sz;
 146       if (OutQueue.length() == StrPos)
 147       {
 148          StrPos = 0;
 149          OutQueue = "";
 150          return;
 151       }
 152    }
 153 }
 154                                                                         /*}}}*/
 155 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 156 // ---------------------------------------------------------------------
 157 /* This empties the buffer into the FD. */
 158 bool CircleBuf::Write(int Fd)
 159 {
 160    while (1)
 161    {
 162       FillOut();
 163
 164       // Woops, buffer is empty
 165       if (OutP == InP)
 166          return true;
 167
 168       if (OutP == MaxGet)
 169          return true;
 170
 171       // Write the buffer segment
 172       int Res;
 173       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 174
 175       if (Res == 0)
 176          return false;
 177       if (Res < 0)
 178       {
 179          if (errno == EAGAIN)
 180             return true;
 181
 182          return false;
 183       }
 184
 185       if (MD5 != 0)
 186          MD5->Add(Buf + (OutP%Size),Res);
 187
 188       OutP += Res;
 189    }
 190 }
 191                                                                         /*}}}*/
 192 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 193 // ---------------------------------------------------------------------
 194 /* This copies till the first empty line */
 195 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 196 {
 197    // We cheat and assume it is unneeded to have more than one buffer load
 198    for (unsigned long I = OutP; I < InP; I++)
 199    {
 200       if (Buf[I%Size] != '\n')
 201          continue;
 202       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 203
 204       if (Single == false)
 205       {
 206          if (Buf[I%Size] != '\n')
 207             continue;
 208          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 209       }
 210
 211       if (I > InP)
 212          I = InP;
 213
 214       Data = "";
 215       while (OutP < I)
 216       {
 217          unsigned long Sz = LeftWrite();
 218          if (Sz == 0)
 219             return false;
 220          if (I - OutP < LeftWrite())
 221             Sz = I - OutP;
 222          Data += string((char *)(Buf + (OutP%Size)),Sz);
 223          OutP += Sz;
 224       }
 225       return true;
 226    }
 227    return false;
 228 }
 229                                                                         /*}}}*/
 230 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 231 // ---------------------------------------------------------------------
 232 /* */
 233 void CircleBuf::Stats()
 234 {
 235    if (InP == 0)
 236       return;
 237
 238    struct timeval Stop;
 239    gettimeofday(&Stop,0);
 240 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 241              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 242    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 243 }
 244                                                                         /*}}}*/
 245
 246 // ServerState::ServerState - Constructor                               /*{{{*/
 247 // ---------------------------------------------------------------------
 248 /* */
 249 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 250                         In(64*1024), Out(4*1024),
 251                         ServerName(Srv)
 252 {
 253    Reset();
 254 }
 255                                                                         /*}}}*/
 256 // ServerState::Open - Open a connection to the server                  /*{{{*/
 257 // ---------------------------------------------------------------------
 258 /* This opens a connection to the server. */
 259 string LastHost;
 260 in_addr LastHostA;
 261 bool ServerState::Open()
 262 {
 263    // Use the already open connection if possible.
 264    if (ServerFd != -1)
 265       return true;
 266
 267    Close();
 268    In.Reset();
 269    Out.Reset();
 270
 271    // Determine the proxy setting
 272    if (getenv("http_proxy") == 0)
 273    {
 274       string DefProxy = _config->Find("Acquire::http::Proxy");
 275       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 276       if (SpecificProxy.empty() == false)
 277       {
 278          if (SpecificProxy == "DIRECT")
 279             Proxy = "";
 280          else
 281             Proxy = SpecificProxy;
 282       }
 283       else
 284          Proxy = DefProxy;
 285    }
 286    else
 287       Proxy = getenv("http_proxy");
 288
 289    // Determine what host and port to use based on the proxy settings
 290    int Port = 80;
 291    string Host;
 292    if (Proxy.empty() == true)
 293    {
 294       if (ServerName.Port != 0)
 295          Port = ServerName.Port;
 296       Host = ServerName.Host;
 297    }
 298    else
 299    {
 300       if (Proxy.Port != 0)
 301          Port = Proxy.Port;
 302       Host = Proxy.Host;
 303    }
 304
 305    /* We used a cached address record.. Yes this is against the spec but
 306       the way we have setup our rotating dns suggests that this is more
 307       sensible */
 308    if (LastHost != Host)
 309    {
 310       Owner->Status("Connecting to %s",Host.c_str());
 311
 312       // Lookup the host
 313       hostent *Addr = gethostbyname(Host.c_str());
 314       if (Addr == 0 || Addr->h_addr_list[0] == 0)
 315          return _error->Error("Could not resolve '%s'",Host.c_str());
 316       LastHost = Host;
 317       LastHostA = *(in_addr *)(Addr->h_addr_list[0]);
 318    }
 319
 320    Owner->Status("Connecting to %s (%s)",Host.c_str(),inet_ntoa(LastHostA));
 321
 322    // Get a socket
 323    if ((ServerFd = socket(AF_INET,SOCK_STREAM,0)) < 0)
 324       return _error->Errno("socket","Could not create a socket");
 325
 326    // Connect to the server
 327    struct sockaddr_in server;
 328    server.sin_family = AF_INET;
 329    server.sin_port = htons(Port);
 330    server.sin_addr = LastHostA;
 331    SetNonBlock(ServerFd,true);
 332    if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0 &&
 333        errno != EINPROGRESS)
 334       return _error->Errno("socket","Could not create a socket");
 335
 336    /* This implements a timeout for connect by opening the connection
 337       nonblocking */
 338    fd_set wfds;
 339    FD_ZERO(&wfds);
 340    FD_SET(ServerFd,&wfds);
 341    struct timeval tv;
 342    tv.tv_sec = TimeOut;
 343    tv.tv_usec = 0;
 344    int Res = 0;
 345    if ((Res = select(ServerFd+1,0,&wfds,0,&tv)) < 0)
 346       return _error->Errno("select","Select failed");
 347    if (Res == 0)
 348       return _error->Error("Could not connect, connection timed out");
 349    unsigned int Err,Len=sizeof(Err);
 350    if (getsockopt(ServerFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 351       return _error->Errno("getsockopt","Failed");
 352    if (Err != 0)
 353       return _error->Error("Could not connect.");
 354
 355    return true;
 356 }
 357                                                                         /*}}}*/
 358 // ServerState::Close - Close a connection to the server                /*{{{*/
 359 // ---------------------------------------------------------------------
 360 /* */
 361 bool ServerState::Close()
 362 {
 363    close(ServerFd);
 364    ServerFd = -1;
 365    return true;
 366 }
 367                                                                         /*}}}*/
 368 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 369 // ---------------------------------------------------------------------
 370 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 371    parse error occured */
 372 int ServerState::RunHeaders()
 373 {
 374    State = Header;
 375
 376    Owner->Status("Waiting for file");
 377
 378    Major = 0;
 379    Minor = 0;
 380    Result = 0;
 381    Size = 0;
 382    StartPos = 0;
 383    Encoding = Closes;
 384    HaveContent = false;
 385    time(&Date);
 386
 387    do
 388    {
 389       string Data;
 390       if (In.WriteTillEl(Data) == false)
 391          continue;
 392
 393       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 394       {
 395          string::const_iterator J = I;
 396          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 397          if (HeaderLine(string(I,J-I)) == false)
 398             return 2;
 399          I = J;
 400       }
 401       return 0;
 402    }
 403    while (Owner->Go(false,this) == true);
 404
 405    return 1;
 406 }
 407                                                                         /*}}}*/
 408 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 409 // ---------------------------------------------------------------------
 410 /* */
 411 bool ServerState::RunData()
 412 {
 413    State = Data;
 414
 415    // Chunked transfer encoding is fun..
 416    if (Encoding == Chunked)
 417    {
 418       while (1)
 419       {
 420          // Grab the block size
 421          bool Last = true;
 422          string Data;
 423          In.Limit(-1);
 424          do
 425          {
 426             if (In.WriteTillEl(Data,true) == true)
 427                break;
 428          }
 429          while ((Last = Owner->Go(false,this)) == true);
 430
 431          if (Last == false)
 432             return false;
 433
 434          // See if we are done
 435          unsigned long Len = strtol(Data.c_str(),0,16);
 436          if (Len == 0)
 437          {
 438             In.Limit(-1);
 439
 440             // We have to remove the entity trailer
 441             Last = true;
 442             do
 443             {
 444                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 445                   break;
 446             }
 447             while ((Last = Owner->Go(false,this)) == true);
 448             if (Last == false)
 449                return false;
 450             return true;
 451          }
 452
 453          // Transfer the block
 454          In.Limit(Len);
 455          while (Owner->Go(true,this) == true)
 456             if (In.IsLimit() == true)
 457                break;
 458
 459          // Error
 460          if (In.IsLimit() == false)
 461             return false;
 462
 463          // The server sends an extra new line before the next block specifier..
 464          In.Limit(-1);
 465          Last = true;
 466          do
 467          {
 468             if (In.WriteTillEl(Data,true) == true)
 469                break;
 470          }
 471          while ((Last = Owner->Go(false,this)) == true);
 472          if (Last == false)
 473             return false;
 474       }
 475    }
 476    else
 477    {
 478       /* Closes encoding is used when the server did not specify a size, the
 479          loss of the connection means we are done */
 480       if (Encoding == Closes)
 481          In.Limit(-1);
 482       else
 483          In.Limit(Size - StartPos);
 484
 485       // Just transfer the whole block.
 486       do
 487       {
 488          if (In.IsLimit() == false)
 489             continue;
 490
 491          In.Limit(-1);
 492          return true;
 493       }
 494       while (Owner->Go(true,this) == true);
 495    }
 496
 497    return Owner->Flush(this);
 498 }
 499                                                                         /*}}}*/
 500 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 501 // ---------------------------------------------------------------------
 502 /* */
 503 bool ServerState::HeaderLine(string Line)
 504 {
 505    if (Line.empty() == true)
 506       return true;
 507
 508    // The http server might be trying to do something evil.
 509    if (Line.length() >= MAXLEN)
 510       return _error->Error("Got a single header line over %u chars",MAXLEN);
 511
 512    string::size_type Pos = Line.find(' ');
 513    if (Pos == string::npos || Pos+1 > Line.length())
 514       return _error->Error("Bad header line");
 515
 516    string Tag = string(Line,0,Pos);
 517    string Val = string(Line,Pos+1);
 518
 519    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 520    {
 521       // Evil servers return no version
 522       if (Line[4] == '/')
 523       {
 524          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 525                     &Result,Code) != 4)
 526             return _error->Error("The http server sent an invalid reply header");
 527       }
 528       else
 529       {
 530          Major = 0;
 531          Minor = 9;
 532          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 533             return _error->Error("The http server sent an invalid reply header");
 534       }
 535
 536       return true;
 537    }
 538
 539    if (stringcasecmp(Tag,"Content-Length:") == 0)
 540    {
 541       if (Encoding == Closes)
 542          Encoding = Stream;
 543       HaveContent = true;
 544
 545       // The length is already set from the Content-Range header
 546       if (StartPos != 0)
 547          return true;
 548
 549       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 550          return _error->Error("The http server sent an invalid Content-Length header");
 551       return true;
 552    }
 553
 554    if (stringcasecmp(Tag,"Content-Type:") == 0)
 555    {
 556       HaveContent = true;
 557       return true;
 558    }
 559
 560    if (stringcasecmp(Tag,"Content-Range:") == 0)
 561    {
 562       HaveContent = true;
 563
 564       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 565          return _error->Error("The http server sent an invalid Content-Range header");
 566       if ((unsigned)StartPos > Size)
 567          return _error->Error("This http server has broken range support");
 568       return true;
 569    }
 570
 571    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 572    {
 573       HaveContent = true;
 574       if (stringcasecmp(Val,"chunked") == 0)
 575          Encoding = Chunked;
 576
 577       return true;
 578    }
 579
 580    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 581    {
 582       if (StrToTime(Val,Date) == false)
 583          return _error->Error("Unknown date format");
 584       return true;
 585    }
 586
 587    return true;
 588 }
 589                                                                         /*}}}*/
 590
 591 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 592 // ---------------------------------------------------------------------
 593 /* This places the http request in the outbound buffer */
 594 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 595 {
 596    URI Uri = Itm->Uri;
 597
 598    // The HTTP server expects a hostname with a trailing :port
 599    char Buf[1000];
 600    string ProperHost = Uri.Host;
 601    if (Uri.Port != 0)
 602    {
 603       sprintf(Buf,":%u",Uri.Port);
 604       ProperHost += Buf;
 605    }
 606
 607    // Just in case.
 608    if (Itm->Uri.length() >= sizeof(Buf))
 609        abort();
 610
 611    /* Build the request. We include a keep-alive header only for non-proxy
 612       requests. This is to tweak old http/1.0 servers that do support keep-alive
 613       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 614       will glitch HTTP/1.0 proxies because they do not filter it out and
 615       pass it on, HTTP/1.1 says the connection should default to keep alive
 616       and we expect the proxy to do this */
 617    if (Proxy.empty() == true)
 618       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 619               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 620    else
 621    {
 622       /* Generate a cache control header if necessary. We place a max
 623          cache age on index files, optionally set a no-cache directive
 624          and a no-store directive for archives. */
 625       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 626               Itm->Uri.c_str(),ProperHost.c_str());
 627       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 628          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 629       else
 630       {
 631          if (Itm->IndexFile == true)
 632             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 633                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 634          else
 635          {
 636             if (_config->FindB("Acquire::http::No-Store",false) == true)
 637                strcat(Buf,"Cache-Control: no-store\r\n");
 638          }
 639       }
 640    }
 641
 642    string Req = Buf;
 643
 644    // Check for a partial file
 645    struct stat SBuf;
 646    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 647    {
 648       // In this case we send an if-range query with a range header
 649       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 650               TimeRFC1123(SBuf.st_mtime).c_str());
 651       Req += Buf;
 652    }
 653    else
 654    {
 655       if (Itm->LastModified != 0)
 656       {
 657          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 658          Req += Buf;
 659       }
 660    }
 661
 662    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 663       Req += string("Proxy-Authorization: Basic ") +
 664           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 665
 666    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 667 //   cerr << Req << endl;
 668
 669    Out.Read(Req);
 670 }
 671                                                                         /*}}}*/
 672 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 673 // ---------------------------------------------------------------------
 674 /* This runs the select loop over the server FDs, Output file FDs and
 675    stdin. */
 676 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 677 {
 678    // Server has closed the connection
 679    if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
 680       return false;
 681
 682    fd_set rfds,wfds,efds;
 683    FD_ZERO(&rfds);
 684    FD_ZERO(&wfds);
 685    FD_ZERO(&efds);
 686
 687    // Add the server
 688    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 689       FD_SET(Srv->ServerFd,&wfds);
 690    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 691       FD_SET(Srv->ServerFd,&rfds);
 692
 693    // Add the file
 694    int FileFD = -1;
 695    if (File != 0)
 696       FileFD = File->Fd();
 697
 698    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 699       FD_SET(FileFD,&wfds);
 700
 701    // Add stdin
 702    FD_SET(STDIN_FILENO,&rfds);
 703
 704    // Error Set
 705    if (FileFD != -1)
 706       FD_SET(FileFD,&efds);
 707    if (Srv->ServerFd != -1)
 708       FD_SET(Srv->ServerFd,&efds);
 709
 710    // Figure out the max fd
 711    int MaxFd = FileFD;
 712    if (MaxFd < Srv->ServerFd)
 713       MaxFd = Srv->ServerFd;
 714
 715    // Select
 716    struct timeval tv;
 717    tv.tv_sec = TimeOut;
 718    tv.tv_usec = 0;
 719    int Res = 0;
 720    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 721       return _error->Errno("select","Select failed");
 722
 723    if (Res == 0)
 724    {
 725       _error->Error("Connection timed out");
 726       return ServerDie(Srv);
 727    }
 728
 729    // Some kind of exception (error) on the sockets, die
 730    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 731        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 732       return _error->Error("Socket Exception");
 733
 734    // Handle server IO
 735    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 736    {
 737       errno = 0;
 738       if (Srv->In.Read(Srv->ServerFd) == false)
 739          return ServerDie(Srv);
 740    }
 741
 742    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 743    {
 744       errno = 0;
 745       if (Srv->Out.Write(Srv->ServerFd) == false)
 746          return ServerDie(Srv);
 747    }
 748
 749    // Send data to the file
 750    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 751    {
 752       if (Srv->In.Write(FileFD) == false)
 753          return _error->Errno("write","Error writing to output file");
 754    }
 755
 756    // Handle commands from APT
 757    if (FD_ISSET(STDIN_FILENO,&rfds))
 758    {
 759       if (Run(true) != 0)
 760          exit(100);
 761    }
 762
 763    return true;
 764 }
 765                                                                         /*}}}*/
 766 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 767 // ---------------------------------------------------------------------
 768 /* This takes the current input buffer from the Server FD and writes it
 769    into the file */
 770 bool HttpMethod::Flush(ServerState *Srv)
 771 {
 772    if (File != 0)
 773    {
 774       SetNonBlock(File->Fd(),false);
 775       if (Srv->In.WriteSpace() == false)
 776          return true;
 777
 778       while (Srv->In.WriteSpace() == true)
 779       {
 780          if (Srv->In.Write(File->Fd()) == false)
 781             return _error->Errno("write","Error writing to file");
 782          if (Srv->In.IsLimit() == true)
 783             return true;
 784       }
 785
 786       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 787          return true;
 788    }
 789    return false;
 790 }
 791                                                                         /*}}}*/
 792 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 793 // ---------------------------------------------------------------------
 794 /* */
 795 bool HttpMethod::ServerDie(ServerState *Srv)
 796 {
 797    // Dump the buffer to the file
 798    if (Srv->State == ServerState::Data)
 799    {
 800       SetNonBlock(File->Fd(),false);
 801       while (Srv->In.WriteSpace() == true)
 802       {
 803          if (Srv->In.Write(File->Fd()) == false)
 804             return _error->Errno("write","Error writing to the file");
 805
 806          // Done
 807          if (Srv->In.IsLimit() == true)
 808             return true;
 809       }
 810    }
 811
 812    // See if this is because the server finished the data stream
 813    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 814        Srv->Encoding != ServerState::Closes)
 815    {
 816       Srv->Close();
 817       if (errno == 0)
 818          return _error->Error("Error reading from server Remote end closed connection");
 819       return _error->Errno("read","Error reading from server");
 820    }
 821    else
 822    {
 823       Srv->In.Limit(-1);
 824
 825       // Nothing left in the buffer
 826       if (Srv->In.WriteSpace() == false)
 827          return false;
 828
 829       // We may have got multiple responses back in one packet..
 830       Srv->Close();
 831       return true;
 832    }
 833
 834    return false;
 835 }
 836                                                                         /*}}}*/
 837 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 838 // ---------------------------------------------------------------------
 839 /* We look at the header data we got back from the server and decide what
 840    to do. Returns
 841      0 - File is open,
 842      1 - IMS hit
 843      3 - Unrecoverable error
 844      4 - Error with error content page
 845      5 - Unrecoverable non-server error (close the connection) */
 846 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 847 {
 848    // Not Modified
 849    if (Srv->Result == 304)
 850    {
 851       unlink(Queue->DestFile.c_str());
 852       Res.IMSHit = true;
 853       Res.LastModified = Queue->LastModified;
 854       return 1;
 855    }
 856
 857    /* We have a reply we dont handle. This should indicate a perm server
 858       failure */
 859    if (Srv->Result < 200 || Srv->Result >= 300)
 860    {
 861       _error->Error("%u %s",Srv->Result,Srv->Code);
 862       if (Srv->HaveContent == true)
 863          return 4;
 864       return 3;
 865    }
 866
 867    // This is some sort of 2xx 'data follows' reply
 868    Res.LastModified = Srv->Date;
 869    Res.Size = Srv->Size;
 870
 871    // Open the file
 872    delete File;
 873    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 874    if (_error->PendingError() == true)
 875       return 5;
 876
 877    FailFile = Queue->DestFile;
 878    FailFile.c_str();   // Make sure we don't do a malloc in the signal handler
 879    FailFd = File->Fd();
 880    FailTime = Srv->Date;
 881
 882    // Set the expected size
 883    if (Srv->StartPos >= 0)
 884    {
 885       Res.ResumePoint = Srv->StartPos;
 886       ftruncate(File->Fd(),Srv->StartPos);
 887    }
 888
 889    // Set the start point
 890    lseek(File->Fd(),0,SEEK_END);
 891
 892    delete Srv->In.MD5;
 893    Srv->In.MD5 = new MD5Summation;
 894
 895    // Fill the MD5 Hash if the file is non-empty (resume)
 896    if (Srv->StartPos > 0)
 897    {
 898       lseek(File->Fd(),0,SEEK_SET);
 899       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 900       {
 901          _error->Errno("read","Problem hashing file");
 902          return 5;
 903       }
 904       lseek(File->Fd(),0,SEEK_END);
 905    }
 906
 907    SetNonBlock(File->Fd(),true);
 908    return 0;
 909 }
 910                                                                         /*}}}*/
 911 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 912 // ---------------------------------------------------------------------
 913 /* This closes and timestamps the open file. This is neccessary to get
 914    resume behavoir on user abort */
 915 void HttpMethod::SigTerm(int)
 916 {
 917    if (FailFd == -1)
 918       exit(100);
 919    close(FailFd);
 920
 921    // Timestamp
 922    struct utimbuf UBuf;
 923    time(&UBuf.actime);
 924    UBuf.actime = FailTime;
 925    UBuf.modtime = FailTime;
 926    utime(FailFile.c_str(),&UBuf);
 927
 928    exit(100);
 929 }
 930                                                                         /*}}}*/
 931 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 932 // ---------------------------------------------------------------------
 933 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 934    depth. */
 935 bool HttpMethod::Fetch(FetchItem *)
 936 {
 937    if (Server == 0)
 938       return true;
 939
 940    // Queue the requests
 941    int Depth = -1;
 942    bool Tail = false;
 943    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++)
 944    {
 945       // Make sure we stick with the same server
 946       if (Server->Comp(I->Uri) == false)
 947          break;
 948       if (QueueBack == I)
 949          Tail = true;
 950       if (Tail == true)
 951       {
 952          QueueBack = I->Next;
 953          SendReq(I,Server->Out);
 954          continue;
 955       }
 956    }
 957
 958    return true;
 959 };
 960                                                                         /*}}}*/
 961 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 962 // ---------------------------------------------------------------------
 963 /* We stash the desired pipeline depth */
 964 bool HttpMethod::Configuration(string Message)
 965 {
 966    if (pkgAcqMethod::Configuration(Message) == false)
 967       return false;
 968
 969    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 970    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 971                                   PipelineDepth);
 972
 973    return true;
 974 }
 975                                                                         /*}}}*/
 976 // HttpMethod::Loop - Main loop                                         /*{{{*/
 977 // ---------------------------------------------------------------------
 978 /* */
 979 int HttpMethod::Loop()
 980 {
 981    signal(SIGTERM,SigTerm);
 982    signal(SIGINT,SigTerm);
 983
 984    Server = 0;
 985
 986    int FailCounter = 0;
 987    while (1)
 988    {
 989       if (FailCounter >= 2)
 990       {
 991          Fail("Massive Server Brain Damage",true);
 992          FailCounter = 0;
 993       }
 994
 995       // We have no commands, wait for some to arrive
 996       if (Queue == 0)
 997       {
 998          if (WaitFd(STDIN_FILENO) == false)
 999             return 0;
1000       }
1001
1002       // Run messages
1003       if (Run(true) != 0)
1004          return 100;
1005
1006       if (Queue == 0)
1007          continue;
1008
1009       // Connect to the server
1010       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1011       {
1012          delete Server;
1013          Server = new ServerState(Queue->Uri,this);
1014       }
1015
1016       // Reset the pipeline
1017       if (Server->ServerFd == -1)
1018          QueueBack = Queue;
1019
1020       // Connnect to the host
1021       if (Server->Open() == false)
1022       {
1023          Fail(true);
1024          delete Server;
1025          Server = 0;
1026          continue;
1027       }
1028
1029       // Fill the pipeline.
1030       Fetch(0);
1031
1032       // Fetch the next URL header data from the server.
1033       switch (Server->RunHeaders())
1034       {
1035          case 0:
1036          break;
1037
1038          // The header data is bad
1039          case 2:
1040          {
1041             _error->Error("Bad header Data");
1042             Fail(true);
1043             continue;
1044          }
1045
1046          // The server closed a connection during the header get..
1047          default:
1048          case 1:
1049          {
1050             FailCounter++;
1051             _error->Discard();
1052             Server->Close();
1053             continue;
1054          }
1055       };
1056
1057       // Decide what to do.
1058       FetchResult Res;
1059       Res.Filename = Queue->DestFile;
1060       switch (DealWithHeaders(Res,Server))
1061       {
1062          // Ok, the file is Open
1063          case 0:
1064          {
1065             URIStart(Res);
1066
1067             // Run the data
1068             bool Result =  Server->RunData();
1069
1070             // Close the file, destroy the FD object and timestamp it
1071             FailFd = -1;
1072             delete File;
1073             File = 0;
1074
1075             // Timestamp
1076             struct utimbuf UBuf;
1077             time(&UBuf.actime);
1078             UBuf.actime = Server->Date;
1079             UBuf.modtime = Server->Date;
1080             utime(Queue->DestFile.c_str(),&UBuf);
1081
1082             // Send status to APT
1083             if (Result == true)
1084             {
1085                Res.MD5Sum = Server->In.MD5->Result();
1086                URIDone(Res);
1087             }
1088             else
1089                Fail();
1090
1091             break;
1092          }
1093
1094          // IMS hit
1095          case 1:
1096          {
1097             URIDone(Res);
1098             break;
1099          }
1100
1101          // Hard server error, not found or something
1102          case 3:
1103          {
1104             Fail();
1105             break;
1106          }
1107
1108          // Hard internal error, kill the connection and fail
1109          case 5:
1110          {
1111             Fail();
1112             Server->Close();
1113             break;
1114          }
1115
1116          // We need to flush the data, the header is like a 404 w/ error text
1117          case 4:
1118          {
1119             Fail();
1120
1121             // Send to content to dev/null
1122             File = new FileFd("/dev/null",FileFd::WriteExists);
1123             Server->RunData();
1124             delete File;
1125             File = 0;
1126             break;
1127          }
1128
1129          default:
1130          Fail("Internal error");
1131          break;
1132       }
1133
1134       FailCounter = 0;
1135    }
1136
1137    return 0;
1138 }
1139                                                                         /*}}}*/
1140
1141 int main()
1142 {
1143    HttpMethod Mth;
1144
1145    return Mth.Loop();
1146 }