methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.15 1998/12/11 01:45:36 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41
  42 // Internet stuff
  43 #include <netinet/in.h>
  44 #include <sys/socket.h>
  45 #include <arpa/inet.h>
  46 #include <netdb.h>
  47
  48 #include "http.h"
  49                                                                         /*}}}*/
  50
  51 string HttpMethod::FailFile;
  52 int HttpMethod::FailFd = -1;
  53 time_t HttpMethod::FailTime = 0;
  54 unsigned long PipelineDepth = 10;
  55 unsigned long TimeOut = 120;
  56
  57 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  58 // ---------------------------------------------------------------------
  59 /* */
  60 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  61 {
  62    Buf = new unsigned char[Size];
  63    Reset();
  64 }
  65                                                                         /*}}}*/
  66 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  67 // ---------------------------------------------------------------------
  68 /* */
  69 void CircleBuf::Reset()
  70 {
  71    InP = 0;
  72    OutP = 0;
  73    StrPos = 0;
  74    MaxGet = (unsigned int)-1;
  75    OutQueue = string();
  76    if (MD5 != 0)
  77    {
  78       delete MD5;
  79       MD5 = new MD5Summation;
  80    }
  81 };
  82                                                                         /*}}}*/
  83 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  84 // ---------------------------------------------------------------------
  85 /* This fills up the buffer with as much data as is in the FD, assuming it
  86    is non-blocking.. */
  87 bool CircleBuf::Read(int Fd)
  88 {
  89    while (1)
  90    {
  91       // Woops, buffer is full
  92       if (InP - OutP == Size)
  93          return true;
  94
  95       // Write the buffer segment
  96       int Res;
  97       Res = read(Fd,Buf + (InP%Size),LeftRead());
  98
  99       if (Res == 0)
 100          return false;
 101       if (Res < 0)
 102       {
 103          if (errno == EAGAIN)
 104             return true;
 105          return false;
 106       }
 107
 108       if (InP == 0)
 109          gettimeofday(&Start,0);
 110       InP += Res;
 111    }
 112 }
 113                                                                         /*}}}*/
 114 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 115 // ---------------------------------------------------------------------
 116 /* This will hold the string in and fill the buffer with it as it empties */
 117 bool CircleBuf::Read(string Data)
 118 {
 119    OutQueue += Data;
 120    FillOut();
 121    return true;
 122 }
 123                                                                         /*}}}*/
 124 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 125 // ---------------------------------------------------------------------
 126 /* */
 127 void CircleBuf::FillOut()
 128 {
 129    if (OutQueue.empty() == true)
 130       return;
 131    while (1)
 132    {
 133       // Woops, buffer is full
 134       if (InP - OutP == Size)
 135          return;
 136
 137       // Write the buffer segment
 138       unsigned long Sz = LeftRead();
 139       if (OutQueue.length() - StrPos < Sz)
 140          Sz = OutQueue.length() - StrPos;
 141       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 142
 143       // Advance
 144       StrPos += Sz;
 145       InP += Sz;
 146       if (OutQueue.length() == StrPos)
 147       {
 148          StrPos = 0;
 149          OutQueue = "";
 150          return;
 151       }
 152    }
 153 }
 154                                                                         /*}}}*/
 155 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 156 // ---------------------------------------------------------------------
 157 /* This empties the buffer into the FD. */
 158 bool CircleBuf::Write(int Fd)
 159 {
 160    while (1)
 161    {
 162       FillOut();
 163
 164       // Woops, buffer is empty
 165       if (OutP == InP)
 166          return true;
 167
 168       if (OutP == MaxGet)
 169          return true;
 170
 171       // Write the buffer segment
 172       int Res;
 173       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 174
 175       if (Res == 0)
 176          return false;
 177       if (Res < 0)
 178       {
 179          if (errno == EAGAIN)
 180             return true;
 181
 182          return false;
 183       }
 184
 185       if (MD5 != 0)
 186          MD5->Add(Buf + (OutP%Size),Res);
 187
 188       OutP += Res;
 189    }
 190 }
 191                                                                         /*}}}*/
 192 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 193 // ---------------------------------------------------------------------
 194 /* This copies till the first empty line */
 195 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 196 {
 197    // We cheat and assume it is unneeded to have more than one buffer load
 198    for (unsigned long I = OutP; I < InP; I++)
 199    {
 200       if (Buf[I%Size] != '\n')
 201          continue;
 202       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 203
 204       if (Single == false)
 205       {
 206          if (Buf[I%Size] != '\n')
 207             continue;
 208          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 209       }
 210
 211       if (I > InP)
 212          I = InP;
 213
 214       Data = "";
 215       while (OutP < I)
 216       {
 217          unsigned long Sz = LeftWrite();
 218          if (Sz == 0)
 219             return false;
 220          if (I - OutP < LeftWrite())
 221             Sz = I - OutP;
 222          Data += string((char *)(Buf + (OutP%Size)),Sz);
 223          OutP += Sz;
 224       }
 225       return true;
 226    }
 227    return false;
 228 }
 229                                                                         /*}}}*/
 230 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 231 // ---------------------------------------------------------------------
 232 /* */
 233 void CircleBuf::Stats()
 234 {
 235    if (InP == 0)
 236       return;
 237
 238    struct timeval Stop;
 239    gettimeofday(&Stop,0);
 240 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 241              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 242    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 243 }
 244                                                                         /*}}}*/
 245
 246 // ServerState::ServerState - Constructor                               /*{{{*/
 247 // ---------------------------------------------------------------------
 248 /* */
 249 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 250                         In(64*1024), Out(4*1024),
 251                         ServerName(Srv)
 252 {
 253    Reset();
 254 }
 255                                                                         /*}}}*/
 256 // ServerState::Open - Open a connection to the server                  /*{{{*/
 257 // ---------------------------------------------------------------------
 258 /* This opens a connection to the server. */
 259 string LastHost;
 260 in_addr LastHostA;
 261 bool ServerState::Open()
 262 {
 263    // Use the already open connection if possible.
 264    if (ServerFd != -1)
 265       return true;
 266
 267    Close();
 268    In.Reset();
 269    Out.Reset();
 270
 271    // Determine the proxy setting
 272    if (getenv("http_proxy") == 0)
 273    {
 274       string DefProxy = _config->Find("Acquire::http::Proxy");
 275       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 276       if (SpecificProxy.empty() == false)
 277       {
 278          if (SpecificProxy == "DIRECT")
 279             Proxy = "";
 280          else
 281             Proxy = SpecificProxy;
 282       }
 283       else
 284          Proxy = DefProxy;
 285    }
 286    else
 287       Proxy = getenv("http_proxy");
 288
 289    // Determine what host and port to use based on the proxy settings
 290    int Port = 80;
 291    string Host;
 292    if (Proxy.empty() == true)
 293    {
 294       if (ServerName.Port != 0)
 295          Port = ServerName.Port;
 296       Host = ServerName.Host;
 297    }
 298    else
 299    {
 300       if (Proxy.Port != 0)
 301          Port = Proxy.Port;
 302       Host = Proxy.Host;
 303    }
 304
 305    /* We used a cached address record.. Yes this is against the spec but
 306       the way we have setup our rotating dns suggests that this is more
 307       sensible */
 308    if (LastHost != Host)
 309    {
 310       Owner->Status("Connecting to %s",Host.c_str());
 311
 312       // Lookup the host
 313       hostent *Addr = gethostbyname(Host.c_str());
 314       if (Addr == 0 || Addr->h_addr_list[0] == 0)
 315          return _error->Error("Could not resolve '%s'",Host.c_str());
 316       LastHost = Host;
 317       LastHostA = *(in_addr *)(Addr->h_addr_list[0]);
 318    }
 319
 320    Owner->Status("Connecting to %s (%s)",Host.c_str(),inet_ntoa(LastHostA));
 321
 322    // Get a socket
 323    if ((ServerFd = socket(AF_INET,SOCK_STREAM,0)) < 0)
 324       return _error->Errno("socket","Could not create a socket");
 325
 326    // Connect to the server
 327    struct sockaddr_in server;
 328    server.sin_family = AF_INET;
 329    server.sin_port = htons(Port);
 330    server.sin_addr = LastHostA;
 331    SetNonBlock(ServerFd,true);
 332    if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0 &&
 333        errno != EINPROGRESS)
 334       return _error->Errno("socket","Could not create a socket");
 335
 336    /* This implements a timeout for connect by opening the connection
 337       nonblocking */
 338    fd_set wfds;
 339    FD_ZERO(&wfds);
 340    FD_SET(ServerFd,&wfds);
 341    struct timeval tv;
 342    tv.tv_sec = TimeOut;
 343    tv.tv_usec = 0;
 344    int Res = 0;
 345    if ((Res = select(ServerFd+1,0,&wfds,0,&tv)) < 0)
 346       return _error->Errno("select","Select failed");
 347    if (Res == 0)
 348       return _error->Error("Could not connect, connection timed out");
 349    unsigned int Err,Len;
 350    if (getsockopt(ServerFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 351       return _error->Errno("getsockopt","Failed");
 352    if (Err != 0)
 353       return _error->Error("Could not connect.");
 354
 355    return true;
 356 }
 357                                                                         /*}}}*/
 358 // ServerState::Close - Close a connection to the server                /*{{{*/
 359 // ---------------------------------------------------------------------
 360 /* */
 361 bool ServerState::Close()
 362 {
 363    close(ServerFd);
 364    ServerFd = -1;
 365    return true;
 366 }
 367                                                                         /*}}}*/
 368 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 369 // ---------------------------------------------------------------------
 370 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 371    parse error occured */
 372 int ServerState::RunHeaders()
 373 {
 374    State = Header;
 375
 376    Owner->Status("Waiting for file");
 377
 378    Major = 0;
 379    Minor = 0;
 380    Result = 0;
 381    Size = 0;
 382    StartPos = 0;
 383    Encoding = Closes;
 384    HaveContent = false;
 385    time(&Date);
 386
 387    do
 388    {
 389       string Data;
 390       if (In.WriteTillEl(Data) == false)
 391          continue;
 392
 393       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 394       {
 395          string::const_iterator J = I;
 396          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 397          if (HeaderLine(string(I,J-I)) == false)
 398             return 2;
 399          I = J;
 400       }
 401       return 0;
 402    }
 403    while (Owner->Go(false,this) == true);
 404
 405    return 1;
 406 }
 407                                                                         /*}}}*/
 408 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 409 // ---------------------------------------------------------------------
 410 /* */
 411 bool ServerState::RunData()
 412 {
 413    State = Data;
 414
 415    // Chunked transfer encoding is fun..
 416    if (Encoding == Chunked)
 417    {
 418       while (1)
 419       {
 420          // Grab the block size
 421          bool Last = true;
 422          string Data;
 423          In.Limit(-1);
 424          do
 425          {
 426             if (In.WriteTillEl(Data,true) == true)
 427                break;
 428          }
 429          while ((Last = Owner->Go(false,this)) == true);
 430
 431          if (Last == false)
 432             return false;
 433
 434          // See if we are done
 435          unsigned long Len = strtol(Data.c_str(),0,16);
 436          if (Len == 0)
 437          {
 438             In.Limit(-1);
 439
 440             // We have to remove the entity trailer
 441             Last = true;
 442             do
 443             {
 444                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 445                   break;
 446             }
 447             while ((Last = Owner->Go(false,this)) == true);
 448             if (Last == false)
 449                return false;
 450             return true;
 451          }
 452
 453          // Transfer the block
 454          In.Limit(Len);
 455          while (Owner->Go(true,this) == true)
 456             if (In.IsLimit() == true)
 457                break;
 458
 459          // Error
 460          if (In.IsLimit() == false)
 461             return false;
 462
 463          // The server sends an extra new line before the next block specifier..
 464          In.Limit(-1);
 465          Last = true;
 466          do
 467          {
 468             if (In.WriteTillEl(Data,true) == true)
 469                break;
 470          }
 471          while ((Last = Owner->Go(false,this)) == true);
 472          if (Last == false)
 473             return false;
 474       }
 475    }
 476    else
 477    {
 478       /* Closes encoding is used when the server did not specify a size, the
 479          loss of the connection means we are done */
 480       if (Encoding == Closes)
 481          In.Limit(-1);
 482       else
 483          In.Limit(Size - StartPos);
 484
 485       // Just transfer the whole block.
 486       do
 487       {
 488          if (In.IsLimit() == false)
 489             continue;
 490
 491          In.Limit(-1);
 492          return true;
 493       }
 494       while (Owner->Go(true,this) == true);
 495    }
 496
 497    return Owner->Flush(this);
 498 }
 499                                                                         /*}}}*/
 500 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 501 // ---------------------------------------------------------------------
 502 /* */
 503 bool ServerState::HeaderLine(string Line)
 504 {
 505    if (Line.empty() == true)
 506       return true;
 507
 508    // The http server might be trying to do something evil.
 509    if (Line.length() >= MAXLEN)
 510       return _error->Error("Got a single header line over %u chars",MAXLEN);
 511
 512    string::size_type Pos = Line.find(' ');
 513    if (Pos == string::npos || Pos+1 > Line.length())
 514       return _error->Error("Bad header line");
 515
 516    string Tag = string(Line,0,Pos);
 517    string Val = string(Line,Pos+1);
 518
 519    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 520    {
 521       // Evil servers return no version
 522       if (Line[4] == '/')
 523       {
 524          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 525                     &Result,Code) != 4)
 526             return _error->Error("The http server sent an invalid reply header");
 527       }
 528       else
 529       {
 530          Major = 0;
 531          Minor = 9;
 532          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 533             return _error->Error("The http server sent an invalid reply header");
 534       }
 535
 536       return true;
 537    }
 538
 539    if (stringcasecmp(Tag,"Content-Length:") == 0)
 540    {
 541       if (Encoding == Closes)
 542          Encoding = Stream;
 543       HaveContent = true;
 544
 545       // The length is already set from the Content-Range header
 546       if (StartPos != 0)
 547          return true;
 548
 549       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 550          return _error->Error("The http server sent an invalid Content-Length header");
 551       return true;
 552    }
 553
 554    if (stringcasecmp(Tag,"Content-Type:") == 0)
 555    {
 556       HaveContent = true;
 557       return true;
 558    }
 559
 560    if (stringcasecmp(Tag,"Content-Range:") == 0)
 561    {
 562       HaveContent = true;
 563
 564       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 565          return _error->Error("The http server sent an invalid Content-Range header");
 566       if ((unsigned)StartPos > Size)
 567          return _error->Error("This http server has broken range support");
 568       return true;
 569    }
 570
 571    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 572    {
 573       HaveContent = true;
 574       if (stringcasecmp(Val,"chunked") == 0)
 575          Encoding = Chunked;
 576
 577       return true;
 578    }
 579
 580    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 581    {
 582       if (StrToTime(Val,Date) == false)
 583          return _error->Error("Unknown date format");
 584       return true;
 585    }
 586
 587    return true;
 588 }
 589                                                                         /*}}}*/
 590
 591 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 592 // ---------------------------------------------------------------------
 593 /* This places the http request in the outbound buffer */
 594 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 595 {
 596    URI Uri = Itm->Uri;
 597
 598    // The HTTP server expects a hostname with a trailing :port
 599    char Buf[1000];
 600    string ProperHost = Uri.Host;
 601    if (Uri.Port != 0)
 602    {
 603       sprintf(Buf,":%u",Uri.Port);
 604       ProperHost += Buf;
 605    }
 606
 607    // Just in case.
 608    if (Itm->Uri.length() >= sizeof(Buf))
 609        abort();
 610
 611    /* Build the request. We include a keep-alive header only for non-proxy
 612       requests. This is to tweak old http/1.0 servers that do support keep-alive
 613       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 614       will glitch HTTP/1.0 proxies because they do not filter it out and
 615       pass it on, HTTP/1.1 says the connection should default to keep alive
 616       and we expect the proxy to do this */
 617    if (Proxy.empty() == true)
 618       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 619               Uri.Path.c_str(),ProperHost.c_str());
 620    else
 621    {
 622       /* Generate a cache control header if necessary. We place a max
 623          cache age on index files, optionally set a no-cache directive
 624          and a no-store directive for archives. */
 625       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 626               Itm->Uri.c_str(),ProperHost.c_str());
 627       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 628          strcat(Buf,"Cache-Control: no-cache\r\n");
 629       else
 630       {
 631          if (Itm->IndexFile == true)
 632             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 633                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 634          else
 635          {
 636             if (_config->FindB("Acquire::http::No-Store",false) == true)
 637                strcat(Buf,"Cache-Control: no-store\r\n");
 638          }
 639       }
 640    }
 641
 642    string Req = Buf;
 643
 644    // Check for a partial file
 645    struct stat SBuf;
 646    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 647    {
 648       // In this case we send an if-range query with a range header
 649       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 650               TimeRFC1123(SBuf.st_mtime).c_str());
 651       Req += Buf;
 652    }
 653    else
 654    {
 655       if (Itm->LastModified != 0)
 656       {
 657          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 658          Req += Buf;
 659       }
 660    }
 661
 662 /*   if (ProxyAuth.empty() == false)
 663       Req += string("Proxy-Authorization: Basic ") + Base64Encode(ProxyAuth) + "\r\n";*/
 664
 665    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 666 //   cerr << Req << endl;
 667
 668    Out.Read(Req);
 669 }
 670                                                                         /*}}}*/
 671 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 672 // ---------------------------------------------------------------------
 673 /* This runs the select loop over the server FDs, Output file FDs and
 674    stdin. */
 675 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 676 {
 677    // Server has closed the connection
 678    if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
 679       return false;
 680
 681    fd_set rfds,wfds,efds;
 682    FD_ZERO(&rfds);
 683    FD_ZERO(&wfds);
 684    FD_ZERO(&efds);
 685
 686    // Add the server
 687    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 688       FD_SET(Srv->ServerFd,&wfds);
 689    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 690       FD_SET(Srv->ServerFd,&rfds);
 691
 692    // Add the file
 693    int FileFD = -1;
 694    if (File != 0)
 695       FileFD = File->Fd();
 696
 697    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 698       FD_SET(FileFD,&wfds);
 699
 700    // Add stdin
 701    FD_SET(STDIN_FILENO,&rfds);
 702
 703    // Error Set
 704    if (FileFD != -1)
 705       FD_SET(FileFD,&efds);
 706    if (Srv->ServerFd != -1)
 707       FD_SET(Srv->ServerFd,&efds);
 708
 709    // Figure out the max fd
 710    int MaxFd = FileFD;
 711    if (MaxFd < Srv->ServerFd)
 712       MaxFd = Srv->ServerFd;
 713
 714    // Select
 715    struct timeval tv;
 716    tv.tv_sec = TimeOut;
 717    tv.tv_usec = 0;
 718    int Res = 0;
 719    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 720       return _error->Errno("select","Select failed");
 721
 722    if (Res == 0)
 723    {
 724       _error->Error("Connection timed out");
 725       return ServerDie(Srv);
 726    }
 727
 728    // Some kind of exception (error) on the sockets, die
 729    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 730        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 731       return _error->Error("Socket Exception");
 732
 733    // Handle server IO
 734    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 735    {
 736       errno = 0;
 737       if (Srv->In.Read(Srv->ServerFd) == false)
 738          return ServerDie(Srv);
 739    }
 740
 741    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 742    {
 743       errno = 0;
 744       if (Srv->Out.Write(Srv->ServerFd) == false)
 745          return ServerDie(Srv);
 746    }
 747
 748    // Send data to the file
 749    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 750    {
 751       if (Srv->In.Write(FileFD) == false)
 752          return _error->Errno("write","Error writing to output file");
 753    }
 754
 755    // Handle commands from APT
 756    if (FD_ISSET(STDIN_FILENO,&rfds))
 757    {
 758       if (Run(true) != 0)
 759          exit(100);
 760    }
 761
 762    return true;
 763 }
 764                                                                         /*}}}*/
 765 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 766 // ---------------------------------------------------------------------
 767 /* This takes the current input buffer from the Server FD and writes it
 768    into the file */
 769 bool HttpMethod::Flush(ServerState *Srv)
 770 {
 771    if (File != 0)
 772    {
 773       SetNonBlock(File->Fd(),false);
 774       if (Srv->In.WriteSpace() == false)
 775          return true;
 776
 777       while (Srv->In.WriteSpace() == true)
 778       {
 779          if (Srv->In.Write(File->Fd()) == false)
 780             return _error->Errno("write","Error writing to file");
 781          if (Srv->In.IsLimit() == true)
 782             return true;
 783       }
 784
 785       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 786          return true;
 787    }
 788    return false;
 789 }
 790                                                                         /*}}}*/
 791 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 792 // ---------------------------------------------------------------------
 793 /* */
 794 bool HttpMethod::ServerDie(ServerState *Srv)
 795 {
 796    // Dump the buffer to the file
 797    if (Srv->State == ServerState::Data)
 798    {
 799       SetNonBlock(File->Fd(),false);
 800       while (Srv->In.WriteSpace() == true)
 801       {
 802          if (Srv->In.Write(File->Fd()) == false)
 803             return _error->Errno("write","Error writing to the file");
 804
 805          // Done
 806          if (Srv->In.IsLimit() == true)
 807             return true;
 808       }
 809    }
 810
 811    // See if this is because the server finished the data stream
 812    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 813        Srv->Encoding != ServerState::Closes)
 814    {
 815       Srv->Close();
 816       if (errno == 0)
 817          return _error->Error("Error reading from server Remote end closed connection");
 818       return _error->Errno("read","Error reading from server");
 819    }
 820    else
 821    {
 822       Srv->In.Limit(-1);
 823
 824       // Nothing left in the buffer
 825       if (Srv->In.WriteSpace() == false)
 826          return false;
 827
 828       // We may have got multiple responses back in one packet..
 829       Srv->Close();
 830       return true;
 831    }
 832
 833    return false;
 834 }
 835                                                                         /*}}}*/
 836 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 837 // ---------------------------------------------------------------------
 838 /* We look at the header data we got back from the server and decide what
 839    to do. Returns
 840      0 - File is open,
 841      1 - IMS hit
 842      3 - Unrecoverable error
 843      4 - Error with error content page
 844      5 - Unrecoverable non-server error (close the connection) */
 845 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 846 {
 847    // Not Modified
 848    if (Srv->Result == 304)
 849    {
 850       unlink(Queue->DestFile.c_str());
 851       Res.IMSHit = true;
 852       Res.LastModified = Queue->LastModified;
 853       return 1;
 854    }
 855
 856    /* We have a reply we dont handle. This should indicate a perm server
 857       failure */
 858    if (Srv->Result < 200 || Srv->Result >= 300)
 859    {
 860       _error->Error("%u %s",Srv->Result,Srv->Code);
 861       if (Srv->HaveContent == true)
 862          return 4;
 863       return 3;
 864    }
 865
 866    // This is some sort of 2xx 'data follows' reply
 867    Res.LastModified = Srv->Date;
 868    Res.Size = Srv->Size;
 869
 870    // Open the file
 871    delete File;
 872    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 873    if (_error->PendingError() == true)
 874       return 5;
 875
 876    FailFile = Queue->DestFile;
 877    FailFd = File->Fd();
 878    FailTime = Srv->Date;
 879
 880    // Set the expected size
 881    if (Srv->StartPos >= 0)
 882    {
 883       Res.ResumePoint = Srv->StartPos;
 884       ftruncate(File->Fd(),Srv->StartPos);
 885    }
 886
 887    // Set the start point
 888    lseek(File->Fd(),0,SEEK_END);
 889
 890    delete Srv->In.MD5;
 891    Srv->In.MD5 = new MD5Summation;
 892
 893    // Fill the MD5 Hash if the file is non-empty (resume)
 894    if (Srv->StartPos > 0)
 895    {
 896       lseek(File->Fd(),0,SEEK_SET);
 897       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 898       {
 899          _error->Errno("read","Problem hashing file");
 900          return 5;
 901       }
 902       lseek(File->Fd(),0,SEEK_END);
 903    }
 904
 905    SetNonBlock(File->Fd(),true);
 906    return 0;
 907 }
 908                                                                         /*}}}*/
 909 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 910 // ---------------------------------------------------------------------
 911 /* This closes and timestamps the open file. This is neccessary to get
 912    resume behavoir on user abort */
 913 void HttpMethod::SigTerm(int)
 914 {
 915    if (FailFd == -1)
 916       exit(100);
 917    close(FailFd);
 918
 919    // Timestamp
 920    struct utimbuf UBuf;
 921    time(&UBuf.actime);
 922    UBuf.actime = FailTime;
 923    UBuf.modtime = FailTime;
 924    utime(FailFile.c_str(),&UBuf);
 925
 926    exit(100);
 927 }
 928                                                                         /*}}}*/
 929 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 930 // ---------------------------------------------------------------------
 931 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 932    depth. */
 933 bool HttpMethod::Fetch(FetchItem *)
 934 {
 935    if (Server == 0)
 936       return true;
 937
 938    // Queue the requests
 939    int Depth = -1;
 940    bool Tail = false;
 941    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++)
 942    {
 943       // Make sure we stick with the same server
 944       if (Server->Comp(I->Uri) == false)
 945          break;
 946       if (QueueBack == I)
 947          Tail = true;
 948       if (Tail == true)
 949       {
 950          QueueBack = I->Next;
 951          SendReq(I,Server->Out);
 952          continue;
 953       }
 954    }
 955
 956    return true;
 957 };
 958                                                                         /*}}}*/
 959 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 960 // ---------------------------------------------------------------------
 961 /* We stash the desired pipeline depth */
 962 bool HttpMethod::Configuration(string Message)
 963 {
 964    if (pkgAcqMethod::Configuration(Message) == false)
 965       return false;
 966
 967    TimeOut = _config->FindI("Acquire::http::Timeout",120);
 968    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",5);
 969
 970    return true;
 971 }
 972                                                                         /*}}}*/
 973 // HttpMethod::Loop - Main loop                                         /*{{{*/
 974 // ---------------------------------------------------------------------
 975 /* */
 976 int HttpMethod::Loop()
 977 {
 978    signal(SIGTERM,SigTerm);
 979    signal(SIGINT,SigTerm);
 980
 981    Server = 0;
 982
 983    int FailCounter = 0;
 984    while (1)
 985    {
 986       if (FailCounter >= 2)
 987       {
 988          Fail("Massive Server Brain Damage");
 989          FailCounter = 0;
 990       }
 991
 992       // We have no commands, wait for some to arrive
 993       if (Queue == 0)
 994       {
 995          if (WaitFd(STDIN_FILENO) == false)
 996             return 0;
 997       }
 998
 999       // Run messages
1000       if (Run(true) != 0)
1001          return 100;
1002
1003       if (Queue == 0)
1004          continue;
1005
1006       // Connect to the server
1007       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1008       {
1009          delete Server;
1010          Server = new ServerState(Queue->Uri,this);
1011       }
1012
1013       // Reset the pipeline
1014       if (Server->ServerFd == -1)
1015          QueueBack = Queue;
1016
1017       // Connnect to the host
1018       if (Server->Open() == false)
1019       {
1020          Fail();
1021          continue;
1022       }
1023
1024       // Fill the pipeline.
1025       Fetch(0);
1026
1027       // Fetch the next URL header data from the server.
1028       switch (Server->RunHeaders())
1029       {
1030          case 0:
1031          break;
1032
1033          // The header data is bad
1034          case 2:
1035          {
1036             _error->Error("Bad header Data");
1037             Fail();
1038             continue;
1039          }
1040
1041          // The server closed a connection during the header get..
1042          default:
1043          case 1:
1044          {
1045             FailCounter++;
1046             _error->Discard();
1047             Server->Close();
1048             continue;
1049          }
1050       };
1051
1052       // Decide what to do.
1053       FetchResult Res;
1054       Res.Filename = Queue->DestFile;
1055       switch (DealWithHeaders(Res,Server))
1056       {
1057          // Ok, the file is Open
1058          case 0:
1059          {
1060             URIStart(Res);
1061
1062             // Run the data
1063             bool Result =  Server->RunData();
1064
1065             // Close the file, destroy the FD object and timestamp it
1066             FailFd = -1;
1067             delete File;
1068             File = 0;
1069
1070             // Timestamp
1071             struct utimbuf UBuf;
1072             time(&UBuf.actime);
1073             UBuf.actime = Server->Date;
1074             UBuf.modtime = Server->Date;
1075             utime(Queue->DestFile.c_str(),&UBuf);
1076
1077             // Send status to APT
1078             if (Result == true)
1079             {
1080                Res.MD5Sum = Server->In.MD5->Result();
1081                URIDone(Res);
1082             }
1083             else
1084                Fail();
1085
1086             break;
1087          }
1088
1089          // IMS hit
1090          case 1:
1091          {
1092             URIDone(Res);
1093             break;
1094          }
1095
1096          // Hard server error, not found or something
1097          case 3:
1098          {
1099             Fail();
1100             break;
1101          }
1102
1103          // Hard internal error, kill the connection and fail
1104          case 5:
1105          {
1106             Fail();
1107             Server->Close();
1108             break;
1109          }
1110
1111          // We need to flush the data, the header is like a 404 w/ error text
1112          case 4:
1113          {
1114             Fail();
1115
1116             // Send to content to dev/null
1117             File = new FileFd("/dev/null",FileFd::WriteExists);
1118             Server->RunData();
1119             delete File;
1120             File = 0;
1121             break;
1122          }
1123
1124          default:
1125          Fail("Internal error");
1126          break;
1127       }
1128
1129       FailCounter = 0;
1130    }
1131
1132    return 0;
1133 }
1134                                                                         /*}}}*/
1135
1136 int main()
1137 {
1138    HttpMethod Mth;
1139
1140    return Mth.Loop();
1141 }