methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   4 /* ######################################################################
   5
   6    HTTP Acquire Method - This is the HTTP acquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <config.h>
  29
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/configuration.h>
  33 #include <apt-pkg/error.h>
  34 #include <apt-pkg/hashes.h>
  35 #include <apt-pkg/netrc.h>
  36 #include <apt-pkg/strutl.h>
  37
  38 #include <stddef.h>
  39 #include <stdlib.h>
  40 #include <sys/select.h>
  41 #include <cstring>
  42 #include <sys/stat.h>
  43 #include <sys/time.h>
  44 #include <unistd.h>
  45 #include <stdio.h>
  46 #include <errno.h>
  47 #include <iostream>
  48
  49 #include "config.h"
  50 #include "connect.h"
  51 #include "http.h"
  52
  53 #include <apti18n.h>
  54                                                                         /*}}}*/
  55 using namespace std;
  56
  57 unsigned long long CircleBuf::BwReadLimit=0;
  58 unsigned long long CircleBuf::BwTickReadData=0;
  59 struct timeval CircleBuf::BwReadTick={0,0};
  60 const unsigned int CircleBuf::BW_HZ=10;
  61
  62 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  63 // ---------------------------------------------------------------------
  64 /* */
  65 CircleBuf::CircleBuf(unsigned long long Size) : Size(Size), Hash(0)
  66 {
  67    Buf = new unsigned char[Size];
  68    Reset();
  69
  70    CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
  71 }
  72                                                                         /*}}}*/
  73 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  74 // ---------------------------------------------------------------------
  75 /* */
  76 void CircleBuf::Reset()
  77 {
  78    InP = 0;
  79    OutP = 0;
  80    StrPos = 0;
  81    MaxGet = (unsigned long long)-1;
  82    OutQueue = string();
  83    if (Hash != 0)
  84    {
  85       delete Hash;
  86       Hash = new Hashes;
  87    }
  88 }
  89                                                                         /*}}}*/
  90 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  91 // ---------------------------------------------------------------------
  92 /* This fills up the buffer with as much data as is in the FD, assuming it
  93    is non-blocking.. */
  94 bool CircleBuf::Read(int Fd)
  95 {
  96    while (1)
  97    {
  98       // Woops, buffer is full
  99       if (InP - OutP == Size)
 100          return true;
 101
 102       // what's left to read in this tick
 103       unsigned long long const BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
 104
 105       if(CircleBuf::BwReadLimit) {
 106          struct timeval now;
 107          gettimeofday(&now,0);
 108
 109          unsigned long long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
 110             now.tv_usec-CircleBuf::BwReadTick.tv_usec;
 111          if(d > 1000000/BW_HZ) {
 112             CircleBuf::BwReadTick = now;
 113             CircleBuf::BwTickReadData = 0;
 114          }
 115
 116          if(CircleBuf::BwTickReadData >= BwReadMax) {
 117             usleep(1000000/BW_HZ);
 118             return true;
 119          }
 120       }
 121
 122       // Write the buffer segment
 123       ssize_t Res;
 124       if(CircleBuf::BwReadLimit) {
 125          Res = read(Fd,Buf + (InP%Size),
 126                     BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
 127       } else
 128          Res = read(Fd,Buf + (InP%Size),LeftRead());
 129
 130       if(Res > 0 && BwReadLimit > 0)
 131          CircleBuf::BwTickReadData += Res;
 132
 133       if (Res == 0)
 134          return false;
 135       if (Res < 0)
 136       {
 137          if (errno == EAGAIN)
 138             return true;
 139          return false;
 140       }
 141
 142       if (InP == 0)
 143          gettimeofday(&Start,0);
 144       InP += Res;
 145    }
 146 }
 147                                                                         /*}}}*/
 148 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 149 // ---------------------------------------------------------------------
 150 /* This will hold the string in and fill the buffer with it as it empties */
 151 bool CircleBuf::Read(string Data)
 152 {
 153    OutQueue += Data;
 154    FillOut();
 155    return true;
 156 }
 157                                                                         /*}}}*/
 158 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 159 // ---------------------------------------------------------------------
 160 /* */
 161 void CircleBuf::FillOut()
 162 {
 163    if (OutQueue.empty() == true)
 164       return;
 165    while (1)
 166    {
 167       // Woops, buffer is full
 168       if (InP - OutP == Size)
 169          return;
 170
 171       // Write the buffer segment
 172       unsigned long long Sz = LeftRead();
 173       if (OutQueue.length() - StrPos < Sz)
 174          Sz = OutQueue.length() - StrPos;
 175       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 176
 177       // Advance
 178       StrPos += Sz;
 179       InP += Sz;
 180       if (OutQueue.length() == StrPos)
 181       {
 182          StrPos = 0;
 183          OutQueue = "";
 184          return;
 185       }
 186    }
 187 }
 188                                                                         /*}}}*/
 189 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 190 // ---------------------------------------------------------------------
 191 /* This empties the buffer into the FD. */
 192 bool CircleBuf::Write(int Fd)
 193 {
 194    while (1)
 195    {
 196       FillOut();
 197
 198       // Woops, buffer is empty
 199       if (OutP == InP)
 200          return true;
 201
 202       if (OutP == MaxGet)
 203          return true;
 204
 205       // Write the buffer segment
 206       ssize_t Res;
 207       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 208
 209       if (Res == 0)
 210          return false;
 211       if (Res < 0)
 212       {
 213          if (errno == EAGAIN)
 214             return true;
 215
 216          return false;
 217       }
 218
 219       if (Hash != 0)
 220          Hash->Add(Buf + (OutP%Size),Res);
 221
 222       OutP += Res;
 223    }
 224 }
 225                                                                         /*}}}*/
 226 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 227 // ---------------------------------------------------------------------
 228 /* This copies till the first empty line */
 229 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 230 {
 231    // We cheat and assume it is unneeded to have more than one buffer load
 232    for (unsigned long long I = OutP; I < InP; I++)
 233    {
 234       if (Buf[I%Size] != '\n')
 235          continue;
 236       ++I;
 237
 238       if (Single == false)
 239       {
 240          if (I < InP  && Buf[I%Size] == '\r')
 241             ++I;
 242          if (I >= InP || Buf[I%Size] != '\n')
 243             continue;
 244          ++I;
 245       }
 246
 247       Data = "";
 248       while (OutP < I)
 249       {
 250          unsigned long long Sz = LeftWrite();
 251          if (Sz == 0)
 252             return false;
 253          if (I - OutP < Sz)
 254             Sz = I - OutP;
 255          Data += string((char *)(Buf + (OutP%Size)),Sz);
 256          OutP += Sz;
 257       }
 258       return true;
 259    }
 260    return false;
 261 }
 262                                                                         /*}}}*/
 263 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 264 // ---------------------------------------------------------------------
 265 /* */
 266 void CircleBuf::Stats()
 267 {
 268    if (InP == 0)
 269       return;
 270
 271    struct timeval Stop;
 272    gettimeofday(&Stop,0);
 273 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 274              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 275    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 276 }
 277                                                                         /*}}}*/
 278 CircleBuf::~CircleBuf()
 279 {
 280    delete [] Buf;
 281    delete Hash;
 282 }
 283
 284 // HttpServerState::HttpServerState - Constructor                       /*{{{*/
 285 HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(64*1024), Out(4*1024)
 286 {
 287    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 288    Reset();
 289 }
 290                                                                         /*}}}*/
 291 // HttpServerState::Open - Open a connection to the server              /*{{{*/
 292 // ---------------------------------------------------------------------
 293 /* This opens a connection to the server. */
 294 bool HttpServerState::Open()
 295 {
 296    // Use the already open connection if possible.
 297    if (ServerFd != -1)
 298       return true;
 299
 300    Close();
 301    In.Reset();
 302    Out.Reset();
 303    Persistent = true;
 304
 305    // Determine the proxy setting
 306    string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 307    if (!SpecificProxy.empty())
 308    {
 309            if (SpecificProxy == "DIRECT")
 310                    Proxy = "";
 311            else
 312                    Proxy = SpecificProxy;
 313    }
 314    else
 315    {
 316            string DefProxy = _config->Find("Acquire::http::Proxy");
 317            if (!DefProxy.empty())
 318            {
 319                    Proxy = DefProxy;
 320            }
 321            else
 322            {
 323                    char* result = getenv("http_proxy");
 324                    Proxy = result ? result : "";
 325            }
 326    }
 327
 328    // Parse no_proxy, a , separated list of domains
 329    if (getenv("no_proxy") != 0)
 330    {
 331       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 332          Proxy = "";
 333    }
 334
 335    // Determine what host and port to use based on the proxy settings
 336    int Port = 0;
 337    string Host;
 338    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 339    {
 340       if (ServerName.Port != 0)
 341          Port = ServerName.Port;
 342       Host = ServerName.Host;
 343    }
 344    else
 345    {
 346       if (Proxy.Port != 0)
 347          Port = Proxy.Port;
 348       Host = Proxy.Host;
 349    }
 350
 351    // Connect to the remote server
 352    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 353       return false;
 354
 355    return true;
 356 }
 357                                                                         /*}}}*/
 358 // HttpServerState::Close - Close a connection to the server            /*{{{*/
 359 // ---------------------------------------------------------------------
 360 /* */
 361 bool HttpServerState::Close()
 362 {
 363    close(ServerFd);
 364    ServerFd = -1;
 365    return true;
 366 }
 367                                                                         /*}}}*/
 368 // HttpServerState::RunData - Transfer the data from the socket         /*{{{*/
 369 bool HttpServerState::RunData(FileFd * const File)
 370 {
 371    State = Data;
 372
 373    // Chunked transfer encoding is fun..
 374    if (Encoding == Chunked)
 375    {
 376       while (1)
 377       {
 378          // Grab the block size
 379          bool Last = true;
 380          string Data;
 381          In.Limit(-1);
 382          do
 383          {
 384             if (In.WriteTillEl(Data,true) == true)
 385                break;
 386          }
 387          while ((Last = Go(false, File)) == true);
 388
 389          if (Last == false)
 390             return false;
 391
 392          // See if we are done
 393          unsigned long long Len = strtoull(Data.c_str(),0,16);
 394          if (Len == 0)
 395          {
 396             In.Limit(-1);
 397
 398             // We have to remove the entity trailer
 399             Last = true;
 400             do
 401             {
 402                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 403                   break;
 404             }
 405             while ((Last = Go(false, File)) == true);
 406             if (Last == false)
 407                return false;
 408             return !_error->PendingError();
 409          }
 410
 411          // Transfer the block
 412          In.Limit(Len);
 413          while (Go(true, File) == true)
 414             if (In.IsLimit() == true)
 415                break;
 416
 417          // Error
 418          if (In.IsLimit() == false)
 419             return false;
 420
 421          // The server sends an extra new line before the next block specifier..
 422          In.Limit(-1);
 423          Last = true;
 424          do
 425          {
 426             if (In.WriteTillEl(Data,true) == true)
 427                break;
 428          }
 429          while ((Last = Go(false, File)) == true);
 430          if (Last == false)
 431             return false;
 432       }
 433    }
 434    else
 435    {
 436       /* Closes encoding is used when the server did not specify a size, the
 437          loss of the connection means we are done */
 438       if (Encoding == Closes)
 439          In.Limit(-1);
 440       else
 441          In.Limit(Size - StartPos);
 442
 443       // Just transfer the whole block.
 444       do
 445       {
 446          if (In.IsLimit() == false)
 447             continue;
 448
 449          In.Limit(-1);
 450          return !_error->PendingError();
 451       }
 452       while (Go(true, File) == true);
 453    }
 454
 455    return Owner->Flush() && !_error->PendingError();
 456 }
 457                                                                         /*}}}*/
 458 bool HttpServerState::ReadHeaderLines(std::string &Data)                /*{{{*/
 459 {
 460    return In.WriteTillEl(Data);
 461 }
 462                                                                         /*}}}*/
 463 bool HttpServerState::LoadNextResponse(bool const ToFile, FileFd * const File)/*{{{*/
 464 {
 465    return Go(ToFile, File);
 466 }
 467                                                                         /*}}}*/
 468 bool HttpServerState::WriteResponse(const std::string &Data)            /*{{{*/
 469 {
 470    return Out.Read(Data);
 471 }
 472                                                                         /*}}}*/
 473 APT_PURE bool HttpServerState::IsOpen()                                 /*{{{*/
 474 {
 475    return (ServerFd != -1);
 476 }
 477                                                                         /*}}}*/
 478 bool HttpServerState::InitHashes(FileFd &File)                          /*{{{*/
 479 {
 480    delete In.Hash;
 481    In.Hash = new Hashes;
 482
 483    // Set the expected size and read file for the hashes
 484    File.Truncate(StartPos);
 485    return In.Hash->AddFD(File, StartPos);
 486 }
 487                                                                         /*}}}*/
 488 APT_PURE Hashes * HttpServerState::GetHashes()                          /*{{{*/
 489 {
 490    return In.Hash;
 491 }
 492                                                                         /*}}}*/
 493 // HttpServerState::Die - The server has closed the connection.         /*{{{*/
 494 bool HttpServerState::Die(FileFd &File)
 495 {
 496    unsigned int LErrno = errno;
 497
 498    // Dump the buffer to the file
 499    if (State == ServerState::Data)
 500    {
 501       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 502       // can't be set
 503       if (File.Name() != "/dev/null")
 504          SetNonBlock(File.Fd(),false);
 505       while (In.WriteSpace() == true)
 506       {
 507          if (In.Write(File.Fd()) == false)
 508             return _error->Errno("write",_("Error writing to the file"));
 509
 510          // Done
 511          if (In.IsLimit() == true)
 512             return true;
 513       }
 514    }
 515
 516    // See if this is because the server finished the data stream
 517    if (In.IsLimit() == false && State != HttpServerState::Header &&
 518        Encoding != HttpServerState::Closes)
 519    {
 520       Close();
 521       if (LErrno == 0)
 522          return _error->Error(_("Error reading from server. Remote end closed connection"));
 523       errno = LErrno;
 524       return _error->Errno("read",_("Error reading from server"));
 525    }
 526    else
 527    {
 528       In.Limit(-1);
 529
 530       // Nothing left in the buffer
 531       if (In.WriteSpace() == false)
 532          return false;
 533
 534       // We may have got multiple responses back in one packet..
 535       Close();
 536       return true;
 537    }
 538
 539    return false;
 540 }
 541                                                                         /*}}}*/
 542 // HttpServerState::Flush - Dump the buffer into the file               /*{{{*/
 543 // ---------------------------------------------------------------------
 544 /* This takes the current input buffer from the Server FD and writes it
 545    into the file */
 546 bool HttpServerState::Flush(FileFd * const File)
 547 {
 548    if (File != NULL)
 549    {
 550       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 551       // can't be set
 552       if (File->Name() != "/dev/null")
 553          SetNonBlock(File->Fd(),false);
 554       if (In.WriteSpace() == false)
 555          return true;
 556
 557       while (In.WriteSpace() == true)
 558       {
 559          if (In.Write(File->Fd()) == false)
 560             return _error->Errno("write",_("Error writing to file"));
 561          if (In.IsLimit() == true)
 562             return true;
 563       }
 564
 565       if (In.IsLimit() == true || Encoding == ServerState::Closes)
 566          return true;
 567    }
 568    return false;
 569 }
 570                                                                         /*}}}*/
 571 // HttpServerState::Go - Run a single loop                              /*{{{*/
 572 // ---------------------------------------------------------------------
 573 /* This runs the select loop over the server FDs, Output file FDs and
 574    stdin. */
 575 bool HttpServerState::Go(bool ToFile, FileFd * const File)
 576 {
 577    // Server has closed the connection
 578    if (ServerFd == -1 && (In.WriteSpace() == false ||
 579                                ToFile == false))
 580       return false;
 581
 582    fd_set rfds,wfds;
 583    FD_ZERO(&rfds);
 584    FD_ZERO(&wfds);
 585
 586    /* Add the server. We only send more requests if the connection will
 587       be persisting */
 588    if (Out.WriteSpace() == true && ServerFd != -1
 589        && Persistent == true)
 590       FD_SET(ServerFd,&wfds);
 591    if (In.ReadSpace() == true && ServerFd != -1)
 592       FD_SET(ServerFd,&rfds);
 593
 594    // Add the file
 595    int FileFD = -1;
 596    if (File != NULL)
 597       FileFD = File->Fd();
 598
 599    if (In.WriteSpace() == true && ToFile == true && FileFD != -1)
 600       FD_SET(FileFD,&wfds);
 601
 602    // Add stdin
 603    if (_config->FindB("Acquire::http::DependOnSTDIN", true) == true)
 604       FD_SET(STDIN_FILENO,&rfds);
 605
 606    // Figure out the max fd
 607    int MaxFd = FileFD;
 608    if (MaxFd < ServerFd)
 609       MaxFd = ServerFd;
 610
 611    // Select
 612    struct timeval tv;
 613    tv.tv_sec = TimeOut;
 614    tv.tv_usec = 0;
 615    int Res = 0;
 616    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 617    {
 618       if (errno == EINTR)
 619          return true;
 620       return _error->Errno("select",_("Select failed"));
 621    }
 622
 623    if (Res == 0)
 624    {
 625       _error->Error(_("Connection timed out"));
 626       return Die(*File);
 627    }
 628
 629    // Handle server IO
 630    if (ServerFd != -1 && FD_ISSET(ServerFd,&rfds))
 631    {
 632       errno = 0;
 633       if (In.Read(ServerFd) == false)
 634          return Die(*File);
 635    }
 636
 637    if (ServerFd != -1 && FD_ISSET(ServerFd,&wfds))
 638    {
 639       errno = 0;
 640       if (Out.Write(ServerFd) == false)
 641          return Die(*File);
 642    }
 643
 644    // Send data to the file
 645    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 646    {
 647       if (In.Write(FileFD) == false)
 648          return _error->Errno("write",_("Error writing to output file"));
 649    }
 650
 651    // Handle commands from APT
 652    if (FD_ISSET(STDIN_FILENO,&rfds))
 653    {
 654       if (Owner->Run(true) != -1)
 655          exit(100);
 656    }
 657
 658    return true;
 659 }
 660                                                                         /*}}}*/
 661
 662 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 663 // ---------------------------------------------------------------------
 664 /* This places the http request in the outbound buffer */
 665 void HttpMethod::SendReq(FetchItem *Itm)
 666 {
 667    URI Uri = Itm->Uri;
 668
 669    // The HTTP server expects a hostname with a trailing :port
 670    char Buf[1000];
 671    string ProperHost;
 672
 673    if (Uri.Host.find(':') != string::npos)
 674       ProperHost = '[' + Uri.Host + ']';
 675    else
 676       ProperHost = Uri.Host;
 677    if (Uri.Port != 0)
 678    {
 679       sprintf(Buf,":%u",Uri.Port);
 680       ProperHost += Buf;
 681    }
 682
 683    // Just in case.
 684    if (Itm->Uri.length() >= sizeof(Buf))
 685        abort();
 686
 687    /* RFC 2616 §5.1.2 requires absolute URIs for requests to proxies,
 688       but while its a must for all servers to accept absolute URIs,
 689       it is assumed clients will sent an absolute path for non-proxies */
 690    std::string requesturi;
 691    if (Server->Proxy.empty() == true || Server->Proxy.Host.empty())
 692       requesturi = Uri.Path;
 693    else
 694       requesturi = Itm->Uri;
 695
 696    // The "+" is encoded as a workaround for a amazon S3 bug
 697    // see LP bugs #1003633 and #1086997.
 698    requesturi = QuoteString(requesturi, "+~ ");
 699
 700    /* Build the request. No keep-alive is included as it is the default
 701       in 1.1, can cause problems with proxies, and we are an HTTP/1.1
 702       client anyway.
 703       C.f. https://tools.ietf.org/wg/httpbis/trac/ticket/158 */
 704    sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 705            requesturi.c_str(),ProperHost.c_str());
 706
 707    // generate a cache control header (if needed)
 708    if (_config->FindB("Acquire::http::No-Cache",false) == true)
 709    {
 710       strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 711    }
 712    else
 713    {
 714       if (Itm->IndexFile == true)
 715       {
 716          sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 717                  _config->FindI("Acquire::http::Max-Age",0));
 718       }
 719       else
 720       {
 721          if (_config->FindB("Acquire::http::No-Store",false) == true)
 722             strcat(Buf,"Cache-Control: no-store\r\n");
 723       }
 724    }
 725
 726    // If we ask for uncompressed files servers might respond with content-
 727    // negotiation which lets us end up with compressed files we do not support,
 728    // see 657029, 657560 and co, so if we have no extension on the request
 729    // ask for text only. As a sidenote: If there is nothing to negotate servers
 730    // seem to be nice and ignore it.
 731    if (_config->FindB("Acquire::http::SendAccept", true) == true)
 732    {
 733       size_t const filepos = Itm->Uri.find_last_of('/');
 734       string const file = Itm->Uri.substr(filepos + 1);
 735       if (flExtension(file) == file)
 736          strcat(Buf,"Accept: text/*\r\n");
 737    }
 738
 739    string Req = Buf;
 740
 741    // Check for a partial file
 742    struct stat SBuf;
 743    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 744    {
 745       // In this case we send an if-range query with a range header
 746       sprintf(Buf,"Range: bytes=%lli-\r\nIf-Range: %s\r\n",(long long)SBuf.st_size,
 747               TimeRFC1123(SBuf.st_mtime).c_str());
 748       Req += Buf;
 749    }
 750    else
 751    {
 752       if (Itm->LastModified != 0)
 753       {
 754          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 755          Req += Buf;
 756       }
 757    }
 758
 759    if (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false)
 760       Req += string("Proxy-Authorization: Basic ") +
 761           Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) + "\r\n";
 762
 763    maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
 764    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 765    {
 766       Req += string("Authorization: Basic ") +
 767           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 768    }
 769    Req += "User-Agent: " + _config->Find("Acquire::http::User-Agent",
 770                 "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")") + "\r\n\r\n";
 771
 772    if (Debug == true)
 773       cerr << Req << endl;
 774
 775    Server->WriteResponse(Req);
 776 }
 777                                                                         /*}}}*/
 778 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 779 // ---------------------------------------------------------------------
 780 /* We stash the desired pipeline depth */
 781 bool HttpMethod::Configuration(string Message)
 782 {
 783    if (ServerMethod::Configuration(Message) == false)
 784       return false;
 785
 786    AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true);
 787    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 788                                   PipelineDepth);
 789    Debug = _config->FindB("Debug::Acquire::http",false);
 790
 791    // Get the proxy to use
 792    AutoDetectProxy();
 793
 794    return true;
 795 }
 796                                                                         /*}}}*/
 797 // HttpMethod::AutoDetectProxy - auto detect proxy                      /*{{{*/
 798 // ---------------------------------------------------------------------
 799 /* */
 800 bool HttpMethod::AutoDetectProxy()
 801 {
 802    // option is "Acquire::http::Proxy-Auto-Detect" but we allow the old
 803    // name without the dash ("-")
 804    AutoDetectProxyCmd = _config->Find("Acquire::http::Proxy-Auto-Detect",
 805                                       _config->Find("Acquire::http::ProxyAutoDetect"));
 806
 807    if (AutoDetectProxyCmd.empty())
 808       return true;
 809
 810    if (Debug)
 811       clog << "Using auto proxy detect command: " << AutoDetectProxyCmd << endl;
 812
 813    int Pipes[2] = {-1,-1};
 814    if (pipe(Pipes) != 0)
 815       return _error->Errno("pipe", "Failed to create Pipe");
 816
 817    pid_t Process = ExecFork();
 818    if (Process == 0)
 819    {
 820       close(Pipes[0]);
 821       dup2(Pipes[1],STDOUT_FILENO);
 822       SetCloseExec(STDOUT_FILENO,false);
 823
 824       const char *Args[2];
 825       Args[0] = AutoDetectProxyCmd.c_str();
 826       Args[1] = 0;
 827       execv(Args[0],(char **)Args);
 828       cerr << "Failed to exec method " << Args[0] << endl;
 829       _exit(100);
 830    }
 831    char buf[512];
 832    int InFd = Pipes[0];
 833    close(Pipes[1]);
 834    int res = read(InFd, buf, sizeof(buf)-1);
 835    ExecWait(Process, "ProxyAutoDetect", true);
 836
 837    if (res < 0)
 838       return _error->Errno("read", "Failed to read");
 839    if (res == 0)
 840       return _error->Warning("ProxyAutoDetect returned no data");
 841
 842    // add trailing \0
 843    buf[res] = 0;
 844
 845    if (Debug)
 846       clog << "auto detect command returned: '" << buf << "'" << endl;
 847
 848    if (strstr(buf, "http://") == buf)
 849       _config->Set("Acquire::http::proxy", _strstrip(buf));
 850
 851    return true;
 852 }
 853                                                                         /*}}}*/
 854 ServerState * HttpMethod::CreateServerState(URI uri)                    /*{{{*/
 855 {
 856    return new HttpServerState(uri, this);
 857 }
 858                                                                         /*}}}*/
 859 void HttpMethod::RotateDNS()                                            /*{{{*/
 860 {
 861    ::RotateDNS();
 862 }
 863                                                                         /*}}}*/