methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   4 /* ######################################################################
   5
   6    HTTP Acquire Method - This is the HTTP acquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <config.h>
  29
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/configuration.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/hashes.h>
  34 #include <apt-pkg/netrc.h>
  35 #include <apt-pkg/strutl.h>
  36 #include <apt-pkg/proxy.h>
  37
  38 #include <stddef.h>
  39 #include <stdlib.h>
  40 #include <sys/select.h>
  41 #include <cstring>
  42 #include <sys/stat.h>
  43 #include <sys/time.h>
  44 #include <unistd.h>
  45 #include <stdio.h>
  46 #include <errno.h>
  47 #include <arpa/inet.h>
  48 #include <iostream>
  49 #include <sstream>
  50
  51 #include "config.h"
  52 #include "connect.h"
  53 #include "http.h"
  54
  55 #include <apti18n.h>
  56                                                                         /*}}}*/
  57 using namespace std;
  58
  59 unsigned long long CircleBuf::BwReadLimit=0;
  60 unsigned long long CircleBuf::BwTickReadData=0;
  61 struct timeval CircleBuf::BwReadTick={0,0};
  62 const unsigned int CircleBuf::BW_HZ=10;
  63
  64 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* */
  67 CircleBuf::CircleBuf(HttpMethod const * const Owner, unsigned long long Size)
  68    : Size(Size), Hash(NULL), TotalWriten(0)
  69 {
  70    Buf = new unsigned char[Size];
  71    Reset();
  72
  73    CircleBuf::BwReadLimit = Owner->ConfigFindI("Dl-Limit", 0) * 1024;
  74 }
  75                                                                         /*}}}*/
  76 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  77 // ---------------------------------------------------------------------
  78 /* */
  79 void CircleBuf::Reset()
  80 {
  81    InP = 0;
  82    OutP = 0;
  83    StrPos = 0;
  84    TotalWriten = 0;
  85    MaxGet = (unsigned long long)-1;
  86    OutQueue = string();
  87    if (Hash != NULL)
  88    {
  89       delete Hash;
  90       Hash = NULL;
  91    }
  92 }
  93                                                                         /*}}}*/
  94 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  95 // ---------------------------------------------------------------------
  96 /* This fills up the buffer with as much data as is in the FD, assuming it
  97    is non-blocking.. */
  98 bool CircleBuf::Read(int Fd)
  99 {
 100    while (1)
 101    {
 102       // Woops, buffer is full
 103       if (InP - OutP == Size)
 104          return true;
 105
 106       // what's left to read in this tick
 107       unsigned long long const BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
 108
 109       if(CircleBuf::BwReadLimit) {
 110          struct timeval now;
 111          gettimeofday(&now,0);
 112
 113          unsigned long long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
 114             now.tv_usec-CircleBuf::BwReadTick.tv_usec;
 115          if(d > 1000000/BW_HZ) {
 116             CircleBuf::BwReadTick = now;
 117             CircleBuf::BwTickReadData = 0;
 118          }
 119
 120          if(CircleBuf::BwTickReadData >= BwReadMax) {
 121             usleep(1000000/BW_HZ);
 122             return true;
 123          }
 124       }
 125
 126       // Write the buffer segment
 127       ssize_t Res;
 128       if(CircleBuf::BwReadLimit) {
 129          Res = read(Fd,Buf + (InP%Size),
 130                     BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
 131       } else
 132          Res = read(Fd,Buf + (InP%Size),LeftRead());
 133
 134       if(Res > 0 && BwReadLimit > 0)
 135          CircleBuf::BwTickReadData += Res;
 136
 137       if (Res == 0)
 138          return false;
 139       if (Res < 0)
 140       {
 141          if (errno == EAGAIN)
 142             return true;
 143          return false;
 144       }
 145
 146       if (InP == 0)
 147          gettimeofday(&Start,0);
 148       InP += Res;
 149    }
 150 }
 151                                                                         /*}}}*/
 152 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 153 // ---------------------------------------------------------------------
 154 /* This will hold the string in and fill the buffer with it as it empties */
 155 bool CircleBuf::Read(string const &Data)
 156 {
 157    OutQueue.append(Data);
 158    FillOut();
 159    return true;
 160 }
 161                                                                         /*}}}*/
 162 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 163 // ---------------------------------------------------------------------
 164 /* */
 165 void CircleBuf::FillOut()
 166 {
 167    if (OutQueue.empty() == true)
 168       return;
 169    while (1)
 170    {
 171       // Woops, buffer is full
 172       if (InP - OutP == Size)
 173          return;
 174
 175       // Write the buffer segment
 176       unsigned long long Sz = LeftRead();
 177       if (OutQueue.length() - StrPos < Sz)
 178          Sz = OutQueue.length() - StrPos;
 179       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 180
 181       // Advance
 182       StrPos += Sz;
 183       InP += Sz;
 184       if (OutQueue.length() == StrPos)
 185       {
 186          StrPos = 0;
 187          OutQueue = "";
 188          return;
 189       }
 190    }
 191 }
 192                                                                         /*}}}*/
 193 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 194 // ---------------------------------------------------------------------
 195 /* This empties the buffer into the FD. */
 196 bool CircleBuf::Write(int Fd)
 197 {
 198    while (1)
 199    {
 200       FillOut();
 201
 202       // Woops, buffer is empty
 203       if (OutP == InP)
 204          return true;
 205
 206       if (OutP == MaxGet)
 207          return true;
 208
 209       // Write the buffer segment
 210       ssize_t Res;
 211       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 212
 213       if (Res == 0)
 214          return false;
 215       if (Res < 0)
 216       {
 217          if (errno == EAGAIN)
 218             return true;
 219
 220          return false;
 221       }
 222
 223       TotalWriten += Res;
 224
 225       if (Hash != NULL)
 226          Hash->Add(Buf + (OutP%Size),Res);
 227
 228       OutP += Res;
 229    }
 230 }
 231                                                                         /*}}}*/
 232 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 233 // ---------------------------------------------------------------------
 234 /* This copies till the first empty line */
 235 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 236 {
 237    // We cheat and assume it is unneeded to have more than one buffer load
 238    for (unsigned long long I = OutP; I < InP; I++)
 239    {
 240       if (Buf[I%Size] != '\n')
 241          continue;
 242       ++I;
 243
 244       if (Single == false)
 245       {
 246          if (I < InP  && Buf[I%Size] == '\r')
 247             ++I;
 248          if (I >= InP || Buf[I%Size] != '\n')
 249             continue;
 250          ++I;
 251       }
 252
 253       Data = "";
 254       while (OutP < I)
 255       {
 256          unsigned long long Sz = LeftWrite();
 257          if (Sz == 0)
 258             return false;
 259          if (I - OutP < Sz)
 260             Sz = I - OutP;
 261          Data += string((char *)(Buf + (OutP%Size)),Sz);
 262          OutP += Sz;
 263       }
 264       return true;
 265    }
 266    return false;
 267 }
 268                                                                         /*}}}*/
 269 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 270 // ---------------------------------------------------------------------
 271 /* */
 272 void CircleBuf::Stats()
 273 {
 274    if (InP == 0)
 275       return;
 276
 277    struct timeval Stop;
 278    gettimeofday(&Stop,0);
 279 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 280              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 281    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 282 }
 283                                                                         /*}}}*/
 284 CircleBuf::~CircleBuf()
 285 {
 286    delete [] Buf;
 287    delete Hash;
 288 }
 289
 290 // HttpServerState::HttpServerState - Constructor                       /*{{{*/
 291 HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(Owner, 64*1024), Out(Owner, 4*1024)
 292 {
 293    TimeOut = Owner->ConfigFindI("Timeout", TimeOut);
 294    Reset();
 295 }
 296                                                                         /*}}}*/
 297 // HttpServerState::Open - Open a connection to the server              /*{{{*/
 298 // ---------------------------------------------------------------------
 299 /* This opens a connection to the server. */
 300 static bool TalkToSocksProxy(int const ServerFd, std::string const &Proxy,
 301       char const * const type, bool const ReadWrite, uint8_t * const ToFrom,
 302       unsigned int const Size, unsigned int const Timeout)
 303 {
 304    if (WaitFd(ServerFd, ReadWrite, Timeout) == false)
 305       return _error->Error("Waiting for the SOCKS proxy %s to %s timed out", URI::SiteOnly(Proxy).c_str(), type);
 306    if (ReadWrite == false)
 307    {
 308       if (FileFd::Read(ServerFd, ToFrom, Size) == false)
 309          return _error->Error("Reading the %s from SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str());
 310    }
 311    else
 312    {
 313       if (FileFd::Write(ServerFd, ToFrom, Size) == false)
 314          return _error->Error("Writing the %s to SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str());
 315    }
 316    return true;
 317 }
 318 bool HttpServerState::Open()
 319 {
 320    // Use the already open connection if possible.
 321    if (ServerFd != -1)
 322       return true;
 323
 324    Close();
 325    In.Reset();
 326    Out.Reset();
 327    Persistent = true;
 328
 329    // Determine the proxy setting
 330    AutoDetectProxy(ServerName);
 331    string SpecificProxy = Owner->ConfigFind("Proxy::" + ServerName.Host, "");
 332    if (!SpecificProxy.empty())
 333    {
 334            if (SpecificProxy == "DIRECT")
 335                    Proxy = "";
 336            else
 337                    Proxy = SpecificProxy;
 338    }
 339    else
 340    {
 341            string DefProxy = Owner->ConfigFind("Proxy", "");
 342            if (!DefProxy.empty())
 343            {
 344                    Proxy = DefProxy;
 345            }
 346            else
 347            {
 348                    char* result = getenv("http_proxy");
 349                    Proxy = result ? result : "";
 350            }
 351    }
 352
 353    // Parse no_proxy, a , separated list of domains
 354    if (getenv("no_proxy") != 0)
 355    {
 356       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 357          Proxy = "";
 358    }
 359
 360    if (Proxy.Access == "socks5h")
 361    {
 362       if (Connect(Proxy.Host, Proxy.Port, "socks", 1080, ServerFd, TimeOut, Owner) == false)
 363          return false;
 364
 365       /* We implement a very basic SOCKS5 client here complying mostly to RFC1928 expect
 366        * for not offering GSSAPI auth which is a must (we only do no or user/pass auth).
 367        * We also expect the SOCKS5 server to do hostname lookup (aka socks5h) */
 368       std::string const ProxyInfo = URI::SiteOnly(Proxy);
 369       Owner->Status(_("Connecting to %s (%s)"),"SOCKS5h proxy",ProxyInfo.c_str());
 370       auto const Timeout = Owner->ConfigFindI("TimeOut", 120);
 371       #define APT_WriteOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) return false
 372       #define APT_ReadOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) return false
 373       if (ServerName.Host.length() > 255)
 374          return _error->Error("Can't use SOCKS5h as hostname %s is too long!", ServerName.Host.c_str());
 375       if (Proxy.User.length() > 255 || Proxy.Password.length() > 255)
 376          return _error->Error("Can't use user&pass auth as they are too long (%lu and %lu) for the SOCKS5!", Proxy.User.length(), Proxy.Password.length());
 377       if (Proxy.User.empty())
 378       {
 379          uint8_t greeting[] = { 0x05, 0x01, 0x00 };
 380          APT_WriteOrFail("greet-1", greeting, sizeof(greeting));
 381       }
 382       else
 383       {
 384          uint8_t greeting[] = { 0x05, 0x02, 0x00, 0x02 };
 385          APT_WriteOrFail("greet-2", greeting, sizeof(greeting));
 386       }
 387       uint8_t greeting[2];
 388       APT_ReadOrFail("greet back", greeting, sizeof(greeting));
 389       if (greeting[0] != 0x05)
 390          return _error->Error("SOCKS proxy %s greets back with wrong version: %d", ProxyInfo.c_str(), greeting[0]);
 391       if (greeting[1] == 0x00)
 392          ; // no auth has no method-dependent sub-negotiations
 393       else if (greeting[1] == 0x02)
 394       {
 395          if (Proxy.User.empty())
 396             return _error->Error("SOCKS proxy %s negotiated user&pass auth, but we had not offered it!", ProxyInfo.c_str());
 397          // user&pass auth sub-negotiations are defined by RFC1929
 398          std::vector<uint8_t> auth = {{ 0x01, static_cast<uint8_t>(Proxy.User.length()) }};
 399          std::copy(Proxy.User.begin(), Proxy.User.end(), std::back_inserter(auth));
 400          auth.push_back(static_cast<uint8_t>(Proxy.Password.length()));
 401          std::copy(Proxy.Password.begin(), Proxy.Password.end(), std::back_inserter(auth));
 402          APT_WriteOrFail("user&pass auth", auth.data(), auth.size());
 403          uint8_t authstatus[2];
 404          APT_ReadOrFail("auth report", authstatus, sizeof(authstatus));
 405          if (authstatus[0] != 0x01)
 406             return _error->Error("SOCKS proxy %s auth status response with wrong version: %d", ProxyInfo.c_str(), authstatus[0]);
 407          if (authstatus[1] != 0x00)
 408             return _error->Error("SOCKS proxy %s reported authorization failure: username or password incorrect? (%d)", ProxyInfo.c_str(), authstatus[1]);
 409       }
 410       else
 411          return _error->Error("SOCKS proxy %s greets back having not found a common authorization method: %d", ProxyInfo.c_str(), greeting[1]);
 412       union { uint16_t * i; uint8_t * b; } portu;
 413       uint16_t port = htons(static_cast<uint16_t>(ServerName.Port == 0 ? 80 : ServerName.Port));
 414       portu.i = &port;
 415       std::vector<uint8_t> request = {{ 0x05, 0x01, 0x00, 0x03, static_cast<uint8_t>(ServerName.Host.length()) }};
 416       std::copy(ServerName.Host.begin(), ServerName.Host.end(), std::back_inserter(request));
 417       request.push_back(portu.b[0]);
 418       request.push_back(portu.b[1]);
 419       APT_WriteOrFail("request", request.data(), request.size());
 420       uint8_t response[4];
 421       APT_ReadOrFail("first part of response", response, sizeof(response));
 422       if (response[0] != 0x05)
 423          return _error->Error("SOCKS proxy %s response with wrong version: %d", ProxyInfo.c_str(), response[0]);
 424       if (response[2] != 0x00)
 425          return _error->Error("SOCKS proxy %s has unexpected non-zero reserved field value: %d", ProxyInfo.c_str(), response[2]);
 426       std::string bindaddr;
 427       if (response[3] == 0x01) // IPv4 address
 428       {
 429          uint8_t ip4port[6];
 430          APT_ReadOrFail("IPv4+Port of response", ip4port, sizeof(ip4port));
 431          portu.b[0] = ip4port[4];
 432          portu.b[1] = ip4port[5];
 433          port = ntohs(*portu.i);
 434          strprintf(bindaddr, "%d.%d.%d.%d:%d", ip4port[0], ip4port[1], ip4port[2], ip4port[3], port);
 435       }
 436       else if (response[3] == 0x03) // hostname
 437       {
 438          uint8_t namelength;
 439          APT_ReadOrFail("hostname length of response", &namelength, 1);
 440          uint8_t hostname[namelength + 2];
 441          APT_ReadOrFail("hostname of response", hostname, sizeof(hostname));
 442          portu.b[0] = hostname[namelength];
 443          portu.b[1] = hostname[namelength + 1];
 444          port = ntohs(*portu.i);
 445          hostname[namelength] = '\0';
 446          strprintf(bindaddr, "%s:%d", hostname, port);
 447       }
 448       else if (response[3] == 0x04) // IPv6 address
 449       {
 450          uint8_t ip6port[18];
 451          APT_ReadOrFail("IPv6+port of response", ip6port, sizeof(ip6port));
 452          portu.b[0] = ip6port[16];
 453          portu.b[1] = ip6port[17];
 454          port = ntohs(*portu.i);
 455          strprintf(bindaddr, "[%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X]:%d",
 456                ip6port[0], ip6port[1], ip6port[2], ip6port[3], ip6port[4], ip6port[5], ip6port[6], ip6port[7],
 457                ip6port[8], ip6port[9], ip6port[10], ip6port[11], ip6port[12], ip6port[13], ip6port[14], ip6port[15],
 458                port);
 459       }
 460       else
 461          return _error->Error("SOCKS proxy %s destination address is of unknown type: %d",
 462                ProxyInfo.c_str(), response[3]);
 463       if (response[1] != 0x00)
 464       {
 465          char const * errstr;
 466          switch (response[1])
 467          {
 468              case 0x01: errstr = "general SOCKS server failure"; Owner->SetFailReason("SOCKS"); break;
 469              case 0x02: errstr = "connection not allowed by ruleset"; Owner->SetFailReason("SOCKS"); break;
 470              case 0x03: errstr = "Network unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break;
 471              case 0x04: errstr = "Host unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break;
 472              case 0x05: errstr = "Connection refused"; Owner->SetFailReason("ConnectionRefused"); break;
 473              case 0x06: errstr = "TTL expired"; Owner->SetFailReason("Timeout"); break;
 474              case 0x07: errstr = "Command not supported"; Owner->SetFailReason("SOCKS"); break;
 475              case 0x08: errstr = "Address type not supported"; Owner->SetFailReason("SOCKS"); break;
 476              default: errstr = "Unknown error"; Owner->SetFailReason("SOCKS"); break;
 477          }
 478          return _error->Error("SOCKS proxy %s didn't grant the connect to %s due to: %s (%d)", ProxyInfo.c_str(), bindaddr.c_str(), errstr, response[1]);
 479       }
 480       else if (Owner->DebugEnabled())
 481          ioprintf(std::clog, "http: SOCKS proxy %s connection established to %s\n", ProxyInfo.c_str(), bindaddr.c_str());
 482
 483       if (WaitFd(ServerFd, true, Timeout) == false)
 484          return _error->Error("SOCKS proxy %s reported connection, but timed out", ProxyInfo.c_str());
 485       #undef APT_ReadOrFail
 486       #undef APT_WriteOrFail
 487    }
 488    else
 489    {
 490       // Determine what host and port to use based on the proxy settings
 491       int Port = 0;
 492       string Host;
 493       if (Proxy.empty() == true || Proxy.Host.empty() == true)
 494       {
 495          if (ServerName.Port != 0)
 496             Port = ServerName.Port;
 497          Host = ServerName.Host;
 498       }
 499       else if (Proxy.Access != "http")
 500          return _error->Error("Unsupported proxy configured: %s", URI::SiteOnly(Proxy).c_str());
 501       else
 502       {
 503          if (Proxy.Port != 0)
 504             Port = Proxy.Port;
 505          Host = Proxy.Host;
 506       }
 507       return Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner);
 508    }
 509    return true;
 510 }
 511                                                                         /*}}}*/
 512 // HttpServerState::Close - Close a connection to the server            /*{{{*/
 513 // ---------------------------------------------------------------------
 514 /* */
 515 bool HttpServerState::Close()
 516 {
 517    close(ServerFd);
 518    ServerFd = -1;
 519    return true;
 520 }
 521                                                                         /*}}}*/
 522 // HttpServerState::RunData - Transfer the data from the socket         /*{{{*/
 523 bool HttpServerState::RunData(FileFd * const File)
 524 {
 525    State = Data;
 526
 527    // Chunked transfer encoding is fun..
 528    if (Encoding == Chunked)
 529    {
 530       while (1)
 531       {
 532          // Grab the block size
 533          bool Last = true;
 534          string Data;
 535          In.Limit(-1);
 536          do
 537          {
 538             if (In.WriteTillEl(Data,true) == true)
 539                break;
 540          }
 541          while ((Last = Go(false, File)) == true);
 542
 543          if (Last == false)
 544             return false;
 545
 546          // See if we are done
 547          unsigned long long Len = strtoull(Data.c_str(),0,16);
 548          if (Len == 0)
 549          {
 550             In.Limit(-1);
 551
 552             // We have to remove the entity trailer
 553             Last = true;
 554             do
 555             {
 556                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 557                   break;
 558             }
 559             while ((Last = Go(false, File)) == true);
 560             if (Last == false)
 561                return false;
 562             return !_error->PendingError();
 563          }
 564
 565          // Transfer the block
 566          In.Limit(Len);
 567          while (Go(true, File) == true)
 568             if (In.IsLimit() == true)
 569                break;
 570
 571          // Error
 572          if (In.IsLimit() == false)
 573             return false;
 574
 575          // The server sends an extra new line before the next block specifier..
 576          In.Limit(-1);
 577          Last = true;
 578          do
 579          {
 580             if (In.WriteTillEl(Data,true) == true)
 581                break;
 582          }
 583          while ((Last = Go(false, File)) == true);
 584          if (Last == false)
 585             return false;
 586       }
 587    }
 588    else
 589    {
 590       /* Closes encoding is used when the server did not specify a size, the
 591          loss of the connection means we are done */
 592       if (JunkSize != 0)
 593          In.Limit(JunkSize);
 594       else if (DownloadSize != 0)
 595          In.Limit(DownloadSize);
 596       else if (Persistent == false)
 597          In.Limit(-1);
 598
 599       // Just transfer the whole block.
 600       do
 601       {
 602          if (In.IsLimit() == false)
 603             continue;
 604
 605          In.Limit(-1);
 606          return !_error->PendingError();
 607       }
 608       while (Go(true, File) == true);
 609    }
 610
 611    return Owner->Flush() && !_error->PendingError();
 612 }
 613                                                                         /*}}}*/
 614 bool HttpServerState::RunDataToDevNull()                                /*{{{*/
 615 {
 616    FileFd DevNull("/dev/null", FileFd::WriteOnly);
 617    return RunData(&DevNull);
 618 }
 619                                                                         /*}}}*/
 620 bool HttpServerState::ReadHeaderLines(std::string &Data)                /*{{{*/
 621 {
 622    return In.WriteTillEl(Data);
 623 }
 624                                                                         /*}}}*/
 625 bool HttpServerState::LoadNextResponse(bool const ToFile, FileFd * const File)/*{{{*/
 626 {
 627    return Go(ToFile, File);
 628 }
 629                                                                         /*}}}*/
 630 bool HttpServerState::WriteResponse(const std::string &Data)            /*{{{*/
 631 {
 632    return Out.Read(Data);
 633 }
 634                                                                         /*}}}*/
 635 APT_PURE bool HttpServerState::IsOpen()                                 /*{{{*/
 636 {
 637    return (ServerFd != -1);
 638 }
 639                                                                         /*}}}*/
 640 bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes)  /*{{{*/
 641 {
 642    delete In.Hash;
 643    In.Hash = new Hashes(ExpectedHashes);
 644    return true;
 645 }
 646                                                                         /*}}}*/
 647
 648 APT_PURE Hashes * HttpServerState::GetHashes()                          /*{{{*/
 649 {
 650    return In.Hash;
 651 }
 652                                                                         /*}}}*/
 653 // HttpServerState::Die - The server has closed the connection.         /*{{{*/
 654 bool HttpServerState::Die(FileFd * const File)
 655 {
 656    unsigned int LErrno = errno;
 657
 658    // Dump the buffer to the file
 659    if (State == ServerState::Data)
 660    {
 661       if (File == nullptr)
 662          return true;
 663       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 664       // can't be set
 665       if (File->Name() != "/dev/null")
 666          SetNonBlock(File->Fd(),false);
 667       while (In.WriteSpace() == true)
 668       {
 669          if (In.Write(File->Fd()) == false)
 670             return _error->Errno("write",_("Error writing to the file"));
 671
 672          // Done
 673          if (In.IsLimit() == true)
 674             return true;
 675       }
 676    }
 677
 678    // See if this is because the server finished the data stream
 679    if (In.IsLimit() == false && State != HttpServerState::Header &&
 680        Persistent == true)
 681    {
 682       Close();
 683       if (LErrno == 0)
 684          return _error->Error(_("Error reading from server. Remote end closed connection"));
 685       errno = LErrno;
 686       return _error->Errno("read",_("Error reading from server"));
 687    }
 688    else
 689    {
 690       In.Limit(-1);
 691
 692       // Nothing left in the buffer
 693       if (In.WriteSpace() == false)
 694          return false;
 695
 696       // We may have got multiple responses back in one packet..
 697       Close();
 698       return true;
 699    }
 700
 701    return false;
 702 }
 703                                                                         /*}}}*/
 704 // HttpServerState::Flush - Dump the buffer into the file               /*{{{*/
 705 // ---------------------------------------------------------------------
 706 /* This takes the current input buffer from the Server FD and writes it
 707    into the file */
 708 bool HttpServerState::Flush(FileFd * const File)
 709 {
 710    if (File != NULL)
 711    {
 712       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 713       // can't be set
 714       if (File->Name() != "/dev/null")
 715          SetNonBlock(File->Fd(),false);
 716       if (In.WriteSpace() == false)
 717          return true;
 718
 719       while (In.WriteSpace() == true)
 720       {
 721          if (In.Write(File->Fd()) == false)
 722             return _error->Errno("write",_("Error writing to file"));
 723          if (In.IsLimit() == true)
 724             return true;
 725       }
 726
 727       if (In.IsLimit() == true || Persistent == false)
 728          return true;
 729    }
 730    return false;
 731 }
 732                                                                         /*}}}*/
 733 // HttpServerState::Go - Run a single loop                              /*{{{*/
 734 // ---------------------------------------------------------------------
 735 /* This runs the select loop over the server FDs, Output file FDs and
 736    stdin. */
 737 bool HttpServerState::Go(bool ToFile, FileFd * const File)
 738 {
 739    // Server has closed the connection
 740    if (ServerFd == -1 && (In.WriteSpace() == false ||
 741                                ToFile == false))
 742       return false;
 743
 744    fd_set rfds,wfds;
 745    FD_ZERO(&rfds);
 746    FD_ZERO(&wfds);
 747
 748    /* Add the server. We only send more requests if the connection will
 749       be persisting */
 750    if (Out.WriteSpace() == true && ServerFd != -1
 751        && Persistent == true)
 752       FD_SET(ServerFd,&wfds);
 753    if (In.ReadSpace() == true && ServerFd != -1)
 754       FD_SET(ServerFd,&rfds);
 755
 756    // Add the file
 757    int FileFD = -1;
 758    if (File != NULL)
 759       FileFD = File->Fd();
 760
 761    if (In.WriteSpace() == true && ToFile == true && FileFD != -1)
 762       FD_SET(FileFD,&wfds);
 763
 764    // Add stdin
 765    if (Owner->ConfigFindB("DependOnSTDIN", true) == true)
 766       FD_SET(STDIN_FILENO,&rfds);
 767
 768    // Figure out the max fd
 769    int MaxFd = FileFD;
 770    if (MaxFd < ServerFd)
 771       MaxFd = ServerFd;
 772
 773    // Select
 774    struct timeval tv;
 775    tv.tv_sec = TimeOut;
 776    tv.tv_usec = 0;
 777    int Res = 0;
 778    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 779    {
 780       if (errno == EINTR)
 781          return true;
 782       return _error->Errno("select",_("Select failed"));
 783    }
 784
 785    if (Res == 0)
 786    {
 787       _error->Error(_("Connection timed out"));
 788       return Die(File);
 789    }
 790
 791    // Handle server IO
 792    if (ServerFd != -1 && FD_ISSET(ServerFd,&rfds))
 793    {
 794       errno = 0;
 795       if (In.Read(ServerFd) == false)
 796          return Die(File);
 797    }
 798
 799    if (ServerFd != -1 && FD_ISSET(ServerFd,&wfds))
 800    {
 801       errno = 0;
 802       if (Out.Write(ServerFd) == false)
 803          return Die(File);
 804    }
 805
 806    // Send data to the file
 807    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 808    {
 809       if (In.Write(FileFD) == false)
 810          return _error->Errno("write",_("Error writing to output file"));
 811    }
 812
 813    if (MaximumSize > 0 && File && File->Tell() > MaximumSize)
 814    {
 815       Owner->SetFailReason("MaximumSizeExceeded");
 816       return _error->Error("Writing more data than expected (%llu > %llu)",
 817                            File->Tell(), MaximumSize);
 818    }
 819
 820    // Handle commands from APT
 821    if (FD_ISSET(STDIN_FILENO,&rfds))
 822    {
 823       if (Owner->Run(true) != -1)
 824          exit(100);
 825    }
 826
 827    return true;
 828 }
 829                                                                         /*}}}*/
 830
 831 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 832 // ---------------------------------------------------------------------
 833 /* This places the http request in the outbound buffer */
 834 void HttpMethod::SendReq(FetchItem *Itm)
 835 {
 836    URI Uri = Itm->Uri;
 837    {
 838       auto const plus = Binary.find('+');
 839       if (plus != std::string::npos)
 840          Uri.Access = Binary.substr(plus + 1);
 841    }
 842
 843    // The HTTP server expects a hostname with a trailing :port
 844    std::stringstream Req;
 845    string ProperHost;
 846
 847    if (Uri.Host.find(':') != string::npos)
 848       ProperHost = '[' + Uri.Host + ']';
 849    else
 850       ProperHost = Uri.Host;
 851
 852    /* RFC 2616 §5.1.2 requires absolute URIs for requests to proxies,
 853       but while its a must for all servers to accept absolute URIs,
 854       it is assumed clients will sent an absolute path for non-proxies */
 855    std::string requesturi;
 856    if (Server->Proxy.Access != "http" || Server->Proxy.empty() == true || Server->Proxy.Host.empty())
 857       requesturi = Uri.Path;
 858    else
 859       requesturi = Uri;
 860
 861    // The "+" is encoded as a workaround for a amazon S3 bug
 862    // see LP bugs #1003633 and #1086997.
 863    requesturi = QuoteString(requesturi, "+~ ");
 864
 865    /* Build the request. No keep-alive is included as it is the default
 866       in 1.1, can cause problems with proxies, and we are an HTTP/1.1
 867       client anyway.
 868       C.f. https://tools.ietf.org/wg/httpbis/trac/ticket/158 */
 869    Req << "GET " << requesturi << " HTTP/1.1\r\n";
 870    if (Uri.Port != 0)
 871       Req << "Host: " << ProperHost << ":" << std::to_string(Uri.Port) << "\r\n";
 872    else
 873       Req << "Host: " << ProperHost << "\r\n";
 874
 875    // generate a cache control header (if needed)
 876    if (ConfigFindB("No-Cache",false) == true)
 877       Req << "Cache-Control: no-cache\r\n"
 878          << "Pragma: no-cache\r\n";
 879    else if (Itm->IndexFile == true)
 880       Req << "Cache-Control: max-age=" << std::to_string(ConfigFindI("Max-Age", 0)) << "\r\n";
 881    else if (ConfigFindB("No-Store", false) == true)
 882       Req << "Cache-Control: no-store\r\n";
 883
 884    // If we ask for uncompressed files servers might respond with content-
 885    // negotiation which lets us end up with compressed files we do not support,
 886    // see 657029, 657560 and co, so if we have no extension on the request
 887    // ask for text only. As a sidenote: If there is nothing to negotate servers
 888    // seem to be nice and ignore it.
 889    if (ConfigFindB("SendAccept", true) == true)
 890    {
 891       size_t const filepos = Itm->Uri.find_last_of('/');
 892       string const file = Itm->Uri.substr(filepos + 1);
 893       if (flExtension(file) == file)
 894          Req << "Accept: text/*\r\n";
 895    }
 896
 897    // Check for a partial file and send if-queries accordingly
 898    struct stat SBuf;
 899    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 900       Req << "Range: bytes=" << std::to_string(SBuf.st_size) << "-\r\n"
 901          << "If-Range: " << TimeRFC1123(SBuf.st_mtime, false) << "\r\n";
 902    else if (Itm->LastModified != 0)
 903       Req << "If-Modified-Since: " << TimeRFC1123(Itm->LastModified, false).c_str() << "\r\n";
 904
 905    if (Server->Proxy.Access == "http" &&
 906          (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false))
 907       Req << "Proxy-Authorization: Basic "
 908          << Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) << "\r\n";
 909
 910    maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
 911    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 912       Req << "Authorization: Basic "
 913          << Base64Encode(Uri.User + ":" + Uri.Password) << "\r\n";
 914
 915    Req << "User-Agent: " << ConfigFind("User-Agent",
 916                 "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")") << "\r\n";
 917
 918    Req << "\r\n";
 919
 920    if (Debug == true)
 921       cerr << Req.str() << endl;
 922
 923    Server->WriteResponse(Req.str());
 924 }
 925                                                                         /*}}}*/
 926 std::unique_ptr<ServerState> HttpMethod::CreateServerState(URI const &uri)/*{{{*/
 927 {
 928    return std::unique_ptr<ServerState>(new HttpServerState(uri, this));
 929 }
 930                                                                         /*}}}*/
 931 void HttpMethod::RotateDNS()                                            /*{{{*/
 932 {
 933    ::RotateDNS();
 934 }
 935                                                                         /*}}}*/
 936 ServerMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &Res)/*{{{*/
 937 {
 938    auto ret = ServerMethod::DealWithHeaders(Res);
 939    if (ret != ServerMethod::FILE_IS_OPEN)
 940       return ret;
 941
 942    // Open the file
 943    delete File;
 944    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 945    if (_error->PendingError() == true)
 946       return ERROR_NOT_FROM_SERVER;
 947
 948    FailFile = Queue->DestFile;
 949    FailFile.c_str();   // Make sure we don't do a malloc in the signal handler
 950    FailFd = File->Fd();
 951    FailTime = Server->Date;
 952
 953    if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false)
 954    {
 955       _error->Errno("read",_("Problem hashing file"));
 956       return ERROR_NOT_FROM_SERVER;
 957    }
 958    if (Server->StartPos > 0)
 959       Res.ResumePoint = Server->StartPos;
 960
 961    SetNonBlock(File->Fd(),true);
 962    return FILE_IS_OPEN;
 963 }
 964                                                                         /*}}}*/
 965 HttpMethod::HttpMethod(std::string &&pProg) : ServerMethod(pProg.c_str(), "1.2", Pipeline | SendConfig)/*{{{*/
 966 {
 967    auto addName = std::inserter(methodNames, methodNames.begin());
 968    if (Binary != "http")
 969       addName = "http";
 970    auto const plus = Binary.find('+');
 971    if (plus != std::string::npos)
 972       addName = Binary.substr(0, plus);
 973    File = 0;
 974    Server = 0;
 975 }
 976                                                                         /*}}}*/