methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   4 /* ######################################################################
   5
   6    HTTP Acquire Method - This is the HTTP acquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <config.h>
  29
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/configuration.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/hashes.h>
  34 #include <apt-pkg/netrc.h>
  35 #include <apt-pkg/strutl.h>
  36 #include <apt-pkg/proxy.h>
  37
  38 #include <stddef.h>
  39 #include <stdlib.h>
  40 #include <sys/select.h>
  41 #include <cstring>
  42 #include <sys/stat.h>
  43 #include <sys/time.h>
  44 #include <unistd.h>
  45 #include <stdio.h>
  46 #include <errno.h>
  47 #include <arpa/inet.h>
  48 #include <iostream>
  49 #include <sstream>
  50
  51 #include "config.h"
  52 #include "connect.h"
  53 #include "http.h"
  54
  55 #include <apti18n.h>
  56                                                                         /*}}}*/
  57 using namespace std;
  58
  59 unsigned long long CircleBuf::BwReadLimit=0;
  60 unsigned long long CircleBuf::BwTickReadData=0;
  61 struct timeval CircleBuf::BwReadTick={0,0};
  62 const unsigned int CircleBuf::BW_HZ=10;
  63
  64 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* */
  67 CircleBuf::CircleBuf(HttpMethod const * const Owner, unsigned long long Size)
  68    : Size(Size), Hash(NULL), TotalWriten(0)
  69 {
  70    Buf = new unsigned char[Size];
  71    Reset();
  72
  73    CircleBuf::BwReadLimit = Owner->ConfigFindI("Dl-Limit", 0) * 1024;
  74 }
  75                                                                         /*}}}*/
  76 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  77 // ---------------------------------------------------------------------
  78 /* */
  79 void CircleBuf::Reset()
  80 {
  81    InP = 0;
  82    OutP = 0;
  83    StrPos = 0;
  84    TotalWriten = 0;
  85    MaxGet = (unsigned long long)-1;
  86    OutQueue = string();
  87    if (Hash != NULL)
  88    {
  89       delete Hash;
  90       Hash = NULL;
  91    }
  92 }
  93                                                                         /*}}}*/
  94 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  95 // ---------------------------------------------------------------------
  96 /* This fills up the buffer with as much data as is in the FD, assuming it
  97    is non-blocking.. */
  98 bool CircleBuf::Read(int Fd)
  99 {
 100    while (1)
 101    {
 102       // Woops, buffer is full
 103       if (InP - OutP == Size)
 104          return true;
 105
 106       // what's left to read in this tick
 107       unsigned long long const BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
 108
 109       if(CircleBuf::BwReadLimit) {
 110          struct timeval now;
 111          gettimeofday(&now,0);
 112
 113          unsigned long long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
 114             now.tv_usec-CircleBuf::BwReadTick.tv_usec;
 115          if(d > 1000000/BW_HZ) {
 116             CircleBuf::BwReadTick = now;
 117             CircleBuf::BwTickReadData = 0;
 118          }
 119
 120          if(CircleBuf::BwTickReadData >= BwReadMax) {
 121             usleep(1000000/BW_HZ);
 122             return true;
 123          }
 124       }
 125
 126       // Write the buffer segment
 127       ssize_t Res;
 128       if(CircleBuf::BwReadLimit) {
 129          Res = read(Fd,Buf + (InP%Size),
 130                     BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
 131       } else
 132          Res = read(Fd,Buf + (InP%Size),LeftRead());
 133
 134       if(Res > 0 && BwReadLimit > 0)
 135          CircleBuf::BwTickReadData += Res;
 136
 137       if (Res == 0)
 138          return false;
 139       if (Res < 0)
 140       {
 141          if (errno == EAGAIN)
 142             return true;
 143          return false;
 144       }
 145
 146       if (InP == 0)
 147          gettimeofday(&Start,0);
 148       InP += Res;
 149    }
 150 }
 151                                                                         /*}}}*/
 152 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 153 // ---------------------------------------------------------------------
 154 /* This will hold the string in and fill the buffer with it as it empties */
 155 bool CircleBuf::Read(string const &Data)
 156 {
 157    OutQueue.append(Data);
 158    FillOut();
 159    return true;
 160 }
 161                                                                         /*}}}*/
 162 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 163 // ---------------------------------------------------------------------
 164 /* */
 165 void CircleBuf::FillOut()
 166 {
 167    if (OutQueue.empty() == true)
 168       return;
 169    while (1)
 170    {
 171       // Woops, buffer is full
 172       if (InP - OutP == Size)
 173          return;
 174
 175       // Write the buffer segment
 176       unsigned long long Sz = LeftRead();
 177       if (OutQueue.length() - StrPos < Sz)
 178          Sz = OutQueue.length() - StrPos;
 179       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 180
 181       // Advance
 182       StrPos += Sz;
 183       InP += Sz;
 184       if (OutQueue.length() == StrPos)
 185       {
 186          StrPos = 0;
 187          OutQueue = "";
 188          return;
 189       }
 190    }
 191 }
 192                                                                         /*}}}*/
 193 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 194 // ---------------------------------------------------------------------
 195 /* This empties the buffer into the FD. */
 196 bool CircleBuf::Write(int Fd)
 197 {
 198    while (1)
 199    {
 200       FillOut();
 201
 202       // Woops, buffer is empty
 203       if (OutP == InP)
 204          return true;
 205
 206       if (OutP == MaxGet)
 207          return true;
 208
 209       // Write the buffer segment
 210       ssize_t Res;
 211       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 212
 213       if (Res == 0)
 214          return false;
 215       if (Res < 0)
 216       {
 217          if (errno == EAGAIN)
 218             return true;
 219
 220          return false;
 221       }
 222
 223       TotalWriten += Res;
 224
 225       if (Hash != NULL)
 226          Hash->Add(Buf + (OutP%Size),Res);
 227
 228       OutP += Res;
 229    }
 230 }
 231                                                                         /*}}}*/
 232 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 233 // ---------------------------------------------------------------------
 234 /* This copies till the first empty line */
 235 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 236 {
 237    // We cheat and assume it is unneeded to have more than one buffer load
 238    for (unsigned long long I = OutP; I < InP; I++)
 239    {
 240       if (Buf[I%Size] != '\n')
 241          continue;
 242       ++I;
 243
 244       if (Single == false)
 245       {
 246          if (I < InP  && Buf[I%Size] == '\r')
 247             ++I;
 248          if (I >= InP || Buf[I%Size] != '\n')
 249             continue;
 250          ++I;
 251       }
 252
 253       Data = "";
 254       while (OutP < I)
 255       {
 256          unsigned long long Sz = LeftWrite();
 257          if (Sz == 0)
 258             return false;
 259          if (I - OutP < Sz)
 260             Sz = I - OutP;
 261          Data += string((char *)(Buf + (OutP%Size)),Sz);
 262          OutP += Sz;
 263       }
 264       return true;
 265    }
 266    return false;
 267 }
 268                                                                         /*}}}*/
 269 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 270 // ---------------------------------------------------------------------
 271 /* */
 272 void CircleBuf::Stats()
 273 {
 274    if (InP == 0)
 275       return;
 276
 277    struct timeval Stop;
 278    gettimeofday(&Stop,0);
 279 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 280              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 281    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 282 }
 283                                                                         /*}}}*/
 284 CircleBuf::~CircleBuf()
 285 {
 286    delete [] Buf;
 287    delete Hash;
 288 }
 289
 290 // HttpServerState::HttpServerState - Constructor                       /*{{{*/
 291 HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(Owner, 64*1024), Out(Owner, 4*1024)
 292 {
 293    TimeOut = Owner->ConfigFindI("Timeout", TimeOut);
 294    Reset();
 295 }
 296                                                                         /*}}}*/
 297 // HttpServerState::Open - Open a connection to the server              /*{{{*/
 298 // ---------------------------------------------------------------------
 299 /* This opens a connection to the server. */
 300 static bool TalkToSocksProxy(int const ServerFd, std::string const &Proxy,
 301       char const * const type, bool const ReadWrite, uint8_t * const ToFrom,
 302       unsigned int const Size, unsigned int const Timeout)
 303 {
 304    if (WaitFd(ServerFd, ReadWrite, Timeout) == false)
 305       return _error->Error("Waiting for the SOCKS proxy %s to %s timed out", URI::SiteOnly(Proxy).c_str(), type);
 306    if (ReadWrite == false)
 307    {
 308       if (FileFd::Read(ServerFd, ToFrom, Size) == false)
 309          return _error->Error("Reading the %s from SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str());
 310    }
 311    else
 312    {
 313       if (FileFd::Write(ServerFd, ToFrom, Size) == false)
 314          return _error->Error("Writing the %s to SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str());
 315    }
 316    return true;
 317 }
 318 bool HttpServerState::Open()
 319 {
 320    // Use the already open connection if possible.
 321    if (ServerFd != -1)
 322       return true;
 323
 324    Close();
 325    In.Reset();
 326    Out.Reset();
 327    Persistent = true;
 328
 329    // Determine the proxy setting
 330    AutoDetectProxy(ServerName);
 331    string SpecificProxy = Owner->ConfigFind("Proxy::" + ServerName.Host, "");
 332    if (!SpecificProxy.empty())
 333    {
 334            if (SpecificProxy == "DIRECT")
 335                    Proxy = "";
 336            else
 337                    Proxy = SpecificProxy;
 338    }
 339    else
 340    {
 341            string DefProxy = Owner->ConfigFind("Proxy", "");
 342            if (!DefProxy.empty())
 343            {
 344                    Proxy = DefProxy;
 345            }
 346            else
 347            {
 348                    char* result = getenv("http_proxy");
 349                    Proxy = result ? result : "";
 350            }
 351    }
 352
 353    // Parse no_proxy, a , separated list of domains
 354    if (getenv("no_proxy") != 0)
 355    {
 356       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 357          Proxy = "";
 358    }
 359
 360    if (Proxy.empty() == false)
 361       Owner->AddProxyAuth(Proxy, ServerName);
 362
 363    if (Proxy.Access == "socks5h")
 364    {
 365       if (Connect(Proxy.Host, Proxy.Port, "socks", 1080, ServerFd, TimeOut, Owner) == false)
 366          return false;
 367
 368       /* We implement a very basic SOCKS5 client here complying mostly to RFC1928 expect
 369        * for not offering GSSAPI auth which is a must (we only do no or user/pass auth).
 370        * We also expect the SOCKS5 server to do hostname lookup (aka socks5h) */
 371       std::string const ProxyInfo = URI::SiteOnly(Proxy);
 372       Owner->Status(_("Connecting to %s (%s)"),"SOCKS5h proxy",ProxyInfo.c_str());
 373       auto const Timeout = Owner->ConfigFindI("TimeOut", 120);
 374       #define APT_WriteOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) return false
 375       #define APT_ReadOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) return false
 376       if (ServerName.Host.length() > 255)
 377          return _error->Error("Can't use SOCKS5h as hostname %s is too long!", ServerName.Host.c_str());
 378       if (Proxy.User.length() > 255 || Proxy.Password.length() > 255)
 379          return _error->Error("Can't use user&pass auth as they are too long (%lu and %lu) for the SOCKS5!", Proxy.User.length(), Proxy.Password.length());
 380       if (Proxy.User.empty())
 381       {
 382          uint8_t greeting[] = { 0x05, 0x01, 0x00 };
 383          APT_WriteOrFail("greet-1", greeting, sizeof(greeting));
 384       }
 385       else
 386       {
 387          uint8_t greeting[] = { 0x05, 0x02, 0x00, 0x02 };
 388          APT_WriteOrFail("greet-2", greeting, sizeof(greeting));
 389       }
 390       uint8_t greeting[2];
 391       APT_ReadOrFail("greet back", greeting, sizeof(greeting));
 392       if (greeting[0] != 0x05)
 393          return _error->Error("SOCKS proxy %s greets back with wrong version: %d", ProxyInfo.c_str(), greeting[0]);
 394       if (greeting[1] == 0x00)
 395          ; // no auth has no method-dependent sub-negotiations
 396       else if (greeting[1] == 0x02)
 397       {
 398          if (Proxy.User.empty())
 399             return _error->Error("SOCKS proxy %s negotiated user&pass auth, but we had not offered it!", ProxyInfo.c_str());
 400          // user&pass auth sub-negotiations are defined by RFC1929
 401          std::vector<uint8_t> auth = {{ 0x01, static_cast<uint8_t>(Proxy.User.length()) }};
 402          std::copy(Proxy.User.begin(), Proxy.User.end(), std::back_inserter(auth));
 403          auth.push_back(static_cast<uint8_t>(Proxy.Password.length()));
 404          std::copy(Proxy.Password.begin(), Proxy.Password.end(), std::back_inserter(auth));
 405          APT_WriteOrFail("user&pass auth", auth.data(), auth.size());
 406          uint8_t authstatus[2];
 407          APT_ReadOrFail("auth report", authstatus, sizeof(authstatus));
 408          if (authstatus[0] != 0x01)
 409             return _error->Error("SOCKS proxy %s auth status response with wrong version: %d", ProxyInfo.c_str(), authstatus[0]);
 410          if (authstatus[1] != 0x00)
 411             return _error->Error("SOCKS proxy %s reported authorization failure: username or password incorrect? (%d)", ProxyInfo.c_str(), authstatus[1]);
 412       }
 413       else
 414          return _error->Error("SOCKS proxy %s greets back having not found a common authorization method: %d", ProxyInfo.c_str(), greeting[1]);
 415       union { uint16_t * i; uint8_t * b; } portu;
 416       uint16_t port = htons(static_cast<uint16_t>(ServerName.Port == 0 ? 80 : ServerName.Port));
 417       portu.i = &port;
 418       std::vector<uint8_t> request = {{ 0x05, 0x01, 0x00, 0x03, static_cast<uint8_t>(ServerName.Host.length()) }};
 419       std::copy(ServerName.Host.begin(), ServerName.Host.end(), std::back_inserter(request));
 420       request.push_back(portu.b[0]);
 421       request.push_back(portu.b[1]);
 422       APT_WriteOrFail("request", request.data(), request.size());
 423       uint8_t response[4];
 424       APT_ReadOrFail("first part of response", response, sizeof(response));
 425       if (response[0] != 0x05)
 426          return _error->Error("SOCKS proxy %s response with wrong version: %d", ProxyInfo.c_str(), response[0]);
 427       if (response[2] != 0x00)
 428          return _error->Error("SOCKS proxy %s has unexpected non-zero reserved field value: %d", ProxyInfo.c_str(), response[2]);
 429       std::string bindaddr;
 430       if (response[3] == 0x01) // IPv4 address
 431       {
 432          uint8_t ip4port[6];
 433          APT_ReadOrFail("IPv4+Port of response", ip4port, sizeof(ip4port));
 434          portu.b[0] = ip4port[4];
 435          portu.b[1] = ip4port[5];
 436          port = ntohs(*portu.i);
 437          strprintf(bindaddr, "%d.%d.%d.%d:%d", ip4port[0], ip4port[1], ip4port[2], ip4port[3], port);
 438       }
 439       else if (response[3] == 0x03) // hostname
 440       {
 441          uint8_t namelength;
 442          APT_ReadOrFail("hostname length of response", &namelength, 1);
 443          uint8_t hostname[namelength + 2];
 444          APT_ReadOrFail("hostname of response", hostname, sizeof(hostname));
 445          portu.b[0] = hostname[namelength];
 446          portu.b[1] = hostname[namelength + 1];
 447          port = ntohs(*portu.i);
 448          hostname[namelength] = '\0';
 449          strprintf(bindaddr, "%s:%d", hostname, port);
 450       }
 451       else if (response[3] == 0x04) // IPv6 address
 452       {
 453          uint8_t ip6port[18];
 454          APT_ReadOrFail("IPv6+port of response", ip6port, sizeof(ip6port));
 455          portu.b[0] = ip6port[16];
 456          portu.b[1] = ip6port[17];
 457          port = ntohs(*portu.i);
 458          strprintf(bindaddr, "[%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X]:%d",
 459                ip6port[0], ip6port[1], ip6port[2], ip6port[3], ip6port[4], ip6port[5], ip6port[6], ip6port[7],
 460                ip6port[8], ip6port[9], ip6port[10], ip6port[11], ip6port[12], ip6port[13], ip6port[14], ip6port[15],
 461                port);
 462       }
 463       else
 464          return _error->Error("SOCKS proxy %s destination address is of unknown type: %d",
 465                ProxyInfo.c_str(), response[3]);
 466       if (response[1] != 0x00)
 467       {
 468          char const * errstr = nullptr;
 469          auto errcode = response[1];
 470          // Tor error reporting can be a bit arcane, lets try to detect & fix it up
 471          if (bindaddr == "0.0.0.0:0")
 472          {
 473             auto const lastdot = ServerName.Host.rfind('.');
 474             if (lastdot == std::string::npos || ServerName.Host.substr(lastdot) != ".onion")
 475                ;
 476             else if (errcode == 0x01)
 477             {
 478                auto const prevdot = ServerName.Host.rfind('.', lastdot - 1);
 479                if (lastdot == 16 && prevdot == std::string::npos)
 480                   ; // valid .onion address
 481                else if (prevdot != std::string::npos && (lastdot - prevdot) == 17)
 482                   ; // valid .onion address with subdomain(s)
 483                else
 484                {
 485                   errstr = "Invalid hostname: onion service name must be 16 characters long";
 486                   Owner->SetFailReason("SOCKS");
 487                }
 488             }
 489             // in all likelihood the service is either down or the address has
 490             // a typo and so "Host unreachable" is the better understood error
 491             // compared to the technically correct "TLL expired".
 492             else if (errcode == 0x06)
 493                errcode = 0x04;
 494          }
 495          if (errstr == nullptr)
 496          {
 497             switch (errcode)
 498             {
 499                case 0x01: errstr = "general SOCKS server failure"; Owner->SetFailReason("SOCKS"); break;
 500                case 0x02: errstr = "connection not allowed by ruleset"; Owner->SetFailReason("SOCKS"); break;
 501                case 0x03: errstr = "Network unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break;
 502                case 0x04: errstr = "Host unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break;
 503                case 0x05: errstr = "Connection refused"; Owner->SetFailReason("ConnectionRefused"); break;
 504                case 0x06: errstr = "TTL expired"; Owner->SetFailReason("Timeout"); break;
 505                case 0x07: errstr = "Command not supported"; Owner->SetFailReason("SOCKS"); break;
 506                case 0x08: errstr = "Address type not supported"; Owner->SetFailReason("SOCKS"); break;
 507                default: errstr = "Unknown error"; Owner->SetFailReason("SOCKS"); break;
 508             }
 509          }
 510          return _error->Error("SOCKS proxy %s could not connect to %s (%s) due to: %s (%d)",
 511                ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str(), errstr, response[1]);
 512       }
 513       else if (Owner->DebugEnabled())
 514          ioprintf(std::clog, "http: SOCKS proxy %s connection established to %s (%s)\n",
 515                ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str());
 516
 517       if (WaitFd(ServerFd, true, Timeout) == false)
 518          return _error->Error("SOCKS proxy %s reported connection to %s (%s), but timed out",
 519                ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str());
 520       #undef APT_ReadOrFail
 521       #undef APT_WriteOrFail
 522    }
 523    else
 524    {
 525       // Determine what host and port to use based on the proxy settings
 526       int Port = 0;
 527       string Host;
 528       if (Proxy.empty() == true || Proxy.Host.empty() == true)
 529       {
 530          if (ServerName.Port != 0)
 531             Port = ServerName.Port;
 532          Host = ServerName.Host;
 533       }
 534       else if (Proxy.Access != "http")
 535          return _error->Error("Unsupported proxy configured: %s", URI::SiteOnly(Proxy).c_str());
 536       else
 537       {
 538          if (Proxy.Port != 0)
 539             Port = Proxy.Port;
 540          Host = Proxy.Host;
 541       }
 542       return Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner);
 543    }
 544    return true;
 545 }
 546                                                                         /*}}}*/
 547 // HttpServerState::Close - Close a connection to the server            /*{{{*/
 548 // ---------------------------------------------------------------------
 549 /* */
 550 bool HttpServerState::Close()
 551 {
 552    close(ServerFd);
 553    ServerFd = -1;
 554    return true;
 555 }
 556                                                                         /*}}}*/
 557 // HttpServerState::RunData - Transfer the data from the socket         /*{{{*/
 558 bool HttpServerState::RunData(FileFd * const File)
 559 {
 560    State = Data;
 561
 562    // Chunked transfer encoding is fun..
 563    if (Encoding == Chunked)
 564    {
 565       while (1)
 566       {
 567          // Grab the block size
 568          bool Last = true;
 569          string Data;
 570          In.Limit(-1);
 571          do
 572          {
 573             if (In.WriteTillEl(Data,true) == true)
 574                break;
 575          }
 576          while ((Last = Go(false, File)) == true);
 577
 578          if (Last == false)
 579             return false;
 580
 581          // See if we are done
 582          unsigned long long Len = strtoull(Data.c_str(),0,16);
 583          if (Len == 0)
 584          {
 585             In.Limit(-1);
 586
 587             // We have to remove the entity trailer
 588             Last = true;
 589             do
 590             {
 591                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 592                   break;
 593             }
 594             while ((Last = Go(false, File)) == true);
 595             if (Last == false)
 596                return false;
 597             return !_error->PendingError();
 598          }
 599
 600          // Transfer the block
 601          In.Limit(Len);
 602          while (Go(true, File) == true)
 603             if (In.IsLimit() == true)
 604                break;
 605
 606          // Error
 607          if (In.IsLimit() == false)
 608             return false;
 609
 610          // The server sends an extra new line before the next block specifier..
 611          In.Limit(-1);
 612          Last = true;
 613          do
 614          {
 615             if (In.WriteTillEl(Data,true) == true)
 616                break;
 617          }
 618          while ((Last = Go(false, File)) == true);
 619          if (Last == false)
 620             return false;
 621       }
 622    }
 623    else
 624    {
 625       /* Closes encoding is used when the server did not specify a size, the
 626          loss of the connection means we are done */
 627       if (JunkSize != 0)
 628          In.Limit(JunkSize);
 629       else if (DownloadSize != 0)
 630          In.Limit(DownloadSize);
 631       else if (Persistent == false)
 632          In.Limit(-1);
 633
 634       // Just transfer the whole block.
 635       do
 636       {
 637          if (In.IsLimit() == false)
 638             continue;
 639
 640          In.Limit(-1);
 641          return !_error->PendingError();
 642       }
 643       while (Go(true, File) == true);
 644    }
 645
 646    return Owner->Flush() && !_error->PendingError();
 647 }
 648                                                                         /*}}}*/
 649 bool HttpServerState::RunDataToDevNull()                                /*{{{*/
 650 {
 651    // no need to clean up if we discard the connection anyhow
 652    if (Persistent == false)
 653       return true;
 654    FileFd DevNull("/dev/null", FileFd::WriteOnly);
 655    return RunData(&DevNull);
 656 }
 657                                                                         /*}}}*/
 658 bool HttpServerState::ReadHeaderLines(std::string &Data)                /*{{{*/
 659 {
 660    return In.WriteTillEl(Data);
 661 }
 662                                                                         /*}}}*/
 663 bool HttpServerState::LoadNextResponse(bool const ToFile, FileFd * const File)/*{{{*/
 664 {
 665    return Go(ToFile, File);
 666 }
 667                                                                         /*}}}*/
 668 bool HttpServerState::WriteResponse(const std::string &Data)            /*{{{*/
 669 {
 670    return Out.Read(Data);
 671 }
 672                                                                         /*}}}*/
 673 APT_PURE bool HttpServerState::IsOpen()                                 /*{{{*/
 674 {
 675    return (ServerFd != -1);
 676 }
 677                                                                         /*}}}*/
 678 bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes)  /*{{{*/
 679 {
 680    delete In.Hash;
 681    In.Hash = new Hashes(ExpectedHashes);
 682    return true;
 683 }
 684                                                                         /*}}}*/
 685 void HttpServerState::Reset(bool const Everything)                      /*{{{*/
 686 {
 687    ServerState::Reset(Everything);
 688    if (Everything)
 689       ServerFd = -1;
 690 }
 691                                                                         /*}}}*/
 692
 693 APT_PURE Hashes * HttpServerState::GetHashes()                          /*{{{*/
 694 {
 695    return In.Hash;
 696 }
 697                                                                         /*}}}*/
 698 // HttpServerState::Die - The server has closed the connection.         /*{{{*/
 699 bool HttpServerState::Die(FileFd * const File)
 700 {
 701    unsigned int LErrno = errno;
 702
 703    // Dump the buffer to the file
 704    if (State == ServerState::Data)
 705    {
 706       if (File == nullptr)
 707          return true;
 708       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 709       // can't be set
 710       if (File->Name() != "/dev/null")
 711          SetNonBlock(File->Fd(),false);
 712       while (In.WriteSpace() == true)
 713       {
 714          if (In.Write(File->Fd()) == false)
 715             return _error->Errno("write",_("Error writing to the file"));
 716
 717          // Done
 718          if (In.IsLimit() == true)
 719             return true;
 720       }
 721    }
 722
 723    // See if this is because the server finished the data stream
 724    if (In.IsLimit() == false && State != HttpServerState::Header &&
 725        Persistent == true)
 726    {
 727       Close();
 728       if (LErrno == 0)
 729          return _error->Error(_("Error reading from server. Remote end closed connection"));
 730       errno = LErrno;
 731       return _error->Errno("read",_("Error reading from server"));
 732    }
 733    else
 734    {
 735       In.Limit(-1);
 736
 737       // Nothing left in the buffer
 738       if (In.WriteSpace() == false)
 739          return false;
 740
 741       // We may have got multiple responses back in one packet..
 742       Close();
 743       return true;
 744    }
 745
 746    return false;
 747 }
 748                                                                         /*}}}*/
 749 // HttpServerState::Flush - Dump the buffer into the file               /*{{{*/
 750 // ---------------------------------------------------------------------
 751 /* This takes the current input buffer from the Server FD and writes it
 752    into the file */
 753 bool HttpServerState::Flush(FileFd * const File)
 754 {
 755    if (File != NULL)
 756    {
 757       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 758       // can't be set
 759       if (File->Name() != "/dev/null")
 760          SetNonBlock(File->Fd(),false);
 761       if (In.WriteSpace() == false)
 762          return true;
 763
 764       while (In.WriteSpace() == true)
 765       {
 766          if (In.Write(File->Fd()) == false)
 767             return _error->Errno("write",_("Error writing to file"));
 768          if (In.IsLimit() == true)
 769             return true;
 770       }
 771
 772       if (In.IsLimit() == true || Persistent == false)
 773          return true;
 774    }
 775    return false;
 776 }
 777                                                                         /*}}}*/
 778 // HttpServerState::Go - Run a single loop                              /*{{{*/
 779 // ---------------------------------------------------------------------
 780 /* This runs the select loop over the server FDs, Output file FDs and
 781    stdin. */
 782 bool HttpServerState::Go(bool ToFile, FileFd * const File)
 783 {
 784    // Server has closed the connection
 785    if (ServerFd == -1 && (In.WriteSpace() == false ||
 786                                ToFile == false))
 787       return false;
 788
 789    fd_set rfds,wfds;
 790    FD_ZERO(&rfds);
 791    FD_ZERO(&wfds);
 792
 793    /* Add the server. We only send more requests if the connection will
 794       be persisting */
 795    if (Out.WriteSpace() == true && ServerFd != -1
 796        && Persistent == true)
 797       FD_SET(ServerFd,&wfds);
 798    if (In.ReadSpace() == true && ServerFd != -1)
 799       FD_SET(ServerFd,&rfds);
 800
 801    // Add the file
 802    int FileFD = -1;
 803    if (File != NULL)
 804       FileFD = File->Fd();
 805
 806    if (In.WriteSpace() == true && ToFile == true && FileFD != -1)
 807       FD_SET(FileFD,&wfds);
 808
 809    // Add stdin
 810    if (Owner->ConfigFindB("DependOnSTDIN", true) == true)
 811       FD_SET(STDIN_FILENO,&rfds);
 812
 813    // Figure out the max fd
 814    int MaxFd = FileFD;
 815    if (MaxFd < ServerFd)
 816       MaxFd = ServerFd;
 817
 818    // Select
 819    struct timeval tv;
 820    tv.tv_sec = TimeOut;
 821    tv.tv_usec = 0;
 822    int Res = 0;
 823    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 824    {
 825       if (errno == EINTR)
 826          return true;
 827       return _error->Errno("select",_("Select failed"));
 828    }
 829
 830    if (Res == 0)
 831    {
 832       _error->Error(_("Connection timed out"));
 833       return Die(File);
 834    }
 835
 836    // Handle server IO
 837    if (ServerFd != -1 && FD_ISSET(ServerFd,&rfds))
 838    {
 839       errno = 0;
 840       if (In.Read(ServerFd) == false)
 841          return Die(File);
 842    }
 843
 844    if (ServerFd != -1 && FD_ISSET(ServerFd,&wfds))
 845    {
 846       errno = 0;
 847       if (Out.Write(ServerFd) == false)
 848          return Die(File);
 849    }
 850
 851    // Send data to the file
 852    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 853    {
 854       if (In.Write(FileFD) == false)
 855          return _error->Errno("write",_("Error writing to output file"));
 856    }
 857
 858    if (MaximumSize > 0 && File && File->Tell() > MaximumSize)
 859    {
 860       Owner->SetFailReason("MaximumSizeExceeded");
 861       return _error->Error("Writing more data than expected (%llu > %llu)",
 862                            File->Tell(), MaximumSize);
 863    }
 864
 865    // Handle commands from APT
 866    if (FD_ISSET(STDIN_FILENO,&rfds))
 867    {
 868       if (Owner->Run(true) != -1)
 869          exit(100);
 870    }
 871
 872    return true;
 873 }
 874                                                                         /*}}}*/
 875
 876 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 877 // ---------------------------------------------------------------------
 878 /* This places the http request in the outbound buffer */
 879 void HttpMethod::SendReq(FetchItem *Itm)
 880 {
 881    URI Uri = Itm->Uri;
 882    {
 883       auto const plus = Binary.find('+');
 884       if (plus != std::string::npos)
 885          Uri.Access = Binary.substr(plus + 1);
 886    }
 887
 888    // The HTTP server expects a hostname with a trailing :port
 889    std::stringstream Req;
 890    string ProperHost;
 891
 892    if (Uri.Host.find(':') != string::npos)
 893       ProperHost = '[' + Uri.Host + ']';
 894    else
 895       ProperHost = Uri.Host;
 896
 897    /* RFC 2616 §5.1.2 requires absolute URIs for requests to proxies,
 898       but while its a must for all servers to accept absolute URIs,
 899       it is assumed clients will sent an absolute path for non-proxies */
 900    std::string requesturi;
 901    if (Server->Proxy.Access != "http" || Server->Proxy.empty() == true || Server->Proxy.Host.empty())
 902       requesturi = Uri.Path;
 903    else
 904       requesturi = Uri;
 905
 906    // The "+" is encoded as a workaround for a amazon S3 bug
 907    // see LP bugs #1003633 and #1086997.
 908    requesturi = QuoteString(requesturi, "+~ ");
 909
 910    /* Build the request. No keep-alive is included as it is the default
 911       in 1.1, can cause problems with proxies, and we are an HTTP/1.1
 912       client anyway.
 913       C.f. https://tools.ietf.org/wg/httpbis/trac/ticket/158 */
 914    Req << "GET " << requesturi << " HTTP/1.1\r\n";
 915    if (Uri.Port != 0)
 916       Req << "Host: " << ProperHost << ":" << std::to_string(Uri.Port) << "\r\n";
 917    else
 918       Req << "Host: " << ProperHost << "\r\n";
 919
 920    // generate a cache control header (if needed)
 921    if (ConfigFindB("No-Cache",false) == true)
 922       Req << "Cache-Control: no-cache\r\n"
 923          << "Pragma: no-cache\r\n";
 924    else if (Itm->IndexFile == true)
 925       Req << "Cache-Control: max-age=" << std::to_string(ConfigFindI("Max-Age", 0)) << "\r\n";
 926    else if (ConfigFindB("No-Store", false) == true)
 927       Req << "Cache-Control: no-store\r\n";
 928
 929    // If we ask for uncompressed files servers might respond with content-
 930    // negotiation which lets us end up with compressed files we do not support,
 931    // see 657029, 657560 and co, so if we have no extension on the request
 932    // ask for text only. As a sidenote: If there is nothing to negotate servers
 933    // seem to be nice and ignore it.
 934    if (ConfigFindB("SendAccept", true) == true)
 935    {
 936       size_t const filepos = Itm->Uri.find_last_of('/');
 937       string const file = Itm->Uri.substr(filepos + 1);
 938       if (flExtension(file) == file)
 939          Req << "Accept: text/*\r\n";
 940    }
 941
 942    // Check for a partial file and send if-queries accordingly
 943    struct stat SBuf;
 944    if (Server->RangesAllowed && stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 945       Req << "Range: bytes=" << std::to_string(SBuf.st_size) << "-\r\n"
 946          << "If-Range: " << TimeRFC1123(SBuf.st_mtime, false) << "\r\n";
 947    else if (Itm->LastModified != 0)
 948       Req << "If-Modified-Since: " << TimeRFC1123(Itm->LastModified, false).c_str() << "\r\n";
 949
 950    if (Server->Proxy.Access == "http" &&
 951          (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false))
 952       Req << "Proxy-Authorization: Basic "
 953          << Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) << "\r\n";
 954
 955    maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
 956    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 957       Req << "Authorization: Basic "
 958          << Base64Encode(Uri.User + ":" + Uri.Password) << "\r\n";
 959
 960    Req << "User-Agent: " << ConfigFind("User-Agent",
 961                 "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")") << "\r\n";
 962
 963    Req << "\r\n";
 964
 965    if (Debug == true)
 966       cerr << Req.str() << endl;
 967
 968    Server->WriteResponse(Req.str());
 969 }
 970                                                                         /*}}}*/
 971 std::unique_ptr<ServerState> HttpMethod::CreateServerState(URI const &uri)/*{{{*/
 972 {
 973    return std::unique_ptr<ServerState>(new HttpServerState(uri, this));
 974 }
 975                                                                         /*}}}*/
 976 void HttpMethod::RotateDNS()                                            /*{{{*/
 977 {
 978    ::RotateDNS();
 979 }
 980                                                                         /*}}}*/
 981 ServerMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &Res)/*{{{*/
 982 {
 983    auto ret = ServerMethod::DealWithHeaders(Res);
 984    if (ret != ServerMethod::FILE_IS_OPEN)
 985       return ret;
 986
 987    // Open the file
 988    delete File;
 989    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 990    if (_error->PendingError() == true)
 991       return ERROR_NOT_FROM_SERVER;
 992
 993    FailFile = Queue->DestFile;
 994    FailFile.c_str();   // Make sure we don't do a malloc in the signal handler
 995    FailFd = File->Fd();
 996    FailTime = Server->Date;
 997
 998    if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false)
 999    {
1000       _error->Errno("read",_("Problem hashing file"));
1001       return ERROR_NOT_FROM_SERVER;
1002    }
1003    if (Server->StartPos > 0)
1004       Res.ResumePoint = Server->StartPos;
1005
1006    SetNonBlock(File->Fd(),true);
1007    return FILE_IS_OPEN;
1008 }
1009                                                                         /*}}}*/
1010 HttpMethod::HttpMethod(std::string &&pProg) : ServerMethod(pProg.c_str(), "1.2", Pipeline | SendConfig)/*{{{*/
1011 {
1012    auto addName = std::inserter(methodNames, methodNames.begin());
1013    if (Binary != "http")
1014       addName = "http";
1015    auto const plus = Binary.find('+');
1016    if (plus != std::string::npos)
1017       addName = Binary.substr(0, plus);
1018    File = 0;
1019    Server = 0;
1020 }
1021                                                                         /*}}}*/