methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   4 /* ######################################################################
   5
   6    HTTP Acquire Method - This is the HTTP acquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <config.h>
  29
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/configuration.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/hashes.h>
  34 #include <apt-pkg/netrc.h>
  35 #include <apt-pkg/strutl.h>
  36 #include <apt-pkg/proxy.h>
  37
  38 #include <stddef.h>
  39 #include <stdlib.h>
  40 #include <sys/select.h>
  41 #include <cstring>
  42 #include <sys/stat.h>
  43 #include <sys/time.h>
  44 #include <unistd.h>
  45 #include <stdio.h>
  46 #include <errno.h>
  47 #include <arpa/inet.h>
  48 #include <iostream>
  49 #include <sstream>
  50
  51 #include "config.h"
  52 #include "connect.h"
  53 #include "http.h"
  54
  55 #include <apti18n.h>
  56                                                                         /*}}}*/
  57 using namespace std;
  58
  59 unsigned long long CircleBuf::BwReadLimit=0;
  60 unsigned long long CircleBuf::BwTickReadData=0;
  61 struct timeval CircleBuf::BwReadTick={0,0};
  62 const unsigned int CircleBuf::BW_HZ=10;
  63
  64 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* */
  67 CircleBuf::CircleBuf(HttpMethod const * const Owner, unsigned long long Size)
  68    : Size(Size), Hash(NULL), TotalWriten(0)
  69 {
  70    Buf = new unsigned char[Size];
  71    Reset();
  72
  73    CircleBuf::BwReadLimit = Owner->ConfigFindI("Dl-Limit", 0) * 1024;
  74 }
  75                                                                         /*}}}*/
  76 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  77 // ---------------------------------------------------------------------
  78 /* */
  79 void CircleBuf::Reset()
  80 {
  81    InP = 0;
  82    OutP = 0;
  83    StrPos = 0;
  84    TotalWriten = 0;
  85    MaxGet = (unsigned long long)-1;
  86    OutQueue = string();
  87    if (Hash != NULL)
  88    {
  89       delete Hash;
  90       Hash = NULL;
  91    }
  92 }
  93                                                                         /*}}}*/
  94 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  95 // ---------------------------------------------------------------------
  96 /* This fills up the buffer with as much data as is in the FD, assuming it
  97    is non-blocking.. */
  98 bool CircleBuf::Read(int Fd)
  99 {
 100    while (1)
 101    {
 102       // Woops, buffer is full
 103       if (InP - OutP == Size)
 104          return true;
 105
 106       // what's left to read in this tick
 107       unsigned long long const BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
 108
 109       if(CircleBuf::BwReadLimit) {
 110          struct timeval now;
 111          gettimeofday(&now,0);
 112
 113          unsigned long long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
 114             now.tv_usec-CircleBuf::BwReadTick.tv_usec;
 115          if(d > 1000000/BW_HZ) {
 116             CircleBuf::BwReadTick = now;
 117             CircleBuf::BwTickReadData = 0;
 118          }
 119
 120          if(CircleBuf::BwTickReadData >= BwReadMax) {
 121             usleep(1000000/BW_HZ);
 122             return true;
 123          }
 124       }
 125
 126       // Write the buffer segment
 127       ssize_t Res;
 128       if(CircleBuf::BwReadLimit) {
 129          Res = read(Fd,Buf + (InP%Size),
 130                     BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
 131       } else
 132          Res = read(Fd,Buf + (InP%Size),LeftRead());
 133
 134       if(Res > 0 && BwReadLimit > 0)
 135          CircleBuf::BwTickReadData += Res;
 136
 137       if (Res == 0)
 138          return false;
 139       if (Res < 0)
 140       {
 141          if (errno == EAGAIN)
 142             return true;
 143          return false;
 144       }
 145
 146       if (InP == 0)
 147          gettimeofday(&Start,0);
 148       InP += Res;
 149    }
 150 }
 151                                                                         /*}}}*/
 152 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 153 // ---------------------------------------------------------------------
 154 /* This will hold the string in and fill the buffer with it as it empties */
 155 bool CircleBuf::Read(string const &Data)
 156 {
 157    OutQueue.append(Data);
 158    FillOut();
 159    return true;
 160 }
 161                                                                         /*}}}*/
 162 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 163 // ---------------------------------------------------------------------
 164 /* */
 165 void CircleBuf::FillOut()
 166 {
 167    if (OutQueue.empty() == true)
 168       return;
 169    while (1)
 170    {
 171       // Woops, buffer is full
 172       if (InP - OutP == Size)
 173          return;
 174
 175       // Write the buffer segment
 176       unsigned long long Sz = LeftRead();
 177       if (OutQueue.length() - StrPos < Sz)
 178          Sz = OutQueue.length() - StrPos;
 179       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 180
 181       // Advance
 182       StrPos += Sz;
 183       InP += Sz;
 184       if (OutQueue.length() == StrPos)
 185       {
 186          StrPos = 0;
 187          OutQueue = "";
 188          return;
 189       }
 190    }
 191 }
 192                                                                         /*}}}*/
 193 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 194 // ---------------------------------------------------------------------
 195 /* This empties the buffer into the FD. */
 196 bool CircleBuf::Write(int Fd)
 197 {
 198    while (1)
 199    {
 200       FillOut();
 201
 202       // Woops, buffer is empty
 203       if (OutP == InP)
 204          return true;
 205
 206       if (OutP == MaxGet)
 207          return true;
 208
 209       // Write the buffer segment
 210       ssize_t Res;
 211       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 212
 213       if (Res == 0)
 214          return false;
 215       if (Res < 0)
 216       {
 217          if (errno == EAGAIN)
 218             return true;
 219
 220          return false;
 221       }
 222
 223       TotalWriten += Res;
 224
 225       if (Hash != NULL)
 226          Hash->Add(Buf + (OutP%Size),Res);
 227
 228       OutP += Res;
 229    }
 230 }
 231                                                                         /*}}}*/
 232 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 233 // ---------------------------------------------------------------------
 234 /* This copies till the first empty line */
 235 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 236 {
 237    // We cheat and assume it is unneeded to have more than one buffer load
 238    for (unsigned long long I = OutP; I < InP; I++)
 239    {
 240       if (Buf[I%Size] != '\n')
 241          continue;
 242       ++I;
 243
 244       if (Single == false)
 245       {
 246          if (I < InP  && Buf[I%Size] == '\r')
 247             ++I;
 248          if (I >= InP || Buf[I%Size] != '\n')
 249             continue;
 250          ++I;
 251       }
 252
 253       Data = "";
 254       while (OutP < I)
 255       {
 256          unsigned long long Sz = LeftWrite();
 257          if (Sz == 0)
 258             return false;
 259          if (I - OutP < Sz)
 260             Sz = I - OutP;
 261          Data += string((char *)(Buf + (OutP%Size)),Sz);
 262          OutP += Sz;
 263       }
 264       return true;
 265    }
 266    return false;
 267 }
 268                                                                         /*}}}*/
 269 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 270 // ---------------------------------------------------------------------
 271 /* */
 272 void CircleBuf::Stats()
 273 {
 274    if (InP == 0)
 275       return;
 276
 277    struct timeval Stop;
 278    gettimeofday(&Stop,0);
 279 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 280              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 281    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 282 }
 283                                                                         /*}}}*/
 284 CircleBuf::~CircleBuf()
 285 {
 286    delete [] Buf;
 287    delete Hash;
 288 }
 289
 290 // HttpServerState::HttpServerState - Constructor                       /*{{{*/
 291 HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(Owner, 64*1024), Out(Owner, 4*1024)
 292 {
 293    TimeOut = Owner->ConfigFindI("Timeout", TimeOut);
 294    Reset();
 295 }
 296                                                                         /*}}}*/
 297 // HttpServerState::Open - Open a connection to the server              /*{{{*/
 298 // ---------------------------------------------------------------------
 299 /* This opens a connection to the server. */
 300 static bool TalkToSocksProxy(int const ServerFd, std::string const &Proxy,
 301       char const * const type, bool const ReadWrite, uint8_t * const ToFrom,
 302       unsigned int const Size, unsigned int const Timeout)
 303 {
 304    if (WaitFd(ServerFd, ReadWrite, Timeout) == false)
 305       return _error->Error("Waiting for the SOCKS proxy %s to %s timed out", URI::SiteOnly(Proxy).c_str(), type);
 306    if (ReadWrite == false)
 307    {
 308       if (FileFd::Read(ServerFd, ToFrom, Size) == false)
 309          return _error->Error("Reading the %s from SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str());
 310    }
 311    else
 312    {
 313       if (FileFd::Write(ServerFd, ToFrom, Size) == false)
 314          return _error->Error("Writing the %s to SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str());
 315    }
 316    return true;
 317 }
 318 bool HttpServerState::Open()
 319 {
 320    // Use the already open connection if possible.
 321    if (ServerFd != -1)
 322       return true;
 323
 324    Close();
 325    In.Reset();
 326    Out.Reset();
 327    Persistent = true;
 328
 329    // Determine the proxy setting
 330    AutoDetectProxy(ServerName);
 331    string SpecificProxy = Owner->ConfigFind("Proxy::" + ServerName.Host, "");
 332    if (!SpecificProxy.empty())
 333    {
 334            if (SpecificProxy == "DIRECT")
 335                    Proxy = "";
 336            else
 337                    Proxy = SpecificProxy;
 338    }
 339    else
 340    {
 341            string DefProxy = Owner->ConfigFind("Proxy", "");
 342            if (!DefProxy.empty())
 343            {
 344                    Proxy = DefProxy;
 345            }
 346            else
 347            {
 348                    char* result = getenv("http_proxy");
 349                    Proxy = result ? result : "";
 350            }
 351    }
 352
 353    // Parse no_proxy, a , separated list of domains
 354    if (getenv("no_proxy") != 0)
 355    {
 356       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 357          Proxy = "";
 358    }
 359
 360    if (Proxy.empty() == false)
 361       Owner->AddProxyAuth(Proxy, ServerName);
 362
 363    if (Proxy.Access == "socks5h")
 364    {
 365       if (Connect(Proxy.Host, Proxy.Port, "socks", 1080, ServerFd, TimeOut, Owner) == false)
 366          return false;
 367
 368       /* We implement a very basic SOCKS5 client here complying mostly to RFC1928 expect
 369        * for not offering GSSAPI auth which is a must (we only do no or user/pass auth).
 370        * We also expect the SOCKS5 server to do hostname lookup (aka socks5h) */
 371       std::string const ProxyInfo = URI::SiteOnly(Proxy);
 372       Owner->Status(_("Connecting to %s (%s)"),"SOCKS5h proxy",ProxyInfo.c_str());
 373       auto const Timeout = Owner->ConfigFindI("TimeOut", 120);
 374       #define APT_WriteOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) return false
 375       #define APT_ReadOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) return false
 376       if (ServerName.Host.length() > 255)
 377          return _error->Error("Can't use SOCKS5h as hostname %s is too long!", ServerName.Host.c_str());
 378       if (Proxy.User.length() > 255 || Proxy.Password.length() > 255)
 379          return _error->Error("Can't use user&pass auth as they are too long (%lu and %lu) for the SOCKS5!", Proxy.User.length(), Proxy.Password.length());
 380       if (Proxy.User.empty())
 381       {
 382          uint8_t greeting[] = { 0x05, 0x01, 0x00 };
 383          APT_WriteOrFail("greet-1", greeting, sizeof(greeting));
 384       }
 385       else
 386       {
 387          uint8_t greeting[] = { 0x05, 0x02, 0x00, 0x02 };
 388          APT_WriteOrFail("greet-2", greeting, sizeof(greeting));
 389       }
 390       uint8_t greeting[2];
 391       APT_ReadOrFail("greet back", greeting, sizeof(greeting));
 392       if (greeting[0] != 0x05)
 393          return _error->Error("SOCKS proxy %s greets back with wrong version: %d", ProxyInfo.c_str(), greeting[0]);
 394       if (greeting[1] == 0x00)
 395          ; // no auth has no method-dependent sub-negotiations
 396       else if (greeting[1] == 0x02)
 397       {
 398          if (Proxy.User.empty())
 399             return _error->Error("SOCKS proxy %s negotiated user&pass auth, but we had not offered it!", ProxyInfo.c_str());
 400          // user&pass auth sub-negotiations are defined by RFC1929
 401          std::vector<uint8_t> auth = {{ 0x01, static_cast<uint8_t>(Proxy.User.length()) }};
 402          std::copy(Proxy.User.begin(), Proxy.User.end(), std::back_inserter(auth));
 403          auth.push_back(static_cast<uint8_t>(Proxy.Password.length()));
 404          std::copy(Proxy.Password.begin(), Proxy.Password.end(), std::back_inserter(auth));
 405          APT_WriteOrFail("user&pass auth", auth.data(), auth.size());
 406          uint8_t authstatus[2];
 407          APT_ReadOrFail("auth report", authstatus, sizeof(authstatus));
 408          if (authstatus[0] != 0x01)
 409             return _error->Error("SOCKS proxy %s auth status response with wrong version: %d", ProxyInfo.c_str(), authstatus[0]);
 410          if (authstatus[1] != 0x00)
 411             return _error->Error("SOCKS proxy %s reported authorization failure: username or password incorrect? (%d)", ProxyInfo.c_str(), authstatus[1]);
 412       }
 413       else
 414          return _error->Error("SOCKS proxy %s greets back having not found a common authorization method: %d", ProxyInfo.c_str(), greeting[1]);
 415       union { uint16_t * i; uint8_t * b; } portu;
 416       uint16_t port = htons(static_cast<uint16_t>(ServerName.Port == 0 ? 80 : ServerName.Port));
 417       portu.i = &port;
 418       std::vector<uint8_t> request = {{ 0x05, 0x01, 0x00, 0x03, static_cast<uint8_t>(ServerName.Host.length()) }};
 419       std::copy(ServerName.Host.begin(), ServerName.Host.end(), std::back_inserter(request));
 420       request.push_back(portu.b[0]);
 421       request.push_back(portu.b[1]);
 422       APT_WriteOrFail("request", request.data(), request.size());
 423       uint8_t response[4];
 424       APT_ReadOrFail("first part of response", response, sizeof(response));
 425       if (response[0] != 0x05)
 426          return _error->Error("SOCKS proxy %s response with wrong version: %d", ProxyInfo.c_str(), response[0]);
 427       if (response[2] != 0x00)
 428          return _error->Error("SOCKS proxy %s has unexpected non-zero reserved field value: %d", ProxyInfo.c_str(), response[2]);
 429       std::string bindaddr;
 430       if (response[3] == 0x01) // IPv4 address
 431       {
 432          uint8_t ip4port[6];
 433          APT_ReadOrFail("IPv4+Port of response", ip4port, sizeof(ip4port));
 434          portu.b[0] = ip4port[4];
 435          portu.b[1] = ip4port[5];
 436          port = ntohs(*portu.i);
 437          strprintf(bindaddr, "%d.%d.%d.%d:%d", ip4port[0], ip4port[1], ip4port[2], ip4port[3], port);
 438       }
 439       else if (response[3] == 0x03) // hostname
 440       {
 441          uint8_t namelength;
 442          APT_ReadOrFail("hostname length of response", &namelength, 1);
 443          uint8_t hostname[namelength + 2];
 444          APT_ReadOrFail("hostname of response", hostname, sizeof(hostname));
 445          portu.b[0] = hostname[namelength];
 446          portu.b[1] = hostname[namelength + 1];
 447          port = ntohs(*portu.i);
 448          hostname[namelength] = '\0';
 449          strprintf(bindaddr, "%s:%d", hostname, port);
 450       }
 451       else if (response[3] == 0x04) // IPv6 address
 452       {
 453          uint8_t ip6port[18];
 454          APT_ReadOrFail("IPv6+port of response", ip6port, sizeof(ip6port));
 455          portu.b[0] = ip6port[16];
 456          portu.b[1] = ip6port[17];
 457          port = ntohs(*portu.i);
 458          strprintf(bindaddr, "[%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X]:%d",
 459                ip6port[0], ip6port[1], ip6port[2], ip6port[3], ip6port[4], ip6port[5], ip6port[6], ip6port[7],
 460                ip6port[8], ip6port[9], ip6port[10], ip6port[11], ip6port[12], ip6port[13], ip6port[14], ip6port[15],
 461                port);
 462       }
 463       else
 464          return _error->Error("SOCKS proxy %s destination address is of unknown type: %d",
 465                ProxyInfo.c_str(), response[3]);
 466       if (response[1] != 0x00)
 467       {
 468          char const * errstr;
 469          switch (response[1])
 470          {
 471              case 0x01: errstr = "general SOCKS server failure"; Owner->SetFailReason("SOCKS"); break;
 472              case 0x02: errstr = "connection not allowed by ruleset"; Owner->SetFailReason("SOCKS"); break;
 473              case 0x03: errstr = "Network unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break;
 474              case 0x04: errstr = "Host unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break;
 475              case 0x05: errstr = "Connection refused"; Owner->SetFailReason("ConnectionRefused"); break;
 476              case 0x06: errstr = "TTL expired"; Owner->SetFailReason("Timeout"); break;
 477              case 0x07: errstr = "Command not supported"; Owner->SetFailReason("SOCKS"); break;
 478              case 0x08: errstr = "Address type not supported"; Owner->SetFailReason("SOCKS"); break;
 479              default: errstr = "Unknown error"; Owner->SetFailReason("SOCKS"); break;
 480          }
 481          return _error->Error("SOCKS proxy %s didn't grant the connect to %s due to: %s (%d)", ProxyInfo.c_str(), bindaddr.c_str(), errstr, response[1]);
 482       }
 483       else if (Owner->DebugEnabled())
 484          ioprintf(std::clog, "http: SOCKS proxy %s connection established to %s\n", ProxyInfo.c_str(), bindaddr.c_str());
 485
 486       if (WaitFd(ServerFd, true, Timeout) == false)
 487          return _error->Error("SOCKS proxy %s reported connection, but timed out", ProxyInfo.c_str());
 488       #undef APT_ReadOrFail
 489       #undef APT_WriteOrFail
 490    }
 491    else
 492    {
 493       // Determine what host and port to use based on the proxy settings
 494       int Port = 0;
 495       string Host;
 496       if (Proxy.empty() == true || Proxy.Host.empty() == true)
 497       {
 498          if (ServerName.Port != 0)
 499             Port = ServerName.Port;
 500          Host = ServerName.Host;
 501       }
 502       else if (Proxy.Access != "http")
 503          return _error->Error("Unsupported proxy configured: %s", URI::SiteOnly(Proxy).c_str());
 504       else
 505       {
 506          if (Proxy.Port != 0)
 507             Port = Proxy.Port;
 508          Host = Proxy.Host;
 509       }
 510       return Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner);
 511    }
 512    return true;
 513 }
 514                                                                         /*}}}*/
 515 // HttpServerState::Close - Close a connection to the server            /*{{{*/
 516 // ---------------------------------------------------------------------
 517 /* */
 518 bool HttpServerState::Close()
 519 {
 520    close(ServerFd);
 521    ServerFd = -1;
 522    return true;
 523 }
 524                                                                         /*}}}*/
 525 // HttpServerState::RunData - Transfer the data from the socket         /*{{{*/
 526 bool HttpServerState::RunData(FileFd * const File)
 527 {
 528    State = Data;
 529
 530    // Chunked transfer encoding is fun..
 531    if (Encoding == Chunked)
 532    {
 533       while (1)
 534       {
 535          // Grab the block size
 536          bool Last = true;
 537          string Data;
 538          In.Limit(-1);
 539          do
 540          {
 541             if (In.WriteTillEl(Data,true) == true)
 542                break;
 543          }
 544          while ((Last = Go(false, File)) == true);
 545
 546          if (Last == false)
 547             return false;
 548
 549          // See if we are done
 550          unsigned long long Len = strtoull(Data.c_str(),0,16);
 551          if (Len == 0)
 552          {
 553             In.Limit(-1);
 554
 555             // We have to remove the entity trailer
 556             Last = true;
 557             do
 558             {
 559                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 560                   break;
 561             }
 562             while ((Last = Go(false, File)) == true);
 563             if (Last == false)
 564                return false;
 565             return !_error->PendingError();
 566          }
 567
 568          // Transfer the block
 569          In.Limit(Len);
 570          while (Go(true, File) == true)
 571             if (In.IsLimit() == true)
 572                break;
 573
 574          // Error
 575          if (In.IsLimit() == false)
 576             return false;
 577
 578          // The server sends an extra new line before the next block specifier..
 579          In.Limit(-1);
 580          Last = true;
 581          do
 582          {
 583             if (In.WriteTillEl(Data,true) == true)
 584                break;
 585          }
 586          while ((Last = Go(false, File)) == true);
 587          if (Last == false)
 588             return false;
 589       }
 590    }
 591    else
 592    {
 593       /* Closes encoding is used when the server did not specify a size, the
 594          loss of the connection means we are done */
 595       if (JunkSize != 0)
 596          In.Limit(JunkSize);
 597       else if (DownloadSize != 0)
 598          In.Limit(DownloadSize);
 599       else if (Persistent == false)
 600          In.Limit(-1);
 601
 602       // Just transfer the whole block.
 603       do
 604       {
 605          if (In.IsLimit() == false)
 606             continue;
 607
 608          In.Limit(-1);
 609          return !_error->PendingError();
 610       }
 611       while (Go(true, File) == true);
 612    }
 613
 614    return Owner->Flush() && !_error->PendingError();
 615 }
 616                                                                         /*}}}*/
 617 bool HttpServerState::RunDataToDevNull()                                /*{{{*/
 618 {
 619    FileFd DevNull("/dev/null", FileFd::WriteOnly);
 620    return RunData(&DevNull);
 621 }
 622                                                                         /*}}}*/
 623 bool HttpServerState::ReadHeaderLines(std::string &Data)                /*{{{*/
 624 {
 625    return In.WriteTillEl(Data);
 626 }
 627                                                                         /*}}}*/
 628 bool HttpServerState::LoadNextResponse(bool const ToFile, FileFd * const File)/*{{{*/
 629 {
 630    return Go(ToFile, File);
 631 }
 632                                                                         /*}}}*/
 633 bool HttpServerState::WriteResponse(const std::string &Data)            /*{{{*/
 634 {
 635    return Out.Read(Data);
 636 }
 637                                                                         /*}}}*/
 638 APT_PURE bool HttpServerState::IsOpen()                                 /*{{{*/
 639 {
 640    return (ServerFd != -1);
 641 }
 642                                                                         /*}}}*/
 643 bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes)  /*{{{*/
 644 {
 645    delete In.Hash;
 646    In.Hash = new Hashes(ExpectedHashes);
 647    return true;
 648 }
 649                                                                         /*}}}*/
 650 void HttpServerState::Reset(bool const Everything)                      /*{{{*/
 651 {
 652    ServerState::Reset(Everything);
 653    if (Everything)
 654       ServerFd = -1;
 655 }
 656                                                                         /*}}}*/
 657
 658 APT_PURE Hashes * HttpServerState::GetHashes()                          /*{{{*/
 659 {
 660    return In.Hash;
 661 }
 662                                                                         /*}}}*/
 663 // HttpServerState::Die - The server has closed the connection.         /*{{{*/
 664 bool HttpServerState::Die(FileFd * const File)
 665 {
 666    unsigned int LErrno = errno;
 667
 668    // Dump the buffer to the file
 669    if (State == ServerState::Data)
 670    {
 671       if (File == nullptr)
 672          return true;
 673       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 674       // can't be set
 675       if (File->Name() != "/dev/null")
 676          SetNonBlock(File->Fd(),false);
 677       while (In.WriteSpace() == true)
 678       {
 679          if (In.Write(File->Fd()) == false)
 680             return _error->Errno("write",_("Error writing to the file"));
 681
 682          // Done
 683          if (In.IsLimit() == true)
 684             return true;
 685       }
 686    }
 687
 688    // See if this is because the server finished the data stream
 689    if (In.IsLimit() == false && State != HttpServerState::Header &&
 690        Persistent == true)
 691    {
 692       Close();
 693       if (LErrno == 0)
 694          return _error->Error(_("Error reading from server. Remote end closed connection"));
 695       errno = LErrno;
 696       return _error->Errno("read",_("Error reading from server"));
 697    }
 698    else
 699    {
 700       In.Limit(-1);
 701
 702       // Nothing left in the buffer
 703       if (In.WriteSpace() == false)
 704          return false;
 705
 706       // We may have got multiple responses back in one packet..
 707       Close();
 708       return true;
 709    }
 710
 711    return false;
 712 }
 713                                                                         /*}}}*/
 714 // HttpServerState::Flush - Dump the buffer into the file               /*{{{*/
 715 // ---------------------------------------------------------------------
 716 /* This takes the current input buffer from the Server FD and writes it
 717    into the file */
 718 bool HttpServerState::Flush(FileFd * const File)
 719 {
 720    if (File != NULL)
 721    {
 722       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 723       // can't be set
 724       if (File->Name() != "/dev/null")
 725          SetNonBlock(File->Fd(),false);
 726       if (In.WriteSpace() == false)
 727          return true;
 728
 729       while (In.WriteSpace() == true)
 730       {
 731          if (In.Write(File->Fd()) == false)
 732             return _error->Errno("write",_("Error writing to file"));
 733          if (In.IsLimit() == true)
 734             return true;
 735       }
 736
 737       if (In.IsLimit() == true || Persistent == false)
 738          return true;
 739    }
 740    return false;
 741 }
 742                                                                         /*}}}*/
 743 // HttpServerState::Go - Run a single loop                              /*{{{*/
 744 // ---------------------------------------------------------------------
 745 /* This runs the select loop over the server FDs, Output file FDs and
 746    stdin. */
 747 bool HttpServerState::Go(bool ToFile, FileFd * const File)
 748 {
 749    // Server has closed the connection
 750    if (ServerFd == -1 && (In.WriteSpace() == false ||
 751                                ToFile == false))
 752       return false;
 753
 754    fd_set rfds,wfds;
 755    FD_ZERO(&rfds);
 756    FD_ZERO(&wfds);
 757
 758    /* Add the server. We only send more requests if the connection will
 759       be persisting */
 760    if (Out.WriteSpace() == true && ServerFd != -1
 761        && Persistent == true)
 762       FD_SET(ServerFd,&wfds);
 763    if (In.ReadSpace() == true && ServerFd != -1)
 764       FD_SET(ServerFd,&rfds);
 765
 766    // Add the file
 767    int FileFD = -1;
 768    if (File != NULL)
 769       FileFD = File->Fd();
 770
 771    if (In.WriteSpace() == true && ToFile == true && FileFD != -1)
 772       FD_SET(FileFD,&wfds);
 773
 774    // Add stdin
 775    if (Owner->ConfigFindB("DependOnSTDIN", true) == true)
 776       FD_SET(STDIN_FILENO,&rfds);
 777
 778    // Figure out the max fd
 779    int MaxFd = FileFD;
 780    if (MaxFd < ServerFd)
 781       MaxFd = ServerFd;
 782
 783    // Select
 784    struct timeval tv;
 785    tv.tv_sec = TimeOut;
 786    tv.tv_usec = 0;
 787    int Res = 0;
 788    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 789    {
 790       if (errno == EINTR)
 791          return true;
 792       return _error->Errno("select",_("Select failed"));
 793    }
 794
 795    if (Res == 0)
 796    {
 797       _error->Error(_("Connection timed out"));
 798       return Die(File);
 799    }
 800
 801    // Handle server IO
 802    if (ServerFd != -1 && FD_ISSET(ServerFd,&rfds))
 803    {
 804       errno = 0;
 805       if (In.Read(ServerFd) == false)
 806          return Die(File);
 807    }
 808
 809    if (ServerFd != -1 && FD_ISSET(ServerFd,&wfds))
 810    {
 811       errno = 0;
 812       if (Out.Write(ServerFd) == false)
 813          return Die(File);
 814    }
 815
 816    // Send data to the file
 817    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 818    {
 819       if (In.Write(FileFD) == false)
 820          return _error->Errno("write",_("Error writing to output file"));
 821    }
 822
 823    if (MaximumSize > 0 && File && File->Tell() > MaximumSize)
 824    {
 825       Owner->SetFailReason("MaximumSizeExceeded");
 826       return _error->Error("Writing more data than expected (%llu > %llu)",
 827                            File->Tell(), MaximumSize);
 828    }
 829
 830    // Handle commands from APT
 831    if (FD_ISSET(STDIN_FILENO,&rfds))
 832    {
 833       if (Owner->Run(true) != -1)
 834          exit(100);
 835    }
 836
 837    return true;
 838 }
 839                                                                         /*}}}*/
 840
 841 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 842 // ---------------------------------------------------------------------
 843 /* This places the http request in the outbound buffer */
 844 void HttpMethod::SendReq(FetchItem *Itm)
 845 {
 846    URI Uri = Itm->Uri;
 847    {
 848       auto const plus = Binary.find('+');
 849       if (plus != std::string::npos)
 850          Uri.Access = Binary.substr(plus + 1);
 851    }
 852
 853    // The HTTP server expects a hostname with a trailing :port
 854    std::stringstream Req;
 855    string ProperHost;
 856
 857    if (Uri.Host.find(':') != string::npos)
 858       ProperHost = '[' + Uri.Host + ']';
 859    else
 860       ProperHost = Uri.Host;
 861
 862    /* RFC 2616 §5.1.2 requires absolute URIs for requests to proxies,
 863       but while its a must for all servers to accept absolute URIs,
 864       it is assumed clients will sent an absolute path for non-proxies */
 865    std::string requesturi;
 866    if (Server->Proxy.Access != "http" || Server->Proxy.empty() == true || Server->Proxy.Host.empty())
 867       requesturi = Uri.Path;
 868    else
 869       requesturi = Uri;
 870
 871    // The "+" is encoded as a workaround for a amazon S3 bug
 872    // see LP bugs #1003633 and #1086997.
 873    requesturi = QuoteString(requesturi, "+~ ");
 874
 875    /* Build the request. No keep-alive is included as it is the default
 876       in 1.1, can cause problems with proxies, and we are an HTTP/1.1
 877       client anyway.
 878       C.f. https://tools.ietf.org/wg/httpbis/trac/ticket/158 */
 879    Req << "GET " << requesturi << " HTTP/1.1\r\n";
 880    if (Uri.Port != 0)
 881       Req << "Host: " << ProperHost << ":" << std::to_string(Uri.Port) << "\r\n";
 882    else
 883       Req << "Host: " << ProperHost << "\r\n";
 884
 885    // generate a cache control header (if needed)
 886    if (ConfigFindB("No-Cache",false) == true)
 887       Req << "Cache-Control: no-cache\r\n"
 888          << "Pragma: no-cache\r\n";
 889    else if (Itm->IndexFile == true)
 890       Req << "Cache-Control: max-age=" << std::to_string(ConfigFindI("Max-Age", 0)) << "\r\n";
 891    else if (ConfigFindB("No-Store", false) == true)
 892       Req << "Cache-Control: no-store\r\n";
 893
 894    // If we ask for uncompressed files servers might respond with content-
 895    // negotiation which lets us end up with compressed files we do not support,
 896    // see 657029, 657560 and co, so if we have no extension on the request
 897    // ask for text only. As a sidenote: If there is nothing to negotate servers
 898    // seem to be nice and ignore it.
 899    if (ConfigFindB("SendAccept", true) == true)
 900    {
 901       size_t const filepos = Itm->Uri.find_last_of('/');
 902       string const file = Itm->Uri.substr(filepos + 1);
 903       if (flExtension(file) == file)
 904          Req << "Accept: text/*\r\n";
 905    }
 906
 907    // Check for a partial file and send if-queries accordingly
 908    struct stat SBuf;
 909    if (Server->RangesAllowed && stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 910       Req << "Range: bytes=" << std::to_string(SBuf.st_size) << "-\r\n"
 911          << "If-Range: " << TimeRFC1123(SBuf.st_mtime, false) << "\r\n";
 912    else if (Itm->LastModified != 0)
 913       Req << "If-Modified-Since: " << TimeRFC1123(Itm->LastModified, false).c_str() << "\r\n";
 914
 915    if (Server->Proxy.Access == "http" &&
 916          (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false))
 917       Req << "Proxy-Authorization: Basic "
 918          << Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) << "\r\n";
 919
 920    maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
 921    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 922       Req << "Authorization: Basic "
 923          << Base64Encode(Uri.User + ":" + Uri.Password) << "\r\n";
 924
 925    Req << "User-Agent: " << ConfigFind("User-Agent",
 926                 "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")") << "\r\n";
 927
 928    Req << "\r\n";
 929
 930    if (Debug == true)
 931       cerr << Req.str() << endl;
 932
 933    Server->WriteResponse(Req.str());
 934 }
 935                                                                         /*}}}*/
 936 std::unique_ptr<ServerState> HttpMethod::CreateServerState(URI const &uri)/*{{{*/
 937 {
 938    return std::unique_ptr<ServerState>(new HttpServerState(uri, this));
 939 }
 940                                                                         /*}}}*/
 941 void HttpMethod::RotateDNS()                                            /*{{{*/
 942 {
 943    ::RotateDNS();
 944 }
 945                                                                         /*}}}*/
 946 ServerMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &Res)/*{{{*/
 947 {
 948    auto ret = ServerMethod::DealWithHeaders(Res);
 949    if (ret != ServerMethod::FILE_IS_OPEN)
 950       return ret;
 951
 952    // Open the file
 953    delete File;
 954    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 955    if (_error->PendingError() == true)
 956       return ERROR_NOT_FROM_SERVER;
 957
 958    FailFile = Queue->DestFile;
 959    FailFile.c_str();   // Make sure we don't do a malloc in the signal handler
 960    FailFd = File->Fd();
 961    FailTime = Server->Date;
 962
 963    if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false)
 964    {
 965       _error->Errno("read",_("Problem hashing file"));
 966       return ERROR_NOT_FROM_SERVER;
 967    }
 968    if (Server->StartPos > 0)
 969       Res.ResumePoint = Server->StartPos;
 970
 971    SetNonBlock(File->Fd(),true);
 972    return FILE_IS_OPEN;
 973 }
 974                                                                         /*}}}*/
 975 HttpMethod::HttpMethod(std::string &&pProg) : ServerMethod(pProg.c_str(), "1.2", Pipeline | SendConfig)/*{{{*/
 976 {
 977    auto addName = std::inserter(methodNames, methodNames.begin());
 978    if (Binary != "http")
 979       addName = "http";
 980    auto const plus = Binary.find('+');
 981    if (plus != std::string::npos)
 982       addName = Binary.substr(0, plus);
 983    File = 0;
 984    Server = 0;
 985 }
 986                                                                         /*}}}*/