methods/server.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 /* ######################################################################
   4
   5    HTTP and HTTPS share a lot of common code and these classes are
   6    exactly the dumping ground for this common code
   7
   8    ##################################################################### */
   9                                                                         /*}}}*/
  10 // Include Files                                                        /*{{{*/
  11 #include <config.h>
  12
  13 #include <apt-pkg/configuration.h>
  14 #include <apt-pkg/error.h>
  15 #include <apt-pkg/fileutl.h>
  16 #include <apt-pkg/strutl.h>
  17
  18 #include <ctype.h>
  19 #include <signal.h>
  20 #include <stdio.h>
  21 #include <stdlib.h>
  22 #include <sys/stat.h>
  23 #include <sys/time.h>
  24 #include <time.h>
  25 #include <unistd.h>
  26 #include <iostream>
  27 #include <limits>
  28 #include <map>
  29 #include <string>
  30 #include <vector>
  31
  32 #include "server.h"
  33
  34 #include <apti18n.h>
  35                                                                         /*}}}*/
  36 using namespace std;
  37
  38 string ServerMethod::FailFile;
  39 int ServerMethod::FailFd = -1;
  40 time_t ServerMethod::FailTime = 0;
  41
  42 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
  43 // ---------------------------------------------------------------------
  44 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
  45    parse error occurred */
  46 ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
  47                                                       const std::string &Uri)
  48 {
  49    Reset(false);
  50    Owner->Status(_("Waiting for headers"));
  51
  52    do
  53    {
  54       string Data;
  55       if (ReadHeaderLines(Data) == false)
  56          continue;
  57
  58       if (Owner->Debug == true)
  59          clog << "Answer for: " << Uri << endl << Data;
  60
  61       for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
  62       {
  63          string::const_iterator J = I;
  64          for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
  65          if (HeaderLine(string(I,J)) == false)
  66             return RUN_HEADERS_PARSE_ERROR;
  67          I = J;
  68       }
  69
  70       // 100 Continue is a Nop...
  71       if (Result == 100)
  72          continue;
  73
  74       // Tidy up the connection persistence state.
  75       if (Encoding == Closes && HaveContent == true)
  76          Persistent = false;
  77
  78       return RUN_HEADERS_OK;
  79    }
  80    while (LoadNextResponse(false, File) == true);
  81
  82    return RUN_HEADERS_IO_ERROR;
  83 }
  84                                                                         /*}}}*/
  85 // ServerState::HeaderLine - Process a header line                      /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* */
  88 bool ServerState::HeaderLine(string Line)
  89 {
  90    if (Line.empty() == true)
  91       return true;
  92
  93    if (Line.size() > 4 && stringcasecmp(Line.data(), Line.data()+4, "HTTP") == 0)
  94    {
  95       // Evil servers return no version
  96       if (Line[4] == '/')
  97       {
  98          int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
  99          if (elements == 3)
 100          {
 101             Code[0] = '\0';
 102             if (Owner != NULL && Owner->Debug == true)
 103                clog << "HTTP server doesn't give Reason-Phrase for " << std::to_string(Result) << std::endl;
 104          }
 105          else if (elements != 4)
 106             return _error->Error(_("The HTTP server sent an invalid reply header"));
 107       }
 108       else
 109       {
 110          Major = 0;
 111          Minor = 9;
 112          if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
 113             return _error->Error(_("The HTTP server sent an invalid reply header"));
 114       }
 115
 116       /* Check the HTTP response header to get the default persistence
 117          state. */
 118       if (Major < 1)
 119          Persistent = false;
 120       else
 121       {
 122          if (Major == 1 && Minor == 0)
 123          {
 124             Persistent = false;
 125          }
 126          else
 127          {
 128             Persistent = true;
 129             if (PipelineAllowed)
 130                Pipeline = true;
 131          }
 132       }
 133
 134       return true;
 135    }
 136
 137    // Blah, some servers use "connection:closes", evil.
 138    // and some even send empty header fields…
 139    string::size_type Pos = Line.find(':');
 140    if (Pos == string::npos)
 141       return _error->Error(_("Bad header line"));
 142    ++Pos;
 143
 144    // Parse off any trailing spaces between the : and the next word.
 145    string::size_type Pos2 = Pos;
 146    while (Pos2 < Line.length() && isspace_ascii(Line[Pos2]) != 0)
 147       Pos2++;
 148
 149    string const Tag(Line,0,Pos);
 150    string const Val(Line,Pos2);
 151
 152    if (stringcasecmp(Tag,"Content-Length:") == 0)
 153    {
 154       if (Encoding == Closes)
 155          Encoding = Stream;
 156       HaveContent = true;
 157
 158       unsigned long long * DownloadSizePtr = &DownloadSize;
 159       if (Result == 416 || (Result >= 300 && Result < 400))
 160          DownloadSizePtr = &JunkSize;
 161
 162       *DownloadSizePtr = strtoull(Val.c_str(), NULL, 10);
 163       if (*DownloadSizePtr >= std::numeric_limits<unsigned long long>::max())
 164          return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
 165       else if (*DownloadSizePtr == 0)
 166          HaveContent = false;
 167
 168       // On partial content (206) the Content-Length less than the real
 169       // size, so do not set it here but leave that to the Content-Range
 170       // header instead
 171       if(Result != 206 && TotalFileSize == 0)
 172          TotalFileSize = DownloadSize;
 173
 174       return true;
 175    }
 176
 177    if (stringcasecmp(Tag,"Content-Type:") == 0)
 178    {
 179       HaveContent = true;
 180       return true;
 181    }
 182
 183    if (stringcasecmp(Tag,"Content-Range:") == 0)
 184    {
 185       HaveContent = true;
 186
 187       // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
 188       if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&TotalFileSize) == 1)
 189          ; // we got the expected filesize which is all we wanted
 190       else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&TotalFileSize) != 2)
 191          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 192       if ((unsigned long long)StartPos > TotalFileSize)
 193          return _error->Error(_("This HTTP server has broken range support"));
 194
 195       // figure out what we will download
 196       DownloadSize = TotalFileSize - StartPos;
 197       return true;
 198    }
 199
 200    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 201    {
 202       HaveContent = true;
 203       if (stringcasecmp(Val,"chunked") == 0)
 204          Encoding = Chunked;
 205       return true;
 206    }
 207
 208    if (stringcasecmp(Tag,"Connection:") == 0)
 209    {
 210       if (stringcasecmp(Val,"close") == 0)
 211          Persistent = false;
 212       if (stringcasecmp(Val,"keep-alive") == 0)
 213          Persistent = true;
 214       return true;
 215    }
 216
 217    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 218    {
 219       if (RFC1123StrToTime(Val.c_str(), Date) == false)
 220          return _error->Error(_("Unknown date format"));
 221       return true;
 222    }
 223
 224    if (stringcasecmp(Tag,"Location:") == 0)
 225    {
 226       Location = Val;
 227       return true;
 228    }
 229
 230    if (stringcasecmp(Tag, "Accept-Ranges:") == 0)
 231    {
 232       std::string ranges = ',' + Val + ',';
 233       ranges.erase(std::remove(ranges.begin(), ranges.end(), ' '), ranges.end());
 234       if (ranges.find(",bytes,") == std::string::npos)
 235          RangesAllowed = false;
 236       return true;
 237    }
 238
 239    return true;
 240 }
 241                                                                         /*}}}*/
 242 // ServerState::ServerState - Constructor                               /*{{{*/
 243 ServerState::ServerState(URI Srv, ServerMethod *Owner) :
 244    DownloadSize(0), ServerName(Srv), TimeOut(120), Owner(Owner)
 245 {
 246    Reset();
 247 }
 248                                                                         /*}}}*/
 249 bool ServerState::AddPartialFileToHashes(FileFd &File)                  /*{{{*/
 250 {
 251    File.Truncate(StartPos);
 252    return GetHashes()->AddFD(File, StartPos);
 253 }
 254                                                                         /*}}}*/
 255 void ServerState::Reset(bool const Everything)                          /*{{{*/
 256 {
 257    Major = 0; Minor = 0; Result = 0; Code[0] = '\0';
 258    TotalFileSize = 0; JunkSize = 0; StartPos = 0;
 259    Encoding = Closes; time(&Date); HaveContent = false;
 260    State = Header; MaximumSize = 0;
 261    if (Everything)
 262    {
 263       Persistent = false; Pipeline = false; PipelineAllowed = true;
 264       RangesAllowed = true;
 265    }
 266 }
 267                                                                         /*}}}*/
 268
 269 // ServerMethod::DealWithHeaders - Handle the retrieved header data     /*{{{*/
 270 // ---------------------------------------------------------------------
 271 /* We look at the header data we got back from the server and decide what
 272    to do. Returns DealWithHeadersResult (see http.h for details).
 273  */
 274 ServerMethod::DealWithHeadersResult
 275 ServerMethod::DealWithHeaders(FetchResult &Res)
 276 {
 277    // Not Modified
 278    if (Server->Result == 304)
 279    {
 280       RemoveFile("server", Queue->DestFile);
 281       Res.IMSHit = true;
 282       Res.LastModified = Queue->LastModified;
 283       Res.Size = 0;
 284       return IMS_HIT;
 285    }
 286
 287    /* Redirect
 288     *
 289     * Note that it is only OK for us to treat all redirection the same
 290     * because we *always* use GET, not other HTTP methods.  There are
 291     * three redirection codes for which it is not appropriate that we
 292     * redirect.  Pass on those codes so the error handling kicks in.
 293     */
 294    if (AllowRedirect
 295        && (Server->Result > 300 && Server->Result < 400)
 296        && (Server->Result != 300       // Multiple Choices
 297            && Server->Result != 304    // Not Modified
 298            && Server->Result != 306))  // (Not part of HTTP/1.1, reserved)
 299    {
 300       if (Server->Location.empty() == true)
 301          ;
 302       else if (Server->Location[0] == '/' && Queue->Uri.empty() == false)
 303       {
 304          URI Uri = Queue->Uri;
 305          if (Uri.Host.empty() == false)
 306             NextURI = URI::SiteOnly(Uri);
 307          else
 308             NextURI.clear();
 309          NextURI.append(DeQuoteString(Server->Location));
 310          if (Queue->Uri == NextURI)
 311          {
 312             SetFailReason("RedirectionLoop");
 313             _error->Error("Redirection loop encountered");
 314             if (Server->HaveContent == true)
 315                return ERROR_WITH_CONTENT_PAGE;
 316             return ERROR_UNRECOVERABLE;
 317          }
 318          return TRY_AGAIN_OR_REDIRECT;
 319       }
 320       else
 321       {
 322          NextURI = DeQuoteString(Server->Location);
 323          URI tmpURI = NextURI;
 324          if (tmpURI.Access.find('+') != std::string::npos)
 325          {
 326             _error->Error("Server tried to trick us into using a specific implementation: %s", tmpURI.Access.c_str());
 327             if (Server->HaveContent == true)
 328                return ERROR_WITH_CONTENT_PAGE;
 329             return ERROR_UNRECOVERABLE;
 330          }
 331          URI Uri = Queue->Uri;
 332          if (Binary.find('+') != std::string::npos)
 333          {
 334             auto base = Binary.substr(0, Binary.find('+'));
 335             if (base != tmpURI.Access)
 336             {
 337                tmpURI.Access = base + '+' + tmpURI.Access;
 338                if (tmpURI.Access == Binary)
 339                {
 340                   std::string tmpAccess = Uri.Access;
 341                   std::swap(tmpURI.Access, Uri.Access);
 342                   NextURI = tmpURI;
 343                   std::swap(tmpURI.Access, Uri.Access);
 344                }
 345                else
 346                   NextURI = tmpURI;
 347             }
 348          }
 349          if (Queue->Uri == NextURI)
 350          {
 351             SetFailReason("RedirectionLoop");
 352             _error->Error("Redirection loop encountered");
 353             if (Server->HaveContent == true)
 354                return ERROR_WITH_CONTENT_PAGE;
 355             return ERROR_UNRECOVERABLE;
 356          }
 357          Uri.Access = Binary;
 358          // same protocol redirects are okay
 359          if (tmpURI.Access == Uri.Access)
 360             return TRY_AGAIN_OR_REDIRECT;
 361          // as well as http to https
 362          else if ((Uri.Access == "http" || Uri.Access == "https+http") && tmpURI.Access == "https")
 363             return TRY_AGAIN_OR_REDIRECT;
 364          else
 365          {
 366             auto const tmpplus = tmpURI.Access.find('+');
 367             if (tmpplus != std::string::npos && tmpURI.Access.substr(tmpplus + 1) == "https")
 368             {
 369                auto const uriplus = Uri.Access.find('+');
 370                if (uriplus == std::string::npos)
 371                {
 372                   if (Uri.Access == tmpURI.Access.substr(0, tmpplus)) // foo -> foo+https
 373                      return TRY_AGAIN_OR_REDIRECT;
 374                }
 375                else if (Uri.Access.substr(uriplus + 1) == "http" &&
 376                      Uri.Access.substr(0, uriplus) == tmpURI.Access.substr(0, tmpplus)) // foo+http -> foo+https
 377                   return TRY_AGAIN_OR_REDIRECT;
 378             }
 379          }
 380          _error->Error("Redirection from %s to '%s' is forbidden", Uri.Access.c_str(), NextURI.c_str());
 381       }
 382       /* else pass through for error message */
 383    }
 384    // retry after an invalid range response without partial data
 385    else if (Server->Result == 416)
 386    {
 387       struct stat SBuf;
 388       if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 389       {
 390          bool partialHit = false;
 391          if (Queue->ExpectedHashes.usable() == true)
 392          {
 393             Hashes resultHashes(Queue->ExpectedHashes);
 394             FileFd file(Queue->DestFile, FileFd::ReadOnly);
 395             Server->TotalFileSize = file.FileSize();
 396             Server->Date = file.ModificationTime();
 397             resultHashes.AddFD(file);
 398             HashStringList const hashList = resultHashes.GetHashStringList();
 399             partialHit = (Queue->ExpectedHashes == hashList);
 400          }
 401          else if ((unsigned long long)SBuf.st_size == Server->TotalFileSize)
 402             partialHit = true;
 403          if (partialHit == true)
 404          {
 405             // the file is completely downloaded, but was not moved
 406             if (Server->HaveContent == true)
 407             {
 408                // nuke the sent error page
 409                Server->RunDataToDevNull();
 410                Server->HaveContent = false;
 411             }
 412             Server->StartPos = Server->TotalFileSize;
 413             Server->Result = 200;
 414          }
 415          else if (RemoveFile("server", Queue->DestFile))
 416          {
 417             NextURI = Queue->Uri;
 418             return TRY_AGAIN_OR_REDIRECT;
 419          }
 420       }
 421    }
 422
 423    /* We have a reply we don't handle. This should indicate a perm server
 424       failure */
 425    if (Server->Result < 200 || Server->Result >= 300)
 426    {
 427       if (_error->PendingError() == false)
 428       {
 429          std::string err;
 430          strprintf(err, "HttpError%u", Server->Result);
 431          SetFailReason(err);
 432          _error->Error("%u %s", Server->Result, Server->Code);
 433       }
 434       if (Server->HaveContent == true)
 435          return ERROR_WITH_CONTENT_PAGE;
 436       return ERROR_UNRECOVERABLE;
 437    }
 438
 439    // This is some sort of 2xx 'data follows' reply
 440    Res.LastModified = Server->Date;
 441    Res.Size = Server->TotalFileSize;
 442    return FILE_IS_OPEN;
 443 }
 444                                                                         /*}}}*/
 445 // ServerMethod::SigTerm - Handle a fatal signal                        /*{{{*/
 446 // ---------------------------------------------------------------------
 447 /* This closes and timestamps the open file. This is necessary to get
 448    resume behavoir on user abort */
 449 void ServerMethod::SigTerm(int)
 450 {
 451    if (FailFd == -1)
 452       _exit(100);
 453
 454    struct timeval times[2];
 455    times[0].tv_sec = FailTime;
 456    times[1].tv_sec = FailTime;
 457    times[0].tv_usec = times[1].tv_usec = 0;
 458    utimes(FailFile.c_str(), times);
 459    close(FailFd);
 460
 461    _exit(100);
 462 }
 463                                                                         /*}}}*/
 464 // ServerMethod::Fetch - Fetch an item                                  /*{{{*/
 465 // ---------------------------------------------------------------------
 466 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 467    depth. */
 468 bool ServerMethod::Fetch(FetchItem *)
 469 {
 470    if (Server == nullptr || QueueBack == nullptr)
 471       return true;
 472
 473    // If pipelining is disabled, we only queue 1 request
 474    auto const AllowedDepth = Server->Pipeline ? PipelineDepth : 0;
 475    // how deep is our pipeline currently?
 476    decltype(PipelineDepth) CurrentDepth = 0;
 477    for (FetchItem const *I = Queue; I != QueueBack; I = I->Next)
 478       ++CurrentDepth;
 479    if (CurrentDepth > AllowedDepth)
 480       return true;
 481
 482    do {
 483       // Make sure we stick with the same server
 484       if (Server->Comp(QueueBack->Uri) == false)
 485          break;
 486
 487       bool const UsableHashes = QueueBack->ExpectedHashes.usable();
 488       // if we have no hashes, do at most one such request
 489       // as we can't fixup pipeling misbehaviors otherwise
 490       if (CurrentDepth != 0 && UsableHashes == false)
 491          break;
 492
 493       if (UsableHashes && FileExists(QueueBack->DestFile))
 494       {
 495          FileFd partial(QueueBack->DestFile, FileFd::ReadOnly);
 496          Hashes wehave(QueueBack->ExpectedHashes);
 497          if (QueueBack->ExpectedHashes.FileSize() == partial.FileSize())
 498          {
 499             if (wehave.AddFD(partial) &&
 500                   wehave.GetHashStringList() == QueueBack->ExpectedHashes)
 501             {
 502                FetchResult Res;
 503                Res.Filename = QueueBack->DestFile;
 504                Res.ResumePoint = QueueBack->ExpectedHashes.FileSize();
 505                URIStart(Res);
 506                // move item to the start of the queue as URIDone will
 507                // always dequeued the first item in the queue
 508                if (Queue != QueueBack)
 509                {
 510                   FetchItem *Prev = Queue;
 511                   for (; Prev->Next != QueueBack; Prev = Prev->Next)
 512                      /* look for the previous queue item */;
 513                   Prev->Next = QueueBack->Next;
 514                   QueueBack->Next = Queue;
 515                   Queue = QueueBack;
 516                   QueueBack = Prev->Next;
 517                }
 518                Res.TakeHashes(wehave);
 519                URIDone(Res);
 520                continue;
 521             }
 522             else
 523                RemoveFile("Fetch-Partial", QueueBack->DestFile);
 524          }
 525       }
 526       auto const Tmp = QueueBack;
 527       QueueBack = QueueBack->Next;
 528       SendReq(Tmp);
 529       ++CurrentDepth;
 530    } while (CurrentDepth <= AllowedDepth && QueueBack != nullptr);
 531
 532    return true;
 533 }
 534                                                                         /*}}}*/
 535 // ServerMethod::Loop - Main loop                                       /*{{{*/
 536 int ServerMethod::Loop()
 537 {
 538    signal(SIGTERM,SigTerm);
 539    signal(SIGINT,SigTerm);
 540
 541    Server = 0;
 542
 543    int FailCounter = 0;
 544    while (1)
 545    {
 546       // We have no commands, wait for some to arrive
 547       if (Queue == 0)
 548       {
 549          if (WaitFd(STDIN_FILENO) == false)
 550             return 0;
 551       }
 552
 553       /* Run messages, we can accept 0 (no message) if we didn't
 554          do a WaitFd above.. Otherwise the FD is closed. */
 555       int Result = Run(true);
 556       if (Result != -1 && (Result != 0 || Queue == 0))
 557       {
 558          if(FailReason.empty() == false ||
 559             ConfigFindB("DependOnSTDIN", true) == true)
 560             return 100;
 561          else
 562             return 0;
 563       }
 564
 565       if (Queue == 0)
 566          continue;
 567
 568       // Connect to the server
 569       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 570       {
 571          Server = CreateServerState(Queue->Uri);
 572          setPostfixForMethodNames(::URI(Queue->Uri).Host.c_str());
 573          AllowRedirect = ConfigFindB("AllowRedirect", true);
 574          PipelineDepth = ConfigFindI("Pipeline-Depth", 10);
 575          Debug = DebugEnabled();
 576       }
 577
 578       /* If the server has explicitly said this is the last connection
 579          then we pre-emptively shut down the pipeline and tear down
 580          the connection. This will speed up HTTP/1.0 servers a tad
 581          since we don't have to wait for the close sequence to
 582          complete */
 583       if (Server->Persistent == false)
 584          Server->Close();
 585
 586       // Reset the pipeline
 587       if (Server->IsOpen() == false)
 588          QueueBack = Queue;
 589
 590       // Connnect to the host
 591       if (Server->Open() == false)
 592       {
 593          Fail(true);
 594          Server = nullptr;
 595          continue;
 596       }
 597
 598       // Fill the pipeline.
 599       Fetch(0);
 600
 601       // Fetch the next URL header data from the server.
 602       switch (Server->RunHeaders(File, Queue->Uri))
 603       {
 604          case ServerState::RUN_HEADERS_OK:
 605          break;
 606
 607          // The header data is bad
 608          case ServerState::RUN_HEADERS_PARSE_ERROR:
 609          {
 610             _error->Error(_("Bad header data"));
 611             Fail(true);
 612             Server->Close();
 613             RotateDNS();
 614             continue;
 615          }
 616
 617          // The server closed a connection during the header get..
 618          default:
 619          case ServerState::RUN_HEADERS_IO_ERROR:
 620          {
 621             FailCounter++;
 622             _error->Discard();
 623             Server->Close();
 624             Server->Pipeline = false;
 625             Server->PipelineAllowed = false;
 626
 627             if (FailCounter >= 2)
 628             {
 629                Fail(_("Connection failed"),true);
 630                FailCounter = 0;
 631             }
 632
 633             RotateDNS();
 634             continue;
 635          }
 636       };
 637
 638       // Decide what to do.
 639       FetchResult Res;
 640       Res.Filename = Queue->DestFile;
 641       switch (DealWithHeaders(Res))
 642       {
 643          // Ok, the file is Open
 644          case FILE_IS_OPEN:
 645          {
 646             URIStart(Res);
 647
 648             // Run the data
 649             bool Result = true;
 650
 651             // ensure we don't fetch too much
 652             // we could do "Server->MaximumSize = Queue->MaximumSize" here
 653             // but that would break the clever pipeline messup detection
 654             // so instead we use the size of the biggest item in the queue
 655             Server->MaximumSize = FindMaximumObjectSizeInQueue();
 656
 657             if (Server->HaveContent)
 658                Result = Server->RunData(File);
 659
 660             /* If the server is sending back sizeless responses then fill in
 661                the size now */
 662             if (Res.Size == 0)
 663                Res.Size = File->Size();
 664
 665             // Close the file, destroy the FD object and timestamp it
 666             FailFd = -1;
 667             delete File;
 668             File = 0;
 669
 670             // Timestamp
 671             struct timeval times[2];
 672             times[0].tv_sec = times[1].tv_sec = Server->Date;
 673             times[0].tv_usec = times[1].tv_usec = 0;
 674             utimes(Queue->DestFile.c_str(), times);
 675
 676             // Send status to APT
 677             if (Result == true)
 678             {
 679                Hashes * const resultHashes = Server->GetHashes();
 680                HashStringList const hashList = resultHashes->GetHashStringList();
 681                if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList)
 682                {
 683                   // we did not get the expected hash… mhhh:
 684                   // could it be that server/proxy messed up pipelining?
 685                   FetchItem * BeforeI = Queue;
 686                   for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next)
 687                   {
 688                      if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList)
 689                      {
 690                         // yes, he did! Disable pipelining and rewrite queue
 691                         if (Server->Pipeline == true)
 692                         {
 693                            Warning(_("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::Pipeline-Depth");
 694                            Server->Pipeline = false;
 695                            Server->PipelineAllowed = false;
 696                            // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well
 697                         }
 698                         Rename(Res.Filename, I->DestFile);
 699                         Res.Filename = I->DestFile;
 700                         BeforeI->Next = I->Next;
 701                         I->Next = Queue;
 702                         Queue = I;
 703                         break;
 704                      }
 705                      BeforeI = I;
 706                   }
 707                }
 708                Res.TakeHashes(*resultHashes);
 709                URIDone(Res);
 710             }
 711             else
 712             {
 713                if (Server->IsOpen() == false)
 714                {
 715                   FailCounter++;
 716                   _error->Discard();
 717                   Server->Close();
 718
 719                   if (FailCounter >= 2)
 720                   {
 721                      Fail(_("Connection failed"),true);
 722                      FailCounter = 0;
 723                   }
 724
 725                   QueueBack = Queue;
 726                }
 727                else
 728                {
 729                   Server->Close();
 730                   Fail(true);
 731                }
 732             }
 733             break;
 734          }
 735
 736          // IMS hit
 737          case IMS_HIT:
 738          {
 739             URIDone(Res);
 740             break;
 741          }
 742
 743          // Hard server error, not found or something
 744          case ERROR_UNRECOVERABLE:
 745          {
 746             Fail();
 747             break;
 748          }
 749
 750          // Hard internal error, kill the connection and fail
 751          case ERROR_NOT_FROM_SERVER:
 752          {
 753             delete File;
 754             File = 0;
 755
 756             Fail();
 757             RotateDNS();
 758             Server->Close();
 759             break;
 760          }
 761
 762          // We need to flush the data, the header is like a 404 w/ error text
 763          case ERROR_WITH_CONTENT_PAGE:
 764          {
 765             Fail();
 766             Server->RunDataToDevNull();
 767             break;
 768          }
 769
 770          // Try again with a new URL
 771          case TRY_AGAIN_OR_REDIRECT:
 772          {
 773             // Clear rest of response if there is content
 774             if (Server->HaveContent)
 775                Server->RunDataToDevNull();
 776             Redirect(NextURI);
 777             break;
 778          }
 779
 780          default:
 781          Fail(_("Internal error"));
 782          break;
 783       }
 784
 785       FailCounter = 0;
 786    }
 787
 788    return 0;
 789 }
 790                                                                         /*}}}*/
 791 unsigned long long ServerMethod::FindMaximumObjectSizeInQueue() const   /*{{{*/
 792 {
 793    unsigned long long MaxSizeInQueue = 0;
 794    for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next)
 795       MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize);
 796    return MaxSizeInQueue;
 797 }
 798                                                                         /*}}}*/
 799 ServerMethod::ServerMethod(std::string &&Binary, char const * const Ver,unsigned long const Flags) :/*{{{*/
 800    aptMethod(std::move(Binary), Ver, Flags), Server(nullptr), File(NULL), PipelineDepth(10),
 801    AllowRedirect(false), Debug(false)
 802 {
 803 }
 804                                                                         /*}}}*/
 805 bool ServerMethod::Configuration(std::string Message)                   /*{{{*/
 806 {
 807    if (aptMethod::Configuration(Message) == false)
 808       return false;
 809
 810    _config->CndSet("Acquire::tor::Proxy",
 811          "socks5h://apt-transport-tor@localhost:9050");
 812    return true;
 813 }
 814                                                                         /*}}}*/
 815 bool ServerMethod::AddProxyAuth(URI &Proxy, URI const &Server) const    /*{{{*/
 816 {
 817    if (std::find(methodNames.begin(), methodNames.end(), "tor") != methodNames.end() &&
 818          Proxy.User == "apt-transport-tor" && Proxy.Password.empty())
 819    {
 820       std::string pass = Server.Host;
 821       pass.erase(std::remove_if(pass.begin(), pass.end(), [](char const c) { return std::isalnum(c) == 0; }), pass.end());
 822       if (pass.length() > 255)
 823          Proxy.Password = pass.substr(0, 255);
 824       else
 825          Proxy.Password = std::move(pass);
 826    }
 827    // FIXME: should we support auth.conf for proxies?
 828    return true;
 829 }
 830                                                                         /*}}}*/