apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/tagfile.h>
  26 #include <apt-pkg/metaindex.h>
  27 #include <apt-pkg/acquire.h>
  28 #include <apt-pkg/hashes.h>
  29 #include <apt-pkg/indexfile.h>
  30 #include <apt-pkg/pkgcache.h>
  31 #include <apt-pkg/cacheiterators.h>
  32 #include <apt-pkg/pkgrecords.h>
  33 #include <apt-pkg/gpgv.h>
  34
  35 #include <algorithm>
  36 #include <stddef.h>
  37 #include <stdlib.h>
  38 #include <string.h>
  39 #include <iostream>
  40 #include <vector>
  41 #include <sys/stat.h>
  42 #include <unistd.h>
  43 #include <errno.h>
  44 #include <string>
  45 #include <stdio.h>
  46 #include <ctime>
  47 #include <sstream>
  48 #include <numeric>
  49
  50 #include <apti18n.h>
  51                                                                         /*}}}*/
  52
  53 using namespace std;
  54
  55 static void printHashSumComparison(std::string const &URI, HashStringList const &Expected, HashStringList const &Actual) /*{{{*/
  56 {
  57    if (_config->FindB("Debug::Acquire::HashSumMismatch", false) == false)
  58       return;
  59    std::cerr << std::endl << URI << ":" << std::endl << " Expected Hash: " << std::endl;
  60    for (HashStringList::const_iterator hs = Expected.begin(); hs != Expected.end(); ++hs)
  61       std::cerr <<  "\t- " << hs->toStr() << std::endl;
  62    std::cerr << " Actual Hash: " << std::endl;
  63    for (HashStringList::const_iterator hs = Actual.begin(); hs != Actual.end(); ++hs)
  64       std::cerr <<  "\t- " << hs->toStr() << std::endl;
  65 }
  66                                                                         /*}}}*/
  67 static std::string GetPartialFileName(std::string const &file)          /*{{{*/
  68 {
  69    std::string DestFile = _config->FindDir("Dir::State::lists") + "partial/";
  70    DestFile += file;
  71    return DestFile;
  72 }
  73                                                                         /*}}}*/
  74 static std::string GetPartialFileNameFromURI(std::string const &uri)    /*{{{*/
  75 {
  76    return GetPartialFileName(URItoFileName(uri));
  77 }
  78                                                                         /*}}}*/
  79 static std::string GetFinalFileNameFromURI(std::string const &uri)      /*{{{*/
  80 {
  81    return _config->FindDir("Dir::State::lists") + URItoFileName(uri);
  82 }
  83                                                                         /*}}}*/
  84 static std::string GetKeepCompressedFileName(std::string file, IndexTarget const &Target)/*{{{*/
  85 {
  86    if (Target.KeepCompressed == false)
  87       return file;
  88
  89    std::string const KeepCompressedAs = Target.Option(IndexTarget::KEEPCOMPRESSEDAS);
  90    if (KeepCompressedAs.empty() == false)
  91    {
  92       std::string const ext = KeepCompressedAs.substr(0, KeepCompressedAs.find(' '));
  93       if (ext != "uncompressed")
  94          file.append(".").append(ext);
  95    }
  96    return file;
  97 }
  98                                                                         /*}}}*/
  99 static std::string GetMergeDiffsPatchFileName(std::string const &Final, std::string const &Patch)/*{{{*/
 100 {
 101    // rred expects the patch as $FinalFile.ed.$patchname.gz
 102    return Final + ".ed." + Patch + ".gz";
 103 }
 104                                                                         /*}}}*/
 105 static std::string GetDiffsPatchFileName(std::string const &Final)      /*{{{*/
 106 {
 107    // rred expects the patch as $FinalFile.ed
 108    return Final + ".ed";
 109 }
 110                                                                         /*}}}*/
 111 static std::string GetExistingFilename(std::string const &File)         /*{{{*/
 112 {
 113    if (RealFileExists(File))
 114       return File;
 115    for (auto const &type : APT::Configuration::getCompressorExtensions())
 116    {
 117       std::string const Final = File + type;
 118       if (RealFileExists(Final))
 119          return Final;
 120    }
 121    return "";
 122 }
 123                                                                         /*}}}*/
 124 static std::string GetDiffIndexFileName(std::string const &Name)        /*{{{*/
 125 {
 126    return Name + ".diff/Index";
 127 }
 128                                                                         /*}}}*/
 129 static std::string GetDiffIndexURI(IndexTarget const &Target)           /*{{{*/
 130 {
 131    return Target.URI + ".diff/Index";
 132 }
 133                                                                         /*}}}*/
 134
 135 static void ReportMirrorFailureToCentral(pkgAcquire::Item const &I, std::string const &FailCode, std::string const &Details)/*{{{*/
 136 {
 137    // we only act if a mirror was used at all
 138    if(I.UsedMirror.empty())
 139       return;
 140 #if 0
 141    std::cerr << "\nReportMirrorFailure: "
 142              << UsedMirror
 143              << " Uri: " << DescURI()
 144              << " FailCode: "
 145              << FailCode << std::endl;
 146 #endif
 147    string const report = _config->Find("Methods::Mirror::ProblemReporting",
 148                                  "/usr/lib/apt/apt-report-mirror-failure");
 149    if(!FileExists(report))
 150       return;
 151
 152    std::vector<char const*> const Args = {
 153       report.c_str(),
 154       I.UsedMirror.c_str(),
 155       I.DescURI().c_str(),
 156       FailCode.c_str(),
 157       Details.c_str(),
 158       NULL
 159    };
 160
 161    pid_t pid = ExecFork();
 162    if(pid < 0)
 163    {
 164       _error->Error("ReportMirrorFailure Fork failed");
 165       return;
 166    }
 167    else if(pid == 0)
 168    {
 169       execvp(Args[0], (char**)Args.data());
 170       std::cerr << "Could not exec " << Args[0] << std::endl;
 171       _exit(100);
 172    }
 173    if(!ExecWait(pid, "report-mirror-failure"))
 174       _error->Warning("Couldn't report problem to '%s'", report.c_str());
 175 }
 176                                                                         /*}}}*/
 177
 178 static APT_NONNULL(2) bool MessageInsecureRepository(bool const isError, char const * const msg, std::string const &repo)/*{{{*/
 179 {
 180    std::string m;
 181    strprintf(m, msg, repo.c_str());
 182    if (isError)
 183    {
 184       _error->Error("%s", m.c_str());
 185       _error->Notice("%s", _("Updating from such a repository can't be done securely, and is therefore disabled by default."));
 186    }
 187    else
 188    {
 189       _error->Warning("%s", m.c_str());
 190       _error->Notice("%s", _("Data from such a repository can't be authenticated and is therefore potentially dangerous to use."));
 191    }
 192    _error->Notice("%s", _("See apt-secure(8) manpage for repository creation and user configuration details."));
 193    return false;
 194 }
 195                                                                         /*}}}*/
 196 // AllowInsecureRepositories                                            /*{{{*/
 197 enum class InsecureType { UNSIGNED, WEAK, NORELEASE };
 198 static bool TargetIsAllowedToBe(IndexTarget const &Target, InsecureType const type)
 199 {
 200    if (_config->FindB("Acquire::AllowInsecureRepositories"))
 201       return true;
 202
 203    if (Target.OptionBool(IndexTarget::ALLOW_INSECURE))
 204       return true;
 205
 206    switch (type)
 207    {
 208       case InsecureType::UNSIGNED: break;
 209       case InsecureType::NORELEASE: break;
 210       case InsecureType::WEAK:
 211          if (_config->FindB("Acquire::AllowWeakRepositories"))
 212             return true;
 213          if (Target.OptionBool(IndexTarget::ALLOW_WEAK))
 214             return true;
 215          break;
 216    }
 217    return false;
 218 }
 219 static bool APT_NONNULL(3, 4, 5) AllowInsecureRepositories(InsecureType const msg, std::string const &repo,
 220       metaIndex const * const MetaIndexParser, pkgAcqMetaClearSig * const TransactionManager, pkgAcquire::Item * const I)
 221 {
 222    // we skip weak downgrades as its unlikely that a repository gets really weaker –
 223    // its more realistic that apt got pickier in a newer version
 224    if (msg != InsecureType::WEAK)
 225    {
 226       std::string const FinalInRelease = TransactionManager->GetFinalFilename();
 227       std::string const FinalReleasegpg = FinalInRelease.substr(0, FinalInRelease.length() - strlen("InRelease")) + "Release.gpg";
 228       if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease))
 229       {
 230          char const * msgstr = nullptr;
 231          switch (msg)
 232          {
 233             case InsecureType::UNSIGNED: msgstr = _("The repository '%s' is no longer signed."); break;
 234             case InsecureType::NORELEASE: msgstr = _("The repository '%s' does no longer have a Release file."); break;
 235             case InsecureType::WEAK: /* unreachable */ break;
 236          }
 237          if (_config->FindB("Acquire::AllowDowngradeToInsecureRepositories") ||
 238                TransactionManager->Target.OptionBool(IndexTarget::ALLOW_DOWNGRADE_TO_INSECURE))
 239          {
 240             // meh, the users wants to take risks (we still mark the packages
 241             // from this repository as unauthenticated)
 242             _error->Warning(msgstr, repo.c_str());
 243             _error->Warning(_("This is normally not allowed, but the option "
 244                      "Acquire::AllowDowngradeToInsecureRepositories was "
 245                      "given to override it."));
 246          } else {
 247             MessageInsecureRepository(true, msgstr, repo);
 248             TransactionManager->AbortTransaction();
 249             I->Status = pkgAcquire::Item::StatError;
 250             return false;
 251          }
 252       }
 253    }
 254
 255    if(MetaIndexParser->GetTrusted() == metaIndex::TRI_YES)
 256       return true;
 257
 258    char const * msgstr = nullptr;
 259    switch (msg)
 260    {
 261       case InsecureType::UNSIGNED: msgstr = _("The repository '%s' is not signed."); break;
 262       case InsecureType::NORELEASE: msgstr = _("The repository '%s' does not have a Release file."); break;
 263       case InsecureType::WEAK: msgstr = _("The repository '%s' provides only weak security information."); break;
 264    }
 265
 266    if (TargetIsAllowedToBe(TransactionManager->Target, msg) == true)
 267    {
 268       MessageInsecureRepository(false, msgstr, repo);
 269       return true;
 270    }
 271
 272    MessageInsecureRepository(true, msgstr, repo);
 273    TransactionManager->AbortTransaction();
 274    I->Status = pkgAcquire::Item::StatError;
 275    return false;
 276 }
 277                                                                         /*}}}*/
 278 static HashStringList GetExpectedHashesFromFor(metaIndex * const Parser, std::string const &MetaKey)/*{{{*/
 279 {
 280    if (Parser == NULL)
 281       return HashStringList();
 282    metaIndex::checkSum * const R = Parser->Lookup(MetaKey);
 283    if (R == NULL)
 284       return HashStringList();
 285    return R->Hashes;
 286 }
 287                                                                         /*}}}*/
 288
 289 // all ::HashesRequired and ::GetExpectedHashes implementations         /*{{{*/
 290 /* ::GetExpectedHashes is abstract and has to be implemented by all subclasses.
 291    It is best to implement it as broadly as possible, while ::HashesRequired defaults
 292    to true and should be as restrictive as possible for false cases. Note that if
 293    a hash is returned by ::GetExpectedHashes it must match. Only if it doesn't
 294    ::HashesRequired is called to evaluate if its okay to have no hashes. */
 295 APT_CONST bool pkgAcqTransactionItem::HashesRequired() const
 296 {
 297    /* signed repositories obviously have a parser and good hashes.
 298       unsigned repositories, too, as even if we can't trust them for security,
 299       we can at least trust them for integrity of the download itself.
 300       Only repositories without a Release file can (obviously) not have
 301       hashes – and they are very uncommon and strongly discouraged */
 302    if (TransactionManager->MetaIndexParser->GetLoadedSuccessfully() != metaIndex::TRI_YES)
 303       return false;
 304    if (TargetIsAllowedToBe(Target, InsecureType::WEAK))
 305    {
 306       /* If we allow weak hashes, we check that we have some (weak) and then
 307          declare hashes not needed. That will tip us in the right direction
 308          as if hashes exist, they will be used, even if not required */
 309       auto const hsl = GetExpectedHashes();
 310       if (hsl.usable())
 311          return true;
 312       if (hsl.empty() == false)
 313          return false;
 314    }
 315    return true;
 316 }
 317 HashStringList pkgAcqTransactionItem::GetExpectedHashes() const
 318 {
 319    return GetExpectedHashesFor(GetMetaKey());
 320 }
 321
 322 APT_CONST bool pkgAcqMetaBase::HashesRequired() const
 323 {
 324    // Release and co have no hashes 'by design'.
 325    return false;
 326 }
 327 HashStringList pkgAcqMetaBase::GetExpectedHashes() const
 328 {
 329    return HashStringList();
 330 }
 331
 332 APT_CONST bool pkgAcqIndexDiffs::HashesRequired() const
 333 {
 334    /* We can't check hashes of rred result as we don't know what the
 335       hash of the file will be. We just know the hash of the patch(es),
 336       the hash of the file they will apply on and the hash of the resulting
 337       file. */
 338    if (State == StateFetchDiff)
 339       return true;
 340    return false;
 341 }
 342 HashStringList pkgAcqIndexDiffs::GetExpectedHashes() const
 343 {
 344    if (State == StateFetchDiff)
 345       return available_patches[0].download_hashes;
 346    return HashStringList();
 347 }
 348
 349 APT_CONST bool pkgAcqIndexMergeDiffs::HashesRequired() const
 350 {
 351    /* @see #pkgAcqIndexDiffs::HashesRequired, with the difference that
 352       we can check the rred result after all patches are applied as
 353       we know the expected result rather than potentially apply more patches */
 354    if (State == StateFetchDiff)
 355       return true;
 356    return State == StateApplyDiff;
 357 }
 358 HashStringList pkgAcqIndexMergeDiffs::GetExpectedHashes() const
 359 {
 360    if (State == StateFetchDiff)
 361       return patch.download_hashes;
 362    else if (State == StateApplyDiff)
 363       return GetExpectedHashesFor(Target.MetaKey);
 364    return HashStringList();
 365 }
 366
 367 APT_CONST bool pkgAcqArchive::HashesRequired() const
 368 {
 369    return LocalSource == false;
 370 }
 371 HashStringList pkgAcqArchive::GetExpectedHashes() const
 372 {
 373    // figured out while parsing the records
 374    return ExpectedHashes;
 375 }
 376
 377 APT_CONST bool pkgAcqFile::HashesRequired() const
 378 {
 379    // supplied as parameter at creation time, so the caller decides
 380    return ExpectedHashes.usable();
 381 }
 382 HashStringList pkgAcqFile::GetExpectedHashes() const
 383 {
 384    return ExpectedHashes;
 385 }
 386                                                                         /*}}}*/
 387 // Acquire::Item::QueueURI and specialisations from child classes       /*{{{*/
 388 bool pkgAcquire::Item::QueueURI(pkgAcquire::ItemDesc &Item)
 389 {
 390    Owner->Enqueue(Item);
 391    return true;
 392 }
 393 /* The idea here is that an item isn't queued if it exists on disk and the
 394    transition manager was a hit as this means that the files it contains
 395    the checksums for can't be updated either (or they are and we are asking
 396    for a hashsum mismatch to happen which helps nobody) */
 397 bool pkgAcqTransactionItem::QueueURI(pkgAcquire::ItemDesc &Item)
 398 {
 399    if (TransactionManager->State != TransactionStarted)
 400    {
 401       if (_config->FindB("Debug::Acquire::Transaction", false))
 402          std::clog << "Skip " << Target.URI << " as transaction was already dealt with!" << std::endl;
 403       return false;
 404    }
 405    std::string const FinalFile = GetFinalFilename();
 406    if (TransactionManager->IMSHit == true && FileExists(FinalFile) == true)
 407    {
 408       PartialFile = DestFile = FinalFile;
 409       Status = StatDone;
 410       return false;
 411    }
 412    // If we got the InRelease file via a mirror, pick all indexes directly from this mirror, too
 413    if (TransactionManager->BaseURI.empty() == false &&
 414          URI::SiteOnly(Item.URI) != URI::SiteOnly(TransactionManager->BaseURI))
 415    {
 416       // this ensures we rewrite only once and only the first step
 417       auto const OldBaseURI = Target.Option(IndexTarget::BASE_URI);
 418       if (OldBaseURI.empty() == false && APT::String::Startswith(Item.URI, OldBaseURI))
 419       {
 420          auto const ExtraPath = Item.URI.substr(OldBaseURI.length());
 421          Item.URI = flCombine(TransactionManager->BaseURI, ExtraPath);
 422          UsedMirror = TransactionManager->UsedMirror;
 423          if (Item.Description.find(" ") != string::npos)
 424             Item.Description.replace(0, Item.Description.find(" "), UsedMirror);
 425       }
 426    }
 427    return pkgAcquire::Item::QueueURI(Item);
 428 }
 429 /* The transition manager InRelease itself (or its older sisters-in-law
 430    Release & Release.gpg) is always queued as this allows us to rerun gpgv
 431    on it to verify that we aren't stalled with old files */
 432 bool pkgAcqMetaBase::QueueURI(pkgAcquire::ItemDesc &Item)
 433 {
 434    return pkgAcquire::Item::QueueURI(Item);
 435 }
 436 /* the Diff/Index needs to queue also the up-to-date complete index file
 437    to ensure that the list cleaner isn't eating it */
 438 bool pkgAcqDiffIndex::QueueURI(pkgAcquire::ItemDesc &Item)
 439 {
 440    if (pkgAcqTransactionItem::QueueURI(Item) == true)
 441       return true;
 442    QueueOnIMSHit();
 443    return false;
 444 }
 445                                                                         /*}}}*/
 446 // Acquire::Item::GetFinalFilename and specialisations for child classes        /*{{{*/
 447 std::string pkgAcquire::Item::GetFinalFilename() const
 448 {
 449    // Beware: Desc.URI is modified by redirections
 450    return GetFinalFileNameFromURI(Desc.URI);
 451 }
 452 std::string pkgAcqDiffIndex::GetFinalFilename() const
 453 {
 454    return GetFinalFileNameFromURI(GetDiffIndexURI(Target));
 455 }
 456 std::string pkgAcqIndex::GetFinalFilename() const
 457 {
 458    std::string const FinalFile = GetFinalFileNameFromURI(Target.URI);
 459    return GetKeepCompressedFileName(FinalFile, Target);
 460 }
 461 std::string pkgAcqMetaSig::GetFinalFilename() const
 462 {
 463    return GetFinalFileNameFromURI(Target.URI);
 464 }
 465 std::string pkgAcqBaseIndex::GetFinalFilename() const
 466 {
 467    return GetFinalFileNameFromURI(Target.URI);
 468 }
 469 std::string pkgAcqMetaBase::GetFinalFilename() const
 470 {
 471    return GetFinalFileNameFromURI(Target.URI);
 472 }
 473 std::string pkgAcqArchive::GetFinalFilename() const
 474 {
 475    return _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
 476 }
 477                                                                         /*}}}*/
 478 // pkgAcqTransactionItem::GetMetaKey and specialisations for child classes      /*{{{*/
 479 std::string pkgAcqTransactionItem::GetMetaKey() const
 480 {
 481    return Target.MetaKey;
 482 }
 483 std::string pkgAcqIndex::GetMetaKey() const
 484 {
 485    if (Stage == STAGE_DECOMPRESS_AND_VERIFY || CurrentCompressionExtension == "uncompressed")
 486       return Target.MetaKey;
 487    return Target.MetaKey + "." + CurrentCompressionExtension;
 488 }
 489 std::string pkgAcqDiffIndex::GetMetaKey() const
 490 {
 491    return GetDiffIndexFileName(Target.MetaKey);
 492 }
 493                                                                         /*}}}*/
 494 //pkgAcqTransactionItem::TransactionState and specialisations for child classes /*{{{*/
 495 bool pkgAcqTransactionItem::TransactionState(TransactionStates const state)
 496 {
 497    bool const Debug = _config->FindB("Debug::Acquire::Transaction", false);
 498    switch(state)
 499    {
 500       case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break;
 501       case TransactionAbort:
 502          if(Debug == true)
 503             std::clog << "  Cancel: " << DestFile << std::endl;
 504          if (Status == pkgAcquire::Item::StatIdle)
 505          {
 506             Status = pkgAcquire::Item::StatDone;
 507             Dequeue();
 508          }
 509          break;
 510       case TransactionCommit:
 511          if(PartialFile.empty() == false)
 512          {
 513             bool sameFile = (PartialFile == DestFile);
 514             // we use symlinks on IMS-Hit to avoid copies
 515             if (RealFileExists(DestFile))
 516             {
 517                struct stat Buf;
 518                if (lstat(PartialFile.c_str(), &Buf) != -1)
 519                {
 520                   if (S_ISLNK(Buf.st_mode) && Buf.st_size > 0)
 521                   {
 522                      char partial[Buf.st_size + 1];
 523                      ssize_t const sp = readlink(PartialFile.c_str(), partial, Buf.st_size);
 524                      if (sp == -1)
 525                         _error->Errno("pkgAcqTransactionItem::TransactionState-sp", _("Failed to readlink %s"), PartialFile.c_str());
 526                      else
 527                      {
 528                         partial[sp] = '\0';
 529                         sameFile = (DestFile == partial);
 530                      }
 531                   }
 532                }
 533                else
 534                   _error->Errno("pkgAcqTransactionItem::TransactionState-stat", _("Failed to stat %s"), PartialFile.c_str());
 535             }
 536             if (sameFile == false)
 537             {
 538                // ensure that even without lists-cleanup all compressions are nuked
 539                std::string FinalFile = GetFinalFileNameFromURI(Target.URI);
 540                if (FileExists(FinalFile))
 541                {
 542                   if(Debug == true)
 543                      std::clog << "rm " << FinalFile << " # " << DescURI() << std::endl;
 544                   if (RemoveFile("TransactionStates-Cleanup", FinalFile) == false)
 545                      return false;
 546                }
 547                for (auto const &ext: APT::Configuration::getCompressorExtensions())
 548                {
 549                   auto const Final = FinalFile + ext;
 550                   if (FileExists(Final))
 551                   {
 552                      if(Debug == true)
 553                         std::clog << "rm " << Final << " # " << DescURI() << std::endl;
 554                      if (RemoveFile("TransactionStates-Cleanup", Final) == false)
 555                         return false;
 556                   }
 557                }
 558                if(Debug == true)
 559                   std::clog << "mv " << PartialFile << " -> "<< DestFile << " # " << DescURI() << std::endl;
 560                if (Rename(PartialFile, DestFile) == false)
 561                   return false;
 562             }
 563             else if(Debug == true)
 564                std::clog << "keep " << PartialFile << " # " << DescURI() << std::endl;
 565
 566          } else {
 567             if(Debug == true)
 568                std::clog << "rm " << DestFile << " # " << DescURI() << std::endl;
 569             if (RemoveFile("TransItem::TransactionCommit", DestFile) == false)
 570                return false;
 571          }
 572          break;
 573    }
 574    return true;
 575 }
 576 bool pkgAcqMetaBase::TransactionState(TransactionStates const state)
 577 {
 578    // Do not remove InRelease on IMSHit of Release.gpg [yes, this is very edgecasey]
 579    if (TransactionManager->IMSHit == false)
 580       return pkgAcqTransactionItem::TransactionState(state);
 581    return true;
 582 }
 583 bool pkgAcqIndex::TransactionState(TransactionStates const state)
 584 {
 585    if (pkgAcqTransactionItem::TransactionState(state) == false)
 586       return false;
 587
 588    switch (state)
 589    {
 590       case TransactionStarted: _error->Fatal("AcqIndex %s changed to invalid transaction start state!", Target.URI.c_str()); break;
 591       case TransactionAbort:
 592          if (Stage == STAGE_DECOMPRESS_AND_VERIFY)
 593          {
 594             // keep the compressed file, but drop the decompressed
 595             EraseFileName.clear();
 596             if (PartialFile.empty() == false && flExtension(PartialFile) != CurrentCompressionExtension)
 597                RemoveFile("TransactionAbort", PartialFile);
 598          }
 599          break;
 600       case TransactionCommit:
 601          if (EraseFileName.empty() == false)
 602             RemoveFile("AcqIndex::TransactionCommit", EraseFileName);
 603          break;
 604    }
 605    return true;
 606 }
 607 bool pkgAcqDiffIndex::TransactionState(TransactionStates const state)
 608 {
 609    if (pkgAcqTransactionItem::TransactionState(state) == false)
 610       return false;
 611
 612    switch (state)
 613    {
 614       case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break;
 615       case TransactionCommit:
 616          break;
 617       case TransactionAbort:
 618          std::string const Partial = GetPartialFileNameFromURI(Target.URI);
 619          RemoveFile("TransactionAbort", Partial);
 620          break;
 621    }
 622
 623    return true;
 624 }
 625                                                                         /*}}}*/
 626
 627 class APT_HIDDEN NoActionItem : public pkgAcquire::Item                 /*{{{*/
 628 /* The sole purpose of this class is having an item which does nothing to
 629    reach its done state to prevent cleanup deleting the mentioned file.
 630    Handy in cases in which we know we have the file already, like IMS-Hits. */
 631 {
 632    IndexTarget const Target;
 633    public:
 634    virtual std::string DescURI() const APT_OVERRIDE {return Target.URI;};
 635    virtual HashStringList GetExpectedHashes()  const APT_OVERRIDE {return HashStringList();};
 636
 637    NoActionItem(pkgAcquire * const Owner, IndexTarget const &Target) :
 638       pkgAcquire::Item(Owner), Target(Target)
 639    {
 640       Status = StatDone;
 641       DestFile = GetFinalFileNameFromURI(Target.URI);
 642    }
 643    NoActionItem(pkgAcquire * const Owner, IndexTarget const &Target, std::string const &FinalFile) :
 644       pkgAcquire::Item(Owner), Target(Target)
 645    {
 646       Status = StatDone;
 647       DestFile = FinalFile;
 648    }
 649 };
 650                                                                         /*}}}*/
 651 class APT_HIDDEN CleanupItem : public pkgAcqTransactionItem             /*{{{*/
 652 /* This class ensures that a file which was configured but isn't downloaded
 653    for various reasons isn't kept in an old version in the lists directory.
 654    In a way its the reverse of NoActionItem as it helps with removing files
 655    even if the lists-cleanup is deactivated. */
 656 {
 657    public:
 658    virtual std::string DescURI() const APT_OVERRIDE {return Target.URI;};
 659    virtual HashStringList GetExpectedHashes()  const APT_OVERRIDE {return HashStringList();};
 660
 661    CleanupItem(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target) :
 662       pkgAcqTransactionItem(Owner, TransactionManager, Target)
 663    {
 664       Status = StatDone;
 665       DestFile = GetFinalFileNameFromURI(Target.URI);
 666    }
 667    bool TransactionState(TransactionStates const state) APT_OVERRIDE
 668    {
 669       switch (state)
 670       {
 671          case TransactionStarted:
 672             break;
 673          case TransactionAbort:
 674             break;
 675          case TransactionCommit:
 676             if (_config->FindB("Debug::Acquire::Transaction", false) == true)
 677                std::clog << "rm " << DestFile << " # " << DescURI() << std::endl;
 678             if (RemoveFile("TransItem::TransactionCommit", DestFile) == false)
 679                return false;
 680             break;
 681       }
 682       return true;
 683    }
 684 };
 685                                                                         /*}}}*/
 686
 687 // Acquire::Item::Item - Constructor                                    /*{{{*/
 688 APT_IGNORE_DEPRECATED_PUSH
 689 pkgAcquire::Item::Item(pkgAcquire * const owner) :
 690    FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false), Local(false),
 691     QueueCounter(0), ExpectedAdditionalItems(0), Owner(owner), d(NULL)
 692 {
 693    Owner->Add(this);
 694    Status = StatIdle;
 695 }
 696 APT_IGNORE_DEPRECATED_POP
 697                                                                         /*}}}*/
 698 // Acquire::Item::~Item - Destructor                                    /*{{{*/
 699 pkgAcquire::Item::~Item()
 700 {
 701    Owner->Remove(this);
 702 }
 703                                                                         /*}}}*/
 704 std::string pkgAcquire::Item::Custom600Headers() const                  /*{{{*/
 705 {
 706    return std::string();
 707 }
 708                                                                         /*}}}*/
 709 std::string pkgAcquire::Item::ShortDesc() const                         /*{{{*/
 710 {
 711    return DescURI();
 712 }
 713                                                                         /*}}}*/
 714 APT_CONST void pkgAcquire::Item::Finished()                             /*{{{*/
 715 {
 716 }
 717                                                                         /*}}}*/
 718 APT_PURE pkgAcquire * pkgAcquire::Item::GetOwner() const                /*{{{*/
 719 {
 720    return Owner;
 721 }
 722                                                                         /*}}}*/
 723 APT_CONST pkgAcquire::ItemDesc &pkgAcquire::Item::GetItemDesc()         /*{{{*/
 724 {
 725    return Desc;
 726 }
 727                                                                         /*}}}*/
 728 APT_CONST bool pkgAcquire::Item::IsTrusted() const                      /*{{{*/
 729 {
 730    return false;
 731 }
 732                                                                         /*}}}*/
 733 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
 734 // ---------------------------------------------------------------------
 735 /* We return to an idle state if there are still other queues that could
 736    fetch this object */
 737 void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
 738 {
 739    if (QueueCounter <= 1)
 740    {
 741       /* This indicates that the file is not available right now but might
 742          be sometime later. If we do a retry cycle then this should be
 743          retried [CDROMs] */
 744       if (Cnf != NULL && Cnf->LocalOnly == true &&
 745           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
 746       {
 747          Status = StatIdle;
 748          Dequeue();
 749          return;
 750       }
 751
 752       switch (Status)
 753       {
 754          case StatIdle:
 755          case StatFetching:
 756          case StatDone:
 757             Status = StatError;
 758             break;
 759          case StatAuthError:
 760          case StatError:
 761          case StatTransientNetworkError:
 762             break;
 763       }
 764       Complete = false;
 765       Dequeue();
 766    }
 767
 768    string const FailReason = LookupTag(Message, "FailReason");
 769    enum { MAXIMUM_SIZE_EXCEEDED, HASHSUM_MISMATCH, WEAK_HASHSUMS, OTHER } failreason = OTHER;
 770    if ( FailReason == "MaximumSizeExceeded")
 771       failreason = MAXIMUM_SIZE_EXCEEDED;
 772    else if ( FailReason == "WeakHashSums")
 773       failreason = WEAK_HASHSUMS;
 774    else if (Status == StatAuthError)
 775       failreason = HASHSUM_MISMATCH;
 776
 777    if(ErrorText.empty())
 778    {
 779       if (Status == StatAuthError)
 780       {
 781          std::ostringstream out;
 782          switch (failreason)
 783          {
 784             case HASHSUM_MISMATCH:
 785                out << _("Hash Sum mismatch") << std::endl;
 786                break;
 787             case WEAK_HASHSUMS:
 788                out << _("Insufficient information available to perform this download securely") << std::endl;
 789                break;
 790             case MAXIMUM_SIZE_EXCEEDED:
 791             case OTHER:
 792                out << LookupTag(Message, "Message") << std::endl;
 793                break;
 794          }
 795          auto const ExpectedHashes = GetExpectedHashes();
 796          if (ExpectedHashes.empty() == false)
 797          {
 798             out << "Hashes of expected file:" << std::endl;
 799             for (auto const &hs: ExpectedHashes)
 800             {
 801                out << " - " << hs.toStr();
 802                if (hs.usable() == false)
 803                   out << " [weak]";
 804                out << std::endl;
 805             }
 806          }
 807          if (failreason == HASHSUM_MISMATCH)
 808          {
 809             out << "Hashes of received file:" << std::endl;
 810             for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
 811             {
 812                std::string const tagname = std::string(*type) + "-Hash";
 813                std::string const hashsum = LookupTag(Message, tagname.c_str());
 814                if (hashsum.empty() == false)
 815                {
 816                   auto const hs = HashString(*type, hashsum);
 817                   out << " - " << hs.toStr();
 818                   if (hs.usable() == false)
 819                      out << " [weak]";
 820                   out << std::endl;
 821                }
 822             }
 823             out << "Last modification reported: " << LookupTag(Message, "Last-Modified", "<none>") << std::endl;
 824          }
 825          ErrorText = out.str();
 826       }
 827       else
 828          ErrorText = LookupTag(Message,"Message");
 829    }
 830
 831    switch (failreason)
 832    {
 833       case MAXIMUM_SIZE_EXCEEDED: RenameOnError(MaximumSizeExceeded); break;
 834       case HASHSUM_MISMATCH: RenameOnError(HashSumMismatch); break;
 835       case WEAK_HASHSUMS: break;
 836       case OTHER: break;
 837    }
 838
 839    if (FailReason.empty() == false)
 840       ReportMirrorFailureToCentral(*this, FailReason, ErrorText);
 841    else
 842       ReportMirrorFailureToCentral(*this, ErrorText, ErrorText);
 843
 844    if (QueueCounter > 1)
 845       Status = StatIdle;
 846 }
 847                                                                         /*}}}*/
 848 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
 849 // ---------------------------------------------------------------------
 850 /* Stash status and the file size. Note that setting Complete means
 851    sub-phases of the acquire process such as decompresion are operating */
 852 void pkgAcquire::Item::Start(string const &/*Message*/, unsigned long long const Size)
 853 {
 854    Status = StatFetching;
 855    ErrorText.clear();
 856    if (FileSize == 0 && Complete == false)
 857       FileSize = Size;
 858 }
 859                                                                         /*}}}*/
 860 // Acquire::Item::VerifyDone - check if Item was downloaded OK          /*{{{*/
 861 /* Note that hash-verification is 'hardcoded' in acquire-worker and has
 862  * already passed if this method is called. */
 863 bool pkgAcquire::Item::VerifyDone(std::string const &Message,
 864          pkgAcquire::MethodConfig const * const /*Cnf*/)
 865 {
 866    std::string const FileName = LookupTag(Message,"Filename");
 867    if (FileName.empty() == true)
 868    {
 869       Status = StatError;
 870       ErrorText = "Method gave a blank filename";
 871       return false;
 872    }
 873
 874    return true;
 875 }
 876                                                                         /*}}}*/
 877 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 878 void pkgAcquire::Item::Done(string const &/*Message*/, HashStringList const &Hashes,
 879                             pkgAcquire::MethodConfig const * const /*Cnf*/)
 880 {
 881    // We just downloaded something..
 882    if (FileSize == 0)
 883    {
 884       unsigned long long const downloadedSize = Hashes.FileSize();
 885       if (downloadedSize != 0)
 886       {
 887          FileSize = downloadedSize;
 888       }
 889    }
 890    Status = StatDone;
 891    ErrorText = string();
 892    Owner->Dequeue(this);
 893 }
 894                                                                         /*}}}*/
 895 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 896 // ---------------------------------------------------------------------
 897 /* This helper function is used by a lot of item methods as their final
 898    step */
 899 bool pkgAcquire::Item::Rename(string const &From,string const &To)
 900 {
 901    if (From == To || rename(From.c_str(),To.c_str()) == 0)
 902       return true;
 903
 904    std::string S;
 905    strprintf(S, _("rename failed, %s (%s -> %s)."), strerror(errno),
 906          From.c_str(),To.c_str());
 907    Status = StatError;
 908    if (ErrorText.empty())
 909       ErrorText = S;
 910    else
 911       ErrorText = ErrorText + ": " + S;
 912    return false;
 913 }
 914                                                                         /*}}}*/
 915 void pkgAcquire::Item::Dequeue()                                        /*{{{*/
 916 {
 917    Owner->Dequeue(this);
 918 }
 919                                                                         /*}}}*/
 920 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 921 {
 922    if (RealFileExists(DestFile))
 923       Rename(DestFile, DestFile + ".FAILED");
 924
 925    std::string errtext;
 926    switch (error)
 927    {
 928       case HashSumMismatch:
 929          errtext = _("Hash Sum mismatch");
 930          break;
 931       case SizeMismatch:
 932          errtext = _("Size mismatch");
 933          Status = StatAuthError;
 934          break;
 935       case InvalidFormat:
 936          errtext = _("Invalid file format");
 937          Status = StatError;
 938          // do not report as usually its not the mirrors fault, but Portal/Proxy
 939          break;
 940       case SignatureError:
 941          errtext = _("Signature error");
 942          Status = StatError;
 943          break;
 944       case NotClearsigned:
 945          strprintf(errtext, _("Clearsigned file isn't valid, got '%s' (does the network require authentication?)"), "NOSPLIT");
 946          Status = StatAuthError;
 947          break;
 948       case MaximumSizeExceeded:
 949          // the method is expected to report a good error for this
 950          break;
 951       case PDiffError:
 952          // no handling here, done by callers
 953          break;
 954    }
 955    if (ErrorText.empty())
 956       ErrorText = errtext;
 957    return false;
 958 }
 959                                                                         /*}}}*/
 960 void pkgAcquire::Item::SetActiveSubprocess(const std::string &subprocess)/*{{{*/
 961 {
 962       ActiveSubprocess = subprocess;
 963       APT_IGNORE_DEPRECATED(Mode = ActiveSubprocess.c_str();)
 964 }
 965                                                                         /*}}}*/
 966 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 967 void pkgAcquire::Item::ReportMirrorFailure(std::string const &FailCode)
 968 {
 969    ReportMirrorFailureToCentral(*this, FailCode, FailCode);
 970 }
 971                                                                         /*}}}*/
 972 std::string pkgAcquire::Item::HashSum() const                           /*{{{*/
 973 {
 974    HashStringList const hashes = GetExpectedHashes();
 975    HashString const * const hs = hashes.find(NULL);
 976    return hs != NULL ? hs->toStr() : "";
 977 }
 978                                                                         /*}}}*/
 979
 980 pkgAcqTransactionItem::pkgAcqTransactionItem(pkgAcquire * const Owner,  /*{{{*/
 981       pkgAcqMetaClearSig * const transactionManager, IndexTarget const &target) :
 982    pkgAcquire::Item(Owner), d(NULL), Target(target), TransactionManager(transactionManager)
 983 {
 984    if (TransactionManager != this)
 985       TransactionManager->Add(this);
 986 }
 987                                                                         /*}}}*/
 988 pkgAcqTransactionItem::~pkgAcqTransactionItem()                         /*{{{*/
 989 {
 990 }
 991                                                                         /*}}}*/
 992 HashStringList pkgAcqTransactionItem::GetExpectedHashesFor(std::string const &MetaKey) const    /*{{{*/
 993 {
 994    return GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, MetaKey);
 995 }
 996                                                                         /*}}}*/
 997
 998 static void LoadLastMetaIndexParser(pkgAcqMetaClearSig * const TransactionManager, std::string const &FinalRelease, std::string const &FinalInRelease)/*{{{*/
 999 {
1000    if (TransactionManager->IMSHit == true)
1001       return;
1002    if (RealFileExists(FinalInRelease) || RealFileExists(FinalRelease))
1003    {
1004       TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone();
1005       if (TransactionManager->LastMetaIndexParser != NULL)
1006       {
1007          _error->PushToStack();
1008          if (RealFileExists(FinalInRelease))
1009             TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL);
1010          else
1011             TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL);
1012          // its unlikely to happen, but if what we have is bad ignore it
1013          if (_error->PendingError())
1014          {
1015             delete TransactionManager->LastMetaIndexParser;
1016             TransactionManager->LastMetaIndexParser = NULL;
1017          }
1018          _error->RevertToStack();
1019       }
1020    }
1021 }
1022                                                                         /*}}}*/
1023
1024 // AcqMetaBase - Constructor                                            /*{{{*/
1025 pkgAcqMetaBase::pkgAcqMetaBase(pkgAcquire * const Owner,
1026       pkgAcqMetaClearSig * const TransactionManager,
1027       IndexTarget const &DataTarget)
1028 : pkgAcqTransactionItem(Owner, TransactionManager, DataTarget), d(NULL),
1029    AuthPass(false), IMSHit(false), State(TransactionStarted)
1030 {
1031 }
1032                                                                         /*}}}*/
1033 // AcqMetaBase::Add - Add a item to the current Transaction             /*{{{*/
1034 void pkgAcqMetaBase::Add(pkgAcqTransactionItem * const I)
1035 {
1036    Transaction.push_back(I);
1037 }
1038                                                                         /*}}}*/
1039 // AcqMetaBase::AbortTransaction - Abort the current Transaction        /*{{{*/
1040 void pkgAcqMetaBase::AbortTransaction()
1041 {
1042    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1043       std::clog << "AbortTransaction: " << TransactionManager << std::endl;
1044
1045    switch (TransactionManager->State)
1046    {
1047       case TransactionStarted: break;
1048       case TransactionAbort: _error->Fatal("Transaction %s was already aborted and is aborted again", TransactionManager->Target.URI.c_str()); return;
1049       case TransactionCommit: _error->Fatal("Transaction %s was already aborted and is now committed", TransactionManager->Target.URI.c_str()); return;
1050    }
1051    TransactionManager->State = TransactionAbort;
1052
1053    // ensure the toplevel is in error state too
1054    for (std::vector<pkgAcqTransactionItem*>::iterator I = Transaction.begin();
1055         I != Transaction.end(); ++I)
1056    {
1057       if ((*I)->Status != pkgAcquire::Item::StatFetching)
1058          Owner->Dequeue(*I);
1059       (*I)->TransactionState(TransactionAbort);
1060    }
1061    Transaction.clear();
1062 }
1063                                                                         /*}}}*/
1064 // AcqMetaBase::TransactionHasError - Check for errors in Transaction   /*{{{*/
1065 APT_PURE bool pkgAcqMetaBase::TransactionHasError() const
1066 {
1067    for (std::vector<pkgAcqTransactionItem*>::const_iterator I = Transaction.begin();
1068         I != Transaction.end(); ++I)
1069    {
1070       switch((*I)->Status) {
1071          case StatDone: break;
1072          case StatIdle: break;
1073          case StatAuthError: return true;
1074          case StatError: return true;
1075          case StatTransientNetworkError: return true;
1076          case StatFetching: break;
1077       }
1078    }
1079    return false;
1080 }
1081                                                                         /*}}}*/
1082 // AcqMetaBase::CommitTransaction - Commit a transaction                /*{{{*/
1083 void pkgAcqMetaBase::CommitTransaction()
1084 {
1085    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1086       std::clog << "CommitTransaction: " << this << std::endl;
1087
1088    switch (TransactionManager->State)
1089    {
1090       case TransactionStarted: break;
1091       case TransactionAbort: _error->Fatal("Transaction %s was already committed and is now aborted", TransactionManager->Target.URI.c_str()); return;
1092       case TransactionCommit: _error->Fatal("Transaction %s was already committed and is again committed", TransactionManager->Target.URI.c_str()); return;
1093    }
1094    TransactionManager->State = TransactionCommit;
1095
1096    // move new files into place *and* remove files that are not
1097    // part of the transaction but are still on disk
1098    for (std::vector<pkgAcqTransactionItem*>::iterator I = Transaction.begin();
1099         I != Transaction.end(); ++I)
1100    {
1101       (*I)->TransactionState(TransactionCommit);
1102    }
1103    Transaction.clear();
1104 }
1105                                                                         /*}}}*/
1106 // AcqMetaBase::TransactionStageCopy - Stage a file for copying         /*{{{*/
1107 void pkgAcqMetaBase::TransactionStageCopy(pkgAcqTransactionItem * const I,
1108                                           const std::string &From,
1109                                           const std::string &To)
1110 {
1111    I->PartialFile = From;
1112    I->DestFile = To;
1113 }
1114                                                                         /*}}}*/
1115 // AcqMetaBase::TransactionStageRemoval - Stage a file for removal      /*{{{*/
1116 void pkgAcqMetaBase::TransactionStageRemoval(pkgAcqTransactionItem * const I,
1117                                              const std::string &FinalFile)
1118 {
1119    I->PartialFile = "";
1120    I->DestFile = FinalFile;
1121 }
1122                                                                         /*}}}*/
1123 // AcqMetaBase::GenerateAuthWarning - Check gpg authentication error    /*{{{*/
1124 /* This method is called from ::Failed handlers. If it returns true,
1125    no fallback to other files or modi is performed */
1126 bool pkgAcqMetaBase::CheckStopAuthentication(pkgAcquire::Item * const I, const std::string &Message)
1127 {
1128    string const Final = I->GetFinalFilename();
1129    std::string const GPGError = LookupTag(Message, "Message");
1130    if (FileExists(Final))
1131    {
1132       I->Status = StatTransientNetworkError;
1133       _error->Warning(_("An error occurred during the signature verification. "
1134                "The repository is not updated and the previous index files will be used. "
1135                "GPG error: %s: %s"),
1136             Desc.Description.c_str(),
1137             GPGError.c_str());
1138       RunScripts("APT::Update::Auth-Failure");
1139       return true;
1140    } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1141       /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1142       _error->Error(_("GPG error: %s: %s"),
1143             Desc.Description.c_str(),
1144             GPGError.c_str());
1145       I->Status = StatAuthError;
1146       return true;
1147    } else {
1148       _error->Warning(_("GPG error: %s: %s"),
1149             Desc.Description.c_str(),
1150             GPGError.c_str());
1151    }
1152    // gpgv method failed
1153    ReportMirrorFailureToCentral(*this, "GPGFailure", GPGError);
1154    return false;
1155 }
1156                                                                         /*}}}*/
1157 // AcqMetaBase::Custom600Headers - Get header for AcqMetaBase           /*{{{*/
1158 // ---------------------------------------------------------------------
1159 string pkgAcqMetaBase::Custom600Headers() const
1160 {
1161    std::string Header = "\nIndex-File: true";
1162    std::string MaximumSize;
1163    strprintf(MaximumSize, "\nMaximum-Size: %i",
1164              _config->FindI("Acquire::MaxReleaseFileSize", 10*1000*1000));
1165    Header += MaximumSize;
1166
1167    string const FinalFile = GetFinalFilename();
1168    struct stat Buf;
1169    if (stat(FinalFile.c_str(),&Buf) == 0)
1170       Header += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);
1171
1172    return Header;
1173 }
1174                                                                         /*}}}*/
1175 // AcqMetaBase::QueueForSignatureVerify                                 /*{{{*/
1176 void pkgAcqMetaBase::QueueForSignatureVerify(pkgAcqTransactionItem * const I, std::string const &File, std::string const &Signature)
1177 {
1178    AuthPass = true;
1179    I->Desc.URI = "gpgv:" + Signature;
1180    I->DestFile = File;
1181    QueueURI(I->Desc);
1182    I->SetActiveSubprocess("gpgv");
1183 }
1184                                                                         /*}}}*/
1185 // AcqMetaBase::CheckDownloadDone                                       /*{{{*/
1186 bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const std::string &Message, HashStringList const &Hashes) const
1187 {
1188    // We have just finished downloading a Release file (it is not
1189    // verified yet)
1190
1191    // Save the final base URI we got this Release file from
1192    if (I->UsedMirror.empty() == false && _config->FindB("Acquire::SameMirrorForAllIndexes", true))
1193    {
1194       if (APT::String::Endswith(I->Desc.URI, "InRelease"))
1195          TransactionManager->BaseURI = I->Desc.URI.substr(0, I->Desc.URI.length() - strlen("InRelease"));
1196       else if (APT::String::Endswith(I->Desc.URI, "Release"))
1197          TransactionManager->BaseURI = I->Desc.URI.substr(0, I->Desc.URI.length() - strlen("Release"));
1198    }
1199
1200    std::string const FileName = LookupTag(Message,"Filename");
1201    if (FileName != I->DestFile && RealFileExists(I->DestFile) == false)
1202    {
1203       I->Local = true;
1204       I->Desc.URI = "copy:" + FileName;
1205       I->QueueURI(I->Desc);
1206       return false;
1207    }
1208
1209    // make sure to verify against the right file on I-M-S hit
1210    bool IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"), false);
1211    if (IMSHit == false && Hashes.usable())
1212    {
1213       // detect IMS-Hits servers haven't detected by Hash comparison
1214       std::string const FinalFile = I->GetFinalFilename();
1215       if (RealFileExists(FinalFile) && Hashes.VerifyFile(FinalFile) == true)
1216       {
1217          IMSHit = true;
1218          RemoveFile("CheckDownloadDone", I->DestFile);
1219       }
1220    }
1221
1222    if(IMSHit == true)
1223    {
1224       // for simplicity, the transaction manager is always InRelease
1225       // even if it doesn't exist.
1226       TransactionManager->IMSHit = true;
1227       I->PartialFile = I->DestFile = I->GetFinalFilename();
1228    }
1229
1230    // set Item to complete as the remaining work is all local (verify etc)
1231    I->Complete = true;
1232
1233    return true;
1234 }
1235                                                                         /*}}}*/
1236 bool pkgAcqMetaBase::CheckAuthDone(string const &Message)               /*{{{*/
1237 {
1238    // At this point, the gpgv method has succeeded, so there is a
1239    // valid signature from a key in the trusted keyring.  We
1240    // perform additional verification of its contents, and use them
1241    // to verify the indexes we are about to download
1242    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1243       std::cerr << "Signature verification succeeded: " << DestFile << std::endl;
1244
1245    if (TransactionManager->IMSHit == false)
1246    {
1247       // open the last (In)Release if we have it
1248       std::string const FinalFile = GetFinalFilename();
1249       std::string FinalRelease;
1250       std::string FinalInRelease;
1251       if (APT::String::Endswith(FinalFile, "InRelease"))
1252       {
1253          FinalInRelease = FinalFile;
1254          FinalRelease = FinalFile.substr(0, FinalFile.length() - strlen("InRelease")) + "Release";
1255       }
1256       else
1257       {
1258          FinalInRelease = FinalFile.substr(0, FinalFile.length() - strlen("Release")) + "InRelease";
1259          FinalRelease = FinalFile;
1260       }
1261       LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
1262    }
1263
1264    bool const GoodAuth = TransactionManager->MetaIndexParser->Load(DestFile, &ErrorText);
1265    if (GoodAuth == false && AllowInsecureRepositories(InsecureType::WEAK, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == false)
1266    {
1267       Status = StatAuthError;
1268       return false;
1269    }
1270
1271    if (!VerifyVendor(Message))
1272    {
1273       Status = StatAuthError;
1274       return false;
1275    }
1276
1277    // Download further indexes with verification
1278    TransactionManager->QueueIndexes(GoodAuth);
1279
1280    return GoodAuth;
1281 }
1282                                                                         /*}}}*/
1283 void pkgAcqMetaClearSig::QueueIndexes(bool const verify)                        /*{{{*/
1284 {
1285    // at this point the real Items are loaded in the fetcher
1286    ExpectedAdditionalItems = 0;
1287
1288    std::set<std::string> targetsSeen;
1289    bool const hasReleaseFile = TransactionManager->MetaIndexParser != NULL;
1290    bool const metaBaseSupportsByHash = hasReleaseFile && TransactionManager->MetaIndexParser->GetSupportsAcquireByHash();
1291    bool hasHashes = true;
1292    auto IndexTargets = TransactionManager->MetaIndexParser->GetIndexTargets();
1293    if (hasReleaseFile && verify == false)
1294       hasHashes = std::any_of(IndexTargets.begin(), IndexTargets.end(),
1295             [&](IndexTarget const &Target) { return TransactionManager->MetaIndexParser->Exists(Target.MetaKey); });
1296    for (auto&& Target: IndexTargets)
1297    {
1298       // if we have seen a target which is created-by a target this one here is declared a
1299       // fallback to, we skip acquiring the fallback (but we make sure we clean up)
1300       if (targetsSeen.find(Target.Option(IndexTarget::FALLBACK_OF)) != targetsSeen.end())
1301       {
1302          targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
1303          new CleanupItem(Owner, TransactionManager, Target);
1304          continue;
1305       }
1306       // all is an implementation detail. Users shouldn't use this as arch
1307       // We need this support trickery here as e.g. Debian has binary-all files already,
1308       // but arch:all packages are still in the arch:any files, so we would waste precious
1309       // download time, bandwidth and diskspace for nothing, BUT Debian doesn't feature all
1310       // in the set of supported architectures, so we can filter based on this property rather
1311       // than invent an entirely new flag we would need to carry for all of eternity.
1312       if (hasReleaseFile && Target.Option(IndexTarget::ARCHITECTURE) == "all")
1313       {
1314          if (TransactionManager->MetaIndexParser->IsArchitectureAllSupportedFor(Target) == false)
1315          {
1316             new CleanupItem(Owner, TransactionManager, Target);
1317             continue;
1318          }
1319       }
1320
1321       bool trypdiff = Target.OptionBool(IndexTarget::PDIFFS);
1322       if (hasReleaseFile == true)
1323       {
1324          if (TransactionManager->MetaIndexParser->Exists(Target.MetaKey) == false)
1325          {
1326             // optional targets that we do not have in the Release file are skipped
1327             if (hasHashes == true && Target.IsOptional)
1328             {
1329                new CleanupItem(Owner, TransactionManager, Target);
1330                continue;
1331             }
1332
1333             std::string const &arch = Target.Option(IndexTarget::ARCHITECTURE);
1334             if (arch.empty() == false)
1335             {
1336                if (TransactionManager->MetaIndexParser->IsArchitectureSupported(arch) == false)
1337                {
1338                   new CleanupItem(Owner, TransactionManager, Target);
1339                   _error->Notice(_("Skipping acquire of configured file '%s' as repository '%s' doesn't support architecture '%s'"),
1340                         Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str(), arch.c_str());
1341                   continue;
1342                }
1343                // if the architecture is officially supported but currently no packages for it available,
1344                // ignore silently as this is pretty much the same as just shipping an empty file.
1345                // if we don't know which architectures are supported, we do NOT ignore it to notify user about this
1346                if (hasHashes == true && TransactionManager->MetaIndexParser->IsArchitectureSupported("*undefined*") == false)
1347                {
1348                   new CleanupItem(Owner, TransactionManager, Target);
1349                   continue;
1350                }
1351             }
1352
1353             if (hasHashes == true)
1354             {
1355                Status = StatAuthError;
1356                strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), Target.MetaKey.c_str());
1357                return;
1358             }
1359             else
1360             {
1361                new pkgAcqIndex(Owner, TransactionManager, Target);
1362                continue;
1363             }
1364          }
1365          else if (verify)
1366          {
1367             auto const hashes = GetExpectedHashesFor(Target.MetaKey);
1368             if (hashes.empty() == false)
1369             {
1370                if (hashes.usable() == false && TargetIsAllowedToBe(TransactionManager->Target, InsecureType::WEAK) == false)
1371                {
1372                   new CleanupItem(Owner, TransactionManager, Target);
1373                   _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"),
1374                         Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str());
1375                   continue;
1376                }
1377                // empty files are skipped as acquiring the very small compressed files is a waste of time
1378                else if (hashes.FileSize() == 0)
1379                {
1380                   new CleanupItem(Owner, TransactionManager, Target);
1381                   targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
1382                   continue;
1383                }
1384             }
1385          }
1386
1387          // autoselect the compression method
1388          std::vector<std::string> types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' ');
1389          types.erase(std::remove_if(types.begin(), types.end(), [&](std::string const &t) {
1390             if (t == "uncompressed")
1391                return TransactionManager->MetaIndexParser->Exists(Target.MetaKey) == false;
1392             std::string const MetaKey = Target.MetaKey + "." + t;
1393             return TransactionManager->MetaIndexParser->Exists(MetaKey) == false;
1394          }), types.end());
1395          if (types.empty() == false)
1396          {
1397             std::ostringstream os;
1398             // add the special compressiontype byhash first if supported
1399             std::string const useByHashConf = Target.Option(IndexTarget::BY_HASH);
1400             bool useByHash = false;
1401             if(useByHashConf == "force")
1402                useByHash = true;
1403             else
1404                useByHash = StringToBool(useByHashConf) == true && metaBaseSupportsByHash;
1405             if (useByHash == true)
1406                os << "by-hash ";
1407             std::copy(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " "));
1408             os << *types.rbegin();
1409             Target.Options["COMPRESSIONTYPES"] = os.str();
1410          }
1411          else
1412             Target.Options["COMPRESSIONTYPES"].clear();
1413
1414          std::string filename = GetExistingFilename(GetFinalFileNameFromURI(Target.URI));
1415          if (filename.empty() == false)
1416          {
1417             // if the Release file is a hit and we have an index it must be the current one
1418             if (TransactionManager->IMSHit == true)
1419                ;
1420             else if (TransactionManager->LastMetaIndexParser != NULL)
1421             {
1422                // see if the file changed since the last Release file
1423                // we use the uncompressed files as we might compress differently compared to the server,
1424                // so the hashes might not match, even if they contain the same data.
1425                HashStringList const newFile = GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, Target.MetaKey);
1426                HashStringList const oldFile = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, Target.MetaKey);
1427                if (newFile != oldFile)
1428                   filename.clear();
1429             }
1430             else
1431                filename.clear();
1432          }
1433          else
1434             trypdiff = false; // no file to patch
1435
1436          if (filename.empty() == false)
1437          {
1438             new NoActionItem(Owner, Target, filename);
1439             std::string const idxfilename = GetFinalFileNameFromURI(GetDiffIndexURI(Target));
1440             if (FileExists(idxfilename))
1441                new NoActionItem(Owner, Target, idxfilename);
1442             targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
1443             continue;
1444          }
1445
1446          // check if we have patches available
1447          trypdiff &= TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target.MetaKey));
1448       }
1449       else
1450       {
1451          // if we have no file to patch, no point in trying
1452          trypdiff &= (GetExistingFilename(GetFinalFileNameFromURI(Target.URI)).empty() == false);
1453       }
1454
1455       // no point in patching from local sources
1456       if (trypdiff)
1457       {
1458          std::string const proto = Target.URI.substr(0, strlen("file:/"));
1459          if (proto == "file:/" || proto == "copy:/" || proto == "cdrom:")
1460             trypdiff = false;
1461       }
1462
1463       // Queue the Index file (Packages, Sources, Translation-$foo, …)
1464       targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
1465       if (trypdiff)
1466          new pkgAcqDiffIndex(Owner, TransactionManager, Target);
1467       else
1468          new pkgAcqIndex(Owner, TransactionManager, Target);
1469    }
1470 }
1471                                                                         /*}}}*/
1472 bool pkgAcqMetaBase::VerifyVendor(string const &)                       /*{{{*/
1473 {
1474    string Transformed = TransactionManager->MetaIndexParser->GetExpectedDist();
1475
1476    if (Transformed == "../project/experimental")
1477    {
1478       Transformed = "experimental";
1479    }
1480
1481    auto pos = Transformed.rfind('/');
1482    if (pos != string::npos)
1483    {
1484       Transformed = Transformed.substr(0, pos);
1485    }
1486
1487    if (Transformed == ".")
1488    {
1489       Transformed = "";
1490    }
1491
1492    if (TransactionManager->MetaIndexParser->GetValidUntil() > 0)
1493    {
1494       time_t const invalid_since = time(NULL) - TransactionManager->MetaIndexParser->GetValidUntil();
1495       if (invalid_since > 0)
1496       {
1497          std::string errmsg;
1498          strprintf(errmsg,
1499                // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1500                // the time since then the file is invalid - formatted in the same way as in
1501                // the download progress display (e.g. 7d 3h 42min 1s)
1502                _("Release file for %s is expired (invalid since %s). "
1503                   "Updates for this repository will not be applied."),
1504                Target.URI.c_str(), TimeToStr(invalid_since).c_str());
1505          if (ErrorText.empty())
1506             ErrorText = errmsg;
1507          return _error->Error("%s", errmsg.c_str());
1508       }
1509    }
1510
1511    /* Did we get a file older than what we have? This is a last minute IMS hit and doubles
1512       as a prevention of downgrading us to older (still valid) files */
1513    if (TransactionManager->IMSHit == false && TransactionManager->LastMetaIndexParser != NULL &&
1514          TransactionManager->LastMetaIndexParser->GetDate() > TransactionManager->MetaIndexParser->GetDate())
1515    {
1516       TransactionManager->IMSHit = true;
1517       RemoveFile("VerifyVendor", DestFile);
1518       PartialFile = DestFile = GetFinalFilename();
1519       // load the 'old' file in the 'new' one instead of flipping pointers as
1520       // the new one isn't owned by us, while the old one is so cleanup would be confused.
1521       TransactionManager->MetaIndexParser->swapLoad(TransactionManager->LastMetaIndexParser);
1522       delete TransactionManager->LastMetaIndexParser;
1523       TransactionManager->LastMetaIndexParser = NULL;
1524    }
1525
1526    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1527    {
1528       std::cerr << "Got Codename: " << TransactionManager->MetaIndexParser->GetCodename() << std::endl;
1529       std::cerr << "Expecting Dist: " << TransactionManager->MetaIndexParser->GetExpectedDist() << std::endl;
1530       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1531    }
1532
1533    if (TransactionManager->MetaIndexParser->CheckDist(Transformed) == false)
1534    {
1535       // This might become fatal one day
1536 //       Status = StatAuthError;
1537 //       ErrorText = "Conflicting distribution; expected "
1538 //          + MetaIndexParser->GetExpectedDist() + " but got "
1539 //          + MetaIndexParser->GetCodename();
1540 //       return false;
1541       if (!Transformed.empty())
1542       {
1543          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1544                          Desc.Description.c_str(),
1545                          Transformed.c_str(),
1546                          TransactionManager->MetaIndexParser->GetCodename().c_str());
1547       }
1548    }
1549
1550    return true;
1551 }
1552                                                                         /*}}}*/
1553 pkgAcqMetaBase::~pkgAcqMetaBase()
1554 {
1555 }
1556
1557 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner,        /*{{{*/
1558       IndexTarget const &ClearsignedTarget,
1559       IndexTarget const &DetachedDataTarget, IndexTarget const &DetachedSigTarget,
1560       metaIndex * const MetaIndexParser) :
1561    pkgAcqMetaIndex(Owner, this, ClearsignedTarget, DetachedSigTarget),
1562    d(NULL), DetachedDataTarget(DetachedDataTarget),
1563    MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL)
1564 {
1565    // index targets + (worst case:) Release/Release.gpg
1566    ExpectedAdditionalItems = std::numeric_limits<decltype(ExpectedAdditionalItems)>::max();
1567    TransactionManager->Add(this);
1568 }
1569                                                                         /*}}}*/
1570 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1571 {
1572    if (LastMetaIndexParser != NULL)
1573       delete LastMetaIndexParser;
1574 }
1575                                                                         /*}}}*/
1576 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1577 string pkgAcqMetaClearSig::Custom600Headers() const
1578 {
1579    string Header = pkgAcqMetaBase::Custom600Headers();
1580    Header += "\nFail-Ignore: true";
1581    std::string const key = TransactionManager->MetaIndexParser->GetSignedBy();
1582    if (key.empty() == false)
1583       Header += "\nSigned-By: " + key;
1584
1585    return Header;
1586 }
1587                                                                         /*}}}*/
1588 void pkgAcqMetaClearSig::Finished()                                     /*{{{*/
1589 {
1590    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1591       std::clog << "Finished: " << DestFile <<std::endl;
1592    if(TransactionManager->State == TransactionStarted &&
1593       TransactionManager->TransactionHasError() == false)
1594       TransactionManager->CommitTransaction();
1595 }
1596                                                                         /*}}}*/
1597 bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message,         /*{{{*/
1598          pkgAcquire::MethodConfig const * const Cnf)
1599 {
1600    Item::VerifyDone(Message, Cnf);
1601
1602    if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile))
1603       return RenameOnError(NotClearsigned);
1604
1605    return true;
1606 }
1607                                                                         /*}}}*/
1608 // pkgAcqMetaClearSig::Done - We got a file                             /*{{{*/
1609 void pkgAcqMetaClearSig::Done(std::string const &Message,
1610                               HashStringList const &Hashes,
1611                               pkgAcquire::MethodConfig const * const Cnf)
1612 {
1613    Item::Done(Message, Hashes, Cnf);
1614
1615    if(AuthPass == false)
1616    {
1617       if(CheckDownloadDone(this, Message, Hashes) == true)
1618          QueueForSignatureVerify(this, DestFile, DestFile);
1619       return;
1620    }
1621    else if(CheckAuthDone(Message) == true)
1622    {
1623       if (TransactionManager->IMSHit == false)
1624          TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
1625       else if (RealFileExists(GetFinalFilename()) == false)
1626       {
1627          // We got an InRelease file IMSHit, but we haven't one, which means
1628          // we had a valid Release/Release.gpg combo stepping in, which we have
1629          // to 'acquire' now to ensure list cleanup isn't removing them
1630          new NoActionItem(Owner, DetachedDataTarget);
1631          new NoActionItem(Owner, DetachedSigTarget);
1632       }
1633    }
1634    else if (Status != StatAuthError)
1635    {
1636       string const FinalFile = GetFinalFileNameFromURI(DetachedDataTarget.URI);
1637       string const OldFile = GetFinalFilename();
1638       if (TransactionManager->IMSHit == false)
1639          TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
1640       else if (RealFileExists(OldFile) == false)
1641          new NoActionItem(Owner, DetachedDataTarget);
1642       else
1643          TransactionManager->TransactionStageCopy(this, OldFile, FinalFile);
1644    }
1645 }
1646                                                                         /*}}}*/
1647 void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) /*{{{*/
1648 {
1649    Item::Failed(Message, Cnf);
1650
1651    if (AuthPass == false)
1652    {
1653       if (Status == StatAuthError || Status == StatTransientNetworkError)
1654       {
1655          // if we expected a ClearTextSignature (InRelease) but got a network
1656          // error or got a file, but it wasn't valid, we end up here (see VerifyDone).
1657          // As these is usually called by web-portals we do not try Release/Release.gpg
1658          // as this is gonna fail anyway and instead abort our try (LP#346386)
1659          TransactionManager->AbortTransaction();
1660          return;
1661       }
1662
1663       // Queue the 'old' InRelease file for removal if we try Release.gpg
1664       // as otherwise the file will stay around and gives a false-auth
1665       // impression (CVE-2012-0214)
1666       TransactionManager->TransactionStageRemoval(this, GetFinalFilename());
1667       Status = StatDone;
1668
1669       new pkgAcqMetaIndex(Owner, TransactionManager, DetachedDataTarget, DetachedSigTarget);
1670    }
1671    else
1672    {
1673       if(CheckStopAuthentication(this, Message))
1674          return;
1675
1676       if(AllowInsecureRepositories(InsecureType::UNSIGNED, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
1677       {
1678          Status = StatDone;
1679
1680          /* InRelease files become Release files, otherwise
1681           * they would be considered as trusted later on */
1682          string const FinalRelease = GetFinalFileNameFromURI(DetachedDataTarget.URI);
1683          string const PartialRelease = GetPartialFileNameFromURI(DetachedDataTarget.URI);
1684          string const FinalReleasegpg = GetFinalFileNameFromURI(DetachedSigTarget.URI);
1685          string const FinalInRelease = GetFinalFilename();
1686          Rename(DestFile, PartialRelease);
1687          TransactionManager->TransactionStageCopy(this, PartialRelease, FinalRelease);
1688          LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
1689
1690          // we parse the indexes here because at this point the user wanted
1691          // a repository that may potentially harm him
1692          if (TransactionManager->MetaIndexParser->Load(PartialRelease, &ErrorText) == false || VerifyVendor(Message) == false)
1693             /* expired Release files are still a problem you need extra force for */;
1694          else
1695             TransactionManager->QueueIndexes(true);
1696       }
1697    }
1698 }
1699                                                                         /*}}}*/
1700
1701 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire * const Owner,              /*{{{*/
1702                                  pkgAcqMetaClearSig * const TransactionManager,
1703                                  IndexTarget const &DataTarget,
1704                                  IndexTarget const &DetachedSigTarget) :
1705    pkgAcqMetaBase(Owner, TransactionManager, DataTarget), d(NULL),
1706    DetachedSigTarget(DetachedSigTarget)
1707 {
1708    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1709       std::clog << "New pkgAcqMetaIndex with TransactionManager "
1710                 << this->TransactionManager << std::endl;
1711
1712    DestFile = GetPartialFileNameFromURI(DataTarget.URI);
1713
1714    // Create the item
1715    Desc.Description = DataTarget.Description;
1716    Desc.Owner = this;
1717    Desc.ShortDesc = DataTarget.ShortDesc;
1718    Desc.URI = DataTarget.URI;
1719    QueueURI(Desc);
1720 }
1721                                                                         /*}}}*/
1722 void pkgAcqMetaIndex::Done(string const &Message,                       /*{{{*/
1723                            HashStringList const &Hashes,
1724                            pkgAcquire::MethodConfig const * const Cfg)
1725 {
1726    Item::Done(Message,Hashes,Cfg);
1727
1728    if(CheckDownloadDone(this, Message, Hashes))
1729    {
1730       // we have a Release file, now download the Signature, all further
1731       // verify/queue for additional downloads will be done in the
1732       // pkgAcqMetaSig::Done() code
1733       new pkgAcqMetaSig(Owner, TransactionManager, DetachedSigTarget, this);
1734    }
1735 }
1736                                                                         /*}}}*/
1737 // pkgAcqMetaIndex::Failed - no Release file present                    /*{{{*/
1738 void pkgAcqMetaIndex::Failed(string const &Message,
1739                              pkgAcquire::MethodConfig const * const Cnf)
1740 {
1741    pkgAcquire::Item::Failed(Message, Cnf);
1742    Status = StatDone;
1743
1744    // No Release file was present so fall
1745    // back to queueing Packages files without verification
1746    // only allow going further if the user explicitly wants it
1747    if(AllowInsecureRepositories(InsecureType::NORELEASE, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
1748    {
1749       // ensure old Release files are removed
1750       TransactionManager->TransactionStageRemoval(this, GetFinalFilename());
1751
1752       // queue without any kind of hashsum support
1753       TransactionManager->QueueIndexes(false);
1754    }
1755 }
1756                                                                         /*}}}*/
1757 std::string pkgAcqMetaIndex::DescURI() const                            /*{{{*/
1758 {
1759    return Target.URI;
1760 }
1761                                                                         /*}}}*/
1762 pkgAcqMetaIndex::~pkgAcqMetaIndex() {}
1763
1764 // AcqMetaSig::AcqMetaSig - Constructor                                 /*{{{*/
1765 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire * const Owner,
1766       pkgAcqMetaClearSig * const TransactionManager,
1767       IndexTarget const &Target,
1768       pkgAcqMetaIndex * const MetaIndex) :
1769    pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL), MetaIndex(MetaIndex)
1770 {
1771    DestFile = GetPartialFileNameFromURI(Target.URI);
1772
1773    // remove any partial downloaded sig-file in partial/.
1774    // it may confuse proxies and is too small to warrant a
1775    // partial download anyway
1776    RemoveFile("pkgAcqMetaSig", DestFile);
1777
1778    // set the TransactionManager
1779    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1780       std::clog << "New pkgAcqMetaSig with TransactionManager "
1781                 << TransactionManager << std::endl;
1782
1783    // Create the item
1784    Desc.Description = Target.Description;
1785    Desc.Owner = this;
1786    Desc.ShortDesc = Target.ShortDesc;
1787    Desc.URI = Target.URI;
1788
1789    // If we got a hit for Release, we will get one for Release.gpg too (or obscure errors),
1790    // so we skip the download step and go instantly to verification
1791    if (TransactionManager->IMSHit == true && RealFileExists(GetFinalFilename()))
1792    {
1793       Complete = true;
1794       Status = StatDone;
1795       PartialFile = DestFile = GetFinalFilename();
1796       MetaIndexFileSignature = DestFile;
1797       MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
1798    }
1799    else
1800       QueueURI(Desc);
1801 }
1802                                                                         /*}}}*/
1803 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1804 {
1805 }
1806                                                                         /*}}}*/
1807 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1808 std::string pkgAcqMetaSig::Custom600Headers() const
1809 {
1810    std::string Header = pkgAcqTransactionItem::Custom600Headers();
1811    std::string const key = TransactionManager->MetaIndexParser->GetSignedBy();
1812    if (key.empty() == false)
1813       Header += "\nSigned-By: " + key;
1814    return Header;
1815 }
1816                                                                         /*}}}*/
1817 // AcqMetaSig::Done - The signature was downloaded/verified             /*{{{*/
1818 void pkgAcqMetaSig::Done(string const &Message, HashStringList const &Hashes,
1819                          pkgAcquire::MethodConfig const * const Cfg)
1820 {
1821    if (MetaIndexFileSignature.empty() == false)
1822    {
1823       DestFile = MetaIndexFileSignature;
1824       MetaIndexFileSignature.clear();
1825    }
1826    Item::Done(Message, Hashes, Cfg);
1827
1828    if(MetaIndex->AuthPass == false)
1829    {
1830       if(MetaIndex->CheckDownloadDone(this, Message, Hashes) == true)
1831       {
1832          // destfile will be modified to point to MetaIndexFile for the
1833          // gpgv method, so we need to save it here
1834          MetaIndexFileSignature = DestFile;
1835          MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
1836       }
1837       return;
1838    }
1839    else if(MetaIndex->CheckAuthDone(Message) == true)
1840    {
1841       if (TransactionManager->IMSHit == false)
1842       {
1843          TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
1844          TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, MetaIndex->GetFinalFilename());
1845       }
1846    }
1847    else if (MetaIndex->Status != StatAuthError)
1848    {
1849       std::string const FinalFile = MetaIndex->GetFinalFilename();
1850       if (TransactionManager->IMSHit == false)
1851          TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, FinalFile);
1852       else
1853          TransactionManager->TransactionStageCopy(MetaIndex, FinalFile, FinalFile);
1854    }
1855 }
1856                                                                         /*}}}*/
1857 void pkgAcqMetaSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
1858 {
1859    Item::Failed(Message,Cnf);
1860
1861    // check if we need to fail at this point
1862    if (MetaIndex->AuthPass == true && MetaIndex->CheckStopAuthentication(this, Message))
1863          return;
1864
1865    // ensures that a Release.gpg file in the lists/ is removed by the transaction
1866    TransactionManager->TransactionStageRemoval(this, DestFile);
1867
1868    // only allow going further if the user explicitly wants it
1869    if (AllowInsecureRepositories(InsecureType::UNSIGNED, MetaIndex->Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
1870    {
1871       string const FinalRelease = MetaIndex->GetFinalFilename();
1872       string const FinalInRelease = TransactionManager->GetFinalFilename();
1873       LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
1874
1875       // we parse the indexes here because at this point the user wanted
1876       // a repository that may potentially harm him
1877       bool const GoodLoad = TransactionManager->MetaIndexParser->Load(MetaIndex->DestFile, &ErrorText);
1878       if (MetaIndex->VerifyVendor(Message) == false)
1879          /* expired Release files are still a problem you need extra force for */;
1880       else
1881          TransactionManager->QueueIndexes(GoodLoad);
1882
1883       TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, FinalRelease);
1884    }
1885    else if (TransactionManager->IMSHit == false)
1886       Rename(MetaIndex->DestFile, MetaIndex->DestFile + ".FAILED");
1887
1888    // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
1889    if (Cnf->LocalOnly == true ||
1890        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1891    {
1892       // Ignore this
1893       Status = StatDone;
1894    }
1895 }
1896                                                                         /*}}}*/
1897
1898
1899 // AcqBaseIndex - Constructor                                           /*{{{*/
1900 pkgAcqBaseIndex::pkgAcqBaseIndex(pkgAcquire * const Owner,
1901       pkgAcqMetaClearSig * const TransactionManager,
1902       IndexTarget const &Target)
1903 : pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL)
1904 {
1905 }
1906                                                                         /*}}}*/
1907 void pkgAcqBaseIndex::Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
1908 {
1909    pkgAcquire::Item::Failed(Message, Cnf);
1910    if (Status != StatAuthError)
1911       return;
1912
1913    ErrorText.append("Release file created at: ");
1914    auto const timespec = TransactionManager->MetaIndexParser->GetDate();
1915    if (timespec == 0)
1916       ErrorText.append("<unknown>");
1917    else
1918       ErrorText.append(TimeRFC1123(timespec, true));
1919    ErrorText.append("\n");
1920 }
1921                                                                         /*}}}*/
1922 pkgAcqBaseIndex::~pkgAcqBaseIndex() {}
1923
1924 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
1925 // ---------------------------------------------------------------------
1926 /* Get the DiffIndex file first and see if there are patches available
1927  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
1928  * patches. If anything goes wrong in that process, it will fall back to
1929  * the original packages file
1930  */
1931 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner,
1932                                  pkgAcqMetaClearSig * const TransactionManager,
1933                                  IndexTarget const &Target)
1934    : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), diffs(NULL)
1935 {
1936    // FIXME: Magic number as an upper bound on pdiffs we will reasonably acquire
1937    ExpectedAdditionalItems = 40;
1938
1939    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
1940
1941    Desc.Owner = this;
1942    Desc.Description = GetDiffIndexFileName(Target.Description);
1943    Desc.ShortDesc = Target.ShortDesc;
1944    Desc.URI = GetDiffIndexURI(Target);
1945
1946    DestFile = GetPartialFileNameFromURI(Desc.URI);
1947
1948    if(Debug)
1949       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
1950
1951    QueueURI(Desc);
1952 }
1953                                                                         /*}}}*/
1954 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1955 // ---------------------------------------------------------------------
1956 /* The only header we use is the last-modified header. */
1957 string pkgAcqDiffIndex::Custom600Headers() const
1958 {
1959    if (TransactionManager->LastMetaIndexParser != NULL)
1960       return "\nIndex-File: true";
1961
1962    string const Final = GetFinalFilename();
1963
1964    if(Debug)
1965       std::clog << "Custom600Header-IMS: " << Final << std::endl;
1966
1967    struct stat Buf;
1968    if (stat(Final.c_str(),&Buf) != 0)
1969       return "\nIndex-File: true";
1970
1971    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);
1972 }
1973                                                                         /*}}}*/
1974 void pkgAcqDiffIndex::QueueOnIMSHit() const                             /*{{{*/
1975 {
1976    // list cleanup needs to know that this file as well as the already
1977    // present index is ours, so we create an empty diff to save it for us
1978    new pkgAcqIndexDiffs(Owner, TransactionManager, Target);
1979 }
1980                                                                         /*}}}*/
1981 bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)       /*{{{*/
1982 {
1983    ExpectedAdditionalItems = 0;
1984    // failing here is fine: our caller will take care of trying to
1985    // get the complete file if patching fails
1986    if(Debug)
1987       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
1988          << std::endl;
1989
1990    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
1991    pkgTagFile TF(&Fd);
1992    if (Fd.IsOpen() == false || Fd.Failed())
1993       return false;
1994
1995    pkgTagSection Tags;
1996    if(unlikely(TF.Step(Tags) == false))
1997       return false;
1998
1999    HashStringList ServerHashes;
2000    unsigned long long ServerSize = 0;
2001
2002    auto const &posix = std::locale("C.UTF-8");
2003    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
2004    {
2005       std::string tagname = *type;
2006       tagname.append("-Current");
2007       std::string const tmp = Tags.FindS(tagname.c_str());
2008       if (tmp.empty() == true)
2009          continue;
2010
2011       string hash;
2012       unsigned long long size;
2013       std::stringstream ss(tmp);
2014       ss.imbue(posix);
2015       ss >> hash >> size;
2016       if (unlikely(hash.empty() == true))
2017          continue;
2018       if (unlikely(ServerSize != 0 && ServerSize != size))
2019          continue;
2020       ServerHashes.push_back(HashString(*type, hash));
2021       ServerSize = size;
2022    }
2023
2024    if (ServerHashes.usable() == false)
2025    {
2026       if (Debug == true)
2027          std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": Did not find a good hashsum in the index" << std::endl;
2028       return false;
2029    }
2030
2031    std::string const CurrentPackagesFile = GetFinalFileNameFromURI(Target.URI);
2032    HashStringList const TargetFileHashes = GetExpectedHashesFor(Target.MetaKey);
2033    if (TargetFileHashes.usable() == false || ServerHashes != TargetFileHashes)
2034    {
2035       if (Debug == true)
2036       {
2037          std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": Index has different hashes than parser, probably older, so fail pdiffing" << std::endl;
2038          printHashSumComparison(CurrentPackagesFile, ServerHashes, TargetFileHashes);
2039       }
2040       return false;
2041    }
2042
2043    HashStringList LocalHashes;
2044    // try avoiding calculating the hash here as this is costly
2045    if (TransactionManager->LastMetaIndexParser != NULL)
2046       LocalHashes = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, Target.MetaKey);
2047    if (LocalHashes.usable() == false)
2048    {
2049       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly, FileFd::Auto);
2050       Hashes LocalHashesCalc(ServerHashes);
2051       LocalHashesCalc.AddFD(fd);
2052       LocalHashes = LocalHashesCalc.GetHashStringList();
2053    }
2054
2055    if (ServerHashes == LocalHashes)
2056    {
2057       // we have the same sha1 as the server so we are done here
2058       if(Debug)
2059          std::clog << "pkgAcqDiffIndex: Package file " << CurrentPackagesFile << " is up-to-date" << std::endl;
2060       QueueOnIMSHit();
2061       return true;
2062    }
2063
2064    if(Debug)
2065       std::clog << "Server-Current: " << ServerHashes.find(NULL)->toStr() << " and we start at "
2066          << CurrentPackagesFile << " " << LocalHashes.FileSize() << " " << LocalHashes.find(NULL)->toStr() << std::endl;
2067
2068    // historically, older hashes have more info than newer ones, so start
2069    // collecting with older ones first to avoid implementing complicated
2070    // information merging techniques… a failure is after all always
2071    // recoverable with a complete file and hashes aren't changed that often.
2072    std::vector<char const *> types;
2073    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
2074       types.push_back(*type);
2075
2076    // parse all of (provided) history
2077    vector<DiffInfo> available_patches;
2078    bool firstAcceptedHashes = true;
2079    for (auto type = types.crbegin(); type != types.crend(); ++type)
2080    {
2081       if (LocalHashes.find(*type) == NULL)
2082          continue;
2083
2084       std::string tagname = *type;
2085       tagname.append("-History");
2086       std::string const tmp = Tags.FindS(tagname.c_str());
2087       if (tmp.empty() == true)
2088          continue;
2089
2090       string hash, filename;
2091       unsigned long long size;
2092       std::stringstream ss(tmp);
2093       ss.imbue(posix);
2094
2095       while (ss >> hash >> size >> filename)
2096       {
2097          if (unlikely(hash.empty() == true || filename.empty() == true))
2098             continue;
2099
2100          // see if we have a record for this file already
2101          std::vector<DiffInfo>::iterator cur = available_patches.begin();
2102          for (; cur != available_patches.end(); ++cur)
2103          {
2104             if (cur->file != filename)
2105                continue;
2106             cur->result_hashes.push_back(HashString(*type, hash));
2107             break;
2108          }
2109          if (cur != available_patches.end())
2110             continue;
2111          if (firstAcceptedHashes == true)
2112          {
2113             DiffInfo next;
2114             next.file = filename;
2115             next.result_hashes.push_back(HashString(*type, hash));
2116             next.result_hashes.FileSize(size);
2117             available_patches.push_back(next);
2118          }
2119          else
2120          {
2121             if (Debug == true)
2122                std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
2123                   << " wasn't in the list for the first parsed hash! (history)" << std::endl;
2124             break;
2125          }
2126       }
2127       firstAcceptedHashes = false;
2128    }
2129
2130    if (unlikely(available_patches.empty() == true))
2131    {
2132       if (Debug)
2133          std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": "
2134             << "Couldn't find any patches for the patch series." << std::endl;
2135       return false;
2136    }
2137
2138    for (auto type = types.crbegin(); type != types.crend(); ++type)
2139    {
2140       if (LocalHashes.find(*type) == NULL)
2141          continue;
2142
2143       std::string tagname = *type;
2144       tagname.append("-Patches");
2145       std::string const tmp = Tags.FindS(tagname.c_str());
2146       if (tmp.empty() == true)
2147          continue;
2148
2149       string hash, filename;
2150       unsigned long long size;
2151       std::stringstream ss(tmp);
2152       ss.imbue(posix);
2153
2154       while (ss >> hash >> size >> filename)
2155       {
2156          if (unlikely(hash.empty() == true || filename.empty() == true))
2157             continue;
2158
2159          // see if we have a record for this file already
2160          std::vector<DiffInfo>::iterator cur = available_patches.begin();
2161          for (; cur != available_patches.end(); ++cur)
2162          {
2163             if (cur->file != filename)
2164                continue;
2165             if (cur->patch_hashes.empty())
2166                cur->patch_hashes.FileSize(size);
2167             cur->patch_hashes.push_back(HashString(*type, hash));
2168             break;
2169          }
2170          if (cur != available_patches.end())
2171                continue;
2172          if (Debug == true)
2173             std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
2174                << " wasn't in the list for the first parsed hash! (patches)" << std::endl;
2175          break;
2176       }
2177    }
2178
2179    for (auto type = types.crbegin(); type != types.crend(); ++type)
2180    {
2181       std::string tagname = *type;
2182       tagname.append("-Download");
2183       std::string const tmp = Tags.FindS(tagname.c_str());
2184       if (tmp.empty() == true)
2185          continue;
2186
2187       string hash, filename;
2188       unsigned long long size;
2189       std::stringstream ss(tmp);
2190       ss.imbue(posix);
2191
2192       // FIXME: all of pdiff supports only .gz compressed patches
2193       while (ss >> hash >> size >> filename)
2194       {
2195          if (unlikely(hash.empty() == true || filename.empty() == true))
2196             continue;
2197          if (unlikely(APT::String::Endswith(filename, ".gz") == false))
2198             continue;
2199          filename.erase(filename.length() - 3);
2200
2201          // see if we have a record for this file already
2202          std::vector<DiffInfo>::iterator cur = available_patches.begin();
2203          for (; cur != available_patches.end(); ++cur)
2204          {
2205             if (cur->file != filename)
2206                continue;
2207             if (cur->download_hashes.empty())
2208                cur->download_hashes.FileSize(size);
2209             cur->download_hashes.push_back(HashString(*type, hash));
2210             break;
2211          }
2212          if (cur != available_patches.end())
2213                continue;
2214          if (Debug == true)
2215             std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
2216                << " wasn't in the list for the first parsed hash! (download)" << std::endl;
2217          break;
2218       }
2219    }
2220
2221
2222    bool foundStart = false;
2223    for (std::vector<DiffInfo>::iterator cur = available_patches.begin();
2224          cur != available_patches.end(); ++cur)
2225    {
2226       if (LocalHashes != cur->result_hashes)
2227          continue;
2228
2229       available_patches.erase(available_patches.begin(), cur);
2230       foundStart = true;
2231       break;
2232    }
2233
2234    if (foundStart == false || unlikely(available_patches.empty() == true))
2235    {
2236       if (Debug)
2237          std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": "
2238             << "Couldn't find the start of the patch series." << std::endl;
2239       return false;
2240    }
2241
2242    for (auto const &patch: available_patches)
2243       if (patch.result_hashes.usable() == false ||
2244             patch.patch_hashes.usable() == false ||
2245             patch.download_hashes.usable() == false)
2246       {
2247          if (Debug)
2248             std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": provides no usable hashes for " << patch.file
2249                << " so fallback to complete download" << std::endl;
2250          return false;
2251       }
2252
2253    // patching with too many files is rather slow compared to a fast download
2254    unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
2255    if (fileLimit != 0 && fileLimit < available_patches.size())
2256    {
2257       if (Debug)
2258          std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
2259             << ") so fallback to complete download" << std::endl;
2260       return false;
2261    }
2262
2263    // calculate the size of all patches we have to get
2264    unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100);
2265    if (sizeLimitPercent > 0)
2266    {
2267       unsigned long long downloadSize = std::accumulate(available_patches.begin(),
2268             available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) {
2269             return T + I.download_hashes.FileSize();
2270             });
2271       if (downloadSize != 0)
2272       {
2273          unsigned long long downloadSizeIdx = 0;
2274          auto const types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' ');
2275          for (auto const &t : types)
2276          {
2277             std::string MetaKey = Target.MetaKey;
2278             if (t != "uncompressed")
2279                MetaKey += '.' + t;
2280             HashStringList const hsl = GetExpectedHashesFor(MetaKey);
2281             if (unlikely(hsl.usable() == false))
2282                continue;
2283             downloadSizeIdx = hsl.FileSize();
2284             break;
2285          }
2286          unsigned long long const sizeLimit = downloadSizeIdx * sizeLimitPercent;
2287          if ((sizeLimit/100) < downloadSize)
2288          {
2289             if (Debug)
2290                std::clog << "Need " << downloadSize << " compressed bytes (Limit is " << (sizeLimit/100) << ", "
2291                   << "original is " << downloadSizeIdx << ") so fallback to complete download" << std::endl;
2292             return false;
2293          }
2294       }
2295    }
2296
2297    // we have something, queue the diffs
2298    string::size_type const last_space = Description.rfind(" ");
2299    if(last_space != string::npos)
2300       Description.erase(last_space, Description.size()-last_space);
2301
2302    /* decide if we should download patches one by one or in one go:
2303       The first is good if the server merges patches, but many don't so client
2304       based merging can be attempt in which case the second is better.
2305       "bad things" will happen if patches are merged on the server,
2306       but client side merging is attempt as well */
2307    bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
2308    if (pdiff_merge == true)
2309    {
2310       // reprepro adds this flag if it has merged patches on the server
2311       std::string const precedence = Tags.FindS("X-Patch-Precedence");
2312       pdiff_merge = (precedence != "merged");
2313    }
2314
2315    // clean the plate
2316    {
2317       std::string const Final = GetExistingFilename(CurrentPackagesFile);
2318       if (unlikely(Final.empty())) // because we wouldn't be called in such a case
2319          return false;
2320       std::string const PartialFile = GetPartialFileNameFromURI(Target.URI);
2321       if (FileExists(PartialFile) && RemoveFile("Bootstrap-linking", PartialFile) == false)
2322       {
2323          if (Debug)
2324             std::clog << "Bootstrap-linking for patching " << CurrentPackagesFile
2325                << " by removing stale " << PartialFile << " failed!" << std::endl;
2326          return false;
2327       }
2328       for (auto const &ext : APT::Configuration::getCompressorExtensions())
2329       {
2330          std::string const Partial = PartialFile + ext;
2331          if (FileExists(Partial) && RemoveFile("Bootstrap-linking", Partial) == false)
2332          {
2333             if (Debug)
2334                std::clog << "Bootstrap-linking for patching " << CurrentPackagesFile
2335                   << " by removing stale " << Partial << " failed!" << std::endl;
2336             return false;
2337          }
2338       }
2339       std::string const Ext = Final.substr(CurrentPackagesFile.length());
2340       std::string const Partial = PartialFile + Ext;
2341       if (symlink(Final.c_str(), Partial.c_str()) != 0)
2342       {
2343          if (Debug)
2344             std::clog << "Bootstrap-linking for patching " << CurrentPackagesFile
2345                << " by linking " << Final << " to " << Partial << " failed!" << std::endl;
2346          return false;
2347       }
2348    }
2349
2350    if (pdiff_merge == false)
2351       new pkgAcqIndexDiffs(Owner, TransactionManager, Target, available_patches);
2352    else
2353    {
2354       diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
2355       for(size_t i = 0; i < available_patches.size(); ++i)
2356          (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, TransactionManager,
2357                Target,
2358                available_patches[i],
2359                diffs);
2360    }
2361
2362    Complete = false;
2363    Status = StatDone;
2364    Dequeue();
2365    return true;
2366 }
2367                                                                         /*}}}*/
2368 void pkgAcqDiffIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
2369 {
2370    pkgAcqBaseIndex::Failed(Message,Cnf);
2371    Status = StatDone;
2372    ExpectedAdditionalItems = 0;
2373
2374    if(Debug)
2375       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
2376                 << "Falling back to normal index file acquire" << std::endl;
2377
2378    new pkgAcqIndex(Owner, TransactionManager, Target);
2379 }
2380                                                                         /*}}}*/
2381 void pkgAcqDiffIndex::Done(string const &Message,HashStringList const &Hashes,  /*{{{*/
2382                            pkgAcquire::MethodConfig const * const Cnf)
2383 {
2384    if(Debug)
2385       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
2386
2387    Item::Done(Message, Hashes, Cnf);
2388
2389    string const FinalFile = GetFinalFilename();
2390    if(StringToBool(LookupTag(Message,"IMS-Hit"),false))
2391       DestFile = FinalFile;
2392
2393    if(ParseDiffIndex(DestFile) == false)
2394    {
2395       Failed("Message: Couldn't parse pdiff index", Cnf);
2396       // queue for final move - this should happen even if we fail
2397       // while parsing (e.g. on sizelimit) and download the complete file.
2398       TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
2399       return;
2400    }
2401
2402    TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
2403
2404    Complete = true;
2405    Status = StatDone;
2406    Dequeue();
2407
2408    return;
2409 }
2410                                                                         /*}}}*/
2411 pkgAcqDiffIndex::~pkgAcqDiffIndex()
2412 {
2413    if (diffs != NULL)
2414       delete diffs;
2415 }
2416
2417 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
2418 // ---------------------------------------------------------------------
2419 /* The package diff is added to the queue. one object is constructed
2420  * for each diff and the index
2421  */
2422 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire * const Owner,
2423                                    pkgAcqMetaClearSig * const TransactionManager,
2424                                    IndexTarget const &Target,
2425                                    vector<DiffInfo> const &diffs)
2426    : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL),
2427      available_patches(diffs)
2428 {
2429    DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
2430
2431    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
2432
2433    Desc.Owner = this;
2434    Description = Target.Description;
2435    Desc.ShortDesc = Target.ShortDesc;
2436
2437    if(available_patches.empty() == true)
2438    {
2439       // we are done (yeah!), check hashes against the final file
2440       DestFile = GetKeepCompressedFileName(GetFinalFileNameFromURI(Target.URI), Target);
2441       Finish(true);
2442    }
2443    else
2444    {
2445       State = StateFetchDiff;
2446       QueueNextDiff();
2447    }
2448 }
2449                                                                         /*}}}*/
2450 void pkgAcqIndexDiffs::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
2451 {
2452    pkgAcqBaseIndex::Failed(Message,Cnf);
2453    Status = StatDone;
2454
2455    DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
2456    if(Debug)
2457       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
2458                 << "Falling back to normal index file acquire " << std::endl;
2459    RenameOnError(PDiffError);
2460    std::string const patchname = GetDiffsPatchFileName(DestFile);
2461    if (RealFileExists(patchname))
2462       Rename(patchname, patchname + ".FAILED");
2463    std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI));
2464    if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile))
2465       Rename(UnpatchedFile, UnpatchedFile + ".FAILED");
2466    new pkgAcqIndex(Owner, TransactionManager, Target);
2467    Finish();
2468 }
2469                                                                         /*}}}*/
2470 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
2471 void pkgAcqIndexDiffs::Finish(bool allDone)
2472 {
2473    if(Debug)
2474       std::clog << "pkgAcqIndexDiffs::Finish(): "
2475                 << allDone << " "
2476                 << Desc.URI << std::endl;
2477
2478    // we restore the original name, this is required, otherwise
2479    // the file will be cleaned
2480    if(allDone)
2481    {
2482       std::string const Final = GetKeepCompressedFileName(GetFinalFilename(), Target);
2483       TransactionManager->TransactionStageCopy(this, DestFile, Final);
2484
2485       // this is for the "real" finish
2486       Complete = true;
2487       Status = StatDone;
2488       Dequeue();
2489       if(Debug)
2490          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
2491       return;
2492    }
2493    else
2494       DestFile.clear();
2495
2496    if(Debug)
2497       std::clog << "Finishing: " << Desc.URI << std::endl;
2498    Complete = false;
2499    Status = StatDone;
2500    Dequeue();
2501    return;
2502 }
2503                                                                         /*}}}*/
2504 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
2505 {
2506    // calc sha1 of the just patched file
2507    std::string const PartialFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI));
2508    if(unlikely(PartialFile.empty()))
2509    {
2510       Failed("Message: The file " + GetPartialFileNameFromURI(Target.URI) + " isn't available", NULL);
2511       return false;
2512    }
2513
2514    FileFd fd(PartialFile, FileFd::ReadOnly, FileFd::Extension);
2515    Hashes LocalHashesCalc;
2516    LocalHashesCalc.AddFD(fd);
2517    HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
2518
2519    if(Debug)
2520       std::clog << "QueueNextDiff: " << PartialFile << " (" << LocalHashes.find(NULL)->toStr() << ")" << std::endl;
2521
2522    HashStringList const TargetFileHashes = GetExpectedHashesFor(Target.MetaKey);
2523    if (unlikely(LocalHashes.usable() == false || TargetFileHashes.usable() == false))
2524    {
2525       Failed("Local/Expected hashes are not usable for " + PartialFile, NULL);
2526       return false;
2527    }
2528
2529    // final file reached before all patches are applied
2530    if(LocalHashes == TargetFileHashes)
2531    {
2532       Finish(true);
2533       return true;
2534    }
2535
2536    // remove all patches until the next matching patch is found
2537    // this requires the Index file to be ordered
2538    available_patches.erase(available_patches.begin(),
2539          std::find_if(available_patches.begin(), available_patches.end(), [&](DiffInfo const &I) {
2540             return I.result_hashes == LocalHashes;
2541             }));
2542
2543    // error checking and falling back if no patch was found
2544    if(available_patches.empty() == true)
2545    {
2546       Failed("No patches left to reach target for " + PartialFile, NULL);
2547       return false;
2548    }
2549
2550    // queue the right diff
2551    Desc.URI = Target.URI + ".diff/" + available_patches[0].file + ".gz";
2552    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
2553    DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI + ".diff/" + available_patches[0].file), Target);
2554
2555    if(Debug)
2556       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
2557
2558    QueueURI(Desc);
2559
2560    return true;
2561 }
2562                                                                         /*}}}*/
2563 void pkgAcqIndexDiffs::Done(string const &Message, HashStringList const &Hashes,        /*{{{*/
2564                             pkgAcquire::MethodConfig const * const Cnf)
2565 {
2566    if (Debug)
2567       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
2568
2569    Item::Done(Message, Hashes, Cnf);
2570
2571    std::string const UncompressedUnpatchedFile = GetPartialFileNameFromURI(Target.URI);
2572    std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile);
2573    std::string const PatchFile = GetDiffsPatchFileName(UnpatchedFile);
2574    std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target);
2575
2576    switch (State)
2577    {
2578       // success in downloading a diff, enter ApplyDiff state
2579       case StateFetchDiff:
2580          Rename(DestFile, PatchFile);
2581          DestFile = GetKeepCompressedFileName(UncompressedUnpatchedFile + "-patched", Target);
2582          if(Debug)
2583             std::clog << "Sending to rred method: " << UnpatchedFile << std::endl;
2584          State = StateApplyDiff;
2585          Local = true;
2586          Desc.URI = "rred:" + UnpatchedFile;
2587          QueueURI(Desc);
2588          SetActiveSubprocess("rred");
2589          return;
2590       // success in download/apply a diff, queue next (if needed)
2591       case StateApplyDiff:
2592          // remove the just applied patch and base file
2593          available_patches.erase(available_patches.begin());
2594          RemoveFile("pkgAcqIndexDiffs::Done", PatchFile);
2595          RemoveFile("pkgAcqIndexDiffs::Done", UnpatchedFile);
2596          if(Debug)
2597             std::clog << "Moving patched file in place: " << std::endl
2598                << DestFile << " -> " << PatchedFile << std::endl;
2599          Rename(DestFile, PatchedFile);
2600
2601          // see if there is more to download
2602          if(available_patches.empty() == false)
2603          {
2604             new pkgAcqIndexDiffs(Owner, TransactionManager, Target, available_patches);
2605             Finish();
2606          } else {
2607             DestFile = PatchedFile;
2608             Finish(true);
2609          }
2610          return;
2611    }
2612 }
2613                                                                         /*}}}*/
2614 std::string pkgAcqIndexDiffs::Custom600Headers() const                  /*{{{*/
2615 {
2616    if(State != StateApplyDiff)
2617       return pkgAcqBaseIndex::Custom600Headers();
2618    std::ostringstream patchhashes;
2619    HashStringList const ExpectedHashes = available_patches[0].patch_hashes;
2620    for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
2621       patchhashes <<  "\nPatch-0-" << hs->HashType() << "-Hash: " << hs->HashValue();
2622    patchhashes << pkgAcqBaseIndex::Custom600Headers();
2623    return patchhashes.str();
2624 }
2625                                                                         /*}}}*/
2626 pkgAcqIndexDiffs::~pkgAcqIndexDiffs() {}
2627
2628 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor                 /*{{{*/
2629 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire * const Owner,
2630                                              pkgAcqMetaClearSig * const TransactionManager,
2631                                              IndexTarget const &Target,
2632                                              DiffInfo const &patch,
2633                                              std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
2634   : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL),
2635      patch(patch), allPatches(allPatches), State(StateFetchDiff)
2636 {
2637    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
2638
2639    Desc.Owner = this;
2640    Description = Target.Description;
2641    Desc.ShortDesc = Target.ShortDesc;
2642    Desc.URI = Target.URI + ".diff/" + patch.file + ".gz";
2643    Desc.Description = Description + " " + patch.file + ".pdiff";
2644    DestFile = GetPartialFileNameFromURI(Desc.URI);
2645
2646    if(Debug)
2647       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
2648
2649    QueueURI(Desc);
2650 }
2651                                                                         /*}}}*/
2652 void pkgAcqIndexMergeDiffs::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
2653 {
2654    if(Debug)
2655       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
2656
2657    pkgAcqBaseIndex::Failed(Message,Cnf);
2658    Status = StatDone;
2659
2660    // check if we are the first to fail, otherwise we are done here
2661    State = StateDoneDiff;
2662    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
2663          I != allPatches->end(); ++I)
2664       if ((*I)->State == StateErrorDiff)
2665       {
2666          State = StateErrorDiff;
2667          return;
2668       }
2669
2670    // first failure means we should fallback
2671    State = StateErrorDiff;
2672    if (Debug)
2673       std::clog << "Falling back to normal index file acquire" << std::endl;
2674    RenameOnError(PDiffError);
2675    if (RealFileExists(DestFile))
2676       Rename(DestFile, DestFile + ".FAILED");
2677    std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI));
2678    if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile))
2679       Rename(UnpatchedFile, UnpatchedFile + ".FAILED");
2680    DestFile.clear();
2681    new pkgAcqIndex(Owner, TransactionManager, Target);
2682 }
2683                                                                         /*}}}*/
2684 void pkgAcqIndexMergeDiffs::Done(string const &Message, HashStringList const &Hashes,   /*{{{*/
2685                             pkgAcquire::MethodConfig const * const Cnf)
2686 {
2687    if(Debug)
2688       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
2689
2690    Item::Done(Message, Hashes, Cnf);
2691
2692    if (std::any_of(allPatches->begin(), allPatches->end(),
2693             [](pkgAcqIndexMergeDiffs const * const P) { return P->State == StateErrorDiff; }))
2694    {
2695       if(Debug)
2696          std::clog << "Another patch failed already, no point in processing this one." << std::endl;
2697       State = StateErrorDiff;
2698       return;
2699    }
2700
2701    std::string const UncompressedUnpatchedFile = GetPartialFileNameFromURI(Target.URI);
2702    std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile);
2703    if (UnpatchedFile.empty())
2704    {
2705       _error->Fatal("Unpatched file %s doesn't exist (anymore)!", UncompressedUnpatchedFile.c_str());
2706       State = StateErrorDiff;
2707       return;
2708    }
2709    std::string const PatchFile = GetMergeDiffsPatchFileName(UnpatchedFile, patch.file);
2710    std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target);
2711
2712    switch (State)
2713    {
2714       case StateFetchDiff:
2715          Rename(DestFile, PatchFile);
2716
2717          // check if this is the last completed diff
2718          State = StateDoneDiff;
2719          for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
2720                I != allPatches->end(); ++I)
2721             if ((*I)->State != StateDoneDiff)
2722             {
2723                if(Debug)
2724                   std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
2725                return;
2726             }
2727          // this is the last completed diff, so we are ready to apply now
2728          DestFile = GetKeepCompressedFileName(UncompressedUnpatchedFile + "-patched", Target);
2729          if(Debug)
2730             std::clog << "Sending to rred method: " << UnpatchedFile << std::endl;
2731          State = StateApplyDiff;
2732          Local = true;
2733          Desc.URI = "rred:" + UnpatchedFile;
2734          QueueURI(Desc);
2735          SetActiveSubprocess("rred");
2736          return;
2737       case StateApplyDiff:
2738          // success in download & apply all diffs, finialize and clean up
2739          if(Debug)
2740             std::clog << "Queue patched file in place: " << std::endl
2741                << DestFile << " -> " << PatchedFile << std::endl;
2742
2743          // queue for copy by the transaction manager
2744          TransactionManager->TransactionStageCopy(this, DestFile, GetKeepCompressedFileName(GetFinalFilename(), Target));
2745
2746          // ensure the ed's are gone regardless of list-cleanup
2747          for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
2748                I != allPatches->end(); ++I)
2749             RemoveFile("pkgAcqIndexMergeDiffs::Done", GetMergeDiffsPatchFileName(UnpatchedFile, (*I)->patch.file));
2750          RemoveFile("pkgAcqIndexMergeDiffs::Done", UnpatchedFile);
2751
2752          // all set and done
2753          Complete = true;
2754          if(Debug)
2755             std::clog << "allDone: " << DestFile << "\n" << std::endl;
2756          return;
2757       case StateDoneDiff: _error->Fatal("Done called for %s which is in an invalid Done state", PatchFile.c_str()); break;
2758       case StateErrorDiff: _error->Fatal("Done called for %s which is in an invalid Error state", PatchFile.c_str()); break;
2759    }
2760 }
2761                                                                         /*}}}*/
2762 std::string pkgAcqIndexMergeDiffs::Custom600Headers() const             /*{{{*/
2763 {
2764    if(State != StateApplyDiff)
2765       return pkgAcqBaseIndex::Custom600Headers();
2766    std::ostringstream patchhashes;
2767    unsigned int seen_patches = 0;
2768    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
2769          I != allPatches->end(); ++I)
2770    {
2771       HashStringList const ExpectedHashes = (*I)->patch.patch_hashes;
2772       for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
2773          patchhashes <<  "\nPatch-" << seen_patches << "-" << hs->HashType() << "-Hash: " << hs->HashValue();
2774       ++seen_patches;
2775    }
2776    patchhashes << pkgAcqBaseIndex::Custom600Headers();
2777    return patchhashes.str();
2778 }
2779                                                                         /*}}}*/
2780 pkgAcqIndexMergeDiffs::~pkgAcqIndexMergeDiffs() {}
2781
2782 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
2783 pkgAcqIndex::pkgAcqIndex(pkgAcquire * const Owner,
2784                          pkgAcqMetaClearSig * const TransactionManager,
2785                          IndexTarget const &Target)
2786    : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), Stage(STAGE_DOWNLOAD),
2787    CompressionExtensions(Target.Option(IndexTarget::COMPRESSIONTYPES))
2788 {
2789    Init(Target.URI, Target.Description, Target.ShortDesc);
2790
2791    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
2792       std::clog << "New pkgIndex with TransactionManager "
2793                 << TransactionManager << std::endl;
2794 }
2795                                                                         /*}}}*/
2796 // AcqIndex::Init - defered Constructor                                 /*{{{*/
2797 static void NextCompressionExtension(std::string &CurrentCompressionExtension, std::string &CompressionExtensions, bool const preview)
2798 {
2799    size_t const nextExt = CompressionExtensions.find(' ');
2800    if (nextExt == std::string::npos)
2801    {
2802       CurrentCompressionExtension = CompressionExtensions;
2803       if (preview == false)
2804          CompressionExtensions.clear();
2805    }
2806    else
2807    {
2808       CurrentCompressionExtension = CompressionExtensions.substr(0, nextExt);
2809       if (preview == false)
2810          CompressionExtensions = CompressionExtensions.substr(nextExt+1);
2811    }
2812 }
2813 void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
2814                        string const &ShortDesc)
2815 {
2816    Stage = STAGE_DOWNLOAD;
2817
2818    DestFile = GetPartialFileNameFromURI(URI);
2819    NextCompressionExtension(CurrentCompressionExtension, CompressionExtensions, false);
2820
2821    // store file size of the download to ensure the fetcher gives
2822    // accurate progress reporting
2823    FileSize = GetExpectedHashes().FileSize();
2824
2825    if (CurrentCompressionExtension == "uncompressed")
2826    {
2827       Desc.URI = URI;
2828    }
2829    else if (CurrentCompressionExtension == "by-hash")
2830    {
2831       NextCompressionExtension(CurrentCompressionExtension, CompressionExtensions, true);
2832       if(unlikely(CurrentCompressionExtension.empty()))
2833          return;
2834       if (CurrentCompressionExtension != "uncompressed")
2835       {
2836          Desc.URI = URI + '.' + CurrentCompressionExtension;
2837          DestFile = DestFile + '.' + CurrentCompressionExtension;
2838       }
2839
2840       HashStringList const Hashes = GetExpectedHashes();
2841       HashString const * const TargetHash = Hashes.find(NULL);
2842       if (unlikely(TargetHash == nullptr))
2843          return;
2844       std::string const ByHash = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue();
2845       size_t const trailing_slash = Desc.URI.find_last_of("/");
2846       if (unlikely(trailing_slash == std::string::npos))
2847          return;
2848       Desc.URI = Desc.URI.replace(
2849             trailing_slash,
2850             Desc.URI.substr(trailing_slash+1).size()+1,
2851             ByHash);
2852    }
2853    else if (unlikely(CurrentCompressionExtension.empty()))
2854       return;
2855    else
2856    {
2857       Desc.URI = URI + '.' + CurrentCompressionExtension;
2858       DestFile = DestFile + '.' + CurrentCompressionExtension;
2859    }
2860
2861
2862    Desc.Description = URIDesc;
2863    Desc.Owner = this;
2864    Desc.ShortDesc = ShortDesc;
2865
2866    QueueURI(Desc);
2867 }
2868                                                                         /*}}}*/
2869 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2870 // ---------------------------------------------------------------------
2871 /* The only header we use is the last-modified header. */
2872 string pkgAcqIndex::Custom600Headers() const
2873 {
2874
2875    string msg = "\nIndex-File: true";
2876
2877    if (TransactionManager->LastMetaIndexParser == NULL)
2878    {
2879       std::string const Final = GetFinalFilename();
2880
2881       struct stat Buf;
2882       if (stat(Final.c_str(),&Buf) == 0)
2883          msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);
2884    }
2885
2886    if(Target.IsOptional)
2887       msg += "\nFail-Ignore: true";
2888
2889    return msg;
2890 }
2891                                                                         /*}}}*/
2892 // AcqIndex::Failed - getting the indexfile failed                      /*{{{*/
2893 void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
2894 {
2895    pkgAcqBaseIndex::Failed(Message,Cnf);
2896
2897    // authorisation matches will not be fixed by other compression types
2898    if (Status != StatAuthError)
2899    {
2900       if (CompressionExtensions.empty() == false)
2901       {
2902          Init(Target.URI, Desc.Description, Desc.ShortDesc);
2903          Status = StatIdle;
2904          return;
2905       }
2906    }
2907
2908    if(Target.IsOptional && GetExpectedHashes().empty() && Stage == STAGE_DOWNLOAD)
2909       Status = StatDone;
2910    else
2911       TransactionManager->AbortTransaction();
2912 }
2913                                                                         /*}}}*/
2914 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
2915 // ---------------------------------------------------------------------
2916 /* This goes through a number of states.. On the initial fetch the
2917    method could possibly return an alternate filename which points
2918    to the uncompressed version of the file. If this is so the file
2919    is copied into the partial directory. In all other cases the file
2920    is decompressed with a compressed uri. */
2921 void pkgAcqIndex::Done(string const &Message,
2922                        HashStringList const &Hashes,
2923                        pkgAcquire::MethodConfig const * const Cfg)
2924 {
2925    Item::Done(Message,Hashes,Cfg);
2926
2927    switch(Stage)
2928    {
2929       case STAGE_DOWNLOAD:
2930          StageDownloadDone(Message);
2931          break;
2932       case STAGE_DECOMPRESS_AND_VERIFY:
2933          StageDecompressDone();
2934          break;
2935    }
2936 }
2937                                                                         /*}}}*/
2938 // AcqIndex::StageDownloadDone - Queue for decompress and verify        /*{{{*/
2939 void pkgAcqIndex::StageDownloadDone(string const &Message)
2940 {
2941    Local = true;
2942    Complete = true;
2943
2944    std::string const AltFilename = LookupTag(Message,"Alt-Filename");
2945    std::string Filename = LookupTag(Message,"Filename");
2946
2947    // we need to verify the file against the current Release file again
2948    // on if-modfied-since hit to avoid a stale attack against us
2949    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2950    {
2951       // copy FinalFile into partial/ so that we check the hash again
2952       string const FinalFile = GetExistingFilename(GetFinalFileNameFromURI(Target.URI));
2953       if (symlink(FinalFile.c_str(), DestFile.c_str()) != 0)
2954          _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking final file %s back to %s failed", FinalFile.c_str(), DestFile.c_str());
2955       else
2956       {
2957          EraseFileName = DestFile;
2958          Filename = DestFile;
2959       }
2960       Stage = STAGE_DECOMPRESS_AND_VERIFY;
2961       Desc.URI = "store:" + Filename;
2962       QueueURI(Desc);
2963       SetActiveSubprocess(::URI(Desc.URI).Access);
2964       return;
2965    }
2966    // methods like file:// give us an alternative (uncompressed) file
2967    else if (Target.KeepCompressed == false && AltFilename.empty() == false)
2968    {
2969       Filename = AltFilename;
2970       EraseFileName.clear();
2971    }
2972    // Methods like e.g. "file:" will give us a (compressed) FileName that is
2973    // not the "DestFile" we set, in this case we uncompress from the local file
2974    else if (Filename != DestFile && RealFileExists(DestFile) == false)
2975    {
2976       // symlinking ensures that the filename can be used for compression detection
2977       // that is e.g. needed for by-hash which has no extension over file
2978       if (symlink(Filename.c_str(),DestFile.c_str()) != 0)
2979          _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", Filename.c_str(), DestFile.c_str());
2980       else
2981       {
2982          EraseFileName = DestFile;
2983          Filename = DestFile;
2984       }
2985    }
2986
2987    Stage = STAGE_DECOMPRESS_AND_VERIFY;
2988    DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
2989    if (Filename != DestFile && flExtension(Filename) == flExtension(DestFile))
2990       Desc.URI = "copy:" + Filename;
2991    else
2992       Desc.URI = "store:" + Filename;
2993    if (DestFile == Filename)
2994    {
2995       if (CurrentCompressionExtension == "uncompressed")
2996          return StageDecompressDone();
2997       DestFile = "/dev/null";
2998    }
2999
3000    if (EraseFileName.empty() && Filename != AltFilename)
3001       EraseFileName = Filename;
3002
3003    // queue uri for the next stage
3004    QueueURI(Desc);
3005    SetActiveSubprocess(::URI(Desc.URI).Access);
3006 }
3007                                                                         /*}}}*/
3008 // AcqIndex::StageDecompressDone - Final verification                   /*{{{*/
3009 void pkgAcqIndex::StageDecompressDone()
3010 {
3011    if (DestFile == "/dev/null")
3012       DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
3013
3014    // Done, queue for rename on transaction finished
3015    TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
3016 }
3017                                                                         /*}}}*/
3018 pkgAcqIndex::~pkgAcqIndex() {}
3019
3020
3021 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
3022 // ---------------------------------------------------------------------
3023 /* This just sets up the initial fetch environment and queues the first
3024    possibilitiy */
3025 pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sources,
3026                              pkgRecords * const Recs,pkgCache::VerIterator const &Version,
3027                              string &StoreFilename) :
3028                Item(Owner), d(NULL), LocalSource(false), Version(Version), Sources(Sources), Recs(Recs),
3029                StoreFilename(StoreFilename), Vf(Version.FileList()),
3030                Trusted(false)
3031 {
3032    Retries = _config->FindI("Acquire::Retries",0);
3033
3034    if (Version.Arch() == 0)
3035    {
3036       _error->Error(_("I wasn't able to locate a file for the %s package. "
3037                       "This might mean you need to manually fix this package. "
3038                       "(due to missing arch)"),
3039                     Version.ParentPkg().FullName().c_str());
3040       return;
3041    }
3042
3043    /* We need to find a filename to determine the extension. We make the
3044       assumption here that all the available sources for this version share
3045       the same extension.. */
3046    // Skip not source sources, they do not have file fields.
3047    for (; Vf.end() == false; ++Vf)
3048    {
3049       if (Vf.File().Flagged(pkgCache::Flag::NotSource))
3050          continue;
3051       break;
3052    }
3053
3054    // Does not really matter here.. we are going to fail out below
3055    if (Vf.end() != true)
3056    {
3057       // If this fails to get a file name we will bomb out below.
3058       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
3059       if (_error->PendingError() == true)
3060          return;
3061
3062       // Generate the final file name as: package_version_arch.foo
3063       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
3064                       QuoteString(Version.VerStr(),"_:") + '_' +
3065                       QuoteString(Version.Arch(),"_:.") +
3066                       "." + flExtension(Parse.FileName());
3067    }
3068
3069    // check if we have one trusted source for the package. if so, switch
3070    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
3071    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
3072    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
3073    bool seenUntrusted = false;
3074    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
3075    {
3076       pkgIndexFile *Index;
3077       if (Sources->FindIndex(i.File(),Index) == false)
3078          continue;
3079
3080       if (debugAuth == true)
3081          std::cerr << "Checking index: " << Index->Describe()
3082                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
3083
3084       if (Index->IsTrusted() == true)
3085       {
3086          Trusted = true;
3087          if (allowUnauth == false)
3088             break;
3089       }
3090       else
3091          seenUntrusted = true;
3092    }
3093
3094    // "allow-unauthenticated" restores apts old fetching behaviour
3095    // that means that e.g. unauthenticated file:// uris are higher
3096    // priority than authenticated http:// uris
3097    if (allowUnauth == true && seenUntrusted == true)
3098       Trusted = false;
3099
3100    // Select a source
3101    if (QueueNext() == false && _error->PendingError() == false)
3102       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
3103                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
3104 }
3105                                                                         /*}}}*/
3106 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
3107 // ---------------------------------------------------------------------
3108 /* This queues the next available file version for download. It checks if
3109    the archive is already available in the cache and stashs the MD5 for
3110    checking later. */
3111 bool pkgAcqArchive::QueueNext()
3112 {
3113    for (; Vf.end() == false; ++Vf)
3114    {
3115       pkgCache::PkgFileIterator const PkgF = Vf.File();
3116       // Ignore not source sources
3117       if (PkgF.Flagged(pkgCache::Flag::NotSource))
3118          continue;
3119
3120       // Try to cross match against the source list
3121       pkgIndexFile *Index;
3122       if (Sources->FindIndex(PkgF, Index) == false)
3123             continue;
3124       LocalSource = PkgF.Flagged(pkgCache::Flag::LocalSource);
3125
3126       // only try to get a trusted package from another source if that source
3127       // is also trusted
3128       if(Trusted && !Index->IsTrusted())
3129          continue;
3130
3131       // Grab the text package record
3132       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
3133       if (_error->PendingError() == true)
3134          return false;
3135
3136       string PkgFile = Parse.FileName();
3137       ExpectedHashes = Parse.Hashes();
3138
3139       if (PkgFile.empty() == true)
3140          return _error->Error(_("The package index files are corrupted. No Filename: "
3141                               "field for package %s."),
3142                               Version.ParentPkg().Name());
3143
3144       Desc.URI = Index->ArchiveURI(PkgFile);
3145       Desc.Description = Index->ArchiveInfo(Version);
3146       Desc.Owner = this;
3147       Desc.ShortDesc = Version.ParentPkg().FullName(true);
3148
3149       // See if we already have the file. (Legacy filenames)
3150       FileSize = Version->Size;
3151       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
3152       struct stat Buf;
3153       if (stat(FinalFile.c_str(),&Buf) == 0)
3154       {
3155          // Make sure the size matches
3156          if ((unsigned long long)Buf.st_size == Version->Size)
3157          {
3158             Complete = true;
3159             Local = true;
3160             Status = StatDone;
3161             StoreFilename = DestFile = FinalFile;
3162             return true;
3163          }
3164
3165          /* Hmm, we have a file and its size does not match, this means it is
3166             an old style mismatched arch */
3167          RemoveFile("pkgAcqArchive::QueueNext", FinalFile);
3168       }
3169
3170       // Check it again using the new style output filenames
3171       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
3172       if (stat(FinalFile.c_str(),&Buf) == 0)
3173       {
3174          // Make sure the size matches
3175          if ((unsigned long long)Buf.st_size == Version->Size)
3176          {
3177             Complete = true;
3178             Local = true;
3179             Status = StatDone;
3180             StoreFilename = DestFile = FinalFile;
3181             return true;
3182          }
3183
3184          /* Hmm, we have a file and its size does not match, this shouldn't
3185             happen.. */
3186          RemoveFile("pkgAcqArchive::QueueNext", FinalFile);
3187       }
3188
3189       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
3190
3191       // Check the destination file
3192       if (stat(DestFile.c_str(),&Buf) == 0)
3193       {
3194          // Hmm, the partial file is too big, erase it
3195          if ((unsigned long long)Buf.st_size > Version->Size)
3196             RemoveFile("pkgAcqArchive::QueueNext", DestFile);
3197          else
3198             PartialSize = Buf.st_size;
3199       }
3200
3201       // Disables download of archives - useful if no real installation follows,
3202       // e.g. if we are just interested in proposed installation order
3203       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
3204       {
3205          Complete = true;
3206          Local = true;
3207          Status = StatDone;
3208          StoreFilename = DestFile = FinalFile;
3209          return true;
3210       }
3211
3212       // Create the item
3213       Local = false;
3214       QueueURI(Desc);
3215
3216       ++Vf;
3217       return true;
3218    }
3219    return false;
3220 }
3221                                                                         /*}}}*/
3222 // AcqArchive::Done - Finished fetching                                 /*{{{*/
3223 // ---------------------------------------------------------------------
3224 /* */
3225 void pkgAcqArchive::Done(string const &Message, HashStringList const &Hashes,
3226                          pkgAcquire::MethodConfig const * const Cfg)
3227 {
3228    Item::Done(Message, Hashes, Cfg);
3229
3230    // Grab the output filename
3231    std::string const FileName = LookupTag(Message,"Filename");
3232    if (DestFile !=  FileName && RealFileExists(DestFile) == false)
3233    {
3234       StoreFilename = DestFile = FileName;
3235       Local = true;
3236       Complete = true;
3237       return;
3238    }
3239
3240    // Done, move it into position
3241    string const FinalFile = GetFinalFilename();
3242    Rename(DestFile,FinalFile);
3243    StoreFilename = DestFile = FinalFile;
3244    Complete = true;
3245 }
3246                                                                         /*}}}*/
3247 // AcqArchive::Failed - Failure handler                                 /*{{{*/
3248 // ---------------------------------------------------------------------
3249 /* Here we try other sources */
3250 void pkgAcqArchive::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
3251 {
3252    Item::Failed(Message,Cnf);
3253
3254    /* We don't really want to retry on failed media swaps, this prevents
3255       that. An interesting observation is that permanent failures are not
3256       recorded. */
3257    if (Cnf->Removable == true &&
3258        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
3259    {
3260       // Vf = Version.FileList();
3261       while (Vf.end() == false) ++Vf;
3262       StoreFilename = string();
3263       return;
3264    }
3265
3266    Status = StatIdle;
3267    if (QueueNext() == false)
3268    {
3269       // This is the retry counter
3270       if (Retries != 0 &&
3271           Cnf->LocalOnly == false &&
3272           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
3273       {
3274          Retries--;
3275          Vf = Version.FileList();
3276          if (QueueNext() == true)
3277             return;
3278       }
3279
3280       StoreFilename = string();
3281       Status = StatError;
3282    }
3283 }
3284                                                                         /*}}}*/
3285 APT_PURE bool pkgAcqArchive::IsTrusted() const                          /*{{{*/
3286 {
3287    return Trusted;
3288 }
3289                                                                         /*}}}*/
3290 void pkgAcqArchive::Finished()                                          /*{{{*/
3291 {
3292    if (Status == pkgAcquire::Item::StatDone &&
3293        Complete == true)
3294       return;
3295    StoreFilename = string();
3296 }
3297                                                                         /*}}}*/
3298 std::string pkgAcqArchive::DescURI() const                              /*{{{*/
3299 {
3300    return Desc.URI;
3301 }
3302                                                                         /*}}}*/
3303 std::string pkgAcqArchive::ShortDesc() const                            /*{{{*/
3304 {
3305    return Desc.ShortDesc;
3306 }
3307                                                                         /*}}}*/
3308 pkgAcqArchive::~pkgAcqArchive() {}
3309
3310 // AcqChangelog::pkgAcqChangelog - Constructors                         /*{{{*/
3311 class pkgAcqChangelog::Private
3312 {
3313    public:
3314    std::string FinalFile;
3315 };
3316 pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::VerIterator const &Ver,
3317       std::string const &DestDir, std::string const &DestFilename) :
3318    pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(Ver.SourcePkgName()), SrcVersion(Ver.SourceVerStr())
3319 {
3320    Desc.URI = URI(Ver);
3321    Init(DestDir, DestFilename);
3322 }
3323 // some parameters are char* here as they come likely from char* interfaces – which can also return NULL
3324 pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::RlsFileIterator const &RlsFile,
3325       char const * const Component, char const * const SrcName, char const * const SrcVersion,
3326       const string &DestDir, const string &DestFilename) :
3327    pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(SrcName), SrcVersion(SrcVersion)
3328 {
3329    Desc.URI = URI(RlsFile, Component, SrcName, SrcVersion);
3330    Init(DestDir, DestFilename);
3331 }
3332 pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner,
3333       std::string const &URI, char const * const SrcName, char const * const SrcVersion,
3334       const string &DestDir, const string &DestFilename) :
3335    pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(SrcName), SrcVersion(SrcVersion)
3336 {
3337    Desc.URI = URI;
3338    Init(DestDir, DestFilename);
3339 }
3340 void pkgAcqChangelog::Init(std::string const &DestDir, std::string const &DestFilename)
3341 {
3342    if (Desc.URI.empty())
3343    {
3344       Status = StatError;
3345       // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1
3346       strprintf(ErrorText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str());
3347       // Let the error message print something sensible rather than "Failed to fetch /"
3348       if (DestFilename.empty())
3349          DestFile = SrcName + ".changelog";
3350       else
3351          DestFile = DestFilename;
3352       Desc.URI = "changelog:/" + DestFile;
3353       return;
3354    }
3355
3356    std::string DestFileName;
3357    if (DestFilename.empty())
3358       DestFileName = flCombine(DestFile, SrcName + ".changelog");
3359    else
3360       DestFileName = flCombine(DestFile, DestFilename);
3361
3362    std::string const SandboxUser = _config->Find("APT::Sandbox::User");
3363    std::string const systemTemp = GetTempDir(SandboxUser);
3364    char tmpname[1000];
3365    snprintf(tmpname, sizeof(tmpname), "%s/apt-changelog-XXXXXX", systemTemp.c_str());
3366    if (NULL == mkdtemp(tmpname))
3367    {
3368       _error->Errno("mkdtemp", "mkdtemp failed in changelog acquire of %s %s", SrcName.c_str(), SrcVersion.c_str());
3369       Status = StatError;
3370       return;
3371    }
3372    TemporaryDirectory = tmpname;
3373
3374    ChangeOwnerAndPermissionOfFile("Item::QueueURI", TemporaryDirectory.c_str(),
3375          SandboxUser.c_str(), "root", 0700);
3376
3377    DestFile = flCombine(TemporaryDirectory, DestFileName);
3378    if (DestDir.empty() == false)
3379    {
3380       d->FinalFile = flCombine(DestDir, DestFileName);
3381       if (RealFileExists(d->FinalFile))
3382       {
3383          FileFd file1, file2;
3384          if (file1.Open(DestFile, FileFd::WriteOnly | FileFd::Create | FileFd::Exclusive) &&
3385                file2.Open(d->FinalFile, FileFd::ReadOnly) && CopyFile(file2, file1))
3386          {
3387             struct timeval times[2];
3388             times[0].tv_sec = times[1].tv_sec = file2.ModificationTime();
3389             times[0].tv_usec = times[1].tv_usec = 0;
3390             utimes(DestFile.c_str(), times);
3391          }
3392       }
3393    }
3394
3395    Desc.ShortDesc = "Changelog";
3396    strprintf(Desc.Description, "%s %s %s Changelog", URI::SiteOnly(Desc.URI).c_str(), SrcName.c_str(), SrcVersion.c_str());
3397    Desc.Owner = this;
3398    QueueURI(Desc);
3399 }
3400                                                                         /*}}}*/
3401 std::string pkgAcqChangelog::URI(pkgCache::VerIterator const &Ver)      /*{{{*/
3402 {
3403    std::string const confOnline = "Acquire::Changelogs::AlwaysOnline";
3404    bool AlwaysOnline = _config->FindB(confOnline, false);
3405    if (AlwaysOnline == false)
3406       for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF)
3407       {
3408          pkgCache::PkgFileIterator const PF = VF.File();
3409          if (PF.Flagged(pkgCache::Flag::NotSource) || PF->Release == 0)
3410             continue;
3411          pkgCache::RlsFileIterator const RF = PF.ReleaseFile();
3412          if (RF->Origin != 0 && _config->FindB(confOnline + "::Origin::" + RF.Origin(), false))
3413          {
3414             AlwaysOnline = true;
3415             break;
3416          }
3417       }
3418    if (AlwaysOnline == false)
3419    {
3420       pkgCache::PkgIterator const Pkg = Ver.ParentPkg();
3421       if (Pkg->CurrentVer != 0 && Pkg.CurrentVer() == Ver)
3422       {
3423          std::string const basename = std::string("/usr/share/doc/") + Pkg.Name() + "/changelog";
3424          std::string const debianname = basename + ".Debian";
3425          if (FileExists(debianname))
3426             return "copy://" + debianname;
3427          else if (FileExists(debianname + ".gz"))
3428             return "gzip://" + debianname + ".gz";
3429          else if (FileExists(basename))
3430             return "copy://" + basename;
3431          else if (FileExists(basename + ".gz"))
3432             return "gzip://" + basename + ".gz";
3433       }
3434    }
3435
3436    char const * const SrcName = Ver.SourcePkgName();
3437    char const * const SrcVersion = Ver.SourceVerStr();
3438    // find the first source for this version which promises a changelog
3439    for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF)
3440    {
3441       pkgCache::PkgFileIterator const PF = VF.File();
3442       if (PF.Flagged(pkgCache::Flag::NotSource) || PF->Release == 0)
3443          continue;
3444       pkgCache::RlsFileIterator const RF = PF.ReleaseFile();
3445       std::string const uri = URI(RF, PF.Component(), SrcName, SrcVersion);
3446       if (uri.empty())
3447          continue;
3448       return uri;
3449    }
3450    return "";
3451 }
3452 std::string pkgAcqChangelog::URITemplate(pkgCache::RlsFileIterator const &Rls)
3453 {
3454    if (Rls.end() == true || (Rls->Label == 0 && Rls->Origin == 0))
3455       return "";
3456    std::string const serverConfig = "Acquire::Changelogs::URI";
3457    std::string server;
3458 #define APT_EMPTY_SERVER \
3459    if (server.empty() == false) \
3460    { \
3461       if (server != "no") \
3462          return server; \
3463       return ""; \
3464    }
3465 #define APT_CHECK_SERVER(X, Y) \
3466    if (Rls->X != 0) \
3467    { \
3468       std::string const specialServerConfig = serverConfig + "::" + Y + #X + "::" + Rls.X(); \
3469       server = _config->Find(specialServerConfig); \
3470       APT_EMPTY_SERVER \
3471    }
3472    // this way e.g. Debian-Security can fallback to Debian
3473    APT_CHECK_SERVER(Label, "Override::")
3474    APT_CHECK_SERVER(Origin, "Override::")
3475
3476    if (RealFileExists(Rls.FileName()))
3477    {
3478       _error->PushToStack();
3479       FileFd rf;
3480       /* This can be costly. A caller wanting to get millions of URIs might
3481          want to do this on its own once and use Override settings.
3482          We don't do this here as Origin/Label are not as unique as they
3483          should be so this could produce request order-dependent anomalies */
3484       if (OpenMaybeClearSignedFile(Rls.FileName(), rf) == true)
3485       {
3486          pkgTagFile TagFile(&rf, rf.Size());
3487          pkgTagSection Section;
3488          if (TagFile.Step(Section) == true)
3489             server = Section.FindS("Changelogs");
3490       }
3491       _error->RevertToStack();
3492       APT_EMPTY_SERVER
3493    }
3494
3495    APT_CHECK_SERVER(Label, "")
3496    APT_CHECK_SERVER(Origin, "")
3497 #undef APT_CHECK_SERVER
3498 #undef APT_EMPTY_SERVER
3499    return "";
3500 }
3501 std::string pkgAcqChangelog::URI(pkgCache::RlsFileIterator const &Rls,
3502          char const * const Component, char const * const SrcName,
3503          char const * const SrcVersion)
3504 {
3505    return URI(URITemplate(Rls), Component, SrcName, SrcVersion);
3506 }
3507 std::string pkgAcqChangelog::URI(std::string const &Template,
3508          char const * const Component, char const * const SrcName,
3509          char const * const SrcVersion)
3510 {
3511    if (Template.find("@CHANGEPATH@") == std::string::npos)
3512       return "";
3513
3514    // the path is: COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER, e.g. main/a/apt/1.1 or contrib/liba/libapt/2.0
3515    std::string Src = SrcName;
3516    std::string path = APT::String::Startswith(SrcName, "lib") ? Src.substr(0, 4) : Src.substr(0,1);
3517    path.append("/").append(Src).append("/");
3518    path.append(Src).append("_").append(StripEpoch(SrcVersion));
3519    // we omit component for releases without one (= flat-style repositories)
3520    if (Component != NULL && strlen(Component) != 0)
3521       path = std::string(Component) + "/" + path;
3522
3523    return SubstVar(Template, "@CHANGEPATH@", path);
3524 }
3525                                                                         /*}}}*/
3526 // AcqChangelog::Failed - Failure handler                               /*{{{*/
3527 void pkgAcqChangelog::Failed(string const &Message, pkgAcquire::MethodConfig const * const Cnf)
3528 {
3529    Item::Failed(Message,Cnf);
3530
3531    std::string errText;
3532    // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1
3533    strprintf(errText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str());
3534
3535    // Error is probably something techy like 404 Not Found
3536    if (ErrorText.empty())
3537       ErrorText = errText;
3538    else
3539       ErrorText = errText + " (" + ErrorText + ")";
3540 }
3541                                                                         /*}}}*/
3542 // AcqChangelog::Done - Item downloaded OK                              /*{{{*/
3543 void pkgAcqChangelog::Done(string const &Message,HashStringList const &CalcHashes,
3544                       pkgAcquire::MethodConfig const * const Cnf)
3545 {
3546    Item::Done(Message,CalcHashes,Cnf);
3547    if (d->FinalFile.empty() == false)
3548    {
3549       if (RemoveFile("pkgAcqChangelog::Done", d->FinalFile) == false ||
3550             Rename(DestFile, d->FinalFile) == false)
3551          Status = StatError;
3552    }
3553
3554    Complete = true;
3555 }
3556                                                                         /*}}}*/
3557 pkgAcqChangelog::~pkgAcqChangelog()                                     /*{{{*/
3558 {
3559    if (TemporaryDirectory.empty() == false)
3560    {
3561       RemoveFile("~pkgAcqChangelog", DestFile);
3562       rmdir(TemporaryDirectory.c_str());
3563    }
3564    delete d;
3565 }
3566                                                                         /*}}}*/
3567
3568 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
3569 pkgAcqFile::pkgAcqFile(pkgAcquire * const Owner,string const &URI, HashStringList const &Hashes,
3570                        unsigned long long const Size,string const &Dsc,string const &ShortDesc,
3571                        const string &DestDir, const string &DestFilename,
3572                        bool const IsIndexFile) :
3573                        Item(Owner), d(NULL), IsIndexFile(IsIndexFile), ExpectedHashes(Hashes)
3574 {
3575    Retries = _config->FindI("Acquire::Retries",0);
3576
3577    if(!DestFilename.empty())
3578       DestFile = DestFilename;
3579    else if(!DestDir.empty())
3580       DestFile = DestDir + "/" + flNotDir(URI);
3581    else
3582       DestFile = flNotDir(URI);
3583
3584    // Create the item
3585    Desc.URI = URI;
3586    Desc.Description = Dsc;
3587    Desc.Owner = this;
3588
3589    // Set the short description to the archive component
3590    Desc.ShortDesc = ShortDesc;
3591
3592    // Get the transfer sizes
3593    FileSize = Size;
3594    struct stat Buf;
3595    if (stat(DestFile.c_str(),&Buf) == 0)
3596    {
3597       // Hmm, the partial file is too big, erase it
3598       if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
3599          RemoveFile("pkgAcqFile", DestFile);
3600       else
3601          PartialSize = Buf.st_size;
3602    }
3603
3604    QueueURI(Desc);
3605 }
3606                                                                         /*}}}*/
3607 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
3608 void pkgAcqFile::Done(string const &Message,HashStringList const &CalcHashes,
3609                       pkgAcquire::MethodConfig const * const Cnf)
3610 {
3611    Item::Done(Message,CalcHashes,Cnf);
3612
3613    std::string const FileName = LookupTag(Message,"Filename");
3614    Complete = true;
3615
3616    // The files timestamp matches
3617    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
3618       return;
3619
3620    // We have to copy it into place
3621    if (RealFileExists(DestFile.c_str()) == false)
3622    {
3623       Local = true;
3624       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
3625           Cnf->Removable == true)
3626       {
3627          Desc.URI = "copy:" + FileName;
3628          QueueURI(Desc);
3629          return;
3630       }
3631
3632       // Erase the file if it is a symlink so we can overwrite it
3633       struct stat St;
3634       if (lstat(DestFile.c_str(),&St) == 0)
3635       {
3636          if (S_ISLNK(St.st_mode) != 0)
3637             RemoveFile("pkgAcqFile::Done", DestFile);
3638       }
3639
3640       // Symlink the file
3641       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
3642       {
3643          _error->PushToStack();
3644          _error->Errno("pkgAcqFile::Done", "Symlinking file %s failed", DestFile.c_str());
3645          std::stringstream msg;
3646          _error->DumpErrors(msg, GlobalError::DEBUG, false);
3647          _error->RevertToStack();
3648          ErrorText = msg.str();
3649          Status = StatError;
3650          Complete = false;
3651       }
3652    }
3653 }
3654                                                                         /*}}}*/
3655 // AcqFile::Failed - Failure handler                                    /*{{{*/
3656 // ---------------------------------------------------------------------
3657 /* Here we try other sources */
3658 void pkgAcqFile::Failed(string const &Message, pkgAcquire::MethodConfig const * const Cnf)
3659 {
3660    Item::Failed(Message,Cnf);
3661
3662    // This is the retry counter
3663    if (Retries != 0 &&
3664        Cnf->LocalOnly == false &&
3665        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
3666    {
3667       --Retries;
3668       QueueURI(Desc);
3669       Status = StatIdle;
3670       return;
3671    }
3672
3673 }
3674                                                                         /*}}}*/
3675 string pkgAcqFile::Custom600Headers() const                             /*{{{*/
3676 {
3677    if (IsIndexFile)
3678       return "\nIndex-File: true";
3679    return "";
3680 }
3681                                                                         /*}}}*/
3682 pkgAcqFile::~pkgAcqFile() {}