apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/tagfile.h>
  26 #include <apt-pkg/metaindex.h>
  27 #include <apt-pkg/acquire.h>
  28 #include <apt-pkg/hashes.h>
  29 #include <apt-pkg/indexfile.h>
  30 #include <apt-pkg/pkgcache.h>
  31 #include <apt-pkg/cacheiterators.h>
  32 #include <apt-pkg/pkgrecords.h>
  33 #include <apt-pkg/gpgv.h>
  34
  35 #include <algorithm>
  36 #include <stddef.h>
  37 #include <stdlib.h>
  38 #include <string.h>
  39 #include <iostream>
  40 #include <vector>
  41 #include <sys/stat.h>
  42 #include <unistd.h>
  43 #include <errno.h>
  44 #include <string>
  45 #include <stdio.h>
  46 #include <ctime>
  47 #include <sstream>
  48 #include <numeric>
  49
  50 #include <apti18n.h>
  51                                                                         /*}}}*/
  52
  53 using namespace std;
  54
  55 static void printHashSumComparison(std::string const &URI, HashStringList const &Expected, HashStringList const &Actual) /*{{{*/
  56 {
  57    if (_config->FindB("Debug::Acquire::HashSumMismatch", false) == false)
  58       return;
  59    std::cerr << std::endl << URI << ":" << std::endl << " Expected Hash: " << std::endl;
  60    for (HashStringList::const_iterator hs = Expected.begin(); hs != Expected.end(); ++hs)
  61       std::cerr <<  "\t- " << hs->toStr() << std::endl;
  62    std::cerr << " Actual Hash: " << std::endl;
  63    for (HashStringList::const_iterator hs = Actual.begin(); hs != Actual.end(); ++hs)
  64       std::cerr <<  "\t- " << hs->toStr() << std::endl;
  65 }
  66                                                                         /*}}}*/
  67 static std::string GetPartialFileName(std::string const &file)          /*{{{*/
  68 {
  69    std::string DestFile = _config->FindDir("Dir::State::lists") + "partial/";
  70    DestFile += file;
  71    return DestFile;
  72 }
  73                                                                         /*}}}*/
  74 static std::string GetPartialFileNameFromURI(std::string const &uri)    /*{{{*/
  75 {
  76    return GetPartialFileName(URItoFileName(uri));
  77 }
  78                                                                         /*}}}*/
  79 static std::string GetFinalFileNameFromURI(std::string const &uri)      /*{{{*/
  80 {
  81    return _config->FindDir("Dir::State::lists") + URItoFileName(uri);
  82 }
  83                                                                         /*}}}*/
  84 static std::string GetKeepCompressedFileName(std::string file, IndexTarget const &Target)/*{{{*/
  85 {
  86    if (Target.KeepCompressed == false)
  87       return file;
  88
  89    std::string const KeepCompressedAs = Target.Option(IndexTarget::KEEPCOMPRESSEDAS);
  90    if (KeepCompressedAs.empty() == false)
  91    {
  92       std::string const ext = KeepCompressedAs.substr(0, KeepCompressedAs.find(' '));
  93       if (ext != "uncompressed")
  94          file.append(".").append(ext);
  95    }
  96    return file;
  97 }
  98                                                                         /*}}}*/
  99 static std::string GetMergeDiffsPatchFileName(std::string const &Final, std::string const &Patch)/*{{{*/
 100 {
 101    // rred expects the patch as $FinalFile.ed.$patchname.gz
 102    return Final + ".ed." + Patch + ".gz";
 103 }
 104                                                                         /*}}}*/
 105 static std::string GetDiffsPatchFileName(std::string const &Final)      /*{{{*/
 106 {
 107    // rred expects the patch as $FinalFile.ed
 108    return Final + ".ed";
 109 }
 110                                                                         /*}}}*/
 111 static std::string GetExistingFilename(std::string const &File)         /*{{{*/
 112 {
 113    if (RealFileExists(File))
 114       return File;
 115    for (auto const &type : APT::Configuration::getCompressorExtensions())
 116    {
 117       std::string const Final = File + type;
 118       if (RealFileExists(Final))
 119          return Final;
 120    }
 121    return "";
 122 }
 123                                                                         /*}}}*/
 124 static std::string GetDiffIndexFileName(std::string const &Name)        /*{{{*/
 125 {
 126    return Name + ".diff/Index";
 127 }
 128                                                                         /*}}}*/
 129 static std::string GetDiffIndexURI(IndexTarget const &Target)           /*{{{*/
 130 {
 131    return Target.URI + ".diff/Index";
 132 }
 133                                                                         /*}}}*/
 134
 135 static void ReportMirrorFailureToCentral(pkgAcquire::Item const &I, std::string const &FailCode, std::string const &Details)/*{{{*/
 136 {
 137    // we only act if a mirror was used at all
 138    if(I.UsedMirror.empty())
 139       return;
 140 #if 0
 141    std::cerr << "\nReportMirrorFailure: "
 142              << UsedMirror
 143              << " Uri: " << DescURI()
 144              << " FailCode: "
 145              << FailCode << std::endl;
 146 #endif
 147    string const report = _config->Find("Methods::Mirror::ProblemReporting",
 148                                  "/usr/lib/apt/apt-report-mirror-failure");
 149    if(!FileExists(report))
 150       return;
 151
 152    std::vector<char const*> const Args = {
 153       report.c_str(),
 154       I.UsedMirror.c_str(),
 155       I.DescURI().c_str(),
 156       FailCode.c_str(),
 157       Details.c_str(),
 158       NULL
 159    };
 160
 161    pid_t pid = ExecFork();
 162    if(pid < 0)
 163    {
 164       _error->Error("ReportMirrorFailure Fork failed");
 165       return;
 166    }
 167    else if(pid == 0)
 168    {
 169       execvp(Args[0], (char**)Args.data());
 170       std::cerr << "Could not exec " << Args[0] << std::endl;
 171       _exit(100);
 172    }
 173    if(!ExecWait(pid, "report-mirror-failure"))
 174       _error->Warning("Couldn't report problem to '%s'", report.c_str());
 175 }
 176                                                                         /*}}}*/
 177
 178 static APT_NONNULL(2) bool MessageInsecureRepository(bool const isError, char const * const msg, std::string const &repo)/*{{{*/
 179 {
 180    std::string m;
 181    strprintf(m, msg, repo.c_str());
 182    if (isError)
 183    {
 184       _error->Error("%s", m.c_str());
 185       _error->Notice("%s", _("Updating from such a repository can't be done securely, and is therefore disabled by default."));
 186    }
 187    else
 188    {
 189       _error->Warning("%s", m.c_str());
 190       _error->Notice("%s", _("Data from such a repository can't be authenticated and is therefore potentially dangerous to use."));
 191    }
 192    _error->Notice("%s", _("See apt-secure(8) manpage for repository creation and user configuration details."));
 193    return false;
 194 }
 195                                                                         /*}}}*/
 196 // AllowInsecureRepositories                                            /*{{{*/
 197 enum class InsecureType { UNSIGNED, WEAK, NORELEASE };
 198 static bool TargetIsAllowedToBe(IndexTarget const &Target, InsecureType const type)
 199 {
 200    if (_config->FindB("Acquire::AllowInsecureRepositories"))
 201       return true;
 202
 203    if (Target.OptionBool(IndexTarget::ALLOW_INSECURE))
 204       return true;
 205
 206    switch (type)
 207    {
 208       case InsecureType::UNSIGNED: break;
 209       case InsecureType::NORELEASE: break;
 210       case InsecureType::WEAK:
 211          if (_config->FindB("Acquire::AllowWeakRepositories"))
 212             return true;
 213          if (Target.OptionBool(IndexTarget::ALLOW_WEAK))
 214             return true;
 215          break;
 216    }
 217    return false;
 218 }
 219 static bool APT_NONNULL(3, 4, 5) AllowInsecureRepositories(InsecureType const msg, std::string const &repo,
 220       metaIndex const * const MetaIndexParser, pkgAcqMetaClearSig * const TransactionManager, pkgAcquire::Item * const I)
 221 {
 222    // we skip weak downgrades as its unlikely that a repository gets really weaker –
 223    // its more realistic that apt got pickier in a newer version
 224    if (msg != InsecureType::WEAK)
 225    {
 226       std::string const FinalInRelease = TransactionManager->GetFinalFilename();
 227       std::string const FinalReleasegpg = FinalInRelease.substr(0, FinalInRelease.length() - strlen("InRelease")) + "Release.gpg";
 228       if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease))
 229       {
 230          char const * msgstr = nullptr;
 231          switch (msg)
 232          {
 233             case InsecureType::UNSIGNED: msgstr = _("The repository '%s' is no longer signed."); break;
 234             case InsecureType::NORELEASE: msgstr = _("The repository '%s' does no longer have a Release file."); break;
 235             case InsecureType::WEAK: /* unreachable */ break;
 236          }
 237          if (_config->FindB("Acquire::AllowDowngradeToInsecureRepositories") ||
 238                TransactionManager->Target.OptionBool(IndexTarget::ALLOW_DOWNGRADE_TO_INSECURE))
 239          {
 240             // meh, the users wants to take risks (we still mark the packages
 241             // from this repository as unauthenticated)
 242             _error->Warning(msgstr, repo.c_str());
 243             _error->Warning(_("This is normally not allowed, but the option "
 244                      "Acquire::AllowDowngradeToInsecureRepositories was "
 245                      "given to override it."));
 246          } else {
 247             MessageInsecureRepository(true, msgstr, repo);
 248             TransactionManager->AbortTransaction();
 249             I->Status = pkgAcquire::Item::StatError;
 250             return false;
 251          }
 252       }
 253    }
 254
 255    if(MetaIndexParser->GetTrusted() == metaIndex::TRI_YES)
 256       return true;
 257
 258    char const * msgstr = nullptr;
 259    switch (msg)
 260    {
 261       case InsecureType::UNSIGNED: msgstr = _("The repository '%s' is not signed."); break;
 262       case InsecureType::NORELEASE: msgstr = _("The repository '%s' does not have a Release file."); break;
 263       case InsecureType::WEAK: msgstr = _("The repository '%s' provides only weak security information."); break;
 264    }
 265
 266    if (TargetIsAllowedToBe(TransactionManager->Target, msg) == true)
 267    {
 268       MessageInsecureRepository(false, msgstr, repo);
 269       return true;
 270    }
 271
 272    MessageInsecureRepository(true, msgstr, repo);
 273    TransactionManager->AbortTransaction();
 274    I->Status = pkgAcquire::Item::StatError;
 275    return false;
 276 }
 277                                                                         /*}}}*/
 278 static HashStringList GetExpectedHashesFromFor(metaIndex * const Parser, std::string const &MetaKey)/*{{{*/
 279 {
 280    if (Parser == NULL)
 281       return HashStringList();
 282    metaIndex::checkSum * const R = Parser->Lookup(MetaKey);
 283    if (R == NULL)
 284       return HashStringList();
 285    return R->Hashes;
 286 }
 287                                                                         /*}}}*/
 288
 289 // all ::HashesRequired and ::GetExpectedHashes implementations         /*{{{*/
 290 /* ::GetExpectedHashes is abstract and has to be implemented by all subclasses.
 291    It is best to implement it as broadly as possible, while ::HashesRequired defaults
 292    to true and should be as restrictive as possible for false cases. Note that if
 293    a hash is returned by ::GetExpectedHashes it must match. Only if it doesn't
 294    ::HashesRequired is called to evaluate if its okay to have no hashes. */
 295 APT_CONST bool pkgAcqTransactionItem::HashesRequired() const
 296 {
 297    /* signed repositories obviously have a parser and good hashes.
 298       unsigned repositories, too, as even if we can't trust them for security,
 299       we can at least trust them for integrity of the download itself.
 300       Only repositories without a Release file can (obviously) not have
 301       hashes – and they are very uncommon and strongly discouraged */
 302    if (TransactionManager->MetaIndexParser->GetLoadedSuccessfully() != metaIndex::TRI_YES)
 303       return false;
 304    if (TargetIsAllowedToBe(Target, InsecureType::WEAK))
 305    {
 306       /* If we allow weak hashes, we check that we have some (weak) and then
 307          declare hashes not needed. That will tip us in the right direction
 308          as if hashes exist, they will be used, even if not required */
 309       auto const hsl = GetExpectedHashes();
 310       if (hsl.usable())
 311          return true;
 312       if (hsl.empty() == false)
 313          return false;
 314    }
 315    return true;
 316 }
 317 HashStringList pkgAcqTransactionItem::GetExpectedHashes() const
 318 {
 319    return GetExpectedHashesFor(GetMetaKey());
 320 }
 321
 322 APT_CONST bool pkgAcqMetaBase::HashesRequired() const
 323 {
 324    // Release and co have no hashes 'by design'.
 325    return false;
 326 }
 327 HashStringList pkgAcqMetaBase::GetExpectedHashes() const
 328 {
 329    return HashStringList();
 330 }
 331
 332 APT_CONST bool pkgAcqIndexDiffs::HashesRequired() const
 333 {
 334    /* We can't check hashes of rred result as we don't know what the
 335       hash of the file will be. We just know the hash of the patch(es),
 336       the hash of the file they will apply on and the hash of the resulting
 337       file. */
 338    if (State == StateFetchDiff)
 339       return true;
 340    return false;
 341 }
 342 HashStringList pkgAcqIndexDiffs::GetExpectedHashes() const
 343 {
 344    if (State == StateFetchDiff)
 345       return available_patches[0].download_hashes;
 346    return HashStringList();
 347 }
 348
 349 APT_CONST bool pkgAcqIndexMergeDiffs::HashesRequired() const
 350 {
 351    /* @see #pkgAcqIndexDiffs::HashesRequired, with the difference that
 352       we can check the rred result after all patches are applied as
 353       we know the expected result rather than potentially apply more patches */
 354    if (State == StateFetchDiff)
 355       return true;
 356    return State == StateApplyDiff;
 357 }
 358 HashStringList pkgAcqIndexMergeDiffs::GetExpectedHashes() const
 359 {
 360    if (State == StateFetchDiff)
 361       return patch.download_hashes;
 362    else if (State == StateApplyDiff)
 363       return GetExpectedHashesFor(Target.MetaKey);
 364    return HashStringList();
 365 }
 366
 367 APT_CONST bool pkgAcqArchive::HashesRequired() const
 368 {
 369    return LocalSource == false;
 370 }
 371 HashStringList pkgAcqArchive::GetExpectedHashes() const
 372 {
 373    // figured out while parsing the records
 374    return ExpectedHashes;
 375 }
 376
 377 APT_CONST bool pkgAcqFile::HashesRequired() const
 378 {
 379    // supplied as parameter at creation time, so the caller decides
 380    return ExpectedHashes.usable();
 381 }
 382 HashStringList pkgAcqFile::GetExpectedHashes() const
 383 {
 384    return ExpectedHashes;
 385 }
 386                                                                         /*}}}*/
 387 // Acquire::Item::QueueURI and specialisations from child classes       /*{{{*/
 388 bool pkgAcquire::Item::QueueURI(pkgAcquire::ItemDesc &Item)
 389 {
 390    Owner->Enqueue(Item);
 391    return true;
 392 }
 393 /* The idea here is that an item isn't queued if it exists on disk and the
 394    transition manager was a hit as this means that the files it contains
 395    the checksums for can't be updated either (or they are and we are asking
 396    for a hashsum mismatch to happen which helps nobody) */
 397 bool pkgAcqTransactionItem::QueueURI(pkgAcquire::ItemDesc &Item)
 398 {
 399    if (TransactionManager->State != TransactionStarted)
 400    {
 401       if (_config->FindB("Debug::Acquire::Transaction", false))
 402          std::clog << "Skip " << Target.URI << " as transaction was already dealt with!" << std::endl;
 403       return false;
 404    }
 405    std::string const FinalFile = GetFinalFilename();
 406    if (TransactionManager->IMSHit == true && FileExists(FinalFile) == true)
 407    {
 408       PartialFile = DestFile = FinalFile;
 409       Status = StatDone;
 410       return false;
 411    }
 412    // If we got the InRelease file via a mirror, pick all indexes directly from this mirror, too
 413    if (TransactionManager->BaseURI.empty() == false &&
 414          URI::SiteOnly(Item.URI) != URI::SiteOnly(TransactionManager->BaseURI))
 415    {
 416       // this ensures we rewrite only once and only the first step
 417       auto const OldBaseURI = Target.Option(IndexTarget::BASE_URI);
 418       if (OldBaseURI.empty() == false && APT::String::Startswith(Item.URI, OldBaseURI))
 419       {
 420          auto const ExtraPath = Item.URI.substr(OldBaseURI.length());
 421          Item.URI = flCombine(TransactionManager->BaseURI, ExtraPath);
 422          UsedMirror = TransactionManager->UsedMirror;
 423          if (Item.Description.find(" ") != string::npos)
 424             Item.Description.replace(0, Item.Description.find(" "), UsedMirror);
 425       }
 426    }
 427    return pkgAcquire::Item::QueueURI(Item);
 428 }
 429 /* The transition manager InRelease itself (or its older sisters-in-law
 430    Release & Release.gpg) is always queued as this allows us to rerun gpgv
 431    on it to verify that we aren't stalled with old files */
 432 bool pkgAcqMetaBase::QueueURI(pkgAcquire::ItemDesc &Item)
 433 {
 434    return pkgAcquire::Item::QueueURI(Item);
 435 }
 436 /* the Diff/Index needs to queue also the up-to-date complete index file
 437    to ensure that the list cleaner isn't eating it */
 438 bool pkgAcqDiffIndex::QueueURI(pkgAcquire::ItemDesc &Item)
 439 {
 440    if (pkgAcqTransactionItem::QueueURI(Item) == true)
 441       return true;
 442    QueueOnIMSHit();
 443    return false;
 444 }
 445                                                                         /*}}}*/
 446 // Acquire::Item::GetFinalFilename and specialisations for child classes        /*{{{*/
 447 std::string pkgAcquire::Item::GetFinalFilename() const
 448 {
 449    // Beware: Desc.URI is modified by redirections
 450    return GetFinalFileNameFromURI(Desc.URI);
 451 }
 452 std::string pkgAcqDiffIndex::GetFinalFilename() const
 453 {
 454    return GetFinalFileNameFromURI(GetDiffIndexURI(Target));
 455 }
 456 std::string pkgAcqIndex::GetFinalFilename() const
 457 {
 458    std::string const FinalFile = GetFinalFileNameFromURI(Target.URI);
 459    return GetKeepCompressedFileName(FinalFile, Target);
 460 }
 461 std::string pkgAcqMetaSig::GetFinalFilename() const
 462 {
 463    return GetFinalFileNameFromURI(Target.URI);
 464 }
 465 std::string pkgAcqBaseIndex::GetFinalFilename() const
 466 {
 467    return GetFinalFileNameFromURI(Target.URI);
 468 }
 469 std::string pkgAcqMetaBase::GetFinalFilename() const
 470 {
 471    return GetFinalFileNameFromURI(Target.URI);
 472 }
 473 std::string pkgAcqArchive::GetFinalFilename() const
 474 {
 475    return _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
 476 }
 477                                                                         /*}}}*/
 478 // pkgAcqTransactionItem::GetMetaKey and specialisations for child classes      /*{{{*/
 479 std::string pkgAcqTransactionItem::GetMetaKey() const
 480 {
 481    return Target.MetaKey;
 482 }
 483 std::string pkgAcqIndex::GetMetaKey() const
 484 {
 485    if (Stage == STAGE_DECOMPRESS_AND_VERIFY || CurrentCompressionExtension == "uncompressed")
 486       return Target.MetaKey;
 487    return Target.MetaKey + "." + CurrentCompressionExtension;
 488 }
 489 std::string pkgAcqDiffIndex::GetMetaKey() const
 490 {
 491    return GetDiffIndexFileName(Target.MetaKey);
 492 }
 493                                                                         /*}}}*/
 494 //pkgAcqTransactionItem::TransactionState and specialisations for child classes /*{{{*/
 495 bool pkgAcqTransactionItem::TransactionState(TransactionStates const state)
 496 {
 497    bool const Debug = _config->FindB("Debug::Acquire::Transaction", false);
 498    switch(state)
 499    {
 500       case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break;
 501       case TransactionAbort:
 502          if(Debug == true)
 503             std::clog << "  Cancel: " << DestFile << std::endl;
 504          if (Status == pkgAcquire::Item::StatIdle)
 505          {
 506             Status = pkgAcquire::Item::StatDone;
 507             Dequeue();
 508          }
 509          break;
 510       case TransactionCommit:
 511          if(PartialFile.empty() == false)
 512          {
 513             bool sameFile = (PartialFile == DestFile);
 514             // we use symlinks on IMS-Hit to avoid copies
 515             if (RealFileExists(DestFile))
 516             {
 517                struct stat Buf;
 518                if (lstat(PartialFile.c_str(), &Buf) != -1)
 519                {
 520                   if (S_ISLNK(Buf.st_mode) && Buf.st_size > 0)
 521                   {
 522                      char partial[Buf.st_size + 1];
 523                      ssize_t const sp = readlink(PartialFile.c_str(), partial, Buf.st_size);
 524                      if (sp == -1)
 525                         _error->Errno("pkgAcqTransactionItem::TransactionState-sp", _("Failed to readlink %s"), PartialFile.c_str());
 526                      else
 527                      {
 528                         partial[sp] = '\0';
 529                         sameFile = (DestFile == partial);
 530                      }
 531                   }
 532                }
 533                else
 534                   _error->Errno("pkgAcqTransactionItem::TransactionState-stat", _("Failed to stat %s"), PartialFile.c_str());
 535             }
 536             if (sameFile == false)
 537             {
 538                // ensure that even without lists-cleanup all compressions are nuked
 539                std::string FinalFile = GetFinalFileNameFromURI(Target.URI);
 540                if (FileExists(FinalFile))
 541                {
 542                   if(Debug == true)
 543                      std::clog << "rm " << FinalFile << " # " << DescURI() << std::endl;
 544                   if (RemoveFile("TransactionStates-Cleanup", FinalFile) == false)
 545                      return false;
 546                }
 547                for (auto const &ext: APT::Configuration::getCompressorExtensions())
 548                {
 549                   auto const Final = FinalFile + ext;
 550                   if (FileExists(Final))
 551                   {
 552                      if(Debug == true)
 553                         std::clog << "rm " << Final << " # " << DescURI() << std::endl;
 554                      if (RemoveFile("TransactionStates-Cleanup", Final) == false)
 555                         return false;
 556                   }
 557                }
 558                if(Debug == true)
 559                   std::clog << "mv " << PartialFile << " -> "<< DestFile << " # " << DescURI() << std::endl;
 560                if (Rename(PartialFile, DestFile) == false)
 561                   return false;
 562             }
 563             else if(Debug == true)
 564                std::clog << "keep " << PartialFile << " # " << DescURI() << std::endl;
 565
 566          } else {
 567             if(Debug == true)
 568                std::clog << "rm " << DestFile << " # " << DescURI() << std::endl;
 569             if (RemoveFile("TransItem::TransactionCommit", DestFile) == false)
 570                return false;
 571          }
 572          break;
 573    }
 574    return true;
 575 }
 576 bool pkgAcqMetaBase::TransactionState(TransactionStates const state)
 577 {
 578    // Do not remove InRelease on IMSHit of Release.gpg [yes, this is very edgecasey]
 579    if (TransactionManager->IMSHit == false)
 580       return pkgAcqTransactionItem::TransactionState(state);
 581    return true;
 582 }
 583 bool pkgAcqIndex::TransactionState(TransactionStates const state)
 584 {
 585    if (pkgAcqTransactionItem::TransactionState(state) == false)
 586       return false;
 587
 588    switch (state)
 589    {
 590       case TransactionStarted: _error->Fatal("AcqIndex %s changed to invalid transaction start state!", Target.URI.c_str()); break;
 591       case TransactionAbort:
 592          if (Stage == STAGE_DECOMPRESS_AND_VERIFY)
 593          {
 594             // keep the compressed file, but drop the decompressed
 595             EraseFileName.clear();
 596             if (PartialFile.empty() == false && flExtension(PartialFile) != CurrentCompressionExtension)
 597                RemoveFile("TransactionAbort", PartialFile);
 598          }
 599          break;
 600       case TransactionCommit:
 601          if (EraseFileName.empty() == false)
 602             RemoveFile("AcqIndex::TransactionCommit", EraseFileName);
 603          break;
 604    }
 605    return true;
 606 }
 607 bool pkgAcqDiffIndex::TransactionState(TransactionStates const state)
 608 {
 609    if (pkgAcqTransactionItem::TransactionState(state) == false)
 610       return false;
 611
 612    switch (state)
 613    {
 614       case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break;
 615       case TransactionCommit:
 616          break;
 617       case TransactionAbort:
 618          std::string const Partial = GetPartialFileNameFromURI(Target.URI);
 619          RemoveFile("TransactionAbort", Partial);
 620          break;
 621    }
 622
 623    return true;
 624 }
 625                                                                         /*}}}*/
 626
 627 class APT_HIDDEN NoActionItem : public pkgAcquire::Item                 /*{{{*/
 628 /* The sole purpose of this class is having an item which does nothing to
 629    reach its done state to prevent cleanup deleting the mentioned file.
 630    Handy in cases in which we know we have the file already, like IMS-Hits. */
 631 {
 632    IndexTarget const Target;
 633    public:
 634    virtual std::string DescURI() const APT_OVERRIDE {return Target.URI;};
 635    virtual HashStringList GetExpectedHashes()  const APT_OVERRIDE {return HashStringList();};
 636
 637    NoActionItem(pkgAcquire * const Owner, IndexTarget const &Target) :
 638       pkgAcquire::Item(Owner), Target(Target)
 639    {
 640       Status = StatDone;
 641       DestFile = GetFinalFileNameFromURI(Target.URI);
 642    }
 643    NoActionItem(pkgAcquire * const Owner, IndexTarget const &Target, std::string const &FinalFile) :
 644       pkgAcquire::Item(Owner), Target(Target)
 645    {
 646       Status = StatDone;
 647       DestFile = FinalFile;
 648    }
 649 };
 650                                                                         /*}}}*/
 651 class APT_HIDDEN CleanupItem : public pkgAcqTransactionItem             /*{{{*/
 652 /* This class ensures that a file which was configured but isn't downloaded
 653    for various reasons isn't kept in an old version in the lists directory.
 654    In a way its the reverse of NoActionItem as it helps with removing files
 655    even if the lists-cleanup is deactivated. */
 656 {
 657    public:
 658    virtual std::string DescURI() const APT_OVERRIDE {return Target.URI;};
 659    virtual HashStringList GetExpectedHashes()  const APT_OVERRIDE {return HashStringList();};
 660
 661    CleanupItem(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target) :
 662       pkgAcqTransactionItem(Owner, TransactionManager, Target)
 663    {
 664       Status = StatDone;
 665       DestFile = GetFinalFileNameFromURI(Target.URI);
 666    }
 667    bool TransactionState(TransactionStates const state) APT_OVERRIDE
 668    {
 669       switch (state)
 670       {
 671          case TransactionStarted:
 672             break;
 673          case TransactionAbort:
 674             break;
 675          case TransactionCommit:
 676             if (_config->FindB("Debug::Acquire::Transaction", false) == true)
 677                std::clog << "rm " << DestFile << " # " << DescURI() << std::endl;
 678             if (RemoveFile("TransItem::TransactionCommit", DestFile) == false)
 679                return false;
 680             break;
 681       }
 682       return true;
 683    }
 684 };
 685                                                                         /*}}}*/
 686
 687 // Acquire::Item::Item - Constructor                                    /*{{{*/
 688 APT_IGNORE_DEPRECATED_PUSH
 689 pkgAcquire::Item::Item(pkgAcquire * const owner) :
 690    FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false), Local(false),
 691     QueueCounter(0), ExpectedAdditionalItems(0), Owner(owner), d(NULL)
 692 {
 693    Owner->Add(this);
 694    Status = StatIdle;
 695 }
 696 APT_IGNORE_DEPRECATED_POP
 697                                                                         /*}}}*/
 698 // Acquire::Item::~Item - Destructor                                    /*{{{*/
 699 pkgAcquire::Item::~Item()
 700 {
 701    Owner->Remove(this);
 702 }
 703                                                                         /*}}}*/
 704 std::string pkgAcquire::Item::Custom600Headers() const                  /*{{{*/
 705 {
 706    return std::string();
 707 }
 708                                                                         /*}}}*/
 709 std::string pkgAcquire::Item::ShortDesc() const                         /*{{{*/
 710 {
 711    return DescURI();
 712 }
 713                                                                         /*}}}*/
 714 APT_CONST void pkgAcquire::Item::Finished()                             /*{{{*/
 715 {
 716 }
 717                                                                         /*}}}*/
 718 APT_PURE pkgAcquire * pkgAcquire::Item::GetOwner() const                /*{{{*/
 719 {
 720    return Owner;
 721 }
 722                                                                         /*}}}*/
 723 APT_CONST pkgAcquire::ItemDesc &pkgAcquire::Item::GetItemDesc()         /*{{{*/
 724 {
 725    return Desc;
 726 }
 727                                                                         /*}}}*/
 728 APT_CONST bool pkgAcquire::Item::IsTrusted() const                      /*{{{*/
 729 {
 730    return false;
 731 }
 732                                                                         /*}}}*/
 733 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
 734 // ---------------------------------------------------------------------
 735 /* We return to an idle state if there are still other queues that could
 736    fetch this object */
 737 void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
 738 {
 739    if (QueueCounter <= 1)
 740    {
 741       /* This indicates that the file is not available right now but might
 742          be sometime later. If we do a retry cycle then this should be
 743          retried [CDROMs] */
 744       if (Cnf != NULL && Cnf->LocalOnly == true &&
 745           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
 746       {
 747          Status = StatIdle;
 748          Dequeue();
 749          return;
 750       }
 751
 752       switch (Status)
 753       {
 754          case StatIdle:
 755          case StatFetching:
 756          case StatDone:
 757             Status = StatError;
 758             break;
 759          case StatAuthError:
 760          case StatError:
 761          case StatTransientNetworkError:
 762             break;
 763       }
 764       Complete = false;
 765       Dequeue();
 766    }
 767
 768    string const FailReason = LookupTag(Message, "FailReason");
 769    enum { MAXIMUM_SIZE_EXCEEDED, HASHSUM_MISMATCH, WEAK_HASHSUMS, OTHER } failreason = OTHER;
 770    if ( FailReason == "MaximumSizeExceeded")
 771       failreason = MAXIMUM_SIZE_EXCEEDED;
 772    else if ( FailReason == "WeakHashSums")
 773       failreason = WEAK_HASHSUMS;
 774    else if (Status == StatAuthError)
 775       failreason = HASHSUM_MISMATCH;
 776
 777    if(ErrorText.empty())
 778    {
 779       if (Status == StatAuthError)
 780       {
 781          std::ostringstream out;
 782          switch (failreason)
 783          {
 784             case HASHSUM_MISMATCH:
 785                out << _("Hash Sum mismatch") << std::endl;
 786                break;
 787             case WEAK_HASHSUMS:
 788                out << _("Insufficient information available to perform this download securely") << std::endl;
 789                break;
 790             case MAXIMUM_SIZE_EXCEEDED:
 791             case OTHER:
 792                out << LookupTag(Message, "Message") << std::endl;
 793                break;
 794          }
 795          auto const ExpectedHashes = GetExpectedHashes();
 796          if (ExpectedHashes.empty() == false)
 797          {
 798             out << "Hashes of expected file:" << std::endl;
 799             for (auto const &hs: ExpectedHashes)
 800             {
 801                out << " - " << hs.toStr();
 802                if (hs.usable() == false)
 803                   out << " [weak]";
 804                out << std::endl;
 805             }
 806          }
 807          if (failreason == HASHSUM_MISMATCH)
 808          {
 809             out << "Hashes of received file:" << std::endl;
 810             for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
 811             {
 812                std::string const tagname = std::string(*type) + "-Hash";
 813                std::string const hashsum = LookupTag(Message, tagname.c_str());
 814                if (hashsum.empty() == false)
 815                {
 816                   auto const hs = HashString(*type, hashsum);
 817                   out << " - " << hs.toStr();
 818                   if (hs.usable() == false)
 819                      out << " [weak]";
 820                   out << std::endl;
 821                }
 822             }
 823             out << "Last modification reported: " << LookupTag(Message, "Last-Modified", "<none>") << std::endl;
 824          }
 825          ErrorText = out.str();
 826       }
 827       else
 828          ErrorText = LookupTag(Message,"Message");
 829    }
 830
 831    switch (failreason)
 832    {
 833       case MAXIMUM_SIZE_EXCEEDED: RenameOnError(MaximumSizeExceeded); break;
 834       case HASHSUM_MISMATCH: RenameOnError(HashSumMismatch); break;
 835       case WEAK_HASHSUMS: break;
 836       case OTHER: break;
 837    }
 838
 839    if (FailReason.empty() == false)
 840       ReportMirrorFailureToCentral(*this, FailReason, ErrorText);
 841    else
 842       ReportMirrorFailureToCentral(*this, ErrorText, ErrorText);
 843
 844    if (QueueCounter > 1)
 845       Status = StatIdle;
 846 }
 847                                                                         /*}}}*/
 848 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
 849 // ---------------------------------------------------------------------
 850 /* Stash status and the file size. Note that setting Complete means
 851    sub-phases of the acquire process such as decompresion are operating */
 852 void pkgAcquire::Item::Start(string const &/*Message*/, unsigned long long const Size)
 853 {
 854    Status = StatFetching;
 855    ErrorText.clear();
 856    if (FileSize == 0 && Complete == false)
 857       FileSize = Size;
 858 }
 859                                                                         /*}}}*/
 860 // Acquire::Item::VerifyDone - check if Item was downloaded OK          /*{{{*/
 861 /* Note that hash-verification is 'hardcoded' in acquire-worker and has
 862  * already passed if this method is called. */
 863 bool pkgAcquire::Item::VerifyDone(std::string const &Message,
 864          pkgAcquire::MethodConfig const * const /*Cnf*/)
 865 {
 866    std::string const FileName = LookupTag(Message,"Filename");
 867    if (FileName.empty() == true)
 868    {
 869       Status = StatError;
 870       ErrorText = "Method gave a blank filename";
 871       return false;
 872    }
 873
 874    return true;
 875 }
 876                                                                         /*}}}*/
 877 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 878 void pkgAcquire::Item::Done(string const &/*Message*/, HashStringList const &Hashes,
 879                             pkgAcquire::MethodConfig const * const /*Cnf*/)
 880 {
 881    // We just downloaded something..
 882    if (FileSize == 0)
 883    {
 884       unsigned long long const downloadedSize = Hashes.FileSize();
 885       if (downloadedSize != 0)
 886       {
 887          FileSize = downloadedSize;
 888       }
 889    }
 890    Status = StatDone;
 891    ErrorText = string();
 892    Owner->Dequeue(this);
 893 }
 894                                                                         /*}}}*/
 895 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 896 // ---------------------------------------------------------------------
 897 /* This helper function is used by a lot of item methods as their final
 898    step */
 899 bool pkgAcquire::Item::Rename(string const &From,string const &To)
 900 {
 901    if (From == To || rename(From.c_str(),To.c_str()) == 0)
 902       return true;
 903
 904    std::string S;
 905    strprintf(S, _("rename failed, %s (%s -> %s)."), strerror(errno),
 906          From.c_str(),To.c_str());
 907    Status = StatError;
 908    if (ErrorText.empty())
 909       ErrorText = S;
 910    else
 911       ErrorText = ErrorText + ": " + S;
 912    return false;
 913 }
 914                                                                         /*}}}*/
 915 void pkgAcquire::Item::Dequeue()                                        /*{{{*/
 916 {
 917    Owner->Dequeue(this);
 918 }
 919                                                                         /*}}}*/
 920 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 921 {
 922    if (RealFileExists(DestFile))
 923       Rename(DestFile, DestFile + ".FAILED");
 924
 925    std::string errtext;
 926    switch (error)
 927    {
 928       case HashSumMismatch:
 929          errtext = _("Hash Sum mismatch");
 930          break;
 931       case SizeMismatch:
 932          errtext = _("Size mismatch");
 933          Status = StatAuthError;
 934          break;
 935       case InvalidFormat:
 936          errtext = _("Invalid file format");
 937          Status = StatError;
 938          // do not report as usually its not the mirrors fault, but Portal/Proxy
 939          break;
 940       case SignatureError:
 941          errtext = _("Signature error");
 942          Status = StatError;
 943          break;
 944       case NotClearsigned:
 945          strprintf(errtext, _("Clearsigned file isn't valid, got '%s' (does the network require authentication?)"), "NOSPLIT");
 946          Status = StatAuthError;
 947          break;
 948       case MaximumSizeExceeded:
 949          // the method is expected to report a good error for this
 950          break;
 951       case PDiffError:
 952          // no handling here, done by callers
 953          break;
 954    }
 955    if (ErrorText.empty())
 956       ErrorText = errtext;
 957    return false;
 958 }
 959                                                                         /*}}}*/
 960 void pkgAcquire::Item::SetActiveSubprocess(const std::string &subprocess)/*{{{*/
 961 {
 962       ActiveSubprocess = subprocess;
 963       APT_IGNORE_DEPRECATED(Mode = ActiveSubprocess.c_str();)
 964 }
 965                                                                         /*}}}*/
 966 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 967 void pkgAcquire::Item::ReportMirrorFailure(std::string const &FailCode)
 968 {
 969    ReportMirrorFailureToCentral(*this, FailCode, FailCode);
 970 }
 971                                                                         /*}}}*/
 972 std::string pkgAcquire::Item::HashSum() const                           /*{{{*/
 973 {
 974    HashStringList const hashes = GetExpectedHashes();
 975    HashString const * const hs = hashes.find(NULL);
 976    return hs != NULL ? hs->toStr() : "";
 977 }
 978                                                                         /*}}}*/
 979
 980 pkgAcqTransactionItem::pkgAcqTransactionItem(pkgAcquire * const Owner,  /*{{{*/
 981       pkgAcqMetaClearSig * const transactionManager, IndexTarget const &target) :
 982    pkgAcquire::Item(Owner), d(NULL), Target(target), TransactionManager(transactionManager)
 983 {
 984    if (TransactionManager != this)
 985       TransactionManager->Add(this);
 986 }
 987                                                                         /*}}}*/
 988 pkgAcqTransactionItem::~pkgAcqTransactionItem()                         /*{{{*/
 989 {
 990 }
 991                                                                         /*}}}*/
 992 HashStringList pkgAcqTransactionItem::GetExpectedHashesFor(std::string const &MetaKey) const    /*{{{*/
 993 {
 994    return GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, MetaKey);
 995 }
 996                                                                         /*}}}*/
 997
 998 static void LoadLastMetaIndexParser(pkgAcqMetaClearSig * const TransactionManager, std::string const &FinalRelease, std::string const &FinalInRelease)/*{{{*/
 999 {
1000    if (TransactionManager->IMSHit == true)
1001       return;
1002    if (RealFileExists(FinalInRelease) || RealFileExists(FinalRelease))
1003    {
1004       TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone();
1005       if (TransactionManager->LastMetaIndexParser != NULL)
1006       {
1007          _error->PushToStack();
1008          if (RealFileExists(FinalInRelease))
1009             TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL);
1010          else
1011             TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL);
1012          // its unlikely to happen, but if what we have is bad ignore it
1013          if (_error->PendingError())
1014          {
1015             delete TransactionManager->LastMetaIndexParser;
1016             TransactionManager->LastMetaIndexParser = NULL;
1017          }
1018          _error->RevertToStack();
1019       }
1020    }
1021 }
1022                                                                         /*}}}*/
1023
1024 // AcqMetaBase - Constructor                                            /*{{{*/
1025 pkgAcqMetaBase::pkgAcqMetaBase(pkgAcquire * const Owner,
1026       pkgAcqMetaClearSig * const TransactionManager,
1027       IndexTarget const &DataTarget)
1028 : pkgAcqTransactionItem(Owner, TransactionManager, DataTarget), d(NULL),
1029    AuthPass(false), IMSHit(false), State(TransactionStarted)
1030 {
1031 }
1032                                                                         /*}}}*/
1033 // AcqMetaBase::Add - Add a item to the current Transaction             /*{{{*/
1034 void pkgAcqMetaBase::Add(pkgAcqTransactionItem * const I)
1035 {
1036    Transaction.push_back(I);
1037 }
1038                                                                         /*}}}*/
1039 // AcqMetaBase::AbortTransaction - Abort the current Transaction        /*{{{*/
1040 void pkgAcqMetaBase::AbortTransaction()
1041 {
1042    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1043       std::clog << "AbortTransaction: " << TransactionManager << std::endl;
1044
1045    switch (TransactionManager->State)
1046    {
1047       case TransactionStarted: break;
1048       case TransactionAbort: _error->Fatal("Transaction %s was already aborted and is aborted again", TransactionManager->Target.URI.c_str()); return;
1049       case TransactionCommit: _error->Fatal("Transaction %s was already aborted and is now committed", TransactionManager->Target.URI.c_str()); return;
1050    }
1051    TransactionManager->State = TransactionAbort;
1052
1053    // ensure the toplevel is in error state too
1054    for (std::vector<pkgAcqTransactionItem*>::iterator I = Transaction.begin();
1055         I != Transaction.end(); ++I)
1056    {
1057       if ((*I)->Status != pkgAcquire::Item::StatFetching)
1058          Owner->Dequeue(*I);
1059       (*I)->TransactionState(TransactionAbort);
1060    }
1061    Transaction.clear();
1062 }
1063                                                                         /*}}}*/
1064 // AcqMetaBase::TransactionHasError - Check for errors in Transaction   /*{{{*/
1065 APT_PURE bool pkgAcqMetaBase::TransactionHasError() const
1066 {
1067    for (std::vector<pkgAcqTransactionItem*>::const_iterator I = Transaction.begin();
1068         I != Transaction.end(); ++I)
1069    {
1070       switch((*I)->Status) {
1071          case StatDone: break;
1072          case StatIdle: break;
1073          case StatAuthError: return true;
1074          case StatError: return true;
1075          case StatTransientNetworkError: return true;
1076          case StatFetching: break;
1077       }
1078    }
1079    return false;
1080 }
1081                                                                         /*}}}*/
1082 // AcqMetaBase::CommitTransaction - Commit a transaction                /*{{{*/
1083 void pkgAcqMetaBase::CommitTransaction()
1084 {
1085    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1086       std::clog << "CommitTransaction: " << this << std::endl;
1087
1088    switch (TransactionManager->State)
1089    {
1090       case TransactionStarted: break;
1091       case TransactionAbort: _error->Fatal("Transaction %s was already committed and is now aborted", TransactionManager->Target.URI.c_str()); return;
1092       case TransactionCommit: _error->Fatal("Transaction %s was already committed and is again committed", TransactionManager->Target.URI.c_str()); return;
1093    }
1094    TransactionManager->State = TransactionCommit;
1095
1096    // move new files into place *and* remove files that are not
1097    // part of the transaction but are still on disk
1098    for (std::vector<pkgAcqTransactionItem*>::iterator I = Transaction.begin();
1099         I != Transaction.end(); ++I)
1100    {
1101       (*I)->TransactionState(TransactionCommit);
1102    }
1103    Transaction.clear();
1104 }
1105                                                                         /*}}}*/
1106 // AcqMetaBase::TransactionStageCopy - Stage a file for copying         /*{{{*/
1107 void pkgAcqMetaBase::TransactionStageCopy(pkgAcqTransactionItem * const I,
1108                                           const std::string &From,
1109                                           const std::string &To)
1110 {
1111    I->PartialFile = From;
1112    I->DestFile = To;
1113 }
1114                                                                         /*}}}*/
1115 // AcqMetaBase::TransactionStageRemoval - Stage a file for removal      /*{{{*/
1116 void pkgAcqMetaBase::TransactionStageRemoval(pkgAcqTransactionItem * const I,
1117                                              const std::string &FinalFile)
1118 {
1119    I->PartialFile = "";
1120    I->DestFile = FinalFile;
1121 }
1122                                                                         /*}}}*/
1123 // AcqMetaBase::GenerateAuthWarning - Check gpg authentication error    /*{{{*/
1124 /* This method is called from ::Failed handlers. If it returns true,
1125    no fallback to other files or modi is performed */
1126 bool pkgAcqMetaBase::CheckStopAuthentication(pkgAcquire::Item * const I, const std::string &Message)
1127 {
1128    string const Final = I->GetFinalFilename();
1129    std::string const GPGError = LookupTag(Message, "Message");
1130    if (FileExists(Final))
1131    {
1132       I->Status = StatTransientNetworkError;
1133       _error->Warning(_("An error occurred during the signature verification. "
1134                "The repository is not updated and the previous index files will be used. "
1135                "GPG error: %s: %s"),
1136             Desc.Description.c_str(),
1137             GPGError.c_str());
1138       RunScripts("APT::Update::Auth-Failure");
1139       return true;
1140    } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1141       /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1142       _error->Error(_("GPG error: %s: %s"),
1143             Desc.Description.c_str(),
1144             GPGError.c_str());
1145       I->Status = StatAuthError;
1146       return true;
1147    } else {
1148       _error->Warning(_("GPG error: %s: %s"),
1149             Desc.Description.c_str(),
1150             GPGError.c_str());
1151    }
1152    // gpgv method failed
1153    ReportMirrorFailureToCentral(*this, "GPGFailure", GPGError);
1154    return false;
1155 }
1156                                                                         /*}}}*/
1157 // AcqMetaBase::Custom600Headers - Get header for AcqMetaBase           /*{{{*/
1158 // ---------------------------------------------------------------------
1159 string pkgAcqMetaBase::Custom600Headers() const
1160 {
1161    std::string Header = "\nIndex-File: true";
1162    std::string MaximumSize;
1163    strprintf(MaximumSize, "\nMaximum-Size: %i",
1164              _config->FindI("Acquire::MaxReleaseFileSize", 10*1000*1000));
1165    Header += MaximumSize;
1166
1167    string const FinalFile = GetFinalFilename();
1168    struct stat Buf;
1169    if (stat(FinalFile.c_str(),&Buf) == 0)
1170       Header += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);
1171
1172    return Header;
1173 }
1174                                                                         /*}}}*/
1175 // AcqMetaBase::QueueForSignatureVerify                                 /*{{{*/
1176 void pkgAcqMetaBase::QueueForSignatureVerify(pkgAcqTransactionItem * const I, std::string const &File, std::string const &Signature)
1177 {
1178    AuthPass = true;
1179    I->Desc.URI = "gpgv:" + Signature;
1180    I->DestFile = File;
1181    QueueURI(I->Desc);
1182    I->SetActiveSubprocess("gpgv");
1183 }
1184                                                                         /*}}}*/
1185 // AcqMetaBase::CheckDownloadDone                                       /*{{{*/
1186 bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const std::string &Message, HashStringList const &Hashes) const
1187 {
1188    // We have just finished downloading a Release file (it is not
1189    // verified yet)
1190
1191    // Save the final base URI we got this Release file from
1192    if (I->UsedMirror.empty() == false && _config->FindB("Acquire::SameMirrorForAllIndexes", true))
1193    {
1194       if (APT::String::Endswith(I->Desc.URI, "InRelease"))
1195          TransactionManager->BaseURI = I->Desc.URI.substr(0, I->Desc.URI.length() - strlen("InRelease"));
1196       else if (APT::String::Endswith(I->Desc.URI, "Release"))
1197          TransactionManager->BaseURI = I->Desc.URI.substr(0, I->Desc.URI.length() - strlen("Release"));
1198    }
1199
1200    std::string const FileName = LookupTag(Message,"Filename");
1201    if (FileName != I->DestFile && RealFileExists(I->DestFile) == false)
1202    {
1203       I->Local = true;
1204       I->Desc.URI = "copy:" + FileName;
1205       I->QueueURI(I->Desc);
1206       return false;
1207    }
1208
1209    // make sure to verify against the right file on I-M-S hit
1210    bool IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"), false);
1211    if (IMSHit == false && Hashes.usable())
1212    {
1213       // detect IMS-Hits servers haven't detected by Hash comparison
1214       std::string const FinalFile = I->GetFinalFilename();
1215       if (RealFileExists(FinalFile) && Hashes.VerifyFile(FinalFile) == true)
1216       {
1217          IMSHit = true;
1218          RemoveFile("CheckDownloadDone", I->DestFile);
1219       }
1220    }
1221
1222    if(IMSHit == true)
1223    {
1224       // for simplicity, the transaction manager is always InRelease
1225       // even if it doesn't exist.
1226       TransactionManager->IMSHit = true;
1227       I->PartialFile = I->DestFile = I->GetFinalFilename();
1228    }
1229
1230    // set Item to complete as the remaining work is all local (verify etc)
1231    I->Complete = true;
1232
1233    return true;
1234 }
1235                                                                         /*}}}*/
1236 bool pkgAcqMetaBase::CheckAuthDone(string const &Message)               /*{{{*/
1237 {
1238    // At this point, the gpgv method has succeeded, so there is a
1239    // valid signature from a key in the trusted keyring.  We
1240    // perform additional verification of its contents, and use them
1241    // to verify the indexes we are about to download
1242    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1243       std::cerr << "Signature verification succeeded: " << DestFile << std::endl;
1244
1245    if (TransactionManager->IMSHit == false)
1246    {
1247       // open the last (In)Release if we have it
1248       std::string const FinalFile = GetFinalFilename();
1249       std::string FinalRelease;
1250       std::string FinalInRelease;
1251       if (APT::String::Endswith(FinalFile, "InRelease"))
1252       {
1253          FinalInRelease = FinalFile;
1254          FinalRelease = FinalFile.substr(0, FinalFile.length() - strlen("InRelease")) + "Release";
1255       }
1256       else
1257       {
1258          FinalInRelease = FinalFile.substr(0, FinalFile.length() - strlen("Release")) + "InRelease";
1259          FinalRelease = FinalFile;
1260       }
1261       LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
1262    }
1263
1264    bool const GoodAuth = TransactionManager->MetaIndexParser->Load(DestFile, &ErrorText);
1265    if (GoodAuth == false && AllowInsecureRepositories(InsecureType::WEAK, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == false)
1266    {
1267       Status = StatAuthError;
1268       return false;
1269    }
1270
1271    if (!VerifyVendor(Message))
1272    {
1273       Status = StatAuthError;
1274       return false;
1275    }
1276
1277    // Download further indexes with verification
1278    TransactionManager->QueueIndexes(GoodAuth);
1279
1280    return GoodAuth;
1281 }
1282                                                                         /*}}}*/
1283 void pkgAcqMetaClearSig::QueueIndexes(bool const verify)                        /*{{{*/
1284 {
1285    // at this point the real Items are loaded in the fetcher
1286    ExpectedAdditionalItems = 0;
1287
1288    std::set<std::string> targetsSeen;
1289    bool const hasReleaseFile = TransactionManager->MetaIndexParser != NULL;
1290    bool const metaBaseSupportsByHash = hasReleaseFile && TransactionManager->MetaIndexParser->GetSupportsAcquireByHash();
1291    bool hasHashes = true;
1292    auto IndexTargets = TransactionManager->MetaIndexParser->GetIndexTargets();
1293    if (hasReleaseFile && verify == false)
1294       hasHashes = std::any_of(IndexTargets.begin(), IndexTargets.end(),
1295             [&](IndexTarget const &Target) { return TransactionManager->MetaIndexParser->Exists(Target.MetaKey); });
1296    for (auto&& Target: IndexTargets)
1297    {
1298       // if we have seen a target which is created-by a target this one here is declared a
1299       // fallback to, we skip acquiring the fallback (but we make sure we clean up)
1300       if (targetsSeen.find(Target.Option(IndexTarget::FALLBACK_OF)) != targetsSeen.end())
1301       {
1302          targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
1303          new CleanupItem(Owner, TransactionManager, Target);
1304          continue;
1305       }
1306       // all is an implementation detail. Users shouldn't use this as arch
1307       // We need this support trickery here as e.g. Debian has binary-all files already,
1308       // but arch:all packages are still in the arch:any files, so we would waste precious
1309       // download time, bandwidth and diskspace for nothing, BUT Debian doesn't feature all
1310       // in the set of supported architectures, so we can filter based on this property rather
1311       // than invent an entirely new flag we would need to carry for all of eternity.
1312       if (hasReleaseFile && Target.Option(IndexTarget::ARCHITECTURE) == "all")
1313       {
1314          if (TransactionManager->MetaIndexParser->IsArchitectureAllSupportedFor(Target) == false)
1315          {
1316             new CleanupItem(Owner, TransactionManager, Target);
1317             continue;
1318          }
1319       }
1320
1321       bool trypdiff = Target.OptionBool(IndexTarget::PDIFFS);
1322       if (hasReleaseFile == true)
1323       {
1324          if (TransactionManager->MetaIndexParser->Exists(Target.MetaKey) == false)
1325          {
1326             // optional targets that we do not have in the Release file are skipped
1327             if (hasHashes == true && Target.IsOptional)
1328             {
1329                new CleanupItem(Owner, TransactionManager, Target);
1330                continue;
1331             }
1332
1333             std::string const &arch = Target.Option(IndexTarget::ARCHITECTURE);
1334             if (arch.empty() == false)
1335             {
1336                if (TransactionManager->MetaIndexParser->IsArchitectureSupported(arch) == false)
1337                {
1338                   new CleanupItem(Owner, TransactionManager, Target);
1339                   _error->Notice(_("Skipping acquire of configured file '%s' as repository '%s' doesn't support architecture '%s'"),
1340                         Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str(), arch.c_str());
1341                   continue;
1342                }
1343                // if the architecture is officially supported but currently no packages for it available,
1344                // ignore silently as this is pretty much the same as just shipping an empty file.
1345                // if we don't know which architectures are supported, we do NOT ignore it to notify user about this
1346                if (hasHashes == true && TransactionManager->MetaIndexParser->IsArchitectureSupported("*undefined*") == false)
1347                {
1348                   new CleanupItem(Owner, TransactionManager, Target);
1349                   continue;
1350                }
1351             }
1352
1353             if (hasHashes == true)
1354             {
1355                Status = StatAuthError;
1356                strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), Target.MetaKey.c_str());
1357                return;
1358             }
1359             else
1360             {
1361                new pkgAcqIndex(Owner, TransactionManager, Target);
1362                continue;
1363             }
1364          }
1365          else if (verify)
1366          {
1367             auto const hashes = GetExpectedHashesFor(Target.MetaKey);
1368             if (hashes.empty() == false)
1369             {
1370                if (hashes.usable() == false && TargetIsAllowedToBe(TransactionManager->Target, InsecureType::WEAK) == false)
1371                {
1372                   new CleanupItem(Owner, TransactionManager, Target);
1373                   _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"),
1374                         Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str());
1375                   continue;
1376                }
1377                // empty files are skipped as acquiring the very small compressed files is a waste of time
1378                else if (hashes.FileSize() == 0)
1379                {
1380                   new CleanupItem(Owner, TransactionManager, Target);
1381                   targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
1382                   continue;
1383                }
1384             }
1385          }
1386
1387          // autoselect the compression method
1388          std::vector<std::string> types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' ');
1389          types.erase(std::remove_if(types.begin(), types.end(), [&](std::string const &t) {
1390             if (t == "uncompressed")
1391                return TransactionManager->MetaIndexParser->Exists(Target.MetaKey) == false;
1392             std::string const MetaKey = Target.MetaKey + "." + t;
1393             return TransactionManager->MetaIndexParser->Exists(MetaKey) == false;
1394          }), types.end());
1395          if (types.empty() == false)
1396          {
1397             std::ostringstream os;
1398             // add the special compressiontype byhash first if supported
1399             std::string const useByHashConf = Target.Option(IndexTarget::BY_HASH);
1400             bool useByHash = false;
1401             if(useByHashConf == "force")
1402                useByHash = true;
1403             else
1404                useByHash = StringToBool(useByHashConf) == true && metaBaseSupportsByHash;
1405             if (useByHash == true)
1406                os << "by-hash ";
1407             std::copy(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " "));
1408             os << *types.rbegin();
1409             Target.Options["COMPRESSIONTYPES"] = os.str();
1410          }
1411          else
1412             Target.Options["COMPRESSIONTYPES"].clear();
1413
1414          std::string filename = GetExistingFilename(GetFinalFileNameFromURI(Target.URI));
1415          if (filename.empty() == false)
1416          {
1417             // if the Release file is a hit and we have an index it must be the current one
1418             if (TransactionManager->IMSHit == true)
1419                ;
1420             else if (TransactionManager->LastMetaIndexParser != NULL)
1421             {
1422                // see if the file changed since the last Release file
1423                // we use the uncompressed files as we might compress differently compared to the server,
1424                // so the hashes might not match, even if they contain the same data.
1425                HashStringList const newFile = GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, Target.MetaKey);
1426                HashStringList const oldFile = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, Target.MetaKey);
1427                if (newFile != oldFile)
1428                   filename.clear();
1429             }
1430             else
1431                filename.clear();
1432          }
1433          else
1434             trypdiff = false; // no file to patch
1435
1436          if (filename.empty() == false)
1437          {
1438             new NoActionItem(Owner, Target, filename);
1439             std::string const idxfilename = GetFinalFileNameFromURI(GetDiffIndexURI(Target));
1440             if (FileExists(idxfilename))
1441                new NoActionItem(Owner, Target, idxfilename);
1442             targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
1443             continue;
1444          }
1445
1446          // check if we have patches available
1447          trypdiff &= TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target.MetaKey));
1448       }
1449       else
1450       {
1451          // if we have no file to patch, no point in trying
1452          trypdiff &= (GetExistingFilename(GetFinalFileNameFromURI(Target.URI)).empty() == false);
1453       }
1454
1455       // no point in patching from local sources
1456       if (trypdiff)
1457       {
1458          std::string const proto = Target.URI.substr(0, strlen("file:/"));
1459          if (proto == "file:/" || proto == "copy:/" || proto == "cdrom:")
1460             trypdiff = false;
1461       }
1462
1463       // Queue the Index file (Packages, Sources, Translation-$foo, …)
1464       targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
1465       if (trypdiff)
1466          new pkgAcqDiffIndex(Owner, TransactionManager, Target);
1467       else
1468          new pkgAcqIndex(Owner, TransactionManager, Target);
1469    }
1470 }
1471                                                                         /*}}}*/
1472 bool pkgAcqMetaBase::VerifyVendor(string const &)                       /*{{{*/
1473 {
1474    string Transformed = TransactionManager->MetaIndexParser->GetExpectedDist();
1475
1476    if (Transformed == "../project/experimental")
1477    {
1478       Transformed = "experimental";
1479    }
1480
1481    auto pos = Transformed.rfind('/');
1482    if (pos != string::npos)
1483    {
1484       Transformed = Transformed.substr(0, pos);
1485    }
1486
1487    if (Transformed == ".")
1488    {
1489       Transformed = "";
1490    }
1491
1492    if (TransactionManager->MetaIndexParser->GetValidUntil() > 0)
1493    {
1494       time_t const invalid_since = time(NULL) - TransactionManager->MetaIndexParser->GetValidUntil();
1495       if (invalid_since > 0)
1496       {
1497          std::string errmsg;
1498          strprintf(errmsg,
1499                // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1500                // the time since then the file is invalid - formatted in the same way as in
1501                // the download progress display (e.g. 7d 3h 42min 1s)
1502                _("Release file for %s is expired (invalid since %s). "
1503                   "Updates for this repository will not be applied."),
1504                Target.URI.c_str(), TimeToStr(invalid_since).c_str());
1505          if (ErrorText.empty())
1506             ErrorText = errmsg;
1507          return _error->Error("%s", errmsg.c_str());
1508       }
1509    }
1510
1511    /* Did we get a file older than what we have? This is a last minute IMS hit and doubles
1512       as a prevention of downgrading us to older (still valid) files */
1513    if (TransactionManager->IMSHit == false && TransactionManager->LastMetaIndexParser != NULL &&
1514          TransactionManager->LastMetaIndexParser->GetDate() > TransactionManager->MetaIndexParser->GetDate())
1515    {
1516       TransactionManager->IMSHit = true;
1517       RemoveFile("VerifyVendor", DestFile);
1518       PartialFile = DestFile = GetFinalFilename();
1519       // load the 'old' file in the 'new' one instead of flipping pointers as
1520       // the new one isn't owned by us, while the old one is so cleanup would be confused.
1521       TransactionManager->MetaIndexParser->swapLoad(TransactionManager->LastMetaIndexParser);
1522       delete TransactionManager->LastMetaIndexParser;
1523       TransactionManager->LastMetaIndexParser = NULL;
1524    }
1525
1526    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1527    {
1528       std::cerr << "Got Codename: " << TransactionManager->MetaIndexParser->GetCodename() << std::endl;
1529       std::cerr << "Expecting Dist: " << TransactionManager->MetaIndexParser->GetExpectedDist() << std::endl;
1530       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1531    }
1532
1533    if (TransactionManager->MetaIndexParser->CheckDist(Transformed) == false)
1534    {
1535       // This might become fatal one day
1536 //       Status = StatAuthError;
1537 //       ErrorText = "Conflicting distribution; expected "
1538 //          + MetaIndexParser->GetExpectedDist() + " but got "
1539 //          + MetaIndexParser->GetCodename();
1540 //       return false;
1541       if (!Transformed.empty())
1542       {
1543          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1544                          Desc.Description.c_str(),
1545                          Transformed.c_str(),
1546                          TransactionManager->MetaIndexParser->GetCodename().c_str());
1547       }
1548    }
1549
1550    return true;
1551 }
1552                                                                         /*}}}*/
1553 pkgAcqMetaBase::~pkgAcqMetaBase()
1554 {
1555 }
1556
1557 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner,        /*{{{*/
1558       IndexTarget const &ClearsignedTarget,
1559       IndexTarget const &DetachedDataTarget, IndexTarget const &DetachedSigTarget,
1560       metaIndex * const MetaIndexParser) :
1561    pkgAcqMetaIndex(Owner, this, ClearsignedTarget, DetachedSigTarget),
1562    d(NULL), DetachedDataTarget(DetachedDataTarget),
1563    MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL)
1564 {
1565    // index targets + (worst case:) Release/Release.gpg
1566    ExpectedAdditionalItems = std::numeric_limits<decltype(ExpectedAdditionalItems)>::max();
1567    TransactionManager->Add(this);
1568 }
1569                                                                         /*}}}*/
1570 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1571 {
1572    if (LastMetaIndexParser != NULL)
1573       delete LastMetaIndexParser;
1574 }
1575                                                                         /*}}}*/
1576 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1577 string pkgAcqMetaClearSig::Custom600Headers() const
1578 {
1579    string Header = pkgAcqMetaBase::Custom600Headers();
1580    Header += "\nFail-Ignore: true";
1581    std::string const key = TransactionManager->MetaIndexParser->GetSignedBy();
1582    if (key.empty() == false)
1583       Header += "\nSigned-By: " + key;
1584
1585    return Header;
1586 }
1587                                                                         /*}}}*/
1588 void pkgAcqMetaClearSig::Finished()                                     /*{{{*/
1589 {
1590    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1591       std::clog << "Finished: " << DestFile <<std::endl;
1592    if(TransactionManager->State == TransactionStarted &&
1593       TransactionManager->TransactionHasError() == false)
1594       TransactionManager->CommitTransaction();
1595 }
1596                                                                         /*}}}*/
1597 bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message,         /*{{{*/
1598          pkgAcquire::MethodConfig const * const Cnf)
1599 {
1600    Item::VerifyDone(Message, Cnf);
1601
1602    if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile))
1603       return RenameOnError(NotClearsigned);
1604
1605    return true;
1606 }
1607                                                                         /*}}}*/
1608 // pkgAcqMetaClearSig::Done - We got a file                             /*{{{*/
1609 void pkgAcqMetaClearSig::Done(std::string const &Message,
1610                               HashStringList const &Hashes,
1611                               pkgAcquire::MethodConfig const * const Cnf)
1612 {
1613    Item::Done(Message, Hashes, Cnf);
1614
1615    if(AuthPass == false)
1616    {
1617       if(CheckDownloadDone(this, Message, Hashes) == true)
1618          QueueForSignatureVerify(this, DestFile, DestFile);
1619       return;
1620    }
1621    else if(CheckAuthDone(Message) == true)
1622    {
1623       if (TransactionManager->IMSHit == false)
1624          TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
1625       else if (RealFileExists(GetFinalFilename()) == false)
1626       {
1627          // We got an InRelease file IMSHit, but we haven't one, which means
1628          // we had a valid Release/Release.gpg combo stepping in, which we have
1629          // to 'acquire' now to ensure list cleanup isn't removing them
1630          new NoActionItem(Owner, DetachedDataTarget);
1631          new NoActionItem(Owner, DetachedSigTarget);
1632       }
1633    }
1634    else if (Status != StatAuthError)
1635    {
1636       string const FinalFile = GetFinalFileNameFromURI(DetachedDataTarget.URI);
1637       string const OldFile = GetFinalFilename();
1638       if (TransactionManager->IMSHit == false)
1639          TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
1640       else if (RealFileExists(OldFile) == false)
1641          new NoActionItem(Owner, DetachedDataTarget);
1642       else
1643          TransactionManager->TransactionStageCopy(this, OldFile, FinalFile);
1644    }
1645 }
1646                                                                         /*}}}*/
1647 void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) /*{{{*/
1648 {
1649    Item::Failed(Message, Cnf);
1650
1651    if (AuthPass == false)
1652    {
1653       if (Status == StatAuthError || Status == StatTransientNetworkError)
1654       {
1655          // if we expected a ClearTextSignature (InRelease) but got a network
1656          // error or got a file, but it wasn't valid, we end up here (see VerifyDone).
1657          // As these is usually called by web-portals we do not try Release/Release.gpg
1658          // as this is gonna fail anyway and instead abort our try (LP#346386)
1659          TransactionManager->AbortTransaction();
1660          return;
1661       }
1662
1663       // Queue the 'old' InRelease file for removal if we try Release.gpg
1664       // as otherwise the file will stay around and gives a false-auth
1665       // impression (CVE-2012-0214)
1666       TransactionManager->TransactionStageRemoval(this, GetFinalFilename());
1667       Status = StatDone;
1668
1669       new pkgAcqMetaIndex(Owner, TransactionManager, DetachedDataTarget, DetachedSigTarget);
1670    }
1671    else
1672    {
1673       if(CheckStopAuthentication(this, Message))
1674          return;
1675
1676       if(AllowInsecureRepositories(InsecureType::UNSIGNED, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
1677       {
1678          Status = StatDone;
1679
1680          /* InRelease files become Release files, otherwise
1681           * they would be considered as trusted later on */
1682          string const FinalRelease = GetFinalFileNameFromURI(DetachedDataTarget.URI);
1683          string const PartialRelease = GetPartialFileNameFromURI(DetachedDataTarget.URI);
1684          string const FinalReleasegpg = GetFinalFileNameFromURI(DetachedSigTarget.URI);
1685          string const FinalInRelease = GetFinalFilename();
1686          Rename(DestFile, PartialRelease);
1687          TransactionManager->TransactionStageCopy(this, PartialRelease, FinalRelease);
1688          LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
1689
1690          // we parse the indexes here because at this point the user wanted
1691          // a repository that may potentially harm him
1692          if (TransactionManager->MetaIndexParser->Load(PartialRelease, &ErrorText) == false || VerifyVendor(Message) == false)
1693             /* expired Release files are still a problem you need extra force for */;
1694          else
1695             TransactionManager->QueueIndexes(true);
1696       }
1697    }
1698 }
1699                                                                         /*}}}*/
1700
1701 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire * const Owner,              /*{{{*/
1702                                  pkgAcqMetaClearSig * const TransactionManager,
1703                                  IndexTarget const &DataTarget,
1704                                  IndexTarget const &DetachedSigTarget) :
1705    pkgAcqMetaBase(Owner, TransactionManager, DataTarget), d(NULL),
1706    DetachedSigTarget(DetachedSigTarget)
1707 {
1708    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1709       std::clog << "New pkgAcqMetaIndex with TransactionManager "
1710                 << this->TransactionManager << std::endl;
1711
1712    DestFile = GetPartialFileNameFromURI(DataTarget.URI);
1713
1714    // Create the item
1715    Desc.Description = DataTarget.Description;
1716    Desc.Owner = this;
1717    Desc.ShortDesc = DataTarget.ShortDesc;
1718    Desc.URI = DataTarget.URI;
1719    QueueURI(Desc);
1720 }
1721                                                                         /*}}}*/
1722 void pkgAcqMetaIndex::Done(string const &Message,                       /*{{{*/
1723                            HashStringList const &Hashes,
1724                            pkgAcquire::MethodConfig const * const Cfg)
1725 {
1726    Item::Done(Message,Hashes,Cfg);
1727
1728    if(CheckDownloadDone(this, Message, Hashes))
1729    {
1730       // we have a Release file, now download the Signature, all further
1731       // verify/queue for additional downloads will be done in the
1732       // pkgAcqMetaSig::Done() code
1733       new pkgAcqMetaSig(Owner, TransactionManager, DetachedSigTarget, this);
1734    }
1735 }
1736                                                                         /*}}}*/
1737 // pkgAcqMetaIndex::Failed - no Release file present                    /*{{{*/
1738 void pkgAcqMetaIndex::Failed(string const &Message,
1739                              pkgAcquire::MethodConfig const * const Cnf)
1740 {
1741    pkgAcquire::Item::Failed(Message, Cnf);
1742    Status = StatDone;
1743
1744    // No Release file was present so fall
1745    // back to queueing Packages files without verification
1746    // only allow going further if the user explicitly wants it
1747    if(AllowInsecureRepositories(InsecureType::NORELEASE, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
1748    {
1749       // ensure old Release files are removed
1750       TransactionManager->TransactionStageRemoval(this, GetFinalFilename());
1751
1752       // queue without any kind of hashsum support
1753       TransactionManager->QueueIndexes(false);
1754    }
1755 }
1756                                                                         /*}}}*/
1757 std::string pkgAcqMetaIndex::DescURI() const                            /*{{{*/
1758 {
1759    return Target.URI;
1760 }
1761                                                                         /*}}}*/
1762 pkgAcqMetaIndex::~pkgAcqMetaIndex() {}
1763
1764 // AcqMetaSig::AcqMetaSig - Constructor                                 /*{{{*/
1765 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire * const Owner,
1766       pkgAcqMetaClearSig * const TransactionManager,
1767       IndexTarget const &Target,
1768       pkgAcqMetaIndex * const MetaIndex) :
1769    pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL), MetaIndex(MetaIndex)
1770 {
1771    DestFile = GetPartialFileNameFromURI(Target.URI);
1772
1773    // remove any partial downloaded sig-file in partial/.
1774    // it may confuse proxies and is too small to warrant a
1775    // partial download anyway
1776    RemoveFile("pkgAcqMetaSig", DestFile);
1777
1778    // set the TransactionManager
1779    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
1780       std::clog << "New pkgAcqMetaSig with TransactionManager "
1781                 << TransactionManager << std::endl;
1782
1783    // Create the item
1784    Desc.Description = Target.Description;
1785    Desc.Owner = this;
1786    Desc.ShortDesc = Target.ShortDesc;
1787    Desc.URI = Target.URI;
1788
1789    // If we got a hit for Release, we will get one for Release.gpg too (or obscure errors),
1790    // so we skip the download step and go instantly to verification
1791    if (TransactionManager->IMSHit == true && RealFileExists(GetFinalFilename()))
1792    {
1793       Complete = true;
1794       Status = StatDone;
1795       PartialFile = DestFile = GetFinalFilename();
1796       MetaIndexFileSignature = DestFile;
1797       MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
1798    }
1799    else
1800       QueueURI(Desc);
1801 }
1802                                                                         /*}}}*/
1803 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1804 {
1805 }
1806                                                                         /*}}}*/
1807 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1808 std::string pkgAcqMetaSig::Custom600Headers() const
1809 {
1810    std::string Header = pkgAcqTransactionItem::Custom600Headers();
1811    std::string const key = TransactionManager->MetaIndexParser->GetSignedBy();
1812    if (key.empty() == false)
1813       Header += "\nSigned-By: " + key;
1814    return Header;
1815 }
1816                                                                         /*}}}*/
1817 // AcqMetaSig::Done - The signature was downloaded/verified             /*{{{*/
1818 void pkgAcqMetaSig::Done(string const &Message, HashStringList const &Hashes,
1819                          pkgAcquire::MethodConfig const * const Cfg)
1820 {
1821    if (MetaIndexFileSignature.empty() == false)
1822    {
1823       DestFile = MetaIndexFileSignature;
1824       MetaIndexFileSignature.clear();
1825    }
1826    Item::Done(Message, Hashes, Cfg);
1827
1828    if(MetaIndex->AuthPass == false)
1829    {
1830       if(MetaIndex->CheckDownloadDone(this, Message, Hashes) == true)
1831       {
1832          // destfile will be modified to point to MetaIndexFile for the
1833          // gpgv method, so we need to save it here
1834          MetaIndexFileSignature = DestFile;
1835          MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
1836       }
1837       return;
1838    }
1839    else if(MetaIndex->CheckAuthDone(Message) == true)
1840    {
1841       if (TransactionManager->IMSHit == false)
1842       {
1843          TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
1844          TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, MetaIndex->GetFinalFilename());
1845       }
1846    }
1847    else if (MetaIndex->Status != StatAuthError)
1848    {
1849       std::string const FinalFile = MetaIndex->GetFinalFilename();
1850       if (TransactionManager->IMSHit == false)
1851          TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, FinalFile);
1852       else
1853          TransactionManager->TransactionStageCopy(MetaIndex, FinalFile, FinalFile);
1854    }
1855 }
1856                                                                         /*}}}*/
1857 void pkgAcqMetaSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
1858 {
1859    Item::Failed(Message,Cnf);
1860
1861    // check if we need to fail at this point
1862    if (MetaIndex->AuthPass == true && MetaIndex->CheckStopAuthentication(this, Message))
1863          return;
1864
1865    // ensures that a Release.gpg file in the lists/ is removed by the transaction
1866    TransactionManager->TransactionStageRemoval(this, DestFile);
1867
1868    // only allow going further if the user explicitly wants it
1869    if (AllowInsecureRepositories(InsecureType::UNSIGNED, MetaIndex->Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
1870    {
1871       string const FinalRelease = MetaIndex->GetFinalFilename();
1872       string const FinalInRelease = TransactionManager->GetFinalFilename();
1873       LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
1874
1875       // we parse the indexes here because at this point the user wanted
1876       // a repository that may potentially harm him
1877       bool const GoodLoad = TransactionManager->MetaIndexParser->Load(MetaIndex->DestFile, &ErrorText);
1878       if (MetaIndex->VerifyVendor(Message) == false)
1879          /* expired Release files are still a problem you need extra force for */;
1880       else
1881          TransactionManager->QueueIndexes(GoodLoad);
1882
1883       TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, FinalRelease);
1884    }
1885    else if (TransactionManager->IMSHit == false)
1886       Rename(MetaIndex->DestFile, MetaIndex->DestFile + ".FAILED");
1887
1888    // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
1889    if (Cnf->LocalOnly == true ||
1890        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1891    {
1892       // Ignore this
1893       Status = StatDone;
1894    }
1895 }
1896                                                                         /*}}}*/
1897
1898
1899 // AcqBaseIndex - Constructor                                           /*{{{*/
1900 pkgAcqBaseIndex::pkgAcqBaseIndex(pkgAcquire * const Owner,
1901       pkgAcqMetaClearSig * const TransactionManager,
1902       IndexTarget const &Target)
1903 : pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL)
1904 {
1905 }
1906                                                                         /*}}}*/
1907 void pkgAcqBaseIndex::Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
1908 {
1909    pkgAcquire::Item::Failed(Message, Cnf);
1910    if (Status != StatAuthError)
1911       return;
1912
1913    ErrorText.append("Release file created at: ");
1914    auto const timespec = TransactionManager->MetaIndexParser->GetDate();
1915    if (timespec == 0)
1916       ErrorText.append("<unknown>");
1917    else
1918       ErrorText.append(TimeRFC1123(timespec, true));
1919    ErrorText.append("\n");
1920 }
1921                                                                         /*}}}*/
1922 pkgAcqBaseIndex::~pkgAcqBaseIndex() {}
1923
1924 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
1925 // ---------------------------------------------------------------------
1926 /* Get the DiffIndex file first and see if there are patches available
1927  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
1928  * patches. If anything goes wrong in that process, it will fall back to
1929  * the original packages file
1930  */
1931 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner,
1932                                  pkgAcqMetaClearSig * const TransactionManager,
1933                                  IndexTarget const &Target)
1934    : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), diffs(NULL)
1935 {
1936    // FIXME: Magic number as an upper bound on pdiffs we will reasonably acquire
1937    ExpectedAdditionalItems = 40;
1938
1939    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
1940
1941    Desc.Owner = this;
1942    Desc.Description = GetDiffIndexFileName(Target.Description);
1943    Desc.ShortDesc = Target.ShortDesc;
1944    Desc.URI = GetDiffIndexURI(Target);
1945
1946    DestFile = GetPartialFileNameFromURI(Desc.URI);
1947
1948    if(Debug)
1949       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
1950
1951    QueueURI(Desc);
1952 }
1953                                                                         /*}}}*/
1954 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1955 // ---------------------------------------------------------------------
1956 /* The only header we use is the last-modified header. */
1957 string pkgAcqDiffIndex::Custom600Headers() const
1958 {
1959    if (TransactionManager->LastMetaIndexParser != NULL)
1960       return "\nIndex-File: true";
1961
1962    string const Final = GetFinalFilename();
1963
1964    if(Debug)
1965       std::clog << "Custom600Header-IMS: " << Final << std::endl;
1966
1967    struct stat Buf;
1968    if (stat(Final.c_str(),&Buf) != 0)
1969       return "\nIndex-File: true";
1970
1971    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);
1972 }
1973                                                                         /*}}}*/
1974 void pkgAcqDiffIndex::QueueOnIMSHit() const                             /*{{{*/
1975 {
1976    // list cleanup needs to know that this file as well as the already
1977    // present index is ours, so we create an empty diff to save it for us
1978    new pkgAcqIndexDiffs(Owner, TransactionManager, Target);
1979 }
1980                                                                         /*}}}*/
1981 bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)       /*{{{*/
1982 {
1983    ExpectedAdditionalItems = 0;
1984    // failing here is fine: our caller will take care of trying to
1985    // get the complete file if patching fails
1986    if(Debug)
1987       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
1988          << std::endl;
1989
1990    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
1991    pkgTagFile TF(&Fd);
1992    if (Fd.IsOpen() == false || Fd.Failed())
1993       return false;
1994
1995    pkgTagSection Tags;
1996    if(unlikely(TF.Step(Tags) == false))
1997       return false;
1998
1999    HashStringList ServerHashes;
2000    unsigned long long ServerSize = 0;
2001
2002    auto const &posix = std::locale("C.UTF-8");
2003    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
2004    {
2005       std::string tagname = *type;
2006       tagname.append("-Current");
2007       std::string const tmp = Tags.FindS(tagname.c_str());
2008       if (tmp.empty() == true)
2009          continue;
2010
2011       string hash;
2012       unsigned long long size;
2013       std::stringstream ss(tmp);
2014       ss.imbue(posix);
2015       ss >> hash >> size;
2016       if (unlikely(hash.empty() == true))
2017          continue;
2018       if (unlikely(ServerSize != 0 && ServerSize != size))
2019          continue;
2020       ServerHashes.push_back(HashString(*type, hash));
2021       ServerSize = size;
2022    }
2023
2024    if (ServerHashes.usable() == false)
2025    {
2026       if (Debug == true)
2027          std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": Did not find a good hashsum in the index" << std::endl;
2028       return false;
2029    }
2030
2031    std::string const CurrentPackagesFile = GetFinalFileNameFromURI(Target.URI);
2032    HashStringList const TargetFileHashes = GetExpectedHashesFor(Target.MetaKey);
2033    if (TargetFileHashes.usable() == false || ServerHashes != TargetFileHashes)
2034    {
2035       if (Debug == true)
2036       {
2037          std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": Index has different hashes than parser, probably older, so fail pdiffing" << std::endl;
2038          printHashSumComparison(CurrentPackagesFile, ServerHashes, TargetFileHashes);
2039       }
2040       return false;
2041    }
2042
2043    HashStringList LocalHashes;
2044    // try avoiding calculating the hash here as this is costly
2045    if (TransactionManager->LastMetaIndexParser != NULL)
2046       LocalHashes = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, Target.MetaKey);
2047    if (LocalHashes.usable() == false)
2048    {
2049       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly, FileFd::Auto);
2050       Hashes LocalHashesCalc(ServerHashes);
2051       LocalHashesCalc.AddFD(fd);
2052       LocalHashes = LocalHashesCalc.GetHashStringList();
2053    }
2054
2055    if (ServerHashes == LocalHashes)
2056    {
2057       // we have the same sha1 as the server so we are done here
2058       if(Debug)
2059          std::clog << "pkgAcqDiffIndex: Package file " << CurrentPackagesFile << " is up-to-date" << std::endl;
2060       QueueOnIMSHit();
2061       return true;
2062    }
2063
2064    if(Debug)
2065       std::clog << "Server-Current: " << ServerHashes.find(NULL)->toStr() << " and we start at "
2066          << CurrentPackagesFile << " " << LocalHashes.FileSize() << " " << LocalHashes.find(NULL)->toStr() << std::endl;
2067
2068    // historically, older hashes have more info than newer ones, so start
2069    // collecting with older ones first to avoid implementing complicated
2070    // information merging techniques… a failure is after all always
2071    // recoverable with a complete file and hashes aren't changed that often.
2072    std::vector<char const *> types;
2073    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
2074       types.push_back(*type);
2075
2076    // parse all of (provided) history
2077    vector<DiffInfo> available_patches;
2078    bool firstAcceptedHashes = true;
2079    for (auto type = types.crbegin(); type != types.crend(); ++type)
2080    {
2081       if (LocalHashes.find(*type) == NULL)
2082          continue;
2083
2084       std::string tagname = *type;
2085       tagname.append("-History");
2086       std::string const tmp = Tags.FindS(tagname.c_str());
2087       if (tmp.empty() == true)
2088          continue;
2089
2090       string hash, filename;
2091       unsigned long long size;
2092       std::stringstream ss(tmp);
2093       ss.imbue(posix);
2094
2095       while (ss >> hash >> size >> filename)
2096       {
2097          if (unlikely(hash.empty() == true || filename.empty() == true))
2098             continue;
2099
2100          // see if we have a record for this file already
2101          std::vector<DiffInfo>::iterator cur = available_patches.begin();
2102          for (; cur != available_patches.end(); ++cur)
2103          {
2104             if (cur->file != filename)
2105                continue;
2106             cur->result_hashes.push_back(HashString(*type, hash));
2107             break;
2108          }
2109          if (cur != available_patches.end())
2110             continue;
2111          if (firstAcceptedHashes == true)
2112          {
2113             DiffInfo next;
2114             next.file = filename;
2115             next.result_hashes.push_back(HashString(*type, hash));
2116             next.result_hashes.FileSize(size);
2117             available_patches.push_back(next);
2118          }
2119          else
2120          {
2121             if (Debug == true)
2122                std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
2123                   << " wasn't in the list for the first parsed hash! (history)" << std::endl;
2124             break;
2125          }
2126       }
2127       firstAcceptedHashes = false;
2128    }
2129
2130    if (unlikely(available_patches.empty() == true))
2131    {
2132       if (Debug)
2133          std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": "
2134             << "Couldn't find any patches for the patch series." << std::endl;
2135       return false;
2136    }
2137
2138    for (auto type = types.crbegin(); type != types.crend(); ++type)
2139    {
2140       if (LocalHashes.find(*type) == NULL)
2141          continue;
2142
2143       std::string tagname = *type;
2144       tagname.append("-Patches");
2145       std::string const tmp = Tags.FindS(tagname.c_str());
2146       if (tmp.empty() == true)
2147          continue;
2148
2149       string hash, filename;
2150       unsigned long long size;
2151       std::stringstream ss(tmp);
2152       ss.imbue(posix);
2153
2154       while (ss >> hash >> size >> filename)
2155       {
2156          if (unlikely(hash.empty() == true || filename.empty() == true))
2157             continue;
2158
2159          // see if we have a record for this file already
2160          std::vector<DiffInfo>::iterator cur = available_patches.begin();
2161          for (; cur != available_patches.end(); ++cur)
2162          {
2163             if (cur->file != filename)
2164                continue;
2165             if (cur->patch_hashes.empty())
2166                cur->patch_hashes.FileSize(size);
2167             cur->patch_hashes.push_back(HashString(*type, hash));
2168             break;
2169          }
2170          if (cur != available_patches.end())
2171                continue;
2172          if (Debug == true)
2173             std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
2174                << " wasn't in the list for the first parsed hash! (patches)" << std::endl;
2175          break;
2176       }
2177    }
2178
2179    for (auto type = types.crbegin(); type != types.crend(); ++type)
2180    {
2181       std::string tagname = *type;
2182       tagname.append("-Download");
2183       std::string const tmp = Tags.FindS(tagname.c_str());
2184       if (tmp.empty() == true)
2185          continue;
2186
2187       string hash, filename;
2188       unsigned long long size;
2189       std::stringstream ss(tmp);
2190       ss.imbue(posix);
2191
2192       // FIXME: all of pdiff supports only .gz compressed patches
2193       while (ss >> hash >> size >> filename)
2194       {
2195          if (unlikely(hash.empty() == true || filename.empty() == true))
2196             continue;
2197          if (unlikely(APT::String::Endswith(filename, ".gz") == false))
2198             continue;
2199          filename.erase(filename.length() - 3);
2200
2201          // see if we have a record for this file already
2202          std::vector<DiffInfo>::iterator cur = available_patches.begin();
2203          for (; cur != available_patches.end(); ++cur)
2204          {
2205             if (cur->file != filename)
2206                continue;
2207             if (cur->download_hashes.empty())
2208                cur->download_hashes.FileSize(size);
2209             cur->download_hashes.push_back(HashString(*type, hash));
2210             break;
2211          }
2212          if (cur != available_patches.end())
2213                continue;
2214          if (Debug == true)
2215             std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
2216                << " wasn't in the list for the first parsed hash! (download)" << std::endl;
2217          break;
2218       }
2219    }
2220
2221
2222    bool foundStart = false;
2223    for (std::vector<DiffInfo>::iterator cur = available_patches.begin();
2224          cur != available_patches.end(); ++cur)
2225    {
2226       if (LocalHashes != cur->result_hashes)
2227          continue;
2228
2229       available_patches.erase(available_patches.begin(), cur);
2230       foundStart = true;
2231       break;
2232    }
2233
2234    if (foundStart == false || unlikely(available_patches.empty() == true))
2235    {
2236       if (Debug)
2237          std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": "
2238             << "Couldn't find the start of the patch series." << std::endl;
2239       return false;
2240    }
2241
2242    for (auto const &patch: available_patches)
2243       if (patch.result_hashes.usable() == false ||
2244             patch.patch_hashes.usable() == false ||
2245             patch.download_hashes.usable() == false)
2246       {
2247          if (Debug)
2248             std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": provides no usable hashes for " << patch.file
2249                << " so fallback to complete download" << std::endl;
2250          return false;
2251       }
2252
2253    // patching with too many files is rather slow compared to a fast download
2254    unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
2255    if (fileLimit != 0 && fileLimit < available_patches.size())
2256    {
2257       if (Debug)
2258          std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
2259             << ") so fallback to complete download" << std::endl;
2260       return false;
2261    }
2262
2263    // calculate the size of all patches we have to get
2264    unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100);
2265    if (sizeLimitPercent > 0)
2266    {
2267       unsigned long long downloadSize = std::accumulate(available_patches.begin(),
2268             available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) {
2269             return T + I.download_hashes.FileSize();
2270             });
2271       if (downloadSize != 0)
2272       {
2273          unsigned long long downloadSizeIdx = 0;
2274          auto const types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' ');
2275          for (auto const &t : types)
2276          {
2277             std::string MetaKey = Target.MetaKey;
2278             if (t != "uncompressed")
2279                MetaKey += '.' + t;
2280             HashStringList const hsl = GetExpectedHashesFor(MetaKey);
2281             if (unlikely(hsl.usable() == false))
2282                continue;
2283             downloadSizeIdx = hsl.FileSize();
2284             break;
2285          }
2286          unsigned long long const sizeLimit = downloadSizeIdx * sizeLimitPercent;
2287          if ((sizeLimit/100) < downloadSize)
2288          {
2289             if (Debug)
2290                std::clog << "Need " << downloadSize << " compressed bytes (Limit is " << (sizeLimit/100) << ", "
2291                   << "original is " << downloadSizeIdx << ") so fallback to complete download" << std::endl;
2292             return false;
2293          }
2294       }
2295    }
2296
2297    // we have something, queue the diffs
2298    string::size_type const last_space = Description.rfind(" ");
2299    if(last_space != string::npos)
2300       Description.erase(last_space, Description.size()-last_space);
2301
2302    /* decide if we should download patches one by one or in one go:
2303       The first is good if the server merges patches, but many don't so client
2304       based merging can be attempt in which case the second is better.
2305       "bad things" will happen if patches are merged on the server,
2306       but client side merging is attempt as well */
2307    bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
2308    if (pdiff_merge == true)
2309    {
2310       // reprepro adds this flag if it has merged patches on the server
2311       std::string const precedence = Tags.FindS("X-Patch-Precedence");
2312       pdiff_merge = (precedence != "merged");
2313    }
2314
2315    // clean the plate
2316    {
2317       std::string const Final = GetExistingFilename(CurrentPackagesFile);
2318       if (unlikely(Final.empty())) // because we wouldn't be called in such a case
2319          return false;
2320       std::string const PartialFile = GetPartialFileNameFromURI(Target.URI);
2321       if (FileExists(PartialFile) && RemoveFile("Bootstrap-linking", PartialFile) == false)
2322       {
2323          if (Debug)
2324             std::clog << "Bootstrap-linking for patching " << CurrentPackagesFile
2325                << " by removing stale " << PartialFile << " failed!" << std::endl;
2326          return false;
2327       }
2328       for (auto const &ext : APT::Configuration::getCompressorExtensions())
2329       {
2330          std::string const Partial = PartialFile + ext;
2331          if (FileExists(Partial) && RemoveFile("Bootstrap-linking", Partial) == false)
2332          {
2333             if (Debug)
2334                std::clog << "Bootstrap-linking for patching " << CurrentPackagesFile
2335                   << " by removing stale " << Partial << " failed!" << std::endl;
2336             return false;
2337          }
2338       }
2339       std::string const Ext = Final.substr(CurrentPackagesFile.length());
2340       std::string const Partial = PartialFile + Ext;
2341       if (symlink(Final.c_str(), Partial.c_str()) != 0)
2342       {
2343          if (Debug)
2344             std::clog << "Bootstrap-linking for patching " << CurrentPackagesFile
2345                << " by linking " << Final << " to " << Partial << " failed!" << std::endl;
2346          return false;
2347       }
2348    }
2349
2350    if (pdiff_merge == false)
2351       new pkgAcqIndexDiffs(Owner, TransactionManager, Target, available_patches);
2352    else
2353    {
2354       diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
2355       for(size_t i = 0; i < available_patches.size(); ++i)
2356          (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, TransactionManager,
2357                Target,
2358                available_patches[i],
2359                diffs);
2360    }
2361
2362    Complete = false;
2363    Status = StatDone;
2364    Dequeue();
2365    return true;
2366 }
2367                                                                         /*}}}*/
2368 void pkgAcqDiffIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
2369 {
2370    pkgAcqBaseIndex::Failed(Message,Cnf);
2371    Status = StatDone;
2372    ExpectedAdditionalItems = 0;
2373
2374    if(Debug)
2375       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
2376                 << "Falling back to normal index file acquire" << std::endl;
2377
2378    new pkgAcqIndex(Owner, TransactionManager, Target);
2379 }
2380                                                                         /*}}}*/
2381 void pkgAcqDiffIndex::Done(string const &Message,HashStringList const &Hashes,  /*{{{*/
2382                            pkgAcquire::MethodConfig const * const Cnf)
2383 {
2384    if(Debug)
2385       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
2386
2387    Item::Done(Message, Hashes, Cnf);
2388
2389    string const FinalFile = GetFinalFilename();
2390    if(StringToBool(LookupTag(Message,"IMS-Hit"),false))
2391       DestFile = FinalFile;
2392
2393    if(ParseDiffIndex(DestFile) == false)
2394    {
2395       Failed("Message: Couldn't parse pdiff index", Cnf);
2396       // queue for final move - this should happen even if we fail
2397       // while parsing (e.g. on sizelimit) and download the complete file.
2398       TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
2399       return;
2400    }
2401
2402    TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
2403
2404    Complete = true;
2405    Status = StatDone;
2406    Dequeue();
2407
2408    return;
2409 }
2410                                                                         /*}}}*/
2411 pkgAcqDiffIndex::~pkgAcqDiffIndex()
2412 {
2413    if (diffs != NULL)
2414       delete diffs;
2415 }
2416
2417 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
2418 // ---------------------------------------------------------------------
2419 /* The package diff is added to the queue. one object is constructed
2420  * for each diff and the index
2421  */
2422 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire * const Owner,
2423                                    pkgAcqMetaClearSig * const TransactionManager,
2424                                    IndexTarget const &Target,
2425                                    vector<DiffInfo> const &diffs)
2426    : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL),
2427      available_patches(diffs)
2428 {
2429    DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
2430
2431    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
2432
2433    Desc.Owner = this;
2434    Description = Target.Description;
2435    Desc.ShortDesc = Target.ShortDesc;
2436
2437    if(available_patches.empty() == true)
2438    {
2439       // we are done (yeah!), check hashes against the final file
2440       DestFile = GetKeepCompressedFileName(GetFinalFileNameFromURI(Target.URI), Target);
2441       Finish(true);
2442    }
2443    else
2444    {
2445       State = StateFetchDiff;
2446       QueueNextDiff();
2447    }
2448 }
2449                                                                         /*}}}*/
2450 void pkgAcqIndexDiffs::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
2451 {
2452    pkgAcqBaseIndex::Failed(Message,Cnf);
2453    Status = StatDone;
2454
2455    DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
2456    if(Debug)
2457       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
2458                 << "Falling back to normal index file acquire " << std::endl;
2459    RenameOnError(PDiffError);
2460    std::string const patchname = GetDiffsPatchFileName(DestFile);
2461    if (RealFileExists(patchname))
2462       Rename(patchname, patchname + ".FAILED");
2463    std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI));
2464    if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile))
2465       Rename(UnpatchedFile, UnpatchedFile + ".FAILED");
2466    new pkgAcqIndex(Owner, TransactionManager, Target);
2467    Finish();
2468 }
2469                                                                         /*}}}*/
2470 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
2471 void pkgAcqIndexDiffs::Finish(bool allDone)
2472 {
2473    if(Debug)
2474       std::clog << "pkgAcqIndexDiffs::Finish(): "
2475                 << allDone << " "
2476                 << Desc.URI << std::endl;
2477
2478    // we restore the original name, this is required, otherwise
2479    // the file will be cleaned
2480    if(allDone)
2481    {
2482       std::string const Final = GetKeepCompressedFileName(GetFinalFilename(), Target);
2483       TransactionManager->TransactionStageCopy(this, DestFile, Final);
2484
2485       // this is for the "real" finish
2486       Complete = true;
2487       Status = StatDone;
2488       Dequeue();
2489       if(Debug)
2490          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
2491       return;
2492    }
2493    else
2494       DestFile.clear();
2495
2496    if(Debug)
2497       std::clog << "Finishing: " << Desc.URI << std::endl;
2498    Complete = false;
2499    Status = StatDone;
2500    Dequeue();
2501    return;
2502 }
2503                                                                         /*}}}*/
2504 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
2505 {
2506    // calc sha1 of the just patched file
2507    std::string const PartialFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI));
2508    if(unlikely(PartialFile.empty()))
2509    {
2510       Failed("Message: The file " + GetPartialFileNameFromURI(Target.URI) + " isn't available", NULL);
2511       return false;
2512    }
2513
2514    FileFd fd(PartialFile, FileFd::ReadOnly, FileFd::Extension);
2515    Hashes LocalHashesCalc;
2516    LocalHashesCalc.AddFD(fd);
2517    HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
2518
2519    if(Debug)
2520       std::clog << "QueueNextDiff: " << PartialFile << " (" << LocalHashes.find(NULL)->toStr() << ")" << std::endl;
2521
2522    HashStringList const TargetFileHashes = GetExpectedHashesFor(Target.MetaKey);
2523    if (unlikely(LocalHashes.usable() == false || TargetFileHashes.usable() == false))
2524    {
2525       Failed("Local/Expected hashes are not usable for " + PartialFile, NULL);
2526       return false;
2527    }
2528
2529    // final file reached before all patches are applied
2530    if(LocalHashes == TargetFileHashes)
2531    {
2532       Finish(true);
2533       return true;
2534    }
2535
2536    // remove all patches until the next matching patch is found
2537    // this requires the Index file to be ordered
2538    available_patches.erase(available_patches.begin(),
2539          std::find_if(available_patches.begin(), available_patches.end(), [&](DiffInfo const &I) {
2540             return I.result_hashes == LocalHashes;
2541             }));
2542
2543    // error checking and falling back if no patch was found
2544    if(available_patches.empty() == true)
2545    {
2546       Failed("No patches left to reach target for " + PartialFile, NULL);
2547       return false;
2548    }
2549
2550    // queue the right diff
2551    Desc.URI = Target.URI + ".diff/" + available_patches[0].file + ".gz";
2552    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
2553    DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI + ".diff/" + available_patches[0].file), Target);
2554
2555    if(Debug)
2556       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
2557
2558    QueueURI(Desc);
2559
2560    return true;
2561 }
2562                                                                         /*}}}*/
2563 void pkgAcqIndexDiffs::Done(string const &Message, HashStringList const &Hashes,        /*{{{*/
2564                             pkgAcquire::MethodConfig const * const Cnf)
2565 {
2566    if (Debug)
2567       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
2568
2569    Item::Done(Message, Hashes, Cnf);
2570
2571    std::string const UncompressedUnpatchedFile = GetPartialFileNameFromURI(Target.URI);
2572    std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile);
2573    std::string const PatchFile = GetDiffsPatchFileName(UnpatchedFile);
2574    std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target);
2575
2576    switch (State)
2577    {
2578       // success in downloading a diff, enter ApplyDiff state
2579       case StateFetchDiff:
2580          Rename(DestFile, PatchFile);
2581          DestFile = GetKeepCompressedFileName(UncompressedUnpatchedFile + "-patched", Target);
2582          if(Debug)
2583             std::clog << "Sending to rred method: " << UnpatchedFile << std::endl;
2584          State = StateApplyDiff;
2585          Local = true;
2586          Desc.URI = "rred:" + UnpatchedFile;
2587          QueueURI(Desc);
2588          SetActiveSubprocess("rred");
2589          return;
2590       // success in download/apply a diff, queue next (if needed)
2591       case StateApplyDiff:
2592          // remove the just applied patch and base file
2593          available_patches.erase(available_patches.begin());
2594          RemoveFile("pkgAcqIndexDiffs::Done", PatchFile);
2595          RemoveFile("pkgAcqIndexDiffs::Done", UnpatchedFile);
2596          if(Debug)
2597             std::clog << "Moving patched file in place: " << std::endl
2598                << DestFile << " -> " << PatchedFile << std::endl;
2599          Rename(DestFile, PatchedFile);
2600
2601          // see if there is more to download
2602          if(available_patches.empty() == false)
2603          {
2604             new pkgAcqIndexDiffs(Owner, TransactionManager, Target, available_patches);
2605             Finish();
2606          } else {
2607             DestFile = PatchedFile;
2608             Finish(true);
2609          }
2610          return;
2611    }
2612 }
2613                                                                         /*}}}*/
2614 std::string pkgAcqIndexDiffs::Custom600Headers() const                  /*{{{*/
2615 {
2616    if(State != StateApplyDiff)
2617       return pkgAcqBaseIndex::Custom600Headers();
2618    std::ostringstream patchhashes;
2619    for (auto && hs : available_patches[0].result_hashes)
2620       patchhashes <<  "\nStart-" << hs.HashType() << "-Hash: " << hs.HashValue();
2621    for (auto && hs : available_patches[0].patch_hashes)
2622       patchhashes <<  "\nPatch-0-" << hs.HashType() << "-Hash: " << hs.HashValue();
2623    patchhashes << pkgAcqBaseIndex::Custom600Headers();
2624    return patchhashes.str();
2625 }
2626                                                                         /*}}}*/
2627 pkgAcqIndexDiffs::~pkgAcqIndexDiffs() {}
2628
2629 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor                 /*{{{*/
2630 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire * const Owner,
2631                                              pkgAcqMetaClearSig * const TransactionManager,
2632                                              IndexTarget const &Target,
2633                                              DiffInfo const &patch,
2634                                              std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
2635   : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL),
2636      patch(patch), allPatches(allPatches), State(StateFetchDiff)
2637 {
2638    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
2639
2640    Desc.Owner = this;
2641    Description = Target.Description;
2642    Desc.ShortDesc = Target.ShortDesc;
2643    Desc.URI = Target.URI + ".diff/" + patch.file + ".gz";
2644    Desc.Description = Description + " " + patch.file + ".pdiff";
2645    DestFile = GetPartialFileNameFromURI(Desc.URI);
2646
2647    if(Debug)
2648       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
2649
2650    QueueURI(Desc);
2651 }
2652                                                                         /*}}}*/
2653 void pkgAcqIndexMergeDiffs::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
2654 {
2655    if(Debug)
2656       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
2657
2658    pkgAcqBaseIndex::Failed(Message,Cnf);
2659    Status = StatDone;
2660
2661    // check if we are the first to fail, otherwise we are done here
2662    State = StateDoneDiff;
2663    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
2664          I != allPatches->end(); ++I)
2665       if ((*I)->State == StateErrorDiff)
2666       {
2667          State = StateErrorDiff;
2668          return;
2669       }
2670
2671    // first failure means we should fallback
2672    State = StateErrorDiff;
2673    if (Debug)
2674       std::clog << "Falling back to normal index file acquire" << std::endl;
2675    RenameOnError(PDiffError);
2676    if (RealFileExists(DestFile))
2677       Rename(DestFile, DestFile + ".FAILED");
2678    std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI));
2679    if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile))
2680       Rename(UnpatchedFile, UnpatchedFile + ".FAILED");
2681    DestFile.clear();
2682    new pkgAcqIndex(Owner, TransactionManager, Target);
2683 }
2684                                                                         /*}}}*/
2685 void pkgAcqIndexMergeDiffs::Done(string const &Message, HashStringList const &Hashes,   /*{{{*/
2686                             pkgAcquire::MethodConfig const * const Cnf)
2687 {
2688    if(Debug)
2689       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
2690
2691    Item::Done(Message, Hashes, Cnf);
2692
2693    if (std::any_of(allPatches->begin(), allPatches->end(),
2694             [](pkgAcqIndexMergeDiffs const * const P) { return P->State == StateErrorDiff; }))
2695    {
2696       if(Debug)
2697          std::clog << "Another patch failed already, no point in processing this one." << std::endl;
2698       State = StateErrorDiff;
2699       return;
2700    }
2701
2702    std::string const UncompressedUnpatchedFile = GetPartialFileNameFromURI(Target.URI);
2703    std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile);
2704    if (UnpatchedFile.empty())
2705    {
2706       _error->Fatal("Unpatched file %s doesn't exist (anymore)!", UncompressedUnpatchedFile.c_str());
2707       State = StateErrorDiff;
2708       return;
2709    }
2710    std::string const PatchFile = GetMergeDiffsPatchFileName(UnpatchedFile, patch.file);
2711    std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target);
2712
2713    switch (State)
2714    {
2715       case StateFetchDiff:
2716          Rename(DestFile, PatchFile);
2717
2718          // check if this is the last completed diff
2719          State = StateDoneDiff;
2720          for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
2721                I != allPatches->end(); ++I)
2722             if ((*I)->State != StateDoneDiff)
2723             {
2724                if(Debug)
2725                   std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
2726                return;
2727             }
2728          // this is the last completed diff, so we are ready to apply now
2729          DestFile = GetKeepCompressedFileName(UncompressedUnpatchedFile + "-patched", Target);
2730          if(Debug)
2731             std::clog << "Sending to rred method: " << UnpatchedFile << std::endl;
2732          State = StateApplyDiff;
2733          Local = true;
2734          Desc.URI = "rred:" + UnpatchedFile;
2735          QueueURI(Desc);
2736          SetActiveSubprocess("rred");
2737          return;
2738       case StateApplyDiff:
2739          // success in download & apply all diffs, finialize and clean up
2740          if(Debug)
2741             std::clog << "Queue patched file in place: " << std::endl
2742                << DestFile << " -> " << PatchedFile << std::endl;
2743
2744          // queue for copy by the transaction manager
2745          TransactionManager->TransactionStageCopy(this, DestFile, GetKeepCompressedFileName(GetFinalFilename(), Target));
2746
2747          // ensure the ed's are gone regardless of list-cleanup
2748          for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
2749                I != allPatches->end(); ++I)
2750             RemoveFile("pkgAcqIndexMergeDiffs::Done", GetMergeDiffsPatchFileName(UnpatchedFile, (*I)->patch.file));
2751          RemoveFile("pkgAcqIndexMergeDiffs::Done", UnpatchedFile);
2752
2753          // all set and done
2754          Complete = true;
2755          if(Debug)
2756             std::clog << "allDone: " << DestFile << "\n" << std::endl;
2757          return;
2758       case StateDoneDiff: _error->Fatal("Done called for %s which is in an invalid Done state", PatchFile.c_str()); break;
2759       case StateErrorDiff: _error->Fatal("Done called for %s which is in an invalid Error state", PatchFile.c_str()); break;
2760    }
2761 }
2762                                                                         /*}}}*/
2763 std::string pkgAcqIndexMergeDiffs::Custom600Headers() const             /*{{{*/
2764 {
2765    if(State != StateApplyDiff)
2766       return pkgAcqBaseIndex::Custom600Headers();
2767    std::ostringstream patchhashes;
2768    unsigned int seen_patches = 0;
2769    for (auto && hs : (*allPatches)[0]->patch.result_hashes)
2770       patchhashes <<  "\nStart-" << hs.HashType() << "-Hash: " << hs.HashValue();
2771    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
2772          I != allPatches->end(); ++I)
2773    {
2774       HashStringList const ExpectedHashes = (*I)->patch.patch_hashes;
2775       for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
2776          patchhashes <<  "\nPatch-" << seen_patches << "-" << hs->HashType() << "-Hash: " << hs->HashValue();
2777       ++seen_patches;
2778    }
2779    patchhashes << pkgAcqBaseIndex::Custom600Headers();
2780    return patchhashes.str();
2781 }
2782                                                                         /*}}}*/
2783 pkgAcqIndexMergeDiffs::~pkgAcqIndexMergeDiffs() {}
2784
2785 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
2786 pkgAcqIndex::pkgAcqIndex(pkgAcquire * const Owner,
2787                          pkgAcqMetaClearSig * const TransactionManager,
2788                          IndexTarget const &Target)
2789    : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), Stage(STAGE_DOWNLOAD),
2790    CompressionExtensions(Target.Option(IndexTarget::COMPRESSIONTYPES))
2791 {
2792    Init(Target.URI, Target.Description, Target.ShortDesc);
2793
2794    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
2795       std::clog << "New pkgIndex with TransactionManager "
2796                 << TransactionManager << std::endl;
2797 }
2798                                                                         /*}}}*/
2799 // AcqIndex::Init - defered Constructor                                 /*{{{*/
2800 static void NextCompressionExtension(std::string &CurrentCompressionExtension, std::string &CompressionExtensions, bool const preview)
2801 {
2802    size_t const nextExt = CompressionExtensions.find(' ');
2803    if (nextExt == std::string::npos)
2804    {
2805       CurrentCompressionExtension = CompressionExtensions;
2806       if (preview == false)
2807          CompressionExtensions.clear();
2808    }
2809    else
2810    {
2811       CurrentCompressionExtension = CompressionExtensions.substr(0, nextExt);
2812       if (preview == false)
2813          CompressionExtensions = CompressionExtensions.substr(nextExt+1);
2814    }
2815 }
2816 void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
2817                        string const &ShortDesc)
2818 {
2819    Stage = STAGE_DOWNLOAD;
2820
2821    DestFile = GetPartialFileNameFromURI(URI);
2822    NextCompressionExtension(CurrentCompressionExtension, CompressionExtensions, false);
2823
2824    // store file size of the download to ensure the fetcher gives
2825    // accurate progress reporting
2826    FileSize = GetExpectedHashes().FileSize();
2827
2828    if (CurrentCompressionExtension == "uncompressed")
2829    {
2830       Desc.URI = URI;
2831    }
2832    else if (CurrentCompressionExtension == "by-hash")
2833    {
2834       NextCompressionExtension(CurrentCompressionExtension, CompressionExtensions, true);
2835       if(unlikely(CurrentCompressionExtension.empty()))
2836          return;
2837       if (CurrentCompressionExtension != "uncompressed")
2838       {
2839          Desc.URI = URI + '.' + CurrentCompressionExtension;
2840          DestFile = DestFile + '.' + CurrentCompressionExtension;
2841       }
2842
2843       HashStringList const Hashes = GetExpectedHashes();
2844       HashString const * const TargetHash = Hashes.find(NULL);
2845       if (unlikely(TargetHash == nullptr))
2846          return;
2847       std::string const ByHash = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue();
2848       size_t const trailing_slash = Desc.URI.find_last_of("/");
2849       if (unlikely(trailing_slash == std::string::npos))
2850          return;
2851       Desc.URI = Desc.URI.replace(
2852             trailing_slash,
2853             Desc.URI.substr(trailing_slash+1).size()+1,
2854             ByHash);
2855    }
2856    else if (unlikely(CurrentCompressionExtension.empty()))
2857       return;
2858    else
2859    {
2860       Desc.URI = URI + '.' + CurrentCompressionExtension;
2861       DestFile = DestFile + '.' + CurrentCompressionExtension;
2862    }
2863
2864
2865    Desc.Description = URIDesc;
2866    Desc.Owner = this;
2867    Desc.ShortDesc = ShortDesc;
2868
2869    QueueURI(Desc);
2870 }
2871                                                                         /*}}}*/
2872 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2873 // ---------------------------------------------------------------------
2874 /* The only header we use is the last-modified header. */
2875 string pkgAcqIndex::Custom600Headers() const
2876 {
2877
2878    string msg = "\nIndex-File: true";
2879
2880    if (TransactionManager->LastMetaIndexParser == NULL)
2881    {
2882       std::string const Final = GetFinalFilename();
2883
2884       struct stat Buf;
2885       if (stat(Final.c_str(),&Buf) == 0)
2886          msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);
2887    }
2888
2889    if(Target.IsOptional)
2890       msg += "\nFail-Ignore: true";
2891
2892    return msg;
2893 }
2894                                                                         /*}}}*/
2895 // AcqIndex::Failed - getting the indexfile failed                      /*{{{*/
2896 void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
2897 {
2898    pkgAcqBaseIndex::Failed(Message,Cnf);
2899
2900    // authorisation matches will not be fixed by other compression types
2901    if (Status != StatAuthError)
2902    {
2903       if (CompressionExtensions.empty() == false)
2904       {
2905          Init(Target.URI, Desc.Description, Desc.ShortDesc);
2906          Status = StatIdle;
2907          return;
2908       }
2909    }
2910
2911    if(Target.IsOptional && GetExpectedHashes().empty() && Stage == STAGE_DOWNLOAD)
2912       Status = StatDone;
2913    else
2914       TransactionManager->AbortTransaction();
2915 }
2916                                                                         /*}}}*/
2917 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
2918 // ---------------------------------------------------------------------
2919 /* This goes through a number of states.. On the initial fetch the
2920    method could possibly return an alternate filename which points
2921    to the uncompressed version of the file. If this is so the file
2922    is copied into the partial directory. In all other cases the file
2923    is decompressed with a compressed uri. */
2924 void pkgAcqIndex::Done(string const &Message,
2925                        HashStringList const &Hashes,
2926                        pkgAcquire::MethodConfig const * const Cfg)
2927 {
2928    Item::Done(Message,Hashes,Cfg);
2929
2930    switch(Stage)
2931    {
2932       case STAGE_DOWNLOAD:
2933          StageDownloadDone(Message);
2934          break;
2935       case STAGE_DECOMPRESS_AND_VERIFY:
2936          StageDecompressDone();
2937          break;
2938    }
2939 }
2940                                                                         /*}}}*/
2941 // AcqIndex::StageDownloadDone - Queue for decompress and verify        /*{{{*/
2942 void pkgAcqIndex::StageDownloadDone(string const &Message)
2943 {
2944    Local = true;
2945    Complete = true;
2946
2947    std::string const AltFilename = LookupTag(Message,"Alt-Filename");
2948    std::string Filename = LookupTag(Message,"Filename");
2949
2950    // we need to verify the file against the current Release file again
2951    // on if-modfied-since hit to avoid a stale attack against us
2952    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2953    {
2954       // copy FinalFile into partial/ so that we check the hash again
2955       string const FinalFile = GetExistingFilename(GetFinalFileNameFromURI(Target.URI));
2956       if (symlink(FinalFile.c_str(), DestFile.c_str()) != 0)
2957          _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking final file %s back to %s failed", FinalFile.c_str(), DestFile.c_str());
2958       else
2959       {
2960          EraseFileName = DestFile;
2961          Filename = DestFile;
2962       }
2963       Stage = STAGE_DECOMPRESS_AND_VERIFY;
2964       Desc.URI = "store:" + Filename;
2965       QueueURI(Desc);
2966       SetActiveSubprocess(::URI(Desc.URI).Access);
2967       return;
2968    }
2969    // methods like file:// give us an alternative (uncompressed) file
2970    else if (Target.KeepCompressed == false && AltFilename.empty() == false)
2971    {
2972       Filename = AltFilename;
2973       EraseFileName.clear();
2974    }
2975    // Methods like e.g. "file:" will give us a (compressed) FileName that is
2976    // not the "DestFile" we set, in this case we uncompress from the local file
2977    else if (Filename != DestFile && RealFileExists(DestFile) == false)
2978    {
2979       // symlinking ensures that the filename can be used for compression detection
2980       // that is e.g. needed for by-hash which has no extension over file
2981       if (symlink(Filename.c_str(),DestFile.c_str()) != 0)
2982          _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", Filename.c_str(), DestFile.c_str());
2983       else
2984       {
2985          EraseFileName = DestFile;
2986          Filename = DestFile;
2987       }
2988    }
2989
2990    Stage = STAGE_DECOMPRESS_AND_VERIFY;
2991    DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
2992    if (Filename != DestFile && flExtension(Filename) == flExtension(DestFile))
2993       Desc.URI = "copy:" + Filename;
2994    else
2995       Desc.URI = "store:" + Filename;
2996    if (DestFile == Filename)
2997    {
2998       if (CurrentCompressionExtension == "uncompressed")
2999          return StageDecompressDone();
3000       DestFile = "/dev/null";
3001    }
3002
3003    if (EraseFileName.empty() && Filename != AltFilename)
3004       EraseFileName = Filename;
3005
3006    // queue uri for the next stage
3007    QueueURI(Desc);
3008    SetActiveSubprocess(::URI(Desc.URI).Access);
3009 }
3010                                                                         /*}}}*/
3011 // AcqIndex::StageDecompressDone - Final verification                   /*{{{*/
3012 void pkgAcqIndex::StageDecompressDone()
3013 {
3014    if (DestFile == "/dev/null")
3015       DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
3016
3017    // Done, queue for rename on transaction finished
3018    TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
3019 }
3020                                                                         /*}}}*/
3021 pkgAcqIndex::~pkgAcqIndex() {}
3022
3023
3024 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
3025 // ---------------------------------------------------------------------
3026 /* This just sets up the initial fetch environment and queues the first
3027    possibilitiy */
3028 pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sources,
3029                              pkgRecords * const Recs,pkgCache::VerIterator const &Version,
3030                              string &StoreFilename) :
3031                Item(Owner), d(NULL), LocalSource(false), Version(Version), Sources(Sources), Recs(Recs),
3032                StoreFilename(StoreFilename), Vf(Version.FileList()),
3033                Trusted(false)
3034 {
3035    Retries = _config->FindI("Acquire::Retries",0);
3036
3037    if (Version.Arch() == 0)
3038    {
3039       _error->Error(_("I wasn't able to locate a file for the %s package. "
3040                       "This might mean you need to manually fix this package. "
3041                       "(due to missing arch)"),
3042                     Version.ParentPkg().FullName().c_str());
3043       return;
3044    }
3045
3046    /* We need to find a filename to determine the extension. We make the
3047       assumption here that all the available sources for this version share
3048       the same extension.. */
3049    // Skip not source sources, they do not have file fields.
3050    for (; Vf.end() == false; ++Vf)
3051    {
3052       if (Vf.File().Flagged(pkgCache::Flag::NotSource))
3053          continue;
3054       break;
3055    }
3056
3057    // Does not really matter here.. we are going to fail out below
3058    if (Vf.end() != true)
3059    {
3060       // If this fails to get a file name we will bomb out below.
3061       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
3062       if (_error->PendingError() == true)
3063          return;
3064
3065       // Generate the final file name as: package_version_arch.foo
3066       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
3067                       QuoteString(Version.VerStr(),"_:") + '_' +
3068                       QuoteString(Version.Arch(),"_:.") +
3069                       "." + flExtension(Parse.FileName());
3070    }
3071
3072    // check if we have one trusted source for the package. if so, switch
3073    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
3074    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
3075    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
3076    bool seenUntrusted = false;
3077    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
3078    {
3079       pkgIndexFile *Index;
3080       if (Sources->FindIndex(i.File(),Index) == false)
3081          continue;
3082
3083       if (debugAuth == true)
3084          std::cerr << "Checking index: " << Index->Describe()
3085                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
3086
3087       if (Index->IsTrusted() == true)
3088       {
3089          Trusted = true;
3090          if (allowUnauth == false)
3091             break;
3092       }
3093       else
3094          seenUntrusted = true;
3095    }
3096
3097    // "allow-unauthenticated" restores apts old fetching behaviour
3098    // that means that e.g. unauthenticated file:// uris are higher
3099    // priority than authenticated http:// uris
3100    if (allowUnauth == true && seenUntrusted == true)
3101       Trusted = false;
3102
3103    // Select a source
3104    if (QueueNext() == false && _error->PendingError() == false)
3105       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
3106                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
3107 }
3108                                                                         /*}}}*/
3109 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
3110 // ---------------------------------------------------------------------
3111 /* This queues the next available file version for download. It checks if
3112    the archive is already available in the cache and stashs the MD5 for
3113    checking later. */
3114 bool pkgAcqArchive::QueueNext()
3115 {
3116    for (; Vf.end() == false; ++Vf)
3117    {
3118       pkgCache::PkgFileIterator const PkgF = Vf.File();
3119       // Ignore not source sources
3120       if (PkgF.Flagged(pkgCache::Flag::NotSource))
3121          continue;
3122
3123       // Try to cross match against the source list
3124       pkgIndexFile *Index;
3125       if (Sources->FindIndex(PkgF, Index) == false)
3126             continue;
3127       LocalSource = PkgF.Flagged(pkgCache::Flag::LocalSource);
3128
3129       // only try to get a trusted package from another source if that source
3130       // is also trusted
3131       if(Trusted && !Index->IsTrusted())
3132          continue;
3133
3134       // Grab the text package record
3135       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
3136       if (_error->PendingError() == true)
3137          return false;
3138
3139       string PkgFile = Parse.FileName();
3140       ExpectedHashes = Parse.Hashes();
3141
3142       if (PkgFile.empty() == true)
3143          return _error->Error(_("The package index files are corrupted. No Filename: "
3144                               "field for package %s."),
3145                               Version.ParentPkg().Name());
3146
3147       Desc.URI = Index->ArchiveURI(PkgFile);
3148       Desc.Description = Index->ArchiveInfo(Version);
3149       Desc.Owner = this;
3150       Desc.ShortDesc = Version.ParentPkg().FullName(true);
3151
3152       // See if we already have the file. (Legacy filenames)
3153       FileSize = Version->Size;
3154       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
3155       struct stat Buf;
3156       if (stat(FinalFile.c_str(),&Buf) == 0)
3157       {
3158          // Make sure the size matches
3159          if ((unsigned long long)Buf.st_size == Version->Size)
3160          {
3161             Complete = true;
3162             Local = true;
3163             Status = StatDone;
3164             StoreFilename = DestFile = FinalFile;
3165             return true;
3166          }
3167
3168          /* Hmm, we have a file and its size does not match, this means it is
3169             an old style mismatched arch */
3170          RemoveFile("pkgAcqArchive::QueueNext", FinalFile);
3171       }
3172
3173       // Check it again using the new style output filenames
3174       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
3175       if (stat(FinalFile.c_str(),&Buf) == 0)
3176       {
3177          // Make sure the size matches
3178          if ((unsigned long long)Buf.st_size == Version->Size)
3179          {
3180             Complete = true;
3181             Local = true;
3182             Status = StatDone;
3183             StoreFilename = DestFile = FinalFile;
3184             return true;
3185          }
3186
3187          /* Hmm, we have a file and its size does not match, this shouldn't
3188             happen.. */
3189          RemoveFile("pkgAcqArchive::QueueNext", FinalFile);
3190       }
3191
3192       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
3193
3194       // Check the destination file
3195       if (stat(DestFile.c_str(),&Buf) == 0)
3196       {
3197          // Hmm, the partial file is too big, erase it
3198          if ((unsigned long long)Buf.st_size > Version->Size)
3199             RemoveFile("pkgAcqArchive::QueueNext", DestFile);
3200          else
3201             PartialSize = Buf.st_size;
3202       }
3203
3204       // Disables download of archives - useful if no real installation follows,
3205       // e.g. if we are just interested in proposed installation order
3206       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
3207       {
3208          Complete = true;
3209          Local = true;
3210          Status = StatDone;
3211          StoreFilename = DestFile = FinalFile;
3212          return true;
3213       }
3214
3215       // Create the item
3216       Local = false;
3217       QueueURI(Desc);
3218
3219       ++Vf;
3220       return true;
3221    }
3222    return false;
3223 }
3224                                                                         /*}}}*/
3225 // AcqArchive::Done - Finished fetching                                 /*{{{*/
3226 // ---------------------------------------------------------------------
3227 /* */
3228 void pkgAcqArchive::Done(string const &Message, HashStringList const &Hashes,
3229                          pkgAcquire::MethodConfig const * const Cfg)
3230 {
3231    Item::Done(Message, Hashes, Cfg);
3232
3233    // Grab the output filename
3234    std::string const FileName = LookupTag(Message,"Filename");
3235    if (DestFile !=  FileName && RealFileExists(DestFile) == false)
3236    {
3237       StoreFilename = DestFile = FileName;
3238       Local = true;
3239       Complete = true;
3240       return;
3241    }
3242
3243    // Done, move it into position
3244    string const FinalFile = GetFinalFilename();
3245    Rename(DestFile,FinalFile);
3246    StoreFilename = DestFile = FinalFile;
3247    Complete = true;
3248 }
3249                                                                         /*}}}*/
3250 // AcqArchive::Failed - Failure handler                                 /*{{{*/
3251 // ---------------------------------------------------------------------
3252 /* Here we try other sources */
3253 void pkgAcqArchive::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
3254 {
3255    Item::Failed(Message,Cnf);
3256
3257    /* We don't really want to retry on failed media swaps, this prevents
3258       that. An interesting observation is that permanent failures are not
3259       recorded. */
3260    if (Cnf->Removable == true &&
3261        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
3262    {
3263       // Vf = Version.FileList();
3264       while (Vf.end() == false) ++Vf;
3265       StoreFilename = string();
3266       return;
3267    }
3268
3269    Status = StatIdle;
3270    if (QueueNext() == false)
3271    {
3272       // This is the retry counter
3273       if (Retries != 0 &&
3274           Cnf->LocalOnly == false &&
3275           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
3276       {
3277          Retries--;
3278          Vf = Version.FileList();
3279          if (QueueNext() == true)
3280             return;
3281       }
3282
3283       StoreFilename = string();
3284       Status = StatError;
3285    }
3286 }
3287                                                                         /*}}}*/
3288 APT_PURE bool pkgAcqArchive::IsTrusted() const                          /*{{{*/
3289 {
3290    return Trusted;
3291 }
3292                                                                         /*}}}*/
3293 void pkgAcqArchive::Finished()                                          /*{{{*/
3294 {
3295    if (Status == pkgAcquire::Item::StatDone &&
3296        Complete == true)
3297       return;
3298    StoreFilename = string();
3299 }
3300                                                                         /*}}}*/
3301 std::string pkgAcqArchive::DescURI() const                              /*{{{*/
3302 {
3303    return Desc.URI;
3304 }
3305                                                                         /*}}}*/
3306 std::string pkgAcqArchive::ShortDesc() const                            /*{{{*/
3307 {
3308    return Desc.ShortDesc;
3309 }
3310                                                                         /*}}}*/
3311 pkgAcqArchive::~pkgAcqArchive() {}
3312
3313 // AcqChangelog::pkgAcqChangelog - Constructors                         /*{{{*/
3314 class pkgAcqChangelog::Private
3315 {
3316    public:
3317    std::string FinalFile;
3318 };
3319 pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::VerIterator const &Ver,
3320       std::string const &DestDir, std::string const &DestFilename) :
3321    pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(Ver.SourcePkgName()), SrcVersion(Ver.SourceVerStr())
3322 {
3323    Desc.URI = URI(Ver);
3324    Init(DestDir, DestFilename);
3325 }
3326 // some parameters are char* here as they come likely from char* interfaces – which can also return NULL
3327 pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::RlsFileIterator const &RlsFile,
3328       char const * const Component, char const * const SrcName, char const * const SrcVersion,
3329       const string &DestDir, const string &DestFilename) :
3330    pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(SrcName), SrcVersion(SrcVersion)
3331 {
3332    Desc.URI = URI(RlsFile, Component, SrcName, SrcVersion);
3333    Init(DestDir, DestFilename);
3334 }
3335 pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner,
3336       std::string const &URI, char const * const SrcName, char const * const SrcVersion,
3337       const string &DestDir, const string &DestFilename) :
3338    pkgAcquire::Item(Owner), d(new pkgAcqChangelog::Private()), SrcName(SrcName), SrcVersion(SrcVersion)
3339 {
3340    Desc.URI = URI;
3341    Init(DestDir, DestFilename);
3342 }
3343 void pkgAcqChangelog::Init(std::string const &DestDir, std::string const &DestFilename)
3344 {
3345    if (Desc.URI.empty())
3346    {
3347       Status = StatError;
3348       // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1
3349       strprintf(ErrorText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str());
3350       // Let the error message print something sensible rather than "Failed to fetch /"
3351       if (DestFilename.empty())
3352          DestFile = SrcName + ".changelog";
3353       else
3354          DestFile = DestFilename;
3355       Desc.URI = "changelog:/" + DestFile;
3356       return;
3357    }
3358
3359    std::string DestFileName;
3360    if (DestFilename.empty())
3361       DestFileName = flCombine(DestFile, SrcName + ".changelog");
3362    else
3363       DestFileName = flCombine(DestFile, DestFilename);
3364
3365    std::string const SandboxUser = _config->Find("APT::Sandbox::User");
3366    std::string const systemTemp = GetTempDir(SandboxUser);
3367    char tmpname[1000];
3368    snprintf(tmpname, sizeof(tmpname), "%s/apt-changelog-XXXXXX", systemTemp.c_str());
3369    if (NULL == mkdtemp(tmpname))
3370    {
3371       _error->Errno("mkdtemp", "mkdtemp failed in changelog acquire of %s %s", SrcName.c_str(), SrcVersion.c_str());
3372       Status = StatError;
3373       return;
3374    }
3375    TemporaryDirectory = tmpname;
3376
3377    ChangeOwnerAndPermissionOfFile("Item::QueueURI", TemporaryDirectory.c_str(),
3378          SandboxUser.c_str(), "root", 0700);
3379
3380    DestFile = flCombine(TemporaryDirectory, DestFileName);
3381    if (DestDir.empty() == false)
3382    {
3383       d->FinalFile = flCombine(DestDir, DestFileName);
3384       if (RealFileExists(d->FinalFile))
3385       {
3386          FileFd file1, file2;
3387          if (file1.Open(DestFile, FileFd::WriteOnly | FileFd::Create | FileFd::Exclusive) &&
3388                file2.Open(d->FinalFile, FileFd::ReadOnly) && CopyFile(file2, file1))
3389          {
3390             struct timeval times[2];
3391             times[0].tv_sec = times[1].tv_sec = file2.ModificationTime();
3392             times[0].tv_usec = times[1].tv_usec = 0;
3393             utimes(DestFile.c_str(), times);
3394          }
3395       }
3396    }
3397
3398    Desc.ShortDesc = "Changelog";
3399    strprintf(Desc.Description, "%s %s %s Changelog", URI::SiteOnly(Desc.URI).c_str(), SrcName.c_str(), SrcVersion.c_str());
3400    Desc.Owner = this;
3401    QueueURI(Desc);
3402 }
3403                                                                         /*}}}*/
3404 std::string pkgAcqChangelog::URI(pkgCache::VerIterator const &Ver)      /*{{{*/
3405 {
3406    std::string const confOnline = "Acquire::Changelogs::AlwaysOnline";
3407    bool AlwaysOnline = _config->FindB(confOnline, false);
3408    if (AlwaysOnline == false)
3409       for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF)
3410       {
3411          pkgCache::PkgFileIterator const PF = VF.File();
3412          if (PF.Flagged(pkgCache::Flag::NotSource) || PF->Release == 0)
3413             continue;
3414          pkgCache::RlsFileIterator const RF = PF.ReleaseFile();
3415          if (RF->Origin != 0 && _config->FindB(confOnline + "::Origin::" + RF.Origin(), false))
3416          {
3417             AlwaysOnline = true;
3418             break;
3419          }
3420       }
3421    if (AlwaysOnline == false)
3422    {
3423       pkgCache::PkgIterator const Pkg = Ver.ParentPkg();
3424       if (Pkg->CurrentVer != 0 && Pkg.CurrentVer() == Ver)
3425       {
3426          std::string const basename = std::string("/usr/share/doc/") + Pkg.Name() + "/changelog";
3427          std::string const debianname = basename + ".Debian";
3428          if (FileExists(debianname))
3429             return "copy://" + debianname;
3430          else if (FileExists(debianname + ".gz"))
3431             return "gzip://" + debianname + ".gz";
3432          else if (FileExists(basename))
3433             return "copy://" + basename;
3434          else if (FileExists(basename + ".gz"))
3435             return "gzip://" + basename + ".gz";
3436       }
3437    }
3438
3439    char const * const SrcName = Ver.SourcePkgName();
3440    char const * const SrcVersion = Ver.SourceVerStr();
3441    // find the first source for this version which promises a changelog
3442    for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF)
3443    {
3444       pkgCache::PkgFileIterator const PF = VF.File();
3445       if (PF.Flagged(pkgCache::Flag::NotSource) || PF->Release == 0)
3446          continue;
3447       pkgCache::RlsFileIterator const RF = PF.ReleaseFile();
3448       std::string const uri = URI(RF, PF.Component(), SrcName, SrcVersion);
3449       if (uri.empty())
3450          continue;
3451       return uri;
3452    }
3453    return "";
3454 }
3455 std::string pkgAcqChangelog::URITemplate(pkgCache::RlsFileIterator const &Rls)
3456 {
3457    if (Rls.end() == true || (Rls->Label == 0 && Rls->Origin == 0))
3458       return "";
3459    std::string const serverConfig = "Acquire::Changelogs::URI";
3460    std::string server;
3461 #define APT_EMPTY_SERVER \
3462    if (server.empty() == false) \
3463    { \
3464       if (server != "no") \
3465          return server; \
3466       return ""; \
3467    }
3468 #define APT_CHECK_SERVER(X, Y) \
3469    if (Rls->X != 0) \
3470    { \
3471       std::string const specialServerConfig = serverConfig + "::" + Y + #X + "::" + Rls.X(); \
3472       server = _config->Find(specialServerConfig); \
3473       APT_EMPTY_SERVER \
3474    }
3475    // this way e.g. Debian-Security can fallback to Debian
3476    APT_CHECK_SERVER(Label, "Override::")
3477    APT_CHECK_SERVER(Origin, "Override::")
3478
3479    if (RealFileExists(Rls.FileName()))
3480    {
3481       _error->PushToStack();
3482       FileFd rf;
3483       /* This can be costly. A caller wanting to get millions of URIs might
3484          want to do this on its own once and use Override settings.
3485          We don't do this here as Origin/Label are not as unique as they
3486          should be so this could produce request order-dependent anomalies */
3487       if (OpenMaybeClearSignedFile(Rls.FileName(), rf) == true)
3488       {
3489          pkgTagFile TagFile(&rf, rf.Size());
3490          pkgTagSection Section;
3491          if (TagFile.Step(Section) == true)
3492             server = Section.FindS("Changelogs");
3493       }
3494       _error->RevertToStack();
3495       APT_EMPTY_SERVER
3496    }
3497
3498    APT_CHECK_SERVER(Label, "")
3499    APT_CHECK_SERVER(Origin, "")
3500 #undef APT_CHECK_SERVER
3501 #undef APT_EMPTY_SERVER
3502    return "";
3503 }
3504 std::string pkgAcqChangelog::URI(pkgCache::RlsFileIterator const &Rls,
3505          char const * const Component, char const * const SrcName,
3506          char const * const SrcVersion)
3507 {
3508    return URI(URITemplate(Rls), Component, SrcName, SrcVersion);
3509 }
3510 std::string pkgAcqChangelog::URI(std::string const &Template,
3511          char const * const Component, char const * const SrcName,
3512          char const * const SrcVersion)
3513 {
3514    if (Template.find("@CHANGEPATH@") == std::string::npos)
3515       return "";
3516
3517    // the path is: COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER, e.g. main/a/apt/1.1 or contrib/liba/libapt/2.0
3518    std::string Src = SrcName;
3519    std::string path = APT::String::Startswith(SrcName, "lib") ? Src.substr(0, 4) : Src.substr(0,1);
3520    path.append("/").append(Src).append("/");
3521    path.append(Src).append("_").append(StripEpoch(SrcVersion));
3522    // we omit component for releases without one (= flat-style repositories)
3523    if (Component != NULL && strlen(Component) != 0)
3524       path = std::string(Component) + "/" + path;
3525
3526    return SubstVar(Template, "@CHANGEPATH@", path);
3527 }
3528                                                                         /*}}}*/
3529 // AcqChangelog::Failed - Failure handler                               /*{{{*/
3530 void pkgAcqChangelog::Failed(string const &Message, pkgAcquire::MethodConfig const * const Cnf)
3531 {
3532    Item::Failed(Message,Cnf);
3533
3534    std::string errText;
3535    // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1
3536    strprintf(errText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str());
3537
3538    // Error is probably something techy like 404 Not Found
3539    if (ErrorText.empty())
3540       ErrorText = errText;
3541    else
3542       ErrorText = errText + " (" + ErrorText + ")";
3543 }
3544                                                                         /*}}}*/
3545 // AcqChangelog::Done - Item downloaded OK                              /*{{{*/
3546 void pkgAcqChangelog::Done(string const &Message,HashStringList const &CalcHashes,
3547                       pkgAcquire::MethodConfig const * const Cnf)
3548 {
3549    Item::Done(Message,CalcHashes,Cnf);
3550    if (d->FinalFile.empty() == false)
3551    {
3552       if (RemoveFile("pkgAcqChangelog::Done", d->FinalFile) == false ||
3553             Rename(DestFile, d->FinalFile) == false)
3554          Status = StatError;
3555    }
3556
3557    Complete = true;
3558 }
3559                                                                         /*}}}*/
3560 pkgAcqChangelog::~pkgAcqChangelog()                                     /*{{{*/
3561 {
3562    if (TemporaryDirectory.empty() == false)
3563    {
3564       RemoveFile("~pkgAcqChangelog", DestFile);
3565       rmdir(TemporaryDirectory.c_str());
3566    }
3567    delete d;
3568 }
3569                                                                         /*}}}*/
3570
3571 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
3572 pkgAcqFile::pkgAcqFile(pkgAcquire * const Owner,string const &URI, HashStringList const &Hashes,
3573                        unsigned long long const Size,string const &Dsc,string const &ShortDesc,
3574                        const string &DestDir, const string &DestFilename,
3575                        bool const IsIndexFile) :
3576                        Item(Owner), d(NULL), IsIndexFile(IsIndexFile), ExpectedHashes(Hashes)
3577 {
3578    Retries = _config->FindI("Acquire::Retries",0);
3579
3580    if(!DestFilename.empty())
3581       DestFile = DestFilename;
3582    else if(!DestDir.empty())
3583       DestFile = DestDir + "/" + flNotDir(URI);
3584    else
3585       DestFile = flNotDir(URI);
3586
3587    // Create the item
3588    Desc.URI = URI;
3589    Desc.Description = Dsc;
3590    Desc.Owner = this;
3591
3592    // Set the short description to the archive component
3593    Desc.ShortDesc = ShortDesc;
3594
3595    // Get the transfer sizes
3596    FileSize = Size;
3597    struct stat Buf;
3598    if (stat(DestFile.c_str(),&Buf) == 0)
3599    {
3600       // Hmm, the partial file is too big, erase it
3601       if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
3602          RemoveFile("pkgAcqFile", DestFile);
3603       else
3604          PartialSize = Buf.st_size;
3605    }
3606
3607    QueueURI(Desc);
3608 }
3609                                                                         /*}}}*/
3610 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
3611 void pkgAcqFile::Done(string const &Message,HashStringList const &CalcHashes,
3612                       pkgAcquire::MethodConfig const * const Cnf)
3613 {
3614    Item::Done(Message,CalcHashes,Cnf);
3615
3616    std::string const FileName = LookupTag(Message,"Filename");
3617    Complete = true;
3618
3619    // The files timestamp matches
3620    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
3621       return;
3622
3623    // We have to copy it into place
3624    if (RealFileExists(DestFile.c_str()) == false)
3625    {
3626       Local = true;
3627       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
3628           Cnf->Removable == true)
3629       {
3630          Desc.URI = "copy:" + FileName;
3631          QueueURI(Desc);
3632          return;
3633       }
3634
3635       // Erase the file if it is a symlink so we can overwrite it
3636       struct stat St;
3637       if (lstat(DestFile.c_str(),&St) == 0)
3638       {
3639          if (S_ISLNK(St.st_mode) != 0)
3640             RemoveFile("pkgAcqFile::Done", DestFile);
3641       }
3642
3643       // Symlink the file
3644       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
3645       {
3646          _error->PushToStack();
3647          _error->Errno("pkgAcqFile::Done", "Symlinking file %s failed", DestFile.c_str());
3648          std::stringstream msg;
3649          _error->DumpErrors(msg, GlobalError::DEBUG, false);
3650          _error->RevertToStack();
3651          ErrorText = msg.str();
3652          Status = StatError;
3653          Complete = false;
3654       }
3655    }
3656 }
3657                                                                         /*}}}*/
3658 // AcqFile::Failed - Failure handler                                    /*{{{*/
3659 // ---------------------------------------------------------------------
3660 /* Here we try other sources */
3661 void pkgAcqFile::Failed(string const &Message, pkgAcquire::MethodConfig const * const Cnf)
3662 {
3663    Item::Failed(Message,Cnf);
3664
3665    // This is the retry counter
3666    if (Retries != 0 &&
3667        Cnf->LocalOnly == false &&
3668        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
3669    {
3670       --Retries;
3671       QueueURI(Desc);
3672       Status = StatIdle;
3673       return;
3674    }
3675
3676 }
3677                                                                         /*}}}*/
3678 string pkgAcqFile::Custom600Headers() const                             /*{{{*/
3679 {
3680    if (IsIndexFile)
3681       return "\nIndex-File: true";
3682    return "";
3683 }
3684                                                                         /*}}}*/
3685 pkgAcqFile::~pkgAcqFile() {}