implement Fallback-Of for IndexTargets

Sometimes index files are in different locations in a repository as it
is currently the case for Contents files which are per-component in
Debian, but aren't in Ubuntu. This has historic reasons and is perhaps
changed soon, but such cases of transitions can always happen in the
future again, so we should prepare:

Introduced is a new field declaring that the current item should only be
downloaded if the mentioned item wasn't allowing for transitions without
a flagday in clients and archives.

This isn't implemented 'simpler' with multiple MetaKeys as items (could)
change their descriptions and perhaps also other configuration bits with
their location.

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 959627a01ed2a6111568767e05ff5b388f31ae70..8c45acdddb9c2117dc4461849c9124c4353ade7e 100644 (file)
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -1196,9 +1196,18 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)                 /*{{{*/
    // at this point the real Items are loaded in the fetcher
    ExpectedAdditionalItems = 0;
 
+   std::set<std::string> targetsSeen;
    bool const metaBaseSupportsByHash = TransactionManager->MetaIndexParser->GetSupportsAcquireByHash();
-   for (auto Target : TransactionManager->MetaIndexParser->GetIndexTargets())
+   for (auto &Target: TransactionManager->MetaIndexParser->GetIndexTargets())
    {
+      // if we have seen a target which is created-by a target this one here is declared a
+      // fallback to, we skip acquiring the fallback (but we make sure we clean up)
+      if (targetsSeen.find(Target.Option(IndexTarget::FALLBACK_OF)) != targetsSeen.end())
+      {
+        targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
+        new CleanupItem(Owner, TransactionManager, Target);
+        continue;
+      }
       // all is an implementation detail. Users shouldn't use this as arch
       // We need this support trickery here as e.g. Debian has binary-all files already,
       // but arch:all packages are still in the arch:any files, so we would waste precious
@@ -1267,6 +1276,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)                  /*{{{*/
               else if (hashes.FileSize() == 0)
               {
                  new CleanupItem(Owner, TransactionManager, Target);
+                 targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
                  continue;
               }
            }
@@ -1327,6 +1337,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)                  /*{{{*/
            std::string const idxfilename = GetFinalFileNameFromURI(GetDiffIndexURI(Target));
            if (FileExists(idxfilename))
               new NoActionItem(Owner, Target, idxfilename);
+           targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
            continue;
         }
 
@@ -1348,6 +1359,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)                  /*{{{*/
       }
 
       // Queue the Index file (Packages, Sources, Translation-$foo, …)
+      targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
       if (trypdiff)
          new pkgAcqDiffIndex(Owner, TransactionManager, Target);
       else

diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index 260686cc78b501387a857acb6eb95c2dd67edc1d..ea32e62357213525a89a0662f11c8f5be9b15ba7 100644 (file)
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -33,13 +33,13 @@ class APT_HIDDEN debReleaseIndexPrivate                                     /*{{{*/
    public:
    struct APT_HIDDEN debSectionEntry
    {
-      std::string sourcesEntry;
-      std::string Name;
-      std::vector<std::string> Targets;
-      std::vector<std::string> Architectures;
-      std::vector<std::string> Languages;
-      bool UsePDiffs;
-      std::string UseByHash;
+      std::string const sourcesEntry;
+      std::string const Name;
+      std::vector<std::string> const Targets;
+      std::vector<std::string> const Architectures;
+      std::vector<std::string> const Languages;
+      bool const UsePDiffs;
+      std::string const UseByHash;
    };
 
    std::vector<debSectionEntry> DebEntries;
@@ -172,6 +172,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
         std::string const UseByHash = APT_T_CONFIG_STR("By-Hash", E->UseByHash);
         std::string const CompressionTypes = APT_T_CONFIG_STR("CompressionTypes", DefCompressionTypes);
         std::string KeepCompressedAs = APT_T_CONFIG_STR("KeepCompressedAs", "");
+        std::string const FallbackOf = APT_T_CONFIG_STR("Fallback-Of", "");
 #undef APT_T_CONFIG_BOOL
 #undef APT_T_CONFIG_STR
         if (tplMetaKey.empty())
@@ -292,6 +293,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
                  Options.insert(std::make_pair("REPO_URI", URI));
                  Options.insert(std::make_pair("TARGET_OF", Type));
                  Options.insert(std::make_pair("CREATED_BY", *T));
+                 Options.insert(std::make_pair("FALLBACK_OF", FallbackOf));
                  Options.insert(std::make_pair("PDIFFS", UsePDiffs ? "yes" : "no"));
                  Options.insert(std::make_pair("BY_HASH", UseByHash));
                  Options.insert(std::make_pair("DEFAULTENABLED", DefaultEnabled ? "yes" : "no"));
@@ -616,7 +618,8 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll)/*{{{*/
    // special case for --print-uris
    if (GetAll)
       for (auto const &Target: GetIndexTargets())
-        new pkgAcqIndex(Owner, TransactionManager, Target);
+        if (Target.Option(IndexTarget::FALLBACK_OF).empty())
+           new pkgAcqIndex(Owner, TransactionManager, Target);
 
    return true;
 }
@@ -949,6 +952,26 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type              /*{{{*/
         else if (optValue == false && tarItr != mytargets.end())
            mytargets.erase(std::remove(mytargets.begin(), mytargets.end(), target), mytargets.end());
       }
+      // if we can't order it in a 1000 steps we give up… probably a cycle
+      for (auto i = 0; i < 1000; ++i)
+      {
+        bool Changed = false;
+        for (auto t = mytargets.begin(); t != mytargets.end(); ++t)
+        {
+           std::string const fallback = _config->Find(std::string("Acquire::IndexTargets::") + Name + "::" + *t + "::Fallback-Of");
+           if (fallback.empty())
+              continue;
+           auto const faller = std::find(mytargets.begin(), mytargets.end(), fallback);
+           if (faller == mytargets.end() || faller < t)
+              continue;
+           Changed = true;
+           auto const tv = *t;
+           mytargets.erase(t);
+           mytargets.emplace_back(tv);
+        }
+        if (Changed == false)
+           break;
+      }
 
       bool UsePDiffs = _config->FindB("Acquire::PDiffs", true);
       {

diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc
index a777c38ac15e902d2c1e7f7834d0370f8dcfe1d2..148ed5d98e115d83e0979f29be09dc60c691bba1 100644 (file)
--- a/apt-pkg/indexfile.cc
+++ b/apt-pkg/indexfile.cc
@@ -139,6 +139,7 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const             /*{{{*/
       APT_CASE(REPO_URI);
       APT_CASE(TARGET_OF);
       APT_CASE(CREATED_BY);
+      APT_CASE(FALLBACK_OF);
       APT_CASE(PDIFFS);
       APT_CASE(DEFAULTENABLED);
       APT_CASE(COMPRESSIONTYPES);

diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h
index 5c666671c6fc796583f3e47b200765a2ef5def9e..eaaded87a7e5741873ba39564b3cf4dec3f18c3b 100644 (file)
--- a/apt-pkg/indexfile.h
+++ b/apt-pkg/indexfile.h
@@ -91,6 +91,7 @@ class IndexTarget                                                     /*{{{*/
       SOURCESENTRY,
       BY_HASH,
       KEEPCOMPRESSEDAS,
+      FALLBACK_OF,
    };
    std::string Option(OptionKeys const Key) const;
    bool OptionBool(OptionKeys const Key) const;

diff --git a/doc/acquire-additional-files.txt b/doc/acquire-additional-files.txt
index 68af9a5b03c2dad6f376453b37ac61e296ec49cd..19c3deb136716f70477fcc0404d3d7df068c76d7 100644 (file)
--- a/doc/acquire-additional-files.txt
+++ b/doc/acquire-additional-files.txt
@@ -90,6 +90,11 @@ Additional optional properties:
   have to ensure your front-end can deal with all compressed fileformats
   supported by apt (libapt users can e.g. use FileFd, others can use
   the cat-file command of /usr/lib/apt/apt-helper).
+* Fallback-Of: Is by default not set. If it is set and specifies another
+  target name (see Created-By) which was found in the Release file the
+  download of this target will be skipped. This can be used to implement
+  fallback(chain)s to allow transitions like the rename of target files.
+  The behavior if cycles are formed with Fallback-Of is undefined!
 * flat{MetaKey,Description}: APT supports two types of repositories:
   dists-style repositories which are the default and by far the most
   common which are named after the fact that the files are in an
@@ -204,11 +209,11 @@ uppercase and wrapped in $(), as in the configuration file.
 To get all the filenames of all Translation-en files you can e.g. call:
        apt-get indextargets --format '$(FILENAME)' "Created-By: Translations" "Language: en"
 
-The line-based filtering and the formating is rather crude and feature-
+The line-based filtering and the formatting is rather crude and feature-
 less by design: The default format is Debians standard format deb822 (in
 particular: Field names are case-insensitive and the order of fields in
 the stanza is undefined), so instead of apt reimplementing powerful
-filters and formating for this command, it is recommend to use piping
+filters and formatting for this command, it is recommend to use piping
 and dedicated tools like 'grep-dctrl' if you need more than the basics
 provided.
 

diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files
index 3420aa29d5a50a0eaff2cc32cdfcffb54f64e9a4..0b81845afb9fcf6ef9be198a7e96f03478518587 100755 (executable)
--- a/test/integration/test-apt-acquire-additional-files
+++ b/test/integration/test-apt-acquire-additional-files
@@ -158,6 +158,64 @@ rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Cont
 testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
 runthistest
 
+msgmsg "Contents with 3 MetaKeys, first match"
+rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz
+rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz
+testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
+cat > rootdir/etc/apt/apt.conf.d/content-target.conf <<EOF
+Acquire::IndexTargets::deb::Contents3 {
+       MetaKey "main/Contents-all";
+       ShortDescription "Contents3";
+       Description "\$(RELEASE) all Contents3";
+       Fallback-Of "Contents2";
+};
+Acquire::IndexTargets::deb::Contents {
+       MetaKey "\$(COMPONENT)/Contents-amd64";
+       ShortDescription "Contents";
+       Description "\$(RELEASE)/\$(COMPONENT) amd64 Contents";
+};
+Acquire::IndexTargets::deb::Contents2 {
+       MetaKey "Contents-all";
+       ShortDescription "Contents2";
+       Description "\$(RELEASE) all Contents2";
+       Fallback-Of "Contents";
+};
+EOF
+testequal "'http://localhost:${APTHTTPPORT}/dists/unstable/InRelease' localhost:${APTHTTPPORT}_dists_unstable_InRelease 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/source/Sources.xz' localhost:${APTHTTPPORT}_dists_unstable_main_source_Sources 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-amd64/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-amd64_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-all/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-all_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/i18n/Translation-en.xz' localhost:${APTHTTPPORT}_dists_unstable_main_i18n_Translation-en 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/Contents-amd64.xz' localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64 0 " aptget update --print-uris
+testsuccessequal "Hit:1 http://localhost:${APTHTTPPORT} unstable InRelease
+Get:2 http://localhost:${APTHTTPPORT} unstable/main amd64 Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
+Reading package lists..." aptget update
+testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64" find rootdir/var/lib/apt/lists -name '*Contents*'
+testequal "$(readfile Contents-amd64)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
+testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents2'
+testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents3'
+testsuccess cmp "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64" 'aptarchive/dists/unstable/main/Contents-amd64'
+
+msgmsg "Contents with 3 MetaKeys, third match"
+rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64
+testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
+echo 'Acquire::IndexTargets::deb::Contents::MetaKey "$(COMPONENT)/Contents-i386";' >> rootdir/etc/apt/apt.conf.d/content-target.conf
+testequal "'http://localhost:${APTHTTPPORT}/dists/unstable/InRelease' localhost:${APTHTTPPORT}_dists_unstable_InRelease 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/source/Sources.xz' localhost:${APTHTTPPORT}_dists_unstable_main_source_Sources 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-amd64/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-amd64_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-all/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-all_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/i18n/Translation-en.xz' localhost:${APTHTTPPORT}_dists_unstable_main_i18n_Translation-en 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/Contents-i386.xz' localhost:${APTHTTPPORT}_dists_unstable_main_Contents-i386 0 " aptget update --print-uris
+testsuccessequal "Hit:1 http://localhost:${APTHTTPPORT} unstable InRelease
+Get:2 http://localhost:${APTHTTPPORT} unstable all Contents3 [$(stat -c%s aptarchive/dists/unstable/main/Contents-all.gz) B]
+Reading package lists..." aptget update
+testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all" find rootdir/var/lib/apt/lists -name '*Contents*'
+testequal "$(readfile Contents-all)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents3'
+testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
+testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents2'
+testsuccess cmp "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all" 'aptarchive/dists/unstable/main/Contents-all'
+
+rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all
 rm -f rootdir/etc/apt/apt.conf.d/content-target.conf
 msgmsg "No Contents file"