From 7f2d1eef183dbebaaabe07a296d9a97e9cfd0f4a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 8 May 2016 11:58:36 +0200 Subject: [PATCH] implement Fallback-Of for IndexTargets Sometimes index files are in different locations in a repository as it is currently the case for Contents files which are per-component in Debian, but aren't in Ubuntu. This has historic reasons and is perhaps changed soon, but such cases of transitions can always happen in the future again, so we should prepare: Introduced is a new field declaring that the current item should only be downloaded if the mentioned item wasn't allowing for transitions without a flagday in clients and archives. This isn't implemented 'simpler' with multiple MetaKeys as items (could) change their descriptions and perhaps also other configuration bits with their location. --- apt-pkg/acquire-item.cc | 14 ++++- apt-pkg/deb/debmetaindex.cc | 39 ++++++++++--- apt-pkg/indexfile.cc | 1 + apt-pkg/indexfile.h | 1 + doc/acquire-additional-files.txt | 9 ++- .../test-apt-acquire-additional-files | 58 +++++++++++++++++++ 6 files changed, 111 insertions(+), 11 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 959627a01..8c45acddd 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1196,9 +1196,18 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify) /*{{{*/ // at this point the real Items are loaded in the fetcher ExpectedAdditionalItems = 0; + std::set targetsSeen; bool const metaBaseSupportsByHash = TransactionManager->MetaIndexParser->GetSupportsAcquireByHash(); - for (auto Target : TransactionManager->MetaIndexParser->GetIndexTargets()) + for (auto &Target: TransactionManager->MetaIndexParser->GetIndexTargets()) { + // if we have seen a target which is created-by a target this one here is declared a + // fallback to, we skip acquiring the fallback (but we make sure we clean up) + if (targetsSeen.find(Target.Option(IndexTarget::FALLBACK_OF)) != targetsSeen.end()) + { + targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY)); + new CleanupItem(Owner, TransactionManager, Target); + continue; + } // all is an implementation detail. Users shouldn't use this as arch // We need this support trickery here as e.g. Debian has binary-all files already, // but arch:all packages are still in the arch:any files, so we would waste precious @@ -1267,6 +1276,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify) /*{{{*/ else if (hashes.FileSize() == 0) { new CleanupItem(Owner, TransactionManager, Target); + targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY)); continue; } } @@ -1327,6 +1337,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify) /*{{{*/ std::string const idxfilename = GetFinalFileNameFromURI(GetDiffIndexURI(Target)); if (FileExists(idxfilename)) new NoActionItem(Owner, Target, idxfilename); + targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY)); continue; } @@ -1348,6 +1359,7 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify) /*{{{*/ } // Queue the Index file (Packages, Sources, Translation-$foo, …) + targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY)); if (trypdiff) new pkgAcqDiffIndex(Owner, TransactionManager, Target); else diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 260686cc7..ea32e6235 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -33,13 +33,13 @@ class APT_HIDDEN debReleaseIndexPrivate /*{{{*/ public: struct APT_HIDDEN debSectionEntry { - std::string sourcesEntry; - std::string Name; - std::vector Targets; - std::vector Architectures; - std::vector Languages; - bool UsePDiffs; - std::string UseByHash; + std::string const sourcesEntry; + std::string const Name; + std::vector const Targets; + std::vector const Architectures; + std::vector const Languages; + bool const UsePDiffs; + std::string const UseByHash; }; std::vector DebEntries; @@ -172,6 +172,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI, std::string const UseByHash = APT_T_CONFIG_STR("By-Hash", E->UseByHash); std::string const CompressionTypes = APT_T_CONFIG_STR("CompressionTypes", DefCompressionTypes); std::string KeepCompressedAs = APT_T_CONFIG_STR("KeepCompressedAs", ""); + std::string const FallbackOf = APT_T_CONFIG_STR("Fallback-Of", ""); #undef APT_T_CONFIG_BOOL #undef APT_T_CONFIG_STR if (tplMetaKey.empty()) @@ -292,6 +293,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI, Options.insert(std::make_pair("REPO_URI", URI)); Options.insert(std::make_pair("TARGET_OF", Type)); Options.insert(std::make_pair("CREATED_BY", *T)); + Options.insert(std::make_pair("FALLBACK_OF", FallbackOf)); Options.insert(std::make_pair("PDIFFS", UsePDiffs ? "yes" : "no")); Options.insert(std::make_pair("BY_HASH", UseByHash)); Options.insert(std::make_pair("DEFAULTENABLED", DefaultEnabled ? "yes" : "no")); @@ -616,7 +618,8 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll)/*{{{*/ // special case for --print-uris if (GetAll) for (auto const &Target: GetIndexTargets()) - new pkgAcqIndex(Owner, TransactionManager, Target); + if (Target.Option(IndexTarget::FALLBACK_OF).empty()) + new pkgAcqIndex(Owner, TransactionManager, Target); return true; } @@ -949,6 +952,26 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ else if (optValue == false && tarItr != mytargets.end()) mytargets.erase(std::remove(mytargets.begin(), mytargets.end(), target), mytargets.end()); } + // if we can't order it in a 1000 steps we give up… probably a cycle + for (auto i = 0; i < 1000; ++i) + { + bool Changed = false; + for (auto t = mytargets.begin(); t != mytargets.end(); ++t) + { + std::string const fallback = _config->Find(std::string("Acquire::IndexTargets::") + Name + "::" + *t + "::Fallback-Of"); + if (fallback.empty()) + continue; + auto const faller = std::find(mytargets.begin(), mytargets.end(), fallback); + if (faller == mytargets.end() || faller < t) + continue; + Changed = true; + auto const tv = *t; + mytargets.erase(t); + mytargets.emplace_back(tv); + } + if (Changed == false) + break; + } bool UsePDiffs = _config->FindB("Acquire::PDiffs", true); { diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc index a777c38ac..148ed5d98 100644 --- a/apt-pkg/indexfile.cc +++ b/apt-pkg/indexfile.cc @@ -139,6 +139,7 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const /*{{{*/ APT_CASE(REPO_URI); APT_CASE(TARGET_OF); APT_CASE(CREATED_BY); + APT_CASE(FALLBACK_OF); APT_CASE(PDIFFS); APT_CASE(DEFAULTENABLED); APT_CASE(COMPRESSIONTYPES); diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h index 5c666671c..eaaded87a 100644 --- a/apt-pkg/indexfile.h +++ b/apt-pkg/indexfile.h @@ -91,6 +91,7 @@ class IndexTarget /*{{{*/ SOURCESENTRY, BY_HASH, KEEPCOMPRESSEDAS, + FALLBACK_OF, }; std::string Option(OptionKeys const Key) const; bool OptionBool(OptionKeys const Key) const; diff --git a/doc/acquire-additional-files.txt b/doc/acquire-additional-files.txt index 68af9a5b0..19c3deb13 100644 --- a/doc/acquire-additional-files.txt +++ b/doc/acquire-additional-files.txt @@ -90,6 +90,11 @@ Additional optional properties: have to ensure your front-end can deal with all compressed fileformats supported by apt (libapt users can e.g. use FileFd, others can use the cat-file command of /usr/lib/apt/apt-helper). +* Fallback-Of: Is by default not set. If it is set and specifies another + target name (see Created-By) which was found in the Release file the + download of this target will be skipped. This can be used to implement + fallback(chain)s to allow transitions like the rename of target files. + The behavior if cycles are formed with Fallback-Of is undefined! * flat{MetaKey,Description}: APT supports two types of repositories: dists-style repositories which are the default and by far the most common which are named after the fact that the files are in an @@ -204,11 +209,11 @@ uppercase and wrapped in $(), as in the configuration file. To get all the filenames of all Translation-en files you can e.g. call: apt-get indextargets --format '$(FILENAME)' "Created-By: Translations" "Language: en" -The line-based filtering and the formating is rather crude and feature- +The line-based filtering and the formatting is rather crude and feature- less by design: The default format is Debians standard format deb822 (in particular: Field names are case-insensitive and the order of fields in the stanza is undefined), so instead of apt reimplementing powerful -filters and formating for this command, it is recommend to use piping +filters and formatting for this command, it is recommend to use piping and dedicated tools like 'grep-dctrl' if you need more than the basics provided. diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files index 3420aa29d..0b81845af 100755 --- a/test/integration/test-apt-acquire-additional-files +++ b/test/integration/test-apt-acquire-additional-files @@ -158,6 +158,64 @@ rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Cont testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents' runthistest +msgmsg "Contents with 3 MetaKeys, first match" +rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz +rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz +testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents' +cat > rootdir/etc/apt/apt.conf.d/content-target.conf <> rootdir/etc/apt/apt.conf.d/content-target.conf +testequal "'http://localhost:${APTHTTPPORT}/dists/unstable/InRelease' localhost:${APTHTTPPORT}_dists_unstable_InRelease 0 +'http://localhost:${APTHTTPPORT}/dists/unstable/main/source/Sources.xz' localhost:${APTHTTPPORT}_dists_unstable_main_source_Sources 0 +'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-amd64/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-amd64_Packages 0 +'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-all/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-all_Packages 0 +'http://localhost:${APTHTTPPORT}/dists/unstable/main/i18n/Translation-en.xz' localhost:${APTHTTPPORT}_dists_unstable_main_i18n_Translation-en 0 +'http://localhost:${APTHTTPPORT}/dists/unstable/main/Contents-i386.xz' localhost:${APTHTTPPORT}_dists_unstable_main_Contents-i386 0 " aptget update --print-uris +testsuccessequal "Hit:1 http://localhost:${APTHTTPPORT} unstable InRelease +Get:2 http://localhost:${APTHTTPPORT} unstable all Contents3 [$(stat -c%s aptarchive/dists/unstable/main/Contents-all.gz) B] +Reading package lists..." aptget update +testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all" find rootdir/var/lib/apt/lists -name '*Contents*' +testequal "$(readfile Contents-all)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents3' +testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents' +testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents2' +testsuccess cmp "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all" 'aptarchive/dists/unstable/main/Contents-all' + +rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all rm -f rootdir/etc/apt/apt.conf.d/content-target.conf msgmsg "No Contents file" -- 2.45.2