From 24e8f24e1e94ec3816b0bfc7a05d1c4e3f73248e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 14 Sep 2015 13:18:29 +0200 Subject: [PATCH] add by-hash sources.list option and document all of by-hash This changes the semantics of the option (which is renamed too) to be a yes/no value with the special additional value "force" as this allows by-hash to be disabled even if the repository indicates it would be supported and is more in line with our other yes/no options like pdiff which disable themselves if no support can be detected. The feature wasn't documented so far and hasn't reached a (un)stable release yet, so changing it without trying too hard to keep compatibility seems okay. --- apt-pkg/acquire-item.cc | 13 +++---- apt-pkg/deb/debmetaindex.cc | 21 ++++++++-- apt-pkg/deb/debmetaindex.h | 2 +- apt-pkg/indexfile.cc | 1 + apt-pkg/indexfile.h | 1 + apt-pkg/sourcelist.cc | 1 + doc/acquire-additional-files.txt | 3 ++ doc/apt.conf.5.xml | 10 +++++ doc/sources.list.5.xml | 16 ++++++++ test/integration/test-apt-by-hash-update | 49 ++++++++++++++++-------- 10 files changed, 90 insertions(+), 27 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index c5b701fdc..1b76f1b7a 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1247,7 +1247,7 @@ string pkgAcqMetaClearSig::Custom600Headers() const return Header; } /*}}}*/ -bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message, +bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message, /*{{{*/ pkgAcquire::MethodConfig const * const Cnf) { Item::VerifyDone(Message, Cnf); @@ -1257,6 +1257,7 @@ bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message, return true; } + /*}}}*/ // pkgAcqMetaClearSig::Done - We got a file /*{{{*/ void pkgAcqMetaClearSig::Done(std::string const &Message, HashStringList const &Hashes, @@ -2480,13 +2481,9 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, // AcqIndex::AdjustForByHash - modify URI for by-hash support /*{{{*/ void pkgAcqIndex::InitByHashIfNeeded() { - // TODO: - // - (maybe?) add support for by-hash into the sources.list as flag - // - make apt-ftparchive generate the hashes (and expire?) - std::string HostKnob = "APT::Acquire::" + ::URI(Desc.URI).Host + "::By-Hash"; - if(_config->FindB("APT::Acquire::By-Hash", false) == true || - _config->FindB(HostKnob, false) == true || - TransactionManager->MetaIndexParser->GetSupportsAcquireByHash()) + std::string const useByHash = Target.Option(IndexTarget::BY_HASH); + if(useByHash == "force" || (StringToBool(useByHash) == true && + TransactionManager->MetaIndexParser->GetSupportsAcquireByHash())) { HashStringList const Hashes = GetExpectedHashes(); if(Hashes.usable()) diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 6ed722e68..a53d32d34 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -39,6 +39,7 @@ class APT_HIDDEN debReleaseIndexPrivate /*{{{*/ std::vector Architectures; std::vector Languages; bool UsePDiffs; + std::string UseByHash; }; std::vector DebEntries; @@ -149,6 +150,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI, bool const KeepCompressed = APT_T_CONFIG_BOOL("KeepCompressed", GzipIndex); bool const DefaultEnabled = APT_T_CONFIG_BOOL("DefaultEnabled", true); bool const UsePDiffs = APT_T_CONFIG_BOOL("PDiffs", E->UsePDiffs); + std::string const UseByHash = APT_T_CONFIG_STR("By-Hash", E->UseByHash); std::string const CompressionTypes = APT_T_CONFIG_STR("CompressionTypes", DefCompressionTypes); #undef APT_T_CONFIG_BOOL #undef APT_T_CONFIG_STR @@ -245,6 +247,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI, Options.insert(std::make_pair("TARGET_OF", Type)); Options.insert(std::make_pair("CREATED_BY", *T)); Options.insert(std::make_pair("PDIFFS", UsePDiffs ? "yes" : "no")); + Options.insert(std::make_pair("BY_HASH", UseByHash)); Options.insert(std::make_pair("DEFAULTENABLED", DefaultEnabled ? "yes" : "no")); Options.insert(std::make_pair("COMPRESSIONTYPES", CompressionTypes)); Options.insert(std::make_pair("SOURCESENTRY", E->sourcesEntry)); @@ -286,12 +289,12 @@ void debReleaseIndex::AddComponent(std::string const &sourcesEntry, /*{{{*/ std::vector const &Targets, std::vector const &Architectures, std::vector Languages, - bool const usePDiffs) + bool const usePDiffs, std::string const &useByHash) { if (Languages.empty() == true) Languages.push_back("none"); debReleaseIndexPrivate::debSectionEntry const entry = { - sourcesEntry, Name, Targets, Architectures, Languages, usePDiffs + sourcesEntry, Name, Targets, Architectures, Languages, usePDiffs, useByHash }; if (isSrc) d->DebSrcEntries.push_back(entry); @@ -822,6 +825,17 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ UsePDiffs = StringToBool(opt->second); } + std::string UseByHash = _config->Find("APT::Acquire::By-Hash", "yes"); + UseByHash = _config->Find("Acquire::By-Hash", UseByHash); + { + std::string const host = ::URI(URI).Host; + UseByHash = _config->Find("APT::Acquire::" + host + "::By-Hash", UseByHash); + UseByHash = _config->Find("Acquire::" + host + "::By-Hash", UseByHash); + std::map::const_iterator const opt = Options.find("by-hash"); + if (opt != Options.end()) + UseByHash = opt->second; + } + auto const entry = Options.find("sourceslist-entry"); Deb->AddComponent( entry->second, @@ -830,7 +844,8 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ mytargets, parsePlusMinusOptions("arch", Options, APT::Configuration::getArchitectures()), parsePlusMinusOptions("lang", Options, APT::Configuration::getLanguages(true)), - UsePDiffs + UsePDiffs, + UseByHash ); if (Deb->SetTrusted(GetTriStateOption(Options, "trusted")) == false || diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h index 419cbdc9d..37515f934 100644 --- a/apt-pkg/deb/debmetaindex.h +++ b/apt-pkg/deb/debmetaindex.h @@ -64,7 +64,7 @@ class APT_HIDDEN debReleaseIndex : public metaIndex std::vector const &Targets, std::vector const &Architectures, std::vector Languages, - bool const usePDiffs); + bool const usePDiffs, std::string const &useByHash); }; #endif diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc index fad339197..f57b442a3 100644 --- a/apt-pkg/indexfile.cc +++ b/apt-pkg/indexfile.cc @@ -148,6 +148,7 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const /*{{{*/ APT_CASE(DEFAULTENABLED); APT_CASE(COMPRESSIONTYPES); APT_CASE(SOURCESENTRY); + APT_CASE(BY_HASH); #undef APT_CASE case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI); case EXISTING_FILENAME: diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h index a09c39057..c3f01c774 100644 --- a/apt-pkg/indexfile.h +++ b/apt-pkg/indexfile.h @@ -89,6 +89,7 @@ class IndexTarget /*{{{*/ COMPRESSIONTYPES, DEFAULTENABLED, SOURCESENTRY, + BY_HASH, }; std::string Option(OptionKeys const Key) const; bool OptionBool(OptionKeys const Key) const; diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 4e5ff0578..2100b5d3c 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -108,6 +108,7 @@ bool pkgSourceList::Type::ParseStanza(vector &List, /*{{{*/ mapping.insert(std::make_pair("Valid-Until-Max", std::make_pair("valid-until-max", false))); mapping.insert(std::make_pair("Signed-By", std::make_pair("signed-by", false))); mapping.insert(std::make_pair("PDiffs", std::make_pair("pdiffs", false))); + mapping.insert(std::make_pair("By-Hash", std::make_pair("by-hash", false))); for (std::map >::const_iterator m = mapping.begin(); m != mapping.end(); ++m) if (Tags.Exists(m->first)) diff --git a/doc/acquire-additional-files.txt b/doc/acquire-additional-files.txt index c52955619..a7acbbe46 100644 --- a/doc/acquire-additional-files.txt +++ b/doc/acquire-additional-files.txt @@ -114,6 +114,9 @@ aren't accidentally used by frontends: Defaults to the value of Acquire::PDiffs which is true by default. Can be overridden per-source by the sources.list option of the same name. See the documentation for both of these for details. +* By-Hash: controls if apt will try to use an URI constructed from + a hashsum of the file to download. See the documentation for config + option Acquire::By-Hash and sources.list option By-Hash for details. * CompressionTypes: The default value is a space separated list of compression types supported by apt (see Acquire::CompressionTypes). You can set this option to prevent apt from downloading a compression diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml index 62dffadc4..bb0c37ff8 100644 --- a/doc/apt.conf.5.xml +++ b/doc/apt.conf.5.xml @@ -347,6 +347,16 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";}; + + Try to download indexes via an URI constructed from a + hashsum of the expected file rather than downloaded via a well-known + stable filename. True by default, but automatically disabled if the + source indicates no support for it. Usage can be forced with the special + value "force". Preferably, this can be set for specific &sources-list; entries + or index files by using the option there. + + + Queuing mode; Queue-Mode can be one of host or access which determines how APT parallelizes outgoing diff --git a/doc/sources.list.5.xml b/doc/sources.list.5.xml index 276e7d46f..71447e84f 100644 --- a/doc/sources.list.5.xml +++ b/doc/sources.list.5.xml @@ -239,6 +239,22 @@ deb-src [ option1=value1 option2=value2 ] uri suite [component1] [component2] [. yes. + () + can have the value "yes", "no" or "force" and controls if APT + should try to acquire indexes via an URI constructed from a + hashsum of the expected file instead of using the well-known + stable filename of the index. Using this can avoid hashsum + mismatches, but requires a supporting mirror. The value + "yes"/"no" activates/disables the use of this feature if this + source indicates support for it, while "force" will enable the + feature regardless of what the source indicates. + Defaults to the value of the option of the same name for a + specific index file defined in the + scope, which itself + defaults to the value of configuration option + which defaults to + yes. + diff --git a/test/integration/test-apt-by-hash-update b/test/integration/test-apt-by-hash-update index 293836861..c00ab497b 100755 --- a/test/integration/test-apt-by-hash-update +++ b/test/integration/test-apt-by-hash-update @@ -5,8 +5,9 @@ TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework setupenvironment -configarchitecture "i386" +configarchitecture 'i386' confighashes 'SHA512' +configcompression '.' 'gz' insertpackage 'unstable' 'foo' 'all' '1.0' @@ -24,29 +25,47 @@ mkdir -p aptarchive/dists/unstable/main/source/by-hash/SHA512 ln -s ../../Sources.gz $(sha512sum ../../Sources.gz|cut -f1 -d' ') ) -# we moved the Packages file away, normal update won't work -testfailure aptget update - -# ensure we do not know about "foo" -testfailureequal "Reading package lists... +ensureitsbroken() { + rm -rf rootdir/var/lib/apt/lists + # we moved the Packages file away, normal update won't work + testfailure aptget update "$@" + # ensure we do not know about "foo" + testfailureequal "Reading package lists... Building dependency tree... E: Unable to locate package foo" aptget install -q -s foo - -# ensure we can apt-get update by hash -testsuccess aptget update -o APT::Acquire::By-Hash=1 -o Acquire::Languages=none +} +ensureitsbroken +ensureitsbroken -o Acquire::By-Hash=1 ensureitworks() { + rm -rf rootdir/var/lib/apt/lists + testsuccess aptget update -o Acquire::Languages=none "$@" testsuccessequal "Inst foo (1.0 unstable [all]) Conf foo (1.0 unstable [all])" aptget install -qq -s foo } -ensureitworks +msgmsg 'Test by-hash via' 'config option' +ensureitworks -o Acquire::By-Hash=force +msgmsg 'Test by-hash via' 'release option' +cp -a aptarchive/dists aptarchive/dists.bak # add magic string to Release file ... -MAGIC="Acquire-By-Hash: true" -sed -i "s#Suite: unstable#Suite: unstable\n$MAGIC#" aptarchive/dists/unstable/Release +sed -i '/^Suite: / a \ +Acquire-By-Hash: yes' aptarchive/dists/unstable/Release signreleasefiles -# ... and verify that it fetches by hash now -rm -rf rootdir/var/lib/apt/lists -testsuccess aptget update -o Acquire::Languages=none +ensureitworks +ensureitsbroken -o Acquire::By-Hash=0 + +msgmsg 'Test by-hash via' 'sources option' +sed -i "s#^\(deb\(-src\)\?\) #\1 [by-hash=yes] #" rootdir/etc/apt/sources.list.d/* +ensureitworks +#ensureitsbroken -o Acquire::By-Hash=0 + +rm -rf aptarchive/dists +cp -a aptarchive/dists.bak aptarchive/dists +#ensureitworks -o Acquire::By-Hash=force +ensureitsbroken -o Acquire::By-Hash=1 +ensureitsbroken -o Acquire::By-Hash=0 +sed -i "s#^\(deb\(-src\)\?\) \[by-hash=yes\] #\1 [by-hash=force] #" rootdir/etc/apt/sources.list.d/* ensureitworks +#ensureitsbroken -o Acquire::By-Hash=0 -- 2.45.2