add by-hash sources.list option and document all of by-hash

This changes the semantics of the option (which is renamed too) to be a
yes/no value with the special additional value "force" as this allows
by-hash to be disabled even if the repository indicates it would be
supported and is more in line with our other yes/no options like pdiff
which disable themselves if no support can be detected.

The feature wasn't documented so far and hasn't reached a (un)stable
release yet, so changing it without trying too hard to keep
compatibility seems okay.

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index c5b701fdcb30613ab539176bb035301621db031b..1b76f1b7acb2de624fca6ae8b3dc409f0058091e 100644 (file)
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -1247,7 +1247,7 @@ string pkgAcqMetaClearSig::Custom600Headers() const
    return Header;
 }
                                                                        /*}}}*/
    return Header;
 }
                                                                        /*}}}*/

-bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message,
+bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message,                /*{{{*/

         pkgAcquire::MethodConfig const * const Cnf)
 {
    Item::VerifyDone(Message, Cnf);
         pkgAcquire::MethodConfig const * const Cnf)
 {
    Item::VerifyDone(Message, Cnf);


@@ -1257,6 +1257,7 @@ bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message,
 
    return true;
 }
 
    return true;
 }

+                                                                       /*}}}*/

 // pkgAcqMetaClearSig::Done - We got a file                            /*{{{*/
 void pkgAcqMetaClearSig::Done(std::string const &Message,
                               HashStringList const &Hashes,
 // pkgAcqMetaClearSig::Done - We got a file                            /*{{{*/
 void pkgAcqMetaClearSig::Done(std::string const &Message,
                               HashStringList const &Hashes,


@@ -2480,13 +2481,9 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
 // AcqIndex::AdjustForByHash - modify URI for by-hash support          /*{{{*/
 void pkgAcqIndex::InitByHashIfNeeded()
 {
 // AcqIndex::AdjustForByHash - modify URI for by-hash support          /*{{{*/
 void pkgAcqIndex::InitByHashIfNeeded()
 {

-   // TODO:
-   //  - (maybe?) add support for by-hash into the sources.list as flag
-   //  - make apt-ftparchive generate the hashes (and expire?)
-   std::string HostKnob = "APT::Acquire::" + ::URI(Desc.URI).Host + "::By-Hash";
-   if(_config->FindB("APT::Acquire::By-Hash", false) == true ||
-      _config->FindB(HostKnob, false) == true ||
-      TransactionManager->MetaIndexParser->GetSupportsAcquireByHash())
+   std::string const useByHash = Target.Option(IndexTarget::BY_HASH);
+   if(useByHash == "force" || (StringToBool(useByHash) == true &&
+           TransactionManager->MetaIndexParser->GetSupportsAcquireByHash()))

    {
       HashStringList const Hashes = GetExpectedHashes();
       if(Hashes.usable())
    {
       HashStringList const Hashes = GetExpectedHashes();
       if(Hashes.usable())

diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index 6ed722e68ed211c9fc3df8e260e0c40e46af15c5..a53d32d346c1c2ef9beca6f1cffc3629dbb9465d 100644 (file)
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -39,6 +39,7 @@ class APT_HIDDEN debReleaseIndexPrivate                                       /*{{{*/
       std::vector<std::string> Architectures;
       std::vector<std::string> Languages;
       bool UsePDiffs;
       std::vector<std::string> Architectures;
       std::vector<std::string> Languages;
       bool UsePDiffs;

+      std::string UseByHash;

    };
 
    std::vector<debSectionEntry> DebEntries;
    };
 
    std::vector<debSectionEntry> DebEntries;


@@ -149,6 +150,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
         bool const KeepCompressed = APT_T_CONFIG_BOOL("KeepCompressed", GzipIndex);
         bool const DefaultEnabled = APT_T_CONFIG_BOOL("DefaultEnabled", true);
         bool const UsePDiffs = APT_T_CONFIG_BOOL("PDiffs", E->UsePDiffs);
         bool const KeepCompressed = APT_T_CONFIG_BOOL("KeepCompressed", GzipIndex);
         bool const DefaultEnabled = APT_T_CONFIG_BOOL("DefaultEnabled", true);
         bool const UsePDiffs = APT_T_CONFIG_BOOL("PDiffs", E->UsePDiffs);

+        std::string const UseByHash = APT_T_CONFIG_STR("By-Hash", E->UseByHash);

         std::string const CompressionTypes = APT_T_CONFIG_STR("CompressionTypes", DefCompressionTypes);
 #undef APT_T_CONFIG_BOOL
 #undef APT_T_CONFIG_STR
         std::string const CompressionTypes = APT_T_CONFIG_STR("CompressionTypes", DefCompressionTypes);
 #undef APT_T_CONFIG_BOOL
 #undef APT_T_CONFIG_STR


@@ -245,6 +247,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
               Options.insert(std::make_pair("TARGET_OF", Type));
               Options.insert(std::make_pair("CREATED_BY", *T));
               Options.insert(std::make_pair("PDIFFS", UsePDiffs ? "yes" : "no"));
               Options.insert(std::make_pair("TARGET_OF", Type));
               Options.insert(std::make_pair("CREATED_BY", *T));
               Options.insert(std::make_pair("PDIFFS", UsePDiffs ? "yes" : "no"));

+              Options.insert(std::make_pair("BY_HASH", UseByHash));

               Options.insert(std::make_pair("DEFAULTENABLED", DefaultEnabled ? "yes" : "no"));
               Options.insert(std::make_pair("COMPRESSIONTYPES", CompressionTypes));
               Options.insert(std::make_pair("SOURCESENTRY", E->sourcesEntry));
               Options.insert(std::make_pair("DEFAULTENABLED", DefaultEnabled ? "yes" : "no"));
               Options.insert(std::make_pair("COMPRESSIONTYPES", CompressionTypes));
               Options.insert(std::make_pair("SOURCESENTRY", E->sourcesEntry));


@@ -286,12 +289,12 @@ void debReleaseIndex::AddComponent(std::string const &sourcesEntry,       /*{{{*/
         std::vector<std::string> const &Targets,
         std::vector<std::string> const &Architectures,
         std::vector<std::string> Languages,
         std::vector<std::string> const &Targets,
         std::vector<std::string> const &Architectures,
         std::vector<std::string> Languages,

-        bool const usePDiffs)
+        bool const usePDiffs, std::string const &useByHash)

 {
    if (Languages.empty() == true)
       Languages.push_back("none");
    debReleaseIndexPrivate::debSectionEntry const entry = {
 {
    if (Languages.empty() == true)
       Languages.push_back("none");
    debReleaseIndexPrivate::debSectionEntry const entry = {

-      sourcesEntry, Name, Targets, Architectures, Languages, usePDiffs
+      sourcesEntry, Name, Targets, Architectures, Languages, usePDiffs, useByHash

    };
    if (isSrc)
       d->DebSrcEntries.push_back(entry);
    };
    if (isSrc)
       d->DebSrcEntries.push_back(entry);


@@ -822,6 +825,17 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type              /*{{{*/
            UsePDiffs = StringToBool(opt->second);
       }
 
            UsePDiffs = StringToBool(opt->second);
       }
 

+      std::string UseByHash = _config->Find("APT::Acquire::By-Hash", "yes");
+      UseByHash = _config->Find("Acquire::By-Hash", UseByHash);
+      {
+        std::string const host = ::URI(URI).Host;
+        UseByHash = _config->Find("APT::Acquire::" + host + "::By-Hash", UseByHash);
+        UseByHash = _config->Find("Acquire::" + host + "::By-Hash", UseByHash);
+        std::map<std::string, std::string>::const_iterator const opt = Options.find("by-hash");
+        if (opt != Options.end())
+           UseByHash = opt->second;
+      }
+

       auto const entry = Options.find("sourceslist-entry");
       Deb->AddComponent(
            entry->second,
       auto const entry = Options.find("sourceslist-entry");
       Deb->AddComponent(
            entry->second,


@@ -830,7 +844,8 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type               /*{{{*/
            mytargets,
            parsePlusMinusOptions("arch", Options, APT::Configuration::getArchitectures()),
            parsePlusMinusOptions("lang", Options, APT::Configuration::getLanguages(true)),
            mytargets,
            parsePlusMinusOptions("arch", Options, APT::Configuration::getArchitectures()),
            parsePlusMinusOptions("lang", Options, APT::Configuration::getLanguages(true)),

-           UsePDiffs
+           UsePDiffs,
+           UseByHash

            );
 
       if (Deb->SetTrusted(GetTriStateOption(Options, "trusted")) == false ||
            );
 
       if (Deb->SetTrusted(GetTriStateOption(Options, "trusted")) == false ||

diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h
index 419cbdc9dc72e958a23c40ff64062f36a5f7c433..37515f934e3b2b328d032b1fde4b1828a98ea032 100644 (file)
--- a/apt-pkg/deb/debmetaindex.h
+++ b/apt-pkg/deb/debmetaindex.h
@@ -64,7 +64,7 @@ class APT_HIDDEN debReleaseIndex : public metaIndex
         std::vector<std::string> const &Targets,
         std::vector<std::string> const &Architectures,
         std::vector<std::string> Languages,
         std::vector<std::string> const &Targets,
         std::vector<std::string> const &Architectures,
         std::vector<std::string> Languages,

-        bool const usePDiffs);
+        bool const usePDiffs, std::string const &useByHash);

 };
 
 #endif
 };
 
 #endif

diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc
index fad3391973dc9c49474b11453e2059d409b125dd..f57b442a3205f2a5fc017901225a6fa0f6f2983a 100644 (file)
--- a/apt-pkg/indexfile.cc
+++ b/apt-pkg/indexfile.cc
@@ -148,6 +148,7 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const             /*{{{*/
       APT_CASE(DEFAULTENABLED);
       APT_CASE(COMPRESSIONTYPES);
       APT_CASE(SOURCESENTRY);
       APT_CASE(DEFAULTENABLED);
       APT_CASE(COMPRESSIONTYPES);
       APT_CASE(SOURCESENTRY);

+      APT_CASE(BY_HASH);

 #undef APT_CASE
       case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
       case EXISTING_FILENAME:
 #undef APT_CASE
       case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
       case EXISTING_FILENAME:

diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h
index a09c39057518f220f2613a3827a4f887a16d3681..c3f01c7746f51082223f35713b48fc76ac36a141 100644 (file)
--- a/apt-pkg/indexfile.h
+++ b/apt-pkg/indexfile.h
@@ -89,6 +89,7 @@ class IndexTarget                                                     /*{{{*/
       COMPRESSIONTYPES,
       DEFAULTENABLED,
       SOURCESENTRY,
       COMPRESSIONTYPES,
       DEFAULTENABLED,
       SOURCESENTRY,

+      BY_HASH,

    };
    std::string Option(OptionKeys const Key) const;
    bool OptionBool(OptionKeys const Key) const;
    };
    std::string Option(OptionKeys const Key) const;
    bool OptionBool(OptionKeys const Key) const;

diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc
index 4e5ff0578271e1bd81a9525a53dfda04c039bb03..2100b5d3ccac5731ca7c7868cbe035c9eecf2c82 100644 (file)
--- a/apt-pkg/sourcelist.cc
+++ b/apt-pkg/sourcelist.cc
@@ -108,6 +108,7 @@ bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List,    /*{{{*/
    mapping.insert(std::make_pair("Valid-Until-Max", std::make_pair("valid-until-max", false)));
    mapping.insert(std::make_pair("Signed-By", std::make_pair("signed-by", false)));
    mapping.insert(std::make_pair("PDiffs", std::make_pair("pdiffs", false)));
    mapping.insert(std::make_pair("Valid-Until-Max", std::make_pair("valid-until-max", false)));
    mapping.insert(std::make_pair("Signed-By", std::make_pair("signed-by", false)));
    mapping.insert(std::make_pair("PDiffs", std::make_pair("pdiffs", false)));

+   mapping.insert(std::make_pair("By-Hash", std::make_pair("by-hash", false)));

 
    for (std::map<char const * const, std::pair<char const * const, bool> >::const_iterator m = mapping.begin(); m != mapping.end(); ++m)
       if (Tags.Exists(m->first))
 
    for (std::map<char const * const, std::pair<char const * const, bool> >::const_iterator m = mapping.begin(); m != mapping.end(); ++m)
       if (Tags.Exists(m->first))

diff --git a/doc/acquire-additional-files.txt b/doc/acquire-additional-files.txt
index c52955619164dea7f7d6beda0f056cb62d39abeb..a7acbbe465a6ef766612ab956d87e3f24fb52500 100644 (file)
--- a/doc/acquire-additional-files.txt
+++ b/doc/acquire-additional-files.txt
@@ -114,6 +114,9 @@ aren't accidentally used by frontends:
   Defaults to the value of Acquire::PDiffs which is true by default.
   Can be overridden per-source by the sources.list option of the same
   name. See the documentation for both of these for details.
   Defaults to the value of Acquire::PDiffs which is true by default.
   Can be overridden per-source by the sources.list option of the same
   name. See the documentation for both of these for details.

+* By-Hash: controls if apt will try to use an URI constructed from
+  a hashsum of the file to download. See the documentation for config
+  option Acquire::By-Hash and sources.list option By-Hash for details.

 * CompressionTypes: The default value is a space separated list of
   compression types supported by apt (see Acquire::CompressionTypes).
   You can set this option to prevent apt from downloading a compression
 * CompressionTypes: The default value is a space separated list of
   compression types supported by apt (see Acquire::CompressionTypes).
   You can set this option to prevent apt from downloading a compression

diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml
index 62dffadc4d271d1996443c97041c21843a217c5b..bb0c37ff806b86456ed938de31bc02e6b917f269 100644 (file)
--- a/doc/apt.conf.5.xml
+++ b/doc/apt.conf.5.xml
@@ -347,6 +347,16 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";};
         </para></listitem>
      </varlistentry>
 
         </para></listitem>
      </varlistentry>
 

+     <varlistentry><term><option>By-Hash</option></term>
+        <listitem><para>Try to download indexes via an URI constructed from a
+        hashsum of the expected file rather than downloaded via a well-known
+        stable filename. True by default, but automatically disabled if the
+        source indicates no support for it. Usage can be forced with the special
+        value "force". Preferably, this can be set for specific &sources-list; entries
+        or index files by using the <option>By-Hash</option> option there.
+        </para></listitem>
+     </varlistentry>
+

      <varlistentry><term><option>Queue-Mode</option></term>
      <listitem><para>Queuing mode; <literal>Queue-Mode</literal> can be one of <literal>host</literal> or 
      <literal>access</literal> which determines how  APT parallelizes outgoing 
      <varlistentry><term><option>Queue-Mode</option></term>
      <listitem><para>Queuing mode; <literal>Queue-Mode</literal> can be one of <literal>host</literal> or 
      <literal>access</literal> which determines how  APT parallelizes outgoing 

diff --git a/doc/sources.list.5.xml b/doc/sources.list.5.xml
index 276e7d46f0a6aefb247f2c41872442ca60a9cc63..71447e84f1fdc02b7b5fe35469828dd67a81bf61 100644 (file)
--- a/doc/sources.list.5.xml
+++ b/doc/sources.list.5.xml
@@ -239,6 +239,22 @@ deb-src [ option1=value1 option2=value2 ] uri suite [component1] [component2] [.
                <literal>yes</literal>.
          </para></listitem>
 
                <literal>yes</literal>.
          </para></listitem>
 

+         <listitem><para><option>By-Hash</option> (<option>by-hash</option>)
+               can have the value "yes", "no" or "force" and controls if APT
+               should try to acquire indexes via an URI constructed from a
+               hashsum of the expected file instead of using the well-known
+               stable filename of the index. Using this can avoid hashsum
+               mismatches, but requires a supporting mirror. The value
+               "yes"/"no" activates/disables the use of this feature if this
+               source indicates support for it, while "force" will enable the
+               feature regardless of what the source indicates.
+               Defaults to the value of the option of the same name for a
+               specific index file defined in the
+               <option>Acquire::IndexTargets</option> scope, which itself
+               defaults to the value of configuration option
+               <option>Acquire::By-Hash</option> which defaults to
+               <literal>yes</literal>.
+         </para></listitem>

 
        </itemizedlist>
 
 
        </itemizedlist>
 

diff --git a/test/integration/test-apt-by-hash-update b/test/integration/test-apt-by-hash-update
index 29383686114e3ff8e1b3d0c2429df8ad725cfd41..c00ab497b71701175c36815bd9eaf31ee7023193 100755 (executable)
--- a/test/integration/test-apt-by-hash-update
+++ b/test/integration/test-apt-by-hash-update
@@ -5,8 +5,9 @@ TESTDIR=$(readlink -f $(dirname $0))
 . $TESTDIR/framework
 
 setupenvironment
 . $TESTDIR/framework
 
 setupenvironment

-configarchitecture "i386"
+configarchitecture 'i386'

 confighashes 'SHA512'
 confighashes 'SHA512'

+configcompression '.' 'gz'

 
 insertpackage 'unstable' 'foo' 'all' '1.0'
 
 
 insertpackage 'unstable' 'foo' 'all' '1.0'
 


@@ -24,29 +25,47 @@ mkdir -p aptarchive/dists/unstable/main/source/by-hash/SHA512
      ln -s ../../Sources.gz  $(sha512sum ../../Sources.gz|cut -f1 -d' ') 
 )
 
      ln -s ../../Sources.gz  $(sha512sum ../../Sources.gz|cut -f1 -d' ') 
 )
 

-# we moved the Packages file away, normal update won't work
-testfailure aptget update
-
-# ensure we do not know about "foo"
-testfailureequal "Reading package lists...
+ensureitsbroken() {
+       rm -rf rootdir/var/lib/apt/lists
+       # we moved the Packages file away, normal update won't work
+       testfailure aptget update "$@"
+       # ensure we do not know about "foo"
+       testfailureequal "Reading package lists...

 Building dependency tree...
 E: Unable to locate package foo" aptget install -q -s foo
 Building dependency tree...
 E: Unable to locate package foo" aptget install -q -s foo

-
-# ensure we can apt-get update by hash
-testsuccess aptget update -o APT::Acquire::By-Hash=1 -o Acquire::Languages=none
+}
+ensureitsbroken
+ensureitsbroken -o Acquire::By-Hash=1

 
 ensureitworks() {
 
 ensureitworks() {

+       rm -rf rootdir/var/lib/apt/lists
+       testsuccess aptget update -o Acquire::Languages=none "$@"

        testsuccessequal "Inst foo (1.0 unstable [all])
 Conf foo (1.0 unstable [all])" aptget install -qq -s foo
 }
        testsuccessequal "Inst foo (1.0 unstable [all])
 Conf foo (1.0 unstable [all])" aptget install -qq -s foo
 }

-ensureitworks
+msgmsg 'Test by-hash via' 'config option'
+ensureitworks -o Acquire::By-Hash=force

 
 

+msgmsg 'Test by-hash via' 'release option'
+cp -a aptarchive/dists aptarchive/dists.bak

 # add magic string to Release file ...
 # add magic string to Release file ...

-MAGIC="Acquire-By-Hash: true"
-sed -i "s#Suite: unstable#Suite: unstable\n$MAGIC#" aptarchive/dists/unstable/Release
+sed -i '/^Suite: / a \
+Acquire-By-Hash: yes' aptarchive/dists/unstable/Release

 signreleasefiles
 signreleasefiles

-# ... and verify that it fetches by hash now
-rm -rf rootdir/var/lib/apt/lists
-testsuccess aptget update -o Acquire::Languages=none
+ensureitworks
+ensureitsbroken -o Acquire::By-Hash=0
+
+msgmsg 'Test by-hash via' 'sources option'
+sed -i "s#^\(deb\(-src\)\?\) #\1 [by-hash=yes] #" rootdir/etc/apt/sources.list.d/*
+ensureitworks
+#ensureitsbroken -o Acquire::By-Hash=0
+
+rm -rf aptarchive/dists
+cp -a aptarchive/dists.bak aptarchive/dists
+#ensureitworks -o Acquire::By-Hash=force
+ensureitsbroken -o Acquire::By-Hash=1
+ensureitsbroken -o Acquire::By-Hash=0

 
 

+sed -i "s#^\(deb\(-src\)\?\) \[by-hash=yes\] #\1 [by-hash=force] #" rootdir/etc/apt/sources.list.d/*

 ensureitworks
 ensureitworks

+#ensureitsbroken -o Acquire::By-Hash=0