implement default apt-get file --release-info mode

Selecting targets based on the Release they belong to isn't to
unrealistic. In fact, it is assumed to be the most used case so it is
made the default especially as this allows to bundle another thing we
have to be careful with: Filenames and only showing targets we have
acquired.

Closes: 752702

diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index 994b95849470c55d95de563cb7a2ab71793fdf98..b328fcea800e025d1f5363503b7dccb48baa4ee7 100644 (file)
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -422,7 +422,7 @@ bool debReleaseIndex::Merge(pkgCacheGenerator &Gen,OpProgress * /*Prog*/) const/
 }
                                                                        /*}}}*/
 // ReleaseIndex::FindInCache - Find this index                         /*{{{*/
 }
                                                                        /*}}}*/
 // ReleaseIndex::FindInCache - Find this index                         /*{{{*/

-pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache) const
+pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache, bool const ModifyCheck) const

 {
    std::string ReleaseFile;
    bool const releaseExists = ReleaseFileName(this, ReleaseFile);
 {
    std::string ReleaseFile;
    bool const releaseExists = ReleaseFileName(this, ReleaseFile);


@@ -434,7 +434,7 @@ pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache) const
         continue;
 
        // empty means the file does not exist by "design"
         continue;
 
        // empty means the file does not exist by "design"

-       if (releaseExists == false && File->Size == 0)
+       if (ModifyCheck == false || (releaseExists == false && File->Size == 0))

          return File;
 
       struct stat St;
          return File;
 
       struct stat St;

diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h
index 7e99427103bb6f4d1c1aaa82d6c713d07ddf0405..b448ecc53e1b3e8cc10f1f2c90460142a3473d93 100644 (file)
--- a/apt-pkg/deb/debmetaindex.h
+++ b/apt-pkg/deb/debmetaindex.h
@@ -51,7 +51,7 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
    virtual std::vector<IndexTarget> GetIndexTargets() const;
 
    virtual std::string Describe() const;
    virtual std::vector<IndexTarget> GetIndexTargets() const;
 
    virtual std::string Describe() const;

-   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache) const;
+   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache, bool const ModifyCheck) const;

    virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
 
    std::string MetaIndexInfo(const char *Type) const;
    virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
 
    std::string MetaIndexInfo(const char *Type) const;

diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc
index 33fb48e35fae2dfcdb0a1890e6437c5cc4930cdd..605bbeb4721ed5fea9ae82847d34115bbda5f103 100644 (file)
--- a/apt-pkg/indexfile.cc
+++ b/apt-pkg/indexfile.cc
@@ -137,6 +137,18 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const            /*{{{*/
       APT_CASE(CREATED_BY);
 #undef APT_CASE
       case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
       APT_CASE(CREATED_BY);
 #undef APT_CASE
       case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);

+      case EXISTING_FILENAME:
+        std::string const filename = Option(FILENAME);
+        std::vector<std::string> const types = APT::Configuration::getCompressionTypes();
+        for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
+        {
+           if (t->empty())
+              continue;
+           std::string const file = (*t == "uncompressed") ? filename : (filename + "." + *t);
+           if (FileExists(file))
+              return file;
+        }
+        return "";

    }
    std::map<std::string,std::string>::const_iterator const M = Options.find(Key);
    if (M == Options.end())
    }
    std::map<std::string,std::string>::const_iterator const M = Options.find(Key);
    if (M == Options.end())

diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h
index 220c415acb7dbb07b5db3bcd7fbf7a2668ff1360..042e5c2f715882f61f6a7168048025b9dca22a85 100644 (file)
--- a/apt-pkg/indexfile.h
+++ b/apt-pkg/indexfile.h
@@ -81,6 +81,7 @@ class IndexTarget                                                     /*{{{*/
       CREATED_BY,
       TARGET_OF,
       FILENAME,
       CREATED_BY,
       TARGET_OF,
       FILENAME,

+      EXISTING_FILENAME,

    };
    std::string Option(OptionKeys const Key) const;
    std::string Format(std::string format) const;
    };
    std::string Option(OptionKeys const Key) const;
    std::string Format(std::string format) const;

diff --git a/apt-pkg/metaindex.cc b/apt-pkg/metaindex.cc
index 35ab6c53b632c2ddd3f2bf146e83040ed6282263..3c1b696bdf78c8a2cdd5daba8b867c6b13d507d1 100644 (file)
--- a/apt-pkg/metaindex.cc
+++ b/apt-pkg/metaindex.cc
@@ -28,7 +28,7 @@ std::string metaIndex::Describe() const
    return "Release";
 }
 
    return "Release";
 }
 

-pkgCache::RlsFileIterator metaIndex::FindInCache(pkgCache &Cache) const
+pkgCache::RlsFileIterator metaIndex::FindInCache(pkgCache &Cache, bool const) const

 {
    return pkgCache::RlsFileIterator(Cache);
 }
 {
    return pkgCache::RlsFileIterator(Cache);
 }

diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h
index 20c879a89cc6802da4677bd44546c1e0ab8f0d4a..e1810fb278817527573f6e77dca94aea0dd9deec 100644 (file)
--- a/apt-pkg/metaindex.h
+++ b/apt-pkg/metaindex.h
@@ -54,7 +54,7 @@ class metaIndex
    virtual bool IsTrusted() const = 0;
 
    virtual std::string Describe() const;
    virtual bool IsTrusted() const = 0;
 
    virtual std::string Describe() const;

-   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache) const;
+   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache, bool const ModifyCheck) const;

    virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
 
    metaIndex(std::string const &URI, std::string const &Dist,
    virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
 
    metaIndex(std::string const &URI, std::string const &Dist,

diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc
index 2d0e3960d30c4ac6d1b77ce1eda245b3aa194e37..9db4ef41f1dc154226ec11075adcb0cef74ea9ad 100644 (file)
--- a/apt-pkg/pkgcachegen.cc
+++ b/apt-pkg/pkgcachegen.cc
@@ -1224,7 +1224,7 @@ static bool CheckValidity(const string &CacheFile,
    {
       if (Debug == true)
         std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
    {
       if (Debug == true)
         std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";

-      pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache);
+      pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);

       if (RlsFile.end() == true)
       {
         if (Debug == true)
       if (RlsFile.end() == true)
       {
         if (Debug == true)


@@ -1348,7 +1348,7 @@ static bool BuildCache(pkgCacheGenerator &Gen,
    {
       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
       {
    {
       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
       {

-        if ((*i)->FindInCache(Gen.GetCache()).end() == false)
+        if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)

         {
            _error->Warning("Duplicate sources.list entry %s",
                  (*i)->Describe().c_str());
         {
            _error->Warning("Duplicate sources.list entry %s",
                  (*i)->Describe().c_str());

diff --git a/apt-private/private-cmndline.cc b/apt-private/private-cmndline.cc
index 458051bf20f5b162e158fd52b5b9a927f61b2828..11e88b1e7f96c4dd793c40029fb471bcb4a0d9ab 100644 (file)
--- a/apt-private/private-cmndline.cc
+++ b/apt-private/private-cmndline.cc
@@ -166,6 +166,7 @@ static bool addArgumentsAPTGet(std::vector<CommandLine::Args> &Args, char const
    else if (CmdMatches("files"))
    {
       addArg(0,"format","APT::Get::Files::Format", CommandLine::HasArg);
    else if (CmdMatches("files"))
    {
       addArg(0,"format","APT::Get::Files::Format", CommandLine::HasArg);

+      addArg(0,"release-info","APT::Get::Files::ReleaseInfo", 0);

    }
    else if (CmdMatches("clean", "autoclean", "check", "download", "changelog") ||
            CmdMatches("markauto", "unmarkauto")) // deprecated commands
    }
    else if (CmdMatches("clean", "autoclean", "check", "download", "changelog") ||
            CmdMatches("markauto", "unmarkauto")) // deprecated commands

diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc
index 0fff2db5812fae5d4a8486e49ec95a69673bde68..a4cd3c377bc57dd9f117ba7e9fcb40443ec353e1 100644 (file)
--- a/cmdline/apt-get.cc
+++ b/cmdline/apt-get.cc
@@ -1628,15 +1628,34 @@ static bool DoFiles(CommandLine &CmdL)
       return false;
 
    std::string const Format = _config->Find("APT::Get::Files::Format");
       return false;
 
    std::string const Format = _config->Find("APT::Get::Files::Format");

+   bool const ReleaseInfo = _config->FindB("APT::Get::Files::ReleaseInfo", true);

    bool Filtered = CmdL.FileSize() > 1;
    for (pkgSourceList::const_iterator S = SrcList->begin(); S != SrcList->end(); ++S)
    {
       std::vector<IndexTarget> const targets = (*S)->GetIndexTargets();
       std::map<std::string, string> AddOptions;
    bool Filtered = CmdL.FileSize() > 1;
    for (pkgSourceList::const_iterator S = SrcList->begin(); S != SrcList->end(); ++S)
    {
       std::vector<IndexTarget> const targets = (*S)->GetIndexTargets();
       std::map<std::string, string> AddOptions;

-      AddOptions.insert(std::make_pair("TRUSTED", ((*S)->IsTrusted() ? "yes" : "no")));
+      if (ReleaseInfo)
+      {
+        AddOptions.insert(std::make_pair("TRUSTED", ((*S)->IsTrusted() ? "yes" : "no")));
+        pkgCache &Cache = *CacheFile.GetPkgCache();
+        pkgCache::RlsFileIterator const RlsFile = (*S)->FindInCache(Cache, false);
+        if (RlsFile.end())
+           continue;
+#define APT_RELEASE(X,Y) if (RlsFile.Y() != NULL) AddOptions.insert(std::make_pair(X, RlsFile.Y()))
+        APT_RELEASE("CODENAME", Codename);
+        APT_RELEASE("SUITE", Archive);
+        APT_RELEASE("VERSION", Version);
+        APT_RELEASE("ORIGIN", Origin);
+        APT_RELEASE("LABEL", Label);
+#undef APT_RELEASE
+      }

 
       for (std::vector<IndexTarget>::const_iterator T = targets.begin(); T != targets.end(); ++T)
       {
 
       for (std::vector<IndexTarget>::const_iterator T = targets.begin(); T != targets.end(); ++T)
       {

+        std::string filename = T->Option(ReleaseInfo ? IndexTarget::EXISTING_FILENAME : IndexTarget::FILENAME);
+        if (filename.empty())
+           continue;
+

         std::ostringstream stanza;
         if (Filtered || Format.empty())
         {
         std::ostringstream stanza;
         if (Filtered || Format.empty())
         {


@@ -1644,7 +1663,7 @@ static bool DoFiles(CommandLine &CmdL)
               << "ShortDesc: " << T->ShortDesc << "\n"
               << "Description: " << T->Description << "\n"
               << "URI: " << T->URI << "\n"
               << "ShortDesc: " << T->ShortDesc << "\n"
               << "Description: " << T->Description << "\n"
               << "URI: " << T->URI << "\n"

-              << "Filename: " << T->Option(IndexTarget::FILENAME) << "\n"
+              << "Filename: " << filename << "\n"

               << "Optional: " << (T->IsOptional ? "yes" : "no") << "\n";
            for (std::map<std::string,std::string>::const_iterator O = AddOptions.begin(); O != AddOptions.end(); ++O)
               stanza << format_key(O->first) << ": " << O->second << "\n";
               << "Optional: " << (T->IsOptional ? "yes" : "no") << "\n";
            for (std::map<std::string,std::string>::const_iterator O = AddOptions.begin(); O != AddOptions.end(); ++O)
               stanza << format_key(O->first) << ": " << O->second << "\n";


@@ -1677,7 +1696,8 @@ static bool DoFiles(CommandLine &CmdL)
            cout << stanza.str();
         else
         {
            cout << stanza.str();
         else
         {

-           std::string out = T->Format(Format);
+           std::string out = SubstVar(Format, "$(FILENAME)", filename);
+           out = T->Format(out);

            for (std::map<std::string,std::string>::const_iterator O = AddOptions.begin(); O != AddOptions.end(); ++O)
               out = SubstVar(out, std::string("$(") + O->first + ")", O->second);
            cout << out << std::endl;
            for (std::map<std::string,std::string>::const_iterator O = AddOptions.begin(); O != AddOptions.end(); ++O)
               out = SubstVar(out, std::string("$(") + O->first + ")", O->second);
            cout << out << std::endl;

diff --git a/doc/acquire-additional-files.txt b/doc/acquire-additional-files.txt
index a47dac2d48363402e8b027d4f663fd39cfcead2f..f9a16318db74b28792fbac62bc79961eda946e4b 100644 (file)
--- a/doc/acquire-additional-files.txt
+++ b/doc/acquire-additional-files.txt
@@ -18,12 +18,6 @@ might want to use and would therefore need to be downloaded as well
 This file describes the configuration scheme such a frontend can use to
 instruct the Acquire system to download those additional files.
 
 This file describes the configuration scheme such a frontend can use to
 instruct the Acquire system to download those additional files.
 

-Note that you can't download files from other sources. It must be files
-in the same repository and below the Release file. The Release file must
-also contain hashes for the file itself as well as for the compressed
-file if wanted, otherwise a download isn't even tried!
-
-

 # The Configuration Stanza
 
 The Acquire system uses the same configuration settings to implement the
 # The Configuration Stanza
 
 The Acquire system uses the same configuration settings to implement the


@@ -81,15 +75,17 @@ Additional optional properties:
 * Optional: The default value is 'true' and should be kept at this
   value.  If enabled the acquire system will skip the download if the
   file isn't mentioned in the Release file. Otherwise this is treated as
 * Optional: The default value is 'true' and should be kept at this
   value.  If enabled the acquire system will skip the download if the
   file isn't mentioned in the Release file. Otherwise this is treated as

-  a hard error and the update process fails.
+  a hard error and the update process fails. Note that failures while
+  downloading (e.g. 404 or hash verification errors) are failures,
+  regardless of this setting.

 
 
 
 

-Note that the acquire system will automatically choose to download
-a compressed file if it is available and uncompress it for you, just as
-it will also use pdiff patching if provided by the repository and
-enabled by the user. You only have to ensure that the Release file
-contains the information about the compressed files/pdiffs to make this
-happen. NO properties have to be set to enable this.
+The acquire system will automatically choose to download a compressed
+file if it is available and uncompress it for you, just as it will also
+use pdiff patching if provided by the repository and enabled by the
+user. You only have to ensure that the Release file contains the
+information about the compressed files/pdiffs to make this happen.
+NO properties have to be set to enable this.

 
 # More examples
 
 
 # More examples
 


@@ -123,10 +119,12 @@ APT::Acquire::Targets {
 As seen in the examples, properties can contain placeholders filled in
 by the acquire system. The following variables are known; note that
 unknown variables have no default value nor are they touched: They are
 As seen in the examples, properties can contain placeholders filled in
 by the acquire system. The following variables are known; note that
 unknown variables have no default value nor are they touched: They are

-printed literally.
+printed as-is.

 
 * $(SITE): An identifier of the site we access as seen in sources.list,
 
 * $(SITE): An identifier of the site we access as seen in sources.list,

-  e.g. "http://example.org/debian" or "file:/path/to/a/repository".
+  e.g. "http://example.org/debian" or "file:/path/to/a/repository". You
+  can't use this field in {,flat}MetaKey, it is for description proposes
+  only.

 * $(RELEASE): This is usually an archive- or codename, e.g. "stable" or
   "stretch".  Note that flat-style repositories do not have a archive-
   or codename per-se, so the value might very well be just "/" or so.
 * $(RELEASE): This is usually an archive- or codename, e.g. "stable" or
   "stretch".  Note that flat-style repositories do not have a archive-
   or codename per-se, so the value might very well be just "/" or so.


@@ -143,7 +141,7 @@ printed literally.
 
 Note that while more variables might exist in the implementation, these
 are to be considered undefined and their usage strongly discouraged. If
 
 Note that while more variables might exist in the implementation, these
 are to be considered undefined and their usage strongly discouraged. If

-you have a need for another variable contact us instead.
+you have a need for other variables contact us.

 
 # Accessing files
 
 
 # Accessing files
 


@@ -170,29 +168,42 @@ sources.lists (pkgSourceList), iterating over the metaIndex objects this
 creates and calling GetIndexTargets() on them. See the sourcecode of
 "apt-get files" for a complete example.
 
 creates and calling GetIndexTargets() on them. See the sourcecode of
 "apt-get files" for a complete example.
 

-Remarks on the available fields:
+Note that by default targets are not listed if they weren't downloaded.
+If you want to see all targets, you can use the --no-release-info, which
+also removes the Codename, Suite, Version, Origin, Label and Trusted
+fields from the output as these also display data which needs to be
+downloaded first and could hence be inaccurate [on the pro-side: This
+mode is faster as it doesn't require a valid binary cache to operate].
+The most notable difference perhaps is in the Filename field through: By
+default it indicates an existing file, potentially compressed (Hint:
+libapt users can use FileFd to open compressed files transparently). In
+the --no-release-info mode the indicated file doesn't need to exist and
+it will always refer to an uncompressed file, even if the index would be
+(or is) stored compressed.
+
+Remarks on fields only available in (default) --release-info mode:
+* Trusted: Denotes with a 'yes' or 'no' if the data in this file is
+  authenticated by a trustchain rooted in a trusted gpg key. You should
+  be careful with untrusted data and warn the user if you use it.
+* Codename, Suite, Version, Origin and Label are fields from the Release
+  file, are only present if they are present in the Release file and
+  contain the same data.
+
+Remarks on other available fields:

 * MetaKey, ShortDesc, Description, Site, Release: as defined
   by the configuration and described further above.
 * Created-By: configuration entity responsible for this target
 * Target-Of: type of the sources.list entry
 * URI, Repo-URI: avoid using. Contains potentially username/password.
   Prefer 'Site', especially for display.
 * MetaKey, ShortDesc, Description, Site, Release: as defined
   by the configuration and described further above.
 * Created-By: configuration entity responsible for this target
 * Target-Of: type of the sources.list entry
 * URI, Repo-URI: avoid using. Contains potentially username/password.
   Prefer 'Site', especially for display.

-* Filename: The mentioned file doesn't need to exist, e.g. because the
-  file wasn't downloaded yet – or it does exist compressed. libapt users
-  are encouraged to use FileFd to open such files as it can handle
-  decompression transparently.
-* Trusted: As of the last 'apt-get update' call denoting if e.g. apt-get
-  would print the unauthenticated warning while installing packages
-  mentioned in this file (example assumes Packages file) [Not really
-  a property of the target itself, but of the metaIndex].

 * Optional: Decodes the option of the same name from the configuration.
   Note that it is using 'yes' and 'no' instead of 'true' and 'false'.
 * Language, Architecture, Component: as defined further above, but with
   the catch that they might be missing if they don't effect the target
   (aka: They weren't used while evaluating the MetaKey template).
 
 * Optional: Decodes the option of the same name from the configuration.
   Note that it is using 'yes' and 'no' instead of 'true' and 'false'.
 * Language, Architecture, Component: as defined further above, but with
   the catch that they might be missing if they don't effect the target
   (aka: They weren't used while evaluating the MetaKey template).
 

-Again, additional fields might be visible in certain implementation, but
-you should avoid using them and instead talk to us about a portable
+Again, additional fields might be visible in certain implementations,
+but you should avoid using them and instead talk to us about a portable

 implementation.
 
 # Multiple application requiring the same files
 implementation.
 
 # Multiple application requiring the same files

diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files
index 971cfac2e56cd6d667a09a4914c6734d5a85e62e..ffb9f3135525a001c89af9d5aa42b91f651590f7 100755 (executable)
--- a/test/integration/test-apt-acquire-additional-files
+++ b/test/integration/test-apt-acquire-additional-files
@@ -37,6 +37,12 @@ APT::Acquire::Targets::deb::Contents {
 };
 EOF
 
 };
 EOF
 

+testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64)" aptget files --no-release-info --format '$(FILENAME)' 'Created-By: Contents'
+testempty aptget files --format '$(FILENAME)' 'Created-By: Contents'
+# lets fake the existence of a compressed Contents file
+touch ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz
+testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz)" aptget files --format '$(FILENAME)' 'Created-By: Contents'
+

 testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 
 'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 
 'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 
 testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 
 'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 
 'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0