implement indextargets option 'DefaultEnabled'

Some targets like Contents-udeb are special-needs targets. Shipping the
configuration snippet for them is okay, but they shouldn't be downloaded
by default. Forcing the user to enable targets by uncommenting targets
is wrong and this would still not really solve the problem completely as
even if you want to download some -udebs it will probably not be for all
sources you have enabled, so having the possibility of disabling a
target by default, but giving the user the option to enable it on a
per-source entry basis is better.

diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index a26b94d82de6ca0214c8c19a109a493d4e739e7d..08d8ef05e795c671645ec485f328214dfc59c966 100644 (file)
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -145,6 +145,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
         std::string const tplLongDesc = "$(SITE) " + APT_T_CONFIG_STR(flatArchive ? "flatDescription" : "Description", "");
         bool const IsOptional = APT_T_CONFIG_BOOL("Optional", true);
         bool const KeepCompressed = APT_T_CONFIG_BOOL("KeepCompressed", GzipIndex);
         std::string const tplLongDesc = "$(SITE) " + APT_T_CONFIG_STR(flatArchive ? "flatDescription" : "Description", "");
         bool const IsOptional = APT_T_CONFIG_BOOL("Optional", true);
         bool const KeepCompressed = APT_T_CONFIG_BOOL("KeepCompressed", GzipIndex);

+        bool const DefaultEnabled = APT_T_CONFIG_BOOL("DefaultEnabled", true);

         bool const UsePDiffs = APT_T_CONFIG_BOOL("PDiffs", E->UsePDiffs);
         std::string const CompressionTypes = APT_T_CONFIG_STR("CompressionTypes", DefCompressionTypes);
 #undef APT_T_CONFIG_BOOL
         bool const UsePDiffs = APT_T_CONFIG_BOOL("PDiffs", E->UsePDiffs);
         std::string const CompressionTypes = APT_T_CONFIG_STR("CompressionTypes", DefCompressionTypes);
 #undef APT_T_CONFIG_BOOL


@@ -185,10 +186,8 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
               Options.insert(std::make_pair("REPO_URI", URI));
               Options.insert(std::make_pair("TARGET_OF", Type));
               Options.insert(std::make_pair("CREATED_BY", *T));
               Options.insert(std::make_pair("REPO_URI", URI));
               Options.insert(std::make_pair("TARGET_OF", Type));
               Options.insert(std::make_pair("CREATED_BY", *T));

-              if (UsePDiffs)
-                 Options.insert(std::make_pair("PDIFFS", "yes"));
-              else
-                 Options.insert(std::make_pair("PDIFFS", "no"));
+              Options.insert(std::make_pair("PDIFFS", UsePDiffs ? "yes" : "no"));
+              Options.insert(std::make_pair("DEFAULTENABLED", DefaultEnabled ? "yes" : "no"));

               Options.insert(std::make_pair("COMPRESSIONTYPES", CompressionTypes));
 
               IndexTarget Target(
               Options.insert(std::make_pair("COMPRESSIONTYPES", CompressionTypes));
 
               IndexTarget Target(


@@ -736,7 +735,14 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type              /*{{{*/
       }
 
       std::vector<std::string> const alltargets = _config->FindVector(std::string("Acquire::IndexTargets::") + Name, "", true);
       }
 
       std::vector<std::string> const alltargets = _config->FindVector(std::string("Acquire::IndexTargets::") + Name, "", true);

-      std::vector<std::string> mytargets = parsePlusMinusOptions("target", Options, alltargets);
+      std::vector<std::string> deftargets;
+      deftargets.reserve(alltargets.size());
+      std::copy_if(alltargets.begin(), alltargets.end(), std::back_inserter(deftargets), [&](std::string const &t) {
+        std::string c = "Acquire::IndexTargets::";
+        c.append(Name).append("::").append(t).append("::DefaultEnabled");
+        return _config->FindB(c, true);
+      });
+      std::vector<std::string> mytargets = parsePlusMinusOptions("target", Options, deftargets);

       for (auto const &target : alltargets)
       {
         std::map<std::string, std::string>::const_iterator const opt = Options.find(target);
       for (auto const &target : alltargets)
       {
         std::map<std::string, std::string>::const_iterator const opt = Options.find(target);

diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc
index 775ddf7122ac2081ab12dad83f156797b617363c..1ed7984101f03bf8a2a418ce21cd07c140462620 100644 (file)
--- a/apt-pkg/indexfile.cc
+++ b/apt-pkg/indexfile.cc
@@ -145,6 +145,7 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const             /*{{{*/
       APT_CASE(TARGET_OF);
       APT_CASE(CREATED_BY);
       APT_CASE(PDIFFS);
       APT_CASE(TARGET_OF);
       APT_CASE(CREATED_BY);
       APT_CASE(PDIFFS);

+      APT_CASE(DEFAULTENABLED);

       APT_CASE(COMPRESSIONTYPES);
 #undef APT_CASE
       case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
       APT_CASE(COMPRESSIONTYPES);
 #undef APT_CASE
       case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);

diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h
index 8f7e9dc0548ba502547abd9362097bb764802c98..f267247c17409046eb910f906e4a8a1bbc98adf7 100644 (file)
--- a/apt-pkg/indexfile.h
+++ b/apt-pkg/indexfile.h
@@ -87,6 +87,7 @@ class IndexTarget                                                     /*{{{*/
       EXISTING_FILENAME,
       PDIFFS,
       COMPRESSIONTYPES,
       EXISTING_FILENAME,
       PDIFFS,
       COMPRESSIONTYPES,

+      DEFAULTENABLED,

    };
    std::string Option(OptionKeys const Key) const;
    bool OptionBool(OptionKeys const Key) const;
    };
    std::string Option(OptionKeys const Key) const;
    bool OptionBool(OptionKeys const Key) const;

diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc
index f5c42b279d5dbc9a05a8aebbb83c56dbe92cefc9..d3b3da240b7b00c786d7c1fd4e7d95dd9c619361 100644 (file)
--- a/cmdline/apt-get.cc
+++ b/cmdline/apt-get.cc
@@ -1505,6 +1505,8 @@ static bool DoIndexTargets(CommandLine &CmdL)
                  stanza << "PDiffs: " << O->second << "\n";
               else if (O->first == "COMPRESSIONTYPES")
                  stanza << "CompressionTypes: " << O->second << "\n";
                  stanza << "PDiffs: " << O->second << "\n";
               else if (O->first == "COMPRESSIONTYPES")
                  stanza << "CompressionTypes: " << O->second << "\n";

+              else if (O->first == "DEFAULTENABLED")
+                 stanza << "DefaultEnabled: " << O->second << "\n";

               else
                  stanza << format_key(O->first) << ": " << O->second << "\n";
            }
               else
                  stanza << format_key(O->first) << ": " << O->second << "\n";
            }

diff --git a/doc/acquire-additional-files.txt b/doc/acquire-additional-files.txt
index 11f4bb76d9c72f908d11c0f9b590ab9b3e52df52..37773c321d7882e4c50f9aa741b28b58ee40fc55 100644 (file)
--- a/doc/acquire-additional-files.txt
+++ b/doc/acquire-additional-files.txt
@@ -12,8 +12,8 @@ apt-transport-https.
 For its own operation libapt needs or can make use of Packages, Sources
 and Translation-* files, which it will acquire by default, but
 a repository might contain more data files (e.g.  Contents) a frontend
 For its own operation libapt needs or can make use of Packages, Sources
 and Translation-* files, which it will acquire by default, but
 a repository might contain more data files (e.g.  Contents) a frontend

-might want to use and would therefore need to be downloaded as well
-(e.g. apt-file).
+(e.g. apt-file) might want to use and would therefore need to be
+downloaded as well.

 
 This file describes the configuration scheme such a frontend can use to
 instruct the Acquire system to download those additional files.
 
 This file describes the configuration scheme such a frontend can use to
 instruct the Acquire system to download those additional files.


@@ -34,7 +34,7 @@ like this (see also apt.conf(5) manpage for configuration file syntax):
                flatMetaKey "Packages";
                flatDescription "$(RELEASE) Packages";
 
                flatMetaKey "Packages";
                flatDescription "$(RELEASE) Packages";
 

-               Optional "false";
+               Optional "no";

        };
 
 All files which should be downloaded (nicknamed 'Targets') are mentioned
        };
 
 All files which should be downloaded (nicknamed 'Targets') are mentioned


@@ -47,7 +47,9 @@ multiple types!
 After the type you can pick any valid and unique string which preferable
 refers to the file it downloads (In the example we picked 'Packages').
 This string is used as identifier for the target class and accessible as
 After the type you can pick any valid and unique string which preferable
 refers to the file it downloads (In the example we picked 'Packages').
 This string is used as identifier for the target class and accessible as

-'Created-By' e.g. in the "apt-get indextargets" output as detailed below.
+'Created-By' e.g. in the "apt-get indextargets" output as detailed
+below.  It is also used to allow user to enable/disable targets per
+sources.list entry.

 
 All targets have three main properties you can define:
 * MetaKey: The identifier of the file to be downloaded as used in the
 
 All targets have three main properties you can define:
 * MetaKey: The identifier of the file to be downloaded as used in the


@@ -69,17 +71,14 @@ All targets have three main properties you can define:
 
 
 Additional optional properties:
 
 
 Additional optional properties:

-* flat{MetaKey,Description}: APT supports two types of repositories:
-  dists-style repositories which are the default and by far the most
-  common which are named after the fact that the files are in an
-  elaborated directory structure.  In contrast a flat-style repositories
-  lumps all files together in one directory.  Support for these flat
-  repositories exists mainly for legacy purposes only.  It is therefore
-  recommend to not set these values.
-* Optional: The default value is 'true' and should be kept at this
-  value.  If enabled the acquire system will skip the download if the
-  file isn't mentioned in the Release file. Otherwise this is treated as
-  a hard error and the update process fails. Note that failures while
+* DefaultEnabled: The default value is 'yes' which means that apt will
+  try to acquire this target from all sources. If set to 'no' the user
+  has to explicitly enable this target in the sources.list file with the
+  Targets option(s) – or override this value in a config file.
+* Optional: The default value is 'yes' and should be kept at this value.
+  If enabled the acquire system will skip the download if the file isn't
+  mentioned in the Release file. Otherwise this is treated as a hard
+  error and the update process fails. Note that failures while

   downloading (e.g. 404 or hash verification errors) are failures,
   regardless of this setting.
 * KeepCompressed: The default is the value of Acquire::GzipIndexes,
   downloading (e.g. 404 or hash verification errors) are failures,
   regardless of this setting.
 * KeepCompressed: The default is the value of Acquire::GzipIndexes,


@@ -90,6 +89,13 @@ Additional optional properties:
   globally. On the other hand, if you set it to true or don't set it you
   have to ensure your frontend can deal with all compressed fileformats
   supported by apt (libapt users can e.g. use FileFd).
   globally. On the other hand, if you set it to true or don't set it you
   have to ensure your frontend can deal with all compressed fileformats
   supported by apt (libapt users can e.g. use FileFd).

+* flat{MetaKey,Description}: APT supports two types of repositories:
+  dists-style repositories which are the default and by far the most
+  common which are named after the fact that the files are in an
+  elaborated directory structure.  In contrast a flat-style repository
+  lumps all files together in one directory.  Support for these flat
+  repositories exists mainly for legacy purposes only.  It is therefore
+  recommend to not set these values.

 
 
 The acquire system will automatically choose to download a compressed
 
 
 The acquire system will automatically choose to download a compressed


@@ -101,7 +107,9 @@ information about the compressed files/PDiffs to make this happen.
 
 
 More properties exist, but these should *NOT* be set by frontends
 
 
 More properties exist, but these should *NOT* be set by frontends

-requesting files. They exist for internal and end-user usage only:
+requesting files. They exist for internal and end-user usage only.
+Some of these are – which are documented here only to ensure that they
+aren't accidentally used by frontends:

 * PDiffs: controls if apt will try to use PDiffs for this target.
   Defaults to the value of Acquire::PDiffs which is true by default.
   Can be overridden per-source by the sources.list option of the same
 * PDiffs: controls if apt will try to use PDiffs for this target.
   Defaults to the value of Acquire::PDiffs which is true by default.
   Can be overridden per-source by the sources.list option of the same


@@ -137,7 +145,7 @@ Acquire::IndexTargets {
                flatMetaKey "Sources";
                flatDescription "$(RELEASE) Sources";
 
                flatMetaKey "Sources";
                flatDescription "$(RELEASE) Sources";
 

-               Optional "false";
+               Optional "no";

        };
 };
 
        };
 };
 


@@ -149,9 +157,8 @@ unknown variables have no default value nor are they touched: They are
 printed as-is.
 
 * $(RELEASE): This is usually an archive- or codename, e.g. "stable" or
 printed as-is.
 
 * $(RELEASE): This is usually an archive- or codename, e.g. "stable" or

-  "stretch".  Note that flat-style repositories do not have a archive-
+  "stretch".  Note that flat-style repositories do not have an archive-

   or codename per-se, so the value might very well be just "/" or so.
   or codename per-se, so the value might very well be just "/" or so.

-  Again, as seen in the sources.list.

 * $(COMPONENT): as given in the sources.list, e.g. "main", "non-free" or
   "universe".  Note that flat-style repositories again do not really
   have a meaningful value here.
 * $(COMPONENT): as given in the sources.list, e.g. "main", "non-free" or
   "universe".  Note that flat-style repositories again do not really
   have a meaningful value here.


@@ -187,8 +194,12 @@ To get all the filenames of all Translation-en files you can e.g. call:
        apt-get indextargets --format '$(FILENAME)' "Created-By: Translations" "Language: en"
 
 The line-based filtering and the formating is rather crude and feature-
        apt-get indextargets --format '$(FILENAME)' "Created-By: Translations" "Language: en"
 
 The line-based filtering and the formating is rather crude and feature-

-less by design, so it is recommend to use dedicated and more powerful
-tools like 'grep-dctrl'.
+less by design: The default format is Debians standard format deb822 (in
+particular: Field names are case-insensitive and the order of fields in
+the stanza is undefined), so instead of apt reimplementing powerful
+filters and formating for this command, it is recommend to use piping
+and dedicated tools like 'grep-dctrl' if you need more than the basics
+provided.

 
 Accessing this information via libapt is done by reading the
 sources.lists (pkgSourceList), iterating over the metaIndex objects this
 
 Accessing this information via libapt is done by reading the
 sources.lists (pkgSourceList), iterating over the metaIndex objects this


@@ -223,8 +234,8 @@ Remarks on other available fields:
 * Target-Of: type of the sources.list entry
 * URI, Repo-URI: avoid using. Contains potentially username/password.
   Prefer 'Site', especially for display.
 * Target-Of: type of the sources.list entry
 * URI, Repo-URI: avoid using. Contains potentially username/password.
   Prefer 'Site', especially for display.

-* Optional: Decodes the option of the same name from the configuration.
-  Note that it is using 'yes' and 'no' instead of 'true' and 'false'.
+* Optional, DefaultEnabled, KeepCompressed: Decode the options of the
+  same name from the configuration.

 * Language, Architecture, Component: as defined further above, but with
   the catch that they might be missing if they don't effect the target
   (aka: They weren't used while evaluating the MetaKey template).
 * Language, Architecture, Component: as defined further above, but with
   the catch that they might be missing if they don't effect the target
   (aka: They weren't used while evaluating the MetaKey template).


@@ -233,13 +244,17 @@ Again, additional fields might be visible in certain implementations,
 but you should avoid using them and instead talk to us about a portable
 implementation.
 
 but you should avoid using them and instead talk to us about a portable
 implementation.
 

-# Multiple application requiring the same files
+# Multiple applications requiring the same files

 
 It is highly encouraged that applications talk to each other and to us
 about which files they require. It is usually best to have a common
 package ship the configuration needed to get the files, but specific
 needs might require specific solutions. Again: talk to us.
 
 
 It is highly encouraged that applications talk to each other and to us
 about which files they require. It is usually best to have a common
 package ship the configuration needed to get the files, but specific
 needs might require specific solutions. Again: talk to us.
 

+Bad things will happen if multiple frontends request the same file(s)
+via different targets, which is another reason why coordination is very
+important!
+

 # Acquiring files not mentioned in the Release file
 
 You can't. This is by design as these files couldn't be verified to not
 # Acquiring files not mentioned in the Release file
 
 You can't. This is by design as these files couldn't be verified to not


@@ -254,3 +269,25 @@ accessible (e.g. proxy settings) or that local sources (file:/, cdrom:/)
 start requesting online files…
 
 In other words: We would be opening Pandora's box.
 start requesting online files…
 
 In other words: We would be opening Pandora's box.

+
+# Acquiring files to a specific location on disk
+
+You can't by design to avoid multiple frontends requesting the same file
+to be downloaded to multiple different places on (different) disks
+(among other reasons).  See the next point for a solution if you really
+have to force a specific location by creating symlinks.
+
+# Post processing the acquired files
+
+You can't modify the files apt has downloaded as apt keeps state with
+e.g. the modification times of the files and advanced features like
+PDiffs break.
+
+You can however install an APT::Update::Post-Invoke{-Success,} hook
+script and use them to copy (modified) files to a different location.
+Use 'apt-get indextargets' (or similar) to get the filenames – do not
+look into /var/lib/apt/lists directly!
+
+Please avoid time consuming calculations in the scripts and instead just
+trigger a background task as there is little to no feedback for the user
+while hook scripts run.

diff --git a/test/integration/test-sourceslist-target-plusminus-options b/test/integration/test-sourceslist-target-plusminus-options
index 00d9085a0a63f0d5eea9443b937c89aee48ffb9a..6f8d05560d333ae32941512587daddbe021990d9 100755 (executable)
--- a/test/integration/test-sourceslist-target-plusminus-options
+++ b/test/integration/test-sourceslist-target-plusminus-options
@@ -20,6 +20,7 @@ testtargets() {
 echo 'deb http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
 testtargets 'default' 'Packages' 'Translations'
 
 echo 'deb http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
 testtargets 'default' 'Packages' 'Translations'
 

+msgmsg 'Contents as a default target'

 cat > rootdir/etc/apt/apt.conf.d/content-target.conf <<EOF
 Acquire::IndexTargets::deb::Contents {
        MetaKey "\$(COMPONENT)/Contents-\$(ARCHITECTURE)";
 cat > rootdir/etc/apt/apt.conf.d/content-target.conf <<EOF
 Acquire::IndexTargets::deb::Contents {
        MetaKey "\$(COMPONENT)/Contents-\$(ARCHITECTURE)";


@@ -27,6 +28,8 @@ Acquire::IndexTargets::deb::Contents {
        Description "\$(RELEASE)/\$(COMPONENT) \$(ARCHITECTURE) Contents";
 };
 EOF
        Description "\$(RELEASE)/\$(COMPONENT) \$(ARCHITECTURE) Contents";
 };
 EOF

+
+echo 'deb http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list

 testtargets 'default + Contents' 'Packages' 'Translations' 'Contents'
 
 echo 'deb [target=Packages] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
 testtargets 'default + Contents' 'Packages' 'Translations' 'Contents'
 
 echo 'deb [target=Packages] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list


@@ -64,3 +67,46 @@ testtargets 'deactivate existing' 'Packages' 'Translations'
 
 echo 'deb [target=Packages Contents=yes] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
 testtargets 'explicit + activate' 'Packages' 'Contents'
 
 echo 'deb [target=Packages Contents=yes] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
 testtargets 'explicit + activate' 'Packages' 'Contents'

+
+
+msgmsg 'Contents NOT as a default target'
+echo 'Acquire::IndexTargets::deb::Contents::DefaultEnabled "no";' > rootdir/etc/apt/apt.conf.d/content-target-notdefault.conf
+
+echo 'deb http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
+testtargets 'default + Contents' 'Packages' 'Translations'
+
+echo 'deb [target=Packages] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
+testtargets 'force Packages target' 'Packages'
+
+echo 'deb [target=Contents] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
+testtargets 'force Contents target' 'Contents'
+
+echo 'deb [target=Translations,Contents] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
+testtargets 'force two targets' 'Contents' 'Translations'
+
+echo 'deb [target+=Translations,Contents] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
+testtargets 'add existing' 'Packages' 'Contents' 'Translations'
+
+echo 'deb [target+=AppStream] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
+testtargets 'add non-existing' 'Packages' 'Translations'
+
+echo 'deb [target-=Translations,Contents] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
+testtargets 'remove existing' 'Packages'
+
+echo 'deb [target-=AppStream] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
+testtargets 'remove non-existing' 'Packages' 'Translations'
+
+echo 'deb [AppStream=yes] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
+testtargets 'activate non-existing' 'Packages' 'Translations'
+
+echo 'deb [AppStream=no] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
+testtargets 'deactivate non-existing' 'Packages' 'Translations'
+
+echo 'deb [Contents=yes] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
+testtargets 'activate existing' 'Packages' 'Contents' 'Translations'
+
+echo 'deb [Contents=no] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
+testtargets 'deactivate existing' 'Packages' 'Translations'
+
+echo 'deb [target=Packages Contents=yes] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list
+testtargets 'explicit + activate' 'Packages' 'Contents'