Read dpkg tables to handle architecture wildcards

Our implementation of wildcards was rudimentary. It worked for some
common ones, but it was also broken: For example, armel matched any-armel,
but should match any-arm.

With this commit, we load the correct tables from dpkg. Supported are
both triplets and quadruplet tables (the latter introduced in dpkg 1.18.11).

There are some odd things we have to deal with in the cache filter for
historical and API reasons:

* The character "*" must be accepted as an alternative to any - in fact
  it may appear anywhere in the wildcard as we also allow fnmatch() style
  wildcard matching on the commandline.

* The code might get passed an arch with a minus at the end, for example
  the cmdline "install apt:any-arm-" will first try to check if any-arm-
  is a valid architecture. We deal with this by rejecting any wildcard
  ending in a minus.

* Triplets are actually implemented by extending them to faux quadruplets
  - by prepending a "base" component for the architecture tuple, and "any"
  if there is a wildcard component.

Once we have constructed a wildcard, it is transformed into an fnmatch()
expression for historical reasons. In the future, we should really get a
tuple class and implement matching in a better, more explicit way.

This does for now though - it passes all the test cases and accepts all
things it should accept.

Closes: #748936
Thanks: James Clarke <jrtc27@jrtc27.com> for the initial patch

diff --git a/CMake/config.h.in b/CMake/config.h.in
index 6f39e2f5808bd20410d7fe259e664a45c3df8935..7b068864cf4f0a9649996f9387494dd7d31e52fd 100644 (file)
--- a/CMake/config.h.in
+++ b/CMake/config.h.in
@@ -63,6 +63,7 @@
 #cmakedefine CONF_DIR "${CONF_DIR}"
 #cmakedefine LIBEXEC_DIR "${LIBEXEC_DIR}"
 #cmakedefine BIN_DIR "${BIN_DIR}"
+#cmakedefine DPKG_DATADIR "${DPKG_DATADIR}"
 
 /* Group of the root user */
 #cmakedefine ROOT_GROUP "${ROOT_GROUP}"

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0afc73c0cfd9cca8690e01e48556aa4856563005..f40e389ae00712901748fb4f1f2ca61436988a40 100644 (file)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -172,6 +172,12 @@ set(PACKAGE ${PROJECT_NAME})
 set(PACKAGE_MAIL "APT Development Team <deity@lists.debian.org>")
 set(PACKAGE_VERSION "1.4~beta3")
 
+if (NOT DEFINED DPKG_DATADIR)
+  execute_process(COMMAND perl -MDpkg -e "print $Dpkg::DATADIR;"
+                  OUTPUT_VARIABLE DPKG_DATADIR_CMD OUTPUT_STRIP_TRAILING_WHITESPACE)
+  message(STATUS "Found dpkg data dir: ${DPKG_DATADIR_CMD}")
+  set(DPKG_DATADIR "${DPKG_DATADIR_CMD}" CACHE PATH "dpkg data directory")
+endif()
 if (NOT DEFINED COMMON_ARCH)
   execute_process(COMMAND dpkg-architecture -qDEB_HOST_ARCH
                   OUTPUT_VARIABLE COMMON_ARCH OUTPUT_STRIP_TRAILING_WHITESPACE)

diff --git a/apt-pkg/cachefilter.cc b/apt-pkg/cachefilter.cc
index b1adf5de3d071670d532dbb65e9b98ca658ed4f2..cc4cdf73cfba41a4326f5218334d967732b23b5b 100644 (file)
--- a/apt-pkg/cachefilter.cc
+++ b/apt-pkg/cachefilter.cc
@@ -14,7 +14,9 @@
 #include <apt-pkg/strutl.h>
 #include <apt-pkg/macros.h>
 
+#include <algorithm>
 #include <string>
+#include <unordered_map>
 #include <string.h>
 #include <regex.h>
 #include <fnmatch.h>
@@ -22,6 +24,9 @@
 #include <apti18n.h>
                                                                        /*}}}*/
 namespace APT {
+
+APT_HIDDEN std::unordered_map<std::string, std::vector<std::string>> ArchToTupleMap;
+
 namespace CacheFilter {
 APT_CONST Matcher::~Matcher() {}
 APT_CONST PackageMatcher::~PackageMatcher() {}
@@ -68,38 +73,72 @@ bool PackageNameMatchesFnmatch::operator() (pkgCache::GrpIterator const &Grp) {
    return fnmatch(Pattern.c_str(), Grp.Name(), FNM_CASEFOLD) == 0;
 }
                                                                        /*}}}*/
-// Architecture matches <libc>-<kernel>-<cpu> specification            /*{{{*/
+// Architecture matches <abi>-<libc>-<kernel>-<cpu> specification      /*{{{*/
 //----------------------------------------------------------------------
-/* The complete architecture, consisting of <libc>-<kernel>-<cpu>. */
-static std::string CompleteArch(std::string const &arch, bool const isPattern) {
-   auto const found = arch.find('-');
-   if (found != std::string::npos)
+
+static std::vector<std::string> ArchToTuple(std::string arch) {
+   // Strip leading linux- from arch if present
+   // dpkg says this may disappear in the future
+   if (APT::String::Startswith(arch, std::string("linux-")))
+      arch = arch.substr(6);
+
+   auto it = ArchToTupleMap.find(arch);
+   if (it != ArchToTupleMap.end())
+   {
+      std::vector<std::string> result = it->second;
+      // Hack in support for triplets
+      if (result.size() == 3)
+        result.emplace(result.begin(), "base");
+      return result;
+   } else
    {
-      // ensure that only -any- is replaced and not something like company-
-      std::string complete = std::string("-").append(arch).append("-");
-      size_t pos = 0;
-      char const * const search = "-any-";
-      auto const search_len = strlen(search) - 2;
-      while((pos = complete.find(search, pos)) != std::string::npos) {
-        complete.replace(pos + 1, search_len, "*");
-        pos += 2;
+      return {};
+   }
+}
+
+static std::vector<std::string> PatternToTuple(std::string const &arch) {
+   std::vector<std::string> tuple = VectorizeString(arch, '-');
+   if (std::find(tuple.begin(), tuple.end(), std::string("any")) != tuple.end() ||
+       std::find(arch.begin(), arch.end(), '*') != arch.end()) {
+      while (tuple.size() < 4) {
+        tuple.emplace(tuple.begin(), "any");
+      }
+      return tuple;
+   } else
+      return ArchToTuple(arch);
+}
+
+/* The complete architecture, consisting of <abi>-<libc>-<kernel>-<cpu>. */
+static std::string CompleteArch(std::string const &arch, bool const isPattern) {
+   auto tuple = isPattern ? PatternToTuple(arch) : ArchToTuple(arch);
+
+   // Bah, the commandline will try and pass us stuff like amd64- -- we need
+   // that not to match an architecture, but the code below would turn it into
+   // a valid tuple. Let's just use an invalid tuple here.
+   if (APT::String::Endswith(arch, "-") || APT::String::Startswith(arch, "-"))
+      return "invalid-invalid-invalid-invalid";
+
+   if (tuple.empty()) {
+      // Fallback for unknown architectures
+      // Patterns never fail if they contain wildcards, so by this point, arch
+      // has no wildcards.
+      tuple = VectorizeString(arch, '-');
+      switch (tuple.size()) {
+        case 1:
+           tuple.emplace(tuple.begin(), "linux");
+           /* fall through */
+        case 2:
+           tuple.emplace(tuple.begin(), "gnu");
+           /* fall through */
+        case 3:
+           tuple.emplace(tuple.begin(), "base");
+           /* fall through */
+           break;
       }
-      complete = complete.substr(1, complete.size()-2);
-      if (arch.find('-', found+1) != std::string::npos)
-        // <libc>-<kernel>-<cpu> format
-        return complete;
-      // <kernel>-<cpu> format
-      else if (isPattern)
-        return "*-" + complete;
-      else
-        return "gnu-" + complete;
    }
-   else if (arch == "any")
-      return "*-*-*";
-   else if (isPattern)
-      return "*-linux-" + arch;
-   else
-      return "gnu-linux-" + arch;
+
+   std::replace(tuple.begin(), tuple.end(), std::string("any"), std::string("*"));
+   return APT::String::Join(tuple, "-");
 }
 PackageArchitectureMatchesSpecification::PackageArchitectureMatchesSpecification(std::string const &pattern, bool const pisPattern) :
                                        literal(pattern), complete(CompleteArch(pattern, pisPattern)), isPattern(pisPattern) {

diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc
index 292d750719975febde7ba87e4d5ed30ee0f0d21b..8b25ca100f8473fca63915555ee03ed1eb77fa7e 100644 (file)
--- a/apt-pkg/init.cc
+++ b/apt-pkg/init.cc
@@ -20,6 +20,11 @@
 
 #include <string.h>
 #include <cstdlib>
+#include <fstream>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+#include <vector>
 
 #include <apti18n.h>
                                                                        /*}}}*/
@@ -30,6 +35,89 @@ const char *pkgVersion = PACKAGE_VERSION;
 const char *pkgLibVersion = Stringfy(APT_PKG_MAJOR) "."
                             Stringfy(APT_PKG_MINOR) "." 
                             Stringfy(APT_PKG_RELEASE);
+namespace APT {
+   APT_HIDDEN extern std::unordered_map<std::string, std::vector<std::string>> ArchToTupleMap;
+}
+
+// Splits by whitespace. There may be continous spans of whitespace - they
+// will be considered as one.
+static std::vector<std::string> split(std::string const & s)
+{
+   std::vector<std::string> vec;
+   std::istringstream iss(s);
+   iss.imbue(std::locale::classic());
+   for(std::string current_s; iss >> current_s; )
+      vec.push_back(current_s);
+   return vec;
+}
+
+
+// pkgInitArchTupleMap - Initialize the architecture tuple map                         /*{{{*/
+// ---------------------------------------------------------------------
+/* This initializes */
+static bool pkgInitArchTupleMap()
+{
+   auto tuplepath = _config->FindFile("Dir::dpkg::tupletable", DPKG_DATADIR "/tupletable");
+   auto tripletpath = _config->FindFile("Dir::dpkg::triplettable", DPKG_DATADIR "/triplettable");
+   auto cpupath = _config->FindFile("Dir::dpkg::cputable", DPKG_DATADIR "/cputable");
+
+   // Load a list of CPUs
+   std::vector<std::string> cpus;
+   std::ifstream cputable(cpupath);
+   for (std::string cpuline; std::getline(cputable, cpuline); )
+   {
+      if (cpuline[0] == '#' || cpuline[0] == '\0')
+         continue;
+      auto cpurow = split(cpuline);
+      auto cpu = APT::String::Strip(cpurow.at(0));
+
+      cpus.push_back(cpu);
+   }
+   if (!cputable.eof())
+      return _error->Error("Error reading the CPU table");
+
+   // Load the architecture tuple
+   std::ifstream tupletable;
+   if (FileExists(tuplepath))
+      tupletable.open(tuplepath);
+   else if (FileExists(tripletpath))
+      tupletable.open(tripletpath);
+   else
+      return _error->Error("Cannot find dpkg tuplet or triplet table");
+
+   APT::ArchToTupleMap.clear();
+   for (std::string tupleline; std::getline(tupletable, tupleline); )
+   {
+      if (tupleline[0] == '#' || tupleline[0] == '\0')
+         continue;
+
+      std::vector<std::string> tuplerow = split(tupleline);
+
+      auto tuple = APT::String::Strip(tuplerow.at(0));
+      auto arch = APT::String::Strip(tuplerow.at(1));
+
+      if (tuple.find("<cpu>") == tuple.npos && arch.find("<cpu>") == arch.npos)
+      {
+         APT::ArchToTupleMap.insert({arch, VectorizeString(tuple, '-')});
+      }
+      else
+      {
+         for (auto && cpu : cpus)
+         {
+            auto mytuple = SubstVar(tuple, std::string("<cpu>"), cpu);
+            auto myarch = SubstVar(arch, std::string("<cpu>"), cpu);
+
+            APT::ArchToTupleMap.insert({myarch, VectorizeString(mytuple, '-')});
+         }
+      }
+   }
+   if (!tupletable.eof())
+      return _error->Error("Error reading the Tuple table");
+
+   return true;
+}
+                                                                       /*}}}*/
+
     
 // pkgInitConfig - Initialize the configuration class                  /*{{{*/
 // ---------------------------------------------------------------------
@@ -193,6 +281,9 @@ bool pkgInitSystem(Configuration &Cnf,pkgSystem *&Sys)
       if (Sys == 0)
         return _error->Error(_("Unable to determine a suitable packaging system type"));
    }
+
+   if (pkgInitArchTupleMap() == false)
+      return false;
    
    return Sys->Initialize(Cnf);
 }

diff --git a/doc/examples/configure-index b/doc/examples/configure-index
index 7ce5aef51a8932b7f7163c6d762e92417cbd116f..653bdec1efff22063f166139eb375e59724c65d5 100644 (file)
--- a/doc/examples/configure-index
+++ b/doc/examples/configure-index
@@ -751,3 +751,8 @@ translation::compress "<STRING>";
 sources::extensions "<STRING>";
 packages::extensions "<STRING>";
 dir::filelistdir "<STRING>";
+
+// Internal code.
+dir::dpkg::tupletable "<FILE>";
+dir::dpkg::triplettable "<FILE>";
+dir::dpkg::cputable "<FILE>";

diff --git a/test/integration/test-bug-632221-cross-dependency-satisfaction b/test/integration/test-bug-632221-cross-dependency-satisfaction
index 066e29d99657c7760496b20274d38e1967be67c5..d52652caddd814f3aa9c33d4a7fed5373cba5b05 100755 (executable)
--- a/test/integration/test-bug-632221-cross-dependency-satisfaction
+++ b/test/integration/test-bug-632221-cross-dependency-satisfaction
@@ -21,7 +21,7 @@ insertpackage 'unstable' 'foreigner' 'amd64,armel' '1.0' 'Multi-Arch: foreign'
 insertpackage 'unstable' 'arm-stuff' 'armel' '1.0'
 insertpackage 'unstable' 'linux-stuff' 'amd64,armel' '1.0'
 
-insertsource 'unstable' 'apt' 'any' '0.8.15' 'Build-Depends: doxygen, libc6-dev, libc6-dev:native, cool:any, amdboot:amd64, foreigner, libfwibble-dev, arm-stuff [any-armel] | linux-stuff [ linux-any]'
+insertsource 'unstable' 'apt' 'any' '0.8.15' 'Build-Depends: doxygen, libc6-dev, libc6-dev:native, cool:any, amdboot:amd64, foreigner, libfwibble-dev, arm-stuff [eabi-any-any-arm gnueabi-any-arm] | linux-stuff [ linux-any]'
 
 insertsource 'unstable' 'forbidden-no' 'any' '1' 'Build-Depends: amdboot:any'
 insertsource 'unstable' 'forbidden-same' 'any' '1' 'Build-Depends: libc6:any'

diff --git a/test/libapt/cachefilter_test.cc b/test/libapt/cachefilter_test.cc
index 28924b758ae851ad90a26f46934a9c2ad6513047..08812e0dc74818c76a3a69284ae20d415b8b2fa9 100644 (file)
--- a/test/libapt/cachefilter_test.cc
+++ b/test/libapt/cachefilter_test.cc
@@ -1,6 +1,7 @@
 #include <config.h>
 
 #include <apt-pkg/cachefilter.h>
+#include <apt-pkg/fileutl.h>
 
 #include <string>
 
@@ -9,17 +10,22 @@
 TEST(CacheFilterTest, ArchitectureSpecification)
 {
    {
-      SCOPED_TRACE("Pattern is any-armhf");
-      APT::CacheFilter::PackageArchitectureMatchesSpecification ams("any-armhf");
-      EXPECT_TRUE(ams("armhf"));
-      EXPECT_FALSE(ams("armel"));
-      EXPECT_TRUE(ams("linux-armhf"));
-      EXPECT_FALSE(ams("linux-armel"));
-      EXPECT_TRUE(ams("kfreebsd-armhf"));
-      EXPECT_TRUE(ams("gnu-linux-armhf"));
-      EXPECT_FALSE(ams("gnu-linux-armel"));
-      EXPECT_TRUE(ams("gnu-kfreebsd-armhf"));
-      EXPECT_TRUE(ams("musl-linux-armhf"));
+      SCOPED_TRACE("Pattern is *");
+      // * should be treated like any
+      APT::CacheFilter::PackageArchitectureMatchesSpecification ams("*");
+      EXPECT_TRUE(ams("sparc"));
+      EXPECT_TRUE(ams("amd64"));
+      EXPECT_TRUE(ams("kfreebsd-amd64"));
+   }
+   {
+      SCOPED_TRACE("Pattern is any-i386");
+      APT::CacheFilter::PackageArchitectureMatchesSpecification ams("any-i386");
+      EXPECT_TRUE(ams("i386"));
+      EXPECT_FALSE(ams("amd64"));
+      EXPECT_TRUE(ams("linux-i386"));
+      EXPECT_FALSE(ams("linux-amd64"));
+      EXPECT_TRUE(ams("kfreebsd-i386"));
+      EXPECT_TRUE(ams("musl-linux-i386"));
    }
    {
       SCOPED_TRACE("Pattern is linux-any");
@@ -29,11 +35,9 @@ TEST(CacheFilterTest, ArchitectureSpecification)
       EXPECT_TRUE(ams("linux-armhf"));
       EXPECT_TRUE(ams("linux-armel"));
       EXPECT_FALSE(ams("kfreebsd-armhf"));
-      EXPECT_TRUE(ams("gnu-linux-armhf"));
-      EXPECT_TRUE(ams("gnu-linux-armel"));
-      EXPECT_FALSE(ams("gnu-kfreebsd-armhf"));
       EXPECT_TRUE(ams("musl-linux-armhf"));
    }
+   if (FileExists(DPKG_DATADIR "/tupletable"))
    {
       SCOPED_TRACE("Pattern is gnu-any-any");
       APT::CacheFilter::PackageArchitectureMatchesSpecification ams("gnu-any-any"); //really?
@@ -42,11 +46,32 @@ TEST(CacheFilterTest, ArchitectureSpecification)
       EXPECT_TRUE(ams("linux-armhf"));
       EXPECT_TRUE(ams("linux-armel"));
       EXPECT_TRUE(ams("kfreebsd-armhf"));
-      EXPECT_TRUE(ams("gnu-linux-armhf"));
-      EXPECT_TRUE(ams("gnu-linux-armel"));
-      EXPECT_TRUE(ams("gnu-kfreebsd-armhf"));
       EXPECT_FALSE(ams("musl-linux-armhf"));
    }
+   if (FileExists(DPKG_DATADIR "/triplettable"))
+   {
+      SCOPED_TRACE("Pattern is gnueabi-any-any");
+      APT::CacheFilter::PackageArchitectureMatchesSpecification ams("gnueabi-any-any"); //really?
+      EXPECT_TRUE(ams("linux-armel"));
+      EXPECT_TRUE(ams("armel"));
+      EXPECT_FALSE(ams("armhf"));
+      EXPECT_FALSE(ams("linux-armhf"));
+      EXPECT_FALSE(ams("musleabihf-linux-armhf"));
+   }
+   if (FileExists(DPKG_DATADIR "/triplettable"))
+   {
+      SCOPED_TRACE("Pattern is gnueabihf-any-any");
+      APT::CacheFilter::PackageArchitectureMatchesSpecification ams("gnueabihf-any-any"); //really?
+      EXPECT_FALSE(ams("linux-armel"));
+      EXPECT_FALSE(ams("armel"));
+      EXPECT_TRUE(ams("armhf"));
+      EXPECT_TRUE(ams("linux-armhf"));
+      EXPECT_FALSE(ams("musleabihf-linux-armhf"));
+   }
+
+   // Weird ones - armhf's tuple is actually eabihf-gnu-linux-arm
+   //              armel's tuple is actually eabi-gnu-linux-arm
+   //              x32's   tuple is actually x32-gnu-linux-amd64
    {
       SCOPED_TRACE("Architecture is armhf");
       APT::CacheFilter::PackageArchitectureMatchesSpecification ams("armhf", false);
@@ -54,13 +79,41 @@ TEST(CacheFilterTest, ArchitectureSpecification)
       EXPECT_FALSE(ams("armel"));
       EXPECT_TRUE(ams("linux-any"));
       EXPECT_FALSE(ams("kfreebsd-any"));
-      EXPECT_TRUE(ams("any-armhf"));
-      EXPECT_FALSE(ams("any-armel"));
+      EXPECT_TRUE(ams("any-arm"));
       EXPECT_TRUE(ams("linux-armhf"));
       EXPECT_FALSE(ams("kfreebsd-armhf"));
-      EXPECT_TRUE(ams("gnu-linux-armhf"));
-      EXPECT_FALSE(ams("gnu-linux-armel"));
-      EXPECT_FALSE(ams("gnu-kfreebsd-armhf"));
       EXPECT_FALSE(ams("musl-linux-armhf"));
    }
+   {
+      SCOPED_TRACE("Pattern is any-arm");
+      APT::CacheFilter::PackageArchitectureMatchesSpecification ams("any-arm");
+      EXPECT_TRUE(ams("armhf"));
+      EXPECT_TRUE(ams("armel"));
+      EXPECT_TRUE(ams("linux-armhf"));
+      EXPECT_TRUE(ams("linux-armel"));
+      EXPECT_TRUE(ams("musl-linux-armhf"));
+      EXPECT_TRUE(ams("uclibc-linux-armel"));
+
+      EXPECT_FALSE(ams("arm64"));
+      EXPECT_FALSE(ams("linux-arm64"));
+      EXPECT_FALSE(ams("kfreebsd-arm64"));
+      EXPECT_FALSE(ams("musl-linux-arm64"));
+   }
+   {
+      SCOPED_TRACE("Pattern is any-amd64");
+      APT::CacheFilter::PackageArchitectureMatchesSpecification ams("any-amd64");
+      EXPECT_TRUE(ams("amd64"));
+      EXPECT_TRUE(ams("x32"));
+      EXPECT_TRUE(ams("linux-amd64"));
+      EXPECT_TRUE(ams("linux-x32"));
+      EXPECT_TRUE(ams("kfreebsd-amd64"));
+      EXPECT_TRUE(ams("musl-linux-amd64"));
+      EXPECT_TRUE(ams("uclibc-linux-amd64"));
+
+      EXPECT_FALSE(ams("i386"));
+      EXPECT_FALSE(ams("linux-i386"));
+      EXPECT_FALSE(ams("kfreebsd-i386"));
+      EXPECT_FALSE(ams("musl-linux-i386"));
+      EXPECT_FALSE(ams("uclibc-linux-i386"));
+   }
 }