allow individual targets to be kept compressed

There is an option to keep all targets (Packages, Sources, …) compressed
for a while now, but the all-or-nothing approach is a bit limited for
our purposes with additional targets as some of them are very big
(Contents) and rarely used in comparison, so keeping them compressed by
default can make sense, while others are still unpacked.

Most interesting is the copy-change maybe: Copy is used by the acquire
system as an uncompressor and it is hence expected that it returns the
hashes for the "output", not the input. Now, in the case of keeping a
file compressed, the output is never written to disk, but generated in
memory and we should still validated it, so for compressed files copy is
expected to return the hashes of the uncompressed file. We used to use
the config option to enable on-the-fly decompress in the method, but in
reality copy is never used in a way where it shouldn't decompress a
compressed file to get its hashes, so we can save us the trouble of
sending this information to the method and just do it always.

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 01a679fe01c92d425ee0e17f2936b6c190c88be6..b6466115e4751daf8b903b9acb67c6ed174dce84 100644 (file)
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -80,18 +80,18 @@ static std::string GetFinalFileNameFromURI(std::string const &uri)  /*{{{*/
    return _config->FindDir("Dir::State::lists") + URItoFileName(uri);
 }
                                                                        /*}}}*/
-static std::string GetCompressedFileName(std::string const &URI, std::string const &Name, std::string const &Ext) /*{{{*/
+static std::string GetCompressedFileName(IndexTarget const &Target, std::string const &Name, std::string const &Ext) /*{{{*/
 {
    if (Ext.empty() || Ext == "uncompressed")
       return Name;
 
    // do not reverify cdrom sources as apt-cdrom may rewrite the Packages
    // file when its doing the indexcopy
-   if (URI.substr(0,6) == "cdrom:")
+   if (Target.URI.substr(0,6) == "cdrom:")
       return Name;
 
    // adjust DestFile if its compressed on disk
-   if (_config->FindB("Acquire::GzipIndexes",false) == true)
+   if (Target.KeepCompressed == true)
       return Name + '.' + Ext;
    return Name;
 }
@@ -269,7 +269,7 @@ std::string pkgAcqDiffIndex::GetFinalFilename() const
 std::string pkgAcqIndex::GetFinalFilename() const
 {
    std::string const FinalFile = GetFinalFileNameFromURI(Target.URI);
-   return GetCompressedFileName(Target.URI, FinalFile, CurrentCompressionExtension);
+   return GetCompressedFileName(Target, FinalFile, CurrentCompressionExtension);
 }
 std::string pkgAcqMetaSig::GetFinalFilename() const
 {
@@ -2456,7 +2456,7 @@ void pkgAcqIndex::ReverifyAfterIMS()
 {
    // update destfile to *not* include the compression extension when doing
    // a reverify (as its uncompressed on disk already)
-   DestFile = GetCompressedFileName(Target.URI, GetPartialFileNameFromURI(Target.URI), CurrentCompressionExtension);
+   DestFile = GetCompressedFileName(Target, GetPartialFileNameFromURI(Target.URI), CurrentCompressionExtension);
 
    // copy FinalFile into partial/ so that we check the hash again
    string FinalFile = GetFinalFilename();
@@ -2532,8 +2532,8 @@ void pkgAcqIndex::StageDownloadDone(string const &Message, HashStringList const
       return;
    }
 
-   // If we have compressed indexes enabled, queue for hash verification
-   if (_config->FindB("Acquire::GzipIndexes",false))
+   // If we want compressed indexes, just copy in place for hash verification
+   if (Target.KeepCompressed == true)
    {
       DestFile = GetPartialFileNameFromURI(Target.URI + '.' + CurrentCompressionExtension);
       EraseFileName = "";

diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index 4bb03a9420a502c09cd63221a7951d5c4d8982c7..10ab0f8443c4d417451c7a8c7ff605bf65685358 100644 (file)
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -123,6 +123,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
    std::string const Release = (Dist == "/") ? "" : Dist;
    std::string const Site = ::URI::ArchiveOnly(URI);
 
+   bool const GzipIndex = _config->FindB("Acquire::GzipIndexes", false);
    for (std::vector<debReleaseIndexPrivate::debSectionEntry>::const_iterator E = entries.begin(); E != entries.end(); ++E)
    {
       for (std::vector<std::string>::const_iterator T = E->Targets.begin(); T != E->Targets.end(); ++T)
@@ -131,7 +132,8 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
         std::string const tplMetaKey = APT_T_CONFIG(flatArchive ? "flatMetaKey" : "MetaKey");
         std::string const tplShortDesc = APT_T_CONFIG("ShortDescription");
         std::string const tplLongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description");
-        bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb-src::") + *T + "::Optional", true);
+        bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::") + Type + "::" + *T + "::Optional", true);
+        bool const KeepCompressed = _config->FindB(std::string("APT::Acquire::Targets::") + Type + "::" + *T + "::KeepCompressed", GzipIndex);
 #undef APT_T_CONFIG
         if (tplMetaKey.empty())
            continue;
@@ -173,6 +175,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
                     LongDesc,
                     Options.find("BASE_URI")->second + MetaKey,
                     IsOptional,
+                    KeepCompressed,
                     Options
                     );
               IndexTargets.push_back(Target);
@@ -413,7 +416,7 @@ bool debReleaseIndex::parseSumData(const char *&Start, const char *End,     /*{{{*/
 bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll)/*{{{*/
 {
    std::vector<IndexTarget> const targets = GetIndexTargets();
-#define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X), false, std::map<std::string,std::string>())
+#define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X), false, false, std::map<std::string,std::string>())
    pkgAcqMetaClearSig * const TransactionManager = new pkgAcqMetaClearSig(Owner,
         APT_TARGET("InRelease"), APT_TARGET("Release"), APT_TARGET("Release.gpg"),
         targets, this);

diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc
index e9e1b08c381187adb7c816922238f599fe056bc5..cce17403db97d9ce8cc472a8165f85a7468d2894 100644 (file)
--- a/apt-pkg/indexfile.cc
+++ b/apt-pkg/indexfile.cc
@@ -115,8 +115,9 @@ APT_DEPRECATED std::string pkgIndexFile::LanguageCode() {
 // IndexTarget - Constructor                                           /*{{{*/
 IndexTarget::IndexTarget(std::string const &MetaKey, std::string const &ShortDesc,
       std::string const &LongDesc, std::string const &URI, bool const IsOptional,
-      std::map<std::string, std::string> const &Options) :
-   URI(URI), Description(LongDesc), ShortDesc(ShortDesc), MetaKey(MetaKey), IsOptional(IsOptional), Options(Options)
+      bool const KeepCompressed, std::map<std::string, std::string> const &Options) :
+   URI(URI), Description(LongDesc), ShortDesc(ShortDesc), MetaKey(MetaKey),
+   IsOptional(IsOptional), KeepCompressed(KeepCompressed), Options(Options)
 {
 }
                                                                        /*}}}*/

diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h
index 7eeccdbd357cdc5eb0232295f8a17959ee45e875..44d39110d2a6504961e662c06666cf0427895749 100644 (file)
--- a/apt-pkg/indexfile.h
+++ b/apt-pkg/indexfile.h
@@ -60,6 +60,9 @@ class IndexTarget                                                     /*{{{*/
    /** \brief Is it okay if the file isn't found in the meta index */
    bool IsOptional;
 
+   /** \brief If the file is downloaded compressed, do not unpack it */
+   bool KeepCompressed;
+
    /** \brief options with which this target was created
        Prefer the usage of #Option if at all possible.
        Beware: Not all of these options are intended for public use */
@@ -67,7 +70,7 @@ class IndexTarget                                                     /*{{{*/
 
    IndexTarget(std::string const &MetaKey, std::string const &ShortDesc,
         std::string const &LongDesc, std::string const &URI, bool const IsOptional,
-        std::map<std::string, std::string> const &Options);
+        bool const KeepCompressed, std::map<std::string, std::string> const &Options);
 
    enum OptionKeys {
       SITE,

diff --git a/methods/copy.cc b/methods/copy.cc
index a8e289df57a71495ecace461ab0e85381f2bdcbd..4bdc15f754012b933d3323880131e751e887e784 100644 (file)
--- a/methods/copy.cc
+++ b/methods/copy.cc
@@ -38,11 +38,7 @@ class CopyMethod : public pkgAcqMethod
 void CopyMethod::CalculateHashes(FetchItem const * const Itm, FetchResult &Res)
 {
    Hashes Hash(Itm->ExpectedHashes);
-   FileFd::CompressMode CompressMode = FileFd::None;
-   if (_config->FindB("Acquire::GzipIndexes", false) == true)
-      CompressMode = FileFd::Extension;
-
-   FileFd Fd(Res.Filename, FileFd::ReadOnly, CompressMode);
+   FileFd Fd(Res.Filename, FileFd::ReadOnly, FileFd::Extension);
    Hash.AddFD(Fd);
    Res.TakeHashes(Hash);
 }
@@ -53,7 +49,7 @@ void CopyMethod::CalculateHashes(FetchItem const * const Itm, FetchResult &Res)
 bool CopyMethod::Fetch(FetchItem *Itm)
 {
    // this ensures that relative paths work in copy
-   std::string File = Itm->Uri.substr(Itm->Uri.find(':')+1);
+   std::string const File = Itm->Uri.substr(Itm->Uri.find(':')+1);
 
    // Stat the file and send a start message
    struct stat Buf;

diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files
index 3465c0a1631e4e28a3f25293241f6f068e8647ae..fecdf30bfcb5115b390ab209d3922b6b65cbdbb0 100755 (executable)
--- a/test/integration/test-apt-acquire-additional-files
+++ b/test/integration/test-apt-acquire-additional-files
@@ -57,7 +57,23 @@ testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents
 testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64)" aptget files --format '$(FILENAME)' 'Created-By: Contents'
 testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64' 'aptarchive/dists/unstable/main/Contents-amd64'
 
-# no automatic uncompress based on the name please,
+rm ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64
+testempty aptget files --format '$(FILENAME)' 'Created-By: Contents'
+
+# if we asked for keeping it compressed, keep it
+echo 'APT::Acquire::Targets::deb::Contents::KeepCompressed "true";' >> rootdir/etc/apt/apt.conf.d/content-target.conf
+testsuccessequal "Hit:1 http://localhost:8080 unstable InRelease
+Get:2 http://localhost:8080 unstable/main amd64 Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
+Reading package lists..." aptget update
+
+testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' find rootdir/var/lib/apt/lists -name '*Contents*'
+testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz)" aptget files --format '$(FILENAME)' 'Created-By: Contents'
+testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' 'aptarchive/dists/unstable/main/Contents-amd64.gz'
+
+rm ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz
+testempty aptget files --format '$(FILENAME)' 'Created-By: Contents'
+
+# and no automatic uncompress based on the name please,
 # only if we downloaded a compressed file, but target was uncompressed
 cat > rootdir/etc/apt/apt.conf.d/content-target.conf <<EOF
 APT::Acquire::Targets::deb::Contents {

diff --git a/test/integration/test-compressed-indexes b/test/integration/test-compressed-indexes
index c6b292baa52a85d8ddf38d3683578149f790d0dc..b5d6b8ae4cbb314f7a77ce5a094f8089fcca1a44 100755 (executable)
--- a/test/integration/test-compressed-indexes
+++ b/test/integration/test-compressed-indexes
@@ -60,7 +60,7 @@ testrun() {
        msgtest 'Check if package is downloadable'
        cd downloaded
        testsuccess --nomsg aptget download testpkg
-       msgtest '\tdeb file is present'; testsuccess --nomsg test -f testpkg_1.0_i386.deb
+       msgtest 'deb file is present'; testsuccess --nomsg test -f testpkg_1.0_i386.deb
        rm -f testpkg_1.0_i386.deb
        cd - >/dev/null
        testsuccessequal 'Reading package lists...