]> git.saurik.com Git - apt.git/commitdiff
keep compressed indexes in a low-cost format
authorDavid Kalnischkies <david@kalnischkies.de>
Thu, 7 Jan 2016 19:32:09 +0000 (20:32 +0100)
committerDavid Kalnischkies <david@kalnischkies.de>
Fri, 8 Jan 2016 14:40:01 +0000 (15:40 +0100)
Downloading and storing are two different operations were different
compression types can be preferred. For downloading we provide the
choice via Acquire::CompressionTypes::Order as there is a choice to
be made between download size and speed – and limited by whats available
in the repository.

Storage on the other hand has all compressions currently supported by
apt available and to reduce runtime of tools accessing these files the
compression type should be a low-cost format in terms of decompression.

apt traditionally stores its indexes uncompressed on disk, but has
options to keep them compressed. Now that apt downloads additional files
we also deal with files which simply can't be stored uncompressed as
they are just too big (like Contents for apt-file). Traditionally they
are downloaded in a low-cost format (gz) as repositories do not provide
other formats, but there might be even lower-cost formats and for
download we could introduce higher-cost in the repositories.

Downloading an entire index potentially requires recompression to
another format, so an update takes potentially longer – but big files
are usually updated via pdiffs which has to de- and re-compress anyhow
and does it on the fly anyhow, so there is no extra time needed and in
general it seems to be benefitial to invest the time in update to save
time later on file access.

14 files changed:
apt-pkg/acquire-item.cc
apt-pkg/acquire-item.h
apt-pkg/aptconfiguration.cc
apt-pkg/deb/debmetaindex.cc
apt-pkg/indexfile.cc
apt-pkg/indexfile.h
cmdline/apt-get.cc
doc/acquire-additional-files.txt
test/integration/framework
test/integration/test-apt-acquire-additional-files
test/integration/test-apt-update-ims
test/integration/test-apt-update-unauth
test/integration/test-compressed-indexes
test/integration/test-pdiff-usage

index 82ccaff64ebf59adfadb2c9ce48ccd9014560a1c..7c7a204c46a6ef8a0f80aee9fe268d1e1c1367e5 100644 (file)
@@ -85,32 +85,16 @@ static std::string GetKeepCompressedFileName(std::string file, IndexTarget const
    if (Target.KeepCompressed == false)
       return file;
 
-   std::string const CompressionTypes = Target.Option(IndexTarget::COMPRESSIONTYPES);
-   if (CompressionTypes.empty() == false)
+   std::string const KeepCompressedAs = Target.Option(IndexTarget::KEEPCOMPRESSEDAS);
+   if (KeepCompressedAs.empty() == false)
    {
-      std::string const ext = CompressionTypes.substr(0, CompressionTypes.find(' '));
+      std::string const ext = KeepCompressedAs.substr(0, KeepCompressedAs.find(' '));
       if (ext != "uncompressed")
         file.append(".").append(ext);
    }
    return file;
 }
                                                                        /*}}}*/
-static std::string GetCompressedFileName(IndexTarget const &Target, std::string const &Name, std::string const &Ext) /*{{{*/
-{
-   if (Ext.empty() || Ext == "uncompressed")
-      return Name;
-
-   // do not reverify cdrom sources as apt-cdrom may rewrite the Packages
-   // file when its doing the indexcopy
-   if (Target.URI.substr(0,6) == "cdrom:")
-      return Name;
-
-   // adjust DestFile if its compressed on disk
-   if (Target.KeepCompressed == true)
-      return Name + '.' + Ext;
-   return Name;
-}
-                                                                       /*}}}*/
 static std::string GetMergeDiffsPatchFileName(std::string const &Final, std::string const &Patch)/*{{{*/
 {
    // rred expects the patch as $FinalFile.ed.$patchname.gz
@@ -326,7 +310,7 @@ std::string pkgAcqDiffIndex::GetFinalFilename() const
 std::string pkgAcqIndex::GetFinalFilename() const
 {
    std::string const FinalFile = GetFinalFileNameFromURI(Target.URI);
-   return GetCompressedFileName(Target, FinalFile, CurrentCompressionExtension);
+   return GetKeepCompressedFileName(FinalFile, Target);
 }
 std::string pkgAcqMetaSig::GetFinalFilename() const
 {
@@ -379,7 +363,30 @@ bool pkgAcqTransactionItem::TransactionState(TransactionStates const state)
       case TransactionCommit:
         if(PartialFile.empty() == false)
         {
-           if (PartialFile != DestFile)
+           bool sameFile = (PartialFile == DestFile);
+           // we use symlinks on IMS-Hit to avoid copies
+           if (RealFileExists(DestFile))
+           {
+              struct stat Buf;
+              if (lstat(PartialFile.c_str(), &Buf) != -1)
+              {
+                 if (S_ISLNK(Buf.st_mode) && Buf.st_size > 0)
+                 {
+                    char partial[Buf.st_size + 1];
+                    ssize_t const sp = readlink(PartialFile.c_str(), partial, Buf.st_size);
+                    if (sp == -1)
+                       _error->Errno("pkgAcqTransactionItem::TransactionState-sp", _("Failed to readlink %s"), PartialFile.c_str());
+                    else
+                    {
+                       partial[sp] = '\0';
+                       sameFile = (DestFile == partial);
+                    }
+                 }
+              }
+              else
+                 _error->Errno("pkgAcqTransactionItem::TransactionState-stat", _("Failed to stat %s"), PartialFile.c_str());
+           }
+           if (sameFile == false)
            {
               // ensure that even without lists-cleanup all compressions are nuked
               std::string FinalFile = GetFinalFileNameFromURI(Target.URI);
@@ -2621,20 +2628,6 @@ void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const *
       TransactionManager->AbortTransaction();
 }
                                                                        /*}}}*/
-// AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit                /*{{{*/
-void pkgAcqIndex::ReverifyAfterIMS()
-{
-   // update destfile to *not* include the compression extension when doing
-   // a reverify (as its uncompressed on disk already)
-   DestFile = GetCompressedFileName(Target, GetPartialFileNameFromURI(Target.URI), CurrentCompressionExtension);
-
-   // copy FinalFile into partial/ so that we check the hash again
-   string FinalFile = GetFinalFilename();
-   Stage = STAGE_DECOMPRESS_AND_VERIFY;
-   Desc.URI = "copy:" + FinalFile;
-   QueueURI(Desc);
-}
-                                                                       /*}}}*/
 // AcqIndex::Done - Finished a fetch                                   /*{{{*/
 // ---------------------------------------------------------------------
 /* This goes through a number of states.. On the initial fetch the
@@ -2651,107 +2644,93 @@ void pkgAcqIndex::Done(string const &Message,
    switch(Stage) 
    {
       case STAGE_DOWNLOAD:
-         StageDownloadDone(Message, Hashes, Cfg);
+         StageDownloadDone(Message);
          break;
       case STAGE_DECOMPRESS_AND_VERIFY:
-         StageDecompressDone(Message, Hashes, Cfg);
+         StageDecompressDone();
          break;
    }
 }
                                                                        /*}}}*/
 // AcqIndex::StageDownloadDone - Queue for decompress and verify       /*{{{*/
-void pkgAcqIndex::StageDownloadDone(string const &Message, HashStringList const &,
-                                    pkgAcquire::MethodConfig const * const)
+void pkgAcqIndex::StageDownloadDone(string const &Message)
 {
+   Local = true;
    Complete = true;
 
-   // Handle the unzipd case
-   std::string FileName = LookupTag(Message,"Alt-Filename");
-   if (FileName.empty() == false)
+   std::string const AltFilename = LookupTag(Message,"Alt-Filename");
+   std::string Filename = LookupTag(Message,"Filename");
+
+   // we need to verify the file against the current Release file again
+   // on if-modfied-since hit to avoid a stale attack against us
+   if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
    {
+      // copy FinalFile into partial/ so that we check the hash again
+      string const FinalFile = GetExistingFilename(GetFinalFileNameFromURI(Target.URI));
+      if (symlink(FinalFile.c_str(), DestFile.c_str()) != 0)
+        _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking final file %s back to %s failed", FinalFile.c_str(), DestFile.c_str());
+      else
+      {
+        EraseFileName = DestFile;
+        Filename = DestFile;
+      }
       Stage = STAGE_DECOMPRESS_AND_VERIFY;
-      Local = true;
-      if (CurrentCompressionExtension != "uncompressed")
-        DestFile.erase(DestFile.length() - (CurrentCompressionExtension.length() + 1));
-      Desc.URI = "copy:" + FileName;
+      Desc.URI = "store:" + Filename;
       QueueURI(Desc);
-      SetActiveSubprocess("copy");
+      SetActiveSubprocess(::URI(Desc.URI).Access);
       return;
    }
-   FileName = LookupTag(Message,"Filename");
-
+   // methods like file:// give us an alternative (uncompressed) file
+   else if (Target.KeepCompressed == false && AltFilename.empty() == false)
+   {
+      if (CurrentCompressionExtension != "uncompressed")
+        DestFile.erase(DestFile.length() - (CurrentCompressionExtension.length() + 1));
+      Filename = AltFilename;
+   }
    // Methods like e.g. "file:" will give us a (compressed) FileName that is
    // not the "DestFile" we set, in this case we uncompress from the local file
-   if (FileName != DestFile && RealFileExists(DestFile) == false)
+   else if (Filename != DestFile && RealFileExists(DestFile) == false)
    {
-      Local = true;
-      if (Target.KeepCompressed == true)
-      {
-        // but if we don't keep the uncompress we copy the compressed file first
-        Stage = STAGE_DOWNLOAD;
-        Desc.URI = "copy:" + FileName;
-        QueueURI(Desc);
-        SetActiveSubprocess("copy");
-        return;
-      }
+      // symlinking ensures that the filename can be used for compression detection
+      // that is e.g. needed for by-hash which has no extension over file
+      if (symlink(Filename.c_str(),DestFile.c_str()) != 0)
+        _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", Filename.c_str(), DestFile.c_str());
       else
       {
-        // symlinking ensures that the filename can be used for compression detection
-        // that is e.g. needed for by-hash over file
-        if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
-           _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", FileName.c_str(), DestFile.c_str());
-        else
-        {
-           EraseFileName = DestFile;
-           FileName = DestFile;
-        }
+        EraseFileName = DestFile;
+        Filename = DestFile;
       }
    }
-   else
-      EraseFileName = FileName;
-
-   // we need to verify the file against the current Release file again
-   // on if-modfied-since hit to avoid a stale attack against us
-   if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
-   {
-      // The files timestamp matches, reverify by copy into partial/
-      EraseFileName = "";
-      ReverifyAfterIMS();
-      return;
-   }
 
-   string decompProg = "store";
-   if (Target.KeepCompressed == true)
-   {
-      DestFile = "/dev/null";
-      EraseFileName.clear();
-   }
+   Stage = STAGE_DECOMPRESS_AND_VERIFY;
+   DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
+   if (Filename != DestFile && flExtension(Filename) == flExtension(DestFile))
+      Desc.URI = "copy:" + Filename;
    else
+      Desc.URI = "store:" + Filename;
+   if (DestFile == Filename)
    {
       if (CurrentCompressionExtension == "uncompressed")
-        decompProg = "copy";
-      else
-        DestFile.erase(DestFile.length() - (CurrentCompressionExtension.length() + 1));
+        return StageDecompressDone();
+      DestFile = "/dev/null";
    }
 
+   if (EraseFileName.empty())
+      EraseFileName = Filename;
+
    // queue uri for the next stage
-   Stage = STAGE_DECOMPRESS_AND_VERIFY;
-   Desc.URI = decompProg + ":" + FileName;
    QueueURI(Desc);
-   SetActiveSubprocess(decompProg);
+   SetActiveSubprocess(::URI(Desc.URI).Access);
 }
                                                                        /*}}}*/
 // AcqIndex::StageDecompressDone - Final verification                  /*{{{*/
-void pkgAcqIndex::StageDecompressDone(string const &,
-                                      HashStringList const &,
-                                      pkgAcquire::MethodConfig const * const)
+void pkgAcqIndex::StageDecompressDone()
 {
-   if (Target.KeepCompressed == true && DestFile == "/dev/null")
-      DestFile = GetPartialFileNameFromURI(Target.URI + '.' + CurrentCompressionExtension);
+   if (DestFile == "/dev/null")
+      DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target);
 
    // Done, queue for rename on transaction finished
    TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
-   return;
 }
                                                                        /*}}}*/
 pkgAcqIndex::~pkgAcqIndex() {}
index 93ff0a9c3cd68f86853222a024d5e617f14ed61a..306b1772a77be02c52d99bb7e9c3afe7ce595267 100644 (file)
@@ -914,16 +914,12 @@ class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex
    AllStages Stage;
 
    /** \brief Handle what needs to be done when the download is done */
-   void StageDownloadDone(std::string const &Message,
-                          HashStringList const &Hashes,
-                          pkgAcquire::MethodConfig const * const Cfg);
+   void StageDownloadDone(std::string const &Message);
 
    /** \brief Handle what needs to be done when the decompression/copy is
     *         done 
     */
-   void StageDecompressDone(std::string const &Message,
-                            HashStringList const &Hashes,
-                            pkgAcquire::MethodConfig const * const Cfg);
+   void StageDecompressDone();
 
    /** \brief If \b set, this partially downloaded file will be
     *  removed when the download completes.
@@ -941,9 +937,6 @@ class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex
    /** \brief Do the changes needed to fetch via AptByHash (if needed) */
    void InitByHashIfNeeded();
 
-   /** \brief Schedule file for verification after a IMS hit */
-   void ReverifyAfterIMS();
-
    /** \brief Get the full pathname of the final file for the current URI */
    virtual std::string GetFinalFilename() const APT_OVERRIDE;
 
index 53f29df799d4f9effd99905e9954bf5ac535092a..01ae5a64240d0b0ec28e2001a9e813d16de729e0 100644 (file)
@@ -37,6 +37,7 @@ static void setDefaultConfigurationForCompressors() {
        // Set default application paths to check for optional compression types
        _config->CndSet("Dir::Bin::bzip2", "/bin/bzip2");
        _config->CndSet("Dir::Bin::xz", "/usr/bin/xz");
+       _config->CndSet("Dir::Bin::lz4", "/usr/bin/lz4");
        if (FileExists(_config->FindFile("Dir::Bin::xz")) == true) {
                _config->Set("Dir::Bin::lzma", _config->FindFile("Dir::Bin::xz"));
                _config->Set("APT::Compressor::lzma::Binary", "xz");
@@ -79,6 +80,7 @@ const Configuration::getCompressionTypes(bool const &Cached) {
        _config->CndSet("Acquire::CompressionTypes::bz2","bzip2");
        _config->CndSet("Acquire::CompressionTypes::lzma","lzma");
        _config->CndSet("Acquire::CompressionTypes::gz","gzip");
+       _config->CndSet("Acquire::CompressionTypes::lz4","lz4");
 
        setDefaultConfigurationForCompressors();
        std::vector<APT::Configuration::Compressor> const compressors = getCompressors();
index 46a9f9c9740aa56355d53aedcafaf04b396529bb..2521fccd676ff4d52d0c6ca1ece402452bf96578 100644 (file)
@@ -138,6 +138,21 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
         DefCompressionTypes = os.str();
       }
    }
+   std::string DefKeepCompressedAs;
+   {
+      std::vector<APT::Configuration::Compressor> comps = APT::Configuration::getCompressors();
+      if (comps.empty() == false)
+      {
+        std::sort(comps.begin(), comps.end(),
+              [](APT::Configuration::Compressor const &a, APT::Configuration::Compressor const &b) { return a.Cost < b.Cost; });
+        std::ostringstream os;
+        for (auto const &c : comps)
+           if (c.Cost != 0)
+              os << c.Extension.substr(1) << ' ';
+        DefKeepCompressedAs = os.str();
+      }
+      DefKeepCompressedAs += "uncompressed";
+   }
    std::string const NativeArch = _config->Find("APT::Architecture");
    bool const GzipIndex = _config->FindB("Acquire::GzipIndexes", false);
    for (std::vector<debReleaseIndexPrivate::debSectionEntry>::const_iterator E = entries.begin(); E != entries.end(); ++E)
@@ -155,11 +170,35 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
         bool const UsePDiffs = APT_T_CONFIG_BOOL("PDiffs", E->UsePDiffs);
         std::string const UseByHash = APT_T_CONFIG_STR("By-Hash", E->UseByHash);
         std::string const CompressionTypes = APT_T_CONFIG_STR("CompressionTypes", DefCompressionTypes);
+        std::string KeepCompressedAs = APT_T_CONFIG_STR("KeepCompressedAs", "");
 #undef APT_T_CONFIG_BOOL
 #undef APT_T_CONFIG_STR
         if (tplMetaKey.empty())
            continue;
 
+        if (KeepCompressedAs.empty())
+           KeepCompressedAs = DefKeepCompressedAs;
+        else
+        {
+           std::vector<std::string> const defKeep = VectorizeString(DefKeepCompressedAs, ' ');
+           std::vector<std::string> const valKeep = VectorizeString(KeepCompressedAs, ' ');
+           std::vector<std::string> keep;
+           for (auto const &val : valKeep)
+           {
+              if (val.empty())
+                 continue;
+              if (std::find(defKeep.begin(), defKeep.end(), val) == defKeep.end())
+                 continue;
+              keep.push_back(val);
+           }
+           if (std::find(keep.begin(), keep.end(), "uncompressed") == keep.end())
+              keep.push_back("uncompressed");
+           std::ostringstream os;
+           std::copy(keep.begin(), keep.end()-1, std::ostream_iterator<std::string>(os, " "));
+           os << *keep.rbegin();
+           KeepCompressedAs = os.str();
+        }
+
         for (std::vector<std::string>::const_iterator L = E->Languages.begin(); L != E->Languages.end(); ++L)
         {
            if (*L == "none" && tplMetaKey.find("$(LANGUAGE)") != std::string::npos)
@@ -253,6 +292,7 @@ static void GetIndexTargetsFor(char const * const Type, std::string const &URI,
               Options.insert(std::make_pair("BY_HASH", UseByHash));
               Options.insert(std::make_pair("DEFAULTENABLED", DefaultEnabled ? "yes" : "no"));
               Options.insert(std::make_pair("COMPRESSIONTYPES", CompressionTypes));
+              Options.insert(std::make_pair("KEEPCOMPRESSEDAS", KeepCompressedAs));
               Options.insert(std::make_pair("SOURCESENTRY", E->sourcesEntry));
 
               bool IsOpt = IsOptional;
index f0fae907f5cb9f7180c4957d13b1e31279d555da..dfd482c04d09544704012cac04c3d7cb083dbdc4 100644 (file)
@@ -144,6 +144,7 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const             /*{{{*/
       APT_CASE(COMPRESSIONTYPES);
       APT_CASE(SOURCESENTRY);
       APT_CASE(BY_HASH);
+      APT_CASE(KEEPCOMPRESSEDAS);
 #undef APT_CASE
       case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
       case EXISTING_FILENAME:
index f87693a8ff2d953eef30161e2e0be1e5134aa7de..5c666671c6fc796583f3e47b200765a2ef5def9e 100644 (file)
@@ -90,6 +90,7 @@ class IndexTarget                                                     /*{{{*/
       DEFAULTENABLED,
       SOURCESENTRY,
       BY_HASH,
+      KEEPCOMPRESSEDAS,
    };
    std::string Option(OptionKeys const Key) const;
    bool OptionBool(OptionKeys const Key) const;
index 3bd4834260eac6f838e4477bf888f09a60df6e45..4ac15274cd4ff1946a665e4e41ad9424202b1f2f 100644 (file)
@@ -299,6 +299,8 @@ static bool DoIndexTargets(CommandLine &CmdL)
                  stanza << "PDiffs: " << O->second << "\n";
               else if (O->first == "COMPRESSIONTYPES")
                  stanza << "CompressionTypes: " << O->second << "\n";
+              else if (O->first == "KEEPCOMPRESSEDAS")
+                 stanza << "KeepCompressedAs: " << O->second << "\n";
               else if (O->first == "DEFAULTENABLED")
                  stanza << "DefaultEnabled: " << O->second << "\n";
               else
index 32b0e4470b144c837bf865eac165d5138bb9697f..68af9a5b03c2dad6f376453b37ac61e296ec49cd 100644 (file)
@@ -124,6 +124,10 @@ aren't accidentally used by front-ends:
   type a front-end can't open transparently. This should always be
   a temporary workaround through and a bug should be reported against
   the front-end in question.
+* KeepCompressedAs: The default value is a space separated list of
+  compression types supported by apt (see previous option) which is
+  sorted by the cost-value of the compression in ascending order,
+  except that cost=0 "compressions" (like uncompressed) are listed last.
 
 
 # More examples
index e2d545ec232b7831a753786a32322b3e5abf6da2..c97500ffd0bf1ff27780f2bbbe7d56693cecad61 100644 (file)
@@ -1826,7 +1826,7 @@ listcurrentlistsdirectory() {
 }
 forallsupportedcompressors() {
        for COMP in $(aptconfig dump 'APT::Compressor' --format '%f%n' | cut -d':' -f 5 | uniq); do
-               if [ -z "$COMP" -o "$COMP" = '.' -o "$COMP" = 'lz4' ]; then continue; fi
+               if [ -z "$COMP" -o "$COMP" = '.' ]; then continue; fi
                "$@" "$COMP"
        done
 }
index e9bbdbce6567d3533f35320185697ebdfc28485b..bf7850a7ad25815aa61d2bdf7c0bfc85d6ec7dcb 100755 (executable)
@@ -10,6 +10,7 @@ configarchitecture 'amd64'
 # note that in --print-uri we talk about .xz because that is the default.
 # This doesn't mean it is actually attempt to download it.
 configcompression '.' 'gz'
+LOWCOSTEXT='lz4'
 
 buildsimplenativepackage 'foo' 'amd64' '1' 'unstable'
 
@@ -101,14 +102,14 @@ Get:2 http://localhost:${APTHTTPPORT} unstable/main amd64 Contents [$(stat -c%s
 Get:3 http://localhost:${APTHTTPPORT} unstable/main all Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-all.gz) B]
 Reading package lists..." aptget update
 
-testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz" find rootdir/var/lib/apt/lists -name '*Contents-amd64*'
-testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz" find rootdir/var/lib/apt/lists -name '*Contents-all*'
-testequal "$(readfile Contents-amd64.gz Contents-all.gz)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
-testsuccess cmp "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz" 'aptarchive/dists/unstable/main/Contents-amd64.gz'
-testsuccess cmp "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz" 'aptarchive/dists/unstable/main/Contents-all.gz'
+testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT" find rootdir/var/lib/apt/lists -name '*Contents-amd64*'
+testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT" find rootdir/var/lib/apt/lists -name '*Contents-all*'
+testequal "$(readfile Contents-amd64.$LOWCOSTEXT Contents-all.$LOWCOSTEXT)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
+testequal "$(apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT)" apthelper cat-file 'aptarchive/dists/unstable/main/Contents-amd64.gz'
+testempty apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT
 
-rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz
-rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz
+rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT
+rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT
 testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
 
 msgmsg "Compressed Contents file"
index f175ca35596be7386004d143ffee96d1ec33ea9b..d433baeb5aa77ceff73a537276b76cf6be4642b6 100755 (executable)
@@ -24,15 +24,15 @@ runtest() {
     rm -rf rootdir/var/lib/apt/lists/
 
     local TEST="test${1:-success}"
-    $TEST aptget update $APTOPT
+    $TEST aptget update $APTOPT -o Debug::pkgAcquire::Worker=1
     if [ "$1" = 'failure' ]; then
        # accept the outdated Release file so we can check Hit behaviour
        "test${2:-success}" aptget update -o Acquire::Min-ValidTime=9999999 $APTOPT
     fi
     listcurrentlistsdirectory > listsdir.lst
-    testsuccess grep '_Packages\(\.gz\)\?$' listsdir.lst
-    testsuccess grep '_Sources\(\.gz\)\?$' listsdir.lst
-    testsuccess grep '_Translation-en\(\.gz\)\?$' listsdir.lst
+    testsuccess grep '_Packages\(\.[0-9a-z]\+\)\?$' listsdir.lst
+    testsuccess grep '_Sources\(\.[0-9a-z]\+\)\?$' listsdir.lst
+    testsuccess grep '_Translation-en\(\.[0-9a-z]\+\)\?$' listsdir.lst
 
     # ensure no leftovers in partial
     testfailure ls 'rootdir/var/lib/apt/lists/partial/*'
index 2ececc4f6aa1d34ab7ad3f884b4a49b96d53361e..ef1e4eca7b020db1ccea6e2199b56f862e7dae54 100755 (executable)
@@ -12,6 +12,7 @@ umask 022
 
 setupenvironment
 configarchitecture "i386"
+configcompression '.' 'xz'
 
 insertpackage 'unstable' 'foo' 'i386' '1.0'
 insertsource 'unstable' 'foo' 'any' '1.0'
@@ -45,7 +46,7 @@ runtest() {
     listcurrentlistsdirectory > lists.before
 
     # update and ensure all is reverted on the hashsum failure
-    testfailure aptget update -o Debug::Acquire::Transaction=0 -o Debug::pkgAcquire::Auth=1 -o Debug::pkgAcquire::worker=0 -o Debug::acquire::http=0
+    testfailure aptget update -o Debug::Acquire::Transaction=1 -o Debug::pkgAcquire::Auth=1 -o Debug::pkgAcquire::worker=0 -o Debug::acquire::http=0
 
     # ensure we have before what we have after
     msgtest 'Check rollback on going from' 'unauth -> auth'
@@ -53,10 +54,13 @@ runtest() {
     if cmp lists.before lists.after; then
         msgpass
     else
+       echo >&2
        echo >&2 '### Output of previous apt-get update ###'
-       cat >&2 rootdir/tmp/testfailure.output
+       cat >&2 rootdir/tmp/testfailure.output || true
        echo >&2 '### Changes in the lists-directory: ###'
-       diff -u >&2 lists.before lists.after
+       diff -u >&2 lists.before lists.after || true
+       echo >&2 '### Contents of the lists-directory: ###'
+       ls -l rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists/partial || true
        msgfail
     fi
 
index b97e1244bca13fd9ed85d0f74b2d0443d25309ee..573a86511486340426eaa5f0e45a48c4e3056d43 100755 (executable)
@@ -7,6 +7,7 @@ TESTDIR="$(readlink -f "$(dirname "$0")")"
 setupenvironment
 configcompression '.' $(aptconfig dump APT::Compressor --format '%t %v%n' | sed -n 's#^Extension \.\(.*\)$#\1#p')
 configarchitecture 'i386'
+LOWCOSTEXT='lz4'
 
 buildsimplenativepackage 'testpkg' 'i386' '1.0'
 
@@ -35,10 +36,10 @@ testrun() {
                ! test -e rootdir/var/lib/apt/lists/*all_Packages || F=1
                ! test -e rootdir/var/lib/apt/lists/*_Sources || F=1
                ! test -e rootdir/var/lib/apt/lists/*_Translation-en || F=1
-               test -e rootdir/var/lib/apt/lists/*i386_Packages.${COMPRESS} || F=1
-               test -e rootdir/var/lib/apt/lists/*all_Packages.${COMPRESS} || F=1
-               test -e rootdir/var/lib/apt/lists/*_Sources.${COMPRESS} || F=1
-               test -e rootdir/var/lib/apt/lists/*_Translation-en.${COMPRESS} || F=1
+               test -e rootdir/var/lib/apt/lists/*i386_Packages.$LOWCOSTEXT || F=1
+               test -e rootdir/var/lib/apt/lists/*all_Packages.$LOWCOSTEXT || F=1
+               test -e rootdir/var/lib/apt/lists/*_Sources.$LOWCOSTEXT || F=1
+               test -e rootdir/var/lib/apt/lists/*_Translation-en.$LOWCOSTEXT || F=1
                # there is no point in trying pdiff if we have compressed indexes
                # as we can't patch compressed files (well, we can, but what is the point?)
                ! test -e rootdir/var/lib/apt/lists/*diff_Index || F=1
index 547242cadb677c1a89241e3a169b8ad7c4e8ac14..32240da4e9314f42948a9f1e2fb57894c1b1f736 100755 (executable)
@@ -7,6 +7,7 @@ TESTDIR="$(readlink -f "$(dirname "$0")")"
 setupenvironment
 configarchitecture 'i386'
 confighashes 'SHA1' 'SHA256'
+LOWCOSTEXT='lz4'
 
 buildaptarchive
 setupflataptarchive
@@ -274,5 +275,5 @@ testcase() {
 }
 aptautotest_apt_update() { aptautotest_aptget_update "$@"; testsuccess test -e "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_Packages"; }
 testcase -o Acquire::IndexTargets::deb::Packages::KeepCompressed=false
-aptautotest_apt_update() { aptautotest_aptget_update "$@"; testsuccess test -e "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_Packages.gz"; }
+aptautotest_apt_update() { aptautotest_aptget_update "$@"; testsuccess test -e "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_Packages.$LOWCOSTEXT"; }
 testcase -o Acquire::IndexTargets::deb::Packages::KeepCompressed=true