keep compressed indexes in a low-cost format

[apt.git] / apt-pkg / indexfile.cc

diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc
index e9e1b08c381187adb7c816922238f599fe056bc5..dfd482c04d09544704012cac04c3d7cb083dbdc4 100644 (file)
--- a/apt-pkg/indexfile.cc
+++ b/apt-pkg/indexfile.cc
@@ -16,14 +16,23 @@
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/aptconfiguration.h>
 #include <apt-pkg/pkgcache.h>
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/aptconfiguration.h>
 #include <apt-pkg/pkgcache.h>

+#include <apt-pkg/pkgcachegen.h>

 #include <apt-pkg/cacheiterators.h>
 #include <apt-pkg/srcrecords.h>
 #include <apt-pkg/cacheiterators.h>
 #include <apt-pkg/srcrecords.h>

+#include <apt-pkg/strutl.h>
+#include <apt-pkg/progress.h>
+#include <apt-pkg/deblistparser.h>

 #include <apt-pkg/macros.h>
 
 #include <apt-pkg/macros.h>
 

+#include <apt-pkg/debindexfile.h>
+
+#include <sys/stat.h>
+

 #include <string>
 #include <vector>
 #include <clocale>
 #include <cstring>
 #include <string>
 #include <vector>
 #include <clocale>
 #include <cstring>

+#include <memory>

                                                                        /*}}}*/
 
 // Global list of Item supported
                                                                        /*}}}*/
 
 // Global list of Item supported


@@ -52,14 +61,17 @@ pkgIndexFile::Type *pkgIndexFile::Type::GetType(const char *Type)
    return 0;
 }
                                                                        /*}}}*/
    return 0;
 }
                                                                        /*}}}*/

-pkgIndexFile::pkgIndexFile(bool Trusted) :                             /*{{{*/
+pkgIndexFile::pkgIndexFile(bool const Trusted) :                       /*{{{*/

    d(NULL), Trusted(Trusted)
 {
 }
                                                                        /*}}}*/
 // IndexFile::ArchiveInfo - Stub                                       /*{{{*/
    d(NULL), Trusted(Trusted)
 {
 }
                                                                        /*}}}*/
 // IndexFile::ArchiveInfo - Stub                                       /*{{{*/

-std::string pkgIndexFile::ArchiveInfo(pkgCache::VerIterator /*Ver*/) const
+std::string pkgIndexFile::ArchiveInfo(pkgCache::VerIterator const &Ver) const

 {
 {

+   debDebPkgFileIndex const * const debfile = dynamic_cast<debDebPkgFileIndex const*>(this);
+   if (debfile != nullptr)
+      return debfile->ArchiveInfo_impl(Ver);

    return std::string();
 }
                                                                        /*}}}*/
    return std::string();
 }
                                                                        /*}}}*/


@@ -77,19 +89,12 @@ std::string pkgIndexFile::SourceInfo(pkgSrcRecords::Parser const &/*Record*/,
 }
                                                                        /*}}}*/
 // IndexFile::TranslationsAvailable - Check if will use Translation    /*{{{*/
 }
                                                                        /*}}}*/
 // IndexFile::TranslationsAvailable - Check if will use Translation    /*{{{*/

-// ---------------------------------------------------------------------
-/* */

 bool pkgIndexFile::TranslationsAvailable() {
        return (APT::Configuration::getLanguages().empty() != true);
 }
                                                                        /*}}}*/
 // IndexFile::CheckLanguageCode - Check the Language Code              /*{{{*/
 bool pkgIndexFile::TranslationsAvailable() {
        return (APT::Configuration::getLanguages().empty() != true);
 }
                                                                        /*}}}*/
 // IndexFile::CheckLanguageCode - Check the Language Code              /*{{{*/

-// ---------------------------------------------------------------------
-/* No intern need for this method anymore as the check for correctness
-   is already done in getLanguages(). Note also that this check is
-   rather bad (doesn't take three character like ast into account).
-   TODO: Remove method with next API break */
-APT_DEPRECATED bool pkgIndexFile::CheckLanguageCode(const char *Lang)
+bool pkgIndexFile::CheckLanguageCode(const char * const Lang)

 {
   if (strlen(Lang) == 2 || (strlen(Lang) == 5 && Lang[2] == '_'))
     return true;
 {
   if (strlen(Lang) == 2 || (strlen(Lang) == 5 && Lang[2] == '_'))
     return true;


@@ -101,22 +106,21 @@ APT_DEPRECATED bool pkgIndexFile::CheckLanguageCode(const char *Lang)
 }
                                                                        /*}}}*/
 // IndexFile::LanguageCode - Return the Language Code                  /*{{{*/
 }
                                                                        /*}}}*/
 // IndexFile::LanguageCode - Return the Language Code                  /*{{{*/

-// ---------------------------------------------------------------------
-/* As we have now possibly more than one LanguageCode this method is
-   supersided by a) private classmembers or b) getLanguages().
-   TODO: Remove method with next API break */
-APT_DEPRECATED std::string pkgIndexFile::LanguageCode() {
+std::string pkgIndexFile::LanguageCode() {
+APT_IGNORE_DEPRECATED_PUSH

        if (TranslationsAvailable() == false)
                return "";
        return APT::Configuration::getLanguages()[0];
        if (TranslationsAvailable() == false)
                return "";
        return APT::Configuration::getLanguages()[0];

+APT_IGNORE_DEPRECATED_POP

 }
                                                                        /*}}}*/
 
 // IndexTarget - Constructor                                           /*{{{*/
 IndexTarget::IndexTarget(std::string const &MetaKey, std::string const &ShortDesc,
       std::string const &LongDesc, std::string const &URI, bool const IsOptional,
 }
                                                                        /*}}}*/
 
 // IndexTarget - Constructor                                           /*{{{*/
 IndexTarget::IndexTarget(std::string const &MetaKey, std::string const &ShortDesc,
       std::string const &LongDesc, std::string const &URI, bool const IsOptional,

-      std::map<std::string, std::string> const &Options) :
-   URI(URI), Description(LongDesc), ShortDesc(ShortDesc), MetaKey(MetaKey), IsOptional(IsOptional), Options(Options)
+      bool const KeepCompressed, std::map<std::string, std::string> const &Options) :
+   URI(URI), Description(LongDesc), ShortDesc(ShortDesc), MetaKey(MetaKey),
+   IsOptional(IsOptional), KeepCompressed(KeepCompressed), Options(Options)

 {
 }
                                                                        /*}}}*/
 {
 }
                                                                        /*}}}*/


@@ -135,11 +139,17 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const           /*{{{*/
       APT_CASE(REPO_URI);
       APT_CASE(TARGET_OF);
       APT_CASE(CREATED_BY);
       APT_CASE(REPO_URI);
       APT_CASE(TARGET_OF);
       APT_CASE(CREATED_BY);

+      APT_CASE(PDIFFS);
+      APT_CASE(DEFAULTENABLED);
+      APT_CASE(COMPRESSIONTYPES);
+      APT_CASE(SOURCESENTRY);
+      APT_CASE(BY_HASH);
+      APT_CASE(KEEPCOMPRESSEDAS);

 #undef APT_CASE
       case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
       case EXISTING_FILENAME:
         std::string const filename = Option(FILENAME);
 #undef APT_CASE
       case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
       case EXISTING_FILENAME:
         std::string const filename = Option(FILENAME);

-        std::vector<std::string> const types = APT::Configuration::getCompressionTypes();
+        std::vector<std::string> const types = VectorizeString(Option(COMPRESSIONTYPES), ' ');

         for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
         {
            if (t->empty())
         for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
         {
            if (t->empty())


@@ -156,6 +166,11 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const            /*{{{*/
    return M->second;
 }
                                                                        /*}}}*/
    return M->second;
 }
                                                                        /*}}}*/

+bool IndexTarget::OptionBool(OptionKeys const EnumKey) const           /*{{{*/
+{
+   return StringToBool(Option(EnumKey));
+}
+                                                                       /*}}}*/

 std::string IndexTarget::Format(std::string format) const              /*{{{*/
 {
    for (std::map<std::string, std::string>::const_iterator O = Options.begin(); O != Options.end(); ++O)
 std::string IndexTarget::Format(std::string format) const              /*{{{*/
 {
    for (std::map<std::string, std::string>::const_iterator O = Options.begin(); O != Options.end(); ++O)


@@ -171,30 +186,30 @@ std::string IndexTarget::Format(std::string format) const         /*{{{*/
 }
                                                                        /*}}}*/
 
 }
                                                                        /*}}}*/
 

-pkgIndexTargetFile::pkgIndexTargetFile(IndexTarget const &Target, bool const Trusted) :/*{{{*/
-   pkgIndexFile(Trusted), d(NULL), Target(Target)
+pkgDebianIndexTargetFile::pkgDebianIndexTargetFile(IndexTarget const &Target, bool const Trusted) :/*{{{*/
+   pkgDebianIndexFile(Trusted), d(NULL), Target(Target)

 {
 }
                                                                        /*}}}*/
 {
 }
                                                                        /*}}}*/

-std::string pkgIndexTargetFile::ArchiveURI(std::string File) const/*{{{*/
+std::string pkgDebianIndexTargetFile::ArchiveURI(std::string const &File) const/*{{{*/

 {
    return Target.Option(IndexTarget::REPO_URI) + File;
 }
                                                                        /*}}}*/
 {
    return Target.Option(IndexTarget::REPO_URI) + File;
 }
                                                                        /*}}}*/

-std::string pkgIndexTargetFile::Describe(bool Short) const             /*{{{*/
+std::string pkgDebianIndexTargetFile::Describe(bool const Short) const /*{{{*/

 {
    if (Short)
       return Target.Description;
    return Target.Description + " (" + IndexFileName() + ")";
 }
                                                                        /*}}}*/
 {
    if (Short)
       return Target.Description;
    return Target.Description + " (" + IndexFileName() + ")";
 }
                                                                        /*}}}*/

-std::string pkgIndexTargetFile::IndexFileName() const                  /*{{{*/
+std::string pkgDebianIndexTargetFile::IndexFileName() const                    /*{{{*/

 {
    std::string const s = Target.Option(IndexTarget::FILENAME);
    if (FileExists(s))
       return s;
 
 {
    std::string const s = Target.Option(IndexTarget::FILENAME);
    if (FileExists(s))
       return s;
 

-   std::vector<std::string> types = APT::Configuration::getCompressionTypes();
+   std::vector<std::string> const types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' ');

    for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
    {
       std::string p = s + '.' + *t;
    for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
    {
       std::string p = s + '.' + *t;


@@ -204,7 +219,7 @@ std::string pkgIndexTargetFile::IndexFileName() const                       /*{{{*/
    return s;
 }
                                                                        /*}}}*/
    return s;
 }
                                                                        /*}}}*/

-unsigned long pkgIndexTargetFile::Size() const                         /*{{{*/
+unsigned long pkgDebianIndexTargetFile::Size() const                           /*{{{*/

 {
    unsigned long size = 0;
 
 {
    unsigned long size = 0;
 


@@ -222,11 +237,160 @@ unsigned long pkgIndexTargetFile::Size() const                           /*{{{*/
    return size;
 }
                                                                        /*}}}*/
    return size;
 }
                                                                        /*}}}*/

-bool pkgIndexTargetFile::Exists() const                                        /*{{{*/
+bool pkgDebianIndexTargetFile::Exists() const                                  /*{{{*/

 {
    return FileExists(IndexFileName());
 }
                                                                        /*}}}*/
 {
    return FileExists(IndexFileName());
 }
                                                                        /*}}}*/

+std::string pkgDebianIndexTargetFile::GetArchitecture() const                  /*{{{*/
+{
+   return Target.Option(IndexTarget::ARCHITECTURE);
+}
+                                                                       /*}}}*/
+std::string pkgDebianIndexTargetFile::GetComponent() const                     /*{{{*/
+{
+   return Target.Option(IndexTarget::COMPONENT);
+}
+                                                                       /*}}}*/
+bool pkgDebianIndexTargetFile::OpenListFile(FileFd &Pkg, std::string const &FileName)/*{{{*/
+{
+   if (Pkg.Open(FileName, FileFd::ReadOnly, FileFd::Extension) == false)
+      return _error->Error("Problem opening %s",FileName.c_str());
+   return true;
+}
+                                                                       /*}}}*/
+std::string pkgDebianIndexTargetFile::GetProgressDescription() const
+{
+   return Target.Description;
+}
+
+pkgDebianIndexRealFile::pkgDebianIndexRealFile(std::string const &pFile, bool const Trusted) :/*{{{*/
+   pkgDebianIndexFile(Trusted), d(NULL)
+{
+   if (pFile == "/nonexistent/stdin")
+      File = pFile;
+   else
+      File = flAbsPath(pFile);
+}
+                                                                       /*}}}*/
+// IndexRealFile::Size - Return the size of the index                  /*{{{*/
+unsigned long pkgDebianIndexRealFile::Size() const
+{
+   struct stat S;
+   if (stat(File.c_str(),&S) != 0)
+      return 0;
+   return S.st_size;
+}
+                                                                       /*}}}*/
+bool pkgDebianIndexRealFile::Exists() const                                    /*{{{*/
+{
+   return FileExists(File);
+}
+                                                                       /*}}}*/
+std::string pkgDebianIndexRealFile::Describe(bool const /*Short*/) const/*{{{*/
+{
+   return File;
+}
+                                                                       /*}}}*/
+std::string pkgDebianIndexRealFile::ArchiveURI(std::string const &/*File*/) const/*{{{*/
+{
+   return "file:" + File;
+}
+                                                                       /*}}}*/
+std::string pkgDebianIndexRealFile::IndexFileName() const                      /*{{{*/
+{
+   return File;
+}
+                                                                       /*}}}*/
+std::string pkgDebianIndexRealFile::GetProgressDescription() const
+{
+   return File;
+}
+bool pkgDebianIndexRealFile::OpenListFile(FileFd &Pkg, std::string const &FileName)/*{{{*/
+{
+   if (Pkg.Open(FileName, FileFd::ReadOnly, FileFd::None) == false)
+      return _error->Error("Problem opening %s",FileName.c_str());
+   return true;
+}
+                                                                       /*}}}*/
+
+pkgDebianIndexFile::pkgDebianIndexFile(bool const Trusted) : pkgIndexFile(Trusted)
+{
+}
+pkgDebianIndexFile::~pkgDebianIndexFile()
+{
+}
+pkgCacheListParser * pkgDebianIndexFile::CreateListParser(FileFd &Pkg)
+{
+   if (Pkg.IsOpen() == false)
+      return NULL;
+   _error->PushToStack();
+   pkgCacheListParser * const Parser = new debListParser(&Pkg);
+   bool const newError = _error->PendingError();
+   _error->MergeWithStack();
+   return newError ? NULL : Parser;
+}
+bool pkgDebianIndexFile::Merge(pkgCacheGenerator &Gen,OpProgress * const Prog)
+{
+   std::string const PackageFile = IndexFileName();
+   FileFd Pkg;
+   if (OpenListFile(Pkg, PackageFile) == false)
+      return false;
+   _error->PushToStack();
+   std::unique_ptr<pkgCacheListParser> Parser(CreateListParser(Pkg));
+   bool const newError = _error->PendingError();
+   _error->MergeWithStack();
+   if (newError == false && Parser == nullptr)
+      return true;
+   if (Parser == NULL)
+      return false;
+
+   if (Prog != NULL)
+      Prog->SubProgress(0, GetProgressDescription());
+
+   if (Gen.SelectFile(PackageFile, *this, GetArchitecture(), GetComponent(), GetIndexFlags()) == false)
+      return _error->Error("Problem with SelectFile %s",PackageFile.c_str());
+
+   // Store the IMS information
+   pkgCache::PkgFileIterator File = Gen.GetCurFile();
+   pkgCacheGenerator::Dynamic<pkgCache::PkgFileIterator> DynFile(File);
+   File->Size = Pkg.FileSize();
+   File->mtime = Pkg.ModificationTime();
+
+   if (Gen.MergeList(*Parser) == false)
+      return _error->Error("Problem with MergeList %s",PackageFile.c_str());
+   return true;
+}
+pkgCache::PkgFileIterator pkgDebianIndexFile::FindInCache(pkgCache &Cache) const
+{
+   std::string const FileName = IndexFileName();
+   pkgCache::PkgFileIterator File = Cache.FileBegin();
+   for (; File.end() == false; ++File)
+   {
+       if (File.FileName() == NULL || FileName != File.FileName())
+        continue;
+
+      struct stat St;
+      if (stat(File.FileName(),&St) != 0)
+      {
+         if (_config->FindB("Debug::pkgCacheGen", false))
+           std::clog << "DebianIndexFile::FindInCache - stat failed on " << File.FileName() << std::endl;
+        return pkgCache::PkgFileIterator(Cache);
+      }
+      if ((map_filesize_t)St.st_size != File->Size || St.st_mtime != File->mtime)
+      {
+         if (_config->FindB("Debug::pkgCacheGen", false))
+           std::clog << "DebianIndexFile::FindInCache - size (" << St.st_size << " <> " << File->Size
+                       << ") or mtime (" << St.st_mtime << " <> " << File->mtime
+                       << ") doesn't match for " << File.FileName() << std::endl;
+        return pkgCache::PkgFileIterator(Cache);
+      }
+      return File;
+   }
+
+   return File;
+}

 
 APT_CONST pkgIndexFile::~pkgIndexFile() {}
 
 APT_CONST pkgIndexFile::~pkgIndexFile() {}

-APT_CONST pkgIndexTargetFile::~pkgIndexTargetFile() {}
+APT_CONST pkgDebianIndexTargetFile::~pkgDebianIndexTargetFile() {}
+APT_CONST pkgDebianIndexRealFile::~pkgDebianIndexRealFile() {}