* cmdline/apt-get.cc, apt-pkg/cdrom.cc:

[apt.git] / apt-pkg / pkgcachegen.cc

diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc
index f71547f3925e8e6b24186ab2859caa2e3fc7a9f4..3eeb18cae69c2ee85ea1242a7481f1d7a917876b 100644 (file)
--- a/apt-pkg/pkgcachegen.cc
+++ b/apt-pkg/pkgcachegen.cc
@@ -21,6 +21,7 @@
 #include <apt-pkg/strutl.h>
 #include <apt-pkg/sptr.h>
 #include <apt-pkg/pkgsystem.h>
 #include <apt-pkg/strutl.h>
 #include <apt-pkg/sptr.h>
 #include <apt-pkg/pkgsystem.h>

+#include <apt-pkg/macros.h>

 
 #include <apt-pkg/tagfile.h>
 
 
 #include <apt-pkg/tagfile.h>
 


@@ -32,7 +33,6 @@
 #include <unistd.h>
 #include <errno.h>
 #include <stdio.h>
 #include <unistd.h>
 #include <errno.h>
 #include <stdio.h>

-#include <system.h>

                                                                        /*}}}*/
 typedef vector<pkgIndexFile *>::iterator FileIterator;
 
                                                                        /*}}}*/
 typedef vector<pkgIndexFile *>::iterator FileIterator;
 


@@ -53,14 +53,16 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
    {
       // Setup the map interface..
       Cache.HeaderP = (pkgCache::Header *)Map.Data();
    {
       // Setup the map interface..
       Cache.HeaderP = (pkgCache::Header *)Map.Data();

-      Map.RawAllocate(sizeof(pkgCache::Header));
+      if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
+        return;
+

       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));

-      
+

       // Starting header
       *Cache.HeaderP = pkgCache::Header();
       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
       // Starting header
       *Cache.HeaderP = pkgCache::Header();
       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));

-      Cache.ReMap(); 
+      Cache.ReMap();

    }
    else
    {
    }
    else
    {


@@ -135,7 +137,7 @@ bool pkgCacheGenerator::MergeList(ListParser &List,
         pkgCache::VerIterator Ver = Pkg.VersionList();
         map_ptrloc *LastVer = &Pkg->VersionList;
 
         pkgCache::VerIterator Ver = Pkg.VersionList();
         map_ptrloc *LastVer = &Pkg->VersionList;
 

-        for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++) 
+        for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)

         {
            pkgCache::DescIterator Desc = Ver.DescriptionList();
            map_ptrloc *LastDesc = &Ver->DescriptionList;
         {
            pkgCache::DescIterator Desc = Ver.DescriptionList();
            map_ptrloc *LastDesc = &Ver->DescriptionList;


@@ -143,7 +145,7 @@ bool pkgCacheGenerator::MergeList(ListParser &List,
 
            // don't add a new description if we have one for the given
            // md5 && language
 
            // don't add a new description if we have one for the given
            // md5 && language

-           for ( ; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++)
+           for ( ; Desc.end() == false; Desc++)

               if (MD5SumValue(Desc.md5()) == CurMd5 && 
                   Desc.LanguageCode() == List.DescriptionLanguage())
                  duplicate=true;
               if (MD5SumValue(Desc.md5()) == CurMd5 && 
                   Desc.LanguageCode() == List.DescriptionLanguage())
                  duplicate=true;


@@ -151,7 +153,7 @@ bool pkgCacheGenerator::MergeList(ListParser &List,
               continue;
            
            for (Desc = Ver.DescriptionList();
               continue;
            
            for (Desc = Ver.DescriptionList();

-                Desc.end() == false; 
+                Desc.end() == false;

                 LastDesc = &Desc->NextDesc, Desc++)
            {
               if (MD5SumValue(Desc.md5()) == CurMd5) 
                 LastDesc = &Desc->NextDesc, Desc++)
            {
               if (MD5SumValue(Desc.md5()) == CurMd5) 


@@ -160,7 +162,7 @@ bool pkgCacheGenerator::MergeList(ListParser &List,
                  *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
                  Desc->ParentPkg = Pkg.Index();
                  
                  *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
                  Desc->ParentPkg = Pkg.Index();
                  

-                 if (NewFileDesc(Desc,List) == false)
+                 if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)

                     return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
                  break;
               }
                     return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
                  break;
               }


@@ -220,7 +222,7 @@ bool pkgCacheGenerator::MergeList(ListParser &List,
       Ver->ParentPkg = Pkg.Index();
       Ver->Hash = Hash;
 
       Ver->ParentPkg = Pkg.Index();
       Ver->Hash = Hash;
 

-      if (List.NewVersion(Ver) == false)
+      if ((*LastVer == 0 && _error->PendingError()) || List.NewVersion(Ver) == false)

         return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
                              PackageName.c_str());
 
         return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
                              PackageName.c_str());
 


@@ -252,7 +254,7 @@ bool pkgCacheGenerator::MergeList(ListParser &List,
       *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
       Desc->ParentPkg = Pkg.Index();
 
       *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
       Desc->ParentPkg = Pkg.Index();
 

-      if (NewFileDesc(Desc,List) == false)
+      if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)

         return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
    }
 
         return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
    }
 


@@ -419,7 +421,7 @@ bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
    // Get a structure
    unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
    if (DescFile == 0)
    // Get a structure
    unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
    if (DescFile == 0)

-      return 0;
+      return false;

 
    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
    DF->File = CurrentFile - Cache.PkgFileP;
 
    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
    DF->File = CurrentFile - Cache.PkgFileP;


@@ -460,6 +462,8 @@ map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
    Desc->ID = Cache.HeaderP->DescriptionCount++;
    Desc->language_code = Map.WriteString(Lang);
    Desc->md5sum = Map.WriteString(md5sum.Value());
    Desc->ID = Cache.HeaderP->DescriptionCount++;
    Desc->language_code = Map.WriteString(Lang);
    Desc->md5sum = Map.WriteString(md5sum.Value());

+   if (Desc->language_code == 0 || Desc->md5sum == 0)
+      return 0;

 
    return Description;
 }
 
    return Description;
 }


@@ -652,7 +656,6 @@ unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
    return ItemP->String;
 }
                                                                        /*}}}*/
    return ItemP->String;
 }
                                                                        /*}}}*/

-

 // CheckValidity - Check that a cache is up-to-date                    /*{{{*/
 // ---------------------------------------------------------------------
 /* This just verifies that each file in the list of index files exists,
 // CheckValidity - Check that a cache is up-to-date                    /*{{{*/
 // ---------------------------------------------------------------------
 /* This just verifies that each file in the list of index files exists,


@@ -661,16 +664,23 @@ unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 static bool CheckValidity(const string &CacheFile, FileIterator Start, 
                           FileIterator End,MMap **OutMap = 0)
 {
 static bool CheckValidity(const string &CacheFile, FileIterator Start, 
                           FileIterator End,MMap **OutMap = 0)
 {

+   bool const Debug = _config->FindB("Debug::pkgCacheGen", false);

    // No file, certainly invalid
    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
    // No file, certainly invalid
    if (CacheFile.empty() == true || FileExists(CacheFile) == false)

+   {
+      if (Debug == true)
+        std::clog << "CacheFile doesn't exist" << std::endl;

       return false;
       return false;

-   
+   }
+

    // Map it
    FileFd CacheF(CacheFile,FileFd::ReadOnly);
    // Map it
    FileFd CacheF(CacheFile,FileFd::ReadOnly);

-   SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
+   SPtr<MMap> Map = new MMap(CacheF,0);

    pkgCache Cache(Map);
    if (_error->PendingError() == true || Map->Size() == 0)
    {
    pkgCache Cache(Map);
    if (_error->PendingError() == true || Map->Size() == 0)
    {

+      if (Debug == true)
+        std::clog << "Errors are pending or Map is empty()" << std::endl;

       _error->Discard();
       return false;
    }
       _error->Discard();
       return false;
    }


@@ -680,9 +690,15 @@ static bool CheckValidity(const string &CacheFile, FileIterator Start,
    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
    for (; Start != End; Start++)
    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
    for (; Start != End; Start++)

-   {      
+   {
+      if (Debug == true)
+        std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";

       if ((*Start)->HasPackages() == false)
       if ((*Start)->HasPackages() == false)

+      {
+         if (Debug == true)
+           std::clog << "Has NO packages" << std::endl;

         continue;
         continue;

+      }

     
       if ((*Start)->Exists() == false)
       {
     
       if ((*Start)->Exists() == false)
       {


@@ -690,23 +706,40 @@ static bool CheckValidity(const string &CacheFile, FileIterator Start,
         _error->WarningE("stat",_("Couldn't stat source package list %s"),
                          (*Start)->Describe().c_str());
 #endif
         _error->WarningE("stat",_("Couldn't stat source package list %s"),
                          (*Start)->Describe().c_str());
 #endif

+         if (Debug == true)
+           std::clog << "file doesn't exist" << std::endl;

         continue;
       }
 
       // FindInCache is also expected to do an IMS check.
       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
       if (File.end() == true)
         continue;
       }
 
       // FindInCache is also expected to do an IMS check.
       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
       if (File.end() == true)

+      {
+        if (Debug == true)
+           std::clog << "FindInCache returned end-Pointer" << std::endl;

         return false;
         return false;

+      }

 
       Visited[File->ID] = true;
 
       Visited[File->ID] = true;

+      if (Debug == true)
+        std::clog << "with ID " << File->ID << " is valid" << std::endl;

    }
    
    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
       if (Visited[I] == false)
    }
    
    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
       if (Visited[I] == false)

+      {
+        if (Debug == true)
+           std::clog << "File with ID" << I << " wasn't visited" << std::endl;

         return false;
         return false;

+      }

    
    if (_error->PendingError() == true)
    {
    
    if (_error->PendingError() == true)
    {

+      if (Debug == true)
+      {
+        std::clog << "Validity failed because of pending errors:" << std::endl;
+        _error->DumpErrors();
+      }

       _error->Discard();
       return false;
    }
       _error->Discard();
       return false;
    }


@@ -793,7 +826,8 @@ static bool BuildCache(pkgCacheGenerator &Gen,
 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
                        MMap **OutMap,bool AllowMem)
 {
 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
                        MMap **OutMap,bool AllowMem)
 {

-   unsigned long MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);
+   bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
+   unsigned long const MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);

    
    vector<pkgIndexFile *> Files;
    for (vector<metaIndex *>::const_iterator i = List.begin();
    
    vector<pkgIndexFile *> Files;
    for (vector<metaIndex *>::const_iterator i = List.begin();


@@ -807,13 +841,13 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
          Files.push_back (*j);
    }
    
          Files.push_back (*j);
    }
    

-   unsigned long EndOfSource = Files.size();
+   unsigned long const EndOfSource = Files.size();

    if (_system->AddStatusFiles(Files) == false)
       return false;
    if (_system->AddStatusFiles(Files) == false)
       return false;

-   
+

    // Decide if we can write to the files..
    // Decide if we can write to the files..

-   string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
-   string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
+   string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
+   string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");

    
    // Decide if we can write to the cache
    bool Writeable = false;
    
    // Decide if we can write to the cache
    bool Writeable = false;


@@ -822,7 +856,9 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
    else
       if (SrcCacheFile.empty() == false)
         Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
    else
       if (SrcCacheFile.empty() == false)
         Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;

-   
+   if (Debug == true)
+      std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
+

    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
    
    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
    


@@ -832,8 +868,12 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
    {
       Progress.OverallProgress(1,1,1,_("Reading package lists"));
    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
    {
       Progress.OverallProgress(1,1,1,_("Reading package lists"));

+      if (Debug == true)
+        std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;

       return true;
    }
       return true;
    }

+   else if (Debug == true)
+        std::clog << "pkgcache.bin is NOT valid" << std::endl;

    
    /* At this point we know we need to reconstruct the package cache,
       begin. */
    
    /* At this point we know we need to reconstruct the package cache,
       begin. */


@@ -847,11 +887,15 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
       if (_error->PendingError() == true)
         return false;
       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
       if (_error->PendingError() == true)
         return false;

+      if (Debug == true)
+        std::clog << "Open filebased MMap" << std::endl;

    }
    else
    {
       // Just build it in memory..
    }
    else
    {
       // Just build it in memory..

-      Map = new DynamicMMap(MMap::Public,MapSize);
+      Map = new DynamicMMap(0,MapSize);
+      if (Debug == true)
+        std::clog << "Open memory Map (not filebased)" << std::endl;

    }
    
    // Lets try the source cache.
    }
    
    // Lets try the source cache.


@@ -860,14 +904,18 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
    if (CheckValidity(SrcCacheFile,Files.begin(),
                     Files.begin()+EndOfSource) == true)
    {
    if (CheckValidity(SrcCacheFile,Files.begin(),
                     Files.begin()+EndOfSource) == true)
    {

+      if (Debug == true)
+        std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;

       // Preload the map with the source cache
       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
       // Preload the map with the source cache
       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);

-      if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
-                      SCacheF.Size()) == false)
+      unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
+      if ((alloc == 0 && _error->PendingError())
+               || SCacheF.Read((unsigned char *)Map->Data() + alloc,
+                               SCacheF.Size()) == false)

         return false;
 
       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
         return false;
 
       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());

-      
+

       // Build the status cache
       pkgCacheGenerator Gen(Map.Get(),&Progress);
       if (_error->PendingError() == true)
       // Build the status cache
       pkgCacheGenerator Gen(Map.Get(),&Progress);
       if (_error->PendingError() == true)


@@ -878,6 +926,8 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
    }
    else
    {
    }
    else
    {

+      if (Debug == true)
+        std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;

       TotalSize = ComputeSize(Files.begin(),Files.end());
       
       // Build the source cache
       TotalSize = ComputeSize(Files.begin(),Files.end());
       
       // Build the source cache


@@ -916,6 +966,8 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
                     Files.begin()+EndOfSource,Files.end()) == false)
         return false;
    }
                     Files.begin()+EndOfSource,Files.end()) == false)
         return false;
    }

+   if (Debug == true)
+      std::clog << "Caches are ready for shipping" << std::endl;

 
    if (_error->PendingError() == true)
       return false;
 
    if (_error->PendingError() == true)
       return false;


@@ -924,7 +976,7 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
       if (CacheF != 0)
       {
         delete Map.UnGuard();
       if (CacheF != 0)
       {
         delete Map.UnGuard();

-        *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
+        *OutMap = new MMap(*CacheF,0);

       }
       else
       {
       }
       else
       {


@@ -946,8 +998,7 @@ bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
    if (_system->AddStatusFiles(Files) == false)
       return false;
    
    if (_system->AddStatusFiles(Files) == false)
       return false;
    

-   SPtr<DynamicMMap> Map;   
-   Map = new DynamicMMap(MMap::Public,MapSize);
+   SPtr<DynamicMMap> Map = new DynamicMMap(0,MapSize);

    unsigned long CurrentSize = 0;
    unsigned long TotalSize = 0;
    
    unsigned long CurrentSize = 0;
    unsigned long TotalSize = 0;