report warnings&errors consistently in edit-sources

[apt.git] / apt-pkg / pkgcachegen.cc

diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc
index 6e307fba9b32df572709ea1a324815bb7aeeb3d5..f0b5a982eb2d4c50248c886584efb701e568b1ae 100644 (file)
--- a/apt-pkg/pkgcachegen.cc
+++ b/apt-pkg/pkgcachegen.cc
@@ -40,8 +40,8 @@
 #include <unistd.h>
 
 #include <apti18n.h>
 #include <unistd.h>
 
 #include <apti18n.h>

-
-template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;                                                       /*}}}*/
+                                                                       /*}}}*/
+template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;

 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
 
 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
 


@@ -57,6 +57,9 @@ using APT::StringView;
 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
                    Map(*pMap), Cache(pMap,false), Progress(Prog),
                     CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
                    Map(*pMap), Cache(pMap,false), Progress(Prog),
                     CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)

+{
+}
+bool pkgCacheGenerator::Start()

 {
    if (Map.Size() == 0)
    {
 {
    if (Map.Size() == 0)
    {


@@ -67,7 +70,9 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
       bool const newError = _error->PendingError();
       _error->MergeWithStack();
       if (newError)
       bool const newError = _error->PendingError();
       _error->MergeWithStack();
       if (newError)

-        return;
+        return false;
+      if (Map.Size() <= 0)
+        return false;

 
       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 
 
       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 


@@ -76,16 +81,15 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
 
       // make room for the hashtables for packages and groups
       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
 
       // make room for the hashtables for packages and groups
       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)

-        return;
+        return false;

 
       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
       if (unlikely(idxVerSysName == 0))
 
       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
       if (unlikely(idxVerSysName == 0))

-        return;
-      Cache.HeaderP->VerSysName = idxVerSysName;
+        return false;

       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
       if (unlikely(idxArchitecture == 0))
       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
       if (unlikely(idxArchitecture == 0))

-        return;
-      Cache.HeaderP->Architecture = idxArchitecture;
+        return false;
+      map_stringitem_t idxArchitectures;

 
       std::vector<std::string> archs = APT::Configuration::getArchitectures();
       if (archs.size() > 1)
 
       std::vector<std::string> archs = APT::Configuration::getArchitectures();
       if (archs.size() > 1)


@@ -94,13 +98,17 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
         std::string list = *a;
         for (++a; a != archs.end(); ++a)
            list.append(",").append(*a);
         std::string list = *a;
         for (++a; a != archs.end(); ++a)
            list.append(",").append(*a);

-        map_stringitem_t const idxArchitectures = WriteStringInMap(list);
+        idxArchitectures = WriteStringInMap(list);

         if (unlikely(idxArchitectures == 0))
         if (unlikely(idxArchitectures == 0))

-           return;
-        Cache.HeaderP->SetArchitectures(idxArchitectures);
+           return false;

       }
       else
       }
       else

-        Cache.HeaderP->SetArchitectures(idxArchitecture);
+        idxArchitectures = idxArchitecture;
+
+      Cache.HeaderP = (pkgCache::Header *)Map.Data();
+      Cache.HeaderP->VerSysName = idxVerSysName;
+      Cache.HeaderP->Architecture = idxArchitecture;
+      Cache.HeaderP->SetArchitectures(idxArchitectures);

 
       // Calculate the hash for the empty map, so ReMap does not fail
       Cache.HeaderP->CacheFileSize = Cache.CacheHash();
 
       // Calculate the hash for the empty map, so ReMap does not fail
       Cache.HeaderP->CacheFileSize = Cache.CacheHash();


@@ -112,14 +120,12 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
       Cache.ReMap(); 
       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
       if (Cache.VS != _system->VS)
       Cache.ReMap(); 
       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
       if (Cache.VS != _system->VS)

-      {
-        _error->Error(_("Cache has an incompatible versioning system"));
-        return;
-      }
+        return _error->Error(_("Cache has an incompatible versioning system"));

    }
 
    Cache.HeaderP->Dirty = true;
    Map.Sync(0,sizeof(pkgCache::Header));
    }
 
    Cache.HeaderP->Dirty = true;
    Map.Sync(0,sizeof(pkgCache::Header));

+   return true;

 }
                                                                        /*}}}*/
 // CacheGenerator::~pkgCacheGenerator - Destructor                     /*{{{*/
 }
                                                                        /*}}}*/
 // CacheGenerator::~pkgCacheGenerator - Destructor                     /*{{{*/


@@ -142,6 +148,10 @@ pkgCacheGenerator::~pkgCacheGenerator()
 }
                                                                        /*}}}*/
 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
 }
                                                                        /*}}}*/
 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/

+   // Prevent multiple remaps of the same iterator. If seen.insert(iterator)
+   // returns (something, true) the iterator was not yet seen and we can
+   // remap it.
+   std::unordered_set<void *> seen;

    if (oldMap == newMap)
       return;
 
    if (oldMap == newMap)
       return;
 


@@ -155,29 +165,39 @@ void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newM
 
    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
        i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 
    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
        i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)

-      (*i)->ReMap(oldMap, newMap);
+      if (std::get<1>(seen.insert(*i)) == true)
+        (*i)->ReMap(oldMap, newMap);

    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
        i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
        i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)

-      (*i)->ReMap(oldMap, newMap);
+      if (std::get<1>(seen.insert(*i)) == true)
+        (*i)->ReMap(oldMap, newMap);

    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
        i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
        i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)

-      (*i)->ReMap(oldMap, newMap);
+      if (std::get<1>(seen.insert(*i)) == true)
+        (*i)->ReMap(oldMap, newMap);

    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
        i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
        i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)

-      (*i)->ReMap(oldMap, newMap);
+       if (std::get<1>(seen.insert(*i)) == true)
+        (*i)->ReMap(oldMap, newMap);

    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
        i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
        i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)

-      (*i)->ReMap(oldMap, newMap);
+      if (std::get<1>(seen.insert(*i)) == true)
+        (*i)->ReMap(oldMap, newMap);

    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
        i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
        i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)

-      (*i)->ReMap(oldMap, newMap);
+      if (std::get<1>(seen.insert(*i)) == true)
+        (*i)->ReMap(oldMap, newMap);

    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
        i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
        i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)

-      (*i)->ReMap(oldMap, newMap);
+      if (std::get<1>(seen.insert(*i)) == true)
+        (*i)->ReMap(oldMap, newMap);

    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
        i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
        i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)

-      (*i)->ReMap(oldMap, newMap);
+      if (std::get<1>(seen.insert(*i)) == true)
+        (*i)->ReMap(oldMap, newMap);

    for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
    for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {

+      if (std::get<1>(seen.insert(ViewP)) == false)
+        continue;

       // Ignore views outside of the cache.
       if (ViewP->data() < static_cast<const char*>(oldMap)
         || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)
       // Ignore views outside of the cache.
       if (ViewP->data() < static_cast<const char*>(oldMap)
         || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)


@@ -236,8 +256,10 @@ bool pkgCacheGenerator::MergeList(ListParser &List,
       if (Counter % 100 == 0 && Progress != 0)
         Progress->Progress(List.Offset());
 
       if (Counter % 100 == 0 && Progress != 0)
         Progress->Progress(List.Offset());
 

-      string Arch = List.Architecture();
-      string const Version = List.Version();
+      APT::StringView Arch = List.Architecture();
+      Dynamic<APT::StringView> DynArch(Arch);
+      APT::StringView Version = List.Version();
+      Dynamic<APT::StringView> DynVersion(Version);

       if (Version.empty() == true && Arch.empty() == true)
       {
         // package descriptions
       if (Version.empty() == true && Arch.empty() == true)
       {
         // package descriptions


@@ -348,7 +370,7 @@ bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator
                                                                        /*}}}*/
 // CacheGenerator::MergeListVersion                                    /*{{{*/
 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
                                                                        /*}}}*/
 // CacheGenerator::MergeListVersion                                    /*{{{*/
 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,

-                                        std::string const &Version, pkgCache::VerIterator* &OutVer)
+                                        APT::StringView const &Version, pkgCache::VerIterator* &OutVer)

 {
    pkgCache::VerIterator Ver = Pkg.VersionList();
    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 {
    pkgCache::VerIterator Ver = Pkg.VersionList();
    Dynamic<pkgCache::VerIterator> DynVer(Ver);


@@ -363,13 +385,28 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
       int Res = 1;
       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
       {
       int Res = 1;
       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
       {

-        Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
+        char const * const VerStr = Ver.VerStr();
+        Res = Cache.VS->DoCmpVersion(Version.data(), Version.data() + Version.length(),
+              VerStr, VerStr + strlen(VerStr));

         // Version is higher as current version - insert here
         if (Res > 0)
            break;
         // Versionstrings are equal - is hash also equal?
         // Version is higher as current version - insert here
         if (Res > 0)
            break;
         // Versionstrings are equal - is hash also equal?

-        if (Res == 0 && List.SameVersion(Hash, Ver) == true)
-           break;
+        if (Res == 0)
+        {
+           if (List.SameVersion(Hash, Ver) == true)
+              break;
+           // sort (volatile) sources above not-sources like the status file
+           if ((CurrentFile->Flags & pkgCache::Flag::NotSource) == 0)
+           {
+              auto VF = Ver.FileList();
+              for (; VF.end() == false; ++VF)
+                 if (VF.File().Flagged(pkgCache::Flag::NotSource) == false)
+                    break;
+              if (VF.end() == true)
+                 break;
+           }
+        }

         // proceed with the next till we have either the right
         // or we found another version (which will be lower)
       }
         // proceed with the next till we have either the right
         // or we found another version (which will be lower)
       }


@@ -659,7 +696,7 @@ bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
    if (Arch == "any")
    {
    // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
    if (Arch == "any")
    {

-      size_t const found = Name.find(':');
+      size_t const found = Name.rfind(':');

       StringView ArchA = Name.substr(found + 1);
       if (ArchA != "any")
       {
       StringView ArchA = Name.substr(found + 1);
       if (ArchA != "any")
       {


@@ -709,9 +746,8 @@ bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
                                           pkgCache::PkgIterator &P,
                                           pkgCache::VerIterator &V)
 {
                                           pkgCache::PkgIterator &P,
                                           pkgCache::VerIterator &V)
 {

-   // copy P.Arch() into a string here as a cache remap
-   // in NewDepends() later may alter the pointer location
-   string Arch = P.Arch() == NULL ? "" : P.Arch();
+   APT::StringView Arch = P.Arch() == NULL ? "" : P.Arch();
+   Dynamic<APT::StringView> DynArch(Arch);

    map_pointer_t *OldDepLast = NULL;
    /* MultiArch handling introduces a lot of implicit Dependencies:
       - MultiArch: same → Co-Installable if they have the same version
    map_pointer_t *OldDepLast = NULL;
    /* MultiArch handling introduces a lot of implicit Dependencies:
       - MultiArch: same → Co-Installable if they have the same version


@@ -812,7 +848,7 @@ bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 // ---------------------------------------------------------------------
 /* This puts a version structure in the linked list */
 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 // ---------------------------------------------------------------------
 /* This puts a version structure in the linked list */
 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,

-                                           const string &VerStr,
+                                           APT::StringView const &VerStr,

                                            map_pointer_t const ParentPkg,
                                            unsigned short const Hash,
                                            map_pointer_t const Next)
                                            map_pointer_t const ParentPkg,
                                            unsigned short const Hash,
                                            map_pointer_t const Next)


@@ -841,8 +877,8 @@ map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
            continue;
         for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
         {
            continue;
         for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
         {

-           int const cmp = strcmp(V.VerStr(), VerStr.c_str());
-           if (cmp == 0)
+           int const cmp = strncmp(V.VerStr(), VerStr.data(), VerStr.length());
+           if (cmp == 0 && V.VerStr()[VerStr.length()] == '\0')

            {
               Ver->VerStr = V->VerStr;
               return Version;
            {
               Ver->VerStr = V->VerStr;
               return Version;


@@ -1604,7 +1640,7 @@ static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
       return false;
    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
    if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
       return false;
    Gen.reset(new pkgCacheGenerator(Map.get(),Progress));

-   return true;
+   return Gen->Start();

 }
 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
                        MMap **OutMap, bool AllowMem)
 }
 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
                        MMap **OutMap, bool AllowMem)


@@ -1710,6 +1746,8 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
       if (Debug == true)
         std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
       if (Debug == true)
         std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
       Gen.reset(new pkgCacheGenerator(Map.get(),Progress));

+      if (Gen->Start() == false)
+        return false;

 
       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
 
       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
       if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,


@@ -1735,7 +1773,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
    }
 
    if (Debug == true)
    }
 
    if (Debug == true)

-      std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;
+      std::clog << "Caches done. " << (volatile_fine ? "No volatile files, so we are done here." : "Now bring in the volatile files") << std::endl;

 
    if (volatile_fine == false)
    {
 
    if (volatile_fine == false)
    {


@@ -1787,7 +1825,7 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O
    if (Progress != NULL)
       Progress->OverallProgress(0,1,1,_("Reading package lists"));
    pkgCacheGenerator Gen(Map.get(),Progress);
    if (Progress != NULL)
       Progress->OverallProgress(0,1,1,_("Reading package lists"));
    pkgCacheGenerator Gen(Map.get(),Progress);

-   if (_error->PendingError() == true)
+   if (Gen.Start() == false || _error->PendingError() == true)

       return false;
    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
                  Files.begin(), Files.end()) == false)
       return false;
    if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
                  Files.begin(), Files.end()) == false)