]> git.saurik.com Git - apt.git/blobdiff - apt-pkg/pkgcachegen.cc
Bug #807012 also involves package dependencies :/.
[apt.git] / apt-pkg / pkgcachegen.cc
index 07a21d27fb1e2c3e5481f90cad046c71b8f78d80..51bb9ec7b3545fb97b479549169e02f2c7c666df 100644 (file)
@@ -18,8 +18,6 @@
 #include <apt-pkg/progress.h>
 #include <apt-pkg/sourcelist.h>
 #include <apt-pkg/configuration.h>
-#include <apt-pkg/strutl.h>
-#include <apt-pkg/sptr.h>
 #include <apt-pkg/pkgsystem.h>
 #include <apt-pkg/macros.h>
 #include <apt-pkg/metaindex.h>
 
 #include <apti18n.h>
                                                                        /*}}}*/
+template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;
 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
 
-static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
-                           MD5SumValue const &CurMd5, std::string const &CurLang);
+static bool IsDuplicateDescription(pkgCache &Cache, pkgCache::DescIterator Desc,
+                           APT::StringView CurMd5, std::string const &CurLang);
 
 using std::string;
+using APT::StringView;
 
 // CacheGenerator::pkgCacheGenerator - Constructor                     /*{{{*/
 // ---------------------------------------------------------------------
@@ -58,15 +58,21 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
                    Map(*pMap), Cache(pMap,false), Progress(Prog),
                     CurrentRlsFile(NULL), CurrentFile(NULL), d(NULL)
 {
-   if (_error->PendingError() == true)
-      return;
-
+}
+bool pkgCacheGenerator::Start()
+{
    if (Map.Size() == 0)
    {
       // Setup the map interface..
       Cache.HeaderP = (pkgCache::Header *)Map.Data();
-      if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
-        return;
+      _error->PushToStack();
+      Map.RawAllocate(sizeof(pkgCache::Header));
+      bool const newError = _error->PendingError();
+      _error->MergeWithStack();
+      if (newError)
+        return false;
+      if (Map.Size() <= 0)
+        return false;
 
       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 
@@ -75,16 +81,15 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
 
       // make room for the hashtables for packages and groups
       if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0)
-        return;
+        return false;
 
       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
       if (unlikely(idxVerSysName == 0))
-        return;
-      Cache.HeaderP->VerSysName = idxVerSysName;
+        return false;
       map_stringitem_t const idxArchitecture = StoreString(MIXED, _config->Find("APT::Architecture"));
       if (unlikely(idxArchitecture == 0))
-        return;
-      Cache.HeaderP->Architecture = idxArchitecture;
+        return false;
+      map_stringitem_t idxArchitectures;
 
       std::vector<std::string> archs = APT::Configuration::getArchitectures();
       if (archs.size() > 1)
@@ -93,14 +98,20 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
         std::string list = *a;
         for (++a; a != archs.end(); ++a)
            list.append(",").append(*a);
-        map_stringitem_t const idxArchitectures = WriteStringInMap(list);
+        idxArchitectures = WriteStringInMap(list);
         if (unlikely(idxArchitectures == 0))
-           return;
-        Cache.HeaderP->SetArchitectures(idxArchitectures);
+           return false;
       }
       else
-        Cache.HeaderP->SetArchitectures(idxArchitecture);
+        idxArchitectures = idxArchitecture;
+
+      Cache.HeaderP = (pkgCache::Header *)Map.Data();
+      Cache.HeaderP->VerSysName = idxVerSysName;
+      Cache.HeaderP->Architecture = idxArchitecture;
+      Cache.HeaderP->SetArchitectures(idxArchitectures);
 
+      // Calculate the hash for the empty map, so ReMap does not fail
+      Cache.HeaderP->CacheFileSize = Cache.CacheHash();
       Cache.ReMap();
    }
    else
@@ -109,14 +120,12 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
       Cache.ReMap(); 
       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
       if (Cache.VS != _system->VS)
-      {
-        _error->Error(_("Cache has an incompatible versioning system"));
-        return;
-      }
+        return _error->Error(_("Cache has an incompatible versioning system"));
    }
 
    Cache.HeaderP->Dirty = true;
    Map.Sync(0,sizeof(pkgCache::Header));
+   return true;
 }
                                                                        /*}}}*/
 // CacheGenerator::~pkgCacheGenerator - Destructor                     /*{{{*/
@@ -125,17 +134,24 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
    advoid a problem during a crash */
 pkgCacheGenerator::~pkgCacheGenerator()
 {
-   if (_error->PendingError() == true)
+   if (_error->PendingError() == true || Map.validData() == false)
       return;
    if (Map.Sync() == false)
       return;
    
    Cache.HeaderP->Dirty = false;
-   Cache.HeaderP->CacheFileSize = Map.Size();
+   Cache.HeaderP->CacheFileSize = Cache.CacheHash();
+
+   if (_config->FindB("Debug::pkgCacheGen", false))
+      std::clog << "Produced cache with hash " << Cache.HeaderP->CacheFileSize << std::endl;
    Map.Sync(0,sizeof(pkgCache::Header));
 }
                                                                        /*}}}*/
-void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
+void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
+   // Prevent multiple remaps of the same iterator. If seen.insert(iterator)
+   // returns (something, true) the iterator was not yet seen and we can
+   // remap it.
+   std::unordered_set<void *> seen;
    if (oldMap == newMap)
       return;
 
@@ -149,53 +165,74 @@ void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newM
 
    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
        i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
-      (*i)->ReMap(oldMap, newMap);
+      if (std::get<1>(seen.insert(*i)) == true)
+        (*i)->ReMap(oldMap, newMap);
    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
        i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
-      (*i)->ReMap(oldMap, newMap);
+      if (std::get<1>(seen.insert(*i)) == true)
+        (*i)->ReMap(oldMap, newMap);
    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
        i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
-      (*i)->ReMap(oldMap, newMap);
+      if (std::get<1>(seen.insert(*i)) == true)
+        (*i)->ReMap(oldMap, newMap);
    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
        i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
-      (*i)->ReMap(oldMap, newMap);
+       if (std::get<1>(seen.insert(*i)) == true)
+        (*i)->ReMap(oldMap, newMap);
    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
        i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
-      (*i)->ReMap(oldMap, newMap);
+      if (std::get<1>(seen.insert(*i)) == true)
+        (*i)->ReMap(oldMap, newMap);
    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
        i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
-      (*i)->ReMap(oldMap, newMap);
+      if (std::get<1>(seen.insert(*i)) == true)
+        (*i)->ReMap(oldMap, newMap);
    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
        i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
-      (*i)->ReMap(oldMap, newMap);
+      if (std::get<1>(seen.insert(*i)) == true)
+        (*i)->ReMap(oldMap, newMap);
    for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
        i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
-      (*i)->ReMap(oldMap, newMap);
+      if (std::get<1>(seen.insert(*i)) == true)
+        (*i)->ReMap(oldMap, newMap);
+   for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
+      if (std::get<1>(seen.insert(ViewP)) == false)
+        continue;
+      // Ignore views outside of the cache.
+      if (ViewP->data() < static_cast<const char*>(oldMap)
+        || ViewP->data() > static_cast<const char*>(oldMap) + oldSize)
+        continue;
+      const char *data = ViewP->data() + (static_cast<const char*>(newMap) - static_cast<const char*>(oldMap));
+      *ViewP = StringView(data , ViewP->size());
+   }
 }                                                                      /*}}}*/
 // CacheGenerator::WriteStringInMap                                    /*{{{*/
 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
                                        const unsigned long &Len) {
+   size_t oldSize = Map.Size();
    void const * const oldMap = Map.Data();
    map_stringitem_t const index = Map.WriteString(String, Len);
    if (index != 0)
-      ReMap(oldMap, Map.Data());
+      ReMap(oldMap, Map.Data(), oldSize);
    return index;
 }
                                                                        /*}}}*/
 // CacheGenerator::WriteStringInMap                                    /*{{{*/
 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
+   size_t oldSize = Map.Size();
    void const * const oldMap = Map.Data();
    map_stringitem_t const index = Map.WriteString(String);
    if (index != 0)
-      ReMap(oldMap, Map.Data());
+      ReMap(oldMap, Map.Data(), oldSize);
    return index;
 }
                                                                        /*}}}*/
 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
+   size_t oldSize = Map.Size();
    void const * const oldMap = Map.Data();
    map_pointer_t const index = Map.Allocate(size);
    if (index != 0)
-      ReMap(oldMap, Map.Data());
+      ReMap(oldMap, Map.Data(), oldSize);
    return index;
 }
                                                                        /*}}}*/
@@ -212,37 +249,43 @@ bool pkgCacheGenerator::MergeList(ListParser &List,
    while (List.Step() == true)
    {
       string const PackageName = List.Package();
-      if (PackageName.empty() == true)
-        return false;
+      if (PackageName.empty() == true) {
+          _error->Warning("Encountered a section with no Package: header");
+          continue;
+      }
 
       Counter++;
       if (Counter % 100 == 0 && Progress != 0)
         Progress->Progress(List.Offset());
 
-      string Arch = List.Architecture();
-      string const Version = List.Version();
+      APT::StringView Arch = List.Architecture();
+      Dynamic<APT::StringView> DynArch(Arch);
+      APT::StringView Version = List.Version();
+      Dynamic<APT::StringView> DynVersion(Version);
       if (Version.empty() == true && Arch.empty() == true)
       {
         // package descriptions
         if (MergeListGroup(List, PackageName) == false)
-           return false;
+           continue;
         continue;
       }
 
       // Get a pointer to the package structure
       pkgCache::PkgIterator Pkg;
       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
-      if (NewPackage(Pkg, PackageName, Arch) == false)
+      if (NewPackage(Pkg, PackageName, Arch) == false) {
         // TRANSLATOR: The first placeholder is a package name,
         // the other two should be copied verbatim as they include debug info
-        return _error->Error(_("Error occurred while processing %s (%s%d)"),
+        _error->Warning(_("Error occurred while processing %s (%s%d)"),
                              PackageName.c_str(), "NewPackage", 1);
+         continue;
+      }
 
 
       if (Version.empty() == true)
       {
         if (MergeListPackage(List, Pkg) == false)
-           return false;
+           continue;
       }
       else
       {
@@ -297,20 +340,18 @@ bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator
    pkgCache::VerIterator Ver(Cache);
    Dynamic<pkgCache::VerIterator> DynVer(Ver);
    if (List.UsePackage(Pkg, Ver) == false)
-      return _error->Error(_("Error occurred while processing %s (%s%d)"),
+      return _error->Warning(_("Error occurred while processing %s (%s%d)"),
                           Pkg.Name(), "UsePackage", 1);
 
    // Find the right version to write the description
-   MD5SumValue CurMd5 = List.Description_md5();
-   if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
-      return true;
+   StringView CurMd5 = List.Description_md5();
    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
    {
       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 
       // a version can only have one md5 describing it
-      if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
+      if (VerDesc.end() == true || Cache.ViewString(VerDesc->md5sum) != CurMd5)
         continue;
 
       map_stringitem_t md5idx = VerDesc->md5sum;
@@ -318,7 +359,7 @@ bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator
       {
         // don't add a new description if we have one for the given
         // md5 && language
-        if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
+        if (IsDuplicateDescription(Cache, VerDesc, CurMd5, *CurLang) == true)
            continue;
 
         AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
@@ -333,7 +374,7 @@ bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator
                                                                        /*}}}*/
 // CacheGenerator::MergeListVersion                                    /*{{{*/
 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
-                                        std::string const &Version, pkgCache::VerIterator* &OutVer)
+                                        APT::StringView const &Version, pkgCache::VerIterator* &OutVer)
 {
    pkgCache::VerIterator Ver = Pkg.VersionList();
    Dynamic<pkgCache::VerIterator> DynVer(Ver);
@@ -348,13 +389,28 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
       int Res = 1;
       for (; Ver.end() == false; LastVer = &Ver->NextVer, ++Ver)
       {
-        Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
+        char const * const VerStr = Ver.VerStr();
+        Res = Cache.VS->DoCmpVersion(Version.data(), Version.data() + Version.length(),
+              VerStr, VerStr + strlen(VerStr));
         // Version is higher as current version - insert here
         if (Res > 0)
            break;
         // Versionstrings are equal - is hash also equal?
-        if (Res == 0 && List.SameVersion(Hash, Ver) == true)
-           break;
+        if (Res == 0)
+        {
+           if (List.SameVersion(Hash, Ver) == true)
+              break;
+           // sort (volatile) sources above not-sources like the status file
+           if ((CurrentFile->Flags & pkgCache::Flag::NotSource) == 0)
+           {
+              auto VF = Ver.FileList();
+              for (; VF.end() == false; ++VF)
+                 if (VF.File().Flagged(pkgCache::Flag::NotSource) == false)
+                    break;
+              if (VF.end() == true)
+                 break;
+           }
+        }
         // proceed with the next till we have either the right
         // or we found another version (which will be lower)
       }
@@ -363,11 +419,11 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
       {
         if (List.UsePackage(Pkg,Ver) == false)
-           return _error->Error(_("Error occurred while processing %s (%s%d)"),
+           return _error->Warning(_("Error occurred while processing %s (%s%d)"),
                                 Pkg.Name(), "UsePackage", 2);
 
         if (NewFileVer(Ver,List) == false)
-           return _error->Error(_("Error occurred while processing %s (%s%d)"),
+           return _error->Warning(_("Error occurred while processing %s (%s%d)"),
                                 Pkg.Name(), "NewFileVer", 1);
 
         // Read only a single record and return
@@ -383,8 +439,8 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
 
    // Add a new version
    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
-   if (verindex == 0 && _error->PendingError())
-      return _error->Error(_("Error occurred while processing %s (%s%d)"),
+   if (unlikely(verindex == 0))
+      return _error->Warning(_("Error occurred while processing %s (%s%d)"),
                           Pkg.Name(), "NewVersion", 1);
 
    if (oldMap != Map.Data())
@@ -392,15 +448,15 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
    *LastVer = verindex;
 
    if (unlikely(List.NewVersion(Ver) == false))
-      return _error->Error(_("Error occurred while processing %s (%s%d)"),
+      return _error->Warning(_("Error occurred while processing %s (%s%d)"),
                           Pkg.Name(), "NewVersion", 2);
 
    if (unlikely(List.UsePackage(Pkg,Ver) == false))
-      return _error->Error(_("Error occurred while processing %s (%s%d)"),
+      return _error->Warning(_("Error occurred while processing %s (%s%d)"),
                           Pkg.Name(), "UsePackage", 3);
 
    if (unlikely(NewFileVer(Ver,List) == false))
-      return _error->Error(_("Error occurred while processing %s (%s%d)"),
+      return _error->Warning(_("Error occurred while processing %s (%s%d)"),
                           Pkg.Name(), "NewFileVer", 2);
 
    pkgCache::GrpIterator Grp = Pkg.Group();
@@ -421,12 +477,12 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
         Dynamic<pkgCache::VerIterator> DynV(V);
         for (; V.end() != true; ++V)
            if (unlikely(AddImplicitDepends(V, Pkg) == false))
-              return _error->Error(_("Error occurred while processing %s (%s%d)"),
+              return _error->Warning(_("Error occurred while processing %s (%s%d)"),
                                    Pkg.Name(), "AddImplicitDepends", 1);
       }
    }
    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
-      return _error->Error(_("Error occurred while processing %s (%s%d)"),
+      return _error->Warning(_("Error occurred while processing %s (%s%d)"),
                           Pkg.Name(), "AddImplicitDepends", 2);
 
    // Read only a single record and return
@@ -437,9 +493,7 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
    }
 
    /* Record the Description(s) based on their master md5sum */
-   MD5SumValue CurMd5 = List.Description_md5();
-   if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
-      return true;
+   StringView CurMd5 = List.Description_md5();
 
    /* Before we add a new description we first search in the group for
       a version with a description of the same MD5 - if so we reuse this
@@ -450,7 +504,7 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
       for (pkgCache::VerIterator V = P.VersionList();
           V.end() == false; ++V)
       {
-        if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
+        if (V->DescriptionList == 0 || Cache.ViewString(V.DescriptionList()->md5sum) != CurMd5)
            continue;
         Ver->DescriptionList = V->DescriptionList;
       }
@@ -465,14 +519,14 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
    return true;
 }
                                                                        /*}}}*/
-bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
+bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, APT::StringView CurMd5, map_stringitem_t &md5idx) /*{{{*/
 {
    pkgCache::DescIterator Desc;
    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 
    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
-   if (unlikely(descindex == 0 && _error->PendingError()))
-      return _error->Error(_("Error occurred while processing %s (%s%d)"),
+   if (unlikely(descindex == 0))
+      return _error->Warning(_("Error occurred while processing %s (%s%d)"),
            Ver.ParentPkg().Name(), "NewDescription", 1);
 
    md5idx = Desc->md5sum;
@@ -486,7 +540,7 @@ bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterato
    *LastNextDesc = descindex;
 
    if (NewFileDesc(Desc,List) == false)
-      return _error->Error(_("Error occurred while processing %s (%s%d)"),
+      return _error->Warning(_("Error occurred while processing %s (%s%d)"),
            Ver.ParentPkg().Name(), "NewFileDesc", 1);
 
    return true;
@@ -496,8 +550,9 @@ bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterato
 // CacheGenerator::NewGroup - Add a new group                          /*{{{*/
 // ---------------------------------------------------------------------
 /* This creates a new group structure and adds it to the hash table */
-bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
+bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, StringView Name)
 {
+   Dynamic<StringView> DName(Name);
    Grp = Cache.FindGrp(Name);
    if (Grp.end() == false)
       return true;
@@ -516,7 +571,8 @@ bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
    // Insert it into the hash table
    unsigned long const Hash = Cache.Hash(Name);
    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash];
-   while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
+
+   while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + *insertAt)->Name)) > 0)
       insertAt = &(Cache.GrpP + *insertAt)->Next;
    Grp->Next = *insertAt;
    *insertAt = Group;
@@ -528,9 +584,11 @@ bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 // CacheGenerator::NewPackage - Add a new package                      /*{{{*/
 // ---------------------------------------------------------------------
 /* This creates a new package structure and adds it to the hash table */
-bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
-                                       const string &Arch) {
+bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg, StringView Name,
+                                       StringView Arch) {
    pkgCache::GrpIterator Grp;
+   Dynamic<StringView> DName(Name);
+   Dynamic<StringView> DArch(Arch);
    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
    if (unlikely(NewGroup(Grp, Name) == false))
       return false;
@@ -562,41 +620,61 @@ bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name
       // Insert it into the hash table
       map_id_t const Hash = Cache.Hash(Name);
       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash];
-      while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0)
+      while (*insertAt != 0 && StringViewCompareFast(Name, Cache.ViewString((Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name)) > 0)
         insertAt = &(Cache.PkgP + *insertAt)->NextPackage;
       Pkg->NextPackage = *insertAt;
       *insertAt = Package;
    }
    else // Group the Packages together
    {
-      // but first get implicit provides done
-      if (APT::Configuration::checkArchitecture(Pkg.Arch()) == true)
+      // if sibling is provided by another package, this one is too
       {
         pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false); // native or any foreign pkg will do
-        if (M.end() == false)
-           for (pkgCache::PrvIterator Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
+        if (M.end() == false) {
+           pkgCache::PrvIterator Prv;
+           Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
+           for (Prv = M.ProvidesList(); Prv.end() == false; ++Prv)
            {
               if ((Prv->Flags & pkgCache::Flag::ArchSpecific) != 0)
                  continue;
               pkgCache::VerIterator Ver = Prv.OwnerVer();
+              Dynamic<pkgCache::VerIterator> DynVer(Ver);
               if ((Ver->MultiArch & pkgCache::Version::Allowed) == pkgCache::Version::Allowed ||
                   ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign &&
                        (Prv->Flags & pkgCache::Flag::MultiArchImplicit) == 0))
+              {
+                 if (APT::Configuration::checkArchitecture(Ver.ParentPkg().Arch()) == false)
+                    continue;
                  if (NewProvides(Ver, Pkg, Prv->ProvideVersion, Prv->Flags) == false)
                     return false;
+              }
            }
+        }
+      }
+      // let M-A:foreign package siblings provide this package
+      {
+        pkgCache::PkgIterator P;
+        pkgCache::VerIterator Ver;
+        Dynamic<pkgCache::PkgIterator> DynP(P);
+        Dynamic<pkgCache::VerIterator> DynVer(Ver);
 
-        for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
-           for (pkgCache::VerIterator Ver = P.VersionList(); Ver.end() == false; ++Ver)
+        for (P = Grp.PackageList(); P.end() == false;  P = Grp.NextPkg(P))
+        {
+           if (APT::Configuration::checkArchitecture(P.Arch()) == false)
+              continue;
+           for (Ver = P.VersionList(); Ver.end() == false; ++Ver)
               if ((Ver->MultiArch & pkgCache::Version::Foreign) == pkgCache::Version::Foreign)
                  if (NewProvides(Ver, Pkg, Ver->VerStr, pkgCache::Flag::MultiArchImplicit) == false)
                     return false;
+        }
       }
       // and negative dependencies, don't forget negative dependencies
       {
         pkgCache::PkgIterator const M = Grp.FindPreferredPkg(false);
-        if (M.end() == false)
-           for (pkgCache::DepIterator Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
+        if (M.end() == false) {
+           pkgCache::DepIterator Dep;
+           Dynamic<pkgCache::DepIterator> DynDep(Dep);
+           for (Dep = M.RevDependsList(); Dep.end() == false; ++Dep)
            {
               if ((Dep->CompareOp & (pkgCache::Dep::ArchSpecific | pkgCache::Dep::MultiArchImplicit)) != 0)
                  continue;
@@ -604,10 +682,12 @@ bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name
                     Dep->Type != pkgCache::Dep::Replaces)
                  continue;
               pkgCache::VerIterator Ver = Dep.ParentVer();
+              Dynamic<pkgCache::VerIterator> DynVer(Ver);
               map_pointer_t * unused = NULL;
               if (NewDepends(Pkg, Ver, Dep->Version, Dep->CompareOp, Dep->Type, unused) == false)
                  return false;
            }
+        }
       }
 
       // this package is the new last package
@@ -616,6 +696,52 @@ bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name
       LastPkg->NextPackage = Package;
    }
    Grp->LastPackage = Package;
+
+   // lazy-create foo (of amd64) provides foo:amd64 at the time we first need it
+   if (Arch == "any")
+   {
+      size_t const found = Name.rfind(':');
+      StringView ArchA = Name.substr(found + 1);
+      if (ArchA != "any")
+      {
+        // ArchA is used inside the loop which might remap (NameA is not used)
+        Dynamic<StringView> DynArchA(ArchA);
+        StringView NameA = Name.substr(0, found);
+        pkgCache::PkgIterator PkgA = Cache.FindPkg(NameA, ArchA);
+        Dynamic<pkgCache::PkgIterator> DynPkgA(PkgA);
+        if (PkgA.end())
+        {
+           Dynamic<StringView> DynNameA(NameA);
+           if (NewPackage(PkgA, NameA, ArchA) == false)
+              return false;
+        }
+        if (unlikely(PkgA.end()))
+           return _error->Fatal("NewPackage was successful for %s:%s,"
+                 "but the package doesn't exist anyhow!",
+                 NameA.to_string().c_str(), ArchA.to_string().c_str());
+        else
+        {
+           pkgCache::PrvIterator Prv = PkgA.ProvidesList();
+           for (; Prv.end() == false; ++Prv)
+           {
+              if (Prv.IsMultiArchImplicit())
+                 continue;
+              pkgCache::VerIterator V = Prv.OwnerVer();
+              if (ArchA != V.ParentPkg().Arch())
+                 continue;
+              if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
+                 return false;
+           }
+           pkgCache::VerIterator V = PkgA.VersionList();
+           Dynamic<pkgCache::VerIterator> DynV(V);
+           for (; V.end() == false; ++V)
+           {
+              if (NewProvides(V, Pkg, V->VerStr, pkgCache::Flag::MultiArchImplicit | pkgCache::Flag::ArchSpecific) == false)
+                 return false;
+           }
+        }
+      }
+   }
    return true;
 }
                                                                        /*}}}*/
@@ -624,9 +750,8 @@ bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
                                           pkgCache::PkgIterator &P,
                                           pkgCache::VerIterator &V)
 {
-   // copy P.Arch() into a string here as a cache remap
-   // in NewDepends() later may alter the pointer location
-   string Arch = P.Arch() == NULL ? "" : P.Arch();
+   APT::StringView Arch = P.Arch() == NULL ? "" : P.Arch();
+   Dynamic<APT::StringView> DynArch(Arch);
    map_pointer_t *OldDepLast = NULL;
    /* MultiArch handling introduces a lot of implicit Dependencies:
       - MultiArch: same â†’ Co-Installable if they have the same version
@@ -727,7 +852,7 @@ bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 // ---------------------------------------------------------------------
 /* This puts a version structure in the linked list */
 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
-                                           const string &VerStr,
+                                           APT::StringView const &VerStr,
                                            map_pointer_t const ParentPkg,
                                            unsigned short const Hash,
                                            map_pointer_t const Next)
@@ -756,8 +881,8 @@ map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
            continue;
         for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
         {
-           int const cmp = strcmp(V.VerStr(), VerStr.c_str());
-           if (cmp == 0)
+           int const cmp = strncmp(V.VerStr(), VerStr.data(), VerStr.length());
+           if (cmp == 0 && V.VerStr()[VerStr.length()] == '\0')
            {
               Ver->VerStr = V->VerStr;
               return Version;
@@ -814,7 +939,7 @@ bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 /* This puts a description structure in the linked list */
 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
                                            const string &Lang,
-                                           const MD5SumValue &md5sum,
+                                           APT::StringView md5sum,
                                            map_stringitem_t const idxmd5str)
 {
    // Get a structure
@@ -834,7 +959,7 @@ map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
       Desc->md5sum = idxmd5str;
    else
    {
-      map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
+      map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum);
       if (unlikely(idxmd5sum == 0))
         return 0;
       Desc->md5sum = idxmd5sum;
@@ -947,14 +1072,17 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 /* This creates a Group and the Package to link this dependency to if
    needed and handles also the caching of the old endpoint */
 bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
-                                              const string &PackageName,
-                                              const string &Arch,
-                                              const string &Version,
+                                              StringView PackageName,
+                                              StringView Arch,
+                                              StringView Version,
                                               uint8_t const Op,
                                               uint8_t const Type)
 {
    pkgCache::GrpIterator Grp;
    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
+   Dynamic<StringView> DynPackageName(PackageName);
+   Dynamic<StringView> DynArch(Arch);
+   Dynamic<StringView> DynVersion(Version);
    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
       return false;
 
@@ -963,7 +1091,7 @@ bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
    {
       int const CmpOp = Op & 0x0F;
       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
-      if (CmpOp == pkgCache::Dep::Equals && strcmp(Version.c_str(), Ver.VerStr()) == 0)
+      if (CmpOp == pkgCache::Dep::Equals && Version == Ver.VerStr())
         idxVersion = Ver->VerStr;
 
       if (idxVersion == 0)
@@ -1016,16 +1144,20 @@ bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver,
                                                                        /*}}}*/
 // ListParser::NewProvides - Create a Provides element                 /*{{{*/
 bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver,
-                                               const string &PkgName,
-                                               const string &PkgArch,
-                                               const string &Version,
+                                               StringView PkgName,
+                                               StringView PkgArch,
+                                               StringView Version,
                                                uint8_t const Flags)
 {
    pkgCache const &Cache = Owner->Cache;
+   Dynamic<StringView> DynPkgName(PkgName);
+   Dynamic<StringView> DynArch(PkgArch);
+   Dynamic<StringView> DynVersion(Version);
 
    // We do not add self referencing provides
    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
-       (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
+       (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)) &&
+        (Version.empty() || Version == Ver.VerStr()))
       return true;
 
    // Locate the target package
@@ -1069,10 +1201,14 @@ bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver,
 }
                                                                        /*}}}*/
 // ListParser::NewProvidesAllArch - add provides for all architectures /*{{{*/
-bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, string const &Package,
-                               string const &Version, uint8_t const Flags) {
+bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, StringView Package,
+                               StringView Version, uint8_t const Flags) {
    pkgCache &Cache = Owner->Cache;
-   pkgCache::GrpIterator const Grp = Cache.FindGrp(Package);
+   pkgCache::GrpIterator Grp = Cache.FindGrp(Package);
+   Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
+   Dynamic<StringView> DynPackage(Package);
+   Dynamic<StringView> DynVersion(Version);
+
    if (Grp.end() == true)
       return NewProvides(Ver, Package, Cache.NativeArch(), Version, Flags);
    else
@@ -1086,8 +1222,11 @@ bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, string c
 
       bool const isImplicit = (Flags & pkgCache::Flag::MultiArchImplicit) == pkgCache::Flag::MultiArchImplicit;
       bool const isArchSpecific = (Flags & pkgCache::Flag::ArchSpecific) == pkgCache::Flag::ArchSpecific;
-      pkgCache::PkgIterator const OwnerPkg = Ver.ParentPkg();
-      for (pkgCache::PkgIterator Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
+      pkgCache::PkgIterator OwnerPkg = Ver.ParentPkg();
+      Dynamic<pkgCache::PkgIterator> DynOwnerPkg(OwnerPkg);
+      pkgCache::PkgIterator Pkg;
+      Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
+      for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
       {
         if (isImplicit && OwnerPkg == Pkg)
            continue;
@@ -1136,6 +1275,36 @@ bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
    Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
    Cache.HeaderP->ReleaseFileCount++;
 
+   return true;
+}
+                                                                       /*}}}*/
+// ListParser::NewTag - Create a Tag element                           /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool pkgCacheListParser::NewTag(pkgCache::VerIterator &Ver,
+                                          const char *NameStart,
+                                          unsigned int NameSize)
+{
+   return Owner->NewTag(Ver, NameStart, NameSize);
+}
+bool pkgCacheGenerator::NewTag(pkgCache::VerIterator &Ver,
+                                          const char *NameStart,
+                                          unsigned int NameSize)
+{
+   // Get a structure
+   unsigned long Tagg = AllocateInMap(sizeof(pkgCache::Tag));
+   if (Tagg == 0)
+      return false;
+   Cache.HeaderP->TagCount++;
+   
+   // Fill it in
+   pkgCache::TagIterator Tg(Cache,Cache.TagP + Tagg);
+   Tg->Name = WriteStringInMap(NameStart,NameSize);
+   if (Tg->Name == 0)
+      return false;
+   Tg->NextTag = Ver->TagList;
+   Ver->TagList = Tg.Index();
+   
    return true;
 }
                                                                        /*}}}*/
@@ -1200,23 +1369,21 @@ bool pkgCacheGenerator::SelectFile(std::string const &File,
 map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, const char *S,
                                                 unsigned int Size)
 {
-   std::string const key(S, Size);
-
-   std::map<std::string,map_stringitem_t> * strings;
+   auto strings = &strMixed;
    switch(type) {
       case MIXED: strings = &strMixed; break;
       case PKGNAME: strings = &strPkgNames; break;
       case VERSIONNUMBER: strings = &strVersions; break;
       case SECTION: strings = &strSections; break;
-      default: _error->Fatal("Unknown enum type used for string storage of '%s'", key.c_str()); return 0;
+      default: _error->Fatal("Unknown enum type used for string storage of '%.*s'", Size, S); return 0;
    }
 
-   std::map<std::string,map_stringitem_t>::const_iterator const item = strings->find(key);
+   auto const item = strings->find({S, Size, nullptr, 0});
    if (item != strings->end())
-      return item->second;
+      return item->item;
 
    map_stringitem_t const idxString = WriteStringInMap(S,Size);
-   strings->insert(std::make_pair(key, idxString));
+   strings->insert({nullptr, Size, this, idxString});
    return idxString;
 }
                                                                        /*}}}*/
@@ -1225,12 +1392,19 @@ map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, cons
 /* This just verifies that each file in the list of index files exists,
    has matching attributes with the cache and the cache does not have
    any extra files. */
+class APT_HIDDEN ScopedErrorRevert {
+public:
+   ScopedErrorRevert() { _error->PushToStack(); }
+   ~ScopedErrorRevert() { _error->RevertToStack(); }
+};
 static bool CheckValidity(const string &CacheFile,
                           pkgSourceList &List,
                           FileIterator const Start,
                           FileIterator const End,
-                          MMap **OutMap = 0)
+                          MMap **OutMap = 0,
+                         pkgCache **OutCache = 0)
 {
+   ScopedErrorRevert ser;
    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
    // No file, certainly invalid
    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
@@ -1249,18 +1423,20 @@ static bool CheckValidity(const string &CacheFile,
 
    // Map it
    FileFd CacheF(CacheFile,FileFd::ReadOnly);
-   SPtr<MMap> Map = new MMap(CacheF,0);
-   pkgCache Cache(Map);
-   if (_error->PendingError() == true || Map->Size() == 0)
+   std::unique_ptr<MMap> Map(new MMap(CacheF,0));
+   if (unlikely(Map->validData()) == false)
+      return false;
+   std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
+   pkgCache &Cache = *CacheP.get();
+   if (_error->PendingError() || Map->Size() == 0)
    {
       if (Debug == true)
         std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
-      _error->Discard();
       return false;
    }
 
-   SPtrArray<bool> RlsVisited = new bool[Cache.HeaderP->ReleaseFileCount];
-   memset(RlsVisited,0,sizeof(*RlsVisited)*Cache.HeaderP->ReleaseFileCount);
+   std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
+   memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
    std::vector<pkgIndexFile *> Files;
    for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
    {
@@ -1294,8 +1470,8 @@ static bool CheckValidity(const string &CacheFile,
 
    /* Now we check every index file, see if it is in the cache,
       verify the IMS data and check that it is on the disk too.. */
-   SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
-   memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
+   std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
+   memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
    for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
    {
       if (Debug == true)
@@ -1334,14 +1510,15 @@ static bool CheckValidity(const string &CacheFile,
       if (Debug == true)
       {
         std::clog << "Validity failed because of pending errors:" << std::endl;
-        _error->DumpErrors();
+        _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
       }
-      _error->Discard();
       return false;
    }
-   
+
    if (OutMap != 0)
-      *OutMap = Map.UnGuard();
+      *OutMap = Map.release();
+   if (OutCache != 0)
+      *OutCache = CacheP.release();
    return true;
 }
                                                                        /*}}}*/
@@ -1373,17 +1550,14 @@ static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator
 }
                                                                        /*}}}*/
 // BuildCache - Merge the list of index files into the cache           /*{{{*/
-static bool BuildCache(pkgCacheGenerator &Gen,
+static void BuildCache(pkgCacheGenerator &Gen,
                       OpProgress * const Progress,
                       map_filesize_t &CurrentSize,map_filesize_t TotalSize,
                       pkgSourceList const * const List,
                       FileIterator const Start, FileIterator const End)
 {
-   std::vector<pkgIndexFile *> Files;
-   bool mergeFailure = false;
-
    auto const indexFileMerge = [&](pkgIndexFile * const I) {
-      if (I->HasPackages() == false || mergeFailure)
+      if (I->HasPackages() == false)
         return;
 
       if (I->Exists() == false)
@@ -1402,7 +1576,7 @@ static bool BuildCache(pkgCacheGenerator &Gen,
       CurrentSize += Size;
 
       if (I->Merge(Gen,Progress) == false)
-        mergeFailure = true;
+        return;
    };
 
    if (List !=  NULL)
@@ -1417,13 +1591,11 @@ static bool BuildCache(pkgCacheGenerator &Gen,
         }
 
         if ((*i)->Merge(Gen, Progress) == false)
-           return false;
+           continue;
 
         std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
         if (Indexes != NULL)
            std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
-        if (mergeFailure)
-           return false;
       }
    }
 
@@ -1431,10 +1603,7 @@ static bool BuildCache(pkgCacheGenerator &Gen,
    {
       Gen.SelectReleaseFile("", "");
       std::for_each(Start, End, indexFileMerge);
-      if (mergeFailure)
-        return false;
    }
-   return true;
 }
                                                                        /*}}}*/
 // CacheGenerator::MakeStatusCache - Construct the status cache                /*{{{*/
@@ -1462,7 +1631,7 @@ static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * con
       std::string const &FileName)
 {
    FileFd SCacheF(FileName, FileFd::WriteAtomic);
-   if (_error->PendingError() == true)
+   if (SCacheF.IsOpen() == false || SCacheF.Failed())
       return false;
 
    fchmod(SCacheF.Fd(),0644);
@@ -1470,36 +1639,48 @@ static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * con
    // Write out the main data
    if (SCacheF.Write(Map->Data(),Map->Size()) == false)
       return _error->Error(_("IO Error saving source cache"));
-   SCacheF.Sync();
 
    // Write out the proper header
    Gen->GetCache().HeaderP->Dirty = false;
+   Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
    if (SCacheF.Seek(0) == false ||
         SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
       return _error->Error(_("IO Error saving source cache"));
    Gen->GetCache().HeaderP->Dirty = true;
-   SCacheF.Sync();
    return true;
 }
 static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
-      SPtr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
+      std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
 {
-   Map = CreateDynamicMMap(NULL, 0);
+   Map.reset(CreateDynamicMMap(NULL, 0));
+   if (unlikely(Map->validData()) == false)
+      return false;
    FileFd CacheF(FileName, FileFd::ReadOnly);
+   if (CacheF.IsOpen() == false || CacheF.Failed())
+      return false;
+   _error->PushToStack();
    map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
-   if ((alloc == 0 && _error->PendingError())
-        || CacheF.Read((unsigned char *)Map->Data() + alloc,
-           CacheF.Size()) == false)
+   bool const newError = _error->PendingError();
+   _error->MergeWithStack();
+   if (alloc == 0 && newError)
       return false;
-   Gen.reset(new pkgCacheGenerator(Map.Get(),Progress));
-   return true;
+   if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
+      return false;
+   Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
+   return Gen->Start();
 }
-APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
+bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
                        MMap **OutMap, bool AllowMem)
    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
-                       MMap **OutMap,bool AllowMem)
+                       MMap **OutMap,bool)
 {
+   return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
+}
+bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
+                       MMap **OutMap,pkgCache **OutCache, bool)
+{
+   // FIXME: deprecate the ignored AllowMem parameter
    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 
    std::vector<pkgIndexFile *> Files;
@@ -1530,7 +1711,8 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
    bool srcpkgcache_fine = false;
    bool volatile_fine = List.GetVolatileFiles().empty();
 
-   if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL) == true)
+   if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
+                    volatile_fine ? OutCache : NULL) == true)
    {
       if (Debug == true)
         std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
@@ -1564,20 +1746,12 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
 
       if (Debug == true)
         std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
-
-      if (Writeable == false && AllowMem == false)
-      {
-        if (CacheFile.empty() == false)
-           return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
-        else if (SrcCacheFile.empty() == false)
-           return _error->Error(_("Unable to write to %s"),flNotFile(SrcCacheFile).c_str());
-        else
-           return _error->Error("Unable to create caches as file usage is disabled, but memory not allowed either!");
-      }
    }
 
    // At this point we know we need to construct something, so get storage ready
-   SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL, 0);
+   std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
+   if (unlikely(Map->validData()) == false)
+      return false;
    if (Debug == true)
       std::clog << "Open memory Map (not filebased)" << std::endl;
 
@@ -1598,15 +1772,16 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
    {
       if (Debug == true)
         std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
-      Gen.reset(new pkgCacheGenerator(Map.Get(),Progress));
+      Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
+      if (Gen->Start() == false)
+        return false;
 
       TotalSize += ComputeSize(&List, Files.begin(),Files.end());
-      if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
-              Files.end(),Files.end()) == false)
-        return false;
+      BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
+              Files.end(),Files.end());
 
       if (Writeable == true && SrcCacheFile.empty() == false)
-        if (writeBackMMapToFile(Gen.get(), Map.Get(), SrcCacheFile) == false)
+        if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
            return false;
    }
 
@@ -1614,17 +1789,16 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
    {
       if (Debug == true)
         std::clog << "Building status cache in pkgcache.bin now" << std::endl;
-      if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
-              Files.begin(), Files.end()) == false)
-        return false;
+      BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
+              Files.begin(), Files.end());
 
       if (Writeable == true && CacheFile.empty() == false)
-        if (writeBackMMapToFile(Gen.get(), Map.Get(), CacheFile) == false)
+        if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
            return false;
    }
 
    if (Debug == true)
-      std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;
+      std::clog << "Caches done. " << (volatile_fine ? "No volatile files, so we are done here." : "Now bring in the volatile files") << std::endl;
 
    if (volatile_fine == false)
    {
@@ -1637,13 +1811,12 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
       }
 
       Files = List.GetVolatileFiles();
-      if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
-              Files.begin(), Files.end()) == false)
-        return false;
+      BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
+              Files.begin(), Files.end());
    }
 
    if (OutMap != nullptr)
-      *OutMap = Map.UnGuard();
+      *OutMap = Map.release();
 
    if (Debug == true)
       std::clog << "Everything is ready for shipping" << std::endl;
@@ -1651,9 +1824,12 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
 }
                                                                        /*}}}*/
 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
+class APT_HIDDEN ScopedErrorMerge {
+public:
+   ScopedErrorMerge() { _error->PushToStack(); }
+   ~ScopedErrorMerge() { _error->MergeWithStack(); }
+};
+bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
 {
@@ -1661,35 +1837,36 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O
    if (_system->AddStatusFiles(Files) == false)
       return false;
 
-   SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL, 0);
+   ScopedErrorMerge sem;
+   std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
+   if (unlikely(Map->validData()) == false)
+      return false;
    map_filesize_t CurrentSize = 0;
    map_filesize_t TotalSize = 0;
-   
    TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
-   
+
    // Build the status cache
    if (Progress != NULL)
       Progress->OverallProgress(0,1,1,_("Reading package lists"));
-   pkgCacheGenerator Gen(Map.Get(),Progress);
-   if (_error->PendingError() == true)
-      return false;
-   if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
-                 Files.begin(), Files.end()) == false)
+   pkgCacheGenerator Gen(Map.get(),Progress);
+   if (Gen.Start() == false || _error->PendingError() == true)
       return false;
+   BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
+                 Files.begin(), Files.end());
 
    if (_error->PendingError() == true)
       return false;
-   *OutMap = Map.UnGuard();
+   *OutMap = Map.release();
    
    return true;
 }
                                                                        /*}}}*/
 // IsDuplicateDescription                                              /*{{{*/
-static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
-                           MD5SumValue const &CurMd5, std::string const &CurLang)
+static bool IsDuplicateDescription(pkgCache &Cache, pkgCache::DescIterator Desc,
+                           APT::StringView CurMd5, std::string const &CurLang)
 {
    // Descriptions in the same link-list have all the same md5
-   if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
+   if (Desc.end() == true || Cache.ViewString(Desc->md5sum) != CurMd5)
       return false;
    for (; Desc.end() == false; ++Desc)
       if (Desc.LanguageCode() == CurLang)