From a9fe592842bfa17d91f4904d7fb0e3af3adebb17 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 8 Jul 2010 15:28:53 +0200 Subject: [PATCH] * apt-pkg/pkgcachegen.{cc,h}: - make the used MMap moveable (and therefore dynamic resizeable) by applying (some) mad pointer magic (Closes: #195018) --- apt-pkg/cacheiterators.h | 6 + apt-pkg/contrib/mmap.cc | 27 +++-- apt-pkg/deb/debindexfile.cc | 1 + apt-pkg/pkgcache.cc | 5 +- apt-pkg/pkgcache.h | 2 +- apt-pkg/pkgcachegen.cc | 229 ++++++++++++++++++++++++++---------- apt-pkg/pkgcachegen.h | 30 ++++- debian/changelog | 3 + 8 files changed, 226 insertions(+), 77 deletions(-) diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h index dfe5707e1..eb8dee5e3 100644 --- a/apt-pkg/cacheiterators.h +++ b/apt-pkg/cacheiterators.h @@ -75,6 +75,12 @@ template class pkgCache::Iterator : inline bool IsGood() const { return S && Owner && ! end();}; inline unsigned long Index() const {return S - OwnerPointer();}; + void ReOwn(pkgCache &newOwner, void const * const oldMap, void const * const newMap) { + if (S == 0) + return; + S += (Str*)(newMap) - (Str*)(oldMap); + } + // Constructors - look out for the variable assigning inline Iterator() : S(0), Owner(0) {}; inline Iterator(pkgCache &Owner,Str *T = 0) : S(T), Owner(&Owner) {}; diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc index d71066243..aa184b130 100644 --- a/apt-pkg/contrib/mmap.cc +++ b/apt-pkg/contrib/mmap.cc @@ -225,22 +225,22 @@ DynamicMMap::DynamicMMap(unsigned long Flags,unsigned long const &WorkSpace, // disable Moveable if we don't grow if (Grow == 0) - Flags &= ~Moveable; + this->Flags &= ~Moveable; #ifndef __linux__ // kfreebsd doesn't have mremap, so we use the fallback - if ((Flags & Moveable) == Moveable) - Flags |= Fallback; + if ((this->Flags & Moveable) == Moveable) + this->Flags |= Fallback; #endif #ifdef _POSIX_MAPPED_FILES - if ((Flags & Fallback) != Fallback) { + if ((this->Flags & Fallback) != Fallback) { // Set the permissions. int Prot = PROT_READ; int Map = MAP_PRIVATE | MAP_ANONYMOUS; - if ((Flags & ReadOnly) != ReadOnly) + if ((this->Flags & ReadOnly) != ReadOnly) Prot |= PROT_WRITE; - if ((Flags & Public) == Public) + if ((this->Flags & Public) == Public) Map = MAP_SHARED | MAP_ANONYMOUS; // use anonymous mmap() to get the memory @@ -314,7 +314,7 @@ unsigned long DynamicMMap::Allocate(unsigned long ItemSize) // Look for a matching pool entry Pool *I; Pool *Empty = 0; - for (I = Pools; I != Pools + PoolCount; I++) + for (I = Pools; I != Pools + PoolCount; ++I) { if (I->ItemSize == 0) Empty = I; @@ -342,7 +342,11 @@ unsigned long DynamicMMap::Allocate(unsigned long ItemSize) { const unsigned long size = 20*1024; I->Count = size/ItemSize; + Pool* oldPools = Pools; Result = RawAllocate(size,ItemSize); + if (Pools != oldPools) + I += Pools - oldPools; + // Does the allocation failed ? if (Result == 0 && _error->PendingError()) return 0; @@ -365,7 +369,7 @@ unsigned long DynamicMMap::WriteString(const char *String, if (Len == (unsigned long)-1) Len = strlen(String); - unsigned long Result = RawAllocate(Len+1,0); + unsigned long const Result = RawAllocate(Len+1,0); if (Result == 0 && _error->PendingError()) return 0; @@ -395,16 +399,20 @@ bool DynamicMMap::Grow() { return _error->Error(_("Unable to increase the size of the MMap as the " "limit of %lu bytes is already reached."), Limit); - unsigned long const newSize = WorkSpace + 1024*1024; + unsigned long const newSize = WorkSpace + GrowFactor; if(Fd != 0) { Fd->Seek(newSize - 1); char C = 0; Fd->Write(&C,sizeof(C)); } + + unsigned long const poolOffset = Pools - ((Pool*) Base); + if ((Flags & Fallback) != Fallback) { #if defined(_POSIX_MAPPED_FILES) && defined(__linux__) #ifdef MREMAP_MAYMOVE + if ((Flags & Moveable) == Moveable) Base = mremap(Base, WorkSpace, newSize, MREMAP_MAYMOVE); else @@ -425,6 +433,7 @@ bool DynamicMMap::Grow() { return false; } + Pools =(Pool*) Base + poolOffset; WorkSpace = newSize; return true; } diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index 6d9e99497..5e6db3f38 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -277,6 +277,7 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const // Store the IMS information pkgCache::PkgFileIterator File = Gen.GetCurFile(); + pkgCacheGenerator::Dynamic DynFile(File); struct stat St; if (fstat(Pkg.Fd(),&St) != 0) return _error->Errno("fstat","Failed to stat"); diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 68aa83d0c..8af8ef7de 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -119,7 +119,7 @@ pkgCache::pkgCache(MMap *Map, bool DoMap) : Map(*Map) // Cache::ReMap - Reopen the cache file /*{{{*/ // --------------------------------------------------------------------- /* If the file is already closed then this will open it open it. */ -bool pkgCache::ReMap() +bool pkgCache::ReMap(bool const &Errorchecks) { // Apply the typecasts. HeaderP = (Header *)Map.Data(); @@ -135,6 +135,9 @@ bool pkgCache::ReMap() StringItemP = (StringItem *)Map.Data(); StrP = (char *)Map.Data(); + if (Errorchecks == false) + return true; + if (Map.Size() == 0 || HeaderP == 0) return _error->Error(_("Empty package cache")); diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 426bb9f13..799521784 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -174,7 +174,7 @@ class pkgCache /*{{{*/ StringItem *StringItemP; char *StrP; - virtual bool ReMap(); + virtual bool ReMap(bool const &Errorchecks = true); inline bool Sync() {return Map.Sync();}; inline MMap &GetMap() {return Map;}; inline void *DataEnd() {return ((unsigned char *)Map.Data()) + Map.Size();}; diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 62e457734..18bad6727 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -36,6 +36,7 @@ #include /*}}}*/ typedef vector::iterator FileIterator; +template std::set pkgCacheGenerator::Dynamic::toReMap; // CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/ // --------------------------------------------------------------------- @@ -61,8 +62,12 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) : // Starting header *Cache.HeaderP = pkgCache::Header(); - Cache.HeaderP->VerSysName = WriteStringInMap(_system->VS->Label); - Cache.HeaderP->Architecture = WriteStringInMap(_config->Find("APT::Architecture")); + map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label); + Cache.HeaderP->VerSysName = idxVerSysName; + map_ptrloc const idxArchitecture = WriteStringInMap(_config->Find("APT::Architecture")); + Cache.HeaderP->Architecture = idxArchitecture; + if (unlikely(idxVerSysName == 0 || idxArchitecture == 0)) + return; Cache.ReMap(); } else @@ -96,19 +101,65 @@ pkgCacheGenerator::~pkgCacheGenerator() Map.Sync(0,sizeof(pkgCache::Header)); } /*}}}*/ +void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/ + if (oldMap == newMap) + return; + + Cache.ReMap(false); + + CurrentFile += (pkgCache::PackageFile*) newMap - (pkgCache::PackageFile*) oldMap; + + for (size_t i = 0; i < _count(UniqHash); ++i) + if (UniqHash[i] != 0) + UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap; + + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); +} /*}}}*/ // CacheGenerator::WriteStringInMap /*{{{*/ -unsigned long pkgCacheGenerator::WriteStringInMap(const char *String, +map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String, const unsigned long &Len) { - return Map.WriteString(String, Len); + void const * const oldMap = Map.Data(); + map_ptrloc const index = Map.WriteString(String, Len); + if (index != 0) + ReMap(oldMap, Map.Data()); + return index; } /*}}}*/ // CacheGenerator::WriteStringInMap /*{{{*/ -unsigned long pkgCacheGenerator::WriteStringInMap(const char *String) { - return Map.WriteString(String); +map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) { + void const * const oldMap = Map.Data(); + map_ptrloc const index = Map.WriteString(String); + if (index != 0) + ReMap(oldMap, Map.Data()); + return index; } /*}}}*/ -unsigned long pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/ - return Map.Allocate(size); +map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/ + void const * const oldMap = Map.Data(); + map_ptrloc const index = Map.Allocate(size); + if (index != 0) + ReMap(oldMap, Map.Data()); + return index; } /*}}}*/ // CacheGenerator::MergeList - Merge the package list /*{{{*/ @@ -142,6 +193,7 @@ bool pkgCacheGenerator::MergeList(ListParser &List, { // Get a pointer to the package structure pkgCache::PkgIterator Pkg; + Dynamic DynPkg(Pkg); if (NewPackage(Pkg, PackageName, *arch) == false) return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str()); Counter++; @@ -158,6 +210,7 @@ bool pkgCacheGenerator::MergeList(ListParser &List, // (this has the bonus that we get MMap error when we run out // of MMap space) pkgCache::VerIterator Ver(Cache); + Dynamic DynVer(Ver); if (List.UsePackage(Pkg, Ver) == false) return _error->Error(_("Error occurred while processing %s (UsePackage1)"), PackageName.c_str()); @@ -165,11 +218,11 @@ bool pkgCacheGenerator::MergeList(ListParser &List, // Find the right version to write the description MD5SumValue CurMd5 = List.Description_md5(); Ver = Pkg.VersionList(); - map_ptrloc *LastVer = &Pkg->VersionList; - for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++) + for (; Ver.end() == false; ++Ver) { pkgCache::DescIterator Desc = Ver.DescriptionList(); + Dynamic DynDesc(Desc); map_ptrloc *LastDesc = &Ver->DescriptionList; bool duplicate=false; @@ -189,7 +242,11 @@ bool pkgCacheGenerator::MergeList(ListParser &List, if (MD5SumValue(Desc.md5()) == CurMd5) { // Add new description - *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc); + void const * const oldMap = Map.Data(); + map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc); + if (oldMap != Map.Data()) + LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; + *LastDesc = descindex; Desc->ParentPkg = Pkg.Index(); if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false) @@ -203,7 +260,9 @@ bool pkgCacheGenerator::MergeList(ListParser &List, } pkgCache::VerIterator Ver = Pkg.VersionList(); + Dynamic DynVer(Ver); map_ptrloc *LastVer = &Pkg->VersionList; + void const * oldMap = Map.Data(); int Res = 1; unsigned long const Hash = List.VersionHash(); for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++) @@ -242,21 +301,28 @@ bool pkgCacheGenerator::MergeList(ListParser &List, } // Add a new version - *LastVer = NewVersion(Ver,Version,*LastVer); + map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer); + if (verindex == 0 && _error->PendingError()) + return _error->Error(_("Error occurred while processing %s (NewVersion%d)"), + PackageName.c_str(), 1); + + if (oldMap != Map.Data()) + LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; + *LastVer = verindex; Ver->ParentPkg = Pkg.Index(); Ver->Hash = Hash; - if ((*LastVer == 0 && _error->PendingError()) || List.NewVersion(Ver) == false) - return _error->Error(_("Error occurred while processing %s (NewVersion1)"), - PackageName.c_str()); + if (List.NewVersion(Ver) == false) + return _error->Error(_("Error occurred while processing %s (NewVersion%d)"), + PackageName.c_str(), 2); if (List.UsePackage(Pkg,Ver) == false) return _error->Error(_("Error occurred while processing %s (UsePackage3)"), PackageName.c_str()); if (NewFileVer(Ver,List) == false) - return _error->Error(_("Error occurred while processing %s (NewVersion2)"), - PackageName.c_str()); + return _error->Error(_("Error occurred while processing %s (NewVersion%d)"), + PackageName.c_str(), 3); // Read only a single record and return if (OutVer != 0) @@ -269,13 +335,18 @@ bool pkgCacheGenerator::MergeList(ListParser &List, /* Record the Description data. Description data always exist in Packages and Translation-* files. */ pkgCache::DescIterator Desc = Ver.DescriptionList(); + Dynamic DynDesc(Desc); map_ptrloc *LastDesc = &Ver->DescriptionList; - + // Skip to the end of description set for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++); // Add new description - *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc); + oldMap = Map.Data(); + map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc); + if (oldMap != Map.Data()) + LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; + *LastDesc = descindex; Desc->ParentPkg = Pkg.Index(); if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false) @@ -322,6 +393,7 @@ bool pkgCacheGenerator::MergeFileProvides(ListParser &List) continue; pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName); + Dynamic DynPkg(Pkg); if (Pkg.end() == true) return _error->Error(_("Error occurred while processing %s (FindPkg)"), PackageName.c_str()); @@ -331,6 +403,7 @@ bool pkgCacheGenerator::MergeFileProvides(ListParser &List) unsigned long Hash = List.VersionHash(); pkgCache::VerIterator Ver = Pkg.VersionList(); + Dynamic DynVer(Ver); for (; Ver.end() == false; Ver++) { if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr()) @@ -358,14 +431,15 @@ bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name) return true; // Get a structure - unsigned long const Group = AllocateInMap(sizeof(pkgCache::Group)); + map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group)); if (unlikely(Group == 0)) return false; Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group); - Grp->Name = WriteStringInMap(Name); - if (unlikely(Grp->Name == 0)) + map_ptrloc const idxName = WriteStringInMap(Name); + if (unlikely(idxName == 0)) return false; + Grp->Name = idxName; // Insert it into the hash table unsigned long const Hash = Cache.Hash(Name); @@ -382,6 +456,7 @@ bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name) bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name, const string &Arch) { pkgCache::GrpIterator Grp; + Dynamic DynGrp(Grp); if (unlikely(NewGroup(Grp, Name) == false)) return false; @@ -390,7 +465,7 @@ bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name return true; // Get a structure - unsigned long const Package = AllocateInMap(sizeof(pkgCache::Package)); + map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package)); if (unlikely(Package == 0)) return false; Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package); @@ -416,9 +491,10 @@ bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name // Set the name, arch and the ID Pkg->Name = Grp->Name; Pkg->Group = Grp.Index(); - Pkg->Arch = WriteUniqString(Arch.c_str()); - if (unlikely(Pkg->Arch == 0)) + map_ptrloc const idxArch = WriteUniqString(Arch.c_str()); + if (unlikely(idxArch == 0)) return false; + Pkg->Arch = idxArch; Pkg->ID = Cache.HeaderP->PackageCount++; return true; @@ -434,7 +510,7 @@ bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver, return true; // Get a structure - unsigned long VerFile = AllocateInMap(sizeof(pkgCache::VerFile)); + map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile)); if (VerFile == 0) return 0; @@ -465,7 +541,7 @@ unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver, unsigned long Next) { // Get a structure - unsigned long Version = AllocateInMap(sizeof(pkgCache::Version)); + map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version)); if (Version == 0) return 0; @@ -473,9 +549,10 @@ unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver, Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version); Ver->NextVer = Next; Ver->ID = Cache.HeaderP->VersionCount++; - Ver->VerStr = WriteStringInMap(VerStr); - if (Ver->VerStr == 0) + map_ptrloc const idxVerStr = WriteStringInMap(VerStr); + if (unlikely(idxVerStr == 0)) return 0; + Ver->VerStr = idxVerStr; return Version; } @@ -490,7 +567,7 @@ bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc, return true; // Get a structure - unsigned long DescFile = AllocateInMap(sizeof(pkgCache::DescFile)); + map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile)); if (DescFile == 0) return false; @@ -523,7 +600,7 @@ map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, map_ptrloc Next) { // Get a structure - map_ptrloc Description = AllocateInMap(sizeof(pkgCache::Description)); + map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description)); if (Description == 0) return 0; @@ -531,10 +608,12 @@ map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description); Desc->NextDesc = Next; Desc->ID = Cache.HeaderP->DescriptionCount++; - Desc->language_code = WriteStringInMap(Lang); - Desc->md5sum = WriteStringInMap(md5sum.Value()); - if (Desc->language_code == 0 || Desc->md5sum == 0) + map_ptrloc const idxlanguage_code = WriteStringInMap(Lang); + map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value()); + if (unlikely(idxlanguage_code == 0 || idxmd5sum == 0)) return 0; + Desc->language_code = idxlanguage_code; + Desc->md5sum = idxmd5sum; return Description; } @@ -550,15 +629,22 @@ bool pkgCacheGenerator::FinishCache(OpProgress *Progress) if (archs.size() > 1) { // Create Conflicts in between the group - for (pkgCache::GrpIterator G = GetCache().GrpBegin(); G.end() != true; G++) + pkgCache::GrpIterator G = GetCache().GrpBegin(); + Dynamic DynG(G); + for (; G.end() != true; G++) { string const PkgName = G.Name(); - for (pkgCache::PkgIterator P = G.PackageList(); P.end() != true; P = G.NextPkg(P)) + pkgCache::PkgIterator P = G.PackageList(); + Dynamic DynP(P); + for (; P.end() != true; P = G.NextPkg(P)) { if (strcmp(P.Arch(),"all") == 0) continue; pkgCache::PkgIterator allPkg; - for (pkgCache::VerIterator V = P.VersionList(); V.end() != true; V++) + Dynamic DynallPkg(allPkg); + pkgCache::VerIterator V = P.VersionList(); + Dynamic DynV(V); + for (; V.end() != true; V++) { string const Arch = V.Arch(true); map_ptrloc *OldDepLast = NULL; @@ -578,6 +664,7 @@ bool pkgCacheGenerator::FinishCache(OpProgress *Progress) per group, therefore each group member conflicts with all other group members */ pkgCache::PkgIterator D = G.FindPkg(*A); + Dynamic DynD(D); if (D.end() == true) continue; if (coInstall == true) @@ -622,13 +709,15 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, unsigned int const &Type, map_ptrloc *OldDepLast) { + void const * const oldMap = Map.Data(); // Get a structure - unsigned long const Dependency = AllocateInMap(sizeof(pkgCache::Dependency)); + map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency)); if (unlikely(Dependency == 0)) return false; // Fill it in pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency); + Dynamic DynDep(Dep); Dep->ParentVer = Ver.Index(); Dep->Type = Type; Dep->CompareOp = Op; @@ -640,9 +729,12 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, /* for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++) if (I->Version != 0 && I.TargetVer() == Version) Dep->Version = I->Version;*/ - if (Dep->Version == 0) - if (unlikely((Dep->Version = WriteStringInMap(Version)) == 0)) + if (Dep->Version == 0) { + map_ptrloc const index = WriteStringInMap(Version); + if (unlikely(index == 0)) return false; + Dep->Version = index; + } } // Link it to the package @@ -656,7 +748,8 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, OldDepLast = &Ver->DependsList; for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++) OldDepLast = &D->NextDepends; - } + } else if (oldMap != Map.Data()) + OldDepLast += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; Dep->NextDepends = *OldDepLast; *OldDepLast = Dep.Index(); @@ -677,11 +770,13 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver, unsigned int Type) { pkgCache::GrpIterator Grp; + Dynamic DynGrp(Grp); if (unlikely(Owner->NewGroup(Grp, PackageName) == false)) return false; // Locate the target package pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch); + Dynamic DynPkg(Pkg); if (Pkg.end() == true) { if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false)) return false; @@ -715,13 +810,14 @@ bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver, return true; // Get a structure - unsigned long const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides)); + map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides)); if (unlikely(Provides == 0)) return false; Cache.HeaderP->ProvidesCount++; // Fill it in pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP); + Dynamic DynPrv(Prv); Prv->Version = Ver.Index(); Prv->NextPkgProv = Ver->ProvidesList; Ver->ProvidesList = Prv.Index(); @@ -730,6 +826,7 @@ bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver, // Locate the target package pkgCache::PkgIterator Pkg; + Dynamic DynPkg(Pkg); if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false)) return false; @@ -750,24 +847,29 @@ bool pkgCacheGenerator::SelectFile(const string &File,const string &Site, unsigned long Flags) { // Get some space for the structure - CurrentFile = Cache.PkgFileP + AllocateInMap(sizeof(*CurrentFile)); - if (CurrentFile == Cache.PkgFileP) + map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile)); + if (unlikely(idxFile == 0)) return false; - + CurrentFile = Cache.PkgFileP + idxFile; + // Fill it in - CurrentFile->FileName = WriteStringInMap(File); - CurrentFile->Site = WriteUniqString(Site); + map_ptrloc const idxFileName = WriteStringInMap(File); + map_ptrloc const idxSite = WriteUniqString(Site); + if (unlikely(idxFileName == 0 || idxSite == 0)) + return false; + CurrentFile->FileName = idxFileName; + CurrentFile->Site = idxSite; CurrentFile->NextFile = Cache.HeaderP->FileList; CurrentFile->Flags = Flags; CurrentFile->ID = Cache.HeaderP->PackageFileCount; - CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label); + map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label); + if (unlikely(idxIndexType == 0)) + return false; + CurrentFile->IndexType = idxIndexType; PkgFileName = File; Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP; Cache.HeaderP->PackageFileCount++; - if (CurrentFile->FileName == 0) - return false; - if (Progress != 0) Progress->SubProgress(Index.Size()); return true; @@ -807,18 +909,25 @@ unsigned long pkgCacheGenerator::WriteUniqString(const char *S, } // Get a structure - unsigned long Item = AllocateInMap(sizeof(pkgCache::StringItem)); + void const * const oldMap = Map.Data(); + map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem)); if (Item == 0) return 0; + map_ptrloc const idxString = WriteStringInMap(S,Size); + if (unlikely(idxString == 0)) + return 0; + if (oldMap != Map.Data()) { + Last += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; + I += (pkgCache::StringItem*) Map.Data() - (pkgCache::StringItem*) oldMap; + } + *Last = Item; + // Fill in the structure pkgCache::StringItem *ItemP = Cache.StringItemP + Item; ItemP->NextItem = I - Cache.StringItemP; - *Last = Item; - ItemP->String = WriteStringInMap(S,Size); - if (ItemP->String == 0) - return 0; - + ItemP->String = idxString; + Bucket = ItemP; return ItemP->String; } @@ -1072,7 +1181,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress unlink(CacheFile.c_str()); CacheF = new FileFd(CacheFile,FileFd::WriteEmpty); fchmod(CacheF->Fd(),0644); - Map = new DynamicMMap(*CacheF,MMap::Public,MapSize); + Map = new DynamicMMap(*CacheF,MMap::Public | MMap::Moveable, MapSize); if (_error->PendingError() == true) return false; if (Debug == true) @@ -1081,7 +1190,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress else { // Just build it in memory.. - Map = new DynamicMMap(0,MapSize); + Map = new DynamicMMap(MMap::Moveable, MapSize); if (Debug == true) std::clog << "Open memory Map (not filebased)" << std::endl; } @@ -1194,7 +1303,7 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O if (_system->AddStatusFiles(Files) == false) return false; - SPtr Map = new DynamicMMap(0,MapSize); + SPtr Map = new DynamicMMap(MMap::Moveable, MapSize); unsigned long CurrentSize = 0; unsigned long TotalSize = 0; diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 8f7739165..3bee1f958 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -23,6 +23,8 @@ #include #include +#include + class pkgSourceList; class OpProgress; class MMap; @@ -33,18 +35,32 @@ class pkgCacheGenerator /*{{{*/ private: pkgCache::StringItem *UniqHash[26]; - unsigned long WriteStringInMap(std::string const &String) { return WriteStringInMap(String.c_str()); }; - unsigned long WriteStringInMap(const char *String); - unsigned long WriteStringInMap(const char *String, const unsigned long &Len); - unsigned long AllocateInMap(const unsigned long &size); + map_ptrloc WriteStringInMap(std::string const &String) { return WriteStringInMap(String.c_str()); }; + map_ptrloc WriteStringInMap(const char *String); + map_ptrloc WriteStringInMap(const char *String, const unsigned long &Len); + map_ptrloc AllocateInMap(const unsigned long &size); public: class ListParser; friend class ListParser; - + + template class Dynamic { + Iter *I; + + public: + static std::set toReMap; + Dynamic(Iter &It) : I(&It) { + toReMap.insert(I); + } + + ~Dynamic() { + toReMap.erase(I); + } + }; + protected: - + DynamicMMap ⤅ pkgCache Cache; OpProgress *Progress; @@ -86,6 +102,8 @@ class pkgCacheGenerator /*{{{*/ MMap **OutMap = 0,bool AllowMem = false); static bool MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap); + void ReMap(void const * const oldMap, void const * const newMap); + pkgCacheGenerator(DynamicMMap *Map,OpProgress *Progress); ~pkgCacheGenerator(); }; diff --git a/debian/changelog b/debian/changelog index 878d5238a..dab703c6b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -37,6 +37,9 @@ apt (0.7.26~exp8) UNRELEASED; urgency=low * apt-pkg/policy.h: - add another round of const& madness as the previous round accidentally NOT overrides the virtual GetCandidateVer() method (Closes: #587725) + * apt-pkg/pkgcachegen.{cc,h}: + - make the used MMap moveable (and therefore dynamic resizeable) by + applying (some) mad pointer magic (Closes: #195018) [ Julian Andres Klode ] * methods/ftp.h: -- 2.45.2