From 5465192b9aeb1ccea778950ccf2d1b7b32f2cd91 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 18 Jul 2015 18:03:54 +0200 Subject: [PATCH] add volatile sources support in libapt-pkg MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Sources are usually defined in sources.list (and co) and are pretty stable, but once in a while a frontend might want to add an additional "source" like a local .deb file to install this package (No support for 'real' sources being added this way as this is a multistep process). We had a hack in place to allow apt-get and apt to pull this of for a short while now, but other frontends are either left in the cold by this and/or the code for it looks dirty with FIXMEs plastering it and has on top of this also some problems (like including these 'volatile' sources in the srcpkgcache.bin file). So the biggest part in this commit is actually the rewrite of the cache generation as it is now potentially a three step process. The biggest problem with adding support now through is that this makes a bunch of previously mostly unusable by externs and therefore hidden classes public, so a bit of further tuneing on this now public API is in order… --- apt-pkg/deb/debindexfile.cc | 27 +- apt-pkg/deb/debindexfile.h | 18 +- apt-pkg/deb/deblistparser.cc | 2 +- apt-pkg/deb/debmetaindex.cc | 29 -- apt-pkg/deb/debmetaindex.h | 39 --- apt-pkg/edsp/edspsystem.cc | 3 +- apt-pkg/pkgcachegen.cc | 401 ++++++++++------------ apt-pkg/pkgcachegen.h | 1 - apt-pkg/pkgsystem.h | 5 +- apt-pkg/sourcelist.cc | 28 +- apt-pkg/sourcelist.h | 19 + apt-private/private-cachefile.h | 21 -- apt-private/private-install.cc | 21 +- test/integration/framework | 12 +- test/integration/test-apt-get-install-deb | 45 ++- 15 files changed, 309 insertions(+), 362 deletions(-) diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index 972feba7f..e67233e5f 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -339,18 +339,16 @@ APT_CONST bool debStatusIndex::Exists() const } /*}}}*/ -// debDebPkgFile - Single .deb file /*{{{*/ -debDebPkgFileIndex::debDebPkgFileIndex(std::string DebFile) +// debDebPkgFileIndex - Single .deb file /*{{{*/ +debDebPkgFileIndex::debDebPkgFileIndex(std::string const &DebFile) : pkgIndexFile(true), d(NULL), DebFile(DebFile) { DebFileFullPath = flAbsPath(DebFile); } - std::string debDebPkgFileIndex::ArchiveURI(std::string /*File*/) const { return "file:" + DebFileFullPath; } - bool debDebPkgFileIndex::Exists() const { return FileExists(DebFile); @@ -403,6 +401,8 @@ bool debDebPkgFileIndex::Merge(pkgCacheGenerator& Gen, OpProgress* Prog) const if (GetContent(content, DebFile) == false) return false; std::string const contentstr = content.str(); + if (contentstr.empty()) + return true; DebControl->Write(contentstr.c_str(), contentstr.length()); // rewind for the listparser DebControl->Seek(0); @@ -431,7 +431,7 @@ pkgCache::PkgFileIterator debDebPkgFileIndex::FindInCache(pkgCache &Cache) const return File; } - + return File; } unsigned long debDebPkgFileIndex::Size() const @@ -443,12 +443,11 @@ unsigned long debDebPkgFileIndex::Size() const } /*}}}*/ -// debDscFileIndex stuff -debDscFileIndex::debDscFileIndex(std::string &DscFile) +// debDscFileIndex - a .dsc file /*{{{*/ +debDscFileIndex::debDscFileIndex(std::string const &DscFile) : pkgIndexFile(true), d(NULL), DscFile(DscFile) { } - bool debDscFileIndex::Exists() const { return FileExists(DscFile); @@ -461,8 +460,6 @@ unsigned long debDscFileIndex::Size() const return buf.st_size; return 0; } - -// DscFileIndex::CreateSrcParser - Get a parser for the .dsc file /*{{{*/ pkgSrcRecords::Parser *debDscFileIndex::CreateSrcParser() const { if (!FileExists(DscFile)) @@ -471,18 +468,17 @@ pkgSrcRecords::Parser *debDscFileIndex::CreateSrcParser() const return new debDscRecordParser(DscFile,this); } /*}}}*/ + // Index File types for Debian /*{{{*/ class APT_HIDDEN debIFTypeSrc : public pkgIndexFile::Type { public: - debIFTypeSrc() {Label = "Debian Source Index";}; }; class APT_HIDDEN debIFTypePkg : public pkgIndexFile::Type { public: - - virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE { return new debRecordParser(File.FileName(),*File.Cache()); }; @@ -496,8 +492,7 @@ class APT_HIDDEN debIFTypeTrans : public debIFTypePkg class APT_HIDDEN debIFTypeStatus : public pkgIndexFile::Type { public: - - virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE { return new debRecordParser(File.FileName(),*File.Cache()); }; @@ -506,7 +501,7 @@ class APT_HIDDEN debIFTypeStatus : public pkgIndexFile::Type class APT_HIDDEN debIFTypeDebPkgFile : public pkgIndexFile::Type { public: - virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE { return new debDebFileRecordParser(File.FileName()); }; diff --git a/apt-pkg/deb/debindexfile.h b/apt-pkg/deb/debindexfile.h index 71ca22e4d..4a818121a 100644 --- a/apt-pkg/deb/debindexfile.h +++ b/apt-pkg/deb/debindexfile.h @@ -27,7 +27,7 @@ class pkgAcquire; class pkgCacheGenerator; -class APT_HIDDEN debStatusIndex : public pkgIndexFile +class debStatusIndex : public pkgIndexFile { void * const d; protected: @@ -51,7 +51,7 @@ class APT_HIDDEN debStatusIndex : public pkgIndexFile virtual ~debStatusIndex(); }; -class APT_HIDDEN debPackagesIndex : public pkgIndexTargetFile +class debPackagesIndex : public pkgIndexTargetFile { void * const d; public: @@ -70,7 +70,7 @@ class APT_HIDDEN debPackagesIndex : public pkgIndexTargetFile virtual ~debPackagesIndex(); }; -class APT_HIDDEN debTranslationsIndex : public pkgIndexTargetFile +class debTranslationsIndex : public pkgIndexTargetFile { void * const d; public: @@ -86,7 +86,7 @@ class APT_HIDDEN debTranslationsIndex : public pkgIndexTargetFile virtual ~debTranslationsIndex(); }; -class APT_HIDDEN debSourcesIndex : public pkgIndexTargetFile +class debSourcesIndex : public pkgIndexTargetFile { void * const d; public: @@ -107,7 +107,7 @@ class APT_HIDDEN debSourcesIndex : public pkgIndexTargetFile virtual ~debSourcesIndex(); }; -class APT_HIDDEN debDebPkgFileIndex : public pkgIndexFile +class debDebPkgFileIndex : public pkgIndexFile { private: void * const d; @@ -141,11 +141,11 @@ class APT_HIDDEN debDebPkgFileIndex : public pkgIndexFile // Interface for acquire virtual std::string ArchiveURI(std::string /*File*/) const APT_OVERRIDE; - debDebPkgFileIndex(std::string DebFile); + debDebPkgFileIndex(std::string const &DebFile); virtual ~debDebPkgFileIndex(); }; -class APT_HIDDEN debDscFileIndex : public pkgIndexFile +class debDscFileIndex : public pkgIndexFile { private: void * const d; @@ -160,11 +160,11 @@ class APT_HIDDEN debDscFileIndex : public pkgIndexFile return DscFile; }; - debDscFileIndex(std::string &DscFile); + debDscFileIndex(std::string const &DscFile); virtual ~debDscFileIndex(); }; -class APT_HIDDEN debDebianSourceDirIndex : public debDscFileIndex +class debDebianSourceDirIndex : public debDscFileIndex { public: virtual const Type *GetType() const APT_CONST; diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 1154016a9..b7988d499 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -959,7 +959,7 @@ bool debDebFileParser::UsePackage(pkgCache::PkgIterator &Pkg, bool res = debListParser::UsePackage(Pkg, Ver); // we use the full file path as a provides so that the file is found // by its name - if(NewProvidesAllArch(Ver, DebFile, Ver.VerStr(), 0) == false) + if(NewProvides(Ver, DebFile, Pkg.Cache()->NativeArch(), Ver.VerStr(), 0) == false) return false; return res; } diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 480317db3..123c91648 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -785,34 +785,5 @@ class APT_HIDDEN debSLTypeDebSrc : public debSLTypeDebian /*{{{*/ }; /*}}}*/ -debDebFileMetaIndex::debDebFileMetaIndex(std::string const &DebFile) /*{{{*/ - : metaIndex(DebFile, "local-uri", "deb-dist"), d(NULL), DebFile(DebFile) -{ - DebIndex = new debDebPkgFileIndex(DebFile); - Indexes = new std::vector(); - Indexes->push_back(DebIndex); -} -debDebFileMetaIndex::~debDebFileMetaIndex() {} - /*}}}*/ -class APT_HIDDEN debSLTypeDebFile : public pkgSourceList::Type /*{{{*/ -{ - public: - - bool CreateItem(std::vector &List, std::string const &URI, - std::string const &/*Dist*/, std::string const &/*Section*/, - std::map const &/*Options*/) const APT_OVERRIDE - { - metaIndex *mi = new debDebFileMetaIndex(URI); - List.push_back(mi); - return true; - } - - debSLTypeDebFile() : Type("deb-file", "Debian local deb file") - { - } -}; - /*}}}*/ - APT_HIDDEN debSLTypeDeb _apt_DebType; APT_HIDDEN debSLTypeDebSrc _apt_DebSrcType; -APT_HIDDEN debSLTypeDebFile _apt_DebFileType; diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h index 8c13237cb..e93959a21 100644 --- a/apt-pkg/deb/debmetaindex.h +++ b/apt-pkg/deb/debmetaindex.h @@ -17,7 +17,6 @@ class pkgAcquire; class pkgIndexFile; -class debDebPkgFileIndex; class IndexTarget; class pkgCacheGenerator; class OpProgress; @@ -66,42 +65,4 @@ class APT_HIDDEN debReleaseIndex : public metaIndex std::vector Languages); }; -class APT_HIDDEN debDebFileMetaIndex : public metaIndex -{ -private: - void * const d; - std::string DebFile; - debDebPkgFileIndex *DebIndex; -public: - virtual std::string ArchiveURI(std::string const& /*File*/) const APT_OVERRIDE { - return DebFile; - } - virtual bool GetIndexes(pkgAcquire* /*Owner*/, const bool& /*GetAll=false*/) APT_OVERRIDE { - return true; - } - virtual std::vector GetIndexTargets() const APT_OVERRIDE { - return std::vector(); - } - virtual std::vector *GetIndexFiles() APT_OVERRIDE { - return Indexes; - } - virtual bool IsTrusted() const APT_OVERRIDE { - return true; - } - virtual bool Load(std::string const &, std::string * const ErrorText) APT_OVERRIDE - { - LoadedSuccessfully = TRI_NO; - if (ErrorText != NULL) - strprintf(*ErrorText, "Unparseable metaindex as it represents the standalone deb file %s", DebFile.c_str()); - return false; - } - virtual metaIndex * UnloadedClone() const APT_OVERRIDE - { - return NULL; - } - debDebFileMetaIndex(std::string const &DebFile); - virtual ~debDebFileMetaIndex(); - -}; - #endif diff --git a/apt-pkg/edsp/edspsystem.cc b/apt-pkg/edsp/edspsystem.cc index 4fb34b896..f65fcc0d2 100644 --- a/apt-pkg/edsp/edspsystem.cc +++ b/apt-pkg/edsp/edspsystem.cc @@ -91,8 +91,7 @@ signed edspSystem::Score(Configuration const &Cnf) return -1000; } /*}}}*/ -// System::AddStatusFiles - Register the status files /*{{{*/ -bool edspSystem::AddStatusFiles(std::vector &List) +bool edspSystem::AddStatusFiles(std::vector &List) /*{{{*/ { if (StatusFile == 0) { diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 26a5e60a6..9529f42dc 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -36,6 +36,8 @@ #include #include #include +#include +#include #include #include @@ -1332,10 +1334,10 @@ map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, cons /* This just verifies that each file in the list of index files exists, has matching attributes with the cache and the cache does not have any extra files. */ -static bool CheckValidity(const string &CacheFile, +static bool CheckValidity(const string &CacheFile, pkgSourceList &List, - FileIterator Start, - FileIterator End, + FileIterator const Start, + FileIterator const End, MMap **OutMap = 0) { bool const Debug = _config->FindB("Debug::pkgCacheGen", false); @@ -1343,7 +1345,7 @@ static bool CheckValidity(const string &CacheFile, if (CacheFile.empty() == true || FileExists(CacheFile) == false) { if (Debug == true) - std::clog << "CacheFile doesn't exist" << std::endl; + std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl; return false; } @@ -1361,7 +1363,7 @@ static bool CheckValidity(const string &CacheFile, if (_error->PendingError() == true || Map->Size() == 0) { if (Debug == true) - std::clog << "Errors are pending or Map is empty()" << std::endl; + std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl; _error->Discard(); return false; } @@ -1385,10 +1387,9 @@ static bool CheckValidity(const string &CacheFile, if (Debug == true) std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl; - std::vector *Indexes = (*i)->GetIndexFiles(); - for (std::vector::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j) - if ((*j)->HasPackages()) - Files.push_back (*j); + std::vector const * const Indexes = (*i)->GetIndexFiles(); + std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files), + [](pkgIndexFile const * const I) { return I->HasPackages(); }); } for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I) if (RlsVisited[I] == false) @@ -1398,8 +1399,7 @@ static bool CheckValidity(const string &CacheFile, return false; } - for (; Start != End; ++Start) - Files.push_back(*Start); + std::copy(Start, End, std::back_inserter(Files)); /* Now we check every index file, see if it is in the cache, verify the IMS data and check that it is on the disk too.. */ @@ -1482,16 +1482,41 @@ static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator } /*}}}*/ // BuildCache - Merge the list of index files into the cache /*{{{*/ -// --------------------------------------------------------------------- -/* */ static bool BuildCache(pkgCacheGenerator &Gen, - OpProgress *Progress, + OpProgress * const Progress, map_filesize_t &CurrentSize,map_filesize_t TotalSize, pkgSourceList const * const List, - FileIterator Start, FileIterator End) + FileIterator const Start, FileIterator const End) { std::vector Files; bool const HasFileDeps = Gen.HasFileDeps(); + bool mergeFailure = false; + + auto const indexFileMerge = [&](pkgIndexFile * const I) { + if (HasFileDeps) + Files.push_back(I); + + if (I->HasPackages() == false || mergeFailure) + return; + + if (I->Exists() == false) + return; + + if (I->FindInCache(Gen.GetCache()).end() == false) + { + _error->Warning("Duplicate sources.list entry %s", + I->Describe().c_str()); + return; + } + + map_filesize_t const Size = I->Size(); + if (Progress != NULL) + Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists")); + CurrentSize += Size; + + if (I->Merge(Gen,Progress) == false) + mergeFailure = true; + }; if (List != NULL) { @@ -1508,63 +1533,20 @@ static bool BuildCache(pkgCacheGenerator &Gen, return false; std::vector *Indexes = (*i)->GetIndexFiles(); - for (std::vector::const_iterator I = Indexes->begin(); I != Indexes->end(); ++I) - { - if (HasFileDeps) - Files.push_back(*I); - - if ((*I)->HasPackages() == false) - continue; - - if ((*I)->Exists() == false) - continue; - - if ((*I)->FindInCache(Gen.GetCache()).end() == false) - { - _error->Warning("Duplicate sources.list entry %s", - (*I)->Describe().c_str()); - continue; - } - - map_filesize_t Size = (*I)->Size(); - if (Progress != NULL) - Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists")); - CurrentSize += Size; - - if ((*I)->Merge(Gen,Progress) == false) - return false; - } + if (Indexes != NULL) + std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge); + if (mergeFailure) + return false; } } - Gen.SelectReleaseFile("", ""); - FileIterator I; - for (I = Start; I != End; ++I) + if (Start != End) { - if (HasFileDeps) - Files.push_back(*I); - - if ((*I)->HasPackages() == false) - continue; - - if ((*I)->Exists() == false) - continue; - - if ((*I)->FindInCache(Gen.GetCache()).end() == false) - { - _error->Warning("Duplicate sources.list entry %s", - (*I)->Describe().c_str()); - continue; - } - - map_filesize_t Size = (*I)->Size(); - if (Progress != NULL) - Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists")); - CurrentSize += Size; - - if ((*I)->Merge(Gen,Progress) == false) + Gen.SelectReleaseFile("", ""); + std::for_each(Start, End, indexFileMerge); + if (mergeFailure) return false; - } + } if (HasFileDeps == true) { @@ -1582,12 +1564,20 @@ static bool BuildCache(pkgCacheGenerator &Gen, return false; } } - + return true; } /*}}}*/ -// CacheGenerator::CreateDynamicMMap - load an mmap with configuration options /*{{{*/ -DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) { +// CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/ +// --------------------------------------------------------------------- +/* This makes sure that the status cache (the cache that has all + index files from the sources list and all local ones) is ready + to be mmaped. If OutMap is not zero then a MMap object representing + the cache will be stored there. This is pretty much mandetory if you + are using AllowMem. AllowMem lets the function be run as non-root + where it builds the cache 'fast' into a memory buffer. */ +static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags) +{ map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024); map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024); map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0); @@ -1599,15 +1589,42 @@ DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long else return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit); } - /*}}}*/ -// CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/ -// --------------------------------------------------------------------- -/* This makes sure that the status cache (the cache that has all - index files from the sources list and all local ones) is ready - to be mmaped. If OutMap is not zero then a MMap object representing - the cache will be stored there. This is pretty much mandetory if you - are using AllowMem. AllowMem lets the function be run as non-root - where it builds the cache 'fast' into a memory buffer. */ +static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map, + std::string const &FileName) +{ + FileFd SCacheF(FileName, FileFd::WriteAtomic); + if (_error->PendingError() == true) + return false; + + fchmod(SCacheF.Fd(),0644); + + // Write out the main data + if (SCacheF.Write(Map->Data(),Map->Size()) == false) + return _error->Error(_("IO Error saving source cache")); + SCacheF.Sync(); + + // Write out the proper header + Gen->GetCache().HeaderP->Dirty = false; + if (SCacheF.Seek(0) == false || + SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false) + return _error->Error(_("IO Error saving source cache")); + Gen->GetCache().HeaderP->Dirty = true; + SCacheF.Sync(); + return true; +} +static bool loadBackMMapFromFile(std::unique_ptr &Gen, + SPtr &Map, OpProgress * const Progress, std::string const &FileName) +{ + Map = CreateDynamicMMap(NULL, 0); + FileFd CacheF(FileName, FileFd::ReadOnly); + map_pointer_t const alloc = Map->RawAllocate(CacheF.Size()); + if ((alloc == 0 && _error->PendingError()) + || CacheF.Read((unsigned char *)Map->Data() + alloc, + CacheF.Size()) == false) + return false; + Gen.reset(new pkgCacheGenerator(Map.Get(),Progress)); + return true; +} APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress, MMap **OutMap, bool AllowMem) { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); } @@ -1617,18 +1634,6 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress bool const Debug = _config->FindB("Debug::pkgCacheGen", false); std::vector Files; - /* - for (std::vector::const_iterator i = List.begin(); - i != List.end(); - ++i) - { - std::vector *Indexes = (*i)->GetIndexFiles(); - for (std::vector::const_iterator j = Indexes->begin(); - j != Indexes->end(); - ++j) - Files.push_back (*j); - } -*/ if (_system->AddStatusFiles(Files) == false) return false; @@ -1649,160 +1654,130 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress CreateDirectory(dir, flNotFile(SrcCacheFile)); } - // Decide if we can write to the cache - bool Writeable = false; - if (CacheFile.empty() == false) - Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0; - else - if (SrcCacheFile.empty() == false) - Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0; - if (Debug == true) - std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl; - - if (Writeable == false && AllowMem == false && CacheFile.empty() == false) - return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str()); - if (Progress != NULL) Progress->OverallProgress(0,1,1,_("Reading package lists")); - // Cache is OK, Fin. - if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true) + bool pkgcache_fine = false; + bool srcpkgcache_fine = false; + bool volatile_fine = List.GetVolatileFiles().empty(); + + if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL) == true) { - if (Progress != NULL) - Progress->OverallProgress(1,1,1,_("Reading package lists")); if (Debug == true) - std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl; - return true; + std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl; + pkgcache_fine = true; + srcpkgcache_fine = true; } - else if (Debug == true) - std::clog << "pkgcache.bin is NOT valid" << std::endl; - - /* At this point we know we need to reconstruct the package cache, - begin. */ - SPtr CacheF; - SPtr Map; - if (Writeable == true && CacheFile.empty() == false) + if (pkgcache_fine == false) { - _error->PushToStack(); - unlink(CacheFile.c_str()); - CacheF = new FileFd(CacheFile,FileFd::WriteAtomic); - fchmod(CacheF->Fd(),0644); - Map = CreateDynamicMMap(CacheF, MMap::Public); - if (_error->PendingError() == true) + if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true) { - delete CacheF.UnGuard(); - delete Map.UnGuard(); if (Debug == true) - std::clog << "Open filebased MMap FAILED" << std::endl; - Writeable = false; - if (AllowMem == false) - { - _error->MergeWithStack(); - return false; - } - _error->RevertToStack(); - } - else - { - _error->MergeWithStack(); - if (Debug == true) - std::clog << "Open filebased MMap" << std::endl; + std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl; + srcpkgcache_fine = true; } } - if (Writeable == false || CacheFile.empty() == true) + + if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true) { - // Just build it in memory.. - Map = CreateDynamicMMap(NULL); - if (Debug == true) - std::clog << "Open memory Map (not filebased)" << std::endl; + if (Progress != NULL) + Progress->OverallProgress(1,1,1,_("Reading package lists")); + return true; } - - // Lets try the source cache. - map_filesize_t CurrentSize = 0; - map_filesize_t TotalSize = 0; - if (CheckValidity(SrcCacheFile, List, Files.end(), - Files.end()) == true) + + bool Writeable = false; + if (srcpkgcache_fine == false || pkgcache_fine == false) { + if (CacheFile.empty() == false) + Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0; + else if (SrcCacheFile.empty() == false) + Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0; + if (Debug == true) - std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl; - // Preload the map with the source cache - FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly); - map_pointer_t const alloc = Map->RawAllocate(SCacheF.Size()); - if ((alloc == 0 && _error->PendingError()) - || SCacheF.Read((unsigned char *)Map->Data() + alloc, - SCacheF.Size()) == false) - return false; + std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl; + + if (Writeable == false && AllowMem == false) + { + if (CacheFile.empty() == false) + return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str()); + else if (SrcCacheFile.empty() == false) + return _error->Error(_("Unable to write to %s"),flNotFile(SrcCacheFile).c_str()); + else + return _error->Error("Unable to create caches as file usage is disabled, but memory not allowed either!"); + } + } - TotalSize = ComputeSize(NULL, Files.begin(), Files.end()); + // At this point we know we need to construct something, so get storage ready + SPtr Map = CreateDynamicMMap(NULL, 0); + if (Debug == true) + std::clog << "Open memory Map (not filebased)" << std::endl; - // Build the status cache - pkgCacheGenerator Gen(Map.Get(),Progress); - if (_error->PendingError() == true) - return false; - if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL, - Files.begin(),Files.end()) == false) + std::unique_ptr Gen{nullptr}; + map_filesize_t CurrentSize = 0; + std::vector VolatileFiles = List.GetVolatileFiles(); + map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end()); + if (srcpkgcache_fine == true && pkgcache_fine == false) + { + if (Debug == true) + std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl; + if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false) return false; + srcpkgcache_fine = true; + TotalSize += ComputeSize(NULL, Files.begin(), Files.end()); } - else + else if (srcpkgcache_fine == false) { if (Debug == true) std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl; - TotalSize = ComputeSize(&List, Files.begin(),Files.end()); - - // Build the source cache - pkgCacheGenerator Gen(Map.Get(),Progress); - if (_error->PendingError() == true) - return false; - if (BuildCache(Gen, Progress, CurrentSize, TotalSize, &List, - Files.end(),Files.end()) == false) + Gen.reset(new pkgCacheGenerator(Map.Get(),Progress)); + + TotalSize += ComputeSize(&List, Files.begin(),Files.end()); + if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List, + Files.end(),Files.end()) == false) return false; - - // Write it back + if (Writeable == true && SrcCacheFile.empty() == false) - { - FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic); - if (_error->PendingError() == true) + if (writeBackMMapToFile(Gen.get(), Map.Get(), SrcCacheFile) == false) return false; - - fchmod(SCacheF.Fd(),0644); - - // Write out the main data - if (SCacheF.Write(Map->Data(),Map->Size()) == false) - return _error->Error(_("IO Error saving source cache")); - SCacheF.Sync(); - - // Write out the proper header - Gen.GetCache().HeaderP->Dirty = false; - if (SCacheF.Seek(0) == false || - SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false) - return _error->Error(_("IO Error saving source cache")); - Gen.GetCache().HeaderP->Dirty = true; - SCacheF.Sync(); - } - - // Build the status cache - if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL, - Files.begin(), Files.end()) == false) + } + + if (pkgcache_fine == false) + { + if (Debug == true) + std::clog << "Building status cache in pkgcache.bin now" << std::endl; + if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL, + Files.begin(), Files.end()) == false) return false; + + if (Writeable == true && CacheFile.empty() == false) + if (writeBackMMapToFile(Gen.get(), Map.Get(), CacheFile) == false) + return false; } + if (Debug == true) - std::clog << "Caches are ready for shipping" << std::endl; + std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl; - if (_error->PendingError() == true) - return false; - if (OutMap != 0) + if (volatile_fine == false) { - if (CacheF != 0) + if (Gen == nullptr) { - delete Map.UnGuard(); - *OutMap = new MMap(*CacheF,0); + if (Debug == true) + std::clog << "Populate new MMap with cachefile contents" << std::endl; + if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false) + return false; } - else - { - *OutMap = Map.UnGuard(); - } + + Files = List.GetVolatileFiles(); + if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL, + Files.begin(), Files.end()) == false) + return false; } - + + if (OutMap != nullptr) + *OutMap = Map.UnGuard(); + + if (Debug == true) + std::clog << "Everything is ready for shipping" << std::endl; return true; } /*}}}*/ @@ -1817,7 +1792,7 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O if (_system->AddStatusFiles(Files) == false) return false; - SPtr Map = CreateDynamicMMap(NULL); + SPtr Map = CreateDynamicMMap(NULL, 0); map_filesize_t CurrentSize = 0; map_filesize_t TotalSize = 0; diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 0d0fb893f..4b6b91992 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -116,7 +116,6 @@ class APT_HIDDEN pkgCacheGenerator /*{{{*/ APT_PUBLIC static bool MakeStatusCache(pkgSourceList &List,OpProgress *Progress, MMap **OutMap = 0,bool AllowMem = false); APT_PUBLIC static bool MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap); - APT_PUBLIC static DynamicMMap* CreateDynamicMMap(FileFd *CacheF, unsigned long Flags = 0); void ReMap(void const * const oldMap, void const * const newMap); diff --git a/apt-pkg/pkgsystem.h b/apt-pkg/pkgsystem.h index 5be93d059..5b31457e0 100644 --- a/apt-pkg/pkgsystem.h +++ b/apt-pkg/pkgsystem.h @@ -52,7 +52,7 @@ class Configuration; class pkgIndexFile; class pkgSystem -{ +{ public: // Global list of supported systems @@ -81,7 +81,8 @@ class pkgSystem virtual bool ArchiveSupported(const char *Type) = 0; // Return a list of system index files.. - virtual bool AddStatusFiles(std::vector &List) = 0; + virtual bool AddStatusFiles(std::vector &List) = 0; + virtual bool FindIndex(pkgCache::PkgFileIterator File, pkgIndexFile *&Found) const = 0; diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index eef0ee709..46e51f592 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -274,6 +274,10 @@ pkgSourceList::~pkgSourceList() { for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I) delete *I; + SrcList.clear(); + for (pkgIndexFile * const File : VolatileFiles) + delete File; + VolatileFiles.clear(); } /*}}}*/ // SourceList::ReadMainList - Read the main source list from etc /*{{{*/ @@ -339,7 +343,7 @@ bool pkgSourceList::ReadAppend(string const &File) else return ParseFileOldStyle(File); } - + /*}}}*/ // SourceList::ReadFileOldStyle - Read Traditional style sources.list /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -454,7 +458,15 @@ bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File, } } } - + for (vector::const_iterator J = VolatileFiles.begin(); + J != VolatileFiles.end(); ++J) + { + if ((*J)->FindInCache(*File.Cache()) == File) + { + Found = (*J); + return true; + } + } return false; } /*}}}*/ @@ -511,4 +523,14 @@ time_t pkgSourceList::GetLastModifiedTime() return mtime_sources; } /*}}}*/ - +std::vector pkgSourceList::GetVolatileFiles() const /*{{{*/ +{ + return VolatileFiles; +} + /*}}}*/ +void pkgSourceList::AddVolatileFile(pkgIndexFile * const File) /*{{{*/ +{ + if (File != NULL) + VolatileFiles.push_back(File); +} + /*}}}*/ diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index d80131438..47a562d18 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -50,6 +50,7 @@ class metaIndex; class pkgSourceList { void * const d; + std::vector VolatileFiles; public: // List of supported source list types @@ -113,6 +114,24 @@ class pkgSourceList // query last-modified time time_t GetLastModifiedTime(); + /** \brief add file for parsing, but not to the cache + * + * pkgIndexFiles origining from pkgSourcesList are included in + * srcpkgcache, the status files added via #AddStatusFiles are + * included in pkgcache, but these files here are not included in + * any cache to have the possibility of having a file included just + * for a single run like a local .deb/.dsc file. + * + * The volatile files do not count as "normal" sourceslist entries, + * can't be iterated over with #begin and #end and can't be + * downloaded, but they can be found via #FindIndex. + * + * @param File is an index file; pointer-ownership is transferred + */ + void AddVolatileFile(pkgIndexFile * const File); + /** @return list of files registered with #AddVolatileFile */ + std::vector GetVolatileFiles() const; + pkgSourceList(); virtual ~pkgSourceList(); }; diff --git a/apt-private/private-cachefile.h b/apt-private/private-cachefile.h index ed9342db0..51703b0ad 100644 --- a/apt-private/private-cachefile.h +++ b/apt-private/private-cachefile.h @@ -11,17 +11,6 @@ #include -// FIXME: we need to find a way to export this -class APT_PUBLIC SourceList : public pkgSourceList -{ - public: - // Add custom metaIndex (e.g. local files) - void AddMetaIndex(metaIndex *mi) { - SrcList.push_back(mi); - } - -}; - // class CacheFile - Cover class for some dependency cache functions /*{{{*/ class APT_PUBLIC CacheFile : public pkgCacheFile { @@ -36,16 +25,6 @@ class APT_PUBLIC CacheFile : public pkgCacheFile return false; return true; } - // FIXME: this can go once the "libapt-pkg" pkgSourceList has a way - // to add custom metaIndexes (or custom local files or so) - bool BuildSourceList(OpProgress */*Progress*/ = NULL) { - if (SrcList != NULL) - return true; - SrcList = new SourceList(); - if (SrcList->ReadMainList() == false) - return _error->Error(_("The list of sources could not be read.")); - return true; - } bool Open(bool WithLock = true) { OpTextProgress Prog(*_config); diff --git a/apt-private/private-install.cc b/apt-private/private-install.cc index 116e01038..074874903 100644 --- a/apt-private/private-install.cc +++ b/apt-private/private-install.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -647,24 +648,8 @@ bool DoInstall(CommandLine &CmdL) // first check for local pkgs and add them to the cache for (const char **I = CmdL.FileList; *I != 0; I++) { - if(FileExists(*I)) - { - // FIXME: make this more elegant - std::string TypeStr = flExtension(*I) + "-file"; - pkgSourceList::Type *Type = pkgSourceList::Type::GetType(TypeStr.c_str()); - if(Type != 0) - { - std::vector List; - std::map Options; - if(Type->CreateItem(List, *I, "", "", Options)) - { - // we have our own CacheFile that gives us a SourceList - // with superpowerz - SourceList *sources = (SourceList*)Cache.GetSourceList(); - sources->AddMetaIndex(List[0]); - } - } - } + if(FileExists(*I) && flExtension(*I) == "deb") + Cache.GetSourceList()->AddVolatileFile(new debDebPkgFileIndex(*I)); } // then open the cache diff --git a/test/integration/framework b/test/integration/framework index 6ae5003f7..f3cc1eff9 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -99,10 +99,14 @@ msgdone() { } getaptconfig() { if [ -f ./aptconfig.conf ]; then - echo "./aptconfig.conf" + echo "$(readlink -f ./aptconfig.conf)" elif [ -f ../aptconfig.conf ]; then - echo "../aptconfig.conf" - fi + echo "$(readlink -f ../aptconfig.conf)" + elif [ -f ../../aptconfig.conf ]; then + echo "$(readlink -f ../../aptconfig.conf)" + elif [ -f "${TMPWORKINGDIRECTORY}/aptconfig.conf" ]; then + echo "$(readlink -f "${TMPWORKINGDIRECTORY}/aptconfig.conf")" + fi } runapt() { msgdebug "Executing: ${CCMD}$*${CDEBUG} " @@ -141,6 +145,8 @@ gdb() { case "$1" in aptget) CMD="apt-get";; aptcache) CMD="apt-cache";; + aptcdrom) CMD="apt-cdrom";; + aptconfig) CMD="apt-config";; aptmark) CMD="apt-mark";; apthelper) CMD="apt-helper";; aptftparchive) CMD="apt-ftparchive";; diff --git a/test/integration/test-apt-get-install-deb b/test/integration/test-apt-get-install-deb index 0f34692fe..991185dea 100755 --- a/test/integration/test-apt-get-install-deb +++ b/test/integration/test-apt-get-install-deb @@ -5,15 +5,21 @@ TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework setupenvironment -configarchitecture "i386" +configarchitecture 'amd64' 'i386' # regression test for #754904 testfailureequal 'E: Unable to locate package /dev/null' aptget install -qq /dev/null -# and ensure we fail for invalid debs -cat > foo.deb < foo.rpm <