+ if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
+ {
+ if (Debug == true)
+ std::clog << "sources.list is newer than the cache" << std::endl;
+ return false;
+ }
+
+ // Map it
+ FileFd CacheF(CacheFile,FileFd::ReadOnly);
+ std::unique_ptr<MMap> Map(new MMap(CacheF,0));
+ if (unlikely(Map->validData()) == false)
+ return false;
+ std::unique_ptr<pkgCache> CacheP(new pkgCache(Map.get()));
+ pkgCache &Cache = *CacheP.get();
+ if (_error->PendingError() || Map->Size() == 0)
+ {
+ if (Debug == true)
+ std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
+ return false;
+ }
+
+ std::unique_ptr<bool[]> RlsVisited(new bool[Cache.HeaderP->ReleaseFileCount]);
+ memset(RlsVisited.get(),0,sizeof(RlsVisited[0])*Cache.HeaderP->ReleaseFileCount);
+ std::vector<pkgIndexFile *> Files;
+ for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
+ {
+ if (Debug == true)
+ std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
+ pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
+ if (RlsFile.end() == true)
+ {
+ if (Debug == true)
+ std::clog << "FindInCache returned end-Pointer" << std::endl;
+ return false;
+ }
+
+ RlsVisited[RlsFile->ID] = true;
+ if (Debug == true)
+ std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
+
+ std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
+ std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
+ [](pkgIndexFile const * const I) { return I->HasPackages(); });
+ }
+ for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
+ if (RlsVisited[I] == false)
+ {
+ if (Debug == true)
+ std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
+ return false;
+ }
+
+ std::copy(Start, End, std::back_inserter(Files));
+
+ /* Now we check every index file, see if it is in the cache,
+ verify the IMS data and check that it is on the disk too.. */
+ std::unique_ptr<bool[]> Visited(new bool[Cache.HeaderP->PackageFileCount]);
+ memset(Visited.get(),0,sizeof(Visited[0])*Cache.HeaderP->PackageFileCount);
+ for (std::vector<pkgIndexFile *>::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
+ {
+ if (Debug == true)
+ std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
+ if ((*PkgFile)->Exists() == false)
+ {
+ if (Debug == true)
+ std::clog << "file doesn't exist" << std::endl;
+ continue;
+ }
+
+ // FindInCache is also expected to do an IMS check.
+ pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
+ if (File.end() == true)
+ {
+ if (Debug == true)
+ std::clog << "FindInCache returned end-Pointer" << std::endl;
+ return false;
+ }
+
+ Visited[File->ID] = true;
+ if (Debug == true)
+ std::clog << "with ID " << File->ID << " is valid" << std::endl;
+ }
+
+ for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
+ if (Visited[I] == false)
+ {
+ if (Debug == true)
+ std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
+ return false;
+ }
+
+ if (_error->PendingError() == true)
+ {
+ if (Debug == true)
+ {
+ std::clog << "Validity failed because of pending errors:" << std::endl;
+ _error->DumpErrors(std::clog, GlobalError::DEBUG, false);
+ }
+ return false;
+ }
+
+ if (OutMap != 0)
+ *OutMap = Map.release();
+ if (OutCache != 0)
+ *OutCache = CacheP.release();
+ return true;
+}
+ /*}}}*/
+// ComputeSize - Compute the total size of a bunch of files /*{{{*/
+// ---------------------------------------------------------------------
+/* Size is kind of an abstract notion that is only used for the progress
+ meter */
+static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
+{
+ map_filesize_t TotalSize = 0;
+ if (List != NULL)
+ {
+ for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
+ {
+ std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
+ for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
+ if ((*j)->HasPackages() == true)
+ TotalSize += (*j)->Size();
+ }
+ }
+
+ for (; Start < End; ++Start)
+ {
+ if ((*Start)->HasPackages() == false)
+ continue;
+ TotalSize += (*Start)->Size();
+ }
+ return TotalSize;
+}
+ /*}}}*/
+// BuildCache - Merge the list of index files into the cache /*{{{*/
+static bool BuildCache(pkgCacheGenerator &Gen,
+ OpProgress * const Progress,
+ map_filesize_t &CurrentSize,map_filesize_t TotalSize,
+ pkgSourceList const * const List,
+ FileIterator const Start, FileIterator const End)
+{
+ bool mergeFailure = false;
+
+ auto const indexFileMerge = [&](pkgIndexFile * const I) {
+ if (I->HasPackages() == false || mergeFailure)
+ return;
+
+ if (I->Exists() == false)
+ return;
+
+ if (I->FindInCache(Gen.GetCache()).end() == false)
+ {
+ _error->Warning("Duplicate sources.list entry %s",
+ I->Describe().c_str());
+ return;
+ }
+
+ map_filesize_t const Size = I->Size();
+ if (Progress != NULL)
+ Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
+ CurrentSize += Size;
+
+ if (I->Merge(Gen,Progress) == false)
+ mergeFailure = true;
+ };
+
+ if (List != NULL)
+ {
+ for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
+ {
+ if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
+ {
+ _error->Warning("Duplicate sources.list entry %s",
+ (*i)->Describe().c_str());
+ continue;
+ }
+
+ if ((*i)->Merge(Gen, Progress) == false)
+ return false;
+
+ std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
+ if (Indexes != NULL)
+ std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
+ if (mergeFailure)
+ return false;
+ }
+ }
+
+ if (Start != End)
+ {
+ Gen.SelectReleaseFile("", "");
+ std::for_each(Start, End, indexFileMerge);
+ if (mergeFailure)
+ return false;
+ }
+ return true;
+}
+ /*}}}*/
+// CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/
+// ---------------------------------------------------------------------
+/* This makes sure that the status cache (the cache that has all
+ index files from the sources list and all local ones) is ready
+ to be mmaped. If OutMap is not zero then a MMap object representing
+ the cache will be stored there. This is pretty much mandetory if you
+ are using AllowMem. AllowMem lets the function be run as non-root
+ where it builds the cache 'fast' into a memory buffer. */
+static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
+{
+ map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
+ map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
+ map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
+ Flags |= MMap::Moveable;
+ if (_config->FindB("APT::Cache-Fallback", false) == true)
+ Flags |= MMap::Fallback;
+ if (CacheF != NULL)
+ return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
+ else
+ return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
+}
+static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
+ std::string const &FileName)
+{
+ FileFd SCacheF(FileName, FileFd::WriteAtomic);
+ if (SCacheF.IsOpen() == false || SCacheF.Failed())
+ return false;
+
+ fchmod(SCacheF.Fd(),0644);
+
+ // Write out the main data
+ if (SCacheF.Write(Map->Data(),Map->Size()) == false)
+ return _error->Error(_("IO Error saving source cache"));
+
+ // Write out the proper header
+ Gen->GetCache().HeaderP->Dirty = false;
+ Gen->GetCache().HeaderP->CacheFileSize = Gen->GetCache().CacheHash();
+ if (SCacheF.Seek(0) == false ||
+ SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
+ return _error->Error(_("IO Error saving source cache"));
+ Gen->GetCache().HeaderP->Dirty = true;
+ return true;
+}
+static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
+ std::unique_ptr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
+{
+ Map.reset(CreateDynamicMMap(NULL, 0));
+ if (unlikely(Map->validData()) == false)
+ return false;
+ FileFd CacheF(FileName, FileFd::ReadOnly);
+ if (CacheF.IsOpen() == false || CacheF.Failed())
+ return false;
+ _error->PushToStack();
+ map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
+ bool const newError = _error->PendingError();
+ _error->MergeWithStack();
+ if (alloc == 0 && newError)
+ return false;
+ if (CacheF.Read((unsigned char *)Map->Data() + alloc, CacheF.Size()) == false)
+ return false;
+ Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
+ return Gen->Start();
+}
+bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
+ MMap **OutMap, bool AllowMem)
+ { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
+bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
+ MMap **OutMap,bool)
+{
+ return pkgCacheGenerator::MakeStatusCache(List, Progress, OutMap, nullptr, true);
+}
+bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
+ MMap **OutMap,pkgCache **OutCache, bool)
+{
+ // FIXME: deprecate the ignored AllowMem parameter
+ bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
+
+ std::vector<pkgIndexFile *> Files;
+ if (_system->AddStatusFiles(Files) == false)
+ return false;
+
+ // Decide if we can write to the files..
+ string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
+ string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
+
+ // ensure the cache directory exists
+ if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
+ {
+ string dir = _config->FindDir("Dir::Cache");
+ size_t const len = dir.size();
+ if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
+ dir = dir.substr(0, len - 5);
+ if (CacheFile.empty() == false)
+ CreateDirectory(dir, flNotFile(CacheFile));
+ if (SrcCacheFile.empty() == false)
+ CreateDirectory(dir, flNotFile(SrcCacheFile));
+ }
+
+ if (Progress != NULL)
+ Progress->OverallProgress(0,1,1,_("Reading package lists"));
+
+ bool pkgcache_fine = false;
+ bool srcpkgcache_fine = false;
+ bool volatile_fine = List.GetVolatileFiles().empty();
+
+ if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL,
+ volatile_fine ? OutCache : NULL) == true)
+ {
+ if (Debug == true)
+ std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
+ pkgcache_fine = true;
+ srcpkgcache_fine = true;
+ }
+ if (pkgcache_fine == false)
+ {
+ if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
+ {
+ if (Debug == true)
+ std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
+ srcpkgcache_fine = true;
+ }
+ }
+
+ if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
+ {
+ if (Progress != NULL)
+ Progress->OverallProgress(1,1,1,_("Reading package lists"));
+ return true;
+ }
+
+ bool Writeable = false;
+ if (srcpkgcache_fine == false || pkgcache_fine == false)
+ {
+ if (CacheFile.empty() == false)
+ Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
+ else if (SrcCacheFile.empty() == false)
+ Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
+
+ if (Debug == true)
+ std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
+ }
+
+ // At this point we know we need to construct something, so get storage ready
+ std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
+ if (unlikely(Map->validData()) == false)
+ return false;
+ if (Debug == true)
+ std::clog << "Open memory Map (not filebased)" << std::endl;
+
+ std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
+ map_filesize_t CurrentSize = 0;
+ std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
+ map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
+ if (srcpkgcache_fine == true && pkgcache_fine == false)
+ {
+ if (Debug == true)
+ std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
+ if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
+ return false;
+ srcpkgcache_fine = true;
+ TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
+ }
+ else if (srcpkgcache_fine == false)
+ {
+ if (Debug == true)
+ std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
+ Gen.reset(new pkgCacheGenerator(Map.get(),Progress));
+ if (Gen->Start() == false)
+ return false;
+
+ TotalSize += ComputeSize(&List, Files.begin(),Files.end());
+ if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
+ Files.end(),Files.end()) == false)
+ return false;
+
+ if (Writeable == true && SrcCacheFile.empty() == false)
+ if (writeBackMMapToFile(Gen.get(), Map.get(), SrcCacheFile) == false)
+ return false;
+ }
+
+ if (pkgcache_fine == false)
+ {
+ if (Debug == true)
+ std::clog << "Building status cache in pkgcache.bin now" << std::endl;
+ if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
+ Files.begin(), Files.end()) == false)
+ return false;
+
+ if (Writeable == true && CacheFile.empty() == false)
+ if (writeBackMMapToFile(Gen.get(), Map.get(), CacheFile) == false)
+ return false;
+ }
+
+ if (Debug == true)
+ std::clog << "Caches done. " << (volatile_fine ? "No volatile files, so we are done here." : "Now bring in the volatile files") << std::endl;
+
+ if (volatile_fine == false)
+ {
+ if (Gen == nullptr)
+ {
+ if (Debug == true)
+ std::clog << "Populate new MMap with cachefile contents" << std::endl;
+ if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
+ return false;
+ }
+
+ Files = List.GetVolatileFiles();
+ if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
+ Files.begin(), Files.end()) == false)
+ return false;
+ }
+
+ if (OutMap != nullptr)
+ *OutMap = Map.release();
+
+ if (Debug == true)
+ std::clog << "Everything is ready for shipping" << std::endl;
+ return true;
+}
+ /*}}}*/
+// CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
+class APT_HIDDEN ScopedErrorMerge {
+public:
+ ScopedErrorMerge() { _error->PushToStack(); }
+ ~ScopedErrorMerge() { _error->MergeWithStack(); }
+};
+bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
+ { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
+bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
+{
+ std::vector<pkgIndexFile *> Files;
+ if (_system->AddStatusFiles(Files) == false)
+ return false;
+
+ ScopedErrorMerge sem;
+ std::unique_ptr<DynamicMMap> Map(CreateDynamicMMap(NULL, 0));
+ if (unlikely(Map->validData()) == false)
+ return false;
+ map_filesize_t CurrentSize = 0;
+ map_filesize_t TotalSize = 0;
+ TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
+
+ // Build the status cache
+ if (Progress != NULL)
+ Progress->OverallProgress(0,1,1,_("Reading package lists"));
+ pkgCacheGenerator Gen(Map.get(),Progress);
+ if (Gen.Start() == false || _error->PendingError() == true)
+ return false;
+ if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
+ Files.begin(), Files.end()) == false)
+ return false;
+
+ if (_error->PendingError() == true)
+ return false;
+ *OutMap = Map.release();
projects / apt.git / blobdiff