From b35d2f5fa0e797bf7062a1a7448a5d54bb396071 Mon Sep 17 00:00:00 2001 From: Arch Librarian Date: Mon, 20 Sep 2004 16:50:54 +0000 Subject: [PATCH] Progress and combined cache generator Author: jgg Date: 1998-07-26 04:49:26 GMT Progress and combined cache generator --- apt-pkg/contrib/mmap.cc | 5 +- apt-pkg/contrib/progress.cc | 9 +- apt-pkg/deb/deblistparser.cc | 14 +- apt-pkg/init.cc | 13 +- apt-pkg/pkgcachegen.cc | 284 +++++++++++++++++++++++++++++++++-- apt-pkg/pkgcachegen.h | 12 +- apt-pkg/sourcelist.cc | 4 +- 7 files changed, 310 insertions(+), 31 deletions(-) diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc index f8582a1ef..e12e077c6 100644 --- a/apt-pkg/contrib/mmap.cc +++ b/apt-pkg/contrib/mmap.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: mmap.cc,v 1.7 1998/07/19 04:42:14 jgg Exp $ +// $Id: mmap.cc,v 1.8 1998/07/26 04:49:34 jgg Exp $ /* ###################################################################### MMap Class - Provides 'real' mmap or a faked mmap using read(). @@ -69,6 +69,9 @@ bool MMap::Map() if ((Flags & Public) != Public) Map = MAP_PRIVATE; + if (iSize == 0) + return _error->Error("Can't mmap an empty file"); + // Map it. Base = mmap(0,iSize,Prot,Map,Fd.Fd(),0); if (Base == (void *)-1) diff --git a/apt-pkg/contrib/progress.cc b/apt-pkg/contrib/progress.cc index 8d49eff2e..aa5acf010 100644 --- a/apt-pkg/contrib/progress.cc +++ b/apt-pkg/contrib/progress.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: progress.cc,v 1.2 1998/07/22 01:45:38 jgg Exp $ +// $Id: progress.cc,v 1.3 1998/07/26 04:49:35 jgg Exp $ /* ###################################################################### OpProgress - Operation Progress @@ -97,7 +97,10 @@ void OpTextProgress::Done() if (NoUpdate == false && OldOp.empty() == false) { char S[300]; - snprintf(S,sizeof(S),"\r%s... Done",OldOp.c_str()); + if (_errors->PendingError() == true) + snprintf(S,sizeof(S),"\r%s... Error!",OldOp.c_str()); + else + snprintf(S,sizeof(S),"\r%s... Done",OldOp.c_str()); Write(S); cout << endl; OldOp = string(); @@ -109,7 +112,7 @@ void OpTextProgress::Done() /* */ void OpTextProgress::Update() { - if (CheckChange(0) == false) + if (CheckChange() == false) return; // No percent spinner diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index c2ae6de9c..f1c1925e1 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: deblistparser.cc,v 1.8 1998/07/19 04:42:16 jgg Exp $ +// $Id: deblistparser.cc,v 1.9 1998/07/26 04:49:37 jgg Exp $ /* ###################################################################### Package Cache Generator - Generator for the cache structure. @@ -133,16 +133,13 @@ string debListParser::Version() bool debListParser::NewVersion(pkgCache::VerIterator Ver) { // Parse the section - if ((Ver->Section = UniqFindTagWrite("Section")) == 0) - return _error->Warning("Missing Section tag"); + Ver->Section = UniqFindTagWrite("Section"); // Archive Size - if ((Ver->Size = (unsigned)FindTagI("Size")) == 0) - return _error->Error("Unparsable Size field"); + Ver->Size = (unsigned)FindTagI("Size"); // Unpacked Size (in K) - if ((Ver->InstalledSize = (unsigned)FindTagI("Installed-Size")) == 0) - return _error->Error("Unparsable Installed-Size field"); + Ver->InstalledSize = (unsigned)FindTagI("Installed-Size"); Ver->InstalledSize *= 1024; // Priority @@ -187,8 +184,7 @@ bool debListParser::UsePackage(pkgCache::PkgIterator Pkg, pkgCache::VerIterator Ver) { if (Pkg->Section == 0) - if ((Pkg->Section = UniqFindTagWrite("Section")) == 0) - return false; + Pkg->Section = UniqFindTagWrite("Section"); if (HandleFlag("Essential",Pkg->Flags,pkgCache::Flag::Essential) == false) return false; if (HandleFlag("Immediate-Configure",Pkg->Flags,pkgCache::Flag::ImmediateConf) == false) diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc index 63e808118..f8cf54ee4 100644 --- a/apt-pkg/init.cc +++ b/apt-pkg/init.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: init.cc,v 1.3 1998/07/16 06:08:36 jgg Exp $ +// $Id: init.cc,v 1.4 1998/07/26 04:49:26 jgg Exp $ /* ###################################################################### Init - Initialize the package library @@ -24,14 +24,19 @@ bool pkgInitialize(Configuration &Cnf) // State Cnf.Set("Dir::State","/var/state/apt/"); Cnf.Set("Dir::State::lists","lists/"); + + /* These really should be jammed into a generic 'Local Database' engine + which is yet to be determined. The functions in pkgcachegen should + be the only users of these */ Cnf.Set("Dir::State::xstatus","xstatus"); - Cnf.Set("Dir::State::userstatus","status.user"); + Cnf.Set("Dir::State::userstatus","status.user"); + Cnf.Set("Dir::State::status","/var/lib/dpkg/status"); // Cache - Cnf.Set("Dir::Cache","/etc/apt/"); + Cnf.Set("Dir::Cache","/tmp/"); Cnf.Set("Dir::Cache::archives","archives/"); Cnf.Set("Dir::Cache::srcpkgcache","srcpkgcache"); - Cnf.Set("Dir::Cache::pkhcache","pkgcache"); + Cnf.Set("Dir::Cache::pkgcache","pkgcache"); // Configuration Cnf.Set("Dir::Etc","/etc/apt/"); diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 3e973bb85..c0297674a 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: pkgcachegen.cc,v 1.12 1998/07/21 05:33:19 jgg Exp $ +// $Id: pkgcachegen.cc,v 1.13 1998/07/26 04:49:31 jgg Exp $ /* ###################################################################### Package Cache Generator - Generator for the cache structure. @@ -17,6 +17,11 @@ #include #include #include +#include +#include +#include +#include + #include #include @@ -71,7 +76,7 @@ bool pkgCacheGenerator::MergeList(ListParser &List) string PackageName = List.Package(); pkgCache::PkgIterator Pkg; if (NewPackage(Pkg,PackageName) == false) - return false; + return _error->Error("Error occured while processing %s (NewPackage)",PackageName.c_str()); Progress.Progress(List.Offset()); /* Get a pointer to the version structure. We know the list is sorted @@ -81,7 +86,7 @@ bool pkgCacheGenerator::MergeList(ListParser &List) if (Version.empty() == true) { if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false) - return false; + return _error->Error("Error occured while processing %s (UsePackage1)",PackageName.c_str()); continue; } @@ -101,10 +106,10 @@ bool pkgCacheGenerator::MergeList(ListParser &List) if (Res == 0) { if (List.UsePackage(Pkg,Ver) == false) - return false; + return _error->Error("Error occured while processing %s (UsePackage2)",PackageName.c_str()); if (NewFileVer(Ver,List) == false) - return false; + return _error->Error("Error occured while processing %s (NewFileVer1)",PackageName.c_str()); continue; } @@ -113,13 +118,13 @@ bool pkgCacheGenerator::MergeList(ListParser &List) *Last = NewVersion(Ver,Version,*Last); Ver->ParentPkg = Pkg.Index(); if (List.NewVersion(Ver) == false) - return false; + return _error->Error("Error occured while processing %s (NewVersion1)",PackageName.c_str()); if (List.UsePackage(Pkg,Ver) == false) - return false; + return _error->Error("Error occured while processing %s (UsePackage3)",PackageName.c_str()); if (NewFileVer(Ver,List) == false) - return false; + return _error->Error("Error occured while processing %s (NewVersion2)",PackageName.c_str()); } return true; @@ -317,7 +322,8 @@ bool pkgCacheGenerator::SelectFile(string File,unsigned long Flags) CurrentFile->Flags = Flags; PkgFileName = File; Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP; - + Cache.HeaderP->PackageFileCount++; + if (CurrentFile->FileName == 0) return false; @@ -364,3 +370,263 @@ unsigned long pkgCacheGenerator::WriteUniqString(const char *S, return ItemP->String; } /*}}}*/ + +// SrcCacheCheck - Check if the source package cache is uptodate /*{{{*/ +// --------------------------------------------------------------------- +/* The source cache is checked against the source list and the files + on disk, any difference results in a false. */ +bool pkgSrcCacheCheck(pkgSourceList &List) +{ + if (_error->PendingError() == true) + return false; + + // Open the source package cache + string CacheFile = _config->FindDir("Dir::Cache::srcpkgcache"); + string ListDir = _config->FindDir("Dir::State::lists"); + if (FileExists(CacheFile) == false) + return false; + + FileFd CacheF(CacheFile,FileFd::ReadOnly); + if (_error->PendingError() == true) + { + _error->Discard(); + return false; + } + + MMap Map(CacheF,MMap::Public | MMap::ReadOnly); + if (_error->PendingError() == true) + { + _error->Discard(); + return false; + } + + pkgCache Cache(Map); + if (_error->PendingError() == true) + { + _error->Discard(); + return false; + } + + // They are certianly out of sync + if (Cache.Head().PackageFileCount != List.size()) + return false; + + for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++) + { + // Search for a match in the source list + bool Bad = true; + for (pkgSourceList::const_iterator I = List.begin(); + I != List.end(); I++) + { + string File = ListDir + URItoFileName(I->PackagesURI()); + if (F.FileName() == File) + { + Bad = false; + break; + } + } + + // Check if the file matches what was cached + Bad |= !F.IsOk(); + if (Bad == true) + return false; + } + + return true; +} + /*}}}*/ +// PkgCacheCheck - Check if the package cache is uptodate /*{{{*/ +// --------------------------------------------------------------------- +/* This does a simple check of all files used to compose the cache */ +bool pkgPkgCacheCheck(string CacheFile) +{ + if (_error->PendingError() == true) + return false; + + // Open the source package cache + if (FileExists(CacheFile) == false) + return false; + + FileFd CacheF(CacheFile,FileFd::ReadOnly); + if (_error->PendingError() == true) + { + _error->Discard(); + return false; + } + + MMap Map(CacheF,MMap::Public | MMap::ReadOnly); + if (_error->PendingError() == true) + { + _error->Discard(); + return false; + } + + pkgCache Cache(Map); + if (_error->PendingError() == true) + { + _error->Discard(); + return false; + } + + // Cheack each file + for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++) + if (F.IsOk() == false) + return false; + return true; +} + /*}}}*/ +// AddSourcesSize - Add the size of the status files /*{{{*/ +// --------------------------------------------------------------------- +/* This adds the size of all the status files to the size counter */ +static bool pkgAddSourcesSize(unsigned long &TotalSize) +{ + // Grab the file names + string xstatus = _config->FindDir("Dir::State::xstatus"); + string userstatus = _config->FindDir("Dir::State::userstatus"); + string status = _config->FindDir("Dir::State::status"); + + // Grab the sizes + struct stat Buf; + if (stat(xstatus.c_str(),&Buf) == 0) + TotalSize += Buf.st_size; + if (stat(userstatus.c_str(),&Buf) == 0) + TotalSize += Buf.st_size; + if (stat(status.c_str(),&Buf) != 0) + return _error->Errno("stat","Couldn't stat the status file %s",status.c_str()); + TotalSize += Buf.st_size; + + return true; +} + /*}}}*/ +// MergeStatus - Add the status files to the cache /*{{{*/ +// --------------------------------------------------------------------- +/* This adds the status files to the map */ +static bool pkgMergeStatus(OpProgress &Progress,pkgCacheGenerator &Gen, + unsigned long &CurrentSize,unsigned long TotalSize) +{ + // Grab the file names + string Status[3]; + Status[0] = _config->FindDir("Dir::State::xstatus"); + Status[1]= _config->FindDir("Dir::State::userstatus"); + Status[2] = _config->FindDir("Dir::State::status"); + + for (int I = 0; I != 3; I++) + { + // Check if the file exists and it is not the primary status file. + string File = Status[I]; + if (I != 2 && FileExists(File) == false) + continue; + + FileFd Pkg(File,FileFd::ReadOnly); + debListParser Parser(Pkg); + Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Generating cache"); + if (_error->PendingError() == true) + return _error->Error("Problem opening %s",File.c_str()); + CurrentSize += Pkg.Size(); + + if (Gen.SelectFile(File,pkgCache::Flag::NotSource) == false) + return _error->Error("Problem with SelectFile %s",File.c_str()); + + if (Gen.MergeList(Parser) == false) + return _error->Error("Problem with MergeList %s",File.c_str()); + } + + return true; +} + /*}}}*/ +// MakeStatusCache - Generates a cache that includes the status files /*{{{*/ +// --------------------------------------------------------------------- +/* This copies the package source cache and then merges the status and + xstatus files into it. */ +bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress) +{ + string CacheFile = _config->FindDir("Dir::Cache::pkgcache"); + bool SrcOk = pkgSrcCacheCheck(List); + bool PkgOk = pkgPkgCacheCheck(CacheFile); + + // Rebuild the source and package caches + if (SrcOk == false) + { + string SCacheFile = _config->FindDir("Dir::Cache::srcpkgcache"); + string ListDir = _config->FindDir("Dir::State::lists"); + + FileFd SCacheF(SCacheFile,FileFd::WriteEmpty); + FileFd CacheF(CacheFile,FileFd::WriteEmpty); + DynamicMMap Map(CacheF,MMap::Public); + if (_error->PendingError() == true) + return false; + + pkgCacheGenerator Gen(Map,Progress); + + // Prepare the progress indicator + unsigned long TotalSize = 0; + struct stat Buf; + for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++) + { + string File = ListDir + URItoFileName(I->PackagesURI()); + if (stat(File.c_str(),&Buf) != 0) + return _error->Errno("stat","Couldn't stat source package list %s",File.c_str()); + TotalSize += Buf.st_size; + } + + if (pkgAddSourcesSize(TotalSize) == false) + return false; + + // Generate the pkg source cache + unsigned long CurrentSize = 0; + for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++) + { + string File = ListDir + URItoFileName(I->PackagesURI()); + FileFd Pkg(File,FileFd::ReadOnly); + debListParser Parser(Pkg); + Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Generating cache"); + if (_error->PendingError() == true) + return _error->Error("Problem opening %s",File.c_str()); + CurrentSize += Pkg.Size(); + + if (Gen.SelectFile(File) == false) + return _error->Error("Problem with SelectFile %s",File.c_str()); + + if (Gen.MergeList(Parser) == false) + return _error->Error("Problem with MergeList %s",File.c_str()); + } + + // Write the src cache + Gen.GetCache().HeaderP->Dirty = false; + if (SCacheF.Write(Map.Data(),Map.Size()) == false) + return _error->Error("IO Error saving source cache"); + Gen.GetCache().HeaderP->Dirty = true; + + // Merge in the source caches + return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize); + } + + if (PkgOk == true) + return true; + + // We use the source cache to generate the package cache + string SCacheFile = _config->FindDir("Dir::Cache::srcpkgcache"); + + FileFd SCacheF(SCacheFile,FileFd::ReadOnly); + FileFd CacheF(CacheFile,FileFd::WriteEmpty); + DynamicMMap Map(CacheF,MMap::Public); + if (_error->PendingError() == true) + return false; + + // Preload the map with the source cache + if (SCacheF.Read((unsigned char *)Map.Data() + Map.RawAllocate(SCacheF.Size()), + SCacheF.Size()) == false) + return false; + + pkgCacheGenerator Gen(Map,Progress); + + // Compute the progress + unsigned long TotalSize = 0; + if (pkgAddSourcesSize(TotalSize) == false) + return false; + + unsigned long CurrentSize = 0; + return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize); +} + /*}}}*/ + diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index ff397bfac..c572f828a 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: pkgcachegen.h,v 1.8 1998/07/21 05:33:20 jgg Exp $ +// $Id: pkgcachegen.h,v 1.9 1998/07/26 04:49:32 jgg Exp $ /* ###################################################################### Package Cache Generator - Generator for the cache structure. @@ -25,7 +25,9 @@ #endif #include -#include + +class pkgSourceList; +class OpProgress; class pkgCacheGenerator { @@ -88,9 +90,13 @@ class pkgCacheGenerator bool SelectFile(string File,unsigned long Flags = 0); bool MergeList(ListParser &List); inline pkgCache &GetCache() {return Cache;}; - + pkgCacheGenerator(DynamicMMap &Map,OpProgress &Progress); ~pkgCacheGenerator(); }; +bool pkgSrcCacheCheck(pkgSourceList &List); +bool pkgPkgCacheCheck(string CacheFile); +bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress); + #endif diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index faae6a330..6fd5e1eb5 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -1,6 +1,6 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: sourcelist.cc,v 1.4 1998/07/19 04:22:04 jgg Exp $ +// $Id: sourcelist.cc,v 1.5 1998/07/26 04:49:33 jgg Exp $ /* ###################################################################### List of Sources @@ -41,7 +41,7 @@ pkgSourceList::pkgSourceList(string File) /* */ bool pkgSourceList::ReadMainList() { - return Read(_config->Find("APT::Etc:sourcelist")); + return Read(_config->FindDir("Dir::Etc::sourcelist")); } /*}}}*/ // SourceList::Read - Parse the sourcelist file /*{{{*/ -- 2.45.2