X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/115ba31b90e0d6643670d7d079f9f27b0e05f1c2..51da0c3587c9d36a3c04b3e28b99d8331a29b230:/ftparchive/apt-ftparchive.cc diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc index 6f9fa7ab3..ba71ee225 100644 --- a/ftparchive/apt-ftparchive.cc +++ b/ftparchive/apt-ftparchive.cc @@ -10,24 +10,34 @@ ##################################################################### */ /*}}}*/ // Include Files /*{{{*/ -#include "apt-ftparchive.h" - +#include + #include #include #include #include #include -#include -#include -#include +#include +#include #include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include -#include "contents.h" +#include "cachedb.h" +#include "override.h" +#include "apt-ftparchive.h" #include "multicompress.h" -#include "writer.h" +#include "writer.h" + +#include /*}}}*/ using namespace std; @@ -52,6 +62,7 @@ struct PackageMap // Stuff for the Package File string PkgFile; string BinCacheDB; + string SrcCacheDB; string BinOverride; string ExtraOverride; @@ -96,6 +107,12 @@ struct PackageMap inline bool operator() (const PackageMap &x,const PackageMap &y) {return x.BinCacheDB < y.BinCacheDB;}; }; + + struct SrcDBCompare : public binary_function + { + inline bool operator() (const PackageMap &x,const PackageMap &y) + {return x.SrcCacheDB < y.SrcCacheDB;}; + }; void GetGeneral(Configuration &Setup,Configuration &Block); bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats); @@ -135,6 +152,8 @@ void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block) PkgExt = Block.Find("Packages::Extensions", Setup.Find("Default::Packages::Extensions",".deb").c_str()); + Permissions = Setup.FindI("Default::FileMode",0644); + if (FLFile.empty() == false) FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile); @@ -203,7 +222,7 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats) Packages.Output = 0; // Just in case // Finish compressing - unsigned long Size; + unsigned long long Size; if (Comp.Finalize(Size) == false) { c0out << endl; @@ -220,11 +239,14 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats) gettimeofday(&NewTime,0); double Delta = NewTime.tv_sec - StartTime.tv_sec + (NewTime.tv_usec - StartTime.tv_usec)/1000000.0; - + c0out << Packages.Stats.Packages << " files " << /* SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */ SizeToStr(Packages.Stats.Bytes) << "B " << TimeToStr((long)Delta) << endl; + + if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true) + c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl; Stats.Add(Packages.Stats); Stats.DeLinkBytes = Packages.Stats.DeLinkBytes; @@ -251,7 +273,8 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats) SrcDone = true; // Create a package writer object. - SourcesWriter Sources(flCombine(OverrideDir,BinOverride), + SourcesWriter Sources(flCombine(CacheDir, SrcCacheDB), + flCombine(OverrideDir,BinOverride), flCombine(OverrideDir,SrcOverride), flCombine(OverrideDir,SrcExtraOverride)); if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false) @@ -289,7 +312,7 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats) Sources.Output = 0; // Just in case // Finish compressing - unsigned long Size; + unsigned long long Size; if (Comp.Finalize(Size) == false) { c0out << endl; @@ -310,6 +333,9 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats) c0out << Sources.Stats.Packages << " pkgs in " << TimeToStr((long)Delta) << endl; + if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true) + c0out << " Misses in Cache: " << Sources.Stats.Misses << endl; + Stats.Add(Sources.Stats); Stats.DeLinkBytes = Sources.Stats.DeLinkBytes; @@ -360,11 +386,11 @@ bool PackageMap::GenContents(Configuration &Setup, if (_error->PendingError() == true) return false; - unsigned long Size = Head.Size(); + unsigned long long Size = Head.Size(); unsigned char Buf[4096]; while (Size != 0) { - unsigned long ToRead = Size; + unsigned long long ToRead = Size; if (Size > sizeof(Buf)) ToRead = sizeof(Buf); @@ -382,7 +408,7 @@ bool PackageMap::GenContents(Configuration &Setup, files associated with this contents file into one great big honking memory structure, then dump the sorted version */ c0out << ' ' << this->Contents << ":" << flush; - for (vector::iterator I = Begin; I != End; I++) + for (vector::iterator I = Begin; I != End; ++I) { if (I->Contents != this->Contents) continue; @@ -398,7 +424,7 @@ bool PackageMap::GenContents(Configuration &Setup, Contents.Finish(); // Finish compressing - unsigned long Size; + unsigned long long Size; if (Comp.Finalize(Size) == false || _error->PendingError() == true) { c0out << endl; @@ -422,6 +448,9 @@ bool PackageMap::GenContents(Configuration &Setup, double Delta = NewTime.tv_sec - StartTime.tv_sec + (NewTime.tv_usec - StartTime.tv_usec)/1000000.0; + if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true) + c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl; + c0out << Contents.Stats.Packages << " files " << SizeToStr(Contents.Stats.Bytes) << "B " << TimeToStr((long)Delta) << endl; @@ -434,7 +463,7 @@ bool PackageMap::GenContents(Configuration &Setup, // --------------------------------------------------------------------- /* This populates the PkgList with all the possible permutations of the section/arch lists. */ -void LoadTree(vector &PkgList,Configuration &Setup) +static void LoadTree(vector &PkgList,Configuration &Setup) { // Load the defaults string DDir = Setup.Find("TreeDefault::Directory", @@ -448,16 +477,18 @@ void LoadTree(vector &PkgList,Configuration &Setup) string DIPrfx = Setup.Find("TreeDefault::InternalPrefix", "$(DIST)/$(SECTION)/"); string DContents = Setup.Find("TreeDefault::Contents", - "$(DIST)/Contents-$(ARCH)"); + "$(DIST)/$(SECTION)/Contents-$(ARCH)"); string DContentsH = Setup.Find("TreeDefault::Contents::Header",""); string DBCache = Setup.Find("TreeDefault::BinCacheDB", "packages-$(ARCH).db"); + string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB", + "sources-$(SECTION).db"); string DSources = Setup.Find("TreeDefault::Sources", "$(DIST)/$(SECTION)/source/Sources"); string DFLFile = Setup.Find("TreeDefault::FileList", ""); string DSFLFile = Setup.Find("TreeDefault::SourceFileList", ""); - int const Permissions = Setup.FindI("Default::FileMode",0644); + mode_t const Permissions = Setup.FindI("Default::FileMode",0644); bool const LongDescription = Setup.FindB("Default::LongDescription", _config->FindB("APT::FTPArchive::LongDescription", true)); @@ -480,7 +511,7 @@ void LoadTree(vector &PkgList,Configuration &Setup) struct SubstVar const Vars[] = {{"$(DIST)",&Dist}, {"$(SECTION)",&Section}, {"$(ARCH)",&Arch}, - {}}; + {NULL, NULL}}; mode_t const Perms = Block.FindI("FileMode", Permissions); bool const LongDesc = Block.FindB("LongDescription", LongDescription); TranslationWriter *TransWriter; @@ -511,6 +542,7 @@ void LoadTree(vector &PkgList,Configuration &Setup) Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars); Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars); Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars); + Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars); } else { @@ -546,8 +578,10 @@ void LoadTree(vector &PkgList,Configuration &Setup) // LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/ // --------------------------------------------------------------------- /* */ -void LoadBinDir(vector &PkgList,Configuration &Setup) +static void LoadBinDir(vector &PkgList,Configuration &Setup) { + mode_t const Permissions = Setup.FindI("Default::FileMode",0644); + // Process 'bindirectory' type sections const Configuration::Item *Top = Setup.Tree("bindirectory"); for (Top = (Top == 0?0:Top->Child); Top != 0;) @@ -558,6 +592,7 @@ void LoadBinDir(vector &PkgList,Configuration &Setup) Itm.PkgFile = Block.Find("Packages"); Itm.SrcFile = Block.Find("Sources"); Itm.BinCacheDB = Block.Find("BinCacheDB"); + Itm.SrcCacheDB = Block.Find("SrcCacheDB"); Itm.BinOverride = Block.Find("BinOverride"); Itm.ExtraOverride = Block.Find("ExtraOverride"); Itm.SrcExtraOverride = Block.Find("SrcExtraOverride"); @@ -567,6 +602,7 @@ void LoadBinDir(vector &PkgList,Configuration &Setup) Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str()); Itm.Contents = Block.Find("Contents"); Itm.ContentsHead = Block.Find("Contents::Header"); + Itm.Permissions = Block.FindI("FileMode", Permissions); Itm.GetGeneral(Setup,Block); PkgList.push_back(Itm); @@ -579,9 +615,9 @@ void LoadBinDir(vector &PkgList,Configuration &Setup) // ShowHelp - Show the help text /*{{{*/ // --------------------------------------------------------------------- /* */ -bool ShowHelp(CommandLine &CmdL) +static bool ShowHelp(CommandLine &) { - ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,VERSION, + ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION, COMMON_ARCH,__DATE__,__TIME__); if (_config->FindB("version") == true) return true; @@ -632,7 +668,7 @@ bool ShowHelp(CommandLine &CmdL) // SimpleGenPackages - Generate a Packages file for a directory tree /*{{{*/ // --------------------------------------------------------------------- /* This emulates dpkg-scanpackages's command line interface. 'mostly' */ -bool SimpleGenPackages(CommandLine &CmdL) +static bool SimpleGenPackages(CommandLine &CmdL) { if (CmdL.FileSize() < 2) return ShowHelp(CmdL); @@ -654,13 +690,17 @@ bool SimpleGenPackages(CommandLine &CmdL) if (Packages.RecursiveScan(CmdL.FileList[1]) == false) return false; + // Give some stats if asked for + if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true) + c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl; + return true; } /*}}}*/ // SimpleGenContents - Generate a Contents listing /*{{{*/ // --------------------------------------------------------------------- /* */ -bool SimpleGenContents(CommandLine &CmdL) +static bool SimpleGenContents(CommandLine &CmdL) { if (CmdL.FileSize() < 2) return ShowHelp(CmdL); @@ -682,7 +722,7 @@ bool SimpleGenContents(CommandLine &CmdL) // SimpleGenSources - Generate a Sources file for a directory tree /*{{{*/ // --------------------------------------------------------------------- /* This emulates dpkg-scanpackages's command line interface. 'mostly' */ -bool SimpleGenSources(CommandLine &CmdL) +static bool SimpleGenSources(CommandLine &CmdL) { if (CmdL.FileSize() < 2) return ShowHelp(CmdL); @@ -699,7 +739,7 @@ bool SimpleGenSources(CommandLine &CmdL) SOverride.c_str()); // Create a package writer object. - SourcesWriter Sources(Override,SOverride); + SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),Override,SOverride); if (_error->PendingError() == true) return false; @@ -710,12 +750,16 @@ bool SimpleGenSources(CommandLine &CmdL) if (Sources.RecursiveScan(CmdL.FileList[1]) == false) return false; + // Give some stats if asked for + if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true) + c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl; + return true; } /*}}}*/ // SimpleGenRelease - Generate a Release file for a directory tree /*{{{*/ // --------------------------------------------------------------------- -bool SimpleGenRelease(CommandLine &CmdL) +static bool SimpleGenRelease(CommandLine &CmdL) { if (CmdL.FileSize() < 2) return ShowHelp(CmdL); @@ -740,7 +784,7 @@ bool SimpleGenRelease(CommandLine &CmdL) // Generate - Full generate, using a config file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool Generate(CommandLine &CmdL) +static bool Generate(CommandLine &CmdL) { struct CacheDB::Stats SrcStats; if (CmdL.FileSize() < 2) @@ -761,14 +805,15 @@ bool Generate(CommandLine &CmdL) // Sort by cache DB to improve IO locality. stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare()); + stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare()); // Generate packages if (CmdL.FileSize() <= 2) { - for (vector::iterator I = PkgList.begin(); I != PkgList.end(); I++) + for (vector::iterator I = PkgList.begin(); I != PkgList.end(); ++I) if (I->GenPackages(Setup,Stats) == false) _error->DumpErrors(); - for (vector::iterator I = PkgList.begin(); I != PkgList.end(); I++) + for (vector::iterator I = PkgList.begin(); I != PkgList.end(); ++I) if (I->GenSources(Setup,SrcStats) == false) _error->DumpErrors(); } @@ -777,7 +822,7 @@ bool Generate(CommandLine &CmdL) // Make a choice list out of the package list.. RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1]; RxChoiceList *End = List; - for (vector::iterator I = PkgList.begin(); I != PkgList.end(); I++) + for (vector::iterator I = PkgList.begin(); I != PkgList.end(); ++I) { End->UserData = &(*I); End->Str = I->BaseDir.c_str(); @@ -827,7 +872,7 @@ bool Generate(CommandLine &CmdL) } // close the Translation master files - for (vector::iterator I = PkgList.begin(); I != PkgList.end(); I++) + for (vector::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I) if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0) delete I->TransWriter; @@ -838,7 +883,7 @@ bool Generate(CommandLine &CmdL) // Sort the contents file list by date string ArchiveDir = Setup.FindDir("Dir::ArchiveDir"); - for (vector::iterator I = PkgList.begin(); I != PkgList.end(); I++) + for (vector::iterator I = PkgList.begin(); I != PkgList.end(); ++I) { struct stat A; if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents), @@ -855,9 +900,9 @@ bool Generate(CommandLine &CmdL) hashes of the .debs this means they have not changed either so the contents must be up to date. */ unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024; - for (vector::iterator I = PkgList.begin(); I != PkgList.end(); I++) + for (vector::iterator I = PkgList.begin(); I != PkgList.end(); ++I) { - // This record is not relevent + // This record is not relevant if (I->ContentsDone == true || I->Contents.empty() == true) continue; @@ -904,7 +949,7 @@ bool Generate(CommandLine &CmdL) // Clean - Clean out the databases /*{{{*/ // --------------------------------------------------------------------- /* */ -bool Clean(CommandLine &CmdL) +static bool Clean(CommandLine &CmdL) { if (CmdL.FileSize() != 2) return ShowHelp(CmdL); @@ -920,20 +965,33 @@ bool Clean(CommandLine &CmdL) // Sort by cache DB to improve IO locality. stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare()); + stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare()); string CacheDir = Setup.FindDir("Dir::CacheDir"); for (vector::iterator I = PkgList.begin(); I != PkgList.end(); ) { - c0out << I->BinCacheDB << endl; + if(I->BinCacheDB != "") + c0out << I->BinCacheDB << endl; + if(I->SrcCacheDB != "") + c0out << I->SrcCacheDB << endl; CacheDB DB(flCombine(CacheDir,I->BinCacheDB)); + CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB)); if (DB.Clean() == false) _error->DumpErrors(); + if (DB_SRC.Clean() == false) + _error->DumpErrors(); string CacheDB = I->BinCacheDB; - for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++); + string SrcCacheDB = I->SrcCacheDB; + while(I != PkgList.end() && + I->BinCacheDB == CacheDB && + I->SrcCacheDB == SrcCacheDB) + ++I; + } - + + return true; } /*}}}*/ @@ -944,6 +1002,8 @@ int main(int argc, const char *argv[]) CommandLine::Args Args[] = { {'h',"help","help",0}, {0,"md5","APT::FTPArchive::MD5",0}, + {0,"sha1","APT::FTPArchive::SHA1",0}, + {0,"sha256","APT::FTPArchive::SHA256",0}, {'v',"version","version",0}, {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg}, {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},