X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/fb3dc57975989a6514577569af9171732b7c5403..1bae10217617c2f79969635d4387fb2a5fe19ecb:/ftparchive/apt-ftparchive.cc

diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc
index d0dea7768..d985646f7 100644
--- a/ftparchive/apt-ftparchive.cc
+++ b/ftparchive/apt-ftparchive.cc
@@ -3,37 +3,48 @@
 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
 /* ######################################################################
 
-   apt-scanpackages - Efficient work-alike for dpkg-scanpackages
+   apt-ftparchive - Efficient work-alike for dpkg-scanpackages
 
    Let contents be disabled from the conf
    
    ##################################################################### */
 									/*}}}*/
 // Include Files							/*{{{*/
-#include "apt-ftparchive.h"
-    
+#include <config.h>
+
 #include <apt-pkg/error.h>
 #include <apt-pkg/configuration.h>
 #include <apt-pkg/cmndline.h>
 #include <apt-pkg/strutl.h>
-#include <config.h>
-#include <apti18n.h>
-#include <algorithm>
+#include <apt-pkg/init.h>
+#include <apt-pkg/fileutl.h>
+
+#include <apt-private/private-cmndline.h>
+#include <apt-private/private-output.h>
+#include <apt-private/private-main.h>
 
+#include <algorithm>
 #include <climits>
 #include <sys/time.h>
-#include <regex.h>
+#include <locale.h>
+#include <stdio.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <functional>
+#include <iostream>
+#include <string>
+#include <vector>
 
-#include "contents.h"
+#include "cachedb.h"
+#include "override.h"
+#include "apt-ftparchive.h"
 #include "multicompress.h"
-#include "writer.h"    
+#include "writer.h"
+
+#include <apti18n.h>
 									/*}}}*/
 
-using namespace std;    
-ostream c0out(0);
-ostream c1out(0);
-ostream c2out(0);
-ofstream devnull("/dev/null");
+using namespace std;
 unsigned Quiet = 0;
 
 // struct PackageMap - List of all package files in the config file	/*{{{*/
@@ -51,17 +62,23 @@ struct PackageMap
    // Stuff for the Package File
    string PkgFile;
    string BinCacheDB;
+   string SrcCacheDB;
    string BinOverride;
    string ExtraOverride;
 
    // We generate for this given arch
    string Arch;
+   bool IncludeArchAll;
    
    // Stuff for the Source File
    string SrcFile;
    string SrcOverride;
    string SrcExtraOverride;
 
+   // Translation master file
+   bool LongDesc;
+   TranslationWriter *TransWriter;
+
    // Contents 
    string Contents;
    string ContentsHead;
@@ -91,6 +108,12 @@ struct PackageMap
       inline bool operator() (const PackageMap &x,const PackageMap &y)
       {return x.BinCacheDB < y.BinCacheDB;};
    };  
+
+   struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
+   {
+      inline bool operator() (const PackageMap &x,const PackageMap &y)
+      {return x.SrcCacheDB < y.SrcCacheDB;};
+   };
    
    void GetGeneral(Configuration &Setup,Configuration &Block);
    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
@@ -100,8 +123,9 @@ struct PackageMap
 		    vector<PackageMap>::iterator End,
 		    unsigned long &Left);
    
-   PackageMap() : DeLinkLimit(0), Permissions(1), ContentsDone(false), 
-        PkgDone(false), SrcDone(false), ContentsMTime(0) {};
+   PackageMap() : IncludeArchAll(true), LongDesc(true), TransWriter(NULL),
+		  DeLinkLimit(0), Permissions(1), ContentsDone(false),
+		  PkgDone(false), SrcDone(false), ContentsMTime(0) {};
 };
 									/*}}}*/
 
@@ -113,7 +137,7 @@ void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
    PathPrefix = Block.Find("PathPrefix");
    
    if (Block.FindB("External-Links",true) == false)
-      DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
+      DeLinkLimit = Setup.FindI("Default::DeLinkLimit", std::numeric_limits<unsigned int>::max());
    else
       DeLinkLimit = 0;
    
@@ -130,7 +154,7 @@ void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 		       Setup.Find("Default::Packages::Extensions",".deb").c_str());
    
    Permissions = Setup.FindI("Default::FileMode",0644);
-   
+
    if (FLFile.empty() == false)
       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
    
@@ -156,10 +180,12 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
    PkgDone = true;
    
    // Create a package writer object.
-   PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
+   MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
+		      PkgCompress,Permissions);
+   PackagesWriter Packages(&Comp.Input, TransWriter, flCombine(CacheDir,BinCacheDB),
 			   flCombine(OverrideDir,BinOverride),
 			   flCombine(OverrideDir,ExtraOverride),
-			   Arch);
+			   Arch, IncludeArchAll);
    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
       return _error->Error(_("Package extension list is too long"));
    if (_error->PendingError() == true)
@@ -169,13 +195,11 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
    Packages.DirStrip = ArchiveDir;
    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 
+   Packages.LongDescription = LongDesc;
+
    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
    Packages.DeLinkLimit = DeLinkLimit;
 
-   // Create a compressor object
-   MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
-		      PkgCompress,Permissions);
-   Packages.Output = Comp.Input;
    if (_error->PendingError() == true)
       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
    
@@ -196,7 +220,7 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
    Packages.Output = 0;      // Just in case
    
    // Finish compressing
-   unsigned long Size;
+   unsigned long long Size;
    if (Comp.Finalize(Size) == false)
    {
       c0out << endl;
@@ -213,11 +237,14 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
    gettimeofday(&NewTime,0);
    double Delta = NewTime.tv_sec - StartTime.tv_sec + 
                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
-   
+
    c0out << Packages.Stats.Packages << " files " <<
 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
       SizeToStr(Packages.Stats.Bytes) << "B " <<
       TimeToStr((long)Delta) << endl;
+
+   if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
+     c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
    
    Stats.Add(Packages.Stats);
    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
@@ -244,7 +271,10 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
    SrcDone = true;
    
    // Create a package writer object.
-   SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
+   MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
+		      SrcCompress,Permissions);
+   SourcesWriter Sources(&Comp.Input, flCombine(CacheDir, SrcCacheDB),
+			 flCombine(OverrideDir,BinOverride),
 			 flCombine(OverrideDir,SrcOverride),
 			 flCombine(OverrideDir,SrcExtraOverride));
    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
@@ -258,11 +288,7 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 
    Sources.DeLinkLimit = DeLinkLimit;
    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
-   
-   // Create a compressor object
-   MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
-		      SrcCompress,Permissions);
-   Sources.Output = Comp.Input;
+
    if (_error->PendingError() == true)
       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 
@@ -282,7 +308,7 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
    Sources.Output = 0;      // Just in case
    
    // Finish compressing
-   unsigned long Size;
+   unsigned long long Size;
    if (Comp.Finalize(Size) == false)
    {
       c0out << endl;
@@ -303,6 +329,9 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
    c0out << Sources.Stats.Packages << " pkgs in " <<
       TimeToStr((long)Delta) << endl;
 
+   if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
+     c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
+
    Stats.Add(Sources.Stats);
    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
    
@@ -333,16 +362,15 @@ bool PackageMap::GenContents(Configuration &Setup,
    gettimeofday(&StartTime,0);   
    
    // Create a package writer object.
-   ContentsWriter Contents("");
+   MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
+		      CntCompress,Permissions);
+   Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
+   ContentsWriter Contents(&Comp.Input, "", Arch, IncludeArchAll);
    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
       return _error->Error(_("Package extension list is too long"));
    if (_error->PendingError() == true)
       return false;
 
-   MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
-		      CntCompress,Permissions);
-   Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
-   Contents.Output = Comp.Input;
    if (_error->PendingError() == true)
       return false;
 
@@ -352,30 +380,30 @@ bool PackageMap::GenContents(Configuration &Setup,
       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
       if (_error->PendingError() == true)
 	 return false;
-      
-      unsigned long Size = Head.Size();
+
+      unsigned long long Size = Head.Size();
       unsigned char Buf[4096];
       while (Size != 0)
       {
-	 unsigned long ToRead = Size;
+	 unsigned long long ToRead = Size;
 	 if (Size > sizeof(Buf))
 	    ToRead = sizeof(Buf);
-	 
+
 	 if (Head.Read(Buf,ToRead) == false)
 	    return false;
-	 
-	 if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
+
+	 if (Comp.Input.Write(Buf, ToRead) == false)
 	    return _error->Errno("fwrite",_("Error writing header to contents file"));
-	 
+
 	 Size -= ToRead;
-      }            
-   }  
-      
+      }
+   }
+
    /* Go over all the package file records and parse all the package
       files associated with this contents file into one great big honking
       memory structure, then dump the sorted version */
    c0out << ' ' << this->Contents << ":" << flush;
-   for (vector<PackageMap>::iterator I = Begin; I != End; I++)
+   for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
    {
       if (I->Contents != this->Contents)
 	 continue;
@@ -391,7 +419,7 @@ bool PackageMap::GenContents(Configuration &Setup,
    Contents.Finish();
    
    // Finish compressing
-   unsigned long Size;
+   unsigned long long Size;
    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
    {
       c0out << endl;
@@ -415,6 +443,9 @@ bool PackageMap::GenContents(Configuration &Setup,
    double Delta = NewTime.tv_sec - StartTime.tv_sec + 
                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
    
+   if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
+     c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
+
    c0out << Contents.Stats.Packages << " files " <<
       SizeToStr(Contents.Stats.Bytes) << "B " <<
       TimeToStr((long)Delta) << endl;
@@ -427,7 +458,7 @@ bool PackageMap::GenContents(Configuration &Setup,
 // ---------------------------------------------------------------------
 /* This populates the PkgList with all the possible permutations of the
    section/arch lists. */
-void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
+static void LoadTree(vector<PackageMap> &PkgList, std::vector<TranslationWriter*> &TransList, Configuration &Setup)
 {   
    // Load the defaults
    string DDir = Setup.Find("TreeDefault::Directory",
@@ -436,18 +467,30 @@ void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 			    "$(DIST)/$(SECTION)/source/");
    string DPkg = Setup.Find("TreeDefault::Packages",
 			    "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
+   string DTrans = Setup.Find("TreeDefault::Translation",
+			    "$(DIST)/$(SECTION)/i18n/Translation-en");
    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 			    "$(DIST)/$(SECTION)/");
    string DContents = Setup.Find("TreeDefault::Contents",
-			    "$(DIST)/Contents-$(ARCH)");
+			    "$(DIST)/$(SECTION)/Contents-$(ARCH)");
    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 			       "packages-$(ARCH).db");
+   string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
+			       "sources-$(SECTION).db");
    string DSources = Setup.Find("TreeDefault::Sources",
 				"$(DIST)/$(SECTION)/source/Sources");
    string DFLFile = Setup.Find("TreeDefault::FileList", "");
    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 
+   mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
+
+   bool const LongDescription = Setup.FindB("Default::LongDescription",
+					_config->FindB("APT::FTPArchive::LongDescription", true));
+   string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
+   bool const ConfIncludeArchAllExists = _config->Exists("APT::FTPArchive::IncludeArchitectureAll");
+   bool const ConfIncludeArchAll = _config->FindB("APT::FTPArchive::IncludeArchitectureAll", true);
+
    // Process 'tree' type sections
    const Configuration::Item *Top = Setup.Tree("tree");
    for (Top = (Top == 0?0:Top->Child); Top != 0;)
@@ -461,21 +504,32 @@ void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
       string Section;
       while (ParseQuoteWord(Sections,Section) == true)
       {
-	 string Tmp2 = Block.Find("Architectures");
-	 string Arch;
-	 const char *Archs = Tmp2.c_str();
-	 while (ParseQuoteWord(Archs,Arch) == true)
+	 struct SubstVar Vars[] = {{"$(DIST)",&Dist},
+					 {"$(SECTION)",&Section},
+					 {"$(ARCH)",nullptr},
+					 {nullptr, nullptr}};
+	 mode_t const Perms = Block.FindI("FileMode", Permissions);
+	 bool const LongDesc = Block.FindB("LongDescription", LongDescription);
+	 TranslationWriter *TransWriter = nullptr;
+
+	 std::string Tmp2 = Block.Find("Architectures");
+	 std::transform(Tmp2.begin(), Tmp2.end(), Tmp2.begin(), ::tolower);
+	 std::vector<std::string> const Archs = VectorizeString(Tmp2, ' ');
+	 bool IncludeArchAll;
+	 if (ConfIncludeArchAllExists == true)
+	    IncludeArchAll = ConfIncludeArchAll;
+	 else
+	    IncludeArchAll = std::find(Archs.begin(), Archs.end(), "all") == Archs.end();
+	 for (auto const& Arch: Archs)
 	 {
-	    struct SubstVar Vars[] = {{"$(DIST)",&Dist},
-	                              {"$(SECTION)",&Section},
-	                              {"$(ARCH)",&Arch},
-	                              {}};
+	    if (Arch.empty()) continue;
+	    Vars[2].Contents = &Arch;
 	    PackageMap Itm;
-	    
+	    Itm.Permissions = Perms;
 	    Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 	    Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 
-	    if (stringcasecmp(Arch,"source") == 0)
+	    if (Arch == "source")
 	    {
 	       Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 	       Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
@@ -483,6 +537,7 @@ void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 	       Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 	       Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 	       Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
+	       Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);
 	    }
 	    else
 	    {
@@ -491,26 +546,45 @@ void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 	       Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 	       Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 	       Itm.Arch = Arch;
+	       Itm.IncludeArchAll = IncludeArchAll;
+	       Itm.LongDesc = LongDesc;
+	       if (TransWriter == NULL && DTrans.empty() == false && LongDesc == false && DTrans != "/dev/null")
+	       {
+		  string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
+			SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
+		  string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
+		  TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
+		  TransList.push_back(TransWriter);
+	       }
+	       Itm.TransWriter = TransWriter;
 	       Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 	       Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 	       Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 	       Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 	    }
 
- 	    Itm.GetGeneral(Setup,Block);
+	    Itm.GetGeneral(Setup,Block);
 	    PkgList.push_back(Itm);
 	 }
       }
-      
+
       Top = Top->Next;
-   }      
+   }
+}
+									/*}}}*/
+static void UnloadTree(std::vector<TranslationWriter*> const &Trans)		/*{{{*/
+{
+   for (std::vector<TranslationWriter*>::const_reverse_iterator T = Trans.rbegin(); T != Trans.rend(); ++T)
+      delete *T;
 }
 									/*}}}*/
 // LoadBinDir - Load a 'bindirectory' section from the Generate Config	/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
+static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 {
+   mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
+
    // Process 'bindirectory' type sections
    const Configuration::Item *Top = Setup.Tree("bindirectory");
    for (Top = (Top == 0?0:Top->Child); Top != 0;)
@@ -521,6 +595,7 @@ void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
       Itm.PkgFile = Block.Find("Packages");
       Itm.SrcFile = Block.Find("Sources");
       Itm.BinCacheDB = Block.Find("BinCacheDB");
+      Itm.SrcCacheDB = Block.Find("SrcCacheDB");
       Itm.BinOverride = Block.Find("BinOverride");
       Itm.ExtraOverride = Block.Find("ExtraOverride");
       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
@@ -530,6 +605,7 @@ void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
       Itm.Contents = Block.Find("Contents");
       Itm.ContentsHead = Block.Find("Contents::Header");
+      Itm.Permissions = Block.FindI("FileMode", Permissions);
       
       Itm.GetGeneral(Setup,Block);
       PkgList.push_back(Itm);
@@ -539,17 +615,9 @@ void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 }
 									/*}}}*/
 
-// ShowHelp - Show the help text					/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool ShowHelp(CommandLine &CmdL)
+static bool ShowHelp(CommandLine &)					/*{{{*/
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
-   if (_config->FindB("version") == true)
-      return true;
-
-   cout << 
+   std::cout <<
     _("Usage: apt-ftparchive [options] command\n"
       "Commands: packages binarypath [overridefile [pathprefix]]\n"
       "          sources srcpath [overridefile [pathprefix]]\n"
@@ -588,14 +656,13 @@ bool ShowHelp(CommandLine &CmdL)
       "  --contents  Control contents file generation\n"
       "  -c=?  Read this configuration file\n"
       "  -o=?  Set an arbitrary configuration option") << endl;
-   
    return true;
 }
 									/*}}}*/
 // SimpleGenPackages - Generate a Packages file for a directory tree	/*{{{*/
 // ---------------------------------------------------------------------
 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
-bool SimpleGenPackages(CommandLine &CmdL)
+static bool SimpleGenPackages(CommandLine &CmdL)
 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);
@@ -605,8 +672,9 @@ bool SimpleGenPackages(CommandLine &CmdL)
       Override = CmdL.FileList[2];
    
    // Create a package writer object.
-   PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
-			   Override, "");   
+   PackagesWriter Packages(NULL, NULL, _config->Find("APT::FTPArchive::DB"),
+			   Override, "", _config->Find("APT::FTPArchive::Architecture"),
+			   _config->FindB("APT::FTPArchive::IncludeArchitectureAll", true));
    if (_error->PendingError() == true)
       return false;
    
@@ -617,19 +685,23 @@ bool SimpleGenPackages(CommandLine &CmdL)
    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
       return false;
 
+   // Give some stats if asked for
+   if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
+     c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
+
    return true;
 }
 									/*}}}*/
 // SimpleGenContents - Generate a Contents listing			/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool SimpleGenContents(CommandLine &CmdL)
+static bool SimpleGenContents(CommandLine &CmdL)
 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);
    
    // Create a package writer object.
-   ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"));
+   ContentsWriter Contents(NULL, _config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
    if (_error->PendingError() == true)
       return false;
    
@@ -645,7 +717,7 @@ bool SimpleGenContents(CommandLine &CmdL)
 // SimpleGenSources - Generate a Sources file for a directory tree	/*{{{*/
 // ---------------------------------------------------------------------
 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
-bool SimpleGenSources(CommandLine &CmdL)
+static bool SimpleGenSources(CommandLine &CmdL)
 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);
@@ -662,7 +734,7 @@ bool SimpleGenSources(CommandLine &CmdL)
 			     SOverride.c_str());
        
    // Create a package writer object.
-   SourcesWriter Sources(Override,SOverride);
+   SourcesWriter Sources(NULL, _config->Find("APT::FTPArchive::DB"),Override,SOverride);
    if (_error->PendingError() == true)
       return false;
    
@@ -673,19 +745,23 @@ bool SimpleGenSources(CommandLine &CmdL)
    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
       return false;
 
+   // Give some stats if asked for
+   if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
+     c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl;
+
    return true;
 }
 									/*}}}*/
 // SimpleGenRelease - Generate a Release file for a directory tree	/*{{{*/
 // ---------------------------------------------------------------------
-bool SimpleGenRelease(CommandLine &CmdL)
+static bool SimpleGenRelease(CommandLine &CmdL)
 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);
 
    string Dir = CmdL.FileList[1];
 
-   ReleaseWriter Release("");
+   ReleaseWriter Release(NULL, "");
    Release.DirStrip = Dir;
 
    if (_error->PendingError() == true)
@@ -700,38 +776,20 @@ bool SimpleGenRelease(CommandLine &CmdL)
 }
 
 									/*}}}*/
-// Generate - Full generate, using a config file			/*{{{*/
+// DoGeneratePackagesAndSources - Helper for Generate                   /*{{{*/
 // ---------------------------------------------------------------------
-/* */
-bool Generate(CommandLine &CmdL)
+static bool DoGeneratePackagesAndSources(Configuration &Setup,
+					 vector<PackageMap> &PkgList,
+					 struct CacheDB::Stats &SrcStats,
+					 struct CacheDB::Stats &Stats,
+					 CommandLine &CmdL)
 {
-   struct CacheDB::Stats SrcStats;
-   if (CmdL.FileSize() < 2)
-      return ShowHelp(CmdL);
-
-   struct timeval StartTime;
-   gettimeofday(&StartTime,0);   
-   struct CacheDB::Stats Stats;
-   
-   // Read the configuration file.
-   Configuration Setup;
-   if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
-      return false;
-
-   vector<PackageMap> PkgList;
-   LoadTree(PkgList,Setup);
-   LoadBinDir(PkgList,Setup);
-
-   // Sort by cache DB to improve IO locality.
-   stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
-		
-   // Generate packages
    if (CmdL.FileSize() <= 2)
    {
-      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
+      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 	 if (I->GenPackages(Setup,Stats) == false)
 	    _error->DumpErrors();
-      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
+      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 	 if (I->GenSources(Setup,SrcStats) == false)
 	    _error->DumpErrors();
    }
@@ -740,7 +798,7 @@ bool Generate(CommandLine &CmdL)
       // Make a choice list out of the package list..
       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
       RxChoiceList *End = List;
-      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
+      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
       {
 	 End->UserData = &(*I);
 	 End->Str = I->BaseDir.c_str();
@@ -761,12 +819,12 @@ bool Generate(CommandLine &CmdL)
       _error->DumpErrors();
       
       // Do the generation for Packages
-      for (End = List; End->Str != 0; End++)
+      for (End = List; End->Str != 0; ++End)
       {
 	 if (End->Hit == false)
 	    continue;
 	 
-	 PackageMap *I = (PackageMap *)End->UserData;
+	 PackageMap * const I = static_cast<PackageMap *>(End->UserData);
 	 if (I->PkgDone == true)
 	    continue;
 	 if (I->GenPackages(Setup,Stats) == false)
@@ -774,12 +832,12 @@ bool Generate(CommandLine &CmdL)
       }
       
       // Do the generation for Sources
-      for (End = List; End->Str != 0; End++)
+      for (End = List; End->Str != 0; ++End)
       {
 	 if (End->Hit == false)
 	    continue;
 	 
-	 PackageMap *I = (PackageMap *)End->UserData;
+	 PackageMap * const I = static_cast<PackageMap *>(End->UserData);
 	 if (I->SrcDone == true)
 	    continue;
 	 if (I->GenSources(Setup,SrcStats) == false)
@@ -788,15 +846,21 @@ bool Generate(CommandLine &CmdL)
       
       delete [] List;
    }
-   
-   if (_config->FindB("APT::FTPArchive::Contents",true) == false)
-      return true;
-   
-   c1out << "Done Packages, Starting contents." << endl;
+   return true;
+}
+
+                                                                        /*}}}*/
+// DoGenerateContents - Helper for Generate to generate the Contents    /*{{{*/
+// ---------------------------------------------------------------------
+static bool DoGenerateContents(Configuration &Setup,
+			       vector<PackageMap> &PkgList,
+			       CommandLine &CmdL)
+{
+   c1out << "Packages done, Starting contents." << endl;
 
    // Sort the contents file list by date
    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
-   for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
+   for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
    {
       struct stat A;
       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
@@ -812,10 +876,11 @@ bool Generate(CommandLine &CmdL)
       that describe the debs it indexes. Since the package files contain 
       hashes of the .debs this means they have not changed either so the 
       contents must be up to date. */
-   unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
-   for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
+   unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",
+                                                 std::numeric_limits<unsigned int>::max())*1024;
+   for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
    {
-      // This record is not relevent
+      // This record is not relevant
       if (I->ContentsDone == true ||
 	  I->Contents.empty() == true)
 	 continue;
@@ -848,21 +913,74 @@ bool Generate(CommandLine &CmdL)
 	 break;
       }      
    }
+
+   return true;
+}
+
+									/*}}}*/
+// Generate - Full generate, using a config file			/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+static bool Generate(CommandLine &CmdL)
+{
+   struct CacheDB::Stats SrcStats;
+   if (CmdL.FileSize() < 2)
+      return ShowHelp(CmdL);
+
+   struct timeval StartTime;
+   gettimeofday(&StartTime,0);
+   struct CacheDB::Stats Stats;
    
+   // Read the configuration file.
+   Configuration Setup;
+   if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
+      return false;
+
+   vector<PackageMap> PkgList;
+   std::vector<TranslationWriter*> TransList;
+   LoadTree(PkgList, TransList, Setup);
+   LoadBinDir(PkgList,Setup);
+
+   // Sort by cache DB to improve IO locality.
+   stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
+   stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
+
+   // Generate packages
+   if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false)
+   {
+      if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false)
+      {
+	 UnloadTree(TransList);
+         return false;
+      }
+   } else {
+      c1out << "Skipping Packages/Sources generation" << endl;
+   }
+
+   // do Contents if needed
+   if (_config->FindB("APT::FTPArchive::Contents", true) == true)
+      if (DoGenerateContents(Setup, PkgList, CmdL) == false)
+      {
+	 UnloadTree(TransList);
+	 return false;
+      }
+
    struct timeval NewTime;
-   gettimeofday(&NewTime,0);   
-   double Delta = NewTime.tv_sec - StartTime.tv_sec + 
+   gettimeofday(&NewTime,0);
+   double Delta = NewTime.tv_sec - StartTime.tv_sec +
                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
-   c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages 
+   c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
          << " archives. Took " << TimeToStr((long)Delta) << endl;
-   
+
+   UnloadTree(TransList);
    return true;
 }
-									/*}}}*/
+
+                                                                        /*}}}*/
 // Clean - Clean out the databases					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool Clean(CommandLine &CmdL)
+static bool Clean(CommandLine &CmdL)
 {
    if (CmdL.FileSize() != 2)
       return ShowHelp(CmdL);
@@ -871,92 +989,71 @@ bool Clean(CommandLine &CmdL)
    Configuration Setup;
    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
       return false;
+   // we don't need translation creation here
+   Setup.Set("TreeDefault::Translation", "/dev/null");
 
    vector<PackageMap> PkgList;
-   LoadTree(PkgList,Setup);
+   std::vector<TranslationWriter*> TransList;
+   LoadTree(PkgList, TransList, Setup);
    LoadBinDir(PkgList,Setup);
 
    // Sort by cache DB to improve IO locality.
    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
+   stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 
    string CacheDir = Setup.FindDir("Dir::CacheDir");
    
    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
    {
-      c0out << I->BinCacheDB << endl;
+      if(I->BinCacheDB != "")
+         c0out << I->BinCacheDB << endl;
+      if(I->SrcCacheDB != "")
+         c0out << I->SrcCacheDB << endl;
       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
+      CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));
       if (DB.Clean() == false)
 	 _error->DumpErrors();
-      
+      if (DB_SRC.Clean() == false)
+	 _error->DumpErrors();
+
       string CacheDB = I->BinCacheDB;
-      for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
+      string SrcCacheDB = I->SrcCacheDB;
+      while(I != PkgList.end() && 
+            I->BinCacheDB == CacheDB && 
+            I->SrcCacheDB == SrcCacheDB)
+         ++I;
    }
-   
+
+  
    return true;
 }
 									/*}}}*/
 
-int main(int argc, const char *argv[])
+static std::vector<aptDispatchWithHelp> GetCommands()			/*{{{*/
+{
+   return {
+      {"packages",&SimpleGenPackages, nullptr},
+      {"contents",&SimpleGenContents, nullptr},
+      {"sources",&SimpleGenSources, nullptr},
+      {"release",&SimpleGenRelease, nullptr},
+      {"generate",&Generate, nullptr},
+      {"clean",&Clean, nullptr},
+      {nullptr, nullptr, nullptr}
+   };
+}
+									/*}}}*/
+int main(int argc, const char *argv[])					/*{{{*/
 {
-   setlocale(LC_ALL, "");
-   CommandLine::Args Args[] = {
-      {'h',"help","help",0},
-      {0,"md5","APT::FTPArchive::MD5",0},
-      {'v',"version","version",0},
-      {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
-      {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
-      {'q',"quiet","quiet",CommandLine::IntLevel},
-      {'q',"silent","quiet",CommandLine::IntLevel},
-      {0,"delink","APT::FTPArchive::DeLinkAct",0},
-      {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
-      {0,"contents","APT::FTPArchive::Contents",0},
-      {'c',"config-file",0,CommandLine::ConfigFile},
-      {'o',"option",0,CommandLine::ArbItem},
-      {0,0,0,0}};
-   CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
-                                   {"contents",&SimpleGenContents},
-                                   {"sources",&SimpleGenSources},
-                                   {"release",&SimpleGenRelease},
-                                   {"generate",&Generate},
-                                   {"clean",&Clean},
-      				   {"help",&ShowHelp},
-                                   {0,0}};
+   InitLocale();
 
    // Parse the command line and initialize the package library
-   CommandLine CmdL(Args,_config);
-   if (CmdL.Parse(argc,argv) == false)
-   {
-      _error->DumpErrors();
-      return 100;
-   }
-   
-   // See if the help should be shown
-   if (_config->FindB("help") == true ||
-       _config->FindB("version") == true ||
-       CmdL.FileSize() == 0)
-   {
-      ShowHelp(CmdL);
-      return 0;
-   }
-   
-   // Setup the output streams
-   c0out.rdbuf(clog.rdbuf());
-   c1out.rdbuf(clog.rdbuf());
-   c2out.rdbuf(clog.rdbuf());
+   CommandLine CmdL;
+   auto const Cmds = ParseCommandLine(CmdL, APT_CMD::APT_FTPARCHIVE, &_config, NULL, argc, argv, ShowHelp, &GetCommands);
+
+   _config->CndSet("quiet",0);
    Quiet = _config->FindI("quiet",0);
-   if (Quiet > 0)
-      c0out.rdbuf(devnull.rdbuf());
-   if (Quiet > 1)
-      c1out.rdbuf(devnull.rdbuf());
- 
-   // Match the operation
-   CmdL.DispatchArg(Cmds);
-   
-   if (_error->empty() == false)
-   {
-      bool Errors = _error->PendingError();
-      _error->DumpErrors();
-      return Errors == true?100:0;
-   }
-   return 0;
+   InitOutput(clog.rdbuf());
+
+   return DispatchCommandLine(CmdL, Cmds);
 }
+									/*}}}*/