Merge remote-tracking branch 'mvo/feature/apt-update-info' into debian/sid

[apt.git] / ftparchive / apt-ftparchive.cc

diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc
index 60fe740b52a00c5d6977d0802cc785cb26664604..ba71ee22582646182081e35e568f11c30224782a 100644 (file)
--- a/ftparchive/apt-ftparchive.cc
+++ b/ftparchive/apt-ftparchive.cc
@@ -1,35 +1,43 @@
 // -*- mode: cpp; mode: fold -*-
 // Description                                                         /*{{{*/
 // -*- mode: cpp; mode: fold -*-
 // Description                                                         /*{{{*/

-// $Id: apt-ftparchive.cc,v 1.7 2003/07/18 14:12:07 mdz Exp $
+// $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $

 /* ######################################################################
 
 /* ######################################################################
 

-   apt-scanpackages - Efficient work-alike for dpkg-scanpackages
+   apt-ftparchive - Efficient work-alike for dpkg-scanpackages

 
    Let contents be disabled from the conf
    
    ##################################################################### */
                                                                        /*}}}*/
 // Include Files                                                       /*{{{*/
 
    Let contents be disabled from the conf
    
    ##################################################################### */
                                                                        /*}}}*/
 // Include Files                                                       /*{{{*/

-#ifdef __GNUG__
-#pragma implementation "apt-ftparchive.h"
-#endif
+#include <config.h>

 
 

-#include "apt-ftparchive.h"
-    

 #include <apt-pkg/error.h>
 #include <apt-pkg/configuration.h>
 #include <apt-pkg/cmndline.h>
 #include <apt-pkg/strutl.h>
 #include <apt-pkg/error.h>
 #include <apt-pkg/configuration.h>
 #include <apt-pkg/cmndline.h>
 #include <apt-pkg/strutl.h>

-#include <config.h>
-#include <apti18n.h>
-#include <algorithm>
+#include <apt-pkg/init.h>
+#include <apt-pkg/fileutl.h>

 
 

+#include <algorithm>
+#include <climits>

 #include <sys/time.h>
 #include <sys/time.h>

-#include <regex.h>
+#include <locale.h>
+#include <stdio.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <functional>
+#include <iostream>
+#include <string>
+#include <vector>

 
 

-#include "contents.h"
+#include "cachedb.h"
+#include "override.h"
+#include "apt-ftparchive.h"

 #include "multicompress.h"
 #include "multicompress.h"

-#include "writer.h"    
+#include "writer.h"
+
+#include <apti18n.h>

                                                                        /*}}}*/
 
 using namespace std;    
                                                                        /*}}}*/
 
 using namespace std;    


@@ -54,14 +62,22 @@ struct PackageMap
    // Stuff for the Package File
    string PkgFile;
    string BinCacheDB;
    // Stuff for the Package File
    string PkgFile;
    string BinCacheDB;

+   string SrcCacheDB;

    string BinOverride;
    string ExtraOverride;
    string BinOverride;
    string ExtraOverride;

+
+   // We generate for this given arch
+   string Arch;

    
    // Stuff for the Source File
    string SrcFile;
    string SrcOverride;
    string SrcExtraOverride;
 
    
    // Stuff for the Source File
    string SrcFile;
    string SrcOverride;
    string SrcExtraOverride;
 

+   // Translation master file
+   bool LongDesc;
+   TranslationWriter *TransWriter;
+

    // Contents 
    string Contents;
    string ContentsHead;
    // Contents 
    string Contents;
    string ContentsHead;


@@ -91,6 +107,12 @@ struct PackageMap
       inline bool operator() (const PackageMap &x,const PackageMap &y)
       {return x.BinCacheDB < y.BinCacheDB;};
    };  
       inline bool operator() (const PackageMap &x,const PackageMap &y)
       {return x.BinCacheDB < y.BinCacheDB;};
    };  

+
+   struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
+   {
+      inline bool operator() (const PackageMap &x,const PackageMap &y)
+      {return x.SrcCacheDB < y.SrcCacheDB;};
+   };

    
    void GetGeneral(Configuration &Setup,Configuration &Block);
    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
    
    void GetGeneral(Configuration &Setup,Configuration &Block);
    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);


@@ -100,8 +122,9 @@ struct PackageMap
                    vector<PackageMap>::iterator End,
                    unsigned long &Left);
    
                    vector<PackageMap>::iterator End,
                    unsigned long &Left);
    

-   PackageMap() : DeLinkLimit(0), Permissions(1), ContentsDone(false), 
-        PkgDone(false), SrcDone(false), ContentsMTime(0) {};
+   PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
+                 ContentsDone(false), PkgDone(false), SrcDone(false),
+                 ContentsMTime(0) {};

 };
                                                                        /*}}}*/
 
 };
                                                                        /*}}}*/
 


@@ -130,7 +153,7 @@ void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
                       Setup.Find("Default::Packages::Extensions",".deb").c_str());
    
    Permissions = Setup.FindI("Default::FileMode",0644);
                       Setup.Find("Default::Packages::Extensions",".deb").c_str());
    
    Permissions = Setup.FindI("Default::FileMode",0644);

-   
+

    if (FLFile.empty() == false)
       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
    
    if (FLFile.empty() == false)
       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
    


@@ -158,16 +181,20 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
    // Create a package writer object.
    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
                           flCombine(OverrideDir,BinOverride),
    // Create a package writer object.
    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
                           flCombine(OverrideDir,BinOverride),

-                          flCombine(OverrideDir,ExtraOverride));
+                          flCombine(OverrideDir,ExtraOverride),
+                          Arch);

    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
       return _error->Error(_("Package extension list is too long"));
    if (_error->PendingError() == true)
    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
       return _error->Error(_("Package extension list is too long"));
    if (_error->PendingError() == true)

-      return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
+      return _error->Error(_("Error processing directory %s"),BaseDir.c_str());

    
    Packages.PathPrefix = PathPrefix;
    Packages.DirStrip = ArchiveDir;
    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 
    
    Packages.PathPrefix = PathPrefix;
    Packages.DirStrip = ArchiveDir;
    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 

+   Packages.TransWriter = TransWriter;
+   Packages.LongDescription = LongDesc;
+

    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
    Packages.DeLinkLimit = DeLinkLimit;
 
    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
    Packages.DeLinkLimit = DeLinkLimit;
 


@@ -176,7 +203,7 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
                      PkgCompress,Permissions);
    Packages.Output = Comp.Input;
    if (_error->PendingError() == true)
                      PkgCompress,Permissions);
    Packages.Output = Comp.Input;
    if (_error->PendingError() == true)

-      return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
+      return _error->Error(_("Error processing directory %s"),BaseDir.c_str());

    
    c0out << ' ' << BaseDir << ":" << flush;
    
    
    c0out << ' ' << BaseDir << ":" << flush;
    


@@ -195,11 +222,11 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
    Packages.Output = 0;      // Just in case
    
    // Finish compressing
    Packages.Output = 0;      // Just in case
    
    // Finish compressing

-   unsigned long Size;
+   unsigned long long Size;

    if (Comp.Finalize(Size) == false)
    {
       c0out << endl;
    if (Comp.Finalize(Size) == false)
    {
       c0out << endl;

-      return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
+      return _error->Error(_("Error processing directory %s"),BaseDir.c_str());

    }
    
    if (Size != 0)
    }
    
    if (Size != 0)


@@ -212,17 +239,21 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
    gettimeofday(&NewTime,0);
    double Delta = NewTime.tv_sec - StartTime.tv_sec + 
                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
    gettimeofday(&NewTime,0);
    double Delta = NewTime.tv_sec - StartTime.tv_sec + 
                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;

-   
+

    c0out << Packages.Stats.Packages << " files " <<
 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
       SizeToStr(Packages.Stats.Bytes) << "B " <<
       TimeToStr((long)Delta) << endl;
    c0out << Packages.Stats.Packages << " files " <<
 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
       SizeToStr(Packages.Stats.Bytes) << "B " <<
       TimeToStr((long)Delta) << endl;

+
+   if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
+     c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;

    
    Stats.Add(Packages.Stats);
    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
    
    return !_error->PendingError();
 }
    
    Stats.Add(Packages.Stats);
    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
    
    return !_error->PendingError();
 }

+

                                                                        /*}}}*/
 // PackageMap::GenSources - Actually generate a Source file            /*{{{*/
 // ---------------------------------------------------------------------
                                                                        /*}}}*/
 // PackageMap::GenSources - Actually generate a Source file            /*{{{*/
 // ---------------------------------------------------------------------


@@ -242,13 +273,14 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
    SrcDone = true;
    
    // Create a package writer object.
    SrcDone = true;
    
    // Create a package writer object.

-   SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
+   SourcesWriter Sources(flCombine(CacheDir, SrcCacheDB),
+                        flCombine(OverrideDir,BinOverride),

                         flCombine(OverrideDir,SrcOverride),
                         flCombine(OverrideDir,SrcExtraOverride));
    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
       return _error->Error(_("Source extension list is too long"));
    if (_error->PendingError() == true)
                         flCombine(OverrideDir,SrcOverride),
                         flCombine(OverrideDir,SrcExtraOverride));
    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
       return _error->Error(_("Source extension list is too long"));
    if (_error->PendingError() == true)

-      return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
+      return _error->Error(_("Error processing directory %s"),BaseDir.c_str());

    
    Sources.PathPrefix = PathPrefix;
    Sources.DirStrip = ArchiveDir;
    
    Sources.PathPrefix = PathPrefix;
    Sources.DirStrip = ArchiveDir;


@@ -262,7 +294,7 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
                      SrcCompress,Permissions);
    Sources.Output = Comp.Input;
    if (_error->PendingError() == true)
                      SrcCompress,Permissions);
    Sources.Output = Comp.Input;
    if (_error->PendingError() == true)

-      return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
+      return _error->Error(_("Error processing directory %s"),BaseDir.c_str());

 
    c0out << ' ' << BaseDir << ":" << flush;
    
 
    c0out << ' ' << BaseDir << ":" << flush;
    


@@ -280,11 +312,11 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
    Sources.Output = 0;      // Just in case
    
    // Finish compressing
    Sources.Output = 0;      // Just in case
    
    // Finish compressing

-   unsigned long Size;
+   unsigned long long Size;

    if (Comp.Finalize(Size) == false)
    {
       c0out << endl;
    if (Comp.Finalize(Size) == false)
    {
       c0out << endl;

-      return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
+      return _error->Error(_("Error processing directory %s"),BaseDir.c_str());

    }
       
    if (Size != 0)
    }
       
    if (Size != 0)


@@ -301,6 +333,9 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
    c0out << Sources.Stats.Packages << " pkgs in " <<
       TimeToStr((long)Delta) << endl;
 
    c0out << Sources.Stats.Packages << " pkgs in " <<
       TimeToStr((long)Delta) << endl;
 

+   if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
+     c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
+

    Stats.Add(Sources.Stats);
    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
    
    Stats.Add(Sources.Stats);
    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
    


@@ -331,7 +366,7 @@ bool PackageMap::GenContents(Configuration &Setup,
    gettimeofday(&StartTime,0);   
    
    // Create a package writer object.
    gettimeofday(&StartTime,0);   
    
    // Create a package writer object.

-   ContentsWriter Contents("");
+   ContentsWriter Contents("", Arch);

    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
       return _error->Error(_("Package extension list is too long"));
    if (_error->PendingError() == true)
    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
       return _error->Error(_("Package extension list is too long"));
    if (_error->PendingError() == true)


@@ -351,11 +386,11 @@ bool PackageMap::GenContents(Configuration &Setup,
       if (_error->PendingError() == true)
         return false;
       
       if (_error->PendingError() == true)
         return false;
       

-      unsigned long Size = Head.Size();
+      unsigned long long Size = Head.Size();

       unsigned char Buf[4096];
       while (Size != 0)
       {
       unsigned char Buf[4096];
       while (Size != 0)
       {

-        unsigned long ToRead = Size;
+        unsigned long long ToRead = Size;

         if (Size > sizeof(Buf))
            ToRead = sizeof(Buf);
         
         if (Size > sizeof(Buf))
            ToRead = sizeof(Buf);
         


@@ -373,7 +408,7 @@ bool PackageMap::GenContents(Configuration &Setup,
       files associated with this contents file into one great big honking
       memory structure, then dump the sorted version */
    c0out << ' ' << this->Contents << ":" << flush;
       files associated with this contents file into one great big honking
       memory structure, then dump the sorted version */
    c0out << ' ' << this->Contents << ":" << flush;

-   for (vector<PackageMap>::iterator I = Begin; I != End; I++)
+   for (vector<PackageMap>::iterator I = Begin; I != End; ++I)

    {
       if (I->Contents != this->Contents)
         continue;
    {
       if (I->Contents != this->Contents)
         continue;


@@ -389,11 +424,11 @@ bool PackageMap::GenContents(Configuration &Setup,
    Contents.Finish();
    
    // Finish compressing
    Contents.Finish();
    
    // Finish compressing

-   unsigned long Size;
+   unsigned long long Size;

    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
    {
       c0out << endl;
    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
    {
       c0out << endl;

-      return _error->Error(_("Error Processing Contents %s"),
+      return _error->Error(_("Error processing contents %s"),

                           this->Contents.c_str());
    }
    
                           this->Contents.c_str());
    }
    


@@ -413,6 +448,9 @@ bool PackageMap::GenContents(Configuration &Setup,
    double Delta = NewTime.tv_sec - StartTime.tv_sec + 
                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
    
    double Delta = NewTime.tv_sec - StartTime.tv_sec + 
                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
    

+   if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
+     c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
+

    c0out << Contents.Stats.Packages << " files " <<
       SizeToStr(Contents.Stats.Bytes) << "B " <<
       TimeToStr((long)Delta) << endl;
    c0out << Contents.Stats.Packages << " files " <<
       SizeToStr(Contents.Stats.Bytes) << "B " <<
       TimeToStr((long)Delta) << endl;


@@ -425,7 +463,7 @@ bool PackageMap::GenContents(Configuration &Setup,
 // ---------------------------------------------------------------------
 /* This populates the PkgList with all the possible permutations of the
    section/arch lists. */
 // ---------------------------------------------------------------------
 /* This populates the PkgList with all the possible permutations of the
    section/arch lists. */

-void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
+static void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)

 {   
    // Load the defaults
    string DDir = Setup.Find("TreeDefault::Directory",
 {   
    // Load the defaults
    string DDir = Setup.Find("TreeDefault::Directory",


@@ -434,18 +472,28 @@ void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
                            "$(DIST)/$(SECTION)/source/");
    string DPkg = Setup.Find("TreeDefault::Packages",
                            "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
                            "$(DIST)/$(SECTION)/source/");
    string DPkg = Setup.Find("TreeDefault::Packages",
                            "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");

+   string DTrans = Setup.Find("TreeDefault::Translation",
+                           "$(DIST)/$(SECTION)/i18n/Translation-en");

    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
                            "$(DIST)/$(SECTION)/");
    string DContents = Setup.Find("TreeDefault::Contents",
    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
                            "$(DIST)/$(SECTION)/");
    string DContents = Setup.Find("TreeDefault::Contents",

-                           "$(DIST)/Contents-$(ARCH)");
+                           "$(DIST)/$(SECTION)/Contents-$(ARCH)");

    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
                               "packages-$(ARCH).db");
    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
                               "packages-$(ARCH).db");

+   string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
+                              "sources-$(SECTION).db");

    string DSources = Setup.Find("TreeDefault::Sources",
                                "$(DIST)/$(SECTION)/source/Sources");
    string DFLFile = Setup.Find("TreeDefault::FileList", "");
    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 
    string DSources = Setup.Find("TreeDefault::Sources",
                                "$(DIST)/$(SECTION)/source/Sources");
    string DFLFile = Setup.Find("TreeDefault::FileList", "");
    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 

+   mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
+
+   bool const LongDescription = Setup.FindB("Default::LongDescription",
+                                       _config->FindB("APT::FTPArchive::LongDescription", true));
+   string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
+

    // Process 'tree' type sections
    const Configuration::Item *Top = Setup.Tree("tree");
    for (Top = (Top == 0?0:Top->Child); Top != 0;)
    // Process 'tree' type sections
    const Configuration::Item *Top = Setup.Tree("tree");
    for (Top = (Top == 0?0:Top->Child); Top != 0;)


@@ -459,17 +507,30 @@ void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
       string Section;
       while (ParseQuoteWord(Sections,Section) == true)
       {
       string Section;
       while (ParseQuoteWord(Sections,Section) == true)
       {

-        string Tmp2 = Block.Find("Architectures");

         string Arch;
         string Arch;

+        struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
+                                        {"$(SECTION)",&Section},
+                                        {"$(ARCH)",&Arch},
+                                        {NULL, NULL}};
+        mode_t const Perms = Block.FindI("FileMode", Permissions);
+        bool const LongDesc = Block.FindB("LongDescription", LongDescription);
+        TranslationWriter *TransWriter;
+        if (DTrans.empty() == false && LongDesc == false)
+        {
+           string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
+                       SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
+           string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
+           TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
+        }
+        else
+           TransWriter = NULL;
+
+        string const Tmp2 = Block.Find("Architectures");

         const char *Archs = Tmp2.c_str();
         while (ParseQuoteWord(Archs,Arch) == true)
         {
         const char *Archs = Tmp2.c_str();
         while (ParseQuoteWord(Archs,Arch) == true)
         {

-           struct SubstVar Vars[] = {{"$(DIST)",&Dist},
-                                     {"$(SECTION)",&Section},
-                                     {"$(ARCH)",&Arch},
-                                     {}};

            PackageMap Itm;
            PackageMap Itm;

-           
+           Itm.Permissions = Perms;

            Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
            Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 
            Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
            Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 


@@ -481,6 +542,7 @@ void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
               Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
               Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
               Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
               Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
               Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
               Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);

+              Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);

            }
            else
            {
            }
            else
            {


@@ -488,6 +550,13 @@ void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
               Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
               Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
               Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
               Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
               Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
               Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);

+              Itm.Arch = Arch;
+              Itm.LongDesc = LongDesc;
+              if (TransWriter != NULL)
+              {
+                 TransWriter->IncreaseRefCounter();
+                 Itm.TransWriter = TransWriter;
+              }

               Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
               Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
               Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
               Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
               Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
               Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);


@@ -497,6 +566,9 @@ void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
            Itm.GetGeneral(Setup,Block);
            PkgList.push_back(Itm);
         }
            Itm.GetGeneral(Setup,Block);
            PkgList.push_back(Itm);
         }

+        // we didn't use this TransWriter, so we can release it
+        if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
+           delete TransWriter;

       }
       
       Top = Top->Next;
       }
       
       Top = Top->Next;


@@ -506,8 +578,10 @@ void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 // LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/
 // ---------------------------------------------------------------------
 /* */
 // LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/
 // ---------------------------------------------------------------------
 /* */

-void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
+static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)

 {
 {

+   mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
+

    // Process 'bindirectory' type sections
    const Configuration::Item *Top = Setup.Tree("bindirectory");
    for (Top = (Top == 0?0:Top->Child); Top != 0;)
    // Process 'bindirectory' type sections
    const Configuration::Item *Top = Setup.Tree("bindirectory");
    for (Top = (Top == 0?0:Top->Child); Top != 0;)


@@ -518,6 +592,7 @@ void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
       Itm.PkgFile = Block.Find("Packages");
       Itm.SrcFile = Block.Find("Sources");
       Itm.BinCacheDB = Block.Find("BinCacheDB");
       Itm.PkgFile = Block.Find("Packages");
       Itm.SrcFile = Block.Find("Sources");
       Itm.BinCacheDB = Block.Find("BinCacheDB");

+      Itm.SrcCacheDB = Block.Find("SrcCacheDB");

       Itm.BinOverride = Block.Find("BinOverride");
       Itm.ExtraOverride = Block.Find("ExtraOverride");
       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
       Itm.BinOverride = Block.Find("BinOverride");
       Itm.ExtraOverride = Block.Find("ExtraOverride");
       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");


@@ -527,6 +602,7 @@ void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
       Itm.Contents = Block.Find("Contents");
       Itm.ContentsHead = Block.Find("Contents::Header");
       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
       Itm.Contents = Block.Find("Contents");
       Itm.ContentsHead = Block.Find("Contents::Header");

+      Itm.Permissions = Block.FindI("FileMode", Permissions);

       
       Itm.GetGeneral(Setup,Block);
       PkgList.push_back(Itm);
       
       Itm.GetGeneral(Setup,Block);
       PkgList.push_back(Itm);


@@ -539,10 +615,10 @@ void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 // ShowHelp - Show the help text                                       /*{{{*/
 // ---------------------------------------------------------------------
 /* */
 // ShowHelp - Show the help text                                       /*{{{*/
 // ---------------------------------------------------------------------
 /* */

-bool ShowHelp(CommandLine &CmdL)
+static bool ShowHelp(CommandLine &)

 {
 {

-   ioprintf(cout,_("%s %s for %s %s compiled on %s %s\n"),PACKAGE,VERSION,
-           COMMON_OS,COMMON_CPU,__DATE__,__TIME__);
+   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
+           COMMON_ARCH,__DATE__,__TIME__);

    if (_config->FindB("version") == true)
       return true;
 
    if (_config->FindB("version") == true)
       return true;
 


@@ -551,6 +627,7 @@ bool ShowHelp(CommandLine &CmdL)
       "Commands: packages binarypath [overridefile [pathprefix]]\n"
       "          sources srcpath [overridefile [pathprefix]]\n"
       "          contents path\n"
       "Commands: packages binarypath [overridefile [pathprefix]]\n"
       "          sources srcpath [overridefile [pathprefix]]\n"
       "          contents path\n"

+      "          release path\n"

       "          generate config [groups]\n"
       "          clean config\n"
       "\n"
       "          generate config [groups]\n"
       "          clean config\n"
       "\n"


@@ -568,9 +645,9 @@ bool ShowHelp(CommandLine &CmdL)
       "\n"
       "The 'packages' and 'sources' command should be run in the root of the\n"
       "tree. BinaryPath should point to the base of the recursive search and \n"
       "\n"
       "The 'packages' and 'sources' command should be run in the root of the\n"
       "tree. BinaryPath should point to the base of the recursive search and \n"

-      "override file should contian the override flags. Pathprefix is\n"
+      "override file should contain the override flags. Pathprefix is\n"

       "appended to the filename fields if present. Example usage from the \n"
       "appended to the filename fields if present. Example usage from the \n"

-      "debian archive:\n"
+      "Debian archive:\n"

       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
       "               dists/potato/main/binary-i386/Packages\n"
       "\n"
       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
       "               dists/potato/main/binary-i386/Packages\n"
       "\n"


@@ -583,7 +660,7 @@ bool ShowHelp(CommandLine &CmdL)
       "  --no-delink Enable delinking debug mode\n"
       "  --contents  Control contents file generation\n"
       "  -c=?  Read this configuration file\n"
       "  --no-delink Enable delinking debug mode\n"
       "  --contents  Control contents file generation\n"
       "  -c=?  Read this configuration file\n"

-      "  -o=?  Set an arbitary configuration option") << endl;
+      "  -o=?  Set an arbitrary configuration option") << endl;

    
    return true;
 }
    
    return true;
 }


@@ -591,7 +668,7 @@ bool ShowHelp(CommandLine &CmdL)
 // SimpleGenPackages - Generate a Packages file for a directory tree   /*{{{*/
 // ---------------------------------------------------------------------
 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 // SimpleGenPackages - Generate a Packages file for a directory tree   /*{{{*/
 // ---------------------------------------------------------------------
 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */

-bool SimpleGenPackages(CommandLine &CmdL)
+static bool SimpleGenPackages(CommandLine &CmdL)

 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);
 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);


@@ -602,7 +679,7 @@ bool SimpleGenPackages(CommandLine &CmdL)
    
    // Create a package writer object.
    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
    
    // Create a package writer object.
    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),

-                          Override, "");   
+                          Override, "", _config->Find("APT::FTPArchive::Architecture"));

    if (_error->PendingError() == true)
       return false;
    
    if (_error->PendingError() == true)
       return false;
    


@@ -613,19 +690,23 @@ bool SimpleGenPackages(CommandLine &CmdL)
    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
       return false;
 
    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
       return false;
 

+   // Give some stats if asked for
+   if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
+     c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
+

    return true;
 }
                                                                        /*}}}*/
 // SimpleGenContents - Generate a Contents listing                     /*{{{*/
 // ---------------------------------------------------------------------
 /* */
    return true;
 }
                                                                        /*}}}*/
 // SimpleGenContents - Generate a Contents listing                     /*{{{*/
 // ---------------------------------------------------------------------
 /* */

-bool SimpleGenContents(CommandLine &CmdL)
+static bool SimpleGenContents(CommandLine &CmdL)

 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);
    
    // Create a package writer object.
 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);
    
    // Create a package writer object.

-   ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"));
+   ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));

    if (_error->PendingError() == true)
       return false;
    
    if (_error->PendingError() == true)
       return false;
    


@@ -641,7 +722,7 @@ bool SimpleGenContents(CommandLine &CmdL)
 // SimpleGenSources - Generate a Sources file for a directory tree     /*{{{*/
 // ---------------------------------------------------------------------
 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 // SimpleGenSources - Generate a Sources file for a directory tree     /*{{{*/
 // ---------------------------------------------------------------------
 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */

-bool SimpleGenSources(CommandLine &CmdL)
+static bool SimpleGenSources(CommandLine &CmdL)

 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);
 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);


@@ -658,7 +739,7 @@ bool SimpleGenSources(CommandLine &CmdL)
                             SOverride.c_str());
        
    // Create a package writer object.
                             SOverride.c_str());
        
    // Create a package writer object.

-   SourcesWriter Sources(Override,SOverride);
+   SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),Override,SOverride);

    if (_error->PendingError() == true)
       return false;
    
    if (_error->PendingError() == true)
       return false;
    


@@ -669,13 +750,41 @@ bool SimpleGenSources(CommandLine &CmdL)
    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
       return false;
 
    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
       return false;
 

+   // Give some stats if asked for
+   if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
+     c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl;
+
+   return true;
+}
+                                                                       /*}}}*/
+// SimpleGenRelease - Generate a Release file for a directory tree     /*{{{*/
+// ---------------------------------------------------------------------
+static bool SimpleGenRelease(CommandLine &CmdL)
+{
+   if (CmdL.FileSize() < 2)
+      return ShowHelp(CmdL);
+
+   string Dir = CmdL.FileList[1];
+
+   ReleaseWriter Release("");
+   Release.DirStrip = Dir;
+
+   if (_error->PendingError() == true)
+      return false;
+
+   if (Release.RecursiveScan(Dir) == false)
+      return false;
+
+   Release.Finish();
+

    return true;
 }
    return true;
 }

+

                                                                        /*}}}*/
 // Generate - Full generate, using a config file                       /*{{{*/
 // ---------------------------------------------------------------------
 /* */
                                                                        /*}}}*/
 // Generate - Full generate, using a config file                       /*{{{*/
 // ---------------------------------------------------------------------
 /* */

-bool Generate(CommandLine &CmdL)
+static bool Generate(CommandLine &CmdL)

 {
    struct CacheDB::Stats SrcStats;
    if (CmdL.FileSize() < 2)
 {
    struct CacheDB::Stats SrcStats;
    if (CmdL.FileSize() < 2)


@@ -696,14 +805,15 @@ bool Generate(CommandLine &CmdL)
 
    // Sort by cache DB to improve IO locality.
    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 
    // Sort by cache DB to improve IO locality.
    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());

+   stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());

                
    // Generate packages
    if (CmdL.FileSize() <= 2)
    {
                
    // Generate packages
    if (CmdL.FileSize() <= 2)
    {

-      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
+      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)

         if (I->GenPackages(Setup,Stats) == false)
            _error->DumpErrors();
         if (I->GenPackages(Setup,Stats) == false)
            _error->DumpErrors();

-      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
+      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)

         if (I->GenSources(Setup,SrcStats) == false)
            _error->DumpErrors();
    }
         if (I->GenSources(Setup,SrcStats) == false)
            _error->DumpErrors();
    }


@@ -712,7 +822,7 @@ bool Generate(CommandLine &CmdL)
       // Make a choice list out of the package list..
       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
       RxChoiceList *End = List;
       // Make a choice list out of the package list..
       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
       RxChoiceList *End = List;

-      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
+      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)

       {
         End->UserData = &(*I);
         End->Str = I->BaseDir.c_str();
       {
         End->UserData = &(*I);
         End->Str = I->BaseDir.c_str();


@@ -760,15 +870,20 @@ bool Generate(CommandLine &CmdL)
       
       delete [] List;
    }
       
       delete [] List;
    }

-   
+
+   // close the Translation master files
+   for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I)
+      if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
+        delete I->TransWriter;
+

    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
       return true;
    
    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
       return true;
    

-   c1out << "Done Packages, Starting contents." << endl;
+   c1out << "Packages done, Starting contents." << endl;

 
    // Sort the contents file list by date
    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 
    // Sort the contents file list by date
    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");

-   for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
+   for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)

    {
       struct stat A;
       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
    {
       struct stat A;
       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),


@@ -785,9 +900,9 @@ bool Generate(CommandLine &CmdL)
       hashes of the .debs this means they have not changed either so the 
       contents must be up to date. */
    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
       hashes of the .debs this means they have not changed either so the 
       contents must be up to date. */
    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;

-   for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
+   for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)

    {
    {

-      // This record is not relevent
+      // This record is not relevant

       if (I->ContentsDone == true ||
          I->Contents.empty() == true)
         continue;
       if (I->ContentsDone == true ||
          I->Contents.empty() == true)
         continue;


@@ -834,7 +949,7 @@ bool Generate(CommandLine &CmdL)
 // Clean - Clean out the databases                                     /*{{{*/
 // ---------------------------------------------------------------------
 /* */
 // Clean - Clean out the databases                                     /*{{{*/
 // ---------------------------------------------------------------------
 /* */

-bool Clean(CommandLine &CmdL)
+static bool Clean(CommandLine &CmdL)

 {
    if (CmdL.FileSize() != 2)
       return ShowHelp(CmdL);
 {
    if (CmdL.FileSize() != 2)
       return ShowHelp(CmdL);


@@ -850,29 +965,45 @@ bool Clean(CommandLine &CmdL)
 
    // Sort by cache DB to improve IO locality.
    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 
    // Sort by cache DB to improve IO locality.
    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());

+   stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());

 
    string CacheDir = Setup.FindDir("Dir::CacheDir");
    
    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
    {
 
    string CacheDir = Setup.FindDir("Dir::CacheDir");
    
    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
    {

-      c0out << I->BinCacheDB << endl;
+      if(I->BinCacheDB != "")
+         c0out << I->BinCacheDB << endl;
+      if(I->SrcCacheDB != "")
+         c0out << I->SrcCacheDB << endl;

       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));

+      CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));

       if (DB.Clean() == false)
         _error->DumpErrors();
       if (DB.Clean() == false)
         _error->DumpErrors();

+      if (DB_SRC.Clean() == false)
+        _error->DumpErrors();

       
       string CacheDB = I->BinCacheDB;
       
       string CacheDB = I->BinCacheDB;

-      for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
+      string SrcCacheDB = I->SrcCacheDB;
+      while(I != PkgList.end() && 
+            I->BinCacheDB == CacheDB && 
+            I->SrcCacheDB == SrcCacheDB)
+         ++I;
+

    }
    }

-   
+
+  

    return true;
 }
                                                                        /*}}}*/
 
 int main(int argc, const char *argv[])
 {
    return true;
 }
                                                                        /*}}}*/
 
 int main(int argc, const char *argv[])
 {

+   setlocale(LC_ALL, "");

    CommandLine::Args Args[] = {
       {'h',"help","help",0},
       {0,"md5","APT::FTPArchive::MD5",0},
    CommandLine::Args Args[] = {
       {'h',"help","help",0},
       {0,"md5","APT::FTPArchive::MD5",0},

+      {0,"sha1","APT::FTPArchive::SHA1",0},
+      {0,"sha256","APT::FTPArchive::SHA256",0},

       {'v',"version","version",0},
       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
       {'v',"version","version",0},
       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},


@@ -881,12 +1012,14 @@ int main(int argc, const char *argv[])
       {0,"delink","APT::FTPArchive::DeLinkAct",0},
       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
       {0,"contents","APT::FTPArchive::Contents",0},
       {0,"delink","APT::FTPArchive::DeLinkAct",0},
       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
       {0,"contents","APT::FTPArchive::Contents",0},

+      {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},

       {'c',"config-file",0,CommandLine::ConfigFile},
       {'o',"option",0,CommandLine::ArbItem},
       {0,0,0,0}};
    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
                                    {"contents",&SimpleGenContents},
                                    {"sources",&SimpleGenSources},
       {'c',"config-file",0,CommandLine::ConfigFile},
       {'o',"option",0,CommandLine::ArbItem},
       {0,0,0,0}};
    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
                                    {"contents",&SimpleGenContents},
                                    {"sources",&SimpleGenSources},

+                                   {"release",&SimpleGenRelease},

                                    {"generate",&Generate},
                                    {"clean",&Clean},
                                   {"help",&ShowHelp},
                                    {"generate",&Generate},
                                    {"clean",&Clean},
                                   {"help",&ShowHelp},


@@ -894,7 +1027,7 @@ int main(int argc, const char *argv[])
 
    // Parse the command line and initialize the package library
    CommandLine CmdL(Args,_config);
 
    // Parse the command line and initialize the package library
    CommandLine CmdL(Args,_config);

-   if (CmdL.Parse(argc,argv) == false)
+   if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false)

    {
       _error->DumpErrors();
       return 100;
    {
       _error->DumpErrors();
       return 100;