Merge branch 'debian/sid' into bugfix/update-progress-reporting

[apt.git] / ftparchive / apt-ftparchive.cc

diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc
index f3e91d90db330749d354604e8c8339a408ab4493..692f19e25669590522a8e55ffbc87281930ebf23 100644 (file)
--- a/ftparchive/apt-ftparchive.cc
+++ b/ftparchive/apt-ftparchive.cc
@@ -10,23 +10,34 @@
    ##################################################################### */
                                                                        /*}}}*/
 // Include Files                                                       /*{{{*/
    ##################################################################### */
                                                                        /*}}}*/
 // Include Files                                                       /*{{{*/

-#include "apt-ftparchive.h"
-    
+#include <config.h>
+

 #include <apt-pkg/error.h>
 #include <apt-pkg/configuration.h>
 #include <apt-pkg/cmndline.h>
 #include <apt-pkg/strutl.h>
 #include <apt-pkg/error.h>
 #include <apt-pkg/configuration.h>
 #include <apt-pkg/cmndline.h>
 #include <apt-pkg/strutl.h>

-#include <config.h>
-#include <apti18n.h>
-#include <algorithm>
+#include <apt-pkg/init.h>
+#include <apt-pkg/fileutl.h>

 
 

+#include <algorithm>

 #include <climits>
 #include <sys/time.h>
 #include <climits>
 #include <sys/time.h>

-#include <regex.h>
-
-#include "contents.h"
+#include <locale.h>
+#include <stdio.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <functional>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "cachedb.h"
+#include "override.h"
+#include "apt-ftparchive.h"

 #include "multicompress.h"
 #include "multicompress.h"

-#include "writer.h"    
+#include "writer.h"
+
+#include <apti18n.h>

                                                                        /*}}}*/
 
 using namespace std;    
                                                                        /*}}}*/
 
 using namespace std;    


@@ -135,7 +146,7 @@ void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
                       Setup.Find("Default::Packages::Extensions",".deb").c_str());
    
    Permissions = Setup.FindI("Default::FileMode",0644);
                       Setup.Find("Default::Packages::Extensions",".deb").c_str());
    
    Permissions = Setup.FindI("Default::FileMode",0644);

-   
+

    if (FLFile.empty() == false)
       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
    
    if (FLFile.empty() == false)
       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
    


@@ -204,7 +215,7 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
    Packages.Output = 0;      // Just in case
    
    // Finish compressing
    Packages.Output = 0;      // Just in case
    
    // Finish compressing

-   unsigned long Size;
+   unsigned long long Size;

    if (Comp.Finalize(Size) == false)
    {
       c0out << endl;
    if (Comp.Finalize(Size) == false)
    {
       c0out << endl;


@@ -252,7 +263,8 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
    SrcDone = true;
    
    // Create a package writer object.
    SrcDone = true;
    
    // Create a package writer object.

-   SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
+   SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),
+                        flCombine(OverrideDir,BinOverride),

                         flCombine(OverrideDir,SrcOverride),
                         flCombine(OverrideDir,SrcExtraOverride));
    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
                         flCombine(OverrideDir,SrcOverride),
                         flCombine(OverrideDir,SrcExtraOverride));
    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)


@@ -290,7 +302,7 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
    Sources.Output = 0;      // Just in case
    
    // Finish compressing
    Sources.Output = 0;      // Just in case
    
    // Finish compressing

-   unsigned long Size;
+   unsigned long long Size;

    if (Comp.Finalize(Size) == false)
    {
       c0out << endl;
    if (Comp.Finalize(Size) == false)
    {
       c0out << endl;


@@ -361,11 +373,11 @@ bool PackageMap::GenContents(Configuration &Setup,
       if (_error->PendingError() == true)
         return false;
       
       if (_error->PendingError() == true)
         return false;
       

-      unsigned long Size = Head.Size();
+      unsigned long long Size = Head.Size();

       unsigned char Buf[4096];
       while (Size != 0)
       {
       unsigned char Buf[4096];
       while (Size != 0)
       {

-        unsigned long ToRead = Size;
+        unsigned long long ToRead = Size;

         if (Size > sizeof(Buf))
            ToRead = sizeof(Buf);
         
         if (Size > sizeof(Buf))
            ToRead = sizeof(Buf);
         


@@ -383,7 +395,7 @@ bool PackageMap::GenContents(Configuration &Setup,
       files associated with this contents file into one great big honking
       memory structure, then dump the sorted version */
    c0out << ' ' << this->Contents << ":" << flush;
       files associated with this contents file into one great big honking
       memory structure, then dump the sorted version */
    c0out << ' ' << this->Contents << ":" << flush;

-   for (vector<PackageMap>::iterator I = Begin; I != End; I++)
+   for (vector<PackageMap>::iterator I = Begin; I != End; ++I)

    {
       if (I->Contents != this->Contents)
         continue;
    {
       if (I->Contents != this->Contents)
         continue;


@@ -399,7 +411,7 @@ bool PackageMap::GenContents(Configuration &Setup,
    Contents.Finish();
    
    // Finish compressing
    Contents.Finish();
    
    // Finish compressing

-   unsigned long Size;
+   unsigned long long Size;

    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
    {
       c0out << endl;
    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
    {
       c0out << endl;


@@ -435,7 +447,7 @@ bool PackageMap::GenContents(Configuration &Setup,
 // ---------------------------------------------------------------------
 /* This populates the PkgList with all the possible permutations of the
    section/arch lists. */
 // ---------------------------------------------------------------------
 /* This populates the PkgList with all the possible permutations of the
    section/arch lists. */

-void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
+static void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)

 {   
    // Load the defaults
    string DDir = Setup.Find("TreeDefault::Directory",
 {   
    // Load the defaults
    string DDir = Setup.Find("TreeDefault::Directory",


@@ -449,7 +461,7 @@ void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
                            "$(DIST)/$(SECTION)/");
    string DContents = Setup.Find("TreeDefault::Contents",
    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
                            "$(DIST)/$(SECTION)/");
    string DContents = Setup.Find("TreeDefault::Contents",

-                           "$(DIST)/Contents-$(ARCH)");
+                           "$(DIST)/$(SECTION)/Contents-$(ARCH)");

    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
                               "packages-$(ARCH).db");
    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
                               "packages-$(ARCH).db");


@@ -458,8 +470,11 @@ void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
    string DFLFile = Setup.Find("TreeDefault::FileList", "");
    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 
    string DFLFile = Setup.Find("TreeDefault::FileList", "");
    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 

-   bool const LongDescription = Setup.FindB("TreeDefault::LongDescription",
+   mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
+
+   bool const LongDescription = Setup.FindB("Default::LongDescription",

                                        _config->FindB("APT::FTPArchive::LongDescription", true));
                                        _config->FindB("APT::FTPArchive::LongDescription", true));

+   string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();

 
    // Process 'tree' type sections
    const Configuration::Item *Top = Setup.Tree("tree");
 
    // Process 'tree' type sections
    const Configuration::Item *Top = Setup.Tree("tree");


@@ -478,14 +493,16 @@ void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
         struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
                                         {"$(SECTION)",&Section},
                                         {"$(ARCH)",&Arch},
         struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
                                         {"$(SECTION)",&Section},
                                         {"$(ARCH)",&Arch},

-                                        {}};
+                                        {NULL, NULL}};
+        mode_t const Perms = Block.FindI("FileMode", Permissions);

         bool const LongDesc = Block.FindB("LongDescription", LongDescription);
         TranslationWriter *TransWriter;
         if (DTrans.empty() == false && LongDesc == false)
         {
            string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
                        SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
         bool const LongDesc = Block.FindB("LongDescription", LongDescription);
         TranslationWriter *TransWriter;
         if (DTrans.empty() == false && LongDesc == false)
         {
            string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
                        SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));

-           TransWriter = new TranslationWriter(TranslationFile);
+           string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
+           TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);

         }
         else
            TransWriter = NULL;
         }
         else
            TransWriter = NULL;


@@ -495,7 +512,7 @@ void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
         while (ParseQuoteWord(Archs,Arch) == true)
         {
            PackageMap Itm;
         while (ParseQuoteWord(Archs,Arch) == true)
         {
            PackageMap Itm;

-           
+           Itm.Permissions = Perms;

            Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
            Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 
            Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
            Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 


@@ -542,8 +559,10 @@ void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 // LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/
 // ---------------------------------------------------------------------
 /* */
 // LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/
 // ---------------------------------------------------------------------
 /* */

-void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
+static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)

 {
 {

+   mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
+

    // Process 'bindirectory' type sections
    const Configuration::Item *Top = Setup.Tree("bindirectory");
    for (Top = (Top == 0?0:Top->Child); Top != 0;)
    // Process 'bindirectory' type sections
    const Configuration::Item *Top = Setup.Tree("bindirectory");
    for (Top = (Top == 0?0:Top->Child); Top != 0;)


@@ -563,6 +582,7 @@ void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
       Itm.Contents = Block.Find("Contents");
       Itm.ContentsHead = Block.Find("Contents::Header");
       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
       Itm.Contents = Block.Find("Contents");
       Itm.ContentsHead = Block.Find("Contents::Header");

+      Itm.Permissions = Block.FindI("FileMode", Permissions);

       
       Itm.GetGeneral(Setup,Block);
       PkgList.push_back(Itm);
       
       Itm.GetGeneral(Setup,Block);
       PkgList.push_back(Itm);


@@ -575,9 +595,9 @@ void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 // ShowHelp - Show the help text                                       /*{{{*/
 // ---------------------------------------------------------------------
 /* */
 // ShowHelp - Show the help text                                       /*{{{*/
 // ---------------------------------------------------------------------
 /* */

-bool ShowHelp(CommandLine &CmdL)
+static bool ShowHelp(CommandLine &)

 {
 {

-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,VERSION,
+   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,

            COMMON_ARCH,__DATE__,__TIME__);
    if (_config->FindB("version") == true)
       return true;
            COMMON_ARCH,__DATE__,__TIME__);
    if (_config->FindB("version") == true)
       return true;


@@ -628,7 +648,7 @@ bool ShowHelp(CommandLine &CmdL)
 // SimpleGenPackages - Generate a Packages file for a directory tree   /*{{{*/
 // ---------------------------------------------------------------------
 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 // SimpleGenPackages - Generate a Packages file for a directory tree   /*{{{*/
 // ---------------------------------------------------------------------
 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */

-bool SimpleGenPackages(CommandLine &CmdL)
+static bool SimpleGenPackages(CommandLine &CmdL)

 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);
 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);


@@ -656,7 +676,7 @@ bool SimpleGenPackages(CommandLine &CmdL)
 // SimpleGenContents - Generate a Contents listing                     /*{{{*/
 // ---------------------------------------------------------------------
 /* */
 // SimpleGenContents - Generate a Contents listing                     /*{{{*/
 // ---------------------------------------------------------------------
 /* */

-bool SimpleGenContents(CommandLine &CmdL)
+static bool SimpleGenContents(CommandLine &CmdL)

 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);
 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);


@@ -678,7 +698,7 @@ bool SimpleGenContents(CommandLine &CmdL)
 // SimpleGenSources - Generate a Sources file for a directory tree     /*{{{*/
 // ---------------------------------------------------------------------
 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 // SimpleGenSources - Generate a Sources file for a directory tree     /*{{{*/
 // ---------------------------------------------------------------------
 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */

-bool SimpleGenSources(CommandLine &CmdL)
+static bool SimpleGenSources(CommandLine &CmdL)

 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);
 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);


@@ -695,7 +715,7 @@ bool SimpleGenSources(CommandLine &CmdL)
                             SOverride.c_str());
        
    // Create a package writer object.
                             SOverride.c_str());
        
    // Create a package writer object.

-   SourcesWriter Sources(Override,SOverride);
+   SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),Override,SOverride);

    if (_error->PendingError() == true)
       return false;
    
    if (_error->PendingError() == true)
       return false;
    


@@ -711,7 +731,7 @@ bool SimpleGenSources(CommandLine &CmdL)
                                                                        /*}}}*/
 // SimpleGenRelease - Generate a Release file for a directory tree     /*{{{*/
 // ---------------------------------------------------------------------
                                                                        /*}}}*/
 // SimpleGenRelease - Generate a Release file for a directory tree     /*{{{*/
 // ---------------------------------------------------------------------

-bool SimpleGenRelease(CommandLine &CmdL)
+static bool SimpleGenRelease(CommandLine &CmdL)

 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);
 {
    if (CmdL.FileSize() < 2)
       return ShowHelp(CmdL);


@@ -736,7 +756,7 @@ bool SimpleGenRelease(CommandLine &CmdL)
 // Generate - Full generate, using a config file                       /*{{{*/
 // ---------------------------------------------------------------------
 /* */
 // Generate - Full generate, using a config file                       /*{{{*/
 // ---------------------------------------------------------------------
 /* */

-bool Generate(CommandLine &CmdL)
+static bool Generate(CommandLine &CmdL)

 {
    struct CacheDB::Stats SrcStats;
    if (CmdL.FileSize() < 2)
 {
    struct CacheDB::Stats SrcStats;
    if (CmdL.FileSize() < 2)


@@ -761,10 +781,10 @@ bool Generate(CommandLine &CmdL)
    // Generate packages
    if (CmdL.FileSize() <= 2)
    {
    // Generate packages
    if (CmdL.FileSize() <= 2)
    {

-      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
+      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)

         if (I->GenPackages(Setup,Stats) == false)
            _error->DumpErrors();
         if (I->GenPackages(Setup,Stats) == false)
            _error->DumpErrors();

-      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
+      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)

         if (I->GenSources(Setup,SrcStats) == false)
            _error->DumpErrors();
    }
         if (I->GenSources(Setup,SrcStats) == false)
            _error->DumpErrors();
    }


@@ -773,7 +793,7 @@ bool Generate(CommandLine &CmdL)
       // Make a choice list out of the package list..
       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
       RxChoiceList *End = List;
       // Make a choice list out of the package list..
       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
       RxChoiceList *End = List;

-      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
+      for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)

       {
         End->UserData = &(*I);
         End->Str = I->BaseDir.c_str();
       {
         End->UserData = &(*I);
         End->Str = I->BaseDir.c_str();


@@ -823,7 +843,7 @@ bool Generate(CommandLine &CmdL)
    }
 
    // close the Translation master files
    }
 
    // close the Translation master files

-   for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
+   for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I)

       if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
         delete I->TransWriter;
 
       if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
         delete I->TransWriter;
 


@@ -834,7 +854,7 @@ bool Generate(CommandLine &CmdL)
 
    // Sort the contents file list by date
    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 
    // Sort the contents file list by date
    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");

-   for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
+   for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)

    {
       struct stat A;
       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
    {
       struct stat A;
       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),


@@ -851,9 +871,9 @@ bool Generate(CommandLine &CmdL)
       hashes of the .debs this means they have not changed either so the 
       contents must be up to date. */
    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
       hashes of the .debs this means they have not changed either so the 
       contents must be up to date. */
    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;

-   for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
+   for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)

    {
    {

-      // This record is not relevent
+      // This record is not relevant

       if (I->ContentsDone == true ||
          I->Contents.empty() == true)
         continue;
       if (I->ContentsDone == true ||
          I->Contents.empty() == true)
         continue;


@@ -900,7 +920,7 @@ bool Generate(CommandLine &CmdL)
 // Clean - Clean out the databases                                     /*{{{*/
 // ---------------------------------------------------------------------
 /* */
 // Clean - Clean out the databases                                     /*{{{*/
 // ---------------------------------------------------------------------
 /* */

-bool Clean(CommandLine &CmdL)
+static bool Clean(CommandLine &CmdL)

 {
    if (CmdL.FileSize() != 2)
       return ShowHelp(CmdL);
 {
    if (CmdL.FileSize() != 2)
       return ShowHelp(CmdL);


@@ -927,7 +947,7 @@ bool Clean(CommandLine &CmdL)
         _error->DumpErrors();
       
       string CacheDB = I->BinCacheDB;
         _error->DumpErrors();
       
       string CacheDB = I->BinCacheDB;

-      for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
+      for (; I != PkgList.end() && I->BinCacheDB == CacheDB; ++I);

    }
    
    return true;
    }
    
    return true;


@@ -940,6 +960,8 @@ int main(int argc, const char *argv[])
    CommandLine::Args Args[] = {
       {'h',"help","help",0},
       {0,"md5","APT::FTPArchive::MD5",0},
    CommandLine::Args Args[] = {
       {'h',"help","help",0},
       {0,"md5","APT::FTPArchive::MD5",0},

+      {0,"sha1","APT::FTPArchive::SHA1",0},
+      {0,"sha256","APT::FTPArchive::SHA256",0},

       {'v',"version","version",0},
       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
       {'v',"version","version",0},
       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},


@@ -963,7 +985,7 @@ int main(int argc, const char *argv[])
 
    // Parse the command line and initialize the package library
    CommandLine CmdL(Args,_config);
 
    // Parse the command line and initialize the package library
    CommandLine CmdL(Args,_config);

-   if (CmdL.Parse(argc,argv) == false)
+   if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false)

    {
       _error->DumpErrors();
       return 100;
    {
       _error->DumpErrors();
       return 100;