ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "apt-ftparchive.h"
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <config.h>
  21 #include <apti18n.h>
  22 #include <algorithm>
  23
  24 #include <climits>
  25 #include <sys/time.h>
  26 #include <regex.h>
  27
  28 #include "contents.h"
  29 #include "multicompress.h"
  30 #include "writer.h"
  31                                                                         /*}}}*/
  32
  33 using namespace std;
  34 ostream c0out(0);
  35 ostream c1out(0);
  36 ostream c2out(0);
  37 ofstream devnull("/dev/null");
  38 unsigned Quiet = 0;
  39
  40 // struct PackageMap - List of all package files in the config file     /*{{{*/
  41 // ---------------------------------------------------------------------
  42 /* */
  43 struct PackageMap
  44 {
  45    // General Stuff
  46    string BaseDir;
  47    string InternalPrefix;
  48    string FLFile;
  49    string PkgExt;
  50    string SrcExt;
  51
  52    // Stuff for the Package File
  53    string PkgFile;
  54    string BinCacheDB;
  55    string BinOverride;
  56    string ExtraOverride;
  57
  58    // We generate for this given arch
  59    string Arch;
  60
  61    // Stuff for the Source File
  62    string SrcFile;
  63    string SrcOverride;
  64    string SrcExtraOverride;
  65
  66    // Translation master file
  67    bool LongDesc;
  68    TranslationWriter *TransWriter;
  69
  70    // Contents
  71    string Contents;
  72    string ContentsHead;
  73
  74    // Random things
  75    string Tag;
  76    string PkgCompress;
  77    string CntCompress;
  78    string SrcCompress;
  79    string PathPrefix;
  80    unsigned int DeLinkLimit;
  81    mode_t Permissions;
  82
  83    bool ContentsDone;
  84    bool PkgDone;
  85    bool SrcDone;
  86    time_t ContentsMTime;
  87
  88    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  89    {
  90       inline bool operator() (const PackageMap &x,const PackageMap &y)
  91       {return x.ContentsMTime < y.ContentsMTime;};
  92    };
  93
  94    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  95    {
  96       inline bool operator() (const PackageMap &x,const PackageMap &y)
  97       {return x.BinCacheDB < y.BinCacheDB;};
  98    };
  99
 100    void GetGeneral(Configuration &Setup,Configuration &Block);
 101    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 102    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 103    bool GenContents(Configuration &Setup,
 104                     vector<PackageMap>::iterator Begin,
 105                     vector<PackageMap>::iterator End,
 106                     unsigned long &Left);
 107
 108    PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 109                   ContentsDone(false), PkgDone(false), SrcDone(false),
 110                   ContentsMTime(0) {};
 111 };
 112                                                                         /*}}}*/
 113
 114 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 115 // ---------------------------------------------------------------------
 116 /* */
 117 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 118 {
 119    PathPrefix = Block.Find("PathPrefix");
 120
 121    if (Block.FindB("External-Links",true) == false)
 122       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 123    else
 124       DeLinkLimit = 0;
 125
 126    PkgCompress = Block.Find("Packages::Compress",
 127                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 128    CntCompress = Block.Find("Contents::Compress",
 129                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 130    SrcCompress = Block.Find("Sources::Compress",
 131                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 132
 133    SrcExt = Block.Find("Sources::Extensions",
 134                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 135    PkgExt = Block.Find("Packages::Extensions",
 136                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 137
 138    Permissions = Setup.FindI("Default::FileMode",0644);
 139
 140    if (FLFile.empty() == false)
 141       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 142
 143    if (Contents == " ")
 144       Contents= string();
 145 }
 146                                                                         /*}}}*/
 147 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 148 // ---------------------------------------------------------------------
 149 /* This generates the Package File described by this object. */
 150 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 151 {
 152    if (PkgFile.empty() == true)
 153       return true;
 154
 155    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 156    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 157    string CacheDir = Setup.FindDir("Dir::CacheDir");
 158
 159    struct timeval StartTime;
 160    gettimeofday(&StartTime,0);
 161
 162    PkgDone = true;
 163
 164    // Create a package writer object.
 165    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 166                            flCombine(OverrideDir,BinOverride),
 167                            flCombine(OverrideDir,ExtraOverride),
 168                            Arch);
 169    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 170       return _error->Error(_("Package extension list is too long"));
 171    if (_error->PendingError() == true)
 172       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 173
 174    Packages.PathPrefix = PathPrefix;
 175    Packages.DirStrip = ArchiveDir;
 176    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 177
 178    Packages.TransWriter = TransWriter;
 179    Packages.LongDescription = LongDesc;
 180
 181    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 182    Packages.DeLinkLimit = DeLinkLimit;
 183
 184    // Create a compressor object
 185    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 186                       PkgCompress,Permissions);
 187    Packages.Output = Comp.Input;
 188    if (_error->PendingError() == true)
 189       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 190
 191    c0out << ' ' << BaseDir << ":" << flush;
 192
 193    // Do recursive directory searching
 194    if (FLFile.empty() == true)
 195    {
 196       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 197          return false;
 198    }
 199    else
 200    {
 201       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 202          return false;
 203    }
 204
 205    Packages.Output = 0;      // Just in case
 206
 207    // Finish compressing
 208    unsigned long Size;
 209    if (Comp.Finalize(Size) == false)
 210    {
 211       c0out << endl;
 212       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 213    }
 214
 215    if (Size != 0)
 216       c0out << " New "
 217              << SizeToStr(Size) << "B ";
 218    else
 219       c0out << ' ';
 220
 221    struct timeval NewTime;
 222    gettimeofday(&NewTime,0);
 223    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 224                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 225
 226    c0out << Packages.Stats.Packages << " files " <<
 227 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 228       SizeToStr(Packages.Stats.Bytes) << "B " <<
 229       TimeToStr((long)Delta) << endl;
 230
 231    Stats.Add(Packages.Stats);
 232    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 233
 234    return !_error->PendingError();
 235 }
 236
 237                                                                         /*}}}*/
 238 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 239 // ---------------------------------------------------------------------
 240 /* This generates the Sources File described by this object. */
 241 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 242 {
 243    if (SrcFile.empty() == true)
 244       return true;
 245
 246    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 247    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 248    string CacheDir = Setup.FindDir("Dir::CacheDir");
 249
 250    struct timeval StartTime;
 251    gettimeofday(&StartTime,0);
 252
 253    SrcDone = true;
 254
 255    // Create a package writer object.
 256    SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
 257                          flCombine(OverrideDir,SrcOverride),
 258                          flCombine(OverrideDir,SrcExtraOverride));
 259    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 260       return _error->Error(_("Source extension list is too long"));
 261    if (_error->PendingError() == true)
 262       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 263
 264    Sources.PathPrefix = PathPrefix;
 265    Sources.DirStrip = ArchiveDir;
 266    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 267
 268    Sources.DeLinkLimit = DeLinkLimit;
 269    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 270
 271    // Create a compressor object
 272    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 273                       SrcCompress,Permissions);
 274    Sources.Output = Comp.Input;
 275    if (_error->PendingError() == true)
 276       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 277
 278    c0out << ' ' << BaseDir << ":" << flush;
 279
 280    // Do recursive directory searching
 281    if (FLFile.empty() == true)
 282    {
 283       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 284          return false;
 285    }
 286    else
 287    {
 288       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 289          return false;
 290    }
 291    Sources.Output = 0;      // Just in case
 292
 293    // Finish compressing
 294    unsigned long Size;
 295    if (Comp.Finalize(Size) == false)
 296    {
 297       c0out << endl;
 298       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 299    }
 300
 301    if (Size != 0)
 302       c0out << " New "
 303              << SizeToStr(Size) << "B ";
 304    else
 305       c0out << ' ';
 306
 307    struct timeval NewTime;
 308    gettimeofday(&NewTime,0);
 309    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 310                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 311
 312    c0out << Sources.Stats.Packages << " pkgs in " <<
 313       TimeToStr((long)Delta) << endl;
 314
 315    Stats.Add(Sources.Stats);
 316    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 317
 318    return !_error->PendingError();
 319 }
 320                                                                         /*}}}*/
 321 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 322 // ---------------------------------------------------------------------
 323 /* This generates the contents file partially described by this object.
 324    It searches the given iterator range for other package files that map
 325    into this contents file and includes their data as well when building. */
 326 bool PackageMap::GenContents(Configuration &Setup,
 327                              vector<PackageMap>::iterator Begin,
 328                              vector<PackageMap>::iterator End,
 329                              unsigned long &Left)
 330 {
 331    if (Contents.empty() == true)
 332       return true;
 333
 334    if (Left == 0)
 335       return true;
 336
 337    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 338    string CacheDir = Setup.FindDir("Dir::CacheDir");
 339    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 340
 341    struct timeval StartTime;
 342    gettimeofday(&StartTime,0);
 343
 344    // Create a package writer object.
 345    ContentsWriter Contents("", Arch);
 346    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 347       return _error->Error(_("Package extension list is too long"));
 348    if (_error->PendingError() == true)
 349       return false;
 350
 351    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 352                       CntCompress,Permissions);
 353    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 354    Contents.Output = Comp.Input;
 355    if (_error->PendingError() == true)
 356       return false;
 357
 358    // Write the header out.
 359    if (ContentsHead.empty() == false)
 360    {
 361       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 362       if (_error->PendingError() == true)
 363          return false;
 364
 365       unsigned long Size = Head.Size();
 366       unsigned char Buf[4096];
 367       while (Size != 0)
 368       {
 369          unsigned long ToRead = Size;
 370          if (Size > sizeof(Buf))
 371             ToRead = sizeof(Buf);
 372
 373          if (Head.Read(Buf,ToRead) == false)
 374             return false;
 375
 376          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 377             return _error->Errno("fwrite",_("Error writing header to contents file"));
 378
 379          Size -= ToRead;
 380       }
 381    }
 382
 383    /* Go over all the package file records and parse all the package
 384       files associated with this contents file into one great big honking
 385       memory structure, then dump the sorted version */
 386    c0out << ' ' << this->Contents << ":" << flush;
 387    for (vector<PackageMap>::iterator I = Begin; I != End; I++)
 388    {
 389       if (I->Contents != this->Contents)
 390          continue;
 391
 392       Contents.Prefix = ArchiveDir;
 393       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 394       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 395                             I->PkgCompress);
 396
 397       I->ContentsDone = true;
 398    }
 399
 400    Contents.Finish();
 401
 402    // Finish compressing
 403    unsigned long Size;
 404    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 405    {
 406       c0out << endl;
 407       return _error->Error(_("Error processing contents %s"),
 408                            this->Contents.c_str());
 409    }
 410
 411    if (Size != 0)
 412    {
 413       c0out << " New " << SizeToStr(Size) << "B ";
 414       if (Left > Size)
 415          Left -= Size;
 416       else
 417          Left = 0;
 418    }
 419    else
 420       c0out << ' ';
 421
 422    struct timeval NewTime;
 423    gettimeofday(&NewTime,0);
 424    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 425                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 426
 427    c0out << Contents.Stats.Packages << " files " <<
 428       SizeToStr(Contents.Stats.Bytes) << "B " <<
 429       TimeToStr((long)Delta) << endl;
 430
 431    return true;
 432 }
 433                                                                         /*}}}*/
 434
 435 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 436 // ---------------------------------------------------------------------
 437 /* This populates the PkgList with all the possible permutations of the
 438    section/arch lists. */
 439 void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 440 {
 441    // Load the defaults
 442    string DDir = Setup.Find("TreeDefault::Directory",
 443                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 444    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 445                             "$(DIST)/$(SECTION)/source/");
 446    string DPkg = Setup.Find("TreeDefault::Packages",
 447                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 448    string DTrans = Setup.Find("TreeDefault::Translation",
 449                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 450    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 451                             "$(DIST)/$(SECTION)/");
 452    string DContents = Setup.Find("TreeDefault::Contents",
 453                             "$(DIST)/Contents-$(ARCH)");
 454    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 455    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 456                                "packages-$(ARCH).db");
 457    string DSources = Setup.Find("TreeDefault::Sources",
 458                                 "$(DIST)/$(SECTION)/source/Sources");
 459    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 460    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 461
 462    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 463
 464    bool const LongDescription = Setup.FindB("Default::LongDescription",
 465                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 466    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 467
 468    // Process 'tree' type sections
 469    const Configuration::Item *Top = Setup.Tree("tree");
 470    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 471    {
 472       Configuration Block(Top);
 473       string Dist = Top->Tag;
 474
 475       // Parse the sections
 476       string Tmp = Block.Find("Sections");
 477       const char *Sections = Tmp.c_str();
 478       string Section;
 479       while (ParseQuoteWord(Sections,Section) == true)
 480       {
 481          string Arch;
 482          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 483                                          {"$(SECTION)",&Section},
 484                                          {"$(ARCH)",&Arch},
 485                                          {}};
 486          mode_t const Perms = Block.FindI("FileMode", Permissions);
 487          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 488          TranslationWriter *TransWriter;
 489          if (DTrans.empty() == false && LongDesc == false)
 490          {
 491             string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 492                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 493             string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 494             TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 495          }
 496          else
 497             TransWriter = NULL;
 498
 499          string const Tmp2 = Block.Find("Architectures");
 500          const char *Archs = Tmp2.c_str();
 501          while (ParseQuoteWord(Archs,Arch) == true)
 502          {
 503             PackageMap Itm;
 504             Itm.Permissions = Perms;
 505             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 506             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 507
 508             if (stringcasecmp(Arch,"source") == 0)
 509             {
 510                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 511                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 512                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 513                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 514                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 515                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 516             }
 517             else
 518             {
 519                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 520                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 521                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 522                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 523                Itm.Arch = Arch;
 524                Itm.LongDesc = LongDesc;
 525                if (TransWriter != NULL)
 526                {
 527                   TransWriter->IncreaseRefCounter();
 528                   Itm.TransWriter = TransWriter;
 529                }
 530                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 531                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 532                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 533                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 534             }
 535
 536             Itm.GetGeneral(Setup,Block);
 537             PkgList.push_back(Itm);
 538          }
 539          // we didn't use this TransWriter, so we can release it
 540          if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
 541             delete TransWriter;
 542       }
 543
 544       Top = Top->Next;
 545    }
 546 }
 547                                                                         /*}}}*/
 548 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 549 // ---------------------------------------------------------------------
 550 /* */
 551 void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 552 {
 553    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 554
 555    // Process 'bindirectory' type sections
 556    const Configuration::Item *Top = Setup.Tree("bindirectory");
 557    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 558    {
 559       Configuration Block(Top);
 560
 561       PackageMap Itm;
 562       Itm.PkgFile = Block.Find("Packages");
 563       Itm.SrcFile = Block.Find("Sources");
 564       Itm.BinCacheDB = Block.Find("BinCacheDB");
 565       Itm.BinOverride = Block.Find("BinOverride");
 566       Itm.ExtraOverride = Block.Find("ExtraOverride");
 567       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 568       Itm.SrcOverride = Block.Find("SrcOverride");
 569       Itm.BaseDir = Top->Tag;
 570       Itm.FLFile = Block.Find("FileList");
 571       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 572       Itm.Contents = Block.Find("Contents");
 573       Itm.ContentsHead = Block.Find("Contents::Header");
 574       Itm.Permissions = Block.FindI("FileMode", Permissions);
 575
 576       Itm.GetGeneral(Setup,Block);
 577       PkgList.push_back(Itm);
 578
 579       Top = Top->Next;
 580    }
 581 }
 582                                                                         /*}}}*/
 583
 584 // ShowHelp - Show the help text                                        /*{{{*/
 585 // ---------------------------------------------------------------------
 586 /* */
 587 bool ShowHelp(CommandLine &CmdL)
 588 {
 589    ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,VERSION,
 590             COMMON_ARCH,__DATE__,__TIME__);
 591    if (_config->FindB("version") == true)
 592       return true;
 593
 594    cout <<
 595     _("Usage: apt-ftparchive [options] command\n"
 596       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 597       "          sources srcpath [overridefile [pathprefix]]\n"
 598       "          contents path\n"
 599       "          release path\n"
 600       "          generate config [groups]\n"
 601       "          clean config\n"
 602       "\n"
 603       "apt-ftparchive generates index files for Debian archives. It supports\n"
 604       "many styles of generation from fully automated to functional replacements\n"
 605       "for dpkg-scanpackages and dpkg-scansources\n"
 606       "\n"
 607       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 608       "Package file contains the contents of all the control fields from\n"
 609       "each package as well as the MD5 hash and filesize. An override file\n"
 610       "is supported to force the value of Priority and Section.\n"
 611       "\n"
 612       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 613       "The --source-override option can be used to specify a src override file\n"
 614       "\n"
 615       "The 'packages' and 'sources' command should be run in the root of the\n"
 616       "tree. BinaryPath should point to the base of the recursive search and \n"
 617       "override file should contain the override flags. Pathprefix is\n"
 618       "appended to the filename fields if present. Example usage from the \n"
 619       "Debian archive:\n"
 620       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 621       "               dists/potato/main/binary-i386/Packages\n"
 622       "\n"
 623       "Options:\n"
 624       "  -h    This help text\n"
 625       "  --md5 Control MD5 generation\n"
 626       "  -s=?  Source override file\n"
 627       "  -q    Quiet\n"
 628       "  -d=?  Select the optional caching database\n"
 629       "  --no-delink Enable delinking debug mode\n"
 630       "  --contents  Control contents file generation\n"
 631       "  -c=?  Read this configuration file\n"
 632       "  -o=?  Set an arbitrary configuration option") << endl;
 633
 634    return true;
 635 }
 636                                                                         /*}}}*/
 637 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 638 // ---------------------------------------------------------------------
 639 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 640 bool SimpleGenPackages(CommandLine &CmdL)
 641 {
 642    if (CmdL.FileSize() < 2)
 643       return ShowHelp(CmdL);
 644
 645    string Override;
 646    if (CmdL.FileSize() >= 3)
 647       Override = CmdL.FileList[2];
 648
 649    // Create a package writer object.
 650    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 651                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 652    if (_error->PendingError() == true)
 653       return false;
 654
 655    if (CmdL.FileSize() >= 4)
 656       Packages.PathPrefix = CmdL.FileList[3];
 657
 658    // Do recursive directory searching
 659    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 660       return false;
 661
 662    return true;
 663 }
 664                                                                         /*}}}*/
 665 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 666 // ---------------------------------------------------------------------
 667 /* */
 668 bool SimpleGenContents(CommandLine &CmdL)
 669 {
 670    if (CmdL.FileSize() < 2)
 671       return ShowHelp(CmdL);
 672
 673    // Create a package writer object.
 674    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 675    if (_error->PendingError() == true)
 676       return false;
 677
 678    // Do recursive directory searching
 679    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 680       return false;
 681
 682    Contents.Finish();
 683
 684    return true;
 685 }
 686                                                                         /*}}}*/
 687 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 688 // ---------------------------------------------------------------------
 689 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 690 bool SimpleGenSources(CommandLine &CmdL)
 691 {
 692    if (CmdL.FileSize() < 2)
 693       return ShowHelp(CmdL);
 694
 695    string Override;
 696    if (CmdL.FileSize() >= 3)
 697       Override = CmdL.FileList[2];
 698
 699    string SOverride;
 700    if (Override.empty() == false)
 701       SOverride = Override + ".src";
 702
 703    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 704                              SOverride.c_str());
 705
 706    // Create a package writer object.
 707    SourcesWriter Sources(Override,SOverride);
 708    if (_error->PendingError() == true)
 709       return false;
 710
 711    if (CmdL.FileSize() >= 4)
 712       Sources.PathPrefix = CmdL.FileList[3];
 713
 714    // Do recursive directory searching
 715    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 716       return false;
 717
 718    return true;
 719 }
 720                                                                         /*}}}*/
 721 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 722 // ---------------------------------------------------------------------
 723 bool SimpleGenRelease(CommandLine &CmdL)
 724 {
 725    if (CmdL.FileSize() < 2)
 726       return ShowHelp(CmdL);
 727
 728    string Dir = CmdL.FileList[1];
 729
 730    ReleaseWriter Release("");
 731    Release.DirStrip = Dir;
 732
 733    if (_error->PendingError() == true)
 734       return false;
 735
 736    if (Release.RecursiveScan(Dir) == false)
 737       return false;
 738
 739    Release.Finish();
 740
 741    return true;
 742 }
 743
 744                                                                         /*}}}*/
 745 // Generate - Full generate, using a config file                        /*{{{*/
 746 // ---------------------------------------------------------------------
 747 /* */
 748 bool Generate(CommandLine &CmdL)
 749 {
 750    struct CacheDB::Stats SrcStats;
 751    if (CmdL.FileSize() < 2)
 752       return ShowHelp(CmdL);
 753
 754    struct timeval StartTime;
 755    gettimeofday(&StartTime,0);
 756    struct CacheDB::Stats Stats;
 757
 758    // Read the configuration file.
 759    Configuration Setup;
 760    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 761       return false;
 762
 763    vector<PackageMap> PkgList;
 764    LoadTree(PkgList,Setup);
 765    LoadBinDir(PkgList,Setup);
 766
 767    // Sort by cache DB to improve IO locality.
 768    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 769
 770    // Generate packages
 771    if (CmdL.FileSize() <= 2)
 772    {
 773       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 774          if (I->GenPackages(Setup,Stats) == false)
 775             _error->DumpErrors();
 776       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 777          if (I->GenSources(Setup,SrcStats) == false)
 778             _error->DumpErrors();
 779    }
 780    else
 781    {
 782       // Make a choice list out of the package list..
 783       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 784       RxChoiceList *End = List;
 785       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 786       {
 787          End->UserData = &(*I);
 788          End->Str = I->BaseDir.c_str();
 789          End++;
 790
 791          End->UserData = &(*I);
 792          End->Str = I->Tag.c_str();
 793          End++;
 794       }
 795       End->Str = 0;
 796
 797       // Regex it
 798       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 799       {
 800          delete [] List;
 801          return _error->Error(_("No selections matched"));
 802       }
 803       _error->DumpErrors();
 804
 805       // Do the generation for Packages
 806       for (End = List; End->Str != 0; End++)
 807       {
 808          if (End->Hit == false)
 809             continue;
 810
 811          PackageMap *I = (PackageMap *)End->UserData;
 812          if (I->PkgDone == true)
 813             continue;
 814          if (I->GenPackages(Setup,Stats) == false)
 815             _error->DumpErrors();
 816       }
 817
 818       // Do the generation for Sources
 819       for (End = List; End->Str != 0; End++)
 820       {
 821          if (End->Hit == false)
 822             continue;
 823
 824          PackageMap *I = (PackageMap *)End->UserData;
 825          if (I->SrcDone == true)
 826             continue;
 827          if (I->GenSources(Setup,SrcStats) == false)
 828             _error->DumpErrors();
 829       }
 830
 831       delete [] List;
 832    }
 833
 834    // close the Translation master files
 835    for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); I++)
 836       if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
 837          delete I->TransWriter;
 838
 839    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 840       return true;
 841
 842    c1out << "Packages done, Starting contents." << endl;
 843
 844    // Sort the contents file list by date
 845    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 846    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 847    {
 848       struct stat A;
 849       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 850                                  I->CntCompress,A) == false)
 851          time(&I->ContentsMTime);
 852       else
 853          I->ContentsMTime = A.st_mtime;
 854    }
 855    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 856
 857    /* Now for Contents.. The process here is to do a make-like dependency
 858       check. Each contents file is verified to be newer than the package files
 859       that describe the debs it indexes. Since the package files contain
 860       hashes of the .debs this means they have not changed either so the
 861       contents must be up to date. */
 862    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 863    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 864    {
 865       // This record is not relevent
 866       if (I->ContentsDone == true ||
 867           I->Contents.empty() == true)
 868          continue;
 869
 870       // Do not do everything if the user specified sections.
 871       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 872          continue;
 873
 874       struct stat A,B;
 875       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 876       {
 877          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 878          {
 879             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 880             continue;
 881          }
 882
 883          if (A.st_mtime > B.st_mtime)
 884             continue;
 885       }
 886
 887       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 888                          MaxContentsChange) == false)
 889          _error->DumpErrors();
 890
 891       // Hit the limit?
 892       if (MaxContentsChange == 0)
 893       {
 894          c1out << "Hit contents update byte limit" << endl;
 895          break;
 896       }
 897    }
 898
 899    struct timeval NewTime;
 900    gettimeofday(&NewTime,0);
 901    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 902                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 903    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 904          << " archives. Took " << TimeToStr((long)Delta) << endl;
 905
 906    return true;
 907 }
 908                                                                         /*}}}*/
 909 // Clean - Clean out the databases                                      /*{{{*/
 910 // ---------------------------------------------------------------------
 911 /* */
 912 bool Clean(CommandLine &CmdL)
 913 {
 914    if (CmdL.FileSize() != 2)
 915       return ShowHelp(CmdL);
 916
 917    // Read the configuration file.
 918    Configuration Setup;
 919    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 920       return false;
 921
 922    vector<PackageMap> PkgList;
 923    LoadTree(PkgList,Setup);
 924    LoadBinDir(PkgList,Setup);
 925
 926    // Sort by cache DB to improve IO locality.
 927    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 928
 929    string CacheDir = Setup.FindDir("Dir::CacheDir");
 930
 931    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 932    {
 933       c0out << I->BinCacheDB << endl;
 934       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 935       if (DB.Clean() == false)
 936          _error->DumpErrors();
 937
 938       string CacheDB = I->BinCacheDB;
 939       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
 940    }
 941
 942    return true;
 943 }
 944                                                                         /*}}}*/
 945
 946 int main(int argc, const char *argv[])
 947 {
 948    setlocale(LC_ALL, "");
 949    CommandLine::Args Args[] = {
 950       {'h',"help","help",0},
 951       {0,"md5","APT::FTPArchive::MD5",0},
 952       {'v',"version","version",0},
 953       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 954       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 955       {'q',"quiet","quiet",CommandLine::IntLevel},
 956       {'q',"silent","quiet",CommandLine::IntLevel},
 957       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 958       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 959       {0,"contents","APT::FTPArchive::Contents",0},
 960       {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
 961       {'c',"config-file",0,CommandLine::ConfigFile},
 962       {'o',"option",0,CommandLine::ArbItem},
 963       {0,0,0,0}};
 964    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 965                                    {"contents",&SimpleGenContents},
 966                                    {"sources",&SimpleGenSources},
 967                                    {"release",&SimpleGenRelease},
 968                                    {"generate",&Generate},
 969                                    {"clean",&Clean},
 970                                    {"help",&ShowHelp},
 971                                    {0,0}};
 972
 973    // Parse the command line and initialize the package library
 974    CommandLine CmdL(Args,_config);
 975    if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false)
 976    {
 977       _error->DumpErrors();
 978       return 100;
 979    }
 980
 981    // See if the help should be shown
 982    if (_config->FindB("help") == true ||
 983        _config->FindB("version") == true ||
 984        CmdL.FileSize() == 0)
 985    {
 986       ShowHelp(CmdL);
 987       return 0;
 988    }
 989
 990    // Setup the output streams
 991    c0out.rdbuf(clog.rdbuf());
 992    c1out.rdbuf(clog.rdbuf());
 993    c2out.rdbuf(clog.rdbuf());
 994    Quiet = _config->FindI("quiet",0);
 995    if (Quiet > 0)
 996       c0out.rdbuf(devnull.rdbuf());
 997    if (Quiet > 1)
 998       c1out.rdbuf(devnull.rdbuf());
 999
1000    // Match the operation
1001    CmdL.DispatchArg(Cmds);
1002
1003    if (_error->empty() == false)
1004    {
1005       bool Errors = _error->PendingError();
1006       _error->DumpErrors();
1007       return Errors == true?100:0;
1008    }
1009    return 0;
1010 }