ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <algorithm>
  21
  22 #include <climits>
  23 #include <sys/time.h>
  24 #include <regex.h>
  25
  26 #include "apt-ftparchive.h"
  27 #include "contents.h"
  28 #include "multicompress.h"
  29 #include "writer.h"
  30
  31 #include <apti18n.h>
  32                                                                         /*}}}*/
  33
  34 using namespace std;
  35 ostream c0out(0);
  36 ostream c1out(0);
  37 ostream c2out(0);
  38 ofstream devnull("/dev/null");
  39 unsigned Quiet = 0;
  40
  41 // struct PackageMap - List of all package files in the config file     /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* */
  44 struct PackageMap
  45 {
  46    // General Stuff
  47    string BaseDir;
  48    string InternalPrefix;
  49    string FLFile;
  50    string PkgExt;
  51    string SrcExt;
  52
  53    // Stuff for the Package File
  54    string PkgFile;
  55    string BinCacheDB;
  56    string BinOverride;
  57    string ExtraOverride;
  58
  59    // We generate for this given arch
  60    string Arch;
  61
  62    // Stuff for the Source File
  63    string SrcFile;
  64    string SrcOverride;
  65    string SrcExtraOverride;
  66
  67    // Translation master file
  68    bool LongDesc;
  69    TranslationWriter *TransWriter;
  70
  71    // Contents
  72    string Contents;
  73    string ContentsHead;
  74
  75    // Random things
  76    string Tag;
  77    string PkgCompress;
  78    string CntCompress;
  79    string SrcCompress;
  80    string PathPrefix;
  81    unsigned int DeLinkLimit;
  82    mode_t Permissions;
  83
  84    bool ContentsDone;
  85    bool PkgDone;
  86    bool SrcDone;
  87    time_t ContentsMTime;
  88
  89    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  90    {
  91       inline bool operator() (const PackageMap &x,const PackageMap &y)
  92       {return x.ContentsMTime < y.ContentsMTime;};
  93    };
  94
  95    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  96    {
  97       inline bool operator() (const PackageMap &x,const PackageMap &y)
  98       {return x.BinCacheDB < y.BinCacheDB;};
  99    };
 100
 101    void GetGeneral(Configuration &Setup,Configuration &Block);
 102    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 103    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 104    bool GenContents(Configuration &Setup,
 105                     vector<PackageMap>::iterator Begin,
 106                     vector<PackageMap>::iterator End,
 107                     unsigned long &Left);
 108
 109    PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 110                   ContentsDone(false), PkgDone(false), SrcDone(false),
 111                   ContentsMTime(0) {};
 112 };
 113                                                                         /*}}}*/
 114
 115 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 116 // ---------------------------------------------------------------------
 117 /* */
 118 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 119 {
 120    PathPrefix = Block.Find("PathPrefix");
 121
 122    if (Block.FindB("External-Links",true) == false)
 123       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 124    else
 125       DeLinkLimit = 0;
 126
 127    PkgCompress = Block.Find("Packages::Compress",
 128                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 129    CntCompress = Block.Find("Contents::Compress",
 130                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 131    SrcCompress = Block.Find("Sources::Compress",
 132                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 133
 134    SrcExt = Block.Find("Sources::Extensions",
 135                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 136    PkgExt = Block.Find("Packages::Extensions",
 137                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 138
 139    Permissions = Setup.FindI("Default::FileMode",0644);
 140
 141    if (FLFile.empty() == false)
 142       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 143
 144    if (Contents == " ")
 145       Contents= string();
 146 }
 147                                                                         /*}}}*/
 148 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 149 // ---------------------------------------------------------------------
 150 /* This generates the Package File described by this object. */
 151 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 152 {
 153    if (PkgFile.empty() == true)
 154       return true;
 155
 156    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 157    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 158    string CacheDir = Setup.FindDir("Dir::CacheDir");
 159
 160    struct timeval StartTime;
 161    gettimeofday(&StartTime,0);
 162
 163    PkgDone = true;
 164
 165    // Create a package writer object.
 166    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 167                            flCombine(OverrideDir,BinOverride),
 168                            flCombine(OverrideDir,ExtraOverride),
 169                            Arch);
 170    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 171       return _error->Error(_("Package extension list is too long"));
 172    if (_error->PendingError() == true)
 173       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 174
 175    Packages.PathPrefix = PathPrefix;
 176    Packages.DirStrip = ArchiveDir;
 177    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 178
 179    Packages.TransWriter = TransWriter;
 180    Packages.LongDescription = LongDesc;
 181
 182    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 183    Packages.DeLinkLimit = DeLinkLimit;
 184
 185    // Create a compressor object
 186    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 187                       PkgCompress,Permissions);
 188    Packages.Output = Comp.Input;
 189    if (_error->PendingError() == true)
 190       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 191
 192    c0out << ' ' << BaseDir << ":" << flush;
 193
 194    // Do recursive directory searching
 195    if (FLFile.empty() == true)
 196    {
 197       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 198          return false;
 199    }
 200    else
 201    {
 202       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 203          return false;
 204    }
 205
 206    Packages.Output = 0;      // Just in case
 207
 208    // Finish compressing
 209    unsigned long Size;
 210    if (Comp.Finalize(Size) == false)
 211    {
 212       c0out << endl;
 213       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 214    }
 215
 216    if (Size != 0)
 217       c0out << " New "
 218              << SizeToStr(Size) << "B ";
 219    else
 220       c0out << ' ';
 221
 222    struct timeval NewTime;
 223    gettimeofday(&NewTime,0);
 224    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 225                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 226
 227    c0out << Packages.Stats.Packages << " files " <<
 228 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 229       SizeToStr(Packages.Stats.Bytes) << "B " <<
 230       TimeToStr((long)Delta) << endl;
 231
 232    Stats.Add(Packages.Stats);
 233    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 234
 235    return !_error->PendingError();
 236 }
 237
 238                                                                         /*}}}*/
 239 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 240 // ---------------------------------------------------------------------
 241 /* This generates the Sources File described by this object. */
 242 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 243 {
 244    if (SrcFile.empty() == true)
 245       return true;
 246
 247    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 248    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 249    string CacheDir = Setup.FindDir("Dir::CacheDir");
 250
 251    struct timeval StartTime;
 252    gettimeofday(&StartTime,0);
 253
 254    SrcDone = true;
 255
 256    // Create a package writer object.
 257    SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
 258                          flCombine(OverrideDir,SrcOverride),
 259                          flCombine(OverrideDir,SrcExtraOverride));
 260    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 261       return _error->Error(_("Source extension list is too long"));
 262    if (_error->PendingError() == true)
 263       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 264
 265    Sources.PathPrefix = PathPrefix;
 266    Sources.DirStrip = ArchiveDir;
 267    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 268
 269    Sources.DeLinkLimit = DeLinkLimit;
 270    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 271
 272    // Create a compressor object
 273    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 274                       SrcCompress,Permissions);
 275    Sources.Output = Comp.Input;
 276    if (_error->PendingError() == true)
 277       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 278
 279    c0out << ' ' << BaseDir << ":" << flush;
 280
 281    // Do recursive directory searching
 282    if (FLFile.empty() == true)
 283    {
 284       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 285          return false;
 286    }
 287    else
 288    {
 289       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 290          return false;
 291    }
 292    Sources.Output = 0;      // Just in case
 293
 294    // Finish compressing
 295    unsigned long Size;
 296    if (Comp.Finalize(Size) == false)
 297    {
 298       c0out << endl;
 299       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 300    }
 301
 302    if (Size != 0)
 303       c0out << " New "
 304              << SizeToStr(Size) << "B ";
 305    else
 306       c0out << ' ';
 307
 308    struct timeval NewTime;
 309    gettimeofday(&NewTime,0);
 310    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 311                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 312
 313    c0out << Sources.Stats.Packages << " pkgs in " <<
 314       TimeToStr((long)Delta) << endl;
 315
 316    Stats.Add(Sources.Stats);
 317    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 318
 319    return !_error->PendingError();
 320 }
 321                                                                         /*}}}*/
 322 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 323 // ---------------------------------------------------------------------
 324 /* This generates the contents file partially described by this object.
 325    It searches the given iterator range for other package files that map
 326    into this contents file and includes their data as well when building. */
 327 bool PackageMap::GenContents(Configuration &Setup,
 328                              vector<PackageMap>::iterator Begin,
 329                              vector<PackageMap>::iterator End,
 330                              unsigned long &Left)
 331 {
 332    if (Contents.empty() == true)
 333       return true;
 334
 335    if (Left == 0)
 336       return true;
 337
 338    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 339    string CacheDir = Setup.FindDir("Dir::CacheDir");
 340    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 341
 342    struct timeval StartTime;
 343    gettimeofday(&StartTime,0);
 344
 345    // Create a package writer object.
 346    ContentsWriter Contents("", Arch);
 347    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 348       return _error->Error(_("Package extension list is too long"));
 349    if (_error->PendingError() == true)
 350       return false;
 351
 352    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 353                       CntCompress,Permissions);
 354    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 355    Contents.Output = Comp.Input;
 356    if (_error->PendingError() == true)
 357       return false;
 358
 359    // Write the header out.
 360    if (ContentsHead.empty() == false)
 361    {
 362       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 363       if (_error->PendingError() == true)
 364          return false;
 365
 366       unsigned long Size = Head.Size();
 367       unsigned char Buf[4096];
 368       while (Size != 0)
 369       {
 370          unsigned long ToRead = Size;
 371          if (Size > sizeof(Buf))
 372             ToRead = sizeof(Buf);
 373
 374          if (Head.Read(Buf,ToRead) == false)
 375             return false;
 376
 377          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 378             return _error->Errno("fwrite",_("Error writing header to contents file"));
 379
 380          Size -= ToRead;
 381       }
 382    }
 383
 384    /* Go over all the package file records and parse all the package
 385       files associated with this contents file into one great big honking
 386       memory structure, then dump the sorted version */
 387    c0out << ' ' << this->Contents << ":" << flush;
 388    for (vector<PackageMap>::iterator I = Begin; I != End; I++)
 389    {
 390       if (I->Contents != this->Contents)
 391          continue;
 392
 393       Contents.Prefix = ArchiveDir;
 394       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 395       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 396                             I->PkgCompress);
 397
 398       I->ContentsDone = true;
 399    }
 400
 401    Contents.Finish();
 402
 403    // Finish compressing
 404    unsigned long Size;
 405    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 406    {
 407       c0out << endl;
 408       return _error->Error(_("Error processing contents %s"),
 409                            this->Contents.c_str());
 410    }
 411
 412    if (Size != 0)
 413    {
 414       c0out << " New " << SizeToStr(Size) << "B ";
 415       if (Left > Size)
 416          Left -= Size;
 417       else
 418          Left = 0;
 419    }
 420    else
 421       c0out << ' ';
 422
 423    struct timeval NewTime;
 424    gettimeofday(&NewTime,0);
 425    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 426                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 427
 428    c0out << Contents.Stats.Packages << " files " <<
 429       SizeToStr(Contents.Stats.Bytes) << "B " <<
 430       TimeToStr((long)Delta) << endl;
 431
 432    return true;
 433 }
 434                                                                         /*}}}*/
 435
 436 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 437 // ---------------------------------------------------------------------
 438 /* This populates the PkgList with all the possible permutations of the
 439    section/arch lists. */
 440 void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 441 {
 442    // Load the defaults
 443    string DDir = Setup.Find("TreeDefault::Directory",
 444                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 445    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 446                             "$(DIST)/$(SECTION)/source/");
 447    string DPkg = Setup.Find("TreeDefault::Packages",
 448                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 449    string DTrans = Setup.Find("TreeDefault::Translation",
 450                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 451    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 452                             "$(DIST)/$(SECTION)/");
 453    string DContents = Setup.Find("TreeDefault::Contents",
 454                             "$(DIST)/Contents-$(ARCH)");
 455    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 456    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 457                                "packages-$(ARCH).db");
 458    string DSources = Setup.Find("TreeDefault::Sources",
 459                                 "$(DIST)/$(SECTION)/source/Sources");
 460    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 461    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 462
 463    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 464
 465    bool const LongDescription = Setup.FindB("Default::LongDescription",
 466                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 467    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 468
 469    // Process 'tree' type sections
 470    const Configuration::Item *Top = Setup.Tree("tree");
 471    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 472    {
 473       Configuration Block(Top);
 474       string Dist = Top->Tag;
 475
 476       // Parse the sections
 477       string Tmp = Block.Find("Sections");
 478       const char *Sections = Tmp.c_str();
 479       string Section;
 480       while (ParseQuoteWord(Sections,Section) == true)
 481       {
 482          string Arch;
 483          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 484                                          {"$(SECTION)",&Section},
 485                                          {"$(ARCH)",&Arch},
 486                                          {}};
 487          mode_t const Perms = Block.FindI("FileMode", Permissions);
 488          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 489          TranslationWriter *TransWriter;
 490          if (DTrans.empty() == false && LongDesc == false)
 491          {
 492             string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 493                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 494             string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 495             TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 496          }
 497          else
 498             TransWriter = NULL;
 499
 500          string const Tmp2 = Block.Find("Architectures");
 501          const char *Archs = Tmp2.c_str();
 502          while (ParseQuoteWord(Archs,Arch) == true)
 503          {
 504             PackageMap Itm;
 505             Itm.Permissions = Perms;
 506             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 507             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 508
 509             if (stringcasecmp(Arch,"source") == 0)
 510             {
 511                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 512                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 513                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 514                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 515                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 516                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 517             }
 518             else
 519             {
 520                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 521                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 522                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 523                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 524                Itm.Arch = Arch;
 525                Itm.LongDesc = LongDesc;
 526                if (TransWriter != NULL)
 527                {
 528                   TransWriter->IncreaseRefCounter();
 529                   Itm.TransWriter = TransWriter;
 530                }
 531                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 532                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 533                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 534                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 535             }
 536
 537             Itm.GetGeneral(Setup,Block);
 538             PkgList.push_back(Itm);
 539          }
 540          // we didn't use this TransWriter, so we can release it
 541          if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
 542             delete TransWriter;
 543       }
 544
 545       Top = Top->Next;
 546    }
 547 }
 548                                                                         /*}}}*/
 549 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 550 // ---------------------------------------------------------------------
 551 /* */
 552 void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 553 {
 554    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 555
 556    // Process 'bindirectory' type sections
 557    const Configuration::Item *Top = Setup.Tree("bindirectory");
 558    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 559    {
 560       Configuration Block(Top);
 561
 562       PackageMap Itm;
 563       Itm.PkgFile = Block.Find("Packages");
 564       Itm.SrcFile = Block.Find("Sources");
 565       Itm.BinCacheDB = Block.Find("BinCacheDB");
 566       Itm.BinOverride = Block.Find("BinOverride");
 567       Itm.ExtraOverride = Block.Find("ExtraOverride");
 568       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 569       Itm.SrcOverride = Block.Find("SrcOverride");
 570       Itm.BaseDir = Top->Tag;
 571       Itm.FLFile = Block.Find("FileList");
 572       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 573       Itm.Contents = Block.Find("Contents");
 574       Itm.ContentsHead = Block.Find("Contents::Header");
 575       Itm.Permissions = Block.FindI("FileMode", Permissions);
 576
 577       Itm.GetGeneral(Setup,Block);
 578       PkgList.push_back(Itm);
 579
 580       Top = Top->Next;
 581    }
 582 }
 583                                                                         /*}}}*/
 584
 585 // ShowHelp - Show the help text                                        /*{{{*/
 586 // ---------------------------------------------------------------------
 587 /* */
 588 bool ShowHelp(CommandLine &CmdL)
 589 {
 590    ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,VERSION,
 591             COMMON_ARCH,__DATE__,__TIME__);
 592    if (_config->FindB("version") == true)
 593       return true;
 594
 595    cout <<
 596     _("Usage: apt-ftparchive [options] command\n"
 597       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 598       "          sources srcpath [overridefile [pathprefix]]\n"
 599       "          contents path\n"
 600       "          release path\n"
 601       "          generate config [groups]\n"
 602       "          clean config\n"
 603       "\n"
 604       "apt-ftparchive generates index files for Debian archives. It supports\n"
 605       "many styles of generation from fully automated to functional replacements\n"
 606       "for dpkg-scanpackages and dpkg-scansources\n"
 607       "\n"
 608       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 609       "Package file contains the contents of all the control fields from\n"
 610       "each package as well as the MD5 hash and filesize. An override file\n"
 611       "is supported to force the value of Priority and Section.\n"
 612       "\n"
 613       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 614       "The --source-override option can be used to specify a src override file\n"
 615       "\n"
 616       "The 'packages' and 'sources' command should be run in the root of the\n"
 617       "tree. BinaryPath should point to the base of the recursive search and \n"
 618       "override file should contain the override flags. Pathprefix is\n"
 619       "appended to the filename fields if present. Example usage from the \n"
 620       "Debian archive:\n"
 621       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 622       "               dists/potato/main/binary-i386/Packages\n"
 623       "\n"
 624       "Options:\n"
 625       "  -h    This help text\n"
 626       "  --md5 Control MD5 generation\n"
 627       "  -s=?  Source override file\n"
 628       "  -q    Quiet\n"
 629       "  -d=?  Select the optional caching database\n"
 630       "  --no-delink Enable delinking debug mode\n"
 631       "  --contents  Control contents file generation\n"
 632       "  -c=?  Read this configuration file\n"
 633       "  -o=?  Set an arbitrary configuration option") << endl;
 634
 635    return true;
 636 }
 637                                                                         /*}}}*/
 638 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 639 // ---------------------------------------------------------------------
 640 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 641 bool SimpleGenPackages(CommandLine &CmdL)
 642 {
 643    if (CmdL.FileSize() < 2)
 644       return ShowHelp(CmdL);
 645
 646    string Override;
 647    if (CmdL.FileSize() >= 3)
 648       Override = CmdL.FileList[2];
 649
 650    // Create a package writer object.
 651    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 652                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 653    if (_error->PendingError() == true)
 654       return false;
 655
 656    if (CmdL.FileSize() >= 4)
 657       Packages.PathPrefix = CmdL.FileList[3];
 658
 659    // Do recursive directory searching
 660    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 661       return false;
 662
 663    return true;
 664 }
 665                                                                         /*}}}*/
 666 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 667 // ---------------------------------------------------------------------
 668 /* */
 669 bool SimpleGenContents(CommandLine &CmdL)
 670 {
 671    if (CmdL.FileSize() < 2)
 672       return ShowHelp(CmdL);
 673
 674    // Create a package writer object.
 675    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 676    if (_error->PendingError() == true)
 677       return false;
 678
 679    // Do recursive directory searching
 680    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 681       return false;
 682
 683    Contents.Finish();
 684
 685    return true;
 686 }
 687                                                                         /*}}}*/
 688 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 689 // ---------------------------------------------------------------------
 690 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 691 bool SimpleGenSources(CommandLine &CmdL)
 692 {
 693    if (CmdL.FileSize() < 2)
 694       return ShowHelp(CmdL);
 695
 696    string Override;
 697    if (CmdL.FileSize() >= 3)
 698       Override = CmdL.FileList[2];
 699
 700    string SOverride;
 701    if (Override.empty() == false)
 702       SOverride = Override + ".src";
 703
 704    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 705                              SOverride.c_str());
 706
 707    // Create a package writer object.
 708    SourcesWriter Sources(Override,SOverride);
 709    if (_error->PendingError() == true)
 710       return false;
 711
 712    if (CmdL.FileSize() >= 4)
 713       Sources.PathPrefix = CmdL.FileList[3];
 714
 715    // Do recursive directory searching
 716    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 717       return false;
 718
 719    return true;
 720 }
 721                                                                         /*}}}*/
 722 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 723 // ---------------------------------------------------------------------
 724 bool SimpleGenRelease(CommandLine &CmdL)
 725 {
 726    if (CmdL.FileSize() < 2)
 727       return ShowHelp(CmdL);
 728
 729    string Dir = CmdL.FileList[1];
 730
 731    ReleaseWriter Release("");
 732    Release.DirStrip = Dir;
 733
 734    if (_error->PendingError() == true)
 735       return false;
 736
 737    if (Release.RecursiveScan(Dir) == false)
 738       return false;
 739
 740    Release.Finish();
 741
 742    return true;
 743 }
 744
 745                                                                         /*}}}*/
 746 // Generate - Full generate, using a config file                        /*{{{*/
 747 // ---------------------------------------------------------------------
 748 /* */
 749 bool Generate(CommandLine &CmdL)
 750 {
 751    struct CacheDB::Stats SrcStats;
 752    if (CmdL.FileSize() < 2)
 753       return ShowHelp(CmdL);
 754
 755    struct timeval StartTime;
 756    gettimeofday(&StartTime,0);
 757    struct CacheDB::Stats Stats;
 758
 759    // Read the configuration file.
 760    Configuration Setup;
 761    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 762       return false;
 763
 764    vector<PackageMap> PkgList;
 765    LoadTree(PkgList,Setup);
 766    LoadBinDir(PkgList,Setup);
 767
 768    // Sort by cache DB to improve IO locality.
 769    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 770
 771    // Generate packages
 772    if (CmdL.FileSize() <= 2)
 773    {
 774       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 775          if (I->GenPackages(Setup,Stats) == false)
 776             _error->DumpErrors();
 777       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 778          if (I->GenSources(Setup,SrcStats) == false)
 779             _error->DumpErrors();
 780    }
 781    else
 782    {
 783       // Make a choice list out of the package list..
 784       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 785       RxChoiceList *End = List;
 786       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 787       {
 788          End->UserData = &(*I);
 789          End->Str = I->BaseDir.c_str();
 790          End++;
 791
 792          End->UserData = &(*I);
 793          End->Str = I->Tag.c_str();
 794          End++;
 795       }
 796       End->Str = 0;
 797
 798       // Regex it
 799       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 800       {
 801          delete [] List;
 802          return _error->Error(_("No selections matched"));
 803       }
 804       _error->DumpErrors();
 805
 806       // Do the generation for Packages
 807       for (End = List; End->Str != 0; End++)
 808       {
 809          if (End->Hit == false)
 810             continue;
 811
 812          PackageMap *I = (PackageMap *)End->UserData;
 813          if (I->PkgDone == true)
 814             continue;
 815          if (I->GenPackages(Setup,Stats) == false)
 816             _error->DumpErrors();
 817       }
 818
 819       // Do the generation for Sources
 820       for (End = List; End->Str != 0; End++)
 821       {
 822          if (End->Hit == false)
 823             continue;
 824
 825          PackageMap *I = (PackageMap *)End->UserData;
 826          if (I->SrcDone == true)
 827             continue;
 828          if (I->GenSources(Setup,SrcStats) == false)
 829             _error->DumpErrors();
 830       }
 831
 832       delete [] List;
 833    }
 834
 835    // close the Translation master files
 836    for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); I++)
 837       if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
 838          delete I->TransWriter;
 839
 840    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 841       return true;
 842
 843    c1out << "Packages done, Starting contents." << endl;
 844
 845    // Sort the contents file list by date
 846    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 847    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 848    {
 849       struct stat A;
 850       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 851                                  I->CntCompress,A) == false)
 852          time(&I->ContentsMTime);
 853       else
 854          I->ContentsMTime = A.st_mtime;
 855    }
 856    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 857
 858    /* Now for Contents.. The process here is to do a make-like dependency
 859       check. Each contents file is verified to be newer than the package files
 860       that describe the debs it indexes. Since the package files contain
 861       hashes of the .debs this means they have not changed either so the
 862       contents must be up to date. */
 863    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 864    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 865    {
 866       // This record is not relevent
 867       if (I->ContentsDone == true ||
 868           I->Contents.empty() == true)
 869          continue;
 870
 871       // Do not do everything if the user specified sections.
 872       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 873          continue;
 874
 875       struct stat A,B;
 876       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 877       {
 878          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 879          {
 880             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 881             continue;
 882          }
 883
 884          if (A.st_mtime > B.st_mtime)
 885             continue;
 886       }
 887
 888       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 889                          MaxContentsChange) == false)
 890          _error->DumpErrors();
 891
 892       // Hit the limit?
 893       if (MaxContentsChange == 0)
 894       {
 895          c1out << "Hit contents update byte limit" << endl;
 896          break;
 897       }
 898    }
 899
 900    struct timeval NewTime;
 901    gettimeofday(&NewTime,0);
 902    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 903                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 904    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 905          << " archives. Took " << TimeToStr((long)Delta) << endl;
 906
 907    return true;
 908 }
 909                                                                         /*}}}*/
 910 // Clean - Clean out the databases                                      /*{{{*/
 911 // ---------------------------------------------------------------------
 912 /* */
 913 bool Clean(CommandLine &CmdL)
 914 {
 915    if (CmdL.FileSize() != 2)
 916       return ShowHelp(CmdL);
 917
 918    // Read the configuration file.
 919    Configuration Setup;
 920    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 921       return false;
 922
 923    vector<PackageMap> PkgList;
 924    LoadTree(PkgList,Setup);
 925    LoadBinDir(PkgList,Setup);
 926
 927    // Sort by cache DB to improve IO locality.
 928    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 929
 930    string CacheDir = Setup.FindDir("Dir::CacheDir");
 931
 932    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 933    {
 934       c0out << I->BinCacheDB << endl;
 935       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 936       if (DB.Clean() == false)
 937          _error->DumpErrors();
 938
 939       string CacheDB = I->BinCacheDB;
 940       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
 941    }
 942
 943    return true;
 944 }
 945                                                                         /*}}}*/
 946
 947 int main(int argc, const char *argv[])
 948 {
 949    setlocale(LC_ALL, "");
 950    CommandLine::Args Args[] = {
 951       {'h',"help","help",0},
 952       {0,"md5","APT::FTPArchive::MD5",0},
 953       {0,"sha1","APT::FTPArchive::SHA1",0},
 954       {0,"sha256","APT::FTPArchive::SHA256",0},
 955       {'v',"version","version",0},
 956       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 957       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 958       {'q',"quiet","quiet",CommandLine::IntLevel},
 959       {'q',"silent","quiet",CommandLine::IntLevel},
 960       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 961       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 962       {0,"contents","APT::FTPArchive::Contents",0},
 963       {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
 964       {'c',"config-file",0,CommandLine::ConfigFile},
 965       {'o',"option",0,CommandLine::ArbItem},
 966       {0,0,0,0}};
 967    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 968                                    {"contents",&SimpleGenContents},
 969                                    {"sources",&SimpleGenSources},
 970                                    {"release",&SimpleGenRelease},
 971                                    {"generate",&Generate},
 972                                    {"clean",&Clean},
 973                                    {"help",&ShowHelp},
 974                                    {0,0}};
 975
 976    // Parse the command line and initialize the package library
 977    CommandLine CmdL(Args,_config);
 978    if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false)
 979    {
 980       _error->DumpErrors();
 981       return 100;
 982    }
 983
 984    // See if the help should be shown
 985    if (_config->FindB("help") == true ||
 986        _config->FindB("version") == true ||
 987        CmdL.FileSize() == 0)
 988    {
 989       ShowHelp(CmdL);
 990       return 0;
 991    }
 992
 993    // Setup the output streams
 994    c0out.rdbuf(clog.rdbuf());
 995    c1out.rdbuf(clog.rdbuf());
 996    c2out.rdbuf(clog.rdbuf());
 997    Quiet = _config->FindI("quiet",0);
 998    if (Quiet > 0)
 999       c0out.rdbuf(devnull.rdbuf());
1000    if (Quiet > 1)
1001       c1out.rdbuf(devnull.rdbuf());
1002
1003    // Match the operation
1004    CmdL.DispatchArg(Cmds);
1005
1006    if (_error->empty() == false)
1007    {
1008       bool Errors = _error->PendingError();
1009       _error->DumpErrors();
1010       return Errors == true?100:0;
1011    }
1012    return 0;
1013 }