ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "apt-ftparchive.h"
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <config.h>
  21 #include <apti18n.h>
  22 #include <algorithm>
  23
  24 #include <climits>
  25 #include <sys/time.h>
  26 #include <regex.h>
  27
  28 #include "contents.h"
  29 #include "multicompress.h"
  30 #include "writer.h"
  31                                                                         /*}}}*/
  32
  33 using namespace std;
  34 ostream c0out(0);
  35 ostream c1out(0);
  36 ostream c2out(0);
  37 ofstream devnull("/dev/null");
  38 unsigned Quiet = 0;
  39
  40 // struct PackageMap - List of all package files in the config file     /*{{{*/
  41 // ---------------------------------------------------------------------
  42 /* */
  43 struct PackageMap
  44 {
  45    // General Stuff
  46    string BaseDir;
  47    string InternalPrefix;
  48    string FLFile;
  49    string PkgExt;
  50    string SrcExt;
  51
  52    // Stuff for the Package File
  53    string PkgFile;
  54    string BinCacheDB;
  55    string BinOverride;
  56    string ExtraOverride;
  57
  58    // We generate for this given arch
  59    string Arch;
  60
  61    // Stuff for the Source File
  62    string SrcFile;
  63    string SrcOverride;
  64    string SrcExtraOverride;
  65
  66    // Translation master file
  67    bool LongDesc;
  68    TranslationWriter *TransWriter;
  69
  70    // Contents
  71    string Contents;
  72    string ContentsHead;
  73
  74    // Random things
  75    string Tag;
  76    string PkgCompress;
  77    string CntCompress;
  78    string SrcCompress;
  79    string PathPrefix;
  80    unsigned int DeLinkLimit;
  81    mode_t Permissions;
  82
  83    bool ContentsDone;
  84    bool PkgDone;
  85    bool SrcDone;
  86    time_t ContentsMTime;
  87
  88    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  89    {
  90       inline bool operator() (const PackageMap &x,const PackageMap &y)
  91       {return x.ContentsMTime < y.ContentsMTime;};
  92    };
  93
  94    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  95    {
  96       inline bool operator() (const PackageMap &x,const PackageMap &y)
  97       {return x.BinCacheDB < y.BinCacheDB;};
  98    };
  99
 100    void GetGeneral(Configuration &Setup,Configuration &Block);
 101    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 102    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 103    bool GenContents(Configuration &Setup,
 104                     vector<PackageMap>::iterator Begin,
 105                     vector<PackageMap>::iterator End,
 106                     unsigned long &Left);
 107
 108    PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 109                   ContentsDone(false), PkgDone(false), SrcDone(false),
 110                   ContentsMTime(0) {};
 111 };
 112                                                                         /*}}}*/
 113
 114 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 115 // ---------------------------------------------------------------------
 116 /* */
 117 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 118 {
 119    PathPrefix = Block.Find("PathPrefix");
 120
 121    if (Block.FindB("External-Links",true) == false)
 122       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 123    else
 124       DeLinkLimit = 0;
 125
 126    PkgCompress = Block.Find("Packages::Compress",
 127                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 128    CntCompress = Block.Find("Contents::Compress",
 129                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 130    SrcCompress = Block.Find("Sources::Compress",
 131                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 132
 133    SrcExt = Block.Find("Sources::Extensions",
 134                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 135    PkgExt = Block.Find("Packages::Extensions",
 136                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 137
 138    if (FLFile.empty() == false)
 139       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 140
 141    if (Contents == " ")
 142       Contents= string();
 143 }
 144                                                                         /*}}}*/
 145 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 146 // ---------------------------------------------------------------------
 147 /* This generates the Package File described by this object. */
 148 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 149 {
 150    if (PkgFile.empty() == true)
 151       return true;
 152
 153    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 154    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 155    string CacheDir = Setup.FindDir("Dir::CacheDir");
 156
 157    struct timeval StartTime;
 158    gettimeofday(&StartTime,0);
 159
 160    PkgDone = true;
 161
 162    // Create a package writer object.
 163    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 164                            flCombine(OverrideDir,BinOverride),
 165                            flCombine(OverrideDir,ExtraOverride),
 166                            Arch);
 167    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 168       return _error->Error(_("Package extension list is too long"));
 169    if (_error->PendingError() == true)
 170       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 171
 172    Packages.PathPrefix = PathPrefix;
 173    Packages.DirStrip = ArchiveDir;
 174    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 175
 176    Packages.TransWriter = TransWriter;
 177    Packages.LongDescription = LongDesc;
 178
 179    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 180    Packages.DeLinkLimit = DeLinkLimit;
 181
 182    // Create a compressor object
 183    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 184                       PkgCompress,Permissions);
 185    Packages.Output = Comp.Input;
 186    if (_error->PendingError() == true)
 187       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 188
 189    c0out << ' ' << BaseDir << ":" << flush;
 190
 191    // Do recursive directory searching
 192    if (FLFile.empty() == true)
 193    {
 194       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 195          return false;
 196    }
 197    else
 198    {
 199       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 200          return false;
 201    }
 202
 203    Packages.Output = 0;      // Just in case
 204
 205    // Finish compressing
 206    unsigned long Size;
 207    if (Comp.Finalize(Size) == false)
 208    {
 209       c0out << endl;
 210       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 211    }
 212
 213    if (Size != 0)
 214       c0out << " New "
 215              << SizeToStr(Size) << "B ";
 216    else
 217       c0out << ' ';
 218
 219    struct timeval NewTime;
 220    gettimeofday(&NewTime,0);
 221    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 222                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 223
 224    c0out << Packages.Stats.Packages << " files " <<
 225 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 226       SizeToStr(Packages.Stats.Bytes) << "B " <<
 227       TimeToStr((long)Delta) << endl;
 228
 229    Stats.Add(Packages.Stats);
 230    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 231
 232    return !_error->PendingError();
 233 }
 234
 235                                                                         /*}}}*/
 236 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 237 // ---------------------------------------------------------------------
 238 /* This generates the Sources File described by this object. */
 239 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 240 {
 241    if (SrcFile.empty() == true)
 242       return true;
 243
 244    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 245    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 246    string CacheDir = Setup.FindDir("Dir::CacheDir");
 247
 248    struct timeval StartTime;
 249    gettimeofday(&StartTime,0);
 250
 251    SrcDone = true;
 252
 253    // Create a package writer object.
 254    SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
 255                          flCombine(OverrideDir,SrcOverride),
 256                          flCombine(OverrideDir,SrcExtraOverride));
 257    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 258       return _error->Error(_("Source extension list is too long"));
 259    if (_error->PendingError() == true)
 260       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 261
 262    Sources.PathPrefix = PathPrefix;
 263    Sources.DirStrip = ArchiveDir;
 264    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 265
 266    Sources.DeLinkLimit = DeLinkLimit;
 267    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 268
 269    // Create a compressor object
 270    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 271                       SrcCompress,Permissions);
 272    Sources.Output = Comp.Input;
 273    if (_error->PendingError() == true)
 274       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 275
 276    c0out << ' ' << BaseDir << ":" << flush;
 277
 278    // Do recursive directory searching
 279    if (FLFile.empty() == true)
 280    {
 281       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 282          return false;
 283    }
 284    else
 285    {
 286       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 287          return false;
 288    }
 289    Sources.Output = 0;      // Just in case
 290
 291    // Finish compressing
 292    unsigned long Size;
 293    if (Comp.Finalize(Size) == false)
 294    {
 295       c0out << endl;
 296       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 297    }
 298
 299    if (Size != 0)
 300       c0out << " New "
 301              << SizeToStr(Size) << "B ";
 302    else
 303       c0out << ' ';
 304
 305    struct timeval NewTime;
 306    gettimeofday(&NewTime,0);
 307    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 308                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 309
 310    c0out << Sources.Stats.Packages << " pkgs in " <<
 311       TimeToStr((long)Delta) << endl;
 312
 313    Stats.Add(Sources.Stats);
 314    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 315
 316    return !_error->PendingError();
 317 }
 318                                                                         /*}}}*/
 319 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 320 // ---------------------------------------------------------------------
 321 /* This generates the contents file partially described by this object.
 322    It searches the given iterator range for other package files that map
 323    into this contents file and includes their data as well when building. */
 324 bool PackageMap::GenContents(Configuration &Setup,
 325                              vector<PackageMap>::iterator Begin,
 326                              vector<PackageMap>::iterator End,
 327                              unsigned long &Left)
 328 {
 329    if (Contents.empty() == true)
 330       return true;
 331
 332    if (Left == 0)
 333       return true;
 334
 335    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 336    string CacheDir = Setup.FindDir("Dir::CacheDir");
 337    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 338
 339    struct timeval StartTime;
 340    gettimeofday(&StartTime,0);
 341
 342    // Create a package writer object.
 343    ContentsWriter Contents("", Arch);
 344    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 345       return _error->Error(_("Package extension list is too long"));
 346    if (_error->PendingError() == true)
 347       return false;
 348
 349    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 350                       CntCompress,Permissions);
 351    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 352    Contents.Output = Comp.Input;
 353    if (_error->PendingError() == true)
 354       return false;
 355
 356    // Write the header out.
 357    if (ContentsHead.empty() == false)
 358    {
 359       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 360       if (_error->PendingError() == true)
 361          return false;
 362
 363       unsigned long Size = Head.Size();
 364       unsigned char Buf[4096];
 365       while (Size != 0)
 366       {
 367          unsigned long ToRead = Size;
 368          if (Size > sizeof(Buf))
 369             ToRead = sizeof(Buf);
 370
 371          if (Head.Read(Buf,ToRead) == false)
 372             return false;
 373
 374          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 375             return _error->Errno("fwrite",_("Error writing header to contents file"));
 376
 377          Size -= ToRead;
 378       }
 379    }
 380
 381    /* Go over all the package file records and parse all the package
 382       files associated with this contents file into one great big honking
 383       memory structure, then dump the sorted version */
 384    c0out << ' ' << this->Contents << ":" << flush;
 385    for (vector<PackageMap>::iterator I = Begin; I != End; I++)
 386    {
 387       if (I->Contents != this->Contents)
 388          continue;
 389
 390       Contents.Prefix = ArchiveDir;
 391       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 392       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 393                             I->PkgCompress);
 394
 395       I->ContentsDone = true;
 396    }
 397
 398    Contents.Finish();
 399
 400    // Finish compressing
 401    unsigned long Size;
 402    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 403    {
 404       c0out << endl;
 405       return _error->Error(_("Error processing contents %s"),
 406                            this->Contents.c_str());
 407    }
 408
 409    if (Size != 0)
 410    {
 411       c0out << " New " << SizeToStr(Size) << "B ";
 412       if (Left > Size)
 413          Left -= Size;
 414       else
 415          Left = 0;
 416    }
 417    else
 418       c0out << ' ';
 419
 420    struct timeval NewTime;
 421    gettimeofday(&NewTime,0);
 422    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 423                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 424
 425    c0out << Contents.Stats.Packages << " files " <<
 426       SizeToStr(Contents.Stats.Bytes) << "B " <<
 427       TimeToStr((long)Delta) << endl;
 428
 429    return true;
 430 }
 431                                                                         /*}}}*/
 432
 433 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 434 // ---------------------------------------------------------------------
 435 /* This populates the PkgList with all the possible permutations of the
 436    section/arch lists. */
 437 void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 438 {
 439    // Load the defaults
 440    string DDir = Setup.Find("TreeDefault::Directory",
 441                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 442    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 443                             "$(DIST)/$(SECTION)/source/");
 444    string DPkg = Setup.Find("TreeDefault::Packages",
 445                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 446    string DTrans = Setup.Find("TreeDefault::Translation",
 447                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 448    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 449                             "$(DIST)/$(SECTION)/");
 450    string DContents = Setup.Find("TreeDefault::Contents",
 451                             "$(DIST)/Contents-$(ARCH)");
 452    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 453    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 454                                "packages-$(ARCH).db");
 455    string DSources = Setup.Find("TreeDefault::Sources",
 456                                 "$(DIST)/$(SECTION)/source/Sources");
 457    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 458    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 459
 460    int const Permissions = Setup.FindI("Default::FileMode",0644);
 461
 462    bool const LongDescription = Setup.FindB("Default::LongDescription",
 463                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 464    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 465
 466    // Process 'tree' type sections
 467    const Configuration::Item *Top = Setup.Tree("tree");
 468    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 469    {
 470       Configuration Block(Top);
 471       string Dist = Top->Tag;
 472
 473       // Parse the sections
 474       string Tmp = Block.Find("Sections");
 475       const char *Sections = Tmp.c_str();
 476       string Section;
 477       while (ParseQuoteWord(Sections,Section) == true)
 478       {
 479          string Arch;
 480          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 481                                          {"$(SECTION)",&Section},
 482                                          {"$(ARCH)",&Arch},
 483                                          {}};
 484          mode_t const Perms = Block.FindI("FileMode", Permissions);
 485          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 486          TranslationWriter *TransWriter;
 487          if (DTrans.empty() == false && LongDesc == false)
 488          {
 489             string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 490                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 491             string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 492             TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 493          }
 494          else
 495             TransWriter = NULL;
 496
 497          string const Tmp2 = Block.Find("Architectures");
 498          const char *Archs = Tmp2.c_str();
 499          while (ParseQuoteWord(Archs,Arch) == true)
 500          {
 501             PackageMap Itm;
 502             Itm.Permissions = Perms;
 503             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 504             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 505
 506             if (stringcasecmp(Arch,"source") == 0)
 507             {
 508                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 509                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 510                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 511                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 512                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 513                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 514             }
 515             else
 516             {
 517                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 518                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 519                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 520                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 521                Itm.Arch = Arch;
 522                Itm.LongDesc = LongDesc;
 523                if (TransWriter != NULL)
 524                {
 525                   TransWriter->IncreaseRefCounter();
 526                   Itm.TransWriter = TransWriter;
 527                }
 528                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 529                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 530                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 531                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 532             }
 533
 534             Itm.GetGeneral(Setup,Block);
 535             PkgList.push_back(Itm);
 536          }
 537          // we didn't use this TransWriter, so we can release it
 538          if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
 539             delete TransWriter;
 540       }
 541
 542       Top = Top->Next;
 543    }
 544 }
 545                                                                         /*}}}*/
 546 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 547 // ---------------------------------------------------------------------
 548 /* */
 549 void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 550 {
 551    // Process 'bindirectory' type sections
 552    const Configuration::Item *Top = Setup.Tree("bindirectory");
 553    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 554    {
 555       Configuration Block(Top);
 556
 557       PackageMap Itm;
 558       Itm.PkgFile = Block.Find("Packages");
 559       Itm.SrcFile = Block.Find("Sources");
 560       Itm.BinCacheDB = Block.Find("BinCacheDB");
 561       Itm.BinOverride = Block.Find("BinOverride");
 562       Itm.ExtraOverride = Block.Find("ExtraOverride");
 563       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 564       Itm.SrcOverride = Block.Find("SrcOverride");
 565       Itm.BaseDir = Top->Tag;
 566       Itm.FLFile = Block.Find("FileList");
 567       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 568       Itm.Contents = Block.Find("Contents");
 569       Itm.ContentsHead = Block.Find("Contents::Header");
 570
 571       Itm.GetGeneral(Setup,Block);
 572       PkgList.push_back(Itm);
 573
 574       Top = Top->Next;
 575    }
 576 }
 577                                                                         /*}}}*/
 578
 579 // ShowHelp - Show the help text                                        /*{{{*/
 580 // ---------------------------------------------------------------------
 581 /* */
 582 bool ShowHelp(CommandLine &CmdL)
 583 {
 584    ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,VERSION,
 585             COMMON_ARCH,__DATE__,__TIME__);
 586    if (_config->FindB("version") == true)
 587       return true;
 588
 589    cout <<
 590     _("Usage: apt-ftparchive [options] command\n"
 591       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 592       "          sources srcpath [overridefile [pathprefix]]\n"
 593       "          contents path\n"
 594       "          release path\n"
 595       "          generate config [groups]\n"
 596       "          clean config\n"
 597       "\n"
 598       "apt-ftparchive generates index files for Debian archives. It supports\n"
 599       "many styles of generation from fully automated to functional replacements\n"
 600       "for dpkg-scanpackages and dpkg-scansources\n"
 601       "\n"
 602       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 603       "Package file contains the contents of all the control fields from\n"
 604       "each package as well as the MD5 hash and filesize. An override file\n"
 605       "is supported to force the value of Priority and Section.\n"
 606       "\n"
 607       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 608       "The --source-override option can be used to specify a src override file\n"
 609       "\n"
 610       "The 'packages' and 'sources' command should be run in the root of the\n"
 611       "tree. BinaryPath should point to the base of the recursive search and \n"
 612       "override file should contain the override flags. Pathprefix is\n"
 613       "appended to the filename fields if present. Example usage from the \n"
 614       "Debian archive:\n"
 615       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 616       "               dists/potato/main/binary-i386/Packages\n"
 617       "\n"
 618       "Options:\n"
 619       "  -h    This help text\n"
 620       "  --md5 Control MD5 generation\n"
 621       "  -s=?  Source override file\n"
 622       "  -q    Quiet\n"
 623       "  -d=?  Select the optional caching database\n"
 624       "  --no-delink Enable delinking debug mode\n"
 625       "  --contents  Control contents file generation\n"
 626       "  -c=?  Read this configuration file\n"
 627       "  -o=?  Set an arbitrary configuration option") << endl;
 628
 629    return true;
 630 }
 631                                                                         /*}}}*/
 632 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 633 // ---------------------------------------------------------------------
 634 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 635 bool SimpleGenPackages(CommandLine &CmdL)
 636 {
 637    if (CmdL.FileSize() < 2)
 638       return ShowHelp(CmdL);
 639
 640    string Override;
 641    if (CmdL.FileSize() >= 3)
 642       Override = CmdL.FileList[2];
 643
 644    // Create a package writer object.
 645    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 646                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 647    if (_error->PendingError() == true)
 648       return false;
 649
 650    if (CmdL.FileSize() >= 4)
 651       Packages.PathPrefix = CmdL.FileList[3];
 652
 653    // Do recursive directory searching
 654    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 655       return false;
 656
 657    return true;
 658 }
 659                                                                         /*}}}*/
 660 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 661 // ---------------------------------------------------------------------
 662 /* */
 663 bool SimpleGenContents(CommandLine &CmdL)
 664 {
 665    if (CmdL.FileSize() < 2)
 666       return ShowHelp(CmdL);
 667
 668    // Create a package writer object.
 669    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 670    if (_error->PendingError() == true)
 671       return false;
 672
 673    // Do recursive directory searching
 674    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 675       return false;
 676
 677    Contents.Finish();
 678
 679    return true;
 680 }
 681                                                                         /*}}}*/
 682 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 683 // ---------------------------------------------------------------------
 684 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 685 bool SimpleGenSources(CommandLine &CmdL)
 686 {
 687    if (CmdL.FileSize() < 2)
 688       return ShowHelp(CmdL);
 689
 690    string Override;
 691    if (CmdL.FileSize() >= 3)
 692       Override = CmdL.FileList[2];
 693
 694    string SOverride;
 695    if (Override.empty() == false)
 696       SOverride = Override + ".src";
 697
 698    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 699                              SOverride.c_str());
 700
 701    // Create a package writer object.
 702    SourcesWriter Sources(Override,SOverride);
 703    if (_error->PendingError() == true)
 704       return false;
 705
 706    if (CmdL.FileSize() >= 4)
 707       Sources.PathPrefix = CmdL.FileList[3];
 708
 709    // Do recursive directory searching
 710    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 711       return false;
 712
 713    return true;
 714 }
 715                                                                         /*}}}*/
 716 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 717 // ---------------------------------------------------------------------
 718 bool SimpleGenRelease(CommandLine &CmdL)
 719 {
 720    if (CmdL.FileSize() < 2)
 721       return ShowHelp(CmdL);
 722
 723    string Dir = CmdL.FileList[1];
 724
 725    ReleaseWriter Release("");
 726    Release.DirStrip = Dir;
 727
 728    if (_error->PendingError() == true)
 729       return false;
 730
 731    if (Release.RecursiveScan(Dir) == false)
 732       return false;
 733
 734    Release.Finish();
 735
 736    return true;
 737 }
 738
 739                                                                         /*}}}*/
 740 // Generate - Full generate, using a config file                        /*{{{*/
 741 // ---------------------------------------------------------------------
 742 /* */
 743 bool Generate(CommandLine &CmdL)
 744 {
 745    struct CacheDB::Stats SrcStats;
 746    if (CmdL.FileSize() < 2)
 747       return ShowHelp(CmdL);
 748
 749    struct timeval StartTime;
 750    gettimeofday(&StartTime,0);
 751    struct CacheDB::Stats Stats;
 752
 753    // Read the configuration file.
 754    Configuration Setup;
 755    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 756       return false;
 757
 758    vector<PackageMap> PkgList;
 759    LoadTree(PkgList,Setup);
 760    LoadBinDir(PkgList,Setup);
 761
 762    // Sort by cache DB to improve IO locality.
 763    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 764
 765    // Generate packages
 766    if (CmdL.FileSize() <= 2)
 767    {
 768       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 769          if (I->GenPackages(Setup,Stats) == false)
 770             _error->DumpErrors();
 771       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 772          if (I->GenSources(Setup,SrcStats) == false)
 773             _error->DumpErrors();
 774    }
 775    else
 776    {
 777       // Make a choice list out of the package list..
 778       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 779       RxChoiceList *End = List;
 780       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 781       {
 782          End->UserData = &(*I);
 783          End->Str = I->BaseDir.c_str();
 784          End++;
 785
 786          End->UserData = &(*I);
 787          End->Str = I->Tag.c_str();
 788          End++;
 789       }
 790       End->Str = 0;
 791
 792       // Regex it
 793       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 794       {
 795          delete [] List;
 796          return _error->Error(_("No selections matched"));
 797       }
 798       _error->DumpErrors();
 799
 800       // Do the generation for Packages
 801       for (End = List; End->Str != 0; End++)
 802       {
 803          if (End->Hit == false)
 804             continue;
 805
 806          PackageMap *I = (PackageMap *)End->UserData;
 807          if (I->PkgDone == true)
 808             continue;
 809          if (I->GenPackages(Setup,Stats) == false)
 810             _error->DumpErrors();
 811       }
 812
 813       // Do the generation for Sources
 814       for (End = List; End->Str != 0; End++)
 815       {
 816          if (End->Hit == false)
 817             continue;
 818
 819          PackageMap *I = (PackageMap *)End->UserData;
 820          if (I->SrcDone == true)
 821             continue;
 822          if (I->GenSources(Setup,SrcStats) == false)
 823             _error->DumpErrors();
 824       }
 825
 826       delete [] List;
 827    }
 828
 829    // close the Translation master files
 830    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 831       if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
 832          delete I->TransWriter;
 833
 834    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 835       return true;
 836
 837    c1out << "Packages done, Starting contents." << endl;
 838
 839    // Sort the contents file list by date
 840    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 841    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 842    {
 843       struct stat A;
 844       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 845                                  I->CntCompress,A) == false)
 846          time(&I->ContentsMTime);
 847       else
 848          I->ContentsMTime = A.st_mtime;
 849    }
 850    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 851
 852    /* Now for Contents.. The process here is to do a make-like dependency
 853       check. Each contents file is verified to be newer than the package files
 854       that describe the debs it indexes. Since the package files contain
 855       hashes of the .debs this means they have not changed either so the
 856       contents must be up to date. */
 857    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 858    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 859    {
 860       // This record is not relevent
 861       if (I->ContentsDone == true ||
 862           I->Contents.empty() == true)
 863          continue;
 864
 865       // Do not do everything if the user specified sections.
 866       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 867          continue;
 868
 869       struct stat A,B;
 870       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 871       {
 872          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 873          {
 874             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 875             continue;
 876          }
 877
 878          if (A.st_mtime > B.st_mtime)
 879             continue;
 880       }
 881
 882       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 883                          MaxContentsChange) == false)
 884          _error->DumpErrors();
 885
 886       // Hit the limit?
 887       if (MaxContentsChange == 0)
 888       {
 889          c1out << "Hit contents update byte limit" << endl;
 890          break;
 891       }
 892    }
 893
 894    struct timeval NewTime;
 895    gettimeofday(&NewTime,0);
 896    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 897                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 898    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 899          << " archives. Took " << TimeToStr((long)Delta) << endl;
 900
 901    return true;
 902 }
 903                                                                         /*}}}*/
 904 // Clean - Clean out the databases                                      /*{{{*/
 905 // ---------------------------------------------------------------------
 906 /* */
 907 bool Clean(CommandLine &CmdL)
 908 {
 909    if (CmdL.FileSize() != 2)
 910       return ShowHelp(CmdL);
 911
 912    // Read the configuration file.
 913    Configuration Setup;
 914    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 915       return false;
 916
 917    vector<PackageMap> PkgList;
 918    LoadTree(PkgList,Setup);
 919    LoadBinDir(PkgList,Setup);
 920
 921    // Sort by cache DB to improve IO locality.
 922    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 923
 924    string CacheDir = Setup.FindDir("Dir::CacheDir");
 925
 926    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 927    {
 928       c0out << I->BinCacheDB << endl;
 929       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 930       if (DB.Clean() == false)
 931          _error->DumpErrors();
 932
 933       string CacheDB = I->BinCacheDB;
 934       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
 935    }
 936
 937    return true;
 938 }
 939                                                                         /*}}}*/
 940
 941 int main(int argc, const char *argv[])
 942 {
 943    setlocale(LC_ALL, "");
 944    CommandLine::Args Args[] = {
 945       {'h',"help","help",0},
 946       {0,"md5","APT::FTPArchive::MD5",0},
 947       {'v',"version","version",0},
 948       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 949       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 950       {'q',"quiet","quiet",CommandLine::IntLevel},
 951       {'q',"silent","quiet",CommandLine::IntLevel},
 952       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 953       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 954       {0,"contents","APT::FTPArchive::Contents",0},
 955       {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
 956       {'c',"config-file",0,CommandLine::ConfigFile},
 957       {'o',"option",0,CommandLine::ArbItem},
 958       {0,0,0,0}};
 959    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 960                                    {"contents",&SimpleGenContents},
 961                                    {"sources",&SimpleGenSources},
 962                                    {"release",&SimpleGenRelease},
 963                                    {"generate",&Generate},
 964                                    {"clean",&Clean},
 965                                    {"help",&ShowHelp},
 966                                    {0,0}};
 967
 968    // Parse the command line and initialize the package library
 969    CommandLine CmdL(Args,_config);
 970    if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false)
 971    {
 972       _error->DumpErrors();
 973       return 100;
 974    }
 975
 976    // See if the help should be shown
 977    if (_config->FindB("help") == true ||
 978        _config->FindB("version") == true ||
 979        CmdL.FileSize() == 0)
 980    {
 981       ShowHelp(CmdL);
 982       return 0;
 983    }
 984
 985    // Setup the output streams
 986    c0out.rdbuf(clog.rdbuf());
 987    c1out.rdbuf(clog.rdbuf());
 988    c2out.rdbuf(clog.rdbuf());
 989    Quiet = _config->FindI("quiet",0);
 990    if (Quiet > 0)
 991       c0out.rdbuf(devnull.rdbuf());
 992    if (Quiet > 1)
 993       c1out.rdbuf(devnull.rdbuf());
 994
 995    // Match the operation
 996    CmdL.DispatchArg(Cmds);
 997
 998    if (_error->empty() == false)
 999    {
1000       bool Errors = _error->PendingError();
1001       _error->DumpErrors();
1002       return Errors == true?100:0;
1003    }
1004    return 0;
1005 }