ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "apt-ftparchive.h"
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <config.h>
  20 #include <apti18n.h>
  21 #include <algorithm>
  22
  23 #include <climits>
  24 #include <sys/time.h>
  25 #include <regex.h>
  26
  27 #include "contents.h"
  28 #include "multicompress.h"
  29 #include "writer.h"
  30                                                                         /*}}}*/
  31
  32 using namespace std;
  33 ostream c0out(0);
  34 ostream c1out(0);
  35 ostream c2out(0);
  36 ofstream devnull("/dev/null");
  37 unsigned Quiet = 0;
  38
  39 // struct PackageMap - List of all package files in the config file     /*{{{*/
  40 // ---------------------------------------------------------------------
  41 /* */
  42 struct PackageMap
  43 {
  44    // General Stuff
  45    string BaseDir;
  46    string InternalPrefix;
  47    string FLFile;
  48    string PkgExt;
  49    string SrcExt;
  50
  51    // Stuff for the Package File
  52    string PkgFile;
  53    string BinCacheDB;
  54    string BinOverride;
  55    string ExtraOverride;
  56
  57    // We generate for this given arch
  58    string Arch;
  59
  60    // Stuff for the Source File
  61    string SrcFile;
  62    string SrcOverride;
  63    string SrcExtraOverride;
  64
  65    // Translation master file
  66    bool LongDesc;
  67    TranslationWriter *TransWriter;
  68
  69    // Contents
  70    string Contents;
  71    string ContentsHead;
  72
  73    // Random things
  74    string Tag;
  75    string PkgCompress;
  76    string CntCompress;
  77    string SrcCompress;
  78    string PathPrefix;
  79    unsigned int DeLinkLimit;
  80    mode_t Permissions;
  81
  82    bool ContentsDone;
  83    bool PkgDone;
  84    bool SrcDone;
  85    time_t ContentsMTime;
  86
  87    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  88    {
  89       inline bool operator() (const PackageMap &x,const PackageMap &y)
  90       {return x.ContentsMTime < y.ContentsMTime;};
  91    };
  92
  93    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  94    {
  95       inline bool operator() (const PackageMap &x,const PackageMap &y)
  96       {return x.BinCacheDB < y.BinCacheDB;};
  97    };
  98
  99    void GetGeneral(Configuration &Setup,Configuration &Block);
 100    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 101    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 102    bool GenContents(Configuration &Setup,
 103                     vector<PackageMap>::iterator Begin,
 104                     vector<PackageMap>::iterator End,
 105                     unsigned long &Left);
 106
 107    PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 108                   ContentsDone(false), PkgDone(false), SrcDone(false),
 109                   ContentsMTime(0) {};
 110 };
 111                                                                         /*}}}*/
 112
 113 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 114 // ---------------------------------------------------------------------
 115 /* */
 116 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 117 {
 118    PathPrefix = Block.Find("PathPrefix");
 119
 120    if (Block.FindB("External-Links",true) == false)
 121       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 122    else
 123       DeLinkLimit = 0;
 124
 125    PkgCompress = Block.Find("Packages::Compress",
 126                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 127    CntCompress = Block.Find("Contents::Compress",
 128                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 129    SrcCompress = Block.Find("Sources::Compress",
 130                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 131
 132    SrcExt = Block.Find("Sources::Extensions",
 133                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 134    PkgExt = Block.Find("Packages::Extensions",
 135                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 136
 137    if (FLFile.empty() == false)
 138       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 139
 140    if (Contents == " ")
 141       Contents= string();
 142 }
 143                                                                         /*}}}*/
 144 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 145 // ---------------------------------------------------------------------
 146 /* This generates the Package File described by this object. */
 147 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 148 {
 149    if (PkgFile.empty() == true)
 150       return true;
 151
 152    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 153    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 154    string CacheDir = Setup.FindDir("Dir::CacheDir");
 155
 156    struct timeval StartTime;
 157    gettimeofday(&StartTime,0);
 158
 159    PkgDone = true;
 160
 161    // Create a package writer object.
 162    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 163                            flCombine(OverrideDir,BinOverride),
 164                            flCombine(OverrideDir,ExtraOverride),
 165                            Arch);
 166    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 167       return _error->Error(_("Package extension list is too long"));
 168    if (_error->PendingError() == true)
 169       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 170
 171    Packages.PathPrefix = PathPrefix;
 172    Packages.DirStrip = ArchiveDir;
 173    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 174
 175    Packages.TransWriter = TransWriter;
 176    Packages.LongDescription = LongDesc;
 177
 178    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 179    Packages.DeLinkLimit = DeLinkLimit;
 180
 181    // Create a compressor object
 182    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 183                       PkgCompress,Permissions);
 184    Packages.Output = Comp.Input;
 185    if (_error->PendingError() == true)
 186       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 187
 188    c0out << ' ' << BaseDir << ":" << flush;
 189
 190    // Do recursive directory searching
 191    if (FLFile.empty() == true)
 192    {
 193       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 194          return false;
 195    }
 196    else
 197    {
 198       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 199          return false;
 200    }
 201
 202    Packages.Output = 0;      // Just in case
 203
 204    // Finish compressing
 205    unsigned long Size;
 206    if (Comp.Finalize(Size) == false)
 207    {
 208       c0out << endl;
 209       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 210    }
 211
 212    if (Size != 0)
 213       c0out << " New "
 214              << SizeToStr(Size) << "B ";
 215    else
 216       c0out << ' ';
 217
 218    struct timeval NewTime;
 219    gettimeofday(&NewTime,0);
 220    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 221                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 222
 223    c0out << Packages.Stats.Packages << " files " <<
 224 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 225       SizeToStr(Packages.Stats.Bytes) << "B " <<
 226       TimeToStr((long)Delta) << endl;
 227
 228    Stats.Add(Packages.Stats);
 229    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 230
 231    return !_error->PendingError();
 232 }
 233
 234                                                                         /*}}}*/
 235 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 236 // ---------------------------------------------------------------------
 237 /* This generates the Sources File described by this object. */
 238 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 239 {
 240    if (SrcFile.empty() == true)
 241       return true;
 242
 243    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 244    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 245    string CacheDir = Setup.FindDir("Dir::CacheDir");
 246
 247    struct timeval StartTime;
 248    gettimeofday(&StartTime,0);
 249
 250    SrcDone = true;
 251
 252    // Create a package writer object.
 253    SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
 254                          flCombine(OverrideDir,SrcOverride),
 255                          flCombine(OverrideDir,SrcExtraOverride));
 256    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 257       return _error->Error(_("Source extension list is too long"));
 258    if (_error->PendingError() == true)
 259       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 260
 261    Sources.PathPrefix = PathPrefix;
 262    Sources.DirStrip = ArchiveDir;
 263    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 264
 265    Sources.DeLinkLimit = DeLinkLimit;
 266    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 267
 268    // Create a compressor object
 269    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 270                       SrcCompress,Permissions);
 271    Sources.Output = Comp.Input;
 272    if (_error->PendingError() == true)
 273       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 274
 275    c0out << ' ' << BaseDir << ":" << flush;
 276
 277    // Do recursive directory searching
 278    if (FLFile.empty() == true)
 279    {
 280       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 281          return false;
 282    }
 283    else
 284    {
 285       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 286          return false;
 287    }
 288    Sources.Output = 0;      // Just in case
 289
 290    // Finish compressing
 291    unsigned long Size;
 292    if (Comp.Finalize(Size) == false)
 293    {
 294       c0out << endl;
 295       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 296    }
 297
 298    if (Size != 0)
 299       c0out << " New "
 300              << SizeToStr(Size) << "B ";
 301    else
 302       c0out << ' ';
 303
 304    struct timeval NewTime;
 305    gettimeofday(&NewTime,0);
 306    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 307                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 308
 309    c0out << Sources.Stats.Packages << " pkgs in " <<
 310       TimeToStr((long)Delta) << endl;
 311
 312    Stats.Add(Sources.Stats);
 313    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 314
 315    return !_error->PendingError();
 316 }
 317                                                                         /*}}}*/
 318 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 319 // ---------------------------------------------------------------------
 320 /* This generates the contents file partially described by this object.
 321    It searches the given iterator range for other package files that map
 322    into this contents file and includes their data as well when building. */
 323 bool PackageMap::GenContents(Configuration &Setup,
 324                              vector<PackageMap>::iterator Begin,
 325                              vector<PackageMap>::iterator End,
 326                              unsigned long &Left)
 327 {
 328    if (Contents.empty() == true)
 329       return true;
 330
 331    if (Left == 0)
 332       return true;
 333
 334    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 335    string CacheDir = Setup.FindDir("Dir::CacheDir");
 336    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 337
 338    struct timeval StartTime;
 339    gettimeofday(&StartTime,0);
 340
 341    // Create a package writer object.
 342    ContentsWriter Contents("", Arch);
 343    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 344       return _error->Error(_("Package extension list is too long"));
 345    if (_error->PendingError() == true)
 346       return false;
 347
 348    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 349                       CntCompress,Permissions);
 350    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 351    Contents.Output = Comp.Input;
 352    if (_error->PendingError() == true)
 353       return false;
 354
 355    // Write the header out.
 356    if (ContentsHead.empty() == false)
 357    {
 358       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 359       if (_error->PendingError() == true)
 360          return false;
 361
 362       unsigned long Size = Head.Size();
 363       unsigned char Buf[4096];
 364       while (Size != 0)
 365       {
 366          unsigned long ToRead = Size;
 367          if (Size > sizeof(Buf))
 368             ToRead = sizeof(Buf);
 369
 370          if (Head.Read(Buf,ToRead) == false)
 371             return false;
 372
 373          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 374             return _error->Errno("fwrite",_("Error writing header to contents file"));
 375
 376          Size -= ToRead;
 377       }
 378    }
 379
 380    /* Go over all the package file records and parse all the package
 381       files associated with this contents file into one great big honking
 382       memory structure, then dump the sorted version */
 383    c0out << ' ' << this->Contents << ":" << flush;
 384    for (vector<PackageMap>::iterator I = Begin; I != End; I++)
 385    {
 386       if (I->Contents != this->Contents)
 387          continue;
 388
 389       Contents.Prefix = ArchiveDir;
 390       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 391       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 392                             I->PkgCompress);
 393
 394       I->ContentsDone = true;
 395    }
 396
 397    Contents.Finish();
 398
 399    // Finish compressing
 400    unsigned long Size;
 401    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 402    {
 403       c0out << endl;
 404       return _error->Error(_("Error processing contents %s"),
 405                            this->Contents.c_str());
 406    }
 407
 408    if (Size != 0)
 409    {
 410       c0out << " New " << SizeToStr(Size) << "B ";
 411       if (Left > Size)
 412          Left -= Size;
 413       else
 414          Left = 0;
 415    }
 416    else
 417       c0out << ' ';
 418
 419    struct timeval NewTime;
 420    gettimeofday(&NewTime,0);
 421    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 422                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 423
 424    c0out << Contents.Stats.Packages << " files " <<
 425       SizeToStr(Contents.Stats.Bytes) << "B " <<
 426       TimeToStr((long)Delta) << endl;
 427
 428    return true;
 429 }
 430                                                                         /*}}}*/
 431
 432 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 433 // ---------------------------------------------------------------------
 434 /* This populates the PkgList with all the possible permutations of the
 435    section/arch lists. */
 436 void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 437 {
 438    // Load the defaults
 439    string DDir = Setup.Find("TreeDefault::Directory",
 440                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 441    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 442                             "$(DIST)/$(SECTION)/source/");
 443    string DPkg = Setup.Find("TreeDefault::Packages",
 444                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 445    string DTrans = Setup.Find("TreeDefault::Translation",
 446                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 447    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 448                             "$(DIST)/$(SECTION)/");
 449    string DContents = Setup.Find("TreeDefault::Contents",
 450                             "$(DIST)/Contents-$(ARCH)");
 451    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 452    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 453                                "packages-$(ARCH).db");
 454    string DSources = Setup.Find("TreeDefault::Sources",
 455                                 "$(DIST)/$(SECTION)/source/Sources");
 456    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 457    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 458
 459    int const Permissions = Setup.FindI("Default::FileMode",0644);
 460
 461    bool const LongDescription = Setup.FindB("Default::LongDescription",
 462                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 463    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 464
 465    // Process 'tree' type sections
 466    const Configuration::Item *Top = Setup.Tree("tree");
 467    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 468    {
 469       Configuration Block(Top);
 470       string Dist = Top->Tag;
 471
 472       // Parse the sections
 473       string Tmp = Block.Find("Sections");
 474       const char *Sections = Tmp.c_str();
 475       string Section;
 476       while (ParseQuoteWord(Sections,Section) == true)
 477       {
 478          string Arch;
 479          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 480                                          {"$(SECTION)",&Section},
 481                                          {"$(ARCH)",&Arch},
 482                                          {}};
 483          mode_t const Perms = Block.FindI("FileMode", Permissions);
 484          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 485          TranslationWriter *TransWriter;
 486          if (DTrans.empty() == false && LongDesc == false)
 487          {
 488             string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 489                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 490             string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 491             TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 492          }
 493          else
 494             TransWriter = NULL;
 495
 496          string const Tmp2 = Block.Find("Architectures");
 497          const char *Archs = Tmp2.c_str();
 498          while (ParseQuoteWord(Archs,Arch) == true)
 499          {
 500             PackageMap Itm;
 501             Itm.Permissions = Perms;
 502             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 503             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 504
 505             if (stringcasecmp(Arch,"source") == 0)
 506             {
 507                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 508                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 509                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 510                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 511                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 512                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 513             }
 514             else
 515             {
 516                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 517                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 518                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 519                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 520                Itm.Arch = Arch;
 521                Itm.LongDesc = LongDesc;
 522                if (TransWriter != NULL)
 523                {
 524                   TransWriter->IncreaseRefCounter();
 525                   Itm.TransWriter = TransWriter;
 526                }
 527                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 528                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 529                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 530                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 531             }
 532
 533             Itm.GetGeneral(Setup,Block);
 534             PkgList.push_back(Itm);
 535          }
 536          // we didn't use this TransWriter, so we can release it
 537          if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
 538             delete TransWriter;
 539       }
 540
 541       Top = Top->Next;
 542    }
 543 }
 544                                                                         /*}}}*/
 545 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 546 // ---------------------------------------------------------------------
 547 /* */
 548 void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 549 {
 550    // Process 'bindirectory' type sections
 551    const Configuration::Item *Top = Setup.Tree("bindirectory");
 552    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 553    {
 554       Configuration Block(Top);
 555
 556       PackageMap Itm;
 557       Itm.PkgFile = Block.Find("Packages");
 558       Itm.SrcFile = Block.Find("Sources");
 559       Itm.BinCacheDB = Block.Find("BinCacheDB");
 560       Itm.BinOverride = Block.Find("BinOverride");
 561       Itm.ExtraOverride = Block.Find("ExtraOverride");
 562       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 563       Itm.SrcOverride = Block.Find("SrcOverride");
 564       Itm.BaseDir = Top->Tag;
 565       Itm.FLFile = Block.Find("FileList");
 566       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 567       Itm.Contents = Block.Find("Contents");
 568       Itm.ContentsHead = Block.Find("Contents::Header");
 569
 570       Itm.GetGeneral(Setup,Block);
 571       PkgList.push_back(Itm);
 572
 573       Top = Top->Next;
 574    }
 575 }
 576                                                                         /*}}}*/
 577
 578 // ShowHelp - Show the help text                                        /*{{{*/
 579 // ---------------------------------------------------------------------
 580 /* */
 581 bool ShowHelp(CommandLine &CmdL)
 582 {
 583    ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,VERSION,
 584             COMMON_ARCH,__DATE__,__TIME__);
 585    if (_config->FindB("version") == true)
 586       return true;
 587
 588    cout <<
 589     _("Usage: apt-ftparchive [options] command\n"
 590       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 591       "          sources srcpath [overridefile [pathprefix]]\n"
 592       "          contents path\n"
 593       "          release path\n"
 594       "          generate config [groups]\n"
 595       "          clean config\n"
 596       "\n"
 597       "apt-ftparchive generates index files for Debian archives. It supports\n"
 598       "many styles of generation from fully automated to functional replacements\n"
 599       "for dpkg-scanpackages and dpkg-scansources\n"
 600       "\n"
 601       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 602       "Package file contains the contents of all the control fields from\n"
 603       "each package as well as the MD5 hash and filesize. An override file\n"
 604       "is supported to force the value of Priority and Section.\n"
 605       "\n"
 606       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 607       "The --source-override option can be used to specify a src override file\n"
 608       "\n"
 609       "The 'packages' and 'sources' command should be run in the root of the\n"
 610       "tree. BinaryPath should point to the base of the recursive search and \n"
 611       "override file should contain the override flags. Pathprefix is\n"
 612       "appended to the filename fields if present. Example usage from the \n"
 613       "Debian archive:\n"
 614       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 615       "               dists/potato/main/binary-i386/Packages\n"
 616       "\n"
 617       "Options:\n"
 618       "  -h    This help text\n"
 619       "  --md5 Control MD5 generation\n"
 620       "  -s=?  Source override file\n"
 621       "  -q    Quiet\n"
 622       "  -d=?  Select the optional caching database\n"
 623       "  --no-delink Enable delinking debug mode\n"
 624       "  --contents  Control contents file generation\n"
 625       "  -c=?  Read this configuration file\n"
 626       "  -o=?  Set an arbitrary configuration option") << endl;
 627
 628    return true;
 629 }
 630                                                                         /*}}}*/
 631 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 632 // ---------------------------------------------------------------------
 633 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 634 bool SimpleGenPackages(CommandLine &CmdL)
 635 {
 636    if (CmdL.FileSize() < 2)
 637       return ShowHelp(CmdL);
 638
 639    string Override;
 640    if (CmdL.FileSize() >= 3)
 641       Override = CmdL.FileList[2];
 642
 643    // Create a package writer object.
 644    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 645                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 646    if (_error->PendingError() == true)
 647       return false;
 648
 649    if (CmdL.FileSize() >= 4)
 650       Packages.PathPrefix = CmdL.FileList[3];
 651
 652    // Do recursive directory searching
 653    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 654       return false;
 655
 656    return true;
 657 }
 658                                                                         /*}}}*/
 659 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 660 // ---------------------------------------------------------------------
 661 /* */
 662 bool SimpleGenContents(CommandLine &CmdL)
 663 {
 664    if (CmdL.FileSize() < 2)
 665       return ShowHelp(CmdL);
 666
 667    // Create a package writer object.
 668    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 669    if (_error->PendingError() == true)
 670       return false;
 671
 672    // Do recursive directory searching
 673    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 674       return false;
 675
 676    Contents.Finish();
 677
 678    return true;
 679 }
 680                                                                         /*}}}*/
 681 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 682 // ---------------------------------------------------------------------
 683 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 684 bool SimpleGenSources(CommandLine &CmdL)
 685 {
 686    if (CmdL.FileSize() < 2)
 687       return ShowHelp(CmdL);
 688
 689    string Override;
 690    if (CmdL.FileSize() >= 3)
 691       Override = CmdL.FileList[2];
 692
 693    string SOverride;
 694    if (Override.empty() == false)
 695       SOverride = Override + ".src";
 696
 697    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 698                              SOverride.c_str());
 699
 700    // Create a package writer object.
 701    SourcesWriter Sources(Override,SOverride);
 702    if (_error->PendingError() == true)
 703       return false;
 704
 705    if (CmdL.FileSize() >= 4)
 706       Sources.PathPrefix = CmdL.FileList[3];
 707
 708    // Do recursive directory searching
 709    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 710       return false;
 711
 712    return true;
 713 }
 714                                                                         /*}}}*/
 715 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 716 // ---------------------------------------------------------------------
 717 bool SimpleGenRelease(CommandLine &CmdL)
 718 {
 719    if (CmdL.FileSize() < 2)
 720       return ShowHelp(CmdL);
 721
 722    string Dir = CmdL.FileList[1];
 723
 724    ReleaseWriter Release("");
 725    Release.DirStrip = Dir;
 726
 727    if (_error->PendingError() == true)
 728       return false;
 729
 730    if (Release.RecursiveScan(Dir) == false)
 731       return false;
 732
 733    Release.Finish();
 734
 735    return true;
 736 }
 737
 738                                                                         /*}}}*/
 739 // Generate - Full generate, using a config file                        /*{{{*/
 740 // ---------------------------------------------------------------------
 741 /* */
 742 bool Generate(CommandLine &CmdL)
 743 {
 744    struct CacheDB::Stats SrcStats;
 745    if (CmdL.FileSize() < 2)
 746       return ShowHelp(CmdL);
 747
 748    struct timeval StartTime;
 749    gettimeofday(&StartTime,0);
 750    struct CacheDB::Stats Stats;
 751
 752    // Read the configuration file.
 753    Configuration Setup;
 754    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 755       return false;
 756
 757    vector<PackageMap> PkgList;
 758    LoadTree(PkgList,Setup);
 759    LoadBinDir(PkgList,Setup);
 760
 761    // Sort by cache DB to improve IO locality.
 762    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 763
 764    // Generate packages
 765    if (CmdL.FileSize() <= 2)
 766    {
 767       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 768          if (I->GenPackages(Setup,Stats) == false)
 769             _error->DumpErrors();
 770       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 771          if (I->GenSources(Setup,SrcStats) == false)
 772             _error->DumpErrors();
 773    }
 774    else
 775    {
 776       // Make a choice list out of the package list..
 777       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 778       RxChoiceList *End = List;
 779       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 780       {
 781          End->UserData = &(*I);
 782          End->Str = I->BaseDir.c_str();
 783          End++;
 784
 785          End->UserData = &(*I);
 786          End->Str = I->Tag.c_str();
 787          End++;
 788       }
 789       End->Str = 0;
 790
 791       // Regex it
 792       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 793       {
 794          delete [] List;
 795          return _error->Error(_("No selections matched"));
 796       }
 797       _error->DumpErrors();
 798
 799       // Do the generation for Packages
 800       for (End = List; End->Str != 0; End++)
 801       {
 802          if (End->Hit == false)
 803             continue;
 804
 805          PackageMap *I = (PackageMap *)End->UserData;
 806          if (I->PkgDone == true)
 807             continue;
 808          if (I->GenPackages(Setup,Stats) == false)
 809             _error->DumpErrors();
 810       }
 811
 812       // Do the generation for Sources
 813       for (End = List; End->Str != 0; End++)
 814       {
 815          if (End->Hit == false)
 816             continue;
 817
 818          PackageMap *I = (PackageMap *)End->UserData;
 819          if (I->SrcDone == true)
 820             continue;
 821          if (I->GenSources(Setup,SrcStats) == false)
 822             _error->DumpErrors();
 823       }
 824
 825       delete [] List;
 826    }
 827
 828    // close the Translation master files
 829    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 830       if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
 831          delete I->TransWriter;
 832
 833    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 834       return true;
 835
 836    c1out << "Packages done, Starting contents." << endl;
 837
 838    // Sort the contents file list by date
 839    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 840    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 841    {
 842       struct stat A;
 843       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 844                                  I->CntCompress,A) == false)
 845          time(&I->ContentsMTime);
 846       else
 847          I->ContentsMTime = A.st_mtime;
 848    }
 849    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 850
 851    /* Now for Contents.. The process here is to do a make-like dependency
 852       check. Each contents file is verified to be newer than the package files
 853       that describe the debs it indexes. Since the package files contain
 854       hashes of the .debs this means they have not changed either so the
 855       contents must be up to date. */
 856    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 857    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 858    {
 859       // This record is not relevent
 860       if (I->ContentsDone == true ||
 861           I->Contents.empty() == true)
 862          continue;
 863
 864       // Do not do everything if the user specified sections.
 865       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 866          continue;
 867
 868       struct stat A,B;
 869       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 870       {
 871          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 872          {
 873             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 874             continue;
 875          }
 876
 877          if (A.st_mtime > B.st_mtime)
 878             continue;
 879       }
 880
 881       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 882                          MaxContentsChange) == false)
 883          _error->DumpErrors();
 884
 885       // Hit the limit?
 886       if (MaxContentsChange == 0)
 887       {
 888          c1out << "Hit contents update byte limit" << endl;
 889          break;
 890       }
 891    }
 892
 893    struct timeval NewTime;
 894    gettimeofday(&NewTime,0);
 895    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 896                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 897    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 898          << " archives. Took " << TimeToStr((long)Delta) << endl;
 899
 900    return true;
 901 }
 902                                                                         /*}}}*/
 903 // Clean - Clean out the databases                                      /*{{{*/
 904 // ---------------------------------------------------------------------
 905 /* */
 906 bool Clean(CommandLine &CmdL)
 907 {
 908    if (CmdL.FileSize() != 2)
 909       return ShowHelp(CmdL);
 910
 911    // Read the configuration file.
 912    Configuration Setup;
 913    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 914       return false;
 915
 916    vector<PackageMap> PkgList;
 917    LoadTree(PkgList,Setup);
 918    LoadBinDir(PkgList,Setup);
 919
 920    // Sort by cache DB to improve IO locality.
 921    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 922
 923    string CacheDir = Setup.FindDir("Dir::CacheDir");
 924
 925    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 926    {
 927       c0out << I->BinCacheDB << endl;
 928       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 929       if (DB.Clean() == false)
 930          _error->DumpErrors();
 931
 932       string CacheDB = I->BinCacheDB;
 933       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
 934    }
 935
 936    return true;
 937 }
 938                                                                         /*}}}*/
 939
 940 int main(int argc, const char *argv[])
 941 {
 942    setlocale(LC_ALL, "");
 943    CommandLine::Args Args[] = {
 944       {'h',"help","help",0},
 945       {0,"md5","APT::FTPArchive::MD5",0},
 946       {'v',"version","version",0},
 947       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 948       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 949       {'q',"quiet","quiet",CommandLine::IntLevel},
 950       {'q',"silent","quiet",CommandLine::IntLevel},
 951       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 952       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 953       {0,"contents","APT::FTPArchive::Contents",0},
 954       {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
 955       {'c',"config-file",0,CommandLine::ConfigFile},
 956       {'o',"option",0,CommandLine::ArbItem},
 957       {0,0,0,0}};
 958    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 959                                    {"contents",&SimpleGenContents},
 960                                    {"sources",&SimpleGenSources},
 961                                    {"release",&SimpleGenRelease},
 962                                    {"generate",&Generate},
 963                                    {"clean",&Clean},
 964                                    {"help",&ShowHelp},
 965                                    {0,0}};
 966
 967    // Parse the command line and initialize the package library
 968    CommandLine CmdL(Args,_config);
 969    if (CmdL.Parse(argc,argv) == false)
 970    {
 971       _error->DumpErrors();
 972       return 100;
 973    }
 974
 975    // See if the help should be shown
 976    if (_config->FindB("help") == true ||
 977        _config->FindB("version") == true ||
 978        CmdL.FileSize() == 0)
 979    {
 980       ShowHelp(CmdL);
 981       return 0;
 982    }
 983
 984    // Setup the output streams
 985    c0out.rdbuf(clog.rdbuf());
 986    c1out.rdbuf(clog.rdbuf());
 987    c2out.rdbuf(clog.rdbuf());
 988    Quiet = _config->FindI("quiet",0);
 989    if (Quiet > 0)
 990       c0out.rdbuf(devnull.rdbuf());
 991    if (Quiet > 1)
 992       c1out.rdbuf(devnull.rdbuf());
 993
 994    // Match the operation
 995    CmdL.DispatchArg(Cmds);
 996
 997    if (_error->empty() == false)
 998    {
 999       bool Errors = _error->PendingError();
1000       _error->DumpErrors();
1001       return Errors == true?100:0;
1002    }
1003    return 0;
1004 }