ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.2 2001/02/20 07:03:18 jgg Exp $
   4 /* ######################################################################
   5
   6    apt-scanpackages - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-ftparchive.h"
  15 #endif
  16
  17 #include "apt-ftparchive.h"
  18
  19 #include <apt-pkg/error.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/cmndline.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <config.h>
  24 #include <apti18n.h>
  25 #include <algorithm>
  26
  27 #include <sys/time.h>
  28 #include <regex.h>
  29
  30 #include "contents.h"
  31 #include "multicompress.h"
  32 #include "writer.h"
  33                                                                         /*}}}*/
  34
  35 ostream c0out;
  36 ostream c1out;
  37 ostream c2out;
  38 ofstream devnull("/dev/null");
  39 unsigned Quiet = 0;
  40
  41 // struct PackageMap - List of all package files in the config file     /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* */
  44 struct PackageMap
  45 {
  46    // General Stuff
  47    string BaseDir;
  48    string InternalPrefix;
  49    string FLFile;
  50    string PkgExt;
  51    string SrcExt;
  52
  53    // Stuff for the Package File
  54    string PkgFile;
  55    string BinCacheDB;
  56    string BinOverride;
  57
  58    // Stuff for the Source File
  59    string SrcFile;
  60    string SrcOverride;
  61
  62    // Contents
  63    string Contents;
  64    string ContentsHead;
  65
  66    // Random things
  67    string Tag;
  68    string PkgCompress;
  69    string CntCompress;
  70    string SrcCompress;
  71    string PathPrefix;
  72    unsigned int DeLinkLimit;
  73    mode_t Permissions;
  74
  75    bool ContentsDone;
  76    bool PkgDone;
  77    bool SrcDone;
  78    time_t ContentsMTime;
  79
  80    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  81    {
  82       inline bool operator() (const PackageMap &x,const PackageMap &y)
  83       {return x.ContentsMTime < y.ContentsMTime;};
  84    };
  85
  86    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  87    {
  88       inline bool operator() (const PackageMap &x,const PackageMap &y)
  89       {return x.BinCacheDB < y.BinCacheDB;};
  90    };
  91
  92    void GetGeneral(Configuration &Setup,Configuration &Block);
  93    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
  94    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
  95    bool GenContents(Configuration &Setup,
  96                     PackageMap *Begin,PackageMap *End,
  97                     unsigned long &Left);
  98
  99    PackageMap() : DeLinkLimit(0), Permissions(1), ContentsDone(false),
 100         PkgDone(false), SrcDone(false), ContentsMTime(0) {};
 101 };
 102                                                                         /*}}}*/
 103
 104 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 105 // ---------------------------------------------------------------------
 106 /* */
 107 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 108 {
 109    PathPrefix = Block.Find("PathPrefix");
 110
 111    if (Block.FindB("External-Links",true) == false)
 112       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 113    else
 114       DeLinkLimit = 0;
 115
 116    PkgCompress = Block.Find("Packages::Compress",
 117                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 118    CntCompress = Block.Find("Contents::Compress",
 119                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 120    SrcCompress = Block.Find("Sources::Compress",
 121                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 122
 123    SrcExt = Block.Find("Sources::Extensions",
 124                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 125    PkgExt = Block.Find("Packages::Extensions",
 126                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 127
 128    Permissions = Setup.FindI("Default::FileMode",0644);
 129
 130    if (FLFile.empty() == false)
 131       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 132
 133    if (Contents == " ")
 134       Contents= string();
 135 }
 136                                                                         /*}}}*/
 137 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 138 // ---------------------------------------------------------------------
 139 /* This generates the Package File described by this object. */
 140 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 141 {
 142    if (PkgFile.empty() == true)
 143       return true;
 144
 145    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 146    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 147    string CacheDir = Setup.FindDir("Dir::CacheDir");
 148
 149    struct timeval StartTime;
 150    gettimeofday(&StartTime,0);
 151
 152    PkgDone = true;
 153
 154    // Create a package writer object.
 155    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 156                            flCombine(OverrideDir,BinOverride));
 157    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 158       return _error->Error("Package extension list is too long");
 159    if (_error->PendingError() == true)
 160       return _error->Error("Error Processing directory %s",BaseDir.c_str());
 161
 162    Packages.PathPrefix = PathPrefix;
 163    Packages.DirStrip = ArchiveDir;
 164    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 165
 166    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 167    Packages.DeLinkLimit = DeLinkLimit;
 168
 169    // Create a compressor object
 170    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 171                       PkgCompress,Permissions);
 172    Packages.Output = Comp.Input;
 173    if (_error->PendingError() == true)
 174       return _error->Error("Error Processing directory %s",BaseDir.c_str());
 175
 176    c0out << ' ' << BaseDir << ":" << flush;
 177
 178    // Do recursive directory searching
 179    if (FLFile.empty() == true)
 180    {
 181       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 182          return false;
 183    }
 184    else
 185    {
 186       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 187          return false;
 188    }
 189
 190    Packages.Output = 0;      // Just in case
 191
 192    // Finish compressing
 193    unsigned long Size;
 194    if (Comp.Finalize(Size) == false)
 195    {
 196       c0out << endl;
 197       return _error->Error("Error Processing directory %s",BaseDir.c_str());
 198    }
 199
 200    if (Size != 0)
 201       c0out << " New "
 202              << SizeToStr(Size) << "B ";
 203    else
 204       c0out << ' ';
 205
 206    struct timeval NewTime;
 207    gettimeofday(&NewTime,0);
 208    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 209                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 210
 211    c0out << Packages.Stats.Packages << " files " <<
 212 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 213       SizeToStr(Packages.Stats.Bytes) << "B " <<
 214       TimeToStr((long)Delta) << endl;
 215
 216    Stats.Add(Packages.Stats);
 217    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 218
 219    return !_error->PendingError();
 220 }
 221                                                                         /*}}}*/
 222 // PackageMap::GenSources - Actually generate a Package file            /*{{{*/
 223 // ---------------------------------------------------------------------
 224 /* This generates the Sources File described by this object. */
 225 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 226 {
 227    if (SrcFile.empty() == true)
 228       return true;
 229
 230    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 231    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 232    string CacheDir = Setup.FindDir("Dir::CacheDir");
 233
 234    struct timeval StartTime;
 235    gettimeofday(&StartTime,0);
 236
 237    SrcDone = true;
 238
 239    // Create a package writer object.
 240    SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
 241                           flCombine(OverrideDir,SrcOverride));
 242    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 243       return _error->Error("Source extension list is too long");
 244    if (_error->PendingError() == true)
 245       return _error->Error("Error Processing directory %s",BaseDir.c_str());
 246
 247    Sources.PathPrefix = PathPrefix;
 248    Sources.DirStrip = ArchiveDir;
 249    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 250
 251    Sources.DeLinkLimit = DeLinkLimit;
 252    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 253
 254    // Create a compressor object
 255    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 256                       SrcCompress,Permissions);
 257    Sources.Output = Comp.Input;
 258    if (_error->PendingError() == true)
 259       return _error->Error("Error Processing directory %s",BaseDir.c_str());
 260
 261    c0out << ' ' << BaseDir << ":" << flush;
 262
 263    // Do recursive directory searching
 264    if (FLFile.empty() == true)
 265    {
 266       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 267          return false;
 268    }
 269    else
 270    {
 271       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 272          return false;
 273    }
 274    Sources.Output = 0;      // Just in case
 275
 276    // Finish compressing
 277    unsigned long Size;
 278    if (Comp.Finalize(Size) == false)
 279    {
 280       c0out << endl;
 281       return _error->Error("Error Processing directory %s",BaseDir.c_str());
 282    }
 283
 284    if (Size != 0)
 285       c0out << " New "
 286              << SizeToStr(Size) << "B ";
 287    else
 288       c0out << ' ';
 289
 290    struct timeval NewTime;
 291    gettimeofday(&NewTime,0);
 292    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 293                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 294
 295    c0out << Sources.Stats.Packages << " pkgs in " <<
 296       TimeToStr((long)Delta) << endl;
 297
 298    Stats.Add(Sources.Stats);
 299    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 300
 301    return !_error->PendingError();
 302 }
 303                                                                         /*}}}*/
 304 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 305 // ---------------------------------------------------------------------
 306 /* This generates the contents file partially described by this object.
 307    It searches the given iterator range for other package files that map
 308    into this contents file and includes their data as well when building. */
 309 bool PackageMap::GenContents(Configuration &Setup,
 310                             PackageMap *Begin,PackageMap *End,
 311                             unsigned long &Left)
 312 {
 313    if (Contents.empty() == true)
 314       return true;
 315
 316    if (Left == 0)
 317       return true;
 318
 319    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 320    string CacheDir = Setup.FindDir("Dir::CacheDir");
 321    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 322
 323    struct timeval StartTime;
 324    gettimeofday(&StartTime,0);
 325
 326    // Create a package writer object.
 327    ContentsWriter Contents("");
 328    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 329       return _error->Error("Package extension list is too long");
 330    if (_error->PendingError() == true)
 331       return false;
 332
 333    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 334                       CntCompress,Permissions);
 335    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 336    Contents.Output = Comp.Input;
 337    if (_error->PendingError() == true)
 338       return false;
 339
 340    // Write the header out.
 341    if (ContentsHead.empty() == false)
 342    {
 343       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 344       if (_error->PendingError() == true)
 345          return false;
 346
 347       unsigned long Size = Head.Size();
 348       unsigned char Buf[4096];
 349       while (Size != 0)
 350       {
 351          unsigned long ToRead = Size;
 352          if (Size > sizeof(Buf))
 353             ToRead = sizeof(Buf);
 354
 355          if (Head.Read(Buf,ToRead) == false)
 356             return false;
 357
 358          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 359             return _error->Errno("fwrite","Error writing header to contents file");
 360
 361          Size -= ToRead;
 362       }
 363    }
 364
 365    /* Go over all the package file records and parse all the package
 366       files associated with this contents file into one great big honking
 367       memory structure, then dump the sorted version */
 368    c0out << ' ' << this->Contents << ":" << flush;
 369    for (PackageMap *I = Begin; I != End; I++)
 370    {
 371       if (I->Contents != this->Contents)
 372          continue;
 373
 374       Contents.Prefix = ArchiveDir;
 375       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 376       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 377                             I->PkgCompress);
 378
 379       I->ContentsDone = true;
 380    }
 381
 382    Contents.Finish();
 383
 384    // Finish compressing
 385    unsigned long Size;
 386    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 387    {
 388       c0out << endl;
 389       return _error->Error("Error Processing Contents %s",
 390                            this->Contents.c_str());
 391    }
 392
 393    if (Size != 0)
 394    {
 395       c0out << " New " << SizeToStr(Size) << "B ";
 396       if (Left > Size)
 397          Left -= Size;
 398       else
 399          Left = 0;
 400    }
 401    else
 402       c0out << ' ';
 403
 404    struct timeval NewTime;
 405    gettimeofday(&NewTime,0);
 406    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 407                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 408
 409    c0out << Contents.Stats.Packages << " files " <<
 410       SizeToStr(Contents.Stats.Bytes) << "B " <<
 411       TimeToStr((long)Delta) << endl;
 412
 413    return true;
 414 }
 415                                                                         /*}}}*/
 416
 417 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 418 // ---------------------------------------------------------------------
 419 /* This populates the PkgList with all the possible permutations of the
 420    section/arch lists. */
 421 void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 422 {
 423    // Load the defaults
 424    string DDir = Setup.Find("TreeDefault::Directory",
 425                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 426    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 427                             "$(DIST)/$(SECTION)/source/");
 428    string DPkg = Setup.Find("TreeDefault::Packages",
 429                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 430    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 431                             "$(DIST)/$(SECTION)/");
 432    string DContents = Setup.Find("TreeDefault::Contents",
 433                             "$(DIST)/Contents-$(ARCH)");
 434    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 435    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 436                                "packages-$(ARCH).db");
 437    string DSources = Setup.Find("TreeDefault::Sources",
 438                                 "$(DIST)/$(SECTION)/source/Sources");
 439    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 440    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 441
 442    // Process 'tree' type sections
 443    const Configuration::Item *Top = Setup.Tree("tree");
 444    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 445    {
 446       Configuration Block(Top);
 447       string Dist = Top->Tag;
 448
 449       // Parse the sections
 450       const char *Sections = Block.Find("Sections").c_str();
 451       string Section;
 452       while (ParseQuoteWord(Sections,Section) == true)
 453       {
 454          const char *Archs = Block.Find("Architectures").c_str();
 455          string Arch;
 456          while (ParseQuoteWord(Archs,Arch) == true)
 457          {
 458             struct SubstVar Vars[] = {{"$(DIST)",&Dist},
 459                                       {"$(SECTION)",&Section},
 460                                       {"$(ARCH)",&Arch},
 461                                       {}};
 462             PackageMap Itm;
 463
 464             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 465             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 466
 467             if (stringcasecmp(Arch,"source") == 0)
 468             {
 469                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 470                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 471                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 472                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 473                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 474             }
 475             else
 476             {
 477                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 478                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 479                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 480                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 481                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 482                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 483                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 484             }
 485
 486             Itm.GetGeneral(Setup,Block);
 487             PkgList.push_back(Itm);
 488          }
 489       }
 490
 491       Top = Top->Next;
 492    }
 493 }
 494                                                                         /*}}}*/
 495 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 496 // ---------------------------------------------------------------------
 497 /* */
 498 void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 499 {
 500    // Process 'bindirectory' type sections
 501    const Configuration::Item *Top = Setup.Tree("bindirectory");
 502    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 503    {
 504       Configuration Block(Top);
 505
 506       PackageMap Itm;
 507       Itm.PkgFile = Block.Find("Packages");
 508       Itm.SrcFile = Block.Find("Sources");
 509       Itm.BinCacheDB = Block.Find("BinCacheDB");
 510       Itm.BinOverride = Block.Find("BinOverride");
 511       Itm.SrcOverride = Block.Find("SrcOverride");
 512       Itm.BaseDir = Top->Tag;
 513       Itm.FLFile = Block.Find("FileList");
 514       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 515       Itm.Contents = Block.Find("Contents");
 516       Itm.ContentsHead = Block.Find("Contents::Header");
 517
 518       Itm.GetGeneral(Setup,Block);
 519       PkgList.push_back(Itm);
 520
 521       Top = Top->Next;
 522    }
 523 }
 524                                                                         /*}}}*/
 525
 526 // ShowHelp - Show the help text                                        /*{{{*/
 527 // ---------------------------------------------------------------------
 528 /* */
 529 bool ShowHelp(CommandLine &CmdL)
 530 {
 531    ioprintf(cout,_("%s %s for %s %s compiled on %s %s\n"),PACKAGE,VERSION,
 532             COMMON_OS,COMMON_CPU,__DATE__,__TIME__);
 533    if (_config->FindB("version") == true)
 534       return true;
 535
 536    cout <<
 537       "Usage: apt-ftparchive [options] command\n"
 538       "Commands: packges binarypath [overridefile [pathprefix]]\n"
 539       "          sources srcpath [overridefile [pathprefix]]\n"
 540       "          contents path\n"
 541       "          generate config [groups]\n"
 542       "          clean config\n"
 543       "\n"
 544       "apt-ftparchive generates index files for Debian archives. It supports\n"
 545       "many styles of generation from fully automated to functional replacements\n"
 546       "for dpkg-scanpackages and dpkg-scansources\n"
 547       "\n"
 548       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 549       "Package file contains the contents of all the control fields from\n"
 550       "each package as well as the MD5 hash and filesize. An override file\n"
 551       "is supported to force the value of Priority and Section.\n"
 552       "\n"
 553       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 554       "The --source-override option can be used to specify a src override file\n"
 555       "\n"
 556       "The 'packages' and 'sources' command should be run in the root of the\n"
 557       "tree. BinaryPath should point to the base of the recursive search and \n"
 558       "override file should contian the override flags. Pathprefix is\n"
 559       "appended to the filename fields if present. Example usage from the \n"
 560       "debian archive:\n"
 561       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 562       "               dists/potato/main/binary-i386/Packages\n"
 563       "\n"
 564       "Options:\n"
 565       "  -h    This help text\n"
 566       "  --md5 Control MD5 generation\n"
 567       "  -s=?  Source override file\n"
 568       "  -q    Quiet\n"
 569       "  -d=?  Select the optional caching database\n"
 570       "  --no-delink Enable delinking debug mode\n"
 571       "  --contents  Control contents file generation\n"
 572       "  -c=?  Read this configuration file\n"
 573       "  -o=?  Set an arbitary configuration option" << endl;
 574
 575    return true;
 576 }
 577                                                                         /*}}}*/
 578 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 579 // ---------------------------------------------------------------------
 580 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 581 bool SimpleGenPackages(CommandLine &CmdL)
 582 {
 583    if (CmdL.FileSize() < 2)
 584       return ShowHelp(CmdL);
 585
 586    string Override;
 587    if (CmdL.FileSize() >= 3)
 588       Override = CmdL.FileList[2];
 589
 590    // Create a package writer object.
 591    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 592                            Override);
 593    if (_error->PendingError() == true)
 594       return false;
 595
 596    if (CmdL.FileSize() >= 4)
 597       Packages.PathPrefix = CmdL.FileList[3];
 598
 599    // Do recursive directory searching
 600    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 601       return false;
 602
 603    return true;
 604 }
 605                                                                         /*}}}*/
 606 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 607 // ---------------------------------------------------------------------
 608 /* */
 609 bool SimpleGenContents(CommandLine &CmdL)
 610 {
 611    if (CmdL.FileSize() < 2)
 612       return ShowHelp(CmdL);
 613
 614    // Create a package writer object.
 615    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"));
 616    if (_error->PendingError() == true)
 617       return false;
 618
 619    // Do recursive directory searching
 620    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 621       return false;
 622
 623    Contents.Finish();
 624
 625    return true;
 626 }
 627                                                                         /*}}}*/
 628 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 629 // ---------------------------------------------------------------------
 630 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 631 bool SimpleGenSources(CommandLine &CmdL)
 632 {
 633    if (CmdL.FileSize() < 2)
 634       return ShowHelp(CmdL);
 635
 636    string Override;
 637    if (CmdL.FileSize() >= 3)
 638       Override = CmdL.FileList[2];
 639
 640    string SOverride;
 641    if (Override.empty() == false)
 642       SOverride = Override + ".src";
 643
 644    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 645                              SOverride.c_str());
 646
 647    // Create a package writer object.
 648    SourcesWriter Sources(Override,SOverride);
 649    if (_error->PendingError() == true)
 650       return false;
 651
 652    if (CmdL.FileSize() >= 4)
 653       Sources.PathPrefix = CmdL.FileList[3];
 654
 655    // Do recursive directory searching
 656    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 657       return false;
 658
 659    return true;
 660 }
 661                                                                         /*}}}*/
 662 // Generate - Full generate, using a config file                        /*{{{*/
 663 // ---------------------------------------------------------------------
 664 /* */
 665 bool Generate(CommandLine &CmdL)
 666 {
 667    struct CacheDB::Stats SrcStats;
 668    if (CmdL.FileSize() < 2)
 669       return ShowHelp(CmdL);
 670
 671    struct timeval StartTime;
 672    gettimeofday(&StartTime,0);
 673    struct CacheDB::Stats Stats;
 674
 675    // Read the configuration file.
 676    Configuration Setup;
 677    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 678       return false;
 679
 680    vector<PackageMap> PkgList;
 681    LoadTree(PkgList,Setup);
 682    LoadBinDir(PkgList,Setup);
 683
 684    // Sort by cache DB to improve IO locality.
 685    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 686
 687    // Generate packages
 688    if (CmdL.FileSize() <= 2)
 689    {
 690       for (PackageMap *I = PkgList.begin(); I != PkgList.end(); I++)
 691          if (I->GenPackages(Setup,Stats) == false)
 692             _error->DumpErrors();
 693       for (PackageMap *I = PkgList.begin(); I != PkgList.end(); I++)
 694          if (I->GenSources(Setup,SrcStats) == false)
 695             _error->DumpErrors();
 696    }
 697    else
 698    {
 699       // Make a choice list out of the package list..
 700       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 701       RxChoiceList *End = List;
 702       for (PackageMap *I = PkgList.begin(); I != PkgList.end(); I++)
 703       {
 704          End->UserData = I;
 705          End->Str = I->BaseDir.c_str();
 706          End++;
 707
 708          End->UserData = I;
 709          End->Str = I->Tag.c_str();
 710          End++;
 711       }
 712       End->Str = 0;
 713
 714       // Regex it
 715       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 716       {
 717          delete [] List;
 718          return _error->Error("No selections matched");
 719       }
 720       _error->DumpErrors();
 721
 722       // Do the generation for Packages
 723       for (End = List; End->Str != 0; End++)
 724       {
 725          if (End->Hit == false)
 726             continue;
 727
 728          PackageMap *I = (PackageMap *)End->UserData;
 729          if (I->PkgDone == true)
 730             continue;
 731          if (I->GenPackages(Setup,Stats) == false)
 732             _error->DumpErrors();
 733       }
 734
 735       // Do the generation for Sources
 736       for (End = List; End->Str != 0; End++)
 737       {
 738          if (End->Hit == false)
 739             continue;
 740
 741          PackageMap *I = (PackageMap *)End->UserData;
 742          if (I->SrcDone == true)
 743             continue;
 744          if (I->GenSources(Setup,SrcStats) == false)
 745             _error->DumpErrors();
 746       }
 747
 748       delete [] List;
 749    }
 750
 751    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 752       return true;
 753
 754    c1out << "Done Packages, Starting contents." << endl;
 755
 756    // Sort the contents file list by date
 757    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 758    for (PackageMap *I = PkgList.begin(); I != PkgList.end(); I++)
 759    {
 760       struct stat A;
 761       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 762                                  I->CntCompress,A) == false)
 763          time(&I->ContentsMTime);
 764       else
 765          I->ContentsMTime = A.st_mtime;
 766    }
 767    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 768
 769    /* Now for Contents.. The process here is to do a make-like dependency
 770       check. Each contents file is verified to be newer than the package files
 771       that describe the debs it indexes. Since the package files contain
 772       hashes of the .debs this means they have not changed either so the
 773       contents must be up to date. */
 774    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 775    for (PackageMap *I = PkgList.begin(); I != PkgList.end(); I++)
 776    {
 777       // This record is not relevent
 778       if (I->ContentsDone == true ||
 779           I->Contents.empty() == true)
 780          continue;
 781
 782       // Do not do everything if the user specified sections.
 783       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 784          continue;
 785
 786       struct stat A,B;
 787       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 788       {
 789          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 790          {
 791             _error->Warning("Some files are missing in the package file group `%s'",I->PkgFile.c_str());
 792             continue;
 793          }
 794
 795          if (A.st_mtime > B.st_mtime)
 796             continue;
 797       }
 798
 799       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 800                          MaxContentsChange) == false)
 801          _error->DumpErrors();
 802
 803       // Hit the limit?
 804       if (MaxContentsChange == 0)
 805       {
 806          c1out << "Hit contents update byte limit" << endl;
 807          break;
 808       }
 809    }
 810
 811    struct timeval NewTime;
 812    gettimeofday(&NewTime,0);
 813    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 814                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 815    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 816          << " archives. Took " << TimeToStr((long)Delta) << endl;
 817
 818    return true;
 819 }
 820                                                                         /*}}}*/
 821 // Clean - Clean out the databases                                      /*{{{*/
 822 // ---------------------------------------------------------------------
 823 /* */
 824 bool Clean(CommandLine &CmdL)
 825 {
 826    if (CmdL.FileSize() != 2)
 827       return ShowHelp(CmdL);
 828
 829    // Read the configuration file.
 830    Configuration Setup;
 831    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 832       return false;
 833
 834    vector<PackageMap> PkgList;
 835    LoadTree(PkgList,Setup);
 836    LoadBinDir(PkgList,Setup);
 837
 838    // Sort by cache DB to improve IO locality.
 839    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 840
 841    string CacheDir = Setup.FindDir("Dir::CacheDir");
 842
 843    for (PackageMap *I = PkgList.begin(); I != PkgList.end(); )
 844    {
 845       c0out << I->BinCacheDB << endl;
 846       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 847       if (DB.Clean() == false)
 848          _error->DumpErrors();
 849
 850       string CacheDB = I->BinCacheDB;
 851       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
 852    }
 853
 854    return true;
 855 }
 856                                                                         /*}}}*/
 857
 858 int main(int argc, const char *argv[])
 859 {
 860    CommandLine::Args Args[] = {
 861       {'h',"help","help",0},
 862       {0,"md5","APT::FTPArchive::MD5",0},
 863       {'v',"version","version",0},
 864       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 865       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 866       {'q',"quiet","quiet",CommandLine::IntLevel},
 867       {'q',"silent","quiet",CommandLine::IntLevel},
 868       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 869       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 870       {0,"contents","APT::FTPArchive::Contents",0},
 871       {'c',"config-file",0,CommandLine::ConfigFile},
 872       {'o',"option",0,CommandLine::ArbItem},
 873       {0,0,0,0}};
 874    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 875                                    {"contents",&SimpleGenContents},
 876                                    {"sources",&SimpleGenSources},
 877                                    {"generate",&Generate},
 878                                    {"clean",&Clean},
 879                                    {"help",&ShowHelp},
 880                                    {0,0}};
 881
 882    // Parse the command line and initialize the package library
 883    CommandLine CmdL(Args,_config);
 884    if (CmdL.Parse(argc,argv) == false)
 885    {
 886       _error->DumpErrors();
 887       return 100;
 888    }
 889
 890    // See if the help should be shown
 891    if (_config->FindB("help") == true ||
 892        _config->FindB("version") == true ||
 893        CmdL.FileSize() == 0)
 894    {
 895       ShowHelp(CmdL);
 896       return 0;
 897    }
 898
 899    // Setup the output streams
 900    c0out.rdbuf(cout.rdbuf());
 901    c1out.rdbuf(cout.rdbuf());
 902    c2out.rdbuf(cout.rdbuf());
 903    Quiet = _config->FindI("quiet",0);
 904    if (Quiet > 0)
 905       c0out.rdbuf(devnull.rdbuf());
 906    if (Quiet > 1)
 907       c1out.rdbuf(devnull.rdbuf());
 908
 909    // Match the operation
 910    CmdL.DispatchArg(Cmds);
 911
 912    if (_error->empty() == false)
 913    {
 914       bool Errors = _error->PendingError();
 915       _error->DumpErrors();
 916       return Errors == true?100:0;
 917    }
 918    return 0;
 919 }