ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.3 2001/05/29 04:39:31 jgg Exp $
   4 /* ######################################################################
   5
   6    apt-scanpackages - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-ftparchive.h"
  15 #endif
  16
  17 #include "apt-ftparchive.h"
  18
  19 #include <apt-pkg/error.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/cmndline.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <config.h>
  24 #include <apti18n.h>
  25 #include <algorithm>
  26
  27 #include <sys/time.h>
  28 #include <regex.h>
  29
  30 #include "contents.h"
  31 #include "multicompress.h"
  32 #include "writer.h"
  33                                                                         /*}}}*/
  34
  35 using namespace std;
  36 ostream c0out(0);
  37 ostream c1out(0);
  38 ostream c2out(0);
  39 ofstream devnull("/dev/null");
  40 unsigned Quiet = 0;
  41
  42 // struct PackageMap - List of all package files in the config file     /*{{{*/
  43 // ---------------------------------------------------------------------
  44 /* */
  45 struct PackageMap
  46 {
  47    // General Stuff
  48    string BaseDir;
  49    string InternalPrefix;
  50    string FLFile;
  51    string PkgExt;
  52    string SrcExt;
  53
  54    // Stuff for the Package File
  55    string PkgFile;
  56    string BinCacheDB;
  57    string BinOverride;
  58
  59    // Stuff for the Source File
  60    string SrcFile;
  61    string SrcOverride;
  62
  63    // Contents
  64    string Contents;
  65    string ContentsHead;
  66
  67    // Random things
  68    string Tag;
  69    string PkgCompress;
  70    string CntCompress;
  71    string SrcCompress;
  72    string PathPrefix;
  73    unsigned int DeLinkLimit;
  74    mode_t Permissions;
  75
  76    bool ContentsDone;
  77    bool PkgDone;
  78    bool SrcDone;
  79    time_t ContentsMTime;
  80
  81    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  82    {
  83       inline bool operator() (const PackageMap &x,const PackageMap &y)
  84       {return x.ContentsMTime < y.ContentsMTime;};
  85    };
  86
  87    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  88    {
  89       inline bool operator() (const PackageMap &x,const PackageMap &y)
  90       {return x.BinCacheDB < y.BinCacheDB;};
  91    };
  92
  93    void GetGeneral(Configuration &Setup,Configuration &Block);
  94    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
  95    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
  96    bool GenContents(Configuration &Setup,
  97                     vector<PackageMap>::iterator Begin,
  98                     vector<PackageMap>::iterator End,
  99                     unsigned long &Left);
 100
 101    PackageMap() : DeLinkLimit(0), Permissions(1), ContentsDone(false),
 102         PkgDone(false), SrcDone(false), ContentsMTime(0) {};
 103 };
 104                                                                         /*}}}*/
 105
 106 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 107 // ---------------------------------------------------------------------
 108 /* */
 109 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 110 {
 111    PathPrefix = Block.Find("PathPrefix");
 112
 113    if (Block.FindB("External-Links",true) == false)
 114       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 115    else
 116       DeLinkLimit = 0;
 117
 118    PkgCompress = Block.Find("Packages::Compress",
 119                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 120    CntCompress = Block.Find("Contents::Compress",
 121                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 122    SrcCompress = Block.Find("Sources::Compress",
 123                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 124
 125    SrcExt = Block.Find("Sources::Extensions",
 126                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 127    PkgExt = Block.Find("Packages::Extensions",
 128                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 129
 130    Permissions = Setup.FindI("Default::FileMode",0644);
 131
 132    if (FLFile.empty() == false)
 133       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 134
 135    if (Contents == " ")
 136       Contents= string();
 137 }
 138                                                                         /*}}}*/
 139 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 140 // ---------------------------------------------------------------------
 141 /* This generates the Package File described by this object. */
 142 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 143 {
 144    if (PkgFile.empty() == true)
 145       return true;
 146
 147    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 148    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 149    string CacheDir = Setup.FindDir("Dir::CacheDir");
 150
 151    struct timeval StartTime;
 152    gettimeofday(&StartTime,0);
 153
 154    PkgDone = true;
 155
 156    // Create a package writer object.
 157    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 158                            flCombine(OverrideDir,BinOverride));
 159    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 160       return _error->Error("Package extension list is too long");
 161    if (_error->PendingError() == true)
 162       return _error->Error("Error Processing directory %s",BaseDir.c_str());
 163
 164    Packages.PathPrefix = PathPrefix;
 165    Packages.DirStrip = ArchiveDir;
 166    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 167
 168    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 169    Packages.DeLinkLimit = DeLinkLimit;
 170
 171    // Create a compressor object
 172    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 173                       PkgCompress,Permissions);
 174    Packages.Output = Comp.Input;
 175    if (_error->PendingError() == true)
 176       return _error->Error("Error Processing directory %s",BaseDir.c_str());
 177
 178    c0out << ' ' << BaseDir << ":" << flush;
 179
 180    // Do recursive directory searching
 181    if (FLFile.empty() == true)
 182    {
 183       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 184          return false;
 185    }
 186    else
 187    {
 188       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 189          return false;
 190    }
 191
 192    Packages.Output = 0;      // Just in case
 193
 194    // Finish compressing
 195    unsigned long Size;
 196    if (Comp.Finalize(Size) == false)
 197    {
 198       c0out << endl;
 199       return _error->Error("Error Processing directory %s",BaseDir.c_str());
 200    }
 201
 202    if (Size != 0)
 203       c0out << " New "
 204              << SizeToStr(Size) << "B ";
 205    else
 206       c0out << ' ';
 207
 208    struct timeval NewTime;
 209    gettimeofday(&NewTime,0);
 210    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 211                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 212
 213    c0out << Packages.Stats.Packages << " files " <<
 214 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 215       SizeToStr(Packages.Stats.Bytes) << "B " <<
 216       TimeToStr((long)Delta) << endl;
 217
 218    Stats.Add(Packages.Stats);
 219    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 220
 221    return !_error->PendingError();
 222 }
 223                                                                         /*}}}*/
 224 // PackageMap::GenSources - Actually generate a Package file            /*{{{*/
 225 // ---------------------------------------------------------------------
 226 /* This generates the Sources File described by this object. */
 227 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 228 {
 229    if (SrcFile.empty() == true)
 230       return true;
 231
 232    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 233    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 234    string CacheDir = Setup.FindDir("Dir::CacheDir");
 235
 236    struct timeval StartTime;
 237    gettimeofday(&StartTime,0);
 238
 239    SrcDone = true;
 240
 241    // Create a package writer object.
 242    SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
 243                           flCombine(OverrideDir,SrcOverride));
 244    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 245       return _error->Error("Source extension list is too long");
 246    if (_error->PendingError() == true)
 247       return _error->Error("Error Processing directory %s",BaseDir.c_str());
 248
 249    Sources.PathPrefix = PathPrefix;
 250    Sources.DirStrip = ArchiveDir;
 251    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 252
 253    Sources.DeLinkLimit = DeLinkLimit;
 254    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 255
 256    // Create a compressor object
 257    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 258                       SrcCompress,Permissions);
 259    Sources.Output = Comp.Input;
 260    if (_error->PendingError() == true)
 261       return _error->Error("Error Processing directory %s",BaseDir.c_str());
 262
 263    c0out << ' ' << BaseDir << ":" << flush;
 264
 265    // Do recursive directory searching
 266    if (FLFile.empty() == true)
 267    {
 268       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 269          return false;
 270    }
 271    else
 272    {
 273       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 274          return false;
 275    }
 276    Sources.Output = 0;      // Just in case
 277
 278    // Finish compressing
 279    unsigned long Size;
 280    if (Comp.Finalize(Size) == false)
 281    {
 282       c0out << endl;
 283       return _error->Error("Error Processing directory %s",BaseDir.c_str());
 284    }
 285
 286    if (Size != 0)
 287       c0out << " New "
 288              << SizeToStr(Size) << "B ";
 289    else
 290       c0out << ' ';
 291
 292    struct timeval NewTime;
 293    gettimeofday(&NewTime,0);
 294    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 295                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 296
 297    c0out << Sources.Stats.Packages << " pkgs in " <<
 298       TimeToStr((long)Delta) << endl;
 299
 300    Stats.Add(Sources.Stats);
 301    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 302
 303    return !_error->PendingError();
 304 }
 305                                                                         /*}}}*/
 306 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 307 // ---------------------------------------------------------------------
 308 /* This generates the contents file partially described by this object.
 309    It searches the given iterator range for other package files that map
 310    into this contents file and includes their data as well when building. */
 311 bool PackageMap::GenContents(Configuration &Setup,
 312                              vector<PackageMap>::iterator Begin,
 313                              vector<PackageMap>::iterator End,
 314                              unsigned long &Left)
 315 {
 316    if (Contents.empty() == true)
 317       return true;
 318
 319    if (Left == 0)
 320       return true;
 321
 322    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 323    string CacheDir = Setup.FindDir("Dir::CacheDir");
 324    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 325
 326    struct timeval StartTime;
 327    gettimeofday(&StartTime,0);
 328
 329    // Create a package writer object.
 330    ContentsWriter Contents("");
 331    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 332       return _error->Error("Package extension list is too long");
 333    if (_error->PendingError() == true)
 334       return false;
 335
 336    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 337                       CntCompress,Permissions);
 338    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 339    Contents.Output = Comp.Input;
 340    if (_error->PendingError() == true)
 341       return false;
 342
 343    // Write the header out.
 344    if (ContentsHead.empty() == false)
 345    {
 346       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 347       if (_error->PendingError() == true)
 348          return false;
 349
 350       unsigned long Size = Head.Size();
 351       unsigned char Buf[4096];
 352       while (Size != 0)
 353       {
 354          unsigned long ToRead = Size;
 355          if (Size > sizeof(Buf))
 356             ToRead = sizeof(Buf);
 357
 358          if (Head.Read(Buf,ToRead) == false)
 359             return false;
 360
 361          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 362             return _error->Errno("fwrite","Error writing header to contents file");
 363
 364          Size -= ToRead;
 365       }
 366    }
 367
 368    /* Go over all the package file records and parse all the package
 369       files associated with this contents file into one great big honking
 370       memory structure, then dump the sorted version */
 371    c0out << ' ' << this->Contents << ":" << flush;
 372    for (vector<PackageMap>::iterator I = Begin; I != End; I++)
 373    {
 374       if (I->Contents != this->Contents)
 375          continue;
 376
 377       Contents.Prefix = ArchiveDir;
 378       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 379       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 380                             I->PkgCompress);
 381
 382       I->ContentsDone = true;
 383    }
 384
 385    Contents.Finish();
 386
 387    // Finish compressing
 388    unsigned long Size;
 389    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 390    {
 391       c0out << endl;
 392       return _error->Error("Error Processing Contents %s",
 393                            this->Contents.c_str());
 394    }
 395
 396    if (Size != 0)
 397    {
 398       c0out << " New " << SizeToStr(Size) << "B ";
 399       if (Left > Size)
 400          Left -= Size;
 401       else
 402          Left = 0;
 403    }
 404    else
 405       c0out << ' ';
 406
 407    struct timeval NewTime;
 408    gettimeofday(&NewTime,0);
 409    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 410                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 411
 412    c0out << Contents.Stats.Packages << " files " <<
 413       SizeToStr(Contents.Stats.Bytes) << "B " <<
 414       TimeToStr((long)Delta) << endl;
 415
 416    return true;
 417 }
 418                                                                         /*}}}*/
 419
 420 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 421 // ---------------------------------------------------------------------
 422 /* This populates the PkgList with all the possible permutations of the
 423    section/arch lists. */
 424 void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 425 {
 426    // Load the defaults
 427    string DDir = Setup.Find("TreeDefault::Directory",
 428                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 429    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 430                             "$(DIST)/$(SECTION)/source/");
 431    string DPkg = Setup.Find("TreeDefault::Packages",
 432                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 433    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 434                             "$(DIST)/$(SECTION)/");
 435    string DContents = Setup.Find("TreeDefault::Contents",
 436                             "$(DIST)/Contents-$(ARCH)");
 437    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 438    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 439                                "packages-$(ARCH).db");
 440    string DSources = Setup.Find("TreeDefault::Sources",
 441                                 "$(DIST)/$(SECTION)/source/Sources");
 442    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 443    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 444
 445    // Process 'tree' type sections
 446    const Configuration::Item *Top = Setup.Tree("tree");
 447    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 448    {
 449       Configuration Block(Top);
 450       string Dist = Top->Tag;
 451
 452       // Parse the sections
 453       string Tmp = Block.Find("Sections");
 454       const char *Sections = Tmp.c_str();
 455       string Section;
 456       while (ParseQuoteWord(Sections,Section) == true)
 457       {
 458          string Tmp2 = Block.Find("Architectures");
 459          string Arch;
 460          const char *Archs = Tmp2.c_str();
 461          while (ParseQuoteWord(Archs,Arch) == true)
 462          {
 463             struct SubstVar Vars[] = {{"$(DIST)",&Dist},
 464                                       {"$(SECTION)",&Section},
 465                                       {"$(ARCH)",&Arch},
 466                                       {}};
 467             PackageMap Itm;
 468
 469             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 470             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 471
 472             if (stringcasecmp(Arch,"source") == 0)
 473             {
 474                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 475                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 476                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 477                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 478                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 479             }
 480             else
 481             {
 482                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 483                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 484                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 485                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 486                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 487                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 488                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 489             }
 490
 491             Itm.GetGeneral(Setup,Block);
 492             PkgList.push_back(Itm);
 493          }
 494       }
 495
 496       Top = Top->Next;
 497    }
 498 }
 499                                                                         /*}}}*/
 500 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 501 // ---------------------------------------------------------------------
 502 /* */
 503 void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 504 {
 505    // Process 'bindirectory' type sections
 506    const Configuration::Item *Top = Setup.Tree("bindirectory");
 507    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 508    {
 509       Configuration Block(Top);
 510
 511       PackageMap Itm;
 512       Itm.PkgFile = Block.Find("Packages");
 513       Itm.SrcFile = Block.Find("Sources");
 514       Itm.BinCacheDB = Block.Find("BinCacheDB");
 515       Itm.BinOverride = Block.Find("BinOverride");
 516       Itm.SrcOverride = Block.Find("SrcOverride");
 517       Itm.BaseDir = Top->Tag;
 518       Itm.FLFile = Block.Find("FileList");
 519       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 520       Itm.Contents = Block.Find("Contents");
 521       Itm.ContentsHead = Block.Find("Contents::Header");
 522
 523       Itm.GetGeneral(Setup,Block);
 524       PkgList.push_back(Itm);
 525
 526       Top = Top->Next;
 527    }
 528 }
 529                                                                         /*}}}*/
 530
 531 // ShowHelp - Show the help text                                        /*{{{*/
 532 // ---------------------------------------------------------------------
 533 /* */
 534 bool ShowHelp(CommandLine &CmdL)
 535 {
 536    ioprintf(cout,_("%s %s for %s %s compiled on %s %s\n"),PACKAGE,VERSION,
 537             COMMON_OS,COMMON_CPU,__DATE__,__TIME__);
 538    if (_config->FindB("version") == true)
 539       return true;
 540
 541    cout <<
 542       "Usage: apt-ftparchive [options] command\n"
 543       "Commands: packges binarypath [overridefile [pathprefix]]\n"
 544       "          sources srcpath [overridefile [pathprefix]]\n"
 545       "          contents path\n"
 546       "          generate config [groups]\n"
 547       "          clean config\n"
 548       "\n"
 549       "apt-ftparchive generates index files for Debian archives. It supports\n"
 550       "many styles of generation from fully automated to functional replacements\n"
 551       "for dpkg-scanpackages and dpkg-scansources\n"
 552       "\n"
 553       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 554       "Package file contains the contents of all the control fields from\n"
 555       "each package as well as the MD5 hash and filesize. An override file\n"
 556       "is supported to force the value of Priority and Section.\n"
 557       "\n"
 558       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 559       "The --source-override option can be used to specify a src override file\n"
 560       "\n"
 561       "The 'packages' and 'sources' command should be run in the root of the\n"
 562       "tree. BinaryPath should point to the base of the recursive search and \n"
 563       "override file should contian the override flags. Pathprefix is\n"
 564       "appended to the filename fields if present. Example usage from the \n"
 565       "debian archive:\n"
 566       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 567       "               dists/potato/main/binary-i386/Packages\n"
 568       "\n"
 569       "Options:\n"
 570       "  -h    This help text\n"
 571       "  --md5 Control MD5 generation\n"
 572       "  -s=?  Source override file\n"
 573       "  -q    Quiet\n"
 574       "  -d=?  Select the optional caching database\n"
 575       "  --no-delink Enable delinking debug mode\n"
 576       "  --contents  Control contents file generation\n"
 577       "  -c=?  Read this configuration file\n"
 578       "  -o=?  Set an arbitary configuration option" << endl;
 579
 580    return true;
 581 }
 582                                                                         /*}}}*/
 583 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 584 // ---------------------------------------------------------------------
 585 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 586 bool SimpleGenPackages(CommandLine &CmdL)
 587 {
 588    if (CmdL.FileSize() < 2)
 589       return ShowHelp(CmdL);
 590
 591    string Override;
 592    if (CmdL.FileSize() >= 3)
 593       Override = CmdL.FileList[2];
 594
 595    // Create a package writer object.
 596    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 597                            Override);
 598    if (_error->PendingError() == true)
 599       return false;
 600
 601    if (CmdL.FileSize() >= 4)
 602       Packages.PathPrefix = CmdL.FileList[3];
 603
 604    // Do recursive directory searching
 605    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 606       return false;
 607
 608    return true;
 609 }
 610                                                                         /*}}}*/
 611 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 612 // ---------------------------------------------------------------------
 613 /* */
 614 bool SimpleGenContents(CommandLine &CmdL)
 615 {
 616    if (CmdL.FileSize() < 2)
 617       return ShowHelp(CmdL);
 618
 619    // Create a package writer object.
 620    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"));
 621    if (_error->PendingError() == true)
 622       return false;
 623
 624    // Do recursive directory searching
 625    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 626       return false;
 627
 628    Contents.Finish();
 629
 630    return true;
 631 }
 632                                                                         /*}}}*/
 633 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 634 // ---------------------------------------------------------------------
 635 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 636 bool SimpleGenSources(CommandLine &CmdL)
 637 {
 638    if (CmdL.FileSize() < 2)
 639       return ShowHelp(CmdL);
 640
 641    string Override;
 642    if (CmdL.FileSize() >= 3)
 643       Override = CmdL.FileList[2];
 644
 645    string SOverride;
 646    if (Override.empty() == false)
 647       SOverride = Override + ".src";
 648
 649    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 650                              SOverride.c_str());
 651
 652    // Create a package writer object.
 653    SourcesWriter Sources(Override,SOverride);
 654    if (_error->PendingError() == true)
 655       return false;
 656
 657    if (CmdL.FileSize() >= 4)
 658       Sources.PathPrefix = CmdL.FileList[3];
 659
 660    // Do recursive directory searching
 661    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 662       return false;
 663
 664    return true;
 665 }
 666                                                                         /*}}}*/
 667 // Generate - Full generate, using a config file                        /*{{{*/
 668 // ---------------------------------------------------------------------
 669 /* */
 670 bool Generate(CommandLine &CmdL)
 671 {
 672    struct CacheDB::Stats SrcStats;
 673    if (CmdL.FileSize() < 2)
 674       return ShowHelp(CmdL);
 675
 676    struct timeval StartTime;
 677    gettimeofday(&StartTime,0);
 678    struct CacheDB::Stats Stats;
 679
 680    // Read the configuration file.
 681    Configuration Setup;
 682    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 683       return false;
 684
 685    vector<PackageMap> PkgList;
 686    LoadTree(PkgList,Setup);
 687    LoadBinDir(PkgList,Setup);
 688
 689    // Sort by cache DB to improve IO locality.
 690    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 691
 692    // Generate packages
 693    if (CmdL.FileSize() <= 2)
 694    {
 695       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 696          if (I->GenPackages(Setup,Stats) == false)
 697             _error->DumpErrors();
 698       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 699          if (I->GenSources(Setup,SrcStats) == false)
 700             _error->DumpErrors();
 701    }
 702    else
 703    {
 704       // Make a choice list out of the package list..
 705       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 706       RxChoiceList *End = List;
 707       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 708       {
 709          End->UserData = &(*I);
 710          End->Str = I->BaseDir.c_str();
 711          End++;
 712
 713          End->UserData = &(*I);
 714          End->Str = I->Tag.c_str();
 715          End++;
 716       }
 717       End->Str = 0;
 718
 719       // Regex it
 720       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 721       {
 722          delete [] List;
 723          return _error->Error("No selections matched");
 724       }
 725       _error->DumpErrors();
 726
 727       // Do the generation for Packages
 728       for (End = List; End->Str != 0; End++)
 729       {
 730          if (End->Hit == false)
 731             continue;
 732
 733          PackageMap *I = (PackageMap *)End->UserData;
 734          if (I->PkgDone == true)
 735             continue;
 736          if (I->GenPackages(Setup,Stats) == false)
 737             _error->DumpErrors();
 738       }
 739
 740       // Do the generation for Sources
 741       for (End = List; End->Str != 0; End++)
 742       {
 743          if (End->Hit == false)
 744             continue;
 745
 746          PackageMap *I = (PackageMap *)End->UserData;
 747          if (I->SrcDone == true)
 748             continue;
 749          if (I->GenSources(Setup,SrcStats) == false)
 750             _error->DumpErrors();
 751       }
 752
 753       delete [] List;
 754    }
 755
 756    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 757       return true;
 758
 759    c1out << "Done Packages, Starting contents." << endl;
 760
 761    // Sort the contents file list by date
 762    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 763    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 764    {
 765       struct stat A;
 766       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 767                                  I->CntCompress,A) == false)
 768          time(&I->ContentsMTime);
 769       else
 770          I->ContentsMTime = A.st_mtime;
 771    }
 772    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 773
 774    /* Now for Contents.. The process here is to do a make-like dependency
 775       check. Each contents file is verified to be newer than the package files
 776       that describe the debs it indexes. Since the package files contain
 777       hashes of the .debs this means they have not changed either so the
 778       contents must be up to date. */
 779    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 780    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 781    {
 782       // This record is not relevent
 783       if (I->ContentsDone == true ||
 784           I->Contents.empty() == true)
 785          continue;
 786
 787       // Do not do everything if the user specified sections.
 788       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 789          continue;
 790
 791       struct stat A,B;
 792       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 793       {
 794          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 795          {
 796             _error->Warning("Some files are missing in the package file group `%s'",I->PkgFile.c_str());
 797             continue;
 798          }
 799
 800          if (A.st_mtime > B.st_mtime)
 801             continue;
 802       }
 803
 804       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 805                          MaxContentsChange) == false)
 806          _error->DumpErrors();
 807
 808       // Hit the limit?
 809       if (MaxContentsChange == 0)
 810       {
 811          c1out << "Hit contents update byte limit" << endl;
 812          break;
 813       }
 814    }
 815
 816    struct timeval NewTime;
 817    gettimeofday(&NewTime,0);
 818    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 819                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 820    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 821          << " archives. Took " << TimeToStr((long)Delta) << endl;
 822
 823    return true;
 824 }
 825                                                                         /*}}}*/
 826 // Clean - Clean out the databases                                      /*{{{*/
 827 // ---------------------------------------------------------------------
 828 /* */
 829 bool Clean(CommandLine &CmdL)
 830 {
 831    if (CmdL.FileSize() != 2)
 832       return ShowHelp(CmdL);
 833
 834    // Read the configuration file.
 835    Configuration Setup;
 836    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 837       return false;
 838
 839    vector<PackageMap> PkgList;
 840    LoadTree(PkgList,Setup);
 841    LoadBinDir(PkgList,Setup);
 842
 843    // Sort by cache DB to improve IO locality.
 844    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 845
 846    string CacheDir = Setup.FindDir("Dir::CacheDir");
 847
 848    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 849    {
 850       c0out << I->BinCacheDB << endl;
 851       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 852       if (DB.Clean() == false)
 853          _error->DumpErrors();
 854
 855       string CacheDB = I->BinCacheDB;
 856       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
 857    }
 858
 859    return true;
 860 }
 861                                                                         /*}}}*/
 862
 863 int main(int argc, const char *argv[])
 864 {
 865    CommandLine::Args Args[] = {
 866       {'h',"help","help",0},
 867       {0,"md5","APT::FTPArchive::MD5",0},
 868       {'v',"version","version",0},
 869       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 870       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 871       {'q',"quiet","quiet",CommandLine::IntLevel},
 872       {'q',"silent","quiet",CommandLine::IntLevel},
 873       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 874       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 875       {0,"contents","APT::FTPArchive::Contents",0},
 876       {'c',"config-file",0,CommandLine::ConfigFile},
 877       {'o',"option",0,CommandLine::ArbItem},
 878       {0,0,0,0}};
 879    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 880                                    {"contents",&SimpleGenContents},
 881                                    {"sources",&SimpleGenSources},
 882                                    {"generate",&Generate},
 883                                    {"clean",&Clean},
 884                                    {"help",&ShowHelp},
 885                                    {0,0}};
 886
 887    // Parse the command line and initialize the package library
 888    CommandLine CmdL(Args,_config);
 889    if (CmdL.Parse(argc,argv) == false)
 890    {
 891       _error->DumpErrors();
 892       return 100;
 893    }
 894
 895    // See if the help should be shown
 896    if (_config->FindB("help") == true ||
 897        _config->FindB("version") == true ||
 898        CmdL.FileSize() == 0)
 899    {
 900       ShowHelp(CmdL);
 901       return 0;
 902    }
 903
 904    // Setup the output streams
 905    c0out.rdbuf(cout.rdbuf());
 906    c1out.rdbuf(cout.rdbuf());
 907    c2out.rdbuf(cout.rdbuf());
 908    Quiet = _config->FindI("quiet",0);
 909    if (Quiet > 0)
 910       c0out.rdbuf(devnull.rdbuf());
 911    if (Quiet > 1)
 912       c1out.rdbuf(devnull.rdbuf());
 913
 914    // Match the operation
 915    CmdL.DispatchArg(Cmds);
 916
 917    if (_error->empty() == false)
 918    {
 919       bool Errors = _error->PendingError();
 920       _error->DumpErrors();
 921       return Errors == true?100:0;
 922    }
 923    return 0;
 924 }