ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-scanpackages - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-ftparchive.h"
  15 #endif
  16
  17 #include "apt-ftparchive.h"
  18
  19 #include <apt-pkg/error.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/cmndline.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <config.h>
  24 #include <apti18n.h>
  25 #include <algorithm>
  26
  27 #include <climits>
  28 #include <sys/time.h>
  29 #include <regex.h>
  30
  31 #include "contents.h"
  32 #include "multicompress.h"
  33 #include "writer.h"
  34                                                                         /*}}}*/
  35
  36 using namespace std;
  37 ostream c0out(0);
  38 ostream c1out(0);
  39 ostream c2out(0);
  40 ofstream devnull("/dev/null");
  41 unsigned Quiet = 0;
  42
  43 // struct PackageMap - List of all package files in the config file     /*{{{*/
  44 // ---------------------------------------------------------------------
  45 /* */
  46 struct PackageMap
  47 {
  48    // General Stuff
  49    string BaseDir;
  50    string InternalPrefix;
  51    string FLFile;
  52    string PkgExt;
  53    string SrcExt;
  54
  55    // Stuff for the Package File
  56    string PkgFile;
  57    string BinCacheDB;
  58    string BinOverride;
  59    string ExtraOverride;
  60
  61    // We generate for this given arch
  62    string Arch;
  63
  64    // Stuff for the Source File
  65    string SrcFile;
  66    string SrcOverride;
  67    string SrcExtraOverride;
  68
  69    // Contents
  70    string Contents;
  71    string ContentsHead;
  72
  73    // Random things
  74    string Tag;
  75    string PkgCompress;
  76    string CntCompress;
  77    string SrcCompress;
  78    string PathPrefix;
  79    unsigned int DeLinkLimit;
  80    mode_t Permissions;
  81
  82    bool ContentsDone;
  83    bool PkgDone;
  84    bool SrcDone;
  85    time_t ContentsMTime;
  86
  87    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  88    {
  89       inline bool operator() (const PackageMap &x,const PackageMap &y)
  90       {return x.ContentsMTime < y.ContentsMTime;};
  91    };
  92
  93    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  94    {
  95       inline bool operator() (const PackageMap &x,const PackageMap &y)
  96       {return x.BinCacheDB < y.BinCacheDB;};
  97    };
  98
  99    void GetGeneral(Configuration &Setup,Configuration &Block);
 100    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 101    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 102    bool GenContents(Configuration &Setup,
 103                     vector<PackageMap>::iterator Begin,
 104                     vector<PackageMap>::iterator End,
 105                     unsigned long &Left);
 106
 107    PackageMap() : DeLinkLimit(0), Permissions(1), ContentsDone(false),
 108         PkgDone(false), SrcDone(false), ContentsMTime(0) {};
 109 };
 110                                                                         /*}}}*/
 111
 112 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 113 // ---------------------------------------------------------------------
 114 /* */
 115 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 116 {
 117    PathPrefix = Block.Find("PathPrefix");
 118
 119    if (Block.FindB("External-Links",true) == false)
 120       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 121    else
 122       DeLinkLimit = 0;
 123
 124    PkgCompress = Block.Find("Packages::Compress",
 125                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 126    CntCompress = Block.Find("Contents::Compress",
 127                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 128    SrcCompress = Block.Find("Sources::Compress",
 129                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 130
 131    SrcExt = Block.Find("Sources::Extensions",
 132                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 133    PkgExt = Block.Find("Packages::Extensions",
 134                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 135
 136    Permissions = Setup.FindI("Default::FileMode",0644);
 137
 138    if (FLFile.empty() == false)
 139       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 140
 141    if (Contents == " ")
 142       Contents= string();
 143 }
 144                                                                         /*}}}*/
 145 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 146 // ---------------------------------------------------------------------
 147 /* This generates the Package File described by this object. */
 148 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 149 {
 150    if (PkgFile.empty() == true)
 151       return true;
 152
 153    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 154    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 155    string CacheDir = Setup.FindDir("Dir::CacheDir");
 156
 157    struct timeval StartTime;
 158    gettimeofday(&StartTime,0);
 159
 160    PkgDone = true;
 161
 162    // Create a package writer object.
 163    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 164                            flCombine(OverrideDir,BinOverride),
 165                            flCombine(OverrideDir,ExtraOverride),
 166                            Arch);
 167    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 168       return _error->Error(_("Package extension list is too long"));
 169    if (_error->PendingError() == true)
 170       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 171
 172    Packages.PathPrefix = PathPrefix;
 173    Packages.DirStrip = ArchiveDir;
 174    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 175
 176    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 177    Packages.DeLinkLimit = DeLinkLimit;
 178
 179    // Create a compressor object
 180    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 181                       PkgCompress,Permissions);
 182    Packages.Output = Comp.Input;
 183    if (_error->PendingError() == true)
 184       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 185
 186    c0out << ' ' << BaseDir << ":" << flush;
 187
 188    // Do recursive directory searching
 189    if (FLFile.empty() == true)
 190    {
 191       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 192          return false;
 193    }
 194    else
 195    {
 196       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 197          return false;
 198    }
 199
 200    Packages.Output = 0;      // Just in case
 201
 202    // Finish compressing
 203    unsigned long Size;
 204    if (Comp.Finalize(Size) == false)
 205    {
 206       c0out << endl;
 207       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 208    }
 209
 210    if (Size != 0)
 211       c0out << " New "
 212              << SizeToStr(Size) << "B ";
 213    else
 214       c0out << ' ';
 215
 216    struct timeval NewTime;
 217    gettimeofday(&NewTime,0);
 218    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 219                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 220
 221    c0out << Packages.Stats.Packages << " files " <<
 222 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 223       SizeToStr(Packages.Stats.Bytes) << "B " <<
 224       TimeToStr((long)Delta) << endl;
 225
 226    Stats.Add(Packages.Stats);
 227    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 228
 229    return !_error->PendingError();
 230 }
 231
 232                                                                         /*}}}*/
 233 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 234 // ---------------------------------------------------------------------
 235 /* This generates the Sources File described by this object. */
 236 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 237 {
 238    if (SrcFile.empty() == true)
 239       return true;
 240
 241    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 242    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 243    string CacheDir = Setup.FindDir("Dir::CacheDir");
 244
 245    struct timeval StartTime;
 246    gettimeofday(&StartTime,0);
 247
 248    SrcDone = true;
 249
 250    // Create a package writer object.
 251    SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
 252                          flCombine(OverrideDir,SrcOverride),
 253                          flCombine(OverrideDir,SrcExtraOverride));
 254    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 255       return _error->Error(_("Source extension list is too long"));
 256    if (_error->PendingError() == true)
 257       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 258
 259    Sources.PathPrefix = PathPrefix;
 260    Sources.DirStrip = ArchiveDir;
 261    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 262
 263    Sources.DeLinkLimit = DeLinkLimit;
 264    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 265
 266    // Create a compressor object
 267    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 268                       SrcCompress,Permissions);
 269    Sources.Output = Comp.Input;
 270    if (_error->PendingError() == true)
 271       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 272
 273    c0out << ' ' << BaseDir << ":" << flush;
 274
 275    // Do recursive directory searching
 276    if (FLFile.empty() == true)
 277    {
 278       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 279          return false;
 280    }
 281    else
 282    {
 283       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 284          return false;
 285    }
 286    Sources.Output = 0;      // Just in case
 287
 288    // Finish compressing
 289    unsigned long Size;
 290    if (Comp.Finalize(Size) == false)
 291    {
 292       c0out << endl;
 293       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 294    }
 295
 296    if (Size != 0)
 297       c0out << " New "
 298              << SizeToStr(Size) << "B ";
 299    else
 300       c0out << ' ';
 301
 302    struct timeval NewTime;
 303    gettimeofday(&NewTime,0);
 304    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 305                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 306
 307    c0out << Sources.Stats.Packages << " pkgs in " <<
 308       TimeToStr((long)Delta) << endl;
 309
 310    Stats.Add(Sources.Stats);
 311    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 312
 313    return !_error->PendingError();
 314 }
 315                                                                         /*}}}*/
 316 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 317 // ---------------------------------------------------------------------
 318 /* This generates the contents file partially described by this object.
 319    It searches the given iterator range for other package files that map
 320    into this contents file and includes their data as well when building. */
 321 bool PackageMap::GenContents(Configuration &Setup,
 322                              vector<PackageMap>::iterator Begin,
 323                              vector<PackageMap>::iterator End,
 324                              unsigned long &Left)
 325 {
 326    if (Contents.empty() == true)
 327       return true;
 328
 329    if (Left == 0)
 330       return true;
 331
 332    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 333    string CacheDir = Setup.FindDir("Dir::CacheDir");
 334    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 335
 336    struct timeval StartTime;
 337    gettimeofday(&StartTime,0);
 338
 339    // Create a package writer object.
 340    ContentsWriter Contents("");
 341    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 342       return _error->Error(_("Package extension list is too long"));
 343    if (_error->PendingError() == true)
 344       return false;
 345
 346    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 347                       CntCompress,Permissions);
 348    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 349    Contents.Output = Comp.Input;
 350    if (_error->PendingError() == true)
 351       return false;
 352
 353    // Write the header out.
 354    if (ContentsHead.empty() == false)
 355    {
 356       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 357       if (_error->PendingError() == true)
 358          return false;
 359
 360       unsigned long Size = Head.Size();
 361       unsigned char Buf[4096];
 362       while (Size != 0)
 363       {
 364          unsigned long ToRead = Size;
 365          if (Size > sizeof(Buf))
 366             ToRead = sizeof(Buf);
 367
 368          if (Head.Read(Buf,ToRead) == false)
 369             return false;
 370
 371          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 372             return _error->Errno("fwrite",_("Error writing header to contents file"));
 373
 374          Size -= ToRead;
 375       }
 376    }
 377
 378    /* Go over all the package file records and parse all the package
 379       files associated with this contents file into one great big honking
 380       memory structure, then dump the sorted version */
 381    c0out << ' ' << this->Contents << ":" << flush;
 382    for (vector<PackageMap>::iterator I = Begin; I != End; I++)
 383    {
 384       if (I->Contents != this->Contents)
 385          continue;
 386
 387       Contents.Prefix = ArchiveDir;
 388       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 389       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 390                             I->PkgCompress);
 391
 392       I->ContentsDone = true;
 393    }
 394
 395    Contents.Finish();
 396
 397    // Finish compressing
 398    unsigned long Size;
 399    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 400    {
 401       c0out << endl;
 402       return _error->Error(_("Error processing contents %s"),
 403                            this->Contents.c_str());
 404    }
 405
 406    if (Size != 0)
 407    {
 408       c0out << " New " << SizeToStr(Size) << "B ";
 409       if (Left > Size)
 410          Left -= Size;
 411       else
 412          Left = 0;
 413    }
 414    else
 415       c0out << ' ';
 416
 417    struct timeval NewTime;
 418    gettimeofday(&NewTime,0);
 419    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 420                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 421
 422    c0out << Contents.Stats.Packages << " files " <<
 423       SizeToStr(Contents.Stats.Bytes) << "B " <<
 424       TimeToStr((long)Delta) << endl;
 425
 426    return true;
 427 }
 428                                                                         /*}}}*/
 429
 430 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 431 // ---------------------------------------------------------------------
 432 /* This populates the PkgList with all the possible permutations of the
 433    section/arch lists. */
 434 void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 435 {
 436    // Load the defaults
 437    string DDir = Setup.Find("TreeDefault::Directory",
 438                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 439    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 440                             "$(DIST)/$(SECTION)/source/");
 441    string DPkg = Setup.Find("TreeDefault::Packages",
 442                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 443    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 444                             "$(DIST)/$(SECTION)/");
 445    string DContents = Setup.Find("TreeDefault::Contents",
 446                             "$(DIST)/Contents-$(ARCH)");
 447    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 448    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 449                                "packages-$(ARCH).db");
 450    string DSources = Setup.Find("TreeDefault::Sources",
 451                                 "$(DIST)/$(SECTION)/source/Sources");
 452    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 453    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 454
 455    // Process 'tree' type sections
 456    const Configuration::Item *Top = Setup.Tree("tree");
 457    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 458    {
 459       Configuration Block(Top);
 460       string Dist = Top->Tag;
 461
 462       // Parse the sections
 463       string Tmp = Block.Find("Sections");
 464       const char *Sections = Tmp.c_str();
 465       string Section;
 466       while (ParseQuoteWord(Sections,Section) == true)
 467       {
 468          string Tmp2 = Block.Find("Architectures");
 469          string Arch;
 470          const char *Archs = Tmp2.c_str();
 471          while (ParseQuoteWord(Archs,Arch) == true)
 472          {
 473             struct SubstVar Vars[] = {{"$(DIST)",&Dist},
 474                                       {"$(SECTION)",&Section},
 475                                       {"$(ARCH)",&Arch},
 476                                       {}};
 477             PackageMap Itm;
 478
 479             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 480             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 481
 482             if (stringcasecmp(Arch,"source") == 0)
 483             {
 484                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 485                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 486                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 487                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 488                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 489                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 490             }
 491             else
 492             {
 493                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 494                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 495                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 496                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 497                Itm.Arch = Arch;
 498                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 499                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 500                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 501                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 502             }
 503
 504             Itm.GetGeneral(Setup,Block);
 505             PkgList.push_back(Itm);
 506          }
 507       }
 508
 509       Top = Top->Next;
 510    }
 511 }
 512                                                                         /*}}}*/
 513 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 514 // ---------------------------------------------------------------------
 515 /* */
 516 void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 517 {
 518    // Process 'bindirectory' type sections
 519    const Configuration::Item *Top = Setup.Tree("bindirectory");
 520    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 521    {
 522       Configuration Block(Top);
 523
 524       PackageMap Itm;
 525       Itm.PkgFile = Block.Find("Packages");
 526       Itm.SrcFile = Block.Find("Sources");
 527       Itm.BinCacheDB = Block.Find("BinCacheDB");
 528       Itm.BinOverride = Block.Find("BinOverride");
 529       Itm.ExtraOverride = Block.Find("ExtraOverride");
 530       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 531       Itm.SrcOverride = Block.Find("SrcOverride");
 532       Itm.BaseDir = Top->Tag;
 533       Itm.FLFile = Block.Find("FileList");
 534       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 535       Itm.Contents = Block.Find("Contents");
 536       Itm.ContentsHead = Block.Find("Contents::Header");
 537
 538       Itm.GetGeneral(Setup,Block);
 539       PkgList.push_back(Itm);
 540
 541       Top = Top->Next;
 542    }
 543 }
 544                                                                         /*}}}*/
 545
 546 // ShowHelp - Show the help text                                        /*{{{*/
 547 // ---------------------------------------------------------------------
 548 /* */
 549 bool ShowHelp(CommandLine &CmdL)
 550 {
 551    ioprintf(cout,_("%s %s for %s %s compiled on %s %s\n"),PACKAGE,VERSION,
 552             COMMON_OS,COMMON_CPU,__DATE__,__TIME__);
 553    if (_config->FindB("version") == true)
 554       return true;
 555
 556    cout <<
 557     _("Usage: apt-ftparchive [options] command\n"
 558       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 559       "          sources srcpath [overridefile [pathprefix]]\n"
 560       "          contents path\n"
 561       "          release path\n"
 562       "          generate config [groups]\n"
 563       "          clean config\n"
 564       "\n"
 565       "apt-ftparchive generates index files for Debian archives. It supports\n"
 566       "many styles of generation from fully automated to functional replacements\n"
 567       "for dpkg-scanpackages and dpkg-scansources\n"
 568       "\n"
 569       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 570       "Package file contains the contents of all the control fields from\n"
 571       "each package as well as the MD5 hash and filesize. An override file\n"
 572       "is supported to force the value of Priority and Section.\n"
 573       "\n"
 574       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 575       "The --source-override option can be used to specify a src override file\n"
 576       "\n"
 577       "The 'packages' and 'sources' command should be run in the root of the\n"
 578       "tree. BinaryPath should point to the base of the recursive search and \n"
 579       "override file should contain the override flags. Pathprefix is\n"
 580       "appended to the filename fields if present. Example usage from the \n"
 581       "Debian archive:\n"
 582       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 583       "               dists/potato/main/binary-i386/Packages\n"
 584       "\n"
 585       "Options:\n"
 586       "  -h    This help text\n"
 587       "  --md5 Control MD5 generation\n"
 588       "  -s=?  Source override file\n"
 589       "  -q    Quiet\n"
 590       "  -d=?  Select the optional caching database\n"
 591       "  --no-delink Enable delinking debug mode\n"
 592       "  --contents  Control contents file generation\n"
 593       "  -c=?  Read this configuration file\n"
 594       "  -o=?  Set an arbitrary configuration option") << endl;
 595
 596    return true;
 597 }
 598                                                                         /*}}}*/
 599 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 600 // ---------------------------------------------------------------------
 601 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 602 bool SimpleGenPackages(CommandLine &CmdL)
 603 {
 604    if (CmdL.FileSize() < 2)
 605       return ShowHelp(CmdL);
 606
 607    string Override;
 608    if (CmdL.FileSize() >= 3)
 609       Override = CmdL.FileList[2];
 610
 611    // Create a package writer object.
 612    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 613                            Override, "");
 614    if (_error->PendingError() == true)
 615       return false;
 616
 617    if (CmdL.FileSize() >= 4)
 618       Packages.PathPrefix = CmdL.FileList[3];
 619
 620    // Do recursive directory searching
 621    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 622       return false;
 623
 624    return true;
 625 }
 626                                                                         /*}}}*/
 627 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 628 // ---------------------------------------------------------------------
 629 /* */
 630 bool SimpleGenContents(CommandLine &CmdL)
 631 {
 632    if (CmdL.FileSize() < 2)
 633       return ShowHelp(CmdL);
 634
 635    // Create a package writer object.
 636    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"));
 637    if (_error->PendingError() == true)
 638       return false;
 639
 640    // Do recursive directory searching
 641    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 642       return false;
 643
 644    Contents.Finish();
 645
 646    return true;
 647 }
 648                                                                         /*}}}*/
 649 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 650 // ---------------------------------------------------------------------
 651 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 652 bool SimpleGenSources(CommandLine &CmdL)
 653 {
 654    if (CmdL.FileSize() < 2)
 655       return ShowHelp(CmdL);
 656
 657    string Override;
 658    if (CmdL.FileSize() >= 3)
 659       Override = CmdL.FileList[2];
 660
 661    string SOverride;
 662    if (Override.empty() == false)
 663       SOverride = Override + ".src";
 664
 665    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 666                              SOverride.c_str());
 667
 668    // Create a package writer object.
 669    SourcesWriter Sources(Override,SOverride);
 670    if (_error->PendingError() == true)
 671       return false;
 672
 673    if (CmdL.FileSize() >= 4)
 674       Sources.PathPrefix = CmdL.FileList[3];
 675
 676    // Do recursive directory searching
 677    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 678       return false;
 679
 680    return true;
 681 }
 682                                                                         /*}}}*/
 683 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 684 // ---------------------------------------------------------------------
 685 bool SimpleGenRelease(CommandLine &CmdL)
 686 {
 687    if (CmdL.FileSize() < 2)
 688       return ShowHelp(CmdL);
 689
 690    string Dir = CmdL.FileList[1];
 691
 692    ReleaseWriter Release("");
 693    Release.DirStrip = Dir;
 694
 695    if (_error->PendingError() == true)
 696       return false;
 697
 698    if (Release.RecursiveScan(Dir) == false)
 699       return false;
 700
 701    Release.Finish();
 702
 703    return true;
 704 }
 705
 706                                                                         /*}}}*/
 707 // Generate - Full generate, using a config file                        /*{{{*/
 708 // ---------------------------------------------------------------------
 709 /* */
 710 bool Generate(CommandLine &CmdL)
 711 {
 712    struct CacheDB::Stats SrcStats;
 713    if (CmdL.FileSize() < 2)
 714       return ShowHelp(CmdL);
 715
 716    struct timeval StartTime;
 717    gettimeofday(&StartTime,0);
 718    struct CacheDB::Stats Stats;
 719
 720    // Read the configuration file.
 721    Configuration Setup;
 722    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 723       return false;
 724
 725    vector<PackageMap> PkgList;
 726    LoadTree(PkgList,Setup);
 727    LoadBinDir(PkgList,Setup);
 728
 729    // Sort by cache DB to improve IO locality.
 730    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 731
 732    // Generate packages
 733    if (CmdL.FileSize() <= 2)
 734    {
 735       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 736          if (I->GenPackages(Setup,Stats) == false)
 737             _error->DumpErrors();
 738       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 739          if (I->GenSources(Setup,SrcStats) == false)
 740             _error->DumpErrors();
 741    }
 742    else
 743    {
 744       // Make a choice list out of the package list..
 745       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 746       RxChoiceList *End = List;
 747       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 748       {
 749          End->UserData = &(*I);
 750          End->Str = I->BaseDir.c_str();
 751          End++;
 752
 753          End->UserData = &(*I);
 754          End->Str = I->Tag.c_str();
 755          End++;
 756       }
 757       End->Str = 0;
 758
 759       // Regex it
 760       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 761       {
 762          delete [] List;
 763          return _error->Error(_("No selections matched"));
 764       }
 765       _error->DumpErrors();
 766
 767       // Do the generation for Packages
 768       for (End = List; End->Str != 0; End++)
 769       {
 770          if (End->Hit == false)
 771             continue;
 772
 773          PackageMap *I = (PackageMap *)End->UserData;
 774          if (I->PkgDone == true)
 775             continue;
 776          if (I->GenPackages(Setup,Stats) == false)
 777             _error->DumpErrors();
 778       }
 779
 780       // Do the generation for Sources
 781       for (End = List; End->Str != 0; End++)
 782       {
 783          if (End->Hit == false)
 784             continue;
 785
 786          PackageMap *I = (PackageMap *)End->UserData;
 787          if (I->SrcDone == true)
 788             continue;
 789          if (I->GenSources(Setup,SrcStats) == false)
 790             _error->DumpErrors();
 791       }
 792
 793       delete [] List;
 794    }
 795
 796    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 797       return true;
 798
 799    c1out << "Done Packages, Starting contents." << endl;
 800
 801    // Sort the contents file list by date
 802    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 803    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 804    {
 805       struct stat A;
 806       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 807                                  I->CntCompress,A) == false)
 808          time(&I->ContentsMTime);
 809       else
 810          I->ContentsMTime = A.st_mtime;
 811    }
 812    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 813
 814    /* Now for Contents.. The process here is to do a make-like dependency
 815       check. Each contents file is verified to be newer than the package files
 816       that describe the debs it indexes. Since the package files contain
 817       hashes of the .debs this means they have not changed either so the
 818       contents must be up to date. */
 819    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 820    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 821    {
 822       // This record is not relevent
 823       if (I->ContentsDone == true ||
 824           I->Contents.empty() == true)
 825          continue;
 826
 827       // Do not do everything if the user specified sections.
 828       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 829          continue;
 830
 831       struct stat A,B;
 832       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 833       {
 834          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 835          {
 836             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 837             continue;
 838          }
 839
 840          if (A.st_mtime > B.st_mtime)
 841             continue;
 842       }
 843
 844       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 845                          MaxContentsChange) == false)
 846          _error->DumpErrors();
 847
 848       // Hit the limit?
 849       if (MaxContentsChange == 0)
 850       {
 851          c1out << "Hit contents update byte limit" << endl;
 852          break;
 853       }
 854    }
 855
 856    struct timeval NewTime;
 857    gettimeofday(&NewTime,0);
 858    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 859                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 860    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 861          << " archives. Took " << TimeToStr((long)Delta) << endl;
 862
 863    return true;
 864 }
 865                                                                         /*}}}*/
 866 // Clean - Clean out the databases                                      /*{{{*/
 867 // ---------------------------------------------------------------------
 868 /* */
 869 bool Clean(CommandLine &CmdL)
 870 {
 871    if (CmdL.FileSize() != 2)
 872       return ShowHelp(CmdL);
 873
 874    // Read the configuration file.
 875    Configuration Setup;
 876    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 877       return false;
 878
 879    vector<PackageMap> PkgList;
 880    LoadTree(PkgList,Setup);
 881    LoadBinDir(PkgList,Setup);
 882
 883    // Sort by cache DB to improve IO locality.
 884    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 885
 886    string CacheDir = Setup.FindDir("Dir::CacheDir");
 887
 888    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 889    {
 890       c0out << I->BinCacheDB << endl;
 891       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 892       if (DB.Clean() == false)
 893          _error->DumpErrors();
 894
 895       string CacheDB = I->BinCacheDB;
 896       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
 897    }
 898
 899    return true;
 900 }
 901                                                                         /*}}}*/
 902
 903 int main(int argc, const char *argv[])
 904 {
 905    CommandLine::Args Args[] = {
 906       {'h',"help","help",0},
 907       {0,"md5","APT::FTPArchive::MD5",0},
 908       {'v',"version","version",0},
 909       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 910       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 911       {'q',"quiet","quiet",CommandLine::IntLevel},
 912       {'q',"silent","quiet",CommandLine::IntLevel},
 913       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 914       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 915       {0,"contents","APT::FTPArchive::Contents",0},
 916       {'c',"config-file",0,CommandLine::ConfigFile},
 917       {'o',"option",0,CommandLine::ArbItem},
 918       {0,0,0,0}};
 919    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 920                                    {"contents",&SimpleGenContents},
 921                                    {"sources",&SimpleGenSources},
 922                                    {"release",&SimpleGenRelease},
 923                                    {"generate",&Generate},
 924                                    {"clean",&Clean},
 925                                    {"help",&ShowHelp},
 926                                    {0,0}};
 927
 928    // Parse the command line and initialize the package library
 929    CommandLine CmdL(Args,_config);
 930    if (CmdL.Parse(argc,argv) == false)
 931    {
 932       _error->DumpErrors();
 933       return 100;
 934    }
 935
 936    // See if the help should be shown
 937    if (_config->FindB("help") == true ||
 938        _config->FindB("version") == true ||
 939        CmdL.FileSize() == 0)
 940    {
 941       ShowHelp(CmdL);
 942       return 0;
 943    }
 944
 945    // Setup the output streams
 946    c0out.rdbuf(clog.rdbuf());
 947    c1out.rdbuf(clog.rdbuf());
 948    c2out.rdbuf(clog.rdbuf());
 949    Quiet = _config->FindI("quiet",0);
 950    if (Quiet > 0)
 951       c0out.rdbuf(devnull.rdbuf());
 952    if (Quiet > 1)
 953       c1out.rdbuf(devnull.rdbuf());
 954
 955    // Match the operation
 956    CmdL.DispatchArg(Cmds);
 957
 958    if (_error->empty() == false)
 959    {
 960       bool Errors = _error->PendingError();
 961       _error->DumpErrors();
 962       return Errors == true?100:0;
 963    }
 964    return 0;
 965 }