ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.12 2004/01/02 21:48:13 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-scanpackages - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-ftparchive.h"
  15 #endif
  16
  17 #include "apt-ftparchive.h"
  18
  19 #include <apt-pkg/error.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/cmndline.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <config.h>
  24 #include <apti18n.h>
  25 #include <algorithm>
  26
  27 #include <sys/time.h>
  28 #include <regex.h>
  29
  30 #include "contents.h"
  31 #include "multicompress.h"
  32 #include "writer.h"
  33                                                                         /*}}}*/
  34
  35 using namespace std;
  36 ostream c0out(0);
  37 ostream c1out(0);
  38 ostream c2out(0);
  39 ofstream devnull("/dev/null");
  40 unsigned Quiet = 0;
  41
  42 // struct PackageMap - List of all package files in the config file     /*{{{*/
  43 // ---------------------------------------------------------------------
  44 /* */
  45 struct PackageMap
  46 {
  47    // General Stuff
  48    string BaseDir;
  49    string InternalPrefix;
  50    string FLFile;
  51    string PkgExt;
  52    string SrcExt;
  53
  54    // Stuff for the Package File
  55    string PkgFile;
  56    string BinCacheDB;
  57    string BinOverride;
  58    string ExtraOverride;
  59
  60    // Stuff for the Source File
  61    string SrcFile;
  62    string SrcOverride;
  63    string SrcExtraOverride;
  64
  65    // Contents
  66    string Contents;
  67    string ContentsHead;
  68
  69    // Random things
  70    string Tag;
  71    string PkgCompress;
  72    string CntCompress;
  73    string SrcCompress;
  74    string PathPrefix;
  75    unsigned int DeLinkLimit;
  76    mode_t Permissions;
  77
  78    bool ContentsDone;
  79    bool PkgDone;
  80    bool SrcDone;
  81    time_t ContentsMTime;
  82
  83    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  84    {
  85       inline bool operator() (const PackageMap &x,const PackageMap &y)
  86       {return x.ContentsMTime < y.ContentsMTime;};
  87    };
  88
  89    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  90    {
  91       inline bool operator() (const PackageMap &x,const PackageMap &y)
  92       {return x.BinCacheDB < y.BinCacheDB;};
  93    };
  94
  95    void GetGeneral(Configuration &Setup,Configuration &Block);
  96    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
  97    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
  98    bool GenContents(Configuration &Setup,
  99                     vector<PackageMap>::iterator Begin,
 100                     vector<PackageMap>::iterator End,
 101                     unsigned long &Left);
 102
 103    PackageMap() : DeLinkLimit(0), Permissions(1), ContentsDone(false),
 104         PkgDone(false), SrcDone(false), ContentsMTime(0) {};
 105 };
 106                                                                         /*}}}*/
 107
 108 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 109 // ---------------------------------------------------------------------
 110 /* */
 111 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 112 {
 113    PathPrefix = Block.Find("PathPrefix");
 114
 115    if (Block.FindB("External-Links",true) == false)
 116       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 117    else
 118       DeLinkLimit = 0;
 119
 120    PkgCompress = Block.Find("Packages::Compress",
 121                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 122    CntCompress = Block.Find("Contents::Compress",
 123                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 124    SrcCompress = Block.Find("Sources::Compress",
 125                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 126
 127    SrcExt = Block.Find("Sources::Extensions",
 128                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 129    PkgExt = Block.Find("Packages::Extensions",
 130                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 131
 132    Permissions = Setup.FindI("Default::FileMode",0644);
 133
 134    if (FLFile.empty() == false)
 135       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 136
 137    if (Contents == " ")
 138       Contents= string();
 139 }
 140                                                                         /*}}}*/
 141 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 142 // ---------------------------------------------------------------------
 143 /* This generates the Package File described by this object. */
 144 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 145 {
 146    if (PkgFile.empty() == true)
 147       return true;
 148
 149    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 150    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 151    string CacheDir = Setup.FindDir("Dir::CacheDir");
 152
 153    struct timeval StartTime;
 154    gettimeofday(&StartTime,0);
 155
 156    PkgDone = true;
 157
 158    // Create a package writer object.
 159    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 160                            flCombine(OverrideDir,BinOverride),
 161                            flCombine(OverrideDir,ExtraOverride));
 162    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 163       return _error->Error(_("Package extension list is too long"));
 164    if (_error->PendingError() == true)
 165       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 166
 167    Packages.PathPrefix = PathPrefix;
 168    Packages.DirStrip = ArchiveDir;
 169    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 170
 171    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 172    Packages.DeLinkLimit = DeLinkLimit;
 173
 174    // Create a compressor object
 175    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 176                       PkgCompress,Permissions);
 177    Packages.Output = Comp.Input;
 178    if (_error->PendingError() == true)
 179       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 180
 181    c0out << ' ' << BaseDir << ":" << flush;
 182
 183    // Do recursive directory searching
 184    if (FLFile.empty() == true)
 185    {
 186       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 187          return false;
 188    }
 189    else
 190    {
 191       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 192          return false;
 193    }
 194
 195    Packages.Output = 0;      // Just in case
 196
 197    // Finish compressing
 198    unsigned long Size;
 199    if (Comp.Finalize(Size) == false)
 200    {
 201       c0out << endl;
 202       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 203    }
 204
 205    if (Size != 0)
 206       c0out << " New "
 207              << SizeToStr(Size) << "B ";
 208    else
 209       c0out << ' ';
 210
 211    struct timeval NewTime;
 212    gettimeofday(&NewTime,0);
 213    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 214                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 215
 216    c0out << Packages.Stats.Packages << " files " <<
 217 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 218       SizeToStr(Packages.Stats.Bytes) << "B " <<
 219       TimeToStr((long)Delta) << endl;
 220
 221    Stats.Add(Packages.Stats);
 222    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 223
 224    return !_error->PendingError();
 225 }
 226
 227                                                                         /*}}}*/
 228 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 229 // ---------------------------------------------------------------------
 230 /* This generates the Sources File described by this object. */
 231 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 232 {
 233    if (SrcFile.empty() == true)
 234       return true;
 235
 236    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 237    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 238    string CacheDir = Setup.FindDir("Dir::CacheDir");
 239
 240    struct timeval StartTime;
 241    gettimeofday(&StartTime,0);
 242
 243    SrcDone = true;
 244
 245    // Create a package writer object.
 246    SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
 247                          flCombine(OverrideDir,SrcOverride),
 248                          flCombine(OverrideDir,SrcExtraOverride));
 249    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 250       return _error->Error(_("Source extension list is too long"));
 251    if (_error->PendingError() == true)
 252       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 253
 254    Sources.PathPrefix = PathPrefix;
 255    Sources.DirStrip = ArchiveDir;
 256    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 257
 258    Sources.DeLinkLimit = DeLinkLimit;
 259    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 260
 261    // Create a compressor object
 262    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 263                       SrcCompress,Permissions);
 264    Sources.Output = Comp.Input;
 265    if (_error->PendingError() == true)
 266       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 267
 268    c0out << ' ' << BaseDir << ":" << flush;
 269
 270    // Do recursive directory searching
 271    if (FLFile.empty() == true)
 272    {
 273       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 274          return false;
 275    }
 276    else
 277    {
 278       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 279          return false;
 280    }
 281    Sources.Output = 0;      // Just in case
 282
 283    // Finish compressing
 284    unsigned long Size;
 285    if (Comp.Finalize(Size) == false)
 286    {
 287       c0out << endl;
 288       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 289    }
 290
 291    if (Size != 0)
 292       c0out << " New "
 293              << SizeToStr(Size) << "B ";
 294    else
 295       c0out << ' ';
 296
 297    struct timeval NewTime;
 298    gettimeofday(&NewTime,0);
 299    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 300                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 301
 302    c0out << Sources.Stats.Packages << " pkgs in " <<
 303       TimeToStr((long)Delta) << endl;
 304
 305    Stats.Add(Sources.Stats);
 306    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 307
 308    return !_error->PendingError();
 309 }
 310                                                                         /*}}}*/
 311 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 312 // ---------------------------------------------------------------------
 313 /* This generates the contents file partially described by this object.
 314    It searches the given iterator range for other package files that map
 315    into this contents file and includes their data as well when building. */
 316 bool PackageMap::GenContents(Configuration &Setup,
 317                              vector<PackageMap>::iterator Begin,
 318                              vector<PackageMap>::iterator End,
 319                              unsigned long &Left)
 320 {
 321    if (Contents.empty() == true)
 322       return true;
 323
 324    if (Left == 0)
 325       return true;
 326
 327    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 328    string CacheDir = Setup.FindDir("Dir::CacheDir");
 329    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 330
 331    struct timeval StartTime;
 332    gettimeofday(&StartTime,0);
 333
 334    // Create a package writer object.
 335    ContentsWriter Contents("");
 336    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 337       return _error->Error(_("Package extension list is too long"));
 338    if (_error->PendingError() == true)
 339       return false;
 340
 341    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 342                       CntCompress,Permissions);
 343    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 344    Contents.Output = Comp.Input;
 345    if (_error->PendingError() == true)
 346       return false;
 347
 348    // Write the header out.
 349    if (ContentsHead.empty() == false)
 350    {
 351       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 352       if (_error->PendingError() == true)
 353          return false;
 354
 355       unsigned long Size = Head.Size();
 356       unsigned char Buf[4096];
 357       while (Size != 0)
 358       {
 359          unsigned long ToRead = Size;
 360          if (Size > sizeof(Buf))
 361             ToRead = sizeof(Buf);
 362
 363          if (Head.Read(Buf,ToRead) == false)
 364             return false;
 365
 366          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 367             return _error->Errno("fwrite",_("Error writing header to contents file"));
 368
 369          Size -= ToRead;
 370       }
 371    }
 372
 373    /* Go over all the package file records and parse all the package
 374       files associated with this contents file into one great big honking
 375       memory structure, then dump the sorted version */
 376    c0out << ' ' << this->Contents << ":" << flush;
 377    for (vector<PackageMap>::iterator I = Begin; I != End; I++)
 378    {
 379       if (I->Contents != this->Contents)
 380          continue;
 381
 382       Contents.Prefix = ArchiveDir;
 383       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 384       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 385                             I->PkgCompress);
 386
 387       I->ContentsDone = true;
 388    }
 389
 390    Contents.Finish();
 391
 392    // Finish compressing
 393    unsigned long Size;
 394    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 395    {
 396       c0out << endl;
 397       return _error->Error(_("Error Processing Contents %s"),
 398                            this->Contents.c_str());
 399    }
 400
 401    if (Size != 0)
 402    {
 403       c0out << " New " << SizeToStr(Size) << "B ";
 404       if (Left > Size)
 405          Left -= Size;
 406       else
 407          Left = 0;
 408    }
 409    else
 410       c0out << ' ';
 411
 412    struct timeval NewTime;
 413    gettimeofday(&NewTime,0);
 414    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 415                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 416
 417    c0out << Contents.Stats.Packages << " files " <<
 418       SizeToStr(Contents.Stats.Bytes) << "B " <<
 419       TimeToStr((long)Delta) << endl;
 420
 421    return true;
 422 }
 423                                                                         /*}}}*/
 424
 425 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 426 // ---------------------------------------------------------------------
 427 /* This populates the PkgList with all the possible permutations of the
 428    section/arch lists. */
 429 void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 430 {
 431    // Load the defaults
 432    string DDir = Setup.Find("TreeDefault::Directory",
 433                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 434    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 435                             "$(DIST)/$(SECTION)/source/");
 436    string DPkg = Setup.Find("TreeDefault::Packages",
 437                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 438    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 439                             "$(DIST)/$(SECTION)/");
 440    string DContents = Setup.Find("TreeDefault::Contents",
 441                             "$(DIST)/Contents-$(ARCH)");
 442    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 443    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 444                                "packages-$(ARCH).db");
 445    string DSources = Setup.Find("TreeDefault::Sources",
 446                                 "$(DIST)/$(SECTION)/source/Sources");
 447    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 448    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 449
 450    // Process 'tree' type sections
 451    const Configuration::Item *Top = Setup.Tree("tree");
 452    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 453    {
 454       Configuration Block(Top);
 455       string Dist = Top->Tag;
 456
 457       // Parse the sections
 458       string Tmp = Block.Find("Sections");
 459       const char *Sections = Tmp.c_str();
 460       string Section;
 461       while (ParseQuoteWord(Sections,Section) == true)
 462       {
 463          string Tmp2 = Block.Find("Architectures");
 464          string Arch;
 465          const char *Archs = Tmp2.c_str();
 466          while (ParseQuoteWord(Archs,Arch) == true)
 467          {
 468             struct SubstVar Vars[] = {{"$(DIST)",&Dist},
 469                                       {"$(SECTION)",&Section},
 470                                       {"$(ARCH)",&Arch},
 471                                       {}};
 472             PackageMap Itm;
 473
 474             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 475             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 476
 477             if (stringcasecmp(Arch,"source") == 0)
 478             {
 479                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 480                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 481                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 482                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 483                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 484                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 485             }
 486             else
 487             {
 488                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 489                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 490                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 491                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 492                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 493                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 494                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 495                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 496             }
 497
 498             Itm.GetGeneral(Setup,Block);
 499             PkgList.push_back(Itm);
 500          }
 501       }
 502
 503       Top = Top->Next;
 504    }
 505 }
 506                                                                         /*}}}*/
 507 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 508 // ---------------------------------------------------------------------
 509 /* */
 510 void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 511 {
 512    // Process 'bindirectory' type sections
 513    const Configuration::Item *Top = Setup.Tree("bindirectory");
 514    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 515    {
 516       Configuration Block(Top);
 517
 518       PackageMap Itm;
 519       Itm.PkgFile = Block.Find("Packages");
 520       Itm.SrcFile = Block.Find("Sources");
 521       Itm.BinCacheDB = Block.Find("BinCacheDB");
 522       Itm.BinOverride = Block.Find("BinOverride");
 523       Itm.ExtraOverride = Block.Find("ExtraOverride");
 524       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 525       Itm.SrcOverride = Block.Find("SrcOverride");
 526       Itm.BaseDir = Top->Tag;
 527       Itm.FLFile = Block.Find("FileList");
 528       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 529       Itm.Contents = Block.Find("Contents");
 530       Itm.ContentsHead = Block.Find("Contents::Header");
 531
 532       Itm.GetGeneral(Setup,Block);
 533       PkgList.push_back(Itm);
 534
 535       Top = Top->Next;
 536    }
 537 }
 538                                                                         /*}}}*/
 539
 540 // ShowHelp - Show the help text                                        /*{{{*/
 541 // ---------------------------------------------------------------------
 542 /* */
 543 bool ShowHelp(CommandLine &CmdL)
 544 {
 545    ioprintf(cout,_("%s %s for %s %s compiled on %s %s\n"),PACKAGE,VERSION,
 546             COMMON_OS,COMMON_CPU,__DATE__,__TIME__);
 547    if (_config->FindB("version") == true)
 548       return true;
 549
 550    cout <<
 551     _("Usage: apt-ftparchive [options] command\n"
 552       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 553       "          sources srcpath [overridefile [pathprefix]]\n"
 554       "          contents path\n"
 555       "          release path\n"
 556       "          generate config [groups]\n"
 557       "          clean config\n"
 558       "\n"
 559       "apt-ftparchive generates index files for Debian archives. It supports\n"
 560       "many styles of generation from fully automated to functional replacements\n"
 561       "for dpkg-scanpackages and dpkg-scansources\n"
 562       "\n"
 563       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 564       "Package file contains the contents of all the control fields from\n"
 565       "each package as well as the MD5 hash and filesize. An override file\n"
 566       "is supported to force the value of Priority and Section.\n"
 567       "\n"
 568       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 569       "The --source-override option can be used to specify a src override file\n"
 570       "\n"
 571       "The 'packages' and 'sources' command should be run in the root of the\n"
 572       "tree. BinaryPath should point to the base of the recursive search and \n"
 573       "override file should contain the override flags. Pathprefix is\n"
 574       "appended to the filename fields if present. Example usage from the \n"
 575       "Debian archive:\n"
 576       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 577       "               dists/potato/main/binary-i386/Packages\n"
 578       "\n"
 579       "Options:\n"
 580       "  -h    This help text\n"
 581       "  --md5 Control MD5 generation\n"
 582       "  -s=?  Source override file\n"
 583       "  -q    Quiet\n"
 584       "  -d=?  Select the optional caching database\n"
 585       "  --no-delink Enable delinking debug mode\n"
 586       "  --contents  Control contents file generation\n"
 587       "  -c=?  Read this configuration file\n"
 588       "  -o=?  Set an arbitary configuration option") << endl;
 589
 590    return true;
 591 }
 592                                                                         /*}}}*/
 593 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 594 // ---------------------------------------------------------------------
 595 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 596 bool SimpleGenPackages(CommandLine &CmdL)
 597 {
 598    if (CmdL.FileSize() < 2)
 599       return ShowHelp(CmdL);
 600
 601    string Override;
 602    if (CmdL.FileSize() >= 3)
 603       Override = CmdL.FileList[2];
 604
 605    // Create a package writer object.
 606    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 607                            Override, "");
 608    if (_error->PendingError() == true)
 609       return false;
 610
 611    if (CmdL.FileSize() >= 4)
 612       Packages.PathPrefix = CmdL.FileList[3];
 613
 614    // Do recursive directory searching
 615    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 616       return false;
 617
 618    return true;
 619 }
 620                                                                         /*}}}*/
 621 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 622 // ---------------------------------------------------------------------
 623 /* */
 624 bool SimpleGenContents(CommandLine &CmdL)
 625 {
 626    if (CmdL.FileSize() < 2)
 627       return ShowHelp(CmdL);
 628
 629    // Create a package writer object.
 630    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"));
 631    if (_error->PendingError() == true)
 632       return false;
 633
 634    // Do recursive directory searching
 635    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 636       return false;
 637
 638    Contents.Finish();
 639
 640    return true;
 641 }
 642                                                                         /*}}}*/
 643 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 644 // ---------------------------------------------------------------------
 645 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 646 bool SimpleGenSources(CommandLine &CmdL)
 647 {
 648    if (CmdL.FileSize() < 2)
 649       return ShowHelp(CmdL);
 650
 651    string Override;
 652    if (CmdL.FileSize() >= 3)
 653       Override = CmdL.FileList[2];
 654
 655    string SOverride;
 656    if (Override.empty() == false)
 657       SOverride = Override + ".src";
 658
 659    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 660                              SOverride.c_str());
 661
 662    // Create a package writer object.
 663    SourcesWriter Sources(Override,SOverride);
 664    if (_error->PendingError() == true)
 665       return false;
 666
 667    if (CmdL.FileSize() >= 4)
 668       Sources.PathPrefix = CmdL.FileList[3];
 669
 670    // Do recursive directory searching
 671    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 672       return false;
 673
 674    return true;
 675 }
 676                                                                         /*}}}*/
 677 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 678 // ---------------------------------------------------------------------
 679 bool SimpleGenRelease(CommandLine &CmdL)
 680 {
 681    if (CmdL.FileSize() < 2)
 682       return ShowHelp(CmdL);
 683
 684    string Dir = CmdL.FileList[1];
 685
 686    ReleaseWriter Release("");
 687    Release.DirStrip = Dir;
 688
 689    if (_error->PendingError() == true)
 690       return false;
 691
 692    if (Release.RecursiveScan(Dir) == false)
 693       return false;
 694
 695    Release.Finish();
 696
 697    return true;
 698 }
 699
 700                                                                         /*}}}*/
 701 // Generate - Full generate, using a config file                        /*{{{*/
 702 // ---------------------------------------------------------------------
 703 /* */
 704 bool Generate(CommandLine &CmdL)
 705 {
 706    struct CacheDB::Stats SrcStats;
 707    if (CmdL.FileSize() < 2)
 708       return ShowHelp(CmdL);
 709
 710    struct timeval StartTime;
 711    gettimeofday(&StartTime,0);
 712    struct CacheDB::Stats Stats;
 713
 714    // Read the configuration file.
 715    Configuration Setup;
 716    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 717       return false;
 718
 719    vector<PackageMap> PkgList;
 720    LoadTree(PkgList,Setup);
 721    LoadBinDir(PkgList,Setup);
 722
 723    // Sort by cache DB to improve IO locality.
 724    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 725
 726    // Generate packages
 727    if (CmdL.FileSize() <= 2)
 728    {
 729       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 730          if (I->GenPackages(Setup,Stats) == false)
 731             _error->DumpErrors();
 732       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 733          if (I->GenSources(Setup,SrcStats) == false)
 734             _error->DumpErrors();
 735    }
 736    else
 737    {
 738       // Make a choice list out of the package list..
 739       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 740       RxChoiceList *End = List;
 741       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 742       {
 743          End->UserData = &(*I);
 744          End->Str = I->BaseDir.c_str();
 745          End++;
 746
 747          End->UserData = &(*I);
 748          End->Str = I->Tag.c_str();
 749          End++;
 750       }
 751       End->Str = 0;
 752
 753       // Regex it
 754       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 755       {
 756          delete [] List;
 757          return _error->Error(_("No selections matched"));
 758       }
 759       _error->DumpErrors();
 760
 761       // Do the generation for Packages
 762       for (End = List; End->Str != 0; End++)
 763       {
 764          if (End->Hit == false)
 765             continue;
 766
 767          PackageMap *I = (PackageMap *)End->UserData;
 768          if (I->PkgDone == true)
 769             continue;
 770          if (I->GenPackages(Setup,Stats) == false)
 771             _error->DumpErrors();
 772       }
 773
 774       // Do the generation for Sources
 775       for (End = List; End->Str != 0; End++)
 776       {
 777          if (End->Hit == false)
 778             continue;
 779
 780          PackageMap *I = (PackageMap *)End->UserData;
 781          if (I->SrcDone == true)
 782             continue;
 783          if (I->GenSources(Setup,SrcStats) == false)
 784             _error->DumpErrors();
 785       }
 786
 787       delete [] List;
 788    }
 789
 790    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 791       return true;
 792
 793    c1out << "Done Packages, Starting contents." << endl;
 794
 795    // Sort the contents file list by date
 796    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 797    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 798    {
 799       struct stat A;
 800       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 801                                  I->CntCompress,A) == false)
 802          time(&I->ContentsMTime);
 803       else
 804          I->ContentsMTime = A.st_mtime;
 805    }
 806    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 807
 808    /* Now for Contents.. The process here is to do a make-like dependency
 809       check. Each contents file is verified to be newer than the package files
 810       that describe the debs it indexes. Since the package files contain
 811       hashes of the .debs this means they have not changed either so the
 812       contents must be up to date. */
 813    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 814    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 815    {
 816       // This record is not relevent
 817       if (I->ContentsDone == true ||
 818           I->Contents.empty() == true)
 819          continue;
 820
 821       // Do not do everything if the user specified sections.
 822       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 823          continue;
 824
 825       struct stat A,B;
 826       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 827       {
 828          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 829          {
 830             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 831             continue;
 832          }
 833
 834          if (A.st_mtime > B.st_mtime)
 835             continue;
 836       }
 837
 838       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 839                          MaxContentsChange) == false)
 840          _error->DumpErrors();
 841
 842       // Hit the limit?
 843       if (MaxContentsChange == 0)
 844       {
 845          c1out << "Hit contents update byte limit" << endl;
 846          break;
 847       }
 848    }
 849
 850    struct timeval NewTime;
 851    gettimeofday(&NewTime,0);
 852    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 853                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 854    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 855          << " archives. Took " << TimeToStr((long)Delta) << endl;
 856
 857    return true;
 858 }
 859                                                                         /*}}}*/
 860 // Clean - Clean out the databases                                      /*{{{*/
 861 // ---------------------------------------------------------------------
 862 /* */
 863 bool Clean(CommandLine &CmdL)
 864 {
 865    if (CmdL.FileSize() != 2)
 866       return ShowHelp(CmdL);
 867
 868    // Read the configuration file.
 869    Configuration Setup;
 870    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 871       return false;
 872
 873    vector<PackageMap> PkgList;
 874    LoadTree(PkgList,Setup);
 875    LoadBinDir(PkgList,Setup);
 876
 877    // Sort by cache DB to improve IO locality.
 878    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 879
 880    string CacheDir = Setup.FindDir("Dir::CacheDir");
 881
 882    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 883    {
 884       c0out << I->BinCacheDB << endl;
 885       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 886       if (DB.Clean() == false)
 887          _error->DumpErrors();
 888
 889       string CacheDB = I->BinCacheDB;
 890       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
 891    }
 892
 893    return true;
 894 }
 895                                                                         /*}}}*/
 896
 897 int main(int argc, const char *argv[])
 898 {
 899    CommandLine::Args Args[] = {
 900       {'h',"help","help",0},
 901       {0,"md5","APT::FTPArchive::MD5",0},
 902       {'v',"version","version",0},
 903       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 904       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 905       {'q',"quiet","quiet",CommandLine::IntLevel},
 906       {'q',"silent","quiet",CommandLine::IntLevel},
 907       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 908       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 909       {0,"contents","APT::FTPArchive::Contents",0},
 910       {'c',"config-file",0,CommandLine::ConfigFile},
 911       {'o',"option",0,CommandLine::ArbItem},
 912       {0,0,0,0}};
 913    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 914                                    {"contents",&SimpleGenContents},
 915                                    {"sources",&SimpleGenSources},
 916                                    {"release",&SimpleGenRelease},
 917                                    {"generate",&Generate},
 918                                    {"clean",&Clean},
 919                                    {"help",&ShowHelp},
 920                                    {0,0}};
 921
 922    // Parse the command line and initialize the package library
 923    CommandLine CmdL(Args,_config);
 924    if (CmdL.Parse(argc,argv) == false)
 925    {
 926       _error->DumpErrors();
 927       return 100;
 928    }
 929
 930    // See if the help should be shown
 931    if (_config->FindB("help") == true ||
 932        _config->FindB("version") == true ||
 933        CmdL.FileSize() == 0)
 934    {
 935       ShowHelp(CmdL);
 936       return 0;
 937    }
 938
 939    // Setup the output streams
 940    c0out.rdbuf(clog.rdbuf());
 941    c1out.rdbuf(clog.rdbuf());
 942    c2out.rdbuf(clog.rdbuf());
 943    Quiet = _config->FindI("quiet",0);
 944    if (Quiet > 0)
 945       c0out.rdbuf(devnull.rdbuf());
 946    if (Quiet > 1)
 947       c1out.rdbuf(devnull.rdbuf());
 948
 949    // Match the operation
 950    CmdL.DispatchArg(Cmds);
 951
 952    if (_error->empty() == false)
 953    {
 954       bool Errors = _error->PendingError();
 955       _error->DumpErrors();
 956       return Errors == true?100:0;
 957    }
 958    return 0;
 959 }