ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.11 2003/12/26 22:50:52 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-scanpackages - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-ftparchive.h"
  15 #endif
  16
  17 #include "apt-ftparchive.h"
  18
  19 #include <apt-pkg/error.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/cmndline.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <config.h>
  24 #include <apti18n.h>
  25 #include <algorithm>
  26
  27 #include <sys/time.h>
  28 #include <regex.h>
  29
  30 #include "contents.h"
  31 #include "multicompress.h"
  32 #include "writer.h"
  33                                                                         /*}}}*/
  34
  35 using namespace std;
  36 ostream c0out(0);
  37 ostream c1out(0);
  38 ostream c2out(0);
  39 ofstream devnull("/dev/null");
  40 unsigned Quiet = 0;
  41
  42 // struct PackageMap - List of all package files in the config file     /*{{{*/
  43 // ---------------------------------------------------------------------
  44 /* */
  45 struct PackageMap
  46 {
  47    // General Stuff
  48    string BaseDir;
  49    string InternalPrefix;
  50    string FLFile;
  51    string PkgExt;
  52    string SrcExt;
  53
  54    // Stuff for the Package File
  55    string PkgFile;
  56    string BinCacheDB;
  57    string BinOverride;
  58    string ExtraOverride;
  59
  60    // Stuff for the Source File
  61    string SrcFile;
  62    string SrcOverride;
  63    string SrcExtraOverride;
  64
  65    // Contents
  66    string Contents;
  67    string ContentsHead;
  68
  69    // Random things
  70    string Tag;
  71    string PkgCompress;
  72    string CntCompress;
  73    string SrcCompress;
  74    string PathPrefix;
  75    unsigned int DeLinkLimit;
  76    mode_t Permissions;
  77
  78    bool ContentsDone;
  79    bool PkgDone;
  80    bool SrcDone;
  81    time_t ContentsMTime;
  82
  83    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  84    {
  85       inline bool operator() (const PackageMap &x,const PackageMap &y)
  86       {return x.ContentsMTime < y.ContentsMTime;};
  87    };
  88
  89    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  90    {
  91       inline bool operator() (const PackageMap &x,const PackageMap &y)
  92       {return x.BinCacheDB < y.BinCacheDB;};
  93    };
  94
  95    void GetGeneral(Configuration &Setup,Configuration &Block);
  96    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
  97    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
  98    bool GenContents(Configuration &Setup,
  99                     vector<PackageMap>::iterator Begin,
 100                     vector<PackageMap>::iterator End,
 101                     unsigned long &Left);
 102
 103    PackageMap() : DeLinkLimit(0), Permissions(1), ContentsDone(false),
 104         PkgDone(false), SrcDone(false), ContentsMTime(0) {};
 105 };
 106                                                                         /*}}}*/
 107
 108 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 109 // ---------------------------------------------------------------------
 110 /* */
 111 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 112 {
 113    PathPrefix = Block.Find("PathPrefix");
 114
 115    if (Block.FindB("External-Links",true) == false)
 116       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 117    else
 118       DeLinkLimit = 0;
 119
 120    PkgCompress = Block.Find("Packages::Compress",
 121                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 122    CntCompress = Block.Find("Contents::Compress",
 123                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 124    SrcCompress = Block.Find("Sources::Compress",
 125                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 126
 127    SrcExt = Block.Find("Sources::Extensions",
 128                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 129    PkgExt = Block.Find("Packages::Extensions",
 130                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 131
 132    Permissions = Setup.FindI("Default::FileMode",0644);
 133
 134    if (FLFile.empty() == false)
 135       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 136
 137    if (Contents == " ")
 138       Contents= string();
 139 }
 140                                                                         /*}}}*/
 141 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 142 // ---------------------------------------------------------------------
 143 /* This generates the Package File described by this object. */
 144 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 145 {
 146    if (PkgFile.empty() == true)
 147       return true;
 148
 149    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 150    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 151    string CacheDir = Setup.FindDir("Dir::CacheDir");
 152
 153    struct timeval StartTime;
 154    gettimeofday(&StartTime,0);
 155
 156    PkgDone = true;
 157
 158    // Create a package writer object.
 159    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 160                            flCombine(OverrideDir,BinOverride),
 161                            flCombine(OverrideDir,ExtraOverride));
 162    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 163       return _error->Error(_("Package extension list is too long"));
 164    if (_error->PendingError() == true)
 165       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 166
 167    Packages.PathPrefix = PathPrefix;
 168    Packages.DirStrip = ArchiveDir;
 169    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 170
 171    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 172    Packages.DeLinkLimit = DeLinkLimit;
 173
 174    // Create a compressor object
 175    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 176                       PkgCompress,Permissions);
 177    Packages.Output = Comp.Input;
 178    if (_error->PendingError() == true)
 179       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 180
 181    c0out << ' ' << BaseDir << ":" << flush;
 182
 183    // Do recursive directory searching
 184    if (FLFile.empty() == true)
 185    {
 186       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 187          return false;
 188    }
 189    else
 190    {
 191       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 192          return false;
 193    }
 194
 195    Packages.Output = 0;      // Just in case
 196
 197    // Finish compressing
 198    unsigned long Size;
 199    if (Comp.Finalize(Size) == false)
 200    {
 201       c0out << endl;
 202       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 203    }
 204
 205    if (Size != 0)
 206       c0out << " New "
 207              << SizeToStr(Size) << "B ";
 208    else
 209       c0out << ' ';
 210
 211    struct timeval NewTime;
 212    gettimeofday(&NewTime,0);
 213    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 214                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 215
 216    c0out << Packages.Stats.Packages << " files " <<
 217 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 218       SizeToStr(Packages.Stats.Bytes) << "B " <<
 219       TimeToStr((long)Delta) << endl;
 220
 221    Stats.Add(Packages.Stats);
 222    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 223
 224    return !_error->PendingError();
 225 }
 226
 227                                                                         /*}}}*/
 228 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 229 // ---------------------------------------------------------------------
 230 /* This generates the Sources File described by this object. */
 231 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 232 {
 233    if (SrcFile.empty() == true)
 234       return true;
 235
 236    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 237    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 238    string CacheDir = Setup.FindDir("Dir::CacheDir");
 239
 240    struct timeval StartTime;
 241    gettimeofday(&StartTime,0);
 242
 243    SrcDone = true;
 244
 245    // Create a package writer object.
 246    SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
 247                          flCombine(OverrideDir,SrcOverride),
 248                          flCombine(OverrideDir,SrcExtraOverride));
 249    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 250       return _error->Error(_("Source extension list is too long"));
 251    if (_error->PendingError() == true)
 252       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 253
 254    Sources.PathPrefix = PathPrefix;
 255    Sources.DirStrip = ArchiveDir;
 256    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 257
 258    Sources.DeLinkLimit = DeLinkLimit;
 259    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 260
 261    // Create a compressor object
 262    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 263                       SrcCompress,Permissions);
 264    Sources.Output = Comp.Input;
 265    if (_error->PendingError() == true)
 266       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 267
 268    c0out << ' ' << BaseDir << ":" << flush;
 269
 270    // Do recursive directory searching
 271    if (FLFile.empty() == true)
 272    {
 273       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 274          return false;
 275    }
 276    else
 277    {
 278       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 279          return false;
 280    }
 281    Sources.Output = 0;      // Just in case
 282
 283    // Finish compressing
 284    unsigned long Size;
 285    if (Comp.Finalize(Size) == false)
 286    {
 287       c0out << endl;
 288       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 289    }
 290
 291    if (Size != 0)
 292       c0out << " New "
 293              << SizeToStr(Size) << "B ";
 294    else
 295       c0out << ' ';
 296
 297    struct timeval NewTime;
 298    gettimeofday(&NewTime,0);
 299    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 300                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 301
 302    c0out << Sources.Stats.Packages << " pkgs in " <<
 303       TimeToStr((long)Delta) << endl;
 304
 305    Stats.Add(Sources.Stats);
 306    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 307
 308    return !_error->PendingError();
 309 }
 310                                                                         /*}}}*/
 311 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 312 // ---------------------------------------------------------------------
 313 /* This generates the contents file partially described by this object.
 314    It searches the given iterator range for other package files that map
 315    into this contents file and includes their data as well when building. */
 316 bool PackageMap::GenContents(Configuration &Setup,
 317                              vector<PackageMap>::iterator Begin,
 318                              vector<PackageMap>::iterator End,
 319                              unsigned long &Left)
 320 {
 321    if (Contents.empty() == true)
 322       return true;
 323
 324    if (Left == 0)
 325       return true;
 326
 327    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 328    string CacheDir = Setup.FindDir("Dir::CacheDir");
 329    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 330
 331    struct timeval StartTime;
 332    gettimeofday(&StartTime,0);
 333
 334    // Create a package writer object.
 335    ContentsWriter Contents("");
 336    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 337       return _error->Error(_("Package extension list is too long"));
 338    if (_error->PendingError() == true)
 339       return false;
 340
 341    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 342                       CntCompress,Permissions);
 343    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 344    Contents.Output = Comp.Input;
 345    if (_error->PendingError() == true)
 346       return false;
 347
 348    // Write the header out.
 349    if (ContentsHead.empty() == false)
 350    {
 351       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 352       if (_error->PendingError() == true)
 353          return false;
 354
 355       unsigned long Size = Head.Size();
 356       unsigned char Buf[4096];
 357       while (Size != 0)
 358       {
 359          unsigned long ToRead = Size;
 360          if (Size > sizeof(Buf))
 361             ToRead = sizeof(Buf);
 362
 363          if (Head.Read(Buf,ToRead) == false)
 364             return false;
 365
 366          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 367             return _error->Errno("fwrite",_("Error writing header to contents file"));
 368
 369          Size -= ToRead;
 370       }
 371    }
 372
 373    /* Go over all the package file records and parse all the package
 374       files associated with this contents file into one great big honking
 375       memory structure, then dump the sorted version */
 376    c0out << ' ' << this->Contents << ":" << flush;
 377    for (vector<PackageMap>::iterator I = Begin; I != End; I++)
 378    {
 379       if (I->Contents != this->Contents)
 380          continue;
 381
 382       Contents.Prefix = ArchiveDir;
 383       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 384       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 385                             I->PkgCompress);
 386
 387       I->ContentsDone = true;
 388    }
 389
 390    Contents.Finish();
 391
 392    // Finish compressing
 393    unsigned long Size;
 394    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 395    {
 396       c0out << endl;
 397       return _error->Error(_("Error Processing Contents %s"),
 398                            this->Contents.c_str());
 399    }
 400
 401    if (Size != 0)
 402    {
 403       c0out << " New " << SizeToStr(Size) << "B ";
 404       if (Left > Size)
 405          Left -= Size;
 406       else
 407          Left = 0;
 408    }
 409    else
 410       c0out << ' ';
 411
 412    struct timeval NewTime;
 413    gettimeofday(&NewTime,0);
 414    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 415                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 416
 417    c0out << Contents.Stats.Packages << " files " <<
 418       SizeToStr(Contents.Stats.Bytes) << "B " <<
 419       TimeToStr((long)Delta) << endl;
 420
 421    return true;
 422 }
 423                                                                         /*}}}*/
 424
 425 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 426 // ---------------------------------------------------------------------
 427 /* This populates the PkgList with all the possible permutations of the
 428    section/arch lists. */
 429 void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 430 {
 431    // Load the defaults
 432    string DDir = Setup.Find("TreeDefault::Directory",
 433                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 434    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 435                             "$(DIST)/$(SECTION)/source/");
 436    string DPkg = Setup.Find("TreeDefault::Packages",
 437                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 438    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 439                             "$(DIST)/$(SECTION)/");
 440    string DContents = Setup.Find("TreeDefault::Contents",
 441                             "$(DIST)/Contents-$(ARCH)");
 442    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 443    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 444                                "packages-$(ARCH).db");
 445    string DSources = Setup.Find("TreeDefault::Sources",
 446                                 "$(DIST)/$(SECTION)/source/Sources");
 447    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 448    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 449
 450    // Process 'tree' type sections
 451    const Configuration::Item *Top = Setup.Tree("tree");
 452    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 453    {
 454       Configuration Block(Top);
 455       string Dist = Top->Tag;
 456
 457       // Parse the sections
 458       string Tmp = Block.Find("Sections");
 459       const char *Sections = Tmp.c_str();
 460       string Section;
 461       while (ParseQuoteWord(Sections,Section) == true)
 462       {
 463          string Tmp2 = Block.Find("Architectures");
 464          string Arch;
 465          const char *Archs = Tmp2.c_str();
 466          while (ParseQuoteWord(Archs,Arch) == true)
 467          {
 468             struct SubstVar Vars[] = {{"$(DIST)",&Dist},
 469                                       {"$(SECTION)",&Section},
 470                                       {"$(ARCH)",&Arch},
 471                                       {}};
 472             PackageMap Itm;
 473
 474             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 475             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 476
 477             if (stringcasecmp(Arch,"source") == 0)
 478             {
 479                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 480                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 481                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 482                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 483                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 484                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 485             }
 486             else
 487             {
 488                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 489                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 490                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 491                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 492                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 493                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 494                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 495                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 496             }
 497
 498             Itm.GetGeneral(Setup,Block);
 499             PkgList.push_back(Itm);
 500          }
 501       }
 502
 503       Top = Top->Next;
 504    }
 505 }
 506                                                                         /*}}}*/
 507 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 508 // ---------------------------------------------------------------------
 509 /* */
 510 void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 511 {
 512    // Process 'bindirectory' type sections
 513    const Configuration::Item *Top = Setup.Tree("bindirectory");
 514    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 515    {
 516       Configuration Block(Top);
 517
 518       PackageMap Itm;
 519       Itm.PkgFile = Block.Find("Packages");
 520       Itm.SrcFile = Block.Find("Sources");
 521       Itm.BinCacheDB = Block.Find("BinCacheDB");
 522       Itm.BinOverride = Block.Find("BinOverride");
 523       Itm.ExtraOverride = Block.Find("ExtraOverride");
 524       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 525       Itm.SrcOverride = Block.Find("SrcOverride");
 526       Itm.BaseDir = Top->Tag;
 527       Itm.FLFile = Block.Find("FileList");
 528       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 529       Itm.Contents = Block.Find("Contents");
 530       Itm.ContentsHead = Block.Find("Contents::Header");
 531
 532       Itm.GetGeneral(Setup,Block);
 533       PkgList.push_back(Itm);
 534
 535       Top = Top->Next;
 536    }
 537 }
 538                                                                         /*}}}*/
 539
 540 // ShowHelp - Show the help text                                        /*{{{*/
 541 // ---------------------------------------------------------------------
 542 /* */
 543 bool ShowHelp(CommandLine &CmdL)
 544 {
 545    ioprintf(cout,_("%s %s for %s %s compiled on %s %s\n"),PACKAGE,VERSION,
 546             COMMON_OS,COMMON_CPU,__DATE__,__TIME__);
 547    if (_config->FindB("version") == true)
 548       return true;
 549
 550    cout <<
 551     _("Usage: apt-ftparchive [options] command\n"
 552       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 553       "          sources srcpath [overridefile [pathprefix]]\n"
 554       "          contents path\n"
 555       "          release path\n"
 556       "          generate config [groups]\n"
 557       "          clean config\n"
 558       "\n"
 559       "apt-ftparchive generates index files for Debian archives. It supports\n"
 560       "many styles of generation from fully automated to functional replacements\n"
 561       "for dpkg-scanpackages and dpkg-scansources\n"
 562       "\n"
 563       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 564       "Package file contains the contents of all the control fields from\n"
 565       "each package as well as the MD5 hash and filesize. An override file\n"
 566       "is supported to force the value of Priority and Section.\n"
 567       "\n"
 568       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 569       "The --source-override option can be used to specify a src override file\n"
 570       "\n"
 571       "The 'packages' and 'sources' command should be run in the root of the\n"
 572       "tree. BinaryPath should point to the base of the recursive search and \n"
 573       "override file should contain the override flags. Pathprefix is\n"
 574       "appended to the filename fields if present. Example usage from the \n"
 575       "Debian archive:\n"
 576       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 577       "               dists/potato/main/binary-i386/Packages\n"
 578       "\n"
 579       "Options:\n"
 580       "  -h    This help text\n"
 581       "  --md5 Control MD5 generation\n"
 582       "  -s=?  Source override file\n"
 583       "  -q    Quiet\n"
 584       "  -d=?  Select the optional caching database\n"
 585       "  --no-delink Enable delinking debug mode\n"
 586       "  --contents  Control contents file generation\n"
 587       "  -c=?  Read this configuration file\n"
 588       "  -o=?  Set an arbitary configuration option") << endl;
 589
 590    return true;
 591 }
 592                                                                         /*}}}*/
 593 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 594 // ---------------------------------------------------------------------
 595 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 596 bool SimpleGenPackages(CommandLine &CmdL)
 597 {
 598    if (CmdL.FileSize() < 2)
 599       return ShowHelp(CmdL);
 600
 601    string Override;
 602    if (CmdL.FileSize() >= 3)
 603       Override = CmdL.FileList[2];
 604
 605    // Create a package writer object.
 606    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 607                            Override, "");
 608    if (_error->PendingError() == true)
 609       return false;
 610
 611    if (CmdL.FileSize() >= 4)
 612       Packages.PathPrefix = CmdL.FileList[3];
 613
 614    // Do recursive directory searching
 615    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 616       return false;
 617
 618    return true;
 619 }
 620                                                                         /*}}}*/
 621 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 622 // ---------------------------------------------------------------------
 623 /* */
 624 bool SimpleGenContents(CommandLine &CmdL)
 625 {
 626    if (CmdL.FileSize() < 2)
 627       return ShowHelp(CmdL);
 628
 629    // Create a package writer object.
 630    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"));
 631    if (_error->PendingError() == true)
 632       return false;
 633
 634    // Do recursive directory searching
 635    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 636       return false;
 637
 638    Contents.Finish();
 639
 640    return true;
 641 }
 642                                                                         /*}}}*/
 643 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 644 // ---------------------------------------------------------------------
 645 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 646 bool SimpleGenSources(CommandLine &CmdL)
 647 {
 648    if (CmdL.FileSize() < 2)
 649       return ShowHelp(CmdL);
 650
 651    string Override;
 652    if (CmdL.FileSize() >= 3)
 653       Override = CmdL.FileList[2];
 654
 655    string SOverride;
 656    if (Override.empty() == false)
 657       SOverride = Override + ".src";
 658
 659    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 660                              SOverride.c_str());
 661
 662    // Create a package writer object.
 663    SourcesWriter Sources(Override,SOverride);
 664    if (_error->PendingError() == true)
 665       return false;
 666
 667    if (CmdL.FileSize() >= 4)
 668       Sources.PathPrefix = CmdL.FileList[3];
 669
 670    // Do recursive directory searching
 671    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 672       return false;
 673
 674    return true;
 675 }
 676                                                                         /*}}}*/
 677 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 678 // ---------------------------------------------------------------------
 679 bool SimpleGenRelease(CommandLine &CmdL)
 680 {
 681    if (CmdL.FileSize() < 2)
 682       return ShowHelp(CmdL);
 683
 684    ReleaseWriter Release("");
 685    if (_error->PendingError() == true)
 686       return false;
 687
 688    if (Release.RecursiveScan(CmdL.FileList[1]) == false)
 689       return false;
 690
 691    Release.Finish();
 692
 693    return true;
 694 }
 695
 696                                                                         /*}}}*/
 697 // Generate - Full generate, using a config file                        /*{{{*/
 698 // ---------------------------------------------------------------------
 699 /* */
 700 bool Generate(CommandLine &CmdL)
 701 {
 702    struct CacheDB::Stats SrcStats;
 703    if (CmdL.FileSize() < 2)
 704       return ShowHelp(CmdL);
 705
 706    struct timeval StartTime;
 707    gettimeofday(&StartTime,0);
 708    struct CacheDB::Stats Stats;
 709
 710    // Read the configuration file.
 711    Configuration Setup;
 712    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 713       return false;
 714
 715    vector<PackageMap> PkgList;
 716    LoadTree(PkgList,Setup);
 717    LoadBinDir(PkgList,Setup);
 718
 719    // Sort by cache DB to improve IO locality.
 720    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 721
 722    // Generate packages
 723    if (CmdL.FileSize() <= 2)
 724    {
 725       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 726          if (I->GenPackages(Setup,Stats) == false)
 727             _error->DumpErrors();
 728       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 729          if (I->GenSources(Setup,SrcStats) == false)
 730             _error->DumpErrors();
 731    }
 732    else
 733    {
 734       // Make a choice list out of the package list..
 735       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 736       RxChoiceList *End = List;
 737       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 738       {
 739          End->UserData = &(*I);
 740          End->Str = I->BaseDir.c_str();
 741          End++;
 742
 743          End->UserData = &(*I);
 744          End->Str = I->Tag.c_str();
 745          End++;
 746       }
 747       End->Str = 0;
 748
 749       // Regex it
 750       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 751       {
 752          delete [] List;
 753          return _error->Error(_("No selections matched"));
 754       }
 755       _error->DumpErrors();
 756
 757       // Do the generation for Packages
 758       for (End = List; End->Str != 0; End++)
 759       {
 760          if (End->Hit == false)
 761             continue;
 762
 763          PackageMap *I = (PackageMap *)End->UserData;
 764          if (I->PkgDone == true)
 765             continue;
 766          if (I->GenPackages(Setup,Stats) == false)
 767             _error->DumpErrors();
 768       }
 769
 770       // Do the generation for Sources
 771       for (End = List; End->Str != 0; End++)
 772       {
 773          if (End->Hit == false)
 774             continue;
 775
 776          PackageMap *I = (PackageMap *)End->UserData;
 777          if (I->SrcDone == true)
 778             continue;
 779          if (I->GenSources(Setup,SrcStats) == false)
 780             _error->DumpErrors();
 781       }
 782
 783       delete [] List;
 784    }
 785
 786    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 787       return true;
 788
 789    c1out << "Done Packages, Starting contents." << endl;
 790
 791    // Sort the contents file list by date
 792    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 793    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 794    {
 795       struct stat A;
 796       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 797                                  I->CntCompress,A) == false)
 798          time(&I->ContentsMTime);
 799       else
 800          I->ContentsMTime = A.st_mtime;
 801    }
 802    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 803
 804    /* Now for Contents.. The process here is to do a make-like dependency
 805       check. Each contents file is verified to be newer than the package files
 806       that describe the debs it indexes. Since the package files contain
 807       hashes of the .debs this means they have not changed either so the
 808       contents must be up to date. */
 809    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 810    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 811    {
 812       // This record is not relevent
 813       if (I->ContentsDone == true ||
 814           I->Contents.empty() == true)
 815          continue;
 816
 817       // Do not do everything if the user specified sections.
 818       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 819          continue;
 820
 821       struct stat A,B;
 822       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 823       {
 824          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 825          {
 826             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 827             continue;
 828          }
 829
 830          if (A.st_mtime > B.st_mtime)
 831             continue;
 832       }
 833
 834       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 835                          MaxContentsChange) == false)
 836          _error->DumpErrors();
 837
 838       // Hit the limit?
 839       if (MaxContentsChange == 0)
 840       {
 841          c1out << "Hit contents update byte limit" << endl;
 842          break;
 843       }
 844    }
 845
 846    struct timeval NewTime;
 847    gettimeofday(&NewTime,0);
 848    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 849                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 850    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 851          << " archives. Took " << TimeToStr((long)Delta) << endl;
 852
 853    return true;
 854 }
 855                                                                         /*}}}*/
 856 // Clean - Clean out the databases                                      /*{{{*/
 857 // ---------------------------------------------------------------------
 858 /* */
 859 bool Clean(CommandLine &CmdL)
 860 {
 861    if (CmdL.FileSize() != 2)
 862       return ShowHelp(CmdL);
 863
 864    // Read the configuration file.
 865    Configuration Setup;
 866    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 867       return false;
 868
 869    vector<PackageMap> PkgList;
 870    LoadTree(PkgList,Setup);
 871    LoadBinDir(PkgList,Setup);
 872
 873    // Sort by cache DB to improve IO locality.
 874    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 875
 876    string CacheDir = Setup.FindDir("Dir::CacheDir");
 877
 878    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 879    {
 880       c0out << I->BinCacheDB << endl;
 881       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 882       if (DB.Clean() == false)
 883          _error->DumpErrors();
 884
 885       string CacheDB = I->BinCacheDB;
 886       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
 887    }
 888
 889    return true;
 890 }
 891                                                                         /*}}}*/
 892
 893 int main(int argc, const char *argv[])
 894 {
 895    CommandLine::Args Args[] = {
 896       {'h',"help","help",0},
 897       {0,"md5","APT::FTPArchive::MD5",0},
 898       {'v',"version","version",0},
 899       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 900       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 901       {'q',"quiet","quiet",CommandLine::IntLevel},
 902       {'q',"silent","quiet",CommandLine::IntLevel},
 903       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 904       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 905       {0,"contents","APT::FTPArchive::Contents",0},
 906       {'c',"config-file",0,CommandLine::ConfigFile},
 907       {'o',"option",0,CommandLine::ArbItem},
 908       {0,0,0,0}};
 909    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 910                                    {"contents",&SimpleGenContents},
 911                                    {"sources",&SimpleGenSources},
 912                                    {"release",&SimpleGenRelease},
 913                                    {"generate",&Generate},
 914                                    {"clean",&Clean},
 915                                    {"help",&ShowHelp},
 916                                    {0,0}};
 917
 918    // Parse the command line and initialize the package library
 919    CommandLine CmdL(Args,_config);
 920    if (CmdL.Parse(argc,argv) == false)
 921    {
 922       _error->DumpErrors();
 923       return 100;
 924    }
 925
 926    // See if the help should be shown
 927    if (_config->FindB("help") == true ||
 928        _config->FindB("version") == true ||
 929        CmdL.FileSize() == 0)
 930    {
 931       ShowHelp(CmdL);
 932       return 0;
 933    }
 934
 935    // Setup the output streams
 936    c0out.rdbuf(clog.rdbuf());
 937    c1out.rdbuf(clog.rdbuf());
 938    c2out.rdbuf(clog.rdbuf());
 939    Quiet = _config->FindI("quiet",0);
 940    if (Quiet > 0)
 941       c0out.rdbuf(devnull.rdbuf());
 942    if (Quiet > 1)
 943       c1out.rdbuf(devnull.rdbuf());
 944
 945    // Match the operation
 946    CmdL.DispatchArg(Cmds);
 947
 948    if (_error->empty() == false)
 949    {
 950       bool Errors = _error->PendingError();
 951       _error->DumpErrors();
 952       return Errors == true?100:0;
 953    }
 954    return 0;
 955 }