ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-scanpackages - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "apt-ftparchive.h"
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <config.h>
  20 #include <apti18n.h>
  21 #include <algorithm>
  22
  23 #include <climits>
  24 #include <sys/time.h>
  25 #include <regex.h>
  26
  27 #include "contents.h"
  28 #include "multicompress.h"
  29 #include "writer.h"
  30                                                                         /*}}}*/
  31
  32 using namespace std;
  33 ostream c0out(0);
  34 ostream c1out(0);
  35 ostream c2out(0);
  36 ofstream devnull("/dev/null");
  37 unsigned Quiet = 0;
  38
  39 // struct PackageMap - List of all package files in the config file     /*{{{*/
  40 // ---------------------------------------------------------------------
  41 /* */
  42 struct PackageMap
  43 {
  44    // General Stuff
  45    string BaseDir;
  46    string InternalPrefix;
  47    string FLFile;
  48    string PkgExt;
  49    string SrcExt;
  50
  51    // Stuff for the Package File
  52    string PkgFile;
  53    string BinCacheDB;
  54    string BinOverride;
  55    string ExtraOverride;
  56
  57    // We generate for this given arch
  58    string Arch;
  59
  60    // Stuff for the Source File
  61    string SrcFile;
  62    string SrcOverride;
  63    string SrcExtraOverride;
  64
  65    // Contents
  66    string Contents;
  67    string ContentsHead;
  68
  69    // Random things
  70    string Tag;
  71    string PkgCompress;
  72    string CntCompress;
  73    string SrcCompress;
  74    string PathPrefix;
  75    unsigned int DeLinkLimit;
  76    mode_t Permissions;
  77
  78    bool ContentsDone;
  79    bool PkgDone;
  80    bool SrcDone;
  81    time_t ContentsMTime;
  82
  83    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  84    {
  85       inline bool operator() (const PackageMap &x,const PackageMap &y)
  86       {return x.ContentsMTime < y.ContentsMTime;};
  87    };
  88
  89    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  90    {
  91       inline bool operator() (const PackageMap &x,const PackageMap &y)
  92       {return x.BinCacheDB < y.BinCacheDB;};
  93    };
  94
  95    void GetGeneral(Configuration &Setup,Configuration &Block);
  96    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
  97    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
  98    bool GenContents(Configuration &Setup,
  99                     vector<PackageMap>::iterator Begin,
 100                     vector<PackageMap>::iterator End,
 101                     unsigned long &Left);
 102
 103    PackageMap() : DeLinkLimit(0), Permissions(1), ContentsDone(false),
 104         PkgDone(false), SrcDone(false), ContentsMTime(0) {};
 105 };
 106                                                                         /*}}}*/
 107
 108 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 109 // ---------------------------------------------------------------------
 110 /* */
 111 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 112 {
 113    PathPrefix = Block.Find("PathPrefix");
 114
 115    if (Block.FindB("External-Links",true) == false)
 116       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 117    else
 118       DeLinkLimit = 0;
 119
 120    PkgCompress = Block.Find("Packages::Compress",
 121                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 122    CntCompress = Block.Find("Contents::Compress",
 123                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 124    SrcCompress = Block.Find("Sources::Compress",
 125                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 126
 127    SrcExt = Block.Find("Sources::Extensions",
 128                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 129    PkgExt = Block.Find("Packages::Extensions",
 130                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 131
 132    Permissions = Setup.FindI("Default::FileMode",0644);
 133
 134    if (FLFile.empty() == false)
 135       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 136
 137    if (Contents == " ")
 138       Contents= string();
 139 }
 140                                                                         /*}}}*/
 141 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 142 // ---------------------------------------------------------------------
 143 /* This generates the Package File described by this object. */
 144 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 145 {
 146    if (PkgFile.empty() == true)
 147       return true;
 148
 149    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 150    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 151    string CacheDir = Setup.FindDir("Dir::CacheDir");
 152
 153    struct timeval StartTime;
 154    gettimeofday(&StartTime,0);
 155
 156    PkgDone = true;
 157
 158    // Create a package writer object.
 159    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 160                            flCombine(OverrideDir,BinOverride),
 161                            flCombine(OverrideDir,ExtraOverride),
 162                            Arch);
 163    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 164       return _error->Error(_("Package extension list is too long"));
 165    if (_error->PendingError() == true)
 166       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 167
 168    Packages.PathPrefix = PathPrefix;
 169    Packages.DirStrip = ArchiveDir;
 170    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 171
 172    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 173    Packages.DeLinkLimit = DeLinkLimit;
 174
 175    // Create a compressor object
 176    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 177                       PkgCompress,Permissions);
 178    Packages.Output = Comp.Input;
 179    if (_error->PendingError() == true)
 180       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 181
 182    c0out << ' ' << BaseDir << ":" << flush;
 183
 184    // Do recursive directory searching
 185    if (FLFile.empty() == true)
 186    {
 187       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 188          return false;
 189    }
 190    else
 191    {
 192       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 193          return false;
 194    }
 195
 196    Packages.Output = 0;      // Just in case
 197
 198    // Finish compressing
 199    unsigned long Size;
 200    if (Comp.Finalize(Size) == false)
 201    {
 202       c0out << endl;
 203       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 204    }
 205
 206    if (Size != 0)
 207       c0out << " New "
 208              << SizeToStr(Size) << "B ";
 209    else
 210       c0out << ' ';
 211
 212    struct timeval NewTime;
 213    gettimeofday(&NewTime,0);
 214    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 215                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 216
 217    c0out << Packages.Stats.Packages << " files " <<
 218 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 219       SizeToStr(Packages.Stats.Bytes) << "B " <<
 220       TimeToStr((long)Delta) << endl;
 221
 222    Stats.Add(Packages.Stats);
 223    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 224
 225    return !_error->PendingError();
 226 }
 227
 228                                                                         /*}}}*/
 229 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 230 // ---------------------------------------------------------------------
 231 /* This generates the Sources File described by this object. */
 232 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 233 {
 234    if (SrcFile.empty() == true)
 235       return true;
 236
 237    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 238    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 239    string CacheDir = Setup.FindDir("Dir::CacheDir");
 240
 241    struct timeval StartTime;
 242    gettimeofday(&StartTime,0);
 243
 244    SrcDone = true;
 245
 246    // Create a package writer object.
 247    SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
 248                          flCombine(OverrideDir,SrcOverride),
 249                          flCombine(OverrideDir,SrcExtraOverride));
 250    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 251       return _error->Error(_("Source extension list is too long"));
 252    if (_error->PendingError() == true)
 253       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 254
 255    Sources.PathPrefix = PathPrefix;
 256    Sources.DirStrip = ArchiveDir;
 257    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 258
 259    Sources.DeLinkLimit = DeLinkLimit;
 260    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 261
 262    // Create a compressor object
 263    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 264                       SrcCompress,Permissions);
 265    Sources.Output = Comp.Input;
 266    if (_error->PendingError() == true)
 267       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 268
 269    c0out << ' ' << BaseDir << ":" << flush;
 270
 271    // Do recursive directory searching
 272    if (FLFile.empty() == true)
 273    {
 274       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 275          return false;
 276    }
 277    else
 278    {
 279       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 280          return false;
 281    }
 282    Sources.Output = 0;      // Just in case
 283
 284    // Finish compressing
 285    unsigned long Size;
 286    if (Comp.Finalize(Size) == false)
 287    {
 288       c0out << endl;
 289       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 290    }
 291
 292    if (Size != 0)
 293       c0out << " New "
 294              << SizeToStr(Size) << "B ";
 295    else
 296       c0out << ' ';
 297
 298    struct timeval NewTime;
 299    gettimeofday(&NewTime,0);
 300    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 301                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 302
 303    c0out << Sources.Stats.Packages << " pkgs in " <<
 304       TimeToStr((long)Delta) << endl;
 305
 306    Stats.Add(Sources.Stats);
 307    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 308
 309    return !_error->PendingError();
 310 }
 311                                                                         /*}}}*/
 312 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 313 // ---------------------------------------------------------------------
 314 /* This generates the contents file partially described by this object.
 315    It searches the given iterator range for other package files that map
 316    into this contents file and includes their data as well when building. */
 317 bool PackageMap::GenContents(Configuration &Setup,
 318                              vector<PackageMap>::iterator Begin,
 319                              vector<PackageMap>::iterator End,
 320                              unsigned long &Left)
 321 {
 322    if (Contents.empty() == true)
 323       return true;
 324
 325    if (Left == 0)
 326       return true;
 327
 328    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 329    string CacheDir = Setup.FindDir("Dir::CacheDir");
 330    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 331
 332    struct timeval StartTime;
 333    gettimeofday(&StartTime,0);
 334
 335    // Create a package writer object.
 336    ContentsWriter Contents("");
 337    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 338       return _error->Error(_("Package extension list is too long"));
 339    if (_error->PendingError() == true)
 340       return false;
 341
 342    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 343                       CntCompress,Permissions);
 344    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 345    Contents.Output = Comp.Input;
 346    if (_error->PendingError() == true)
 347       return false;
 348
 349    // Write the header out.
 350    if (ContentsHead.empty() == false)
 351    {
 352       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 353       if (_error->PendingError() == true)
 354          return false;
 355
 356       unsigned long Size = Head.Size();
 357       unsigned char Buf[4096];
 358       while (Size != 0)
 359       {
 360          unsigned long ToRead = Size;
 361          if (Size > sizeof(Buf))
 362             ToRead = sizeof(Buf);
 363
 364          if (Head.Read(Buf,ToRead) == false)
 365             return false;
 366
 367          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 368             return _error->Errno("fwrite",_("Error writing header to contents file"));
 369
 370          Size -= ToRead;
 371       }
 372    }
 373
 374    /* Go over all the package file records and parse all the package
 375       files associated with this contents file into one great big honking
 376       memory structure, then dump the sorted version */
 377    c0out << ' ' << this->Contents << ":" << flush;
 378    for (vector<PackageMap>::iterator I = Begin; I != End; I++)
 379    {
 380       if (I->Contents != this->Contents)
 381          continue;
 382
 383       Contents.Prefix = ArchiveDir;
 384       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 385       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 386                             I->PkgCompress);
 387
 388       I->ContentsDone = true;
 389    }
 390
 391    Contents.Finish();
 392
 393    // Finish compressing
 394    unsigned long Size;
 395    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 396    {
 397       c0out << endl;
 398       return _error->Error(_("Error processing contents %s"),
 399                            this->Contents.c_str());
 400    }
 401
 402    if (Size != 0)
 403    {
 404       c0out << " New " << SizeToStr(Size) << "B ";
 405       if (Left > Size)
 406          Left -= Size;
 407       else
 408          Left = 0;
 409    }
 410    else
 411       c0out << ' ';
 412
 413    struct timeval NewTime;
 414    gettimeofday(&NewTime,0);
 415    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 416                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 417
 418    c0out << Contents.Stats.Packages << " files " <<
 419       SizeToStr(Contents.Stats.Bytes) << "B " <<
 420       TimeToStr((long)Delta) << endl;
 421
 422    return true;
 423 }
 424                                                                         /*}}}*/
 425
 426 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 427 // ---------------------------------------------------------------------
 428 /* This populates the PkgList with all the possible permutations of the
 429    section/arch lists. */
 430 void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 431 {
 432    // Load the defaults
 433    string DDir = Setup.Find("TreeDefault::Directory",
 434                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 435    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 436                             "$(DIST)/$(SECTION)/source/");
 437    string DPkg = Setup.Find("TreeDefault::Packages",
 438                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 439    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 440                             "$(DIST)/$(SECTION)/");
 441    string DContents = Setup.Find("TreeDefault::Contents",
 442                             "$(DIST)/Contents-$(ARCH)");
 443    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 444    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 445                                "packages-$(ARCH).db");
 446    string DSources = Setup.Find("TreeDefault::Sources",
 447                                 "$(DIST)/$(SECTION)/source/Sources");
 448    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 449    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 450
 451    // Process 'tree' type sections
 452    const Configuration::Item *Top = Setup.Tree("tree");
 453    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 454    {
 455       Configuration Block(Top);
 456       string Dist = Top->Tag;
 457
 458       // Parse the sections
 459       string Tmp = Block.Find("Sections");
 460       const char *Sections = Tmp.c_str();
 461       string Section;
 462       while (ParseQuoteWord(Sections,Section) == true)
 463       {
 464          string Tmp2 = Block.Find("Architectures");
 465          string Arch;
 466          const char *Archs = Tmp2.c_str();
 467          while (ParseQuoteWord(Archs,Arch) == true)
 468          {
 469             struct SubstVar Vars[] = {{"$(DIST)",&Dist},
 470                                       {"$(SECTION)",&Section},
 471                                       {"$(ARCH)",&Arch},
 472                                       {}};
 473             PackageMap Itm;
 474
 475             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 476             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 477
 478             if (stringcasecmp(Arch,"source") == 0)
 479             {
 480                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 481                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 482                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 483                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 484                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 485                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 486             }
 487             else
 488             {
 489                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 490                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 491                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 492                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 493                Itm.Arch = Arch;
 494                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 495                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 496                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 497                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 498             }
 499
 500             Itm.GetGeneral(Setup,Block);
 501             PkgList.push_back(Itm);
 502          }
 503       }
 504
 505       Top = Top->Next;
 506    }
 507 }
 508                                                                         /*}}}*/
 509 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 510 // ---------------------------------------------------------------------
 511 /* */
 512 void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 513 {
 514    // Process 'bindirectory' type sections
 515    const Configuration::Item *Top = Setup.Tree("bindirectory");
 516    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 517    {
 518       Configuration Block(Top);
 519
 520       PackageMap Itm;
 521       Itm.PkgFile = Block.Find("Packages");
 522       Itm.SrcFile = Block.Find("Sources");
 523       Itm.BinCacheDB = Block.Find("BinCacheDB");
 524       Itm.BinOverride = Block.Find("BinOverride");
 525       Itm.ExtraOverride = Block.Find("ExtraOverride");
 526       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 527       Itm.SrcOverride = Block.Find("SrcOverride");
 528       Itm.BaseDir = Top->Tag;
 529       Itm.FLFile = Block.Find("FileList");
 530       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 531       Itm.Contents = Block.Find("Contents");
 532       Itm.ContentsHead = Block.Find("Contents::Header");
 533
 534       Itm.GetGeneral(Setup,Block);
 535       PkgList.push_back(Itm);
 536
 537       Top = Top->Next;
 538    }
 539 }
 540                                                                         /*}}}*/
 541
 542 // ShowHelp - Show the help text                                        /*{{{*/
 543 // ---------------------------------------------------------------------
 544 /* */
 545 bool ShowHelp(CommandLine &CmdL)
 546 {
 547    ioprintf(cout,_("%s %s for %s %s compiled on %s %s\n"),PACKAGE,VERSION,
 548             COMMON_OS,COMMON_CPU,__DATE__,__TIME__);
 549    if (_config->FindB("version") == true)
 550       return true;
 551
 552    cout <<
 553     _("Usage: apt-ftparchive [options] command\n"
 554       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 555       "          sources srcpath [overridefile [pathprefix]]\n"
 556       "          contents path\n"
 557       "          release path\n"
 558       "          generate config [groups]\n"
 559       "          clean config\n"
 560       "\n"
 561       "apt-ftparchive generates index files for Debian archives. It supports\n"
 562       "many styles of generation from fully automated to functional replacements\n"
 563       "for dpkg-scanpackages and dpkg-scansources\n"
 564       "\n"
 565       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 566       "Package file contains the contents of all the control fields from\n"
 567       "each package as well as the MD5 hash and filesize. An override file\n"
 568       "is supported to force the value of Priority and Section.\n"
 569       "\n"
 570       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 571       "The --source-override option can be used to specify a src override file\n"
 572       "\n"
 573       "The 'packages' and 'sources' command should be run in the root of the\n"
 574       "tree. BinaryPath should point to the base of the recursive search and \n"
 575       "override file should contain the override flags. Pathprefix is\n"
 576       "appended to the filename fields if present. Example usage from the \n"
 577       "Debian archive:\n"
 578       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 579       "               dists/potato/main/binary-i386/Packages\n"
 580       "\n"
 581       "Options:\n"
 582       "  -h    This help text\n"
 583       "  --md5 Control MD5 generation\n"
 584       "  -s=?  Source override file\n"
 585       "  -q    Quiet\n"
 586       "  -d=?  Select the optional caching database\n"
 587       "  --no-delink Enable delinking debug mode\n"
 588       "  --contents  Control contents file generation\n"
 589       "  -c=?  Read this configuration file\n"
 590       "  -o=?  Set an arbitrary configuration option") << endl;
 591
 592    return true;
 593 }
 594                                                                         /*}}}*/
 595 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 596 // ---------------------------------------------------------------------
 597 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 598 bool SimpleGenPackages(CommandLine &CmdL)
 599 {
 600    if (CmdL.FileSize() < 2)
 601       return ShowHelp(CmdL);
 602
 603    string Override;
 604    if (CmdL.FileSize() >= 3)
 605       Override = CmdL.FileList[2];
 606
 607    // Create a package writer object.
 608    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 609                            Override, "");
 610    if (_error->PendingError() == true)
 611       return false;
 612
 613    if (CmdL.FileSize() >= 4)
 614       Packages.PathPrefix = CmdL.FileList[3];
 615
 616    // Do recursive directory searching
 617    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 618       return false;
 619
 620    return true;
 621 }
 622                                                                         /*}}}*/
 623 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 624 // ---------------------------------------------------------------------
 625 /* */
 626 bool SimpleGenContents(CommandLine &CmdL)
 627 {
 628    if (CmdL.FileSize() < 2)
 629       return ShowHelp(CmdL);
 630
 631    // Create a package writer object.
 632    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"));
 633    if (_error->PendingError() == true)
 634       return false;
 635
 636    // Do recursive directory searching
 637    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 638       return false;
 639
 640    Contents.Finish();
 641
 642    return true;
 643 }
 644                                                                         /*}}}*/
 645 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 646 // ---------------------------------------------------------------------
 647 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 648 bool SimpleGenSources(CommandLine &CmdL)
 649 {
 650    if (CmdL.FileSize() < 2)
 651       return ShowHelp(CmdL);
 652
 653    string Override;
 654    if (CmdL.FileSize() >= 3)
 655       Override = CmdL.FileList[2];
 656
 657    string SOverride;
 658    if (Override.empty() == false)
 659       SOverride = Override + ".src";
 660
 661    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 662                              SOverride.c_str());
 663
 664    // Create a package writer object.
 665    SourcesWriter Sources(Override,SOverride);
 666    if (_error->PendingError() == true)
 667       return false;
 668
 669    if (CmdL.FileSize() >= 4)
 670       Sources.PathPrefix = CmdL.FileList[3];
 671
 672    // Do recursive directory searching
 673    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 674       return false;
 675
 676    return true;
 677 }
 678                                                                         /*}}}*/
 679 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 680 // ---------------------------------------------------------------------
 681 bool SimpleGenRelease(CommandLine &CmdL)
 682 {
 683    if (CmdL.FileSize() < 2)
 684       return ShowHelp(CmdL);
 685
 686    string Dir = CmdL.FileList[1];
 687
 688    ReleaseWriter Release("");
 689    Release.DirStrip = Dir;
 690
 691    if (_error->PendingError() == true)
 692       return false;
 693
 694    if (Release.RecursiveScan(Dir) == false)
 695       return false;
 696
 697    Release.Finish();
 698
 699    return true;
 700 }
 701
 702                                                                         /*}}}*/
 703 // Generate - Full generate, using a config file                        /*{{{*/
 704 // ---------------------------------------------------------------------
 705 /* */
 706 bool Generate(CommandLine &CmdL)
 707 {
 708    struct CacheDB::Stats SrcStats;
 709    if (CmdL.FileSize() < 2)
 710       return ShowHelp(CmdL);
 711
 712    struct timeval StartTime;
 713    gettimeofday(&StartTime,0);
 714    struct CacheDB::Stats Stats;
 715
 716    // Read the configuration file.
 717    Configuration Setup;
 718    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 719       return false;
 720
 721    vector<PackageMap> PkgList;
 722    LoadTree(PkgList,Setup);
 723    LoadBinDir(PkgList,Setup);
 724
 725    // Sort by cache DB to improve IO locality.
 726    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 727
 728    // Generate packages
 729    if (CmdL.FileSize() <= 2)
 730    {
 731       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 732          if (I->GenPackages(Setup,Stats) == false)
 733             _error->DumpErrors();
 734       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 735          if (I->GenSources(Setup,SrcStats) == false)
 736             _error->DumpErrors();
 737    }
 738    else
 739    {
 740       // Make a choice list out of the package list..
 741       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 742       RxChoiceList *End = List;
 743       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 744       {
 745          End->UserData = &(*I);
 746          End->Str = I->BaseDir.c_str();
 747          End++;
 748
 749          End->UserData = &(*I);
 750          End->Str = I->Tag.c_str();
 751          End++;
 752       }
 753       End->Str = 0;
 754
 755       // Regex it
 756       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 757       {
 758          delete [] List;
 759          return _error->Error(_("No selections matched"));
 760       }
 761       _error->DumpErrors();
 762
 763       // Do the generation for Packages
 764       for (End = List; End->Str != 0; End++)
 765       {
 766          if (End->Hit == false)
 767             continue;
 768
 769          PackageMap *I = (PackageMap *)End->UserData;
 770          if (I->PkgDone == true)
 771             continue;
 772          if (I->GenPackages(Setup,Stats) == false)
 773             _error->DumpErrors();
 774       }
 775
 776       // Do the generation for Sources
 777       for (End = List; End->Str != 0; End++)
 778       {
 779          if (End->Hit == false)
 780             continue;
 781
 782          PackageMap *I = (PackageMap *)End->UserData;
 783          if (I->SrcDone == true)
 784             continue;
 785          if (I->GenSources(Setup,SrcStats) == false)
 786             _error->DumpErrors();
 787       }
 788
 789       delete [] List;
 790    }
 791
 792    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 793       return true;
 794
 795    c1out << "Done Packages, Starting contents." << endl;
 796
 797    // Sort the contents file list by date
 798    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 799    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 800    {
 801       struct stat A;
 802       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 803                                  I->CntCompress,A) == false)
 804          time(&I->ContentsMTime);
 805       else
 806          I->ContentsMTime = A.st_mtime;
 807    }
 808    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 809
 810    /* Now for Contents.. The process here is to do a make-like dependency
 811       check. Each contents file is verified to be newer than the package files
 812       that describe the debs it indexes. Since the package files contain
 813       hashes of the .debs this means they have not changed either so the
 814       contents must be up to date. */
 815    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 816    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 817    {
 818       // This record is not relevent
 819       if (I->ContentsDone == true ||
 820           I->Contents.empty() == true)
 821          continue;
 822
 823       // Do not do everything if the user specified sections.
 824       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 825          continue;
 826
 827       struct stat A,B;
 828       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 829       {
 830          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 831          {
 832             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 833             continue;
 834          }
 835
 836          if (A.st_mtime > B.st_mtime)
 837             continue;
 838       }
 839
 840       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 841                          MaxContentsChange) == false)
 842          _error->DumpErrors();
 843
 844       // Hit the limit?
 845       if (MaxContentsChange == 0)
 846       {
 847          c1out << "Hit contents update byte limit" << endl;
 848          break;
 849       }
 850    }
 851
 852    struct timeval NewTime;
 853    gettimeofday(&NewTime,0);
 854    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 855                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 856    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 857          << " archives. Took " << TimeToStr((long)Delta) << endl;
 858
 859    return true;
 860 }
 861                                                                         /*}}}*/
 862 // Clean - Clean out the databases                                      /*{{{*/
 863 // ---------------------------------------------------------------------
 864 /* */
 865 bool Clean(CommandLine &CmdL)
 866 {
 867    if (CmdL.FileSize() != 2)
 868       return ShowHelp(CmdL);
 869
 870    // Read the configuration file.
 871    Configuration Setup;
 872    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 873       return false;
 874
 875    vector<PackageMap> PkgList;
 876    LoadTree(PkgList,Setup);
 877    LoadBinDir(PkgList,Setup);
 878
 879    // Sort by cache DB to improve IO locality.
 880    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 881
 882    string CacheDir = Setup.FindDir("Dir::CacheDir");
 883
 884    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 885    {
 886       c0out << I->BinCacheDB << endl;
 887       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 888       if (DB.Clean() == false)
 889          _error->DumpErrors();
 890
 891       string CacheDB = I->BinCacheDB;
 892       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
 893    }
 894
 895    return true;
 896 }
 897                                                                         /*}}}*/
 898
 899 int main(int argc, const char *argv[])
 900 {
 901    CommandLine::Args Args[] = {
 902       {'h',"help","help",0},
 903       {0,"md5","APT::FTPArchive::MD5",0},
 904       {'v',"version","version",0},
 905       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 906       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 907       {'q',"quiet","quiet",CommandLine::IntLevel},
 908       {'q',"silent","quiet",CommandLine::IntLevel},
 909       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 910       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 911       {0,"contents","APT::FTPArchive::Contents",0},
 912       {'c',"config-file",0,CommandLine::ConfigFile},
 913       {'o',"option",0,CommandLine::ArbItem},
 914       {0,0,0,0}};
 915    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 916                                    {"contents",&SimpleGenContents},
 917                                    {"sources",&SimpleGenSources},
 918                                    {"release",&SimpleGenRelease},
 919                                    {"generate",&Generate},
 920                                    {"clean",&Clean},
 921                                    {"help",&ShowHelp},
 922                                    {0,0}};
 923
 924    // Parse the command line and initialize the package library
 925    CommandLine CmdL(Args,_config);
 926    if (CmdL.Parse(argc,argv) == false)
 927    {
 928       _error->DumpErrors();
 929       return 100;
 930    }
 931
 932    // See if the help should be shown
 933    if (_config->FindB("help") == true ||
 934        _config->FindB("version") == true ||
 935        CmdL.FileSize() == 0)
 936    {
 937       ShowHelp(CmdL);
 938       return 0;
 939    }
 940
 941    // Setup the output streams
 942    c0out.rdbuf(clog.rdbuf());
 943    c1out.rdbuf(clog.rdbuf());
 944    c2out.rdbuf(clog.rdbuf());
 945    Quiet = _config->FindI("quiet",0);
 946    if (Quiet > 0)
 947       c0out.rdbuf(devnull.rdbuf());
 948    if (Quiet > 1)
 949       c1out.rdbuf(devnull.rdbuf());
 950
 951    // Match the operation
 952    CmdL.DispatchArg(Cmds);
 953
 954    if (_error->empty() == false)
 955    {
 956       bool Errors = _error->PendingError();
 957       _error->DumpErrors();
 958       return Errors == true?100:0;
 959    }
 960    return 0;
 961 }