ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8 2003/07/18 15:28:58 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-scanpackages - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-ftparchive.h"
  15 #endif
  16
  17 #include "apt-ftparchive.h"
  18
  19 #include <apt-pkg/error.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/cmndline.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <config.h>
  24 #include <apti18n.h>
  25 #include <algorithm>
  26
  27 #include <sys/time.h>
  28 #include <regex.h>
  29
  30 #include "contents.h"
  31 #include "multicompress.h"
  32 #include "writer.h"
  33                                                                         /*}}}*/
  34
  35 using namespace std;
  36 ostream c0out(0);
  37 ostream c1out(0);
  38 ostream c2out(0);
  39 ofstream devnull("/dev/null");
  40 unsigned Quiet = 0;
  41
  42 // struct PackageMap - List of all package files in the config file     /*{{{*/
  43 // ---------------------------------------------------------------------
  44 /* */
  45 struct PackageMap
  46 {
  47    // General Stuff
  48    string BaseDir;
  49    string InternalPrefix;
  50    string FLFile;
  51    string PkgExt;
  52    string SrcExt;
  53
  54    // Stuff for the Package File
  55    string PkgFile;
  56    string BinCacheDB;
  57    string BinOverride;
  58    string ExtraOverride;
  59
  60    // Stuff for the Source File
  61    string SrcFile;
  62    string SrcOverride;
  63    string SrcExtraOverride;
  64
  65    // Contents
  66    string Contents;
  67    string ContentsHead;
  68
  69    // Random things
  70    string Tag;
  71    string PkgCompress;
  72    string CntCompress;
  73    string SrcCompress;
  74    string PathPrefix;
  75    unsigned int DeLinkLimit;
  76    mode_t Permissions;
  77
  78    bool ContentsDone;
  79    bool PkgDone;
  80    bool SrcDone;
  81    time_t ContentsMTime;
  82
  83    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  84    {
  85       inline bool operator() (const PackageMap &x,const PackageMap &y)
  86       {return x.ContentsMTime < y.ContentsMTime;};
  87    };
  88
  89    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  90    {
  91       inline bool operator() (const PackageMap &x,const PackageMap &y)
  92       {return x.BinCacheDB < y.BinCacheDB;};
  93    };
  94
  95    void GetGeneral(Configuration &Setup,Configuration &Block);
  96    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
  97    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
  98    bool GenContents(Configuration &Setup,
  99                     vector<PackageMap>::iterator Begin,
 100                     vector<PackageMap>::iterator End,
 101                     unsigned long &Left);
 102
 103    PackageMap() : DeLinkLimit(0), Permissions(1), ContentsDone(false),
 104         PkgDone(false), SrcDone(false), ContentsMTime(0) {};
 105 };
 106                                                                         /*}}}*/
 107
 108 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 109 // ---------------------------------------------------------------------
 110 /* */
 111 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 112 {
 113    PathPrefix = Block.Find("PathPrefix");
 114
 115    if (Block.FindB("External-Links",true) == false)
 116       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 117    else
 118       DeLinkLimit = 0;
 119
 120    PkgCompress = Block.Find("Packages::Compress",
 121                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 122    CntCompress = Block.Find("Contents::Compress",
 123                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 124    SrcCompress = Block.Find("Sources::Compress",
 125                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 126
 127    SrcExt = Block.Find("Sources::Extensions",
 128                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 129    PkgExt = Block.Find("Packages::Extensions",
 130                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 131
 132    Permissions = Setup.FindI("Default::FileMode",0644);
 133
 134    if (FLFile.empty() == false)
 135       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 136
 137    if (Contents == " ")
 138       Contents= string();
 139 }
 140                                                                         /*}}}*/
 141 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 142 // ---------------------------------------------------------------------
 143 /* This generates the Package File described by this object. */
 144 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 145 {
 146    if (PkgFile.empty() == true)
 147       return true;
 148
 149    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 150    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 151    string CacheDir = Setup.FindDir("Dir::CacheDir");
 152
 153    struct timeval StartTime;
 154    gettimeofday(&StartTime,0);
 155
 156    PkgDone = true;
 157
 158    // Create a package writer object.
 159    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 160                            flCombine(OverrideDir,BinOverride),
 161                            flCombine(OverrideDir,ExtraOverride));
 162    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 163       return _error->Error(_("Package extension list is too long"));
 164    if (_error->PendingError() == true)
 165       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 166
 167    Packages.PathPrefix = PathPrefix;
 168    Packages.DirStrip = ArchiveDir;
 169    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 170
 171    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 172    Packages.DeLinkLimit = DeLinkLimit;
 173
 174    // Create a compressor object
 175    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 176                       PkgCompress,Permissions);
 177    Packages.Output = Comp.Input;
 178    if (_error->PendingError() == true)
 179       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 180
 181    c0out << ' ' << BaseDir << ":" << flush;
 182
 183    // Do recursive directory searching
 184    if (FLFile.empty() == true)
 185    {
 186       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 187          return false;
 188    }
 189    else
 190    {
 191       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 192          return false;
 193    }
 194
 195    Packages.Output = 0;      // Just in case
 196
 197    // Finish compressing
 198    unsigned long Size;
 199    if (Comp.Finalize(Size) == false)
 200    {
 201       c0out << endl;
 202       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 203    }
 204
 205    if (Size != 0)
 206       c0out << " New "
 207              << SizeToStr(Size) << "B ";
 208    else
 209       c0out << ' ';
 210
 211    struct timeval NewTime;
 212    gettimeofday(&NewTime,0);
 213    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 214                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 215
 216    c0out << Packages.Stats.Packages << " files " <<
 217 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 218       SizeToStr(Packages.Stats.Bytes) << "B " <<
 219       TimeToStr((long)Delta) << endl;
 220
 221    Stats.Add(Packages.Stats);
 222    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 223
 224    return !_error->PendingError();
 225 }
 226                                                                         /*}}}*/
 227 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 228 // ---------------------------------------------------------------------
 229 /* This generates the Sources File described by this object. */
 230 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 231 {
 232    if (SrcFile.empty() == true)
 233       return true;
 234
 235    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 236    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 237    string CacheDir = Setup.FindDir("Dir::CacheDir");
 238
 239    struct timeval StartTime;
 240    gettimeofday(&StartTime,0);
 241
 242    SrcDone = true;
 243
 244    // Create a package writer object.
 245    SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
 246                          flCombine(OverrideDir,SrcOverride),
 247                          flCombine(OverrideDir,SrcExtraOverride));
 248    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 249       return _error->Error(_("Source extension list is too long"));
 250    if (_error->PendingError() == true)
 251       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 252
 253    Sources.PathPrefix = PathPrefix;
 254    Sources.DirStrip = ArchiveDir;
 255    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 256
 257    Sources.DeLinkLimit = DeLinkLimit;
 258    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 259
 260    // Create a compressor object
 261    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 262                       SrcCompress,Permissions);
 263    Sources.Output = Comp.Input;
 264    if (_error->PendingError() == true)
 265       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 266
 267    c0out << ' ' << BaseDir << ":" << flush;
 268
 269    // Do recursive directory searching
 270    if (FLFile.empty() == true)
 271    {
 272       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 273          return false;
 274    }
 275    else
 276    {
 277       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 278          return false;
 279    }
 280    Sources.Output = 0;      // Just in case
 281
 282    // Finish compressing
 283    unsigned long Size;
 284    if (Comp.Finalize(Size) == false)
 285    {
 286       c0out << endl;
 287       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 288    }
 289
 290    if (Size != 0)
 291       c0out << " New "
 292              << SizeToStr(Size) << "B ";
 293    else
 294       c0out << ' ';
 295
 296    struct timeval NewTime;
 297    gettimeofday(&NewTime,0);
 298    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 299                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 300
 301    c0out << Sources.Stats.Packages << " pkgs in " <<
 302       TimeToStr((long)Delta) << endl;
 303
 304    Stats.Add(Sources.Stats);
 305    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 306
 307    return !_error->PendingError();
 308 }
 309                                                                         /*}}}*/
 310 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 311 // ---------------------------------------------------------------------
 312 /* This generates the contents file partially described by this object.
 313    It searches the given iterator range for other package files that map
 314    into this contents file and includes their data as well when building. */
 315 bool PackageMap::GenContents(Configuration &Setup,
 316                              vector<PackageMap>::iterator Begin,
 317                              vector<PackageMap>::iterator End,
 318                              unsigned long &Left)
 319 {
 320    if (Contents.empty() == true)
 321       return true;
 322
 323    if (Left == 0)
 324       return true;
 325
 326    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 327    string CacheDir = Setup.FindDir("Dir::CacheDir");
 328    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 329
 330    struct timeval StartTime;
 331    gettimeofday(&StartTime,0);
 332
 333    // Create a package writer object.
 334    ContentsWriter Contents("");
 335    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 336       return _error->Error(_("Package extension list is too long"));
 337    if (_error->PendingError() == true)
 338       return false;
 339
 340    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 341                       CntCompress,Permissions);
 342    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 343    Contents.Output = Comp.Input;
 344    if (_error->PendingError() == true)
 345       return false;
 346
 347    // Write the header out.
 348    if (ContentsHead.empty() == false)
 349    {
 350       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 351       if (_error->PendingError() == true)
 352          return false;
 353
 354       unsigned long Size = Head.Size();
 355       unsigned char Buf[4096];
 356       while (Size != 0)
 357       {
 358          unsigned long ToRead = Size;
 359          if (Size > sizeof(Buf))
 360             ToRead = sizeof(Buf);
 361
 362          if (Head.Read(Buf,ToRead) == false)
 363             return false;
 364
 365          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 366             return _error->Errno("fwrite",_("Error writing header to contents file"));
 367
 368          Size -= ToRead;
 369       }
 370    }
 371
 372    /* Go over all the package file records and parse all the package
 373       files associated with this contents file into one great big honking
 374       memory structure, then dump the sorted version */
 375    c0out << ' ' << this->Contents << ":" << flush;
 376    for (vector<PackageMap>::iterator I = Begin; I != End; I++)
 377    {
 378       if (I->Contents != this->Contents)
 379          continue;
 380
 381       Contents.Prefix = ArchiveDir;
 382       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 383       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 384                             I->PkgCompress);
 385
 386       I->ContentsDone = true;
 387    }
 388
 389    Contents.Finish();
 390
 391    // Finish compressing
 392    unsigned long Size;
 393    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 394    {
 395       c0out << endl;
 396       return _error->Error(_("Error Processing Contents %s"),
 397                            this->Contents.c_str());
 398    }
 399
 400    if (Size != 0)
 401    {
 402       c0out << " New " << SizeToStr(Size) << "B ";
 403       if (Left > Size)
 404          Left -= Size;
 405       else
 406          Left = 0;
 407    }
 408    else
 409       c0out << ' ';
 410
 411    struct timeval NewTime;
 412    gettimeofday(&NewTime,0);
 413    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 414                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 415
 416    c0out << Contents.Stats.Packages << " files " <<
 417       SizeToStr(Contents.Stats.Bytes) << "B " <<
 418       TimeToStr((long)Delta) << endl;
 419
 420    return true;
 421 }
 422                                                                         /*}}}*/
 423
 424 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 425 // ---------------------------------------------------------------------
 426 /* This populates the PkgList with all the possible permutations of the
 427    section/arch lists. */
 428 void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 429 {
 430    // Load the defaults
 431    string DDir = Setup.Find("TreeDefault::Directory",
 432                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 433    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 434                             "$(DIST)/$(SECTION)/source/");
 435    string DPkg = Setup.Find("TreeDefault::Packages",
 436                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 437    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 438                             "$(DIST)/$(SECTION)/");
 439    string DContents = Setup.Find("TreeDefault::Contents",
 440                             "$(DIST)/Contents-$(ARCH)");
 441    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 442    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 443                                "packages-$(ARCH).db");
 444    string DSources = Setup.Find("TreeDefault::Sources",
 445                                 "$(DIST)/$(SECTION)/source/Sources");
 446    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 447    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 448
 449    // Process 'tree' type sections
 450    const Configuration::Item *Top = Setup.Tree("tree");
 451    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 452    {
 453       Configuration Block(Top);
 454       string Dist = Top->Tag;
 455
 456       // Parse the sections
 457       string Tmp = Block.Find("Sections");
 458       const char *Sections = Tmp.c_str();
 459       string Section;
 460       while (ParseQuoteWord(Sections,Section) == true)
 461       {
 462          string Tmp2 = Block.Find("Architectures");
 463          string Arch;
 464          const char *Archs = Tmp2.c_str();
 465          while (ParseQuoteWord(Archs,Arch) == true)
 466          {
 467             struct SubstVar Vars[] = {{"$(DIST)",&Dist},
 468                                       {"$(SECTION)",&Section},
 469                                       {"$(ARCH)",&Arch},
 470                                       {}};
 471             PackageMap Itm;
 472
 473             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 474             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 475
 476             if (stringcasecmp(Arch,"source") == 0)
 477             {
 478                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 479                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 480                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 481                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 482                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 483                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 484             }
 485             else
 486             {
 487                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 488                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 489                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 490                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 491                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 492                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 493                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 494                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 495             }
 496
 497             Itm.GetGeneral(Setup,Block);
 498             PkgList.push_back(Itm);
 499          }
 500       }
 501
 502       Top = Top->Next;
 503    }
 504 }
 505                                                                         /*}}}*/
 506 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 507 // ---------------------------------------------------------------------
 508 /* */
 509 void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 510 {
 511    // Process 'bindirectory' type sections
 512    const Configuration::Item *Top = Setup.Tree("bindirectory");
 513    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 514    {
 515       Configuration Block(Top);
 516
 517       PackageMap Itm;
 518       Itm.PkgFile = Block.Find("Packages");
 519       Itm.SrcFile = Block.Find("Sources");
 520       Itm.BinCacheDB = Block.Find("BinCacheDB");
 521       Itm.BinOverride = Block.Find("BinOverride");
 522       Itm.ExtraOverride = Block.Find("ExtraOverride");
 523       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 524       Itm.SrcOverride = Block.Find("SrcOverride");
 525       Itm.BaseDir = Top->Tag;
 526       Itm.FLFile = Block.Find("FileList");
 527       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 528       Itm.Contents = Block.Find("Contents");
 529       Itm.ContentsHead = Block.Find("Contents::Header");
 530
 531       Itm.GetGeneral(Setup,Block);
 532       PkgList.push_back(Itm);
 533
 534       Top = Top->Next;
 535    }
 536 }
 537                                                                         /*}}}*/
 538
 539 // ShowHelp - Show the help text                                        /*{{{*/
 540 // ---------------------------------------------------------------------
 541 /* */
 542 bool ShowHelp(CommandLine &CmdL)
 543 {
 544    ioprintf(cout,_("%s %s for %s %s compiled on %s %s\n"),PACKAGE,VERSION,
 545             COMMON_OS,COMMON_CPU,__DATE__,__TIME__);
 546    if (_config->FindB("version") == true)
 547       return true;
 548
 549    cout <<
 550     _("Usage: apt-ftparchive [options] command\n"
 551       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 552       "          sources srcpath [overridefile [pathprefix]]\n"
 553       "          contents path\n"
 554       "          generate config [groups]\n"
 555       "          clean config\n"
 556       "\n"
 557       "apt-ftparchive generates index files for Debian archives. It supports\n"
 558       "many styles of generation from fully automated to functional replacements\n"
 559       "for dpkg-scanpackages and dpkg-scansources\n"
 560       "\n"
 561       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 562       "Package file contains the contents of all the control fields from\n"
 563       "each package as well as the MD5 hash and filesize. An override file\n"
 564       "is supported to force the value of Priority and Section.\n"
 565       "\n"
 566       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 567       "The --source-override option can be used to specify a src override file\n"
 568       "\n"
 569       "The 'packages' and 'sources' command should be run in the root of the\n"
 570       "tree. BinaryPath should point to the base of the recursive search and \n"
 571       "override file should contain the override flags. Pathprefix is\n"
 572       "appended to the filename fields if present. Example usage from the \n"
 573       "Debian archive:\n"
 574       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 575       "               dists/potato/main/binary-i386/Packages\n"
 576       "\n"
 577       "Options:\n"
 578       "  -h    This help text\n"
 579       "  --md5 Control MD5 generation\n"
 580       "  -s=?  Source override file\n"
 581       "  -q    Quiet\n"
 582       "  -d=?  Select the optional caching database\n"
 583       "  --no-delink Enable delinking debug mode\n"
 584       "  --contents  Control contents file generation\n"
 585       "  -c=?  Read this configuration file\n"
 586       "  -o=?  Set an arbitary configuration option") << endl;
 587
 588    return true;
 589 }
 590                                                                         /*}}}*/
 591 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 592 // ---------------------------------------------------------------------
 593 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 594 bool SimpleGenPackages(CommandLine &CmdL)
 595 {
 596    if (CmdL.FileSize() < 2)
 597       return ShowHelp(CmdL);
 598
 599    string Override;
 600    if (CmdL.FileSize() >= 3)
 601       Override = CmdL.FileList[2];
 602
 603    // Create a package writer object.
 604    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 605                            Override, "");
 606    if (_error->PendingError() == true)
 607       return false;
 608
 609    if (CmdL.FileSize() >= 4)
 610       Packages.PathPrefix = CmdL.FileList[3];
 611
 612    // Do recursive directory searching
 613    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 614       return false;
 615
 616    return true;
 617 }
 618                                                                         /*}}}*/
 619 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 620 // ---------------------------------------------------------------------
 621 /* */
 622 bool SimpleGenContents(CommandLine &CmdL)
 623 {
 624    if (CmdL.FileSize() < 2)
 625       return ShowHelp(CmdL);
 626
 627    // Create a package writer object.
 628    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"));
 629    if (_error->PendingError() == true)
 630       return false;
 631
 632    // Do recursive directory searching
 633    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 634       return false;
 635
 636    Contents.Finish();
 637
 638    return true;
 639 }
 640                                                                         /*}}}*/
 641 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 642 // ---------------------------------------------------------------------
 643 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 644 bool SimpleGenSources(CommandLine &CmdL)
 645 {
 646    if (CmdL.FileSize() < 2)
 647       return ShowHelp(CmdL);
 648
 649    string Override;
 650    if (CmdL.FileSize() >= 3)
 651       Override = CmdL.FileList[2];
 652
 653    string SOverride;
 654    if (Override.empty() == false)
 655       SOverride = Override + ".src";
 656
 657    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 658                              SOverride.c_str());
 659
 660    // Create a package writer object.
 661    SourcesWriter Sources(Override,SOverride);
 662    if (_error->PendingError() == true)
 663       return false;
 664
 665    if (CmdL.FileSize() >= 4)
 666       Sources.PathPrefix = CmdL.FileList[3];
 667
 668    // Do recursive directory searching
 669    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 670       return false;
 671
 672    return true;
 673 }
 674                                                                         /*}}}*/
 675 // Generate - Full generate, using a config file                        /*{{{*/
 676 // ---------------------------------------------------------------------
 677 /* */
 678 bool Generate(CommandLine &CmdL)
 679 {
 680    struct CacheDB::Stats SrcStats;
 681    if (CmdL.FileSize() < 2)
 682       return ShowHelp(CmdL);
 683
 684    struct timeval StartTime;
 685    gettimeofday(&StartTime,0);
 686    struct CacheDB::Stats Stats;
 687
 688    // Read the configuration file.
 689    Configuration Setup;
 690    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 691       return false;
 692
 693    vector<PackageMap> PkgList;
 694    LoadTree(PkgList,Setup);
 695    LoadBinDir(PkgList,Setup);
 696
 697    // Sort by cache DB to improve IO locality.
 698    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 699
 700    // Generate packages
 701    if (CmdL.FileSize() <= 2)
 702    {
 703       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 704          if (I->GenPackages(Setup,Stats) == false)
 705             _error->DumpErrors();
 706       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 707          if (I->GenSources(Setup,SrcStats) == false)
 708             _error->DumpErrors();
 709    }
 710    else
 711    {
 712       // Make a choice list out of the package list..
 713       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 714       RxChoiceList *End = List;
 715       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 716       {
 717          End->UserData = &(*I);
 718          End->Str = I->BaseDir.c_str();
 719          End++;
 720
 721          End->UserData = &(*I);
 722          End->Str = I->Tag.c_str();
 723          End++;
 724       }
 725       End->Str = 0;
 726
 727       // Regex it
 728       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 729       {
 730          delete [] List;
 731          return _error->Error(_("No selections matched"));
 732       }
 733       _error->DumpErrors();
 734
 735       // Do the generation for Packages
 736       for (End = List; End->Str != 0; End++)
 737       {
 738          if (End->Hit == false)
 739             continue;
 740
 741          PackageMap *I = (PackageMap *)End->UserData;
 742          if (I->PkgDone == true)
 743             continue;
 744          if (I->GenPackages(Setup,Stats) == false)
 745             _error->DumpErrors();
 746       }
 747
 748       // Do the generation for Sources
 749       for (End = List; End->Str != 0; End++)
 750       {
 751          if (End->Hit == false)
 752             continue;
 753
 754          PackageMap *I = (PackageMap *)End->UserData;
 755          if (I->SrcDone == true)
 756             continue;
 757          if (I->GenSources(Setup,SrcStats) == false)
 758             _error->DumpErrors();
 759       }
 760
 761       delete [] List;
 762    }
 763
 764    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 765       return true;
 766
 767    c1out << "Done Packages, Starting contents." << endl;
 768
 769    // Sort the contents file list by date
 770    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 771    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 772    {
 773       struct stat A;
 774       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 775                                  I->CntCompress,A) == false)
 776          time(&I->ContentsMTime);
 777       else
 778          I->ContentsMTime = A.st_mtime;
 779    }
 780    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 781
 782    /* Now for Contents.. The process here is to do a make-like dependency
 783       check. Each contents file is verified to be newer than the package files
 784       that describe the debs it indexes. Since the package files contain
 785       hashes of the .debs this means they have not changed either so the
 786       contents must be up to date. */
 787    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 788    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 789    {
 790       // This record is not relevent
 791       if (I->ContentsDone == true ||
 792           I->Contents.empty() == true)
 793          continue;
 794
 795       // Do not do everything if the user specified sections.
 796       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 797          continue;
 798
 799       struct stat A,B;
 800       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 801       {
 802          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 803          {
 804             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 805             continue;
 806          }
 807
 808          if (A.st_mtime > B.st_mtime)
 809             continue;
 810       }
 811
 812       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 813                          MaxContentsChange) == false)
 814          _error->DumpErrors();
 815
 816       // Hit the limit?
 817       if (MaxContentsChange == 0)
 818       {
 819          c1out << "Hit contents update byte limit" << endl;
 820          break;
 821       }
 822    }
 823
 824    struct timeval NewTime;
 825    gettimeofday(&NewTime,0);
 826    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 827                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 828    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 829          << " archives. Took " << TimeToStr((long)Delta) << endl;
 830
 831    return true;
 832 }
 833                                                                         /*}}}*/
 834 // Clean - Clean out the databases                                      /*{{{*/
 835 // ---------------------------------------------------------------------
 836 /* */
 837 bool Clean(CommandLine &CmdL)
 838 {
 839    if (CmdL.FileSize() != 2)
 840       return ShowHelp(CmdL);
 841
 842    // Read the configuration file.
 843    Configuration Setup;
 844    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 845       return false;
 846
 847    vector<PackageMap> PkgList;
 848    LoadTree(PkgList,Setup);
 849    LoadBinDir(PkgList,Setup);
 850
 851    // Sort by cache DB to improve IO locality.
 852    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 853
 854    string CacheDir = Setup.FindDir("Dir::CacheDir");
 855
 856    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 857    {
 858       c0out << I->BinCacheDB << endl;
 859       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 860       if (DB.Clean() == false)
 861          _error->DumpErrors();
 862
 863       string CacheDB = I->BinCacheDB;
 864       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
 865    }
 866
 867    return true;
 868 }
 869                                                                         /*}}}*/
 870
 871 int main(int argc, const char *argv[])
 872 {
 873    CommandLine::Args Args[] = {
 874       {'h',"help","help",0},
 875       {0,"md5","APT::FTPArchive::MD5",0},
 876       {'v',"version","version",0},
 877       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 878       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 879       {'q',"quiet","quiet",CommandLine::IntLevel},
 880       {'q',"silent","quiet",CommandLine::IntLevel},
 881       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 882       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 883       {0,"contents","APT::FTPArchive::Contents",0},
 884       {'c',"config-file",0,CommandLine::ConfigFile},
 885       {'o',"option",0,CommandLine::ArbItem},
 886       {0,0,0,0}};
 887    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 888                                    {"contents",&SimpleGenContents},
 889                                    {"sources",&SimpleGenSources},
 890                                    {"generate",&Generate},
 891                                    {"clean",&Clean},
 892                                    {"help",&ShowHelp},
 893                                    {0,0}};
 894
 895    // Parse the command line and initialize the package library
 896    CommandLine CmdL(Args,_config);
 897    if (CmdL.Parse(argc,argv) == false)
 898    {
 899       _error->DumpErrors();
 900       return 100;
 901    }
 902
 903    // See if the help should be shown
 904    if (_config->FindB("help") == true ||
 905        _config->FindB("version") == true ||
 906        CmdL.FileSize() == 0)
 907    {
 908       ShowHelp(CmdL);
 909       return 0;
 910    }
 911
 912    // Setup the output streams
 913    c0out.rdbuf(clog.rdbuf());
 914    c1out.rdbuf(clog.rdbuf());
 915    c2out.rdbuf(clog.rdbuf());
 916    Quiet = _config->FindI("quiet",0);
 917    if (Quiet > 0)
 918       c0out.rdbuf(devnull.rdbuf());
 919    if (Quiet > 1)
 920       c1out.rdbuf(devnull.rdbuf());
 921
 922    // Match the operation
 923    CmdL.DispatchArg(Cmds);
 924
 925    if (_error->empty() == false)
 926    {
 927       bool Errors = _error->PendingError();
 928       _error->DumpErrors();
 929       return Errors == true?100:0;
 930    }
 931    return 0;
 932 }