ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <apt-pkg/fileutl.h>
  21
  22 #include <apt-private/private-cmndline.h>
  23 #include <apt-private/private-output.h>
  24 #include <apt-private/private-main.h>
  25
  26 #include <algorithm>
  27 #include <climits>
  28 #include <sys/time.h>
  29 #include <locale.h>
  30 #include <stdio.h>
  31 #include <sys/stat.h>
  32 #include <time.h>
  33 #include <functional>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37
  38 #include "cachedb.h"
  39 #include "override.h"
  40 #include "apt-ftparchive.h"
  41 #include "multicompress.h"
  42 #include "writer.h"
  43
  44 #include <apti18n.h>
  45                                                                         /*}}}*/
  46
  47 using namespace std;
  48 unsigned Quiet = 0;
  49
  50 // struct PackageMap - List of all package files in the config file     /*{{{*/
  51 // ---------------------------------------------------------------------
  52 /* */
  53 struct PackageMap
  54 {
  55    // General Stuff
  56    string BaseDir;
  57    string InternalPrefix;
  58    string FLFile;
  59    string PkgExt;
  60    string SrcExt;
  61
  62    // Stuff for the Package File
  63    string PkgFile;
  64    string BinCacheDB;
  65    string SrcCacheDB;
  66    string BinOverride;
  67    string ExtraOverride;
  68
  69    // We generate for this given arch
  70    string Arch;
  71
  72    // Stuff for the Source File
  73    string SrcFile;
  74    string SrcOverride;
  75    string SrcExtraOverride;
  76
  77    // Translation master file
  78    bool LongDesc;
  79    TranslationWriter *TransWriter;
  80
  81    // Contents
  82    string Contents;
  83    string ContentsHead;
  84
  85    // Random things
  86    string Tag;
  87    string PkgCompress;
  88    string CntCompress;
  89    string SrcCompress;
  90    string PathPrefix;
  91    unsigned int DeLinkLimit;
  92    mode_t Permissions;
  93
  94    bool ContentsDone;
  95    bool PkgDone;
  96    bool SrcDone;
  97    time_t ContentsMTime;
  98
  99    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
 100    {
 101       inline bool operator() (const PackageMap &x,const PackageMap &y)
 102       {return x.ContentsMTime < y.ContentsMTime;};
 103    };
 104
 105    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
 106    {
 107       inline bool operator() (const PackageMap &x,const PackageMap &y)
 108       {return x.BinCacheDB < y.BinCacheDB;};
 109    };
 110
 111    struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
 112    {
 113       inline bool operator() (const PackageMap &x,const PackageMap &y)
 114       {return x.SrcCacheDB < y.SrcCacheDB;};
 115    };
 116
 117    void GetGeneral(Configuration &Setup,Configuration &Block);
 118    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 119    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 120    bool GenContents(Configuration &Setup,
 121                     vector<PackageMap>::iterator Begin,
 122                     vector<PackageMap>::iterator End,
 123                     unsigned long &Left);
 124
 125    PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 126                   ContentsDone(false), PkgDone(false), SrcDone(false),
 127                   ContentsMTime(0) {};
 128 };
 129                                                                         /*}}}*/
 130
 131 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 132 // ---------------------------------------------------------------------
 133 /* */
 134 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 135 {
 136    PathPrefix = Block.Find("PathPrefix");
 137
 138    if (Block.FindB("External-Links",true) == false)
 139       DeLinkLimit = Setup.FindI("Default::DeLinkLimit", std::numeric_limits<unsigned int>::max());
 140    else
 141       DeLinkLimit = 0;
 142
 143    PkgCompress = Block.Find("Packages::Compress",
 144                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 145    CntCompress = Block.Find("Contents::Compress",
 146                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 147    SrcCompress = Block.Find("Sources::Compress",
 148                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 149
 150    SrcExt = Block.Find("Sources::Extensions",
 151                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 152    PkgExt = Block.Find("Packages::Extensions",
 153                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 154
 155    Permissions = Setup.FindI("Default::FileMode",0644);
 156
 157    if (FLFile.empty() == false)
 158       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 159
 160    if (Contents == " ")
 161       Contents= string();
 162 }
 163                                                                         /*}}}*/
 164 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 165 // ---------------------------------------------------------------------
 166 /* This generates the Package File described by this object. */
 167 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 168 {
 169    if (PkgFile.empty() == true)
 170       return true;
 171
 172    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 173    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 174    string CacheDir = Setup.FindDir("Dir::CacheDir");
 175
 176    struct timeval StartTime;
 177    gettimeofday(&StartTime,0);
 178
 179    PkgDone = true;
 180
 181    // Create a package writer object.
 182    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 183                       PkgCompress,Permissions);
 184    PackagesWriter Packages(&Comp.Input, TransWriter, flCombine(CacheDir,BinCacheDB),
 185                            flCombine(OverrideDir,BinOverride),
 186                            flCombine(OverrideDir,ExtraOverride),
 187                            Arch);
 188    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 189       return _error->Error(_("Package extension list is too long"));
 190    if (_error->PendingError() == true)
 191       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 192
 193    Packages.PathPrefix = PathPrefix;
 194    Packages.DirStrip = ArchiveDir;
 195    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 196
 197    Packages.LongDescription = LongDesc;
 198
 199    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 200    Packages.DeLinkLimit = DeLinkLimit;
 201
 202    if (_error->PendingError() == true)
 203       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 204
 205    c0out << ' ' << BaseDir << ":" << flush;
 206
 207    // Do recursive directory searching
 208    if (FLFile.empty() == true)
 209    {
 210       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 211          return false;
 212    }
 213    else
 214    {
 215       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 216          return false;
 217    }
 218
 219    Packages.Output = 0;      // Just in case
 220
 221    // Finish compressing
 222    unsigned long long Size;
 223    if (Comp.Finalize(Size) == false)
 224    {
 225       c0out << endl;
 226       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 227    }
 228
 229    if (Size != 0)
 230       c0out << " New "
 231              << SizeToStr(Size) << "B ";
 232    else
 233       c0out << ' ';
 234
 235    struct timeval NewTime;
 236    gettimeofday(&NewTime,0);
 237    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 238                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 239
 240    c0out << Packages.Stats.Packages << " files " <<
 241 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 242       SizeToStr(Packages.Stats.Bytes) << "B " <<
 243       TimeToStr((long)Delta) << endl;
 244
 245    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 246      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 247
 248    Stats.Add(Packages.Stats);
 249    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 250
 251    return !_error->PendingError();
 252 }
 253
 254                                                                         /*}}}*/
 255 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 256 // ---------------------------------------------------------------------
 257 /* This generates the Sources File described by this object. */
 258 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 259 {
 260    if (SrcFile.empty() == true)
 261       return true;
 262
 263    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 264    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 265    string CacheDir = Setup.FindDir("Dir::CacheDir");
 266
 267    struct timeval StartTime;
 268    gettimeofday(&StartTime,0);
 269
 270    SrcDone = true;
 271
 272    // Create a package writer object.
 273    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 274                       SrcCompress,Permissions);
 275    SourcesWriter Sources(&Comp.Input, flCombine(CacheDir, SrcCacheDB),
 276                          flCombine(OverrideDir,BinOverride),
 277                          flCombine(OverrideDir,SrcOverride),
 278                          flCombine(OverrideDir,SrcExtraOverride));
 279    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 280       return _error->Error(_("Source extension list is too long"));
 281    if (_error->PendingError() == true)
 282       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 283
 284    Sources.PathPrefix = PathPrefix;
 285    Sources.DirStrip = ArchiveDir;
 286    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 287
 288    Sources.DeLinkLimit = DeLinkLimit;
 289    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 290
 291    if (_error->PendingError() == true)
 292       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 293
 294    c0out << ' ' << BaseDir << ":" << flush;
 295
 296    // Do recursive directory searching
 297    if (FLFile.empty() == true)
 298    {
 299       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 300          return false;
 301    }
 302    else
 303    {
 304       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 305          return false;
 306    }
 307    Sources.Output = 0;      // Just in case
 308
 309    // Finish compressing
 310    unsigned long long Size;
 311    if (Comp.Finalize(Size) == false)
 312    {
 313       c0out << endl;
 314       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 315    }
 316
 317    if (Size != 0)
 318       c0out << " New "
 319              << SizeToStr(Size) << "B ";
 320    else
 321       c0out << ' ';
 322
 323    struct timeval NewTime;
 324    gettimeofday(&NewTime,0);
 325    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 326                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 327
 328    c0out << Sources.Stats.Packages << " pkgs in " <<
 329       TimeToStr((long)Delta) << endl;
 330
 331    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 332      c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
 333
 334    Stats.Add(Sources.Stats);
 335    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 336
 337    return !_error->PendingError();
 338 }
 339                                                                         /*}}}*/
 340 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 341 // ---------------------------------------------------------------------
 342 /* This generates the contents file partially described by this object.
 343    It searches the given iterator range for other package files that map
 344    into this contents file and includes their data as well when building. */
 345 bool PackageMap::GenContents(Configuration &Setup,
 346                              vector<PackageMap>::iterator Begin,
 347                              vector<PackageMap>::iterator End,
 348                              unsigned long &Left)
 349 {
 350    if (Contents.empty() == true)
 351       return true;
 352
 353    if (Left == 0)
 354       return true;
 355
 356    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 357    string CacheDir = Setup.FindDir("Dir::CacheDir");
 358    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 359
 360    struct timeval StartTime;
 361    gettimeofday(&StartTime,0);
 362
 363    // Create a package writer object.
 364    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 365                       CntCompress,Permissions);
 366    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 367    ContentsWriter Contents(&Comp.Input, "", Arch);
 368    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 369       return _error->Error(_("Package extension list is too long"));
 370    if (_error->PendingError() == true)
 371       return false;
 372
 373    if (_error->PendingError() == true)
 374       return false;
 375
 376    // Write the header out.
 377    if (ContentsHead.empty() == false)
 378    {
 379       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 380       if (_error->PendingError() == true)
 381          return false;
 382
 383       unsigned long long Size = Head.Size();
 384       unsigned char Buf[4096];
 385       while (Size != 0)
 386       {
 387          unsigned long long ToRead = Size;
 388          if (Size > sizeof(Buf))
 389             ToRead = sizeof(Buf);
 390
 391          if (Head.Read(Buf,ToRead) == false)
 392             return false;
 393
 394          if (Comp.Input.Write(Buf, ToRead) == false)
 395             return _error->Errno("fwrite",_("Error writing header to contents file"));
 396
 397          Size -= ToRead;
 398       }
 399    }
 400
 401    /* Go over all the package file records and parse all the package
 402       files associated with this contents file into one great big honking
 403       memory structure, then dump the sorted version */
 404    c0out << ' ' << this->Contents << ":" << flush;
 405    for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
 406    {
 407       if (I->Contents != this->Contents)
 408          continue;
 409
 410       Contents.Prefix = ArchiveDir;
 411       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 412       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 413                             I->PkgCompress);
 414
 415       I->ContentsDone = true;
 416    }
 417
 418    Contents.Finish();
 419
 420    // Finish compressing
 421    unsigned long long Size;
 422    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 423    {
 424       c0out << endl;
 425       return _error->Error(_("Error processing contents %s"),
 426                            this->Contents.c_str());
 427    }
 428
 429    if (Size != 0)
 430    {
 431       c0out << " New " << SizeToStr(Size) << "B ";
 432       if (Left > Size)
 433          Left -= Size;
 434       else
 435          Left = 0;
 436    }
 437    else
 438       c0out << ' ';
 439
 440    struct timeval NewTime;
 441    gettimeofday(&NewTime,0);
 442    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 443                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 444
 445    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 446      c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
 447
 448    c0out << Contents.Stats.Packages << " files " <<
 449       SizeToStr(Contents.Stats.Bytes) << "B " <<
 450       TimeToStr((long)Delta) << endl;
 451
 452    return true;
 453 }
 454                                                                         /*}}}*/
 455
 456 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 457 // ---------------------------------------------------------------------
 458 /* This populates the PkgList with all the possible permutations of the
 459    section/arch lists. */
 460 static void LoadTree(vector<PackageMap> &PkgList, std::vector<TranslationWriter*> &TransList, Configuration &Setup)
 461 {
 462    // Load the defaults
 463    string DDir = Setup.Find("TreeDefault::Directory",
 464                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 465    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 466                             "$(DIST)/$(SECTION)/source/");
 467    string DPkg = Setup.Find("TreeDefault::Packages",
 468                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 469    string DTrans = Setup.Find("TreeDefault::Translation",
 470                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 471    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 472                             "$(DIST)/$(SECTION)/");
 473    string DContents = Setup.Find("TreeDefault::Contents",
 474                             "$(DIST)/$(SECTION)/Contents-$(ARCH)");
 475    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 476    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 477                                "packages-$(ARCH).db");
 478    string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
 479                                "sources-$(SECTION).db");
 480    string DSources = Setup.Find("TreeDefault::Sources",
 481                                 "$(DIST)/$(SECTION)/source/Sources");
 482    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 483    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 484
 485    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 486
 487    bool const LongDescription = Setup.FindB("Default::LongDescription",
 488                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 489    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 490
 491    // Process 'tree' type sections
 492    const Configuration::Item *Top = Setup.Tree("tree");
 493    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 494    {
 495       Configuration Block(Top);
 496       string Dist = Top->Tag;
 497
 498       // Parse the sections
 499       string Tmp = Block.Find("Sections");
 500       const char *Sections = Tmp.c_str();
 501       string Section;
 502       while (ParseQuoteWord(Sections,Section) == true)
 503       {
 504          string Arch;
 505          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 506                                          {"$(SECTION)",&Section},
 507                                          {"$(ARCH)",&Arch},
 508                                          {NULL, NULL}};
 509          mode_t const Perms = Block.FindI("FileMode", Permissions);
 510          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 511          TranslationWriter *TransWriter = NULL;
 512
 513          string const Tmp2 = Block.Find("Architectures");
 514          const char *Archs = Tmp2.c_str();
 515          while (ParseQuoteWord(Archs,Arch) == true)
 516          {
 517             PackageMap Itm;
 518             Itm.Permissions = Perms;
 519             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 520             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 521
 522             if (stringcasecmp(Arch,"source") == 0)
 523             {
 524                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 525                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 526                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 527                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 528                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 529                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 530                Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);
 531             }
 532             else
 533             {
 534                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 535                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 536                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 537                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 538                Itm.Arch = Arch;
 539                Itm.LongDesc = LongDesc;
 540                if (TransWriter == NULL && DTrans.empty() == false && LongDesc == false && DTrans != "/dev/null")
 541                {
 542                   string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 543                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 544                   string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 545                   TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 546                   TransList.push_back(TransWriter);
 547                }
 548                Itm.TransWriter = TransWriter;
 549                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 550                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 551                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 552                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 553             }
 554
 555             Itm.GetGeneral(Setup,Block);
 556             PkgList.push_back(Itm);
 557          }
 558       }
 559
 560       Top = Top->Next;
 561    }
 562 }
 563                                                                         /*}}}*/
 564 static void UnloadTree(std::vector<TranslationWriter*> const &Trans)            /*{{{*/
 565 {
 566    for (std::vector<TranslationWriter*>::const_reverse_iterator T = Trans.rbegin(); T != Trans.rend(); ++T)
 567       delete *T;
 568 }
 569                                                                         /*}}}*/
 570 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 571 // ---------------------------------------------------------------------
 572 /* */
 573 static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 574 {
 575    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 576
 577    // Process 'bindirectory' type sections
 578    const Configuration::Item *Top = Setup.Tree("bindirectory");
 579    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 580    {
 581       Configuration Block(Top);
 582
 583       PackageMap Itm;
 584       Itm.PkgFile = Block.Find("Packages");
 585       Itm.SrcFile = Block.Find("Sources");
 586       Itm.BinCacheDB = Block.Find("BinCacheDB");
 587       Itm.SrcCacheDB = Block.Find("SrcCacheDB");
 588       Itm.BinOverride = Block.Find("BinOverride");
 589       Itm.ExtraOverride = Block.Find("ExtraOverride");
 590       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 591       Itm.SrcOverride = Block.Find("SrcOverride");
 592       Itm.BaseDir = Top->Tag;
 593       Itm.FLFile = Block.Find("FileList");
 594       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 595       Itm.Contents = Block.Find("Contents");
 596       Itm.ContentsHead = Block.Find("Contents::Header");
 597       Itm.Permissions = Block.FindI("FileMode", Permissions);
 598
 599       Itm.GetGeneral(Setup,Block);
 600       PkgList.push_back(Itm);
 601
 602       Top = Top->Next;
 603    }
 604 }
 605                                                                         /*}}}*/
 606
 607 // ShowHelp - Show the help text                                        /*{{{*/
 608 bool ShowHelp(CommandLine &, CommandLine::DispatchWithHelp const *)
 609 {
 610    ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 611    if (_config->FindB("version") == true)
 612       return true;
 613
 614    cout <<
 615     _("Usage: apt-ftparchive [options] command\n"
 616       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 617       "          sources srcpath [overridefile [pathprefix]]\n"
 618       "          contents path\n"
 619       "          release path\n"
 620       "          generate config [groups]\n"
 621       "          clean config\n"
 622       "\n"
 623       "apt-ftparchive generates index files for Debian archives. It supports\n"
 624       "many styles of generation from fully automated to functional replacements\n"
 625       "for dpkg-scanpackages and dpkg-scansources\n"
 626       "\n"
 627       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 628       "Package file contains the contents of all the control fields from\n"
 629       "each package as well as the MD5 hash and filesize. An override file\n"
 630       "is supported to force the value of Priority and Section.\n"
 631       "\n"
 632       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 633       "The --source-override option can be used to specify a src override file\n"
 634       "\n"
 635       "The 'packages' and 'sources' command should be run in the root of the\n"
 636       "tree. BinaryPath should point to the base of the recursive search and \n"
 637       "override file should contain the override flags. Pathprefix is\n"
 638       "appended to the filename fields if present. Example usage from the \n"
 639       "Debian archive:\n"
 640       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 641       "               dists/potato/main/binary-i386/Packages\n"
 642       "\n"
 643       "Options:\n"
 644       "  -h    This help text\n"
 645       "  --md5 Control MD5 generation\n"
 646       "  -s=?  Source override file\n"
 647       "  -q    Quiet\n"
 648       "  -d=?  Select the optional caching database\n"
 649       "  --no-delink Enable delinking debug mode\n"
 650       "  --contents  Control contents file generation\n"
 651       "  -c=?  Read this configuration file\n"
 652       "  -o=?  Set an arbitrary configuration option") << endl;
 653
 654    return true;
 655 }
 656                                                                         /*}}}*/
 657 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 658 // ---------------------------------------------------------------------
 659 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 660 static bool SimpleGenPackages(CommandLine &CmdL)
 661 {
 662    if (CmdL.FileSize() < 2)
 663       return ShowHelp(CmdL, nullptr);
 664
 665    string Override;
 666    if (CmdL.FileSize() >= 3)
 667       Override = CmdL.FileList[2];
 668
 669    // Create a package writer object.
 670    PackagesWriter Packages(NULL, NULL, _config->Find("APT::FTPArchive::DB"),
 671                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 672    if (_error->PendingError() == true)
 673       return false;
 674
 675    if (CmdL.FileSize() >= 4)
 676       Packages.PathPrefix = CmdL.FileList[3];
 677
 678    // Do recursive directory searching
 679    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 680       return false;
 681
 682    // Give some stats if asked for
 683    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 684      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 685
 686    return true;
 687 }
 688                                                                         /*}}}*/
 689 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 690 // ---------------------------------------------------------------------
 691 /* */
 692 static bool SimpleGenContents(CommandLine &CmdL)
 693 {
 694    if (CmdL.FileSize() < 2)
 695       return ShowHelp(CmdL, nullptr);
 696
 697    // Create a package writer object.
 698    ContentsWriter Contents(NULL, _config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 699    if (_error->PendingError() == true)
 700       return false;
 701
 702    // Do recursive directory searching
 703    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 704       return false;
 705
 706    Contents.Finish();
 707
 708    return true;
 709 }
 710                                                                         /*}}}*/
 711 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 712 // ---------------------------------------------------------------------
 713 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 714 static bool SimpleGenSources(CommandLine &CmdL)
 715 {
 716    if (CmdL.FileSize() < 2)
 717       return ShowHelp(CmdL, nullptr);
 718
 719    string Override;
 720    if (CmdL.FileSize() >= 3)
 721       Override = CmdL.FileList[2];
 722
 723    string SOverride;
 724    if (Override.empty() == false)
 725       SOverride = Override + ".src";
 726
 727    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 728                              SOverride.c_str());
 729
 730    // Create a package writer object.
 731    SourcesWriter Sources(NULL, _config->Find("APT::FTPArchive::DB"),Override,SOverride);
 732    if (_error->PendingError() == true)
 733       return false;
 734
 735    if (CmdL.FileSize() >= 4)
 736       Sources.PathPrefix = CmdL.FileList[3];
 737
 738    // Do recursive directory searching
 739    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 740       return false;
 741
 742    // Give some stats if asked for
 743    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 744      c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl;
 745
 746    return true;
 747 }
 748                                                                         /*}}}*/
 749 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 750 // ---------------------------------------------------------------------
 751 static bool SimpleGenRelease(CommandLine &CmdL)
 752 {
 753    if (CmdL.FileSize() < 2)
 754       return ShowHelp(CmdL, nullptr);
 755
 756    string Dir = CmdL.FileList[1];
 757
 758    ReleaseWriter Release(NULL, "");
 759    Release.DirStrip = Dir;
 760
 761    if (_error->PendingError() == true)
 762       return false;
 763
 764    if (Release.RecursiveScan(Dir) == false)
 765       return false;
 766
 767    Release.Finish();
 768
 769    return true;
 770 }
 771
 772                                                                         /*}}}*/
 773 // DoGeneratePackagesAndSources - Helper for Generate                   /*{{{*/
 774 // ---------------------------------------------------------------------
 775 static bool DoGeneratePackagesAndSources(Configuration &Setup,
 776                                          vector<PackageMap> &PkgList,
 777                                          struct CacheDB::Stats &SrcStats,
 778                                          struct CacheDB::Stats &Stats,
 779                                          CommandLine &CmdL)
 780 {
 781    if (CmdL.FileSize() <= 2)
 782    {
 783       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 784          if (I->GenPackages(Setup,Stats) == false)
 785             _error->DumpErrors();
 786       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 787          if (I->GenSources(Setup,SrcStats) == false)
 788             _error->DumpErrors();
 789    }
 790    else
 791    {
 792       // Make a choice list out of the package list..
 793       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 794       RxChoiceList *End = List;
 795       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 796       {
 797          End->UserData = &(*I);
 798          End->Str = I->BaseDir.c_str();
 799          End++;
 800
 801          End->UserData = &(*I);
 802          End->Str = I->Tag.c_str();
 803          End++;
 804       }
 805       End->Str = 0;
 806
 807       // Regex it
 808       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 809       {
 810          delete [] List;
 811          return _error->Error(_("No selections matched"));
 812       }
 813       _error->DumpErrors();
 814
 815       // Do the generation for Packages
 816       for (End = List; End->Str != 0; End++)
 817       {
 818          if (End->Hit == false)
 819             continue;
 820
 821          PackageMap *I = (PackageMap *)End->UserData;
 822          if (I->PkgDone == true)
 823             continue;
 824          if (I->GenPackages(Setup,Stats) == false)
 825             _error->DumpErrors();
 826       }
 827
 828       // Do the generation for Sources
 829       for (End = List; End->Str != 0; End++)
 830       {
 831          if (End->Hit == false)
 832             continue;
 833
 834          PackageMap *I = (PackageMap *)End->UserData;
 835          if (I->SrcDone == true)
 836             continue;
 837          if (I->GenSources(Setup,SrcStats) == false)
 838             _error->DumpErrors();
 839       }
 840
 841       delete [] List;
 842    }
 843    return true;
 844 }
 845
 846                                                                         /*}}}*/
 847 // DoGenerateContents - Helper for Generate to generate the Contents    /*{{{*/
 848 // ---------------------------------------------------------------------
 849 static bool DoGenerateContents(Configuration &Setup,
 850                                vector<PackageMap> &PkgList,
 851                                CommandLine &CmdL)
 852 {
 853    c1out << "Packages done, Starting contents." << endl;
 854
 855    // Sort the contents file list by date
 856    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 857    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 858    {
 859       struct stat A;
 860       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 861                                  I->CntCompress,A) == false)
 862          time(&I->ContentsMTime);
 863       else
 864          I->ContentsMTime = A.st_mtime;
 865    }
 866    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 867
 868    /* Now for Contents.. The process here is to do a make-like dependency
 869       check. Each contents file is verified to be newer than the package files
 870       that describe the debs it indexes. Since the package files contain
 871       hashes of the .debs this means they have not changed either so the
 872       contents must be up to date. */
 873    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",
 874                                                  std::numeric_limits<unsigned int>::max())*1024;
 875    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 876    {
 877       // This record is not relevant
 878       if (I->ContentsDone == true ||
 879           I->Contents.empty() == true)
 880          continue;
 881
 882       // Do not do everything if the user specified sections.
 883       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 884          continue;
 885
 886       struct stat A,B;
 887       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 888       {
 889          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 890          {
 891             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 892             continue;
 893          }
 894
 895          if (A.st_mtime > B.st_mtime)
 896             continue;
 897       }
 898
 899       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 900                          MaxContentsChange) == false)
 901          _error->DumpErrors();
 902
 903       // Hit the limit?
 904       if (MaxContentsChange == 0)
 905       {
 906          c1out << "Hit contents update byte limit" << endl;
 907          break;
 908       }
 909    }
 910
 911    return true;
 912 }
 913
 914                                                                         /*}}}*/
 915 // Generate - Full generate, using a config file                        /*{{{*/
 916 // ---------------------------------------------------------------------
 917 /* */
 918 static bool Generate(CommandLine &CmdL)
 919 {
 920    struct CacheDB::Stats SrcStats;
 921    if (CmdL.FileSize() < 2)
 922       return ShowHelp(CmdL, nullptr);
 923
 924    struct timeval StartTime;
 925    gettimeofday(&StartTime,0);
 926    struct CacheDB::Stats Stats;
 927
 928    // Read the configuration file.
 929    Configuration Setup;
 930    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 931       return false;
 932
 933    vector<PackageMap> PkgList;
 934    std::vector<TranslationWriter*> TransList;
 935    LoadTree(PkgList, TransList, Setup);
 936    LoadBinDir(PkgList,Setup);
 937
 938    // Sort by cache DB to improve IO locality.
 939    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 940    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 941
 942    // Generate packages
 943    if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false)
 944    {
 945       if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false)
 946       {
 947          UnloadTree(TransList);
 948          return false;
 949       }
 950    } else {
 951       c1out << "Skipping Packages/Sources generation" << endl;
 952    }
 953
 954    // do Contents if needed
 955    if (_config->FindB("APT::FTPArchive::Contents", true) == true)
 956       if (DoGenerateContents(Setup, PkgList, CmdL) == false)
 957       {
 958          UnloadTree(TransList);
 959          return false;
 960       }
 961
 962    struct timeval NewTime;
 963    gettimeofday(&NewTime,0);
 964    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 965                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 966    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 967          << " archives. Took " << TimeToStr((long)Delta) << endl;
 968
 969    UnloadTree(TransList);
 970    return true;
 971 }
 972
 973                                                                         /*}}}*/
 974 // Clean - Clean out the databases                                      /*{{{*/
 975 // ---------------------------------------------------------------------
 976 /* */
 977 static bool Clean(CommandLine &CmdL)
 978 {
 979    if (CmdL.FileSize() != 2)
 980       return ShowHelp(CmdL, nullptr);
 981
 982    // Read the configuration file.
 983    Configuration Setup;
 984    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 985       return false;
 986    // we don't need translation creation here
 987    Setup.Set("TreeDefault::Translation", "/dev/null");
 988
 989    vector<PackageMap> PkgList;
 990    std::vector<TranslationWriter*> TransList;
 991    LoadTree(PkgList, TransList, Setup);
 992    LoadBinDir(PkgList,Setup);
 993
 994    // Sort by cache DB to improve IO locality.
 995    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 996    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 997
 998    string CacheDir = Setup.FindDir("Dir::CacheDir");
 999
1000    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
1001    {
1002       if(I->BinCacheDB != "")
1003          c0out << I->BinCacheDB << endl;
1004       if(I->SrcCacheDB != "")
1005          c0out << I->SrcCacheDB << endl;
1006       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
1007       CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));
1008       if (DB.Clean() == false)
1009          _error->DumpErrors();
1010       if (DB_SRC.Clean() == false)
1011          _error->DumpErrors();
1012
1013       string CacheDB = I->BinCacheDB;
1014       string SrcCacheDB = I->SrcCacheDB;
1015       while(I != PkgList.end() &&
1016             I->BinCacheDB == CacheDB &&
1017             I->SrcCacheDB == SrcCacheDB)
1018          ++I;
1019    }
1020
1021
1022    return true;
1023 }
1024                                                                         /*}}}*/
1025
1026 std::vector<CommandLine::DispatchWithHelp> GetCommands()                /*{{{*/
1027 {
1028    return {
1029       {"packages",&SimpleGenPackages, nullptr},
1030       {"contents",&SimpleGenContents, nullptr},
1031       {"sources",&SimpleGenSources, nullptr},
1032       {"release",&SimpleGenRelease, nullptr},
1033       {"generate",&Generate, nullptr},
1034       {"clean",&Clean, nullptr},
1035       {nullptr, nullptr, nullptr}
1036    };
1037 }
1038                                                                         /*}}}*/
1039 int main(int argc, const char *argv[])                                  /*{{{*/
1040 {
1041    InitLocale();
1042
1043    // Parse the command line and initialize the package library
1044    CommandLine CmdL;
1045    auto const Cmds = ParseCommandLine(CmdL, APT_CMD::APT_FTPARCHIVE, &_config, NULL, argc, argv);
1046
1047    _config->CndSet("quiet",0);
1048    Quiet = _config->FindI("quiet",0);
1049    InitOutput(clog.rdbuf());
1050
1051    return DispatchCommandLine(CmdL, Cmds);
1052 }
1053                                                                         /*}}}*/