ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <apt-pkg/fileutl.h>
  21
  22 #include <apt-private/private-cmndline.h>
  23 #include <apt-private/private-output.h>
  24 #include <apt-private/private-main.h>
  25
  26 #include <algorithm>
  27 #include <climits>
  28 #include <sys/time.h>
  29 #include <locale.h>
  30 #include <stdio.h>
  31 #include <sys/stat.h>
  32 #include <time.h>
  33 #include <functional>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37
  38 #include "cachedb.h"
  39 #include "override.h"
  40 #include "apt-ftparchive.h"
  41 #include "multicompress.h"
  42 #include "writer.h"
  43
  44 #include <apti18n.h>
  45                                                                         /*}}}*/
  46
  47 using namespace std;
  48 unsigned Quiet = 0;
  49
  50 // struct PackageMap - List of all package files in the config file     /*{{{*/
  51 // ---------------------------------------------------------------------
  52 /* */
  53 struct PackageMap
  54 {
  55    // General Stuff
  56    string BaseDir;
  57    string InternalPrefix;
  58    string FLFile;
  59    string PkgExt;
  60    string SrcExt;
  61
  62    // Stuff for the Package File
  63    string PkgFile;
  64    string BinCacheDB;
  65    string SrcCacheDB;
  66    string BinOverride;
  67    string ExtraOverride;
  68
  69    // We generate for this given arch
  70    string Arch;
  71    bool IncludeArchAll;
  72
  73    // Stuff for the Source File
  74    string SrcFile;
  75    string SrcOverride;
  76    string SrcExtraOverride;
  77
  78    // Translation master file
  79    bool LongDesc;
  80    TranslationWriter *TransWriter;
  81
  82    // Contents
  83    string Contents;
  84    string ContentsHead;
  85
  86    // Random things
  87    string Tag;
  88    string PkgCompress;
  89    string CntCompress;
  90    string SrcCompress;
  91    string PathPrefix;
  92    unsigned int DeLinkLimit;
  93    mode_t Permissions;
  94
  95    bool ContentsDone;
  96    bool PkgDone;
  97    bool SrcDone;
  98    time_t ContentsMTime;
  99
 100    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
 101    {
 102       inline bool operator() (const PackageMap &x,const PackageMap &y)
 103       {return x.ContentsMTime < y.ContentsMTime;};
 104    };
 105
 106    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
 107    {
 108       inline bool operator() (const PackageMap &x,const PackageMap &y)
 109       {return x.BinCacheDB < y.BinCacheDB;};
 110    };
 111
 112    struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
 113    {
 114       inline bool operator() (const PackageMap &x,const PackageMap &y)
 115       {return x.SrcCacheDB < y.SrcCacheDB;};
 116    };
 117
 118    void GetGeneral(Configuration &Setup,Configuration &Block);
 119    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 120    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 121    bool GenContents(Configuration &Setup,
 122                     vector<PackageMap>::iterator Begin,
 123                     vector<PackageMap>::iterator End,
 124                     unsigned long &Left);
 125
 126    PackageMap() : IncludeArchAll(true), LongDesc(true), TransWriter(NULL),
 127                   DeLinkLimit(0), Permissions(1), ContentsDone(false),
 128                   PkgDone(false), SrcDone(false), ContentsMTime(0) {};
 129 };
 130                                                                         /*}}}*/
 131
 132 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 133 // ---------------------------------------------------------------------
 134 /* */
 135 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 136 {
 137    PathPrefix = Block.Find("PathPrefix");
 138
 139    if (Block.FindB("External-Links",true) == false)
 140       DeLinkLimit = Setup.FindI("Default::DeLinkLimit", std::numeric_limits<unsigned int>::max());
 141    else
 142       DeLinkLimit = 0;
 143
 144    PkgCompress = Block.Find("Packages::Compress",
 145                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 146    CntCompress = Block.Find("Contents::Compress",
 147                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 148    SrcCompress = Block.Find("Sources::Compress",
 149                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 150
 151    SrcExt = Block.Find("Sources::Extensions",
 152                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 153    PkgExt = Block.Find("Packages::Extensions",
 154                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 155
 156    Permissions = Setup.FindI("Default::FileMode",0644);
 157
 158    if (FLFile.empty() == false)
 159       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 160
 161    if (Contents == " ")
 162       Contents= string();
 163 }
 164                                                                         /*}}}*/
 165 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 166 // ---------------------------------------------------------------------
 167 /* This generates the Package File described by this object. */
 168 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 169 {
 170    if (PkgFile.empty() == true)
 171       return true;
 172
 173    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 174    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 175    string CacheDir = Setup.FindDir("Dir::CacheDir");
 176
 177    struct timeval StartTime;
 178    gettimeofday(&StartTime,0);
 179
 180    PkgDone = true;
 181
 182    // Create a package writer object.
 183    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 184                       PkgCompress,Permissions);
 185    PackagesWriter Packages(&Comp.Input, TransWriter, flCombine(CacheDir,BinCacheDB),
 186                            flCombine(OverrideDir,BinOverride),
 187                            flCombine(OverrideDir,ExtraOverride),
 188                            Arch, IncludeArchAll);
 189    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 190       return _error->Error(_("Package extension list is too long"));
 191    if (_error->PendingError() == true)
 192       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 193
 194    Packages.PathPrefix = PathPrefix;
 195    Packages.DirStrip = ArchiveDir;
 196    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 197
 198    Packages.LongDescription = LongDesc;
 199
 200    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 201    Packages.DeLinkLimit = DeLinkLimit;
 202
 203    if (_error->PendingError() == true)
 204       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 205
 206    c0out << ' ' << BaseDir << ":" << flush;
 207
 208    // Do recursive directory searching
 209    if (FLFile.empty() == true)
 210    {
 211       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 212          return false;
 213    }
 214    else
 215    {
 216       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 217          return false;
 218    }
 219
 220    Packages.Output = 0;      // Just in case
 221
 222    // Finish compressing
 223    unsigned long long Size;
 224    if (Comp.Finalize(Size) == false)
 225    {
 226       c0out << endl;
 227       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 228    }
 229
 230    if (Size != 0)
 231       c0out << " New "
 232              << SizeToStr(Size) << "B ";
 233    else
 234       c0out << ' ';
 235
 236    struct timeval NewTime;
 237    gettimeofday(&NewTime,0);
 238    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 239                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 240
 241    c0out << Packages.Stats.Packages << " files " <<
 242 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 243       SizeToStr(Packages.Stats.Bytes) << "B " <<
 244       TimeToStr((long)Delta) << endl;
 245
 246    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 247      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 248
 249    Stats.Add(Packages.Stats);
 250    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 251
 252    return !_error->PendingError();
 253 }
 254
 255                                                                         /*}}}*/
 256 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 257 // ---------------------------------------------------------------------
 258 /* This generates the Sources File described by this object. */
 259 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 260 {
 261    if (SrcFile.empty() == true)
 262       return true;
 263
 264    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 265    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 266    string CacheDir = Setup.FindDir("Dir::CacheDir");
 267
 268    struct timeval StartTime;
 269    gettimeofday(&StartTime,0);
 270
 271    SrcDone = true;
 272
 273    // Create a package writer object.
 274    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 275                       SrcCompress,Permissions);
 276    SourcesWriter Sources(&Comp.Input, flCombine(CacheDir, SrcCacheDB),
 277                          flCombine(OverrideDir,BinOverride),
 278                          flCombine(OverrideDir,SrcOverride),
 279                          flCombine(OverrideDir,SrcExtraOverride));
 280    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 281       return _error->Error(_("Source extension list is too long"));
 282    if (_error->PendingError() == true)
 283       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 284
 285    Sources.PathPrefix = PathPrefix;
 286    Sources.DirStrip = ArchiveDir;
 287    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 288
 289    Sources.DeLinkLimit = DeLinkLimit;
 290    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 291
 292    if (_error->PendingError() == true)
 293       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 294
 295    c0out << ' ' << BaseDir << ":" << flush;
 296
 297    // Do recursive directory searching
 298    if (FLFile.empty() == true)
 299    {
 300       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 301          return false;
 302    }
 303    else
 304    {
 305       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 306          return false;
 307    }
 308    Sources.Output = 0;      // Just in case
 309
 310    // Finish compressing
 311    unsigned long long Size;
 312    if (Comp.Finalize(Size) == false)
 313    {
 314       c0out << endl;
 315       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 316    }
 317
 318    if (Size != 0)
 319       c0out << " New "
 320              << SizeToStr(Size) << "B ";
 321    else
 322       c0out << ' ';
 323
 324    struct timeval NewTime;
 325    gettimeofday(&NewTime,0);
 326    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 327                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 328
 329    c0out << Sources.Stats.Packages << " pkgs in " <<
 330       TimeToStr((long)Delta) << endl;
 331
 332    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 333      c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
 334
 335    Stats.Add(Sources.Stats);
 336    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 337
 338    return !_error->PendingError();
 339 }
 340                                                                         /*}}}*/
 341 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 342 // ---------------------------------------------------------------------
 343 /* This generates the contents file partially described by this object.
 344    It searches the given iterator range for other package files that map
 345    into this contents file and includes their data as well when building. */
 346 bool PackageMap::GenContents(Configuration &Setup,
 347                              vector<PackageMap>::iterator Begin,
 348                              vector<PackageMap>::iterator End,
 349                              unsigned long &Left)
 350 {
 351    if (Contents.empty() == true)
 352       return true;
 353
 354    if (Left == 0)
 355       return true;
 356
 357    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 358    string CacheDir = Setup.FindDir("Dir::CacheDir");
 359    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 360
 361    struct timeval StartTime;
 362    gettimeofday(&StartTime,0);
 363
 364    // Create a package writer object.
 365    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 366                       CntCompress,Permissions);
 367    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 368    ContentsWriter Contents(&Comp.Input, "", Arch, IncludeArchAll);
 369    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 370       return _error->Error(_("Package extension list is too long"));
 371    if (_error->PendingError() == true)
 372       return false;
 373
 374    if (_error->PendingError() == true)
 375       return false;
 376
 377    // Write the header out.
 378    if (ContentsHead.empty() == false)
 379    {
 380       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 381       if (_error->PendingError() == true)
 382          return false;
 383
 384       unsigned long long Size = Head.Size();
 385       unsigned char Buf[4096];
 386       while (Size != 0)
 387       {
 388          unsigned long long ToRead = Size;
 389          if (Size > sizeof(Buf))
 390             ToRead = sizeof(Buf);
 391
 392          if (Head.Read(Buf,ToRead) == false)
 393             return false;
 394
 395          if (Comp.Input.Write(Buf, ToRead) == false)
 396             return _error->Errno("fwrite",_("Error writing header to contents file"));
 397
 398          Size -= ToRead;
 399       }
 400    }
 401
 402    /* Go over all the package file records and parse all the package
 403       files associated with this contents file into one great big honking
 404       memory structure, then dump the sorted version */
 405    c0out << ' ' << this->Contents << ":" << flush;
 406    for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
 407    {
 408       if (I->Contents != this->Contents)
 409          continue;
 410
 411       Contents.Prefix = ArchiveDir;
 412       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 413       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 414                             I->PkgCompress);
 415
 416       I->ContentsDone = true;
 417    }
 418
 419    Contents.Finish();
 420
 421    // Finish compressing
 422    unsigned long long Size;
 423    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 424    {
 425       c0out << endl;
 426       return _error->Error(_("Error processing contents %s"),
 427                            this->Contents.c_str());
 428    }
 429
 430    if (Size != 0)
 431    {
 432       c0out << " New " << SizeToStr(Size) << "B ";
 433       if (Left > Size)
 434          Left -= Size;
 435       else
 436          Left = 0;
 437    }
 438    else
 439       c0out << ' ';
 440
 441    struct timeval NewTime;
 442    gettimeofday(&NewTime,0);
 443    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 444                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 445
 446    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 447      c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
 448
 449    c0out << Contents.Stats.Packages << " files " <<
 450       SizeToStr(Contents.Stats.Bytes) << "B " <<
 451       TimeToStr((long)Delta) << endl;
 452
 453    return true;
 454 }
 455                                                                         /*}}}*/
 456
 457 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 458 // ---------------------------------------------------------------------
 459 /* This populates the PkgList with all the possible permutations of the
 460    section/arch lists. */
 461 static void LoadTree(vector<PackageMap> &PkgList, std::vector<TranslationWriter*> &TransList, Configuration &Setup)
 462 {
 463    // Load the defaults
 464    string DDir = Setup.Find("TreeDefault::Directory",
 465                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 466    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 467                             "$(DIST)/$(SECTION)/source/");
 468    string DPkg = Setup.Find("TreeDefault::Packages",
 469                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 470    string DTrans = Setup.Find("TreeDefault::Translation",
 471                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 472    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 473                             "$(DIST)/$(SECTION)/");
 474    string DContents = Setup.Find("TreeDefault::Contents",
 475                             "$(DIST)/$(SECTION)/Contents-$(ARCH)");
 476    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 477    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 478                                "packages-$(ARCH).db");
 479    string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
 480                                "sources-$(SECTION).db");
 481    string DSources = Setup.Find("TreeDefault::Sources",
 482                                 "$(DIST)/$(SECTION)/source/Sources");
 483    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 484    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 485
 486    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 487
 488    bool const LongDescription = Setup.FindB("Default::LongDescription",
 489                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 490    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 491    bool const ConfIncludeArchAllExists = _config->Exists("APT::FTPArchive::IncludeArchitectureAll");
 492    bool const ConfIncludeArchAll = _config->FindB("APT::FTPArchive::IncludeArchitectureAll", true);
 493
 494    // Process 'tree' type sections
 495    const Configuration::Item *Top = Setup.Tree("tree");
 496    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 497    {
 498       Configuration Block(Top);
 499       string Dist = Top->Tag;
 500
 501       // Parse the sections
 502       string Tmp = Block.Find("Sections");
 503       const char *Sections = Tmp.c_str();
 504       string Section;
 505       while (ParseQuoteWord(Sections,Section) == true)
 506       {
 507          struct SubstVar Vars[] = {{"$(DIST)",&Dist},
 508                                          {"$(SECTION)",&Section},
 509                                          {"$(ARCH)",nullptr},
 510                                          {nullptr, nullptr}};
 511          mode_t const Perms = Block.FindI("FileMode", Permissions);
 512          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 513          TranslationWriter *TransWriter = nullptr;
 514
 515          std::string Tmp2 = Block.Find("Architectures");
 516          std::transform(Tmp2.begin(), Tmp2.end(), Tmp2.begin(), ::tolower);
 517          std::vector<std::string> const Archs = VectorizeString(Tmp2, ' ');
 518          bool IncludeArchAll;
 519          if (ConfIncludeArchAllExists == true)
 520             IncludeArchAll = ConfIncludeArchAll;
 521          else
 522             IncludeArchAll = std::find(Archs.begin(), Archs.end(), "all") == Archs.end();
 523          for (auto const& Arch: Archs)
 524          {
 525             if (Arch.empty()) continue;
 526             Vars[2].Contents = &Arch;
 527             PackageMap Itm;
 528             Itm.Permissions = Perms;
 529             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 530             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 531
 532             if (Arch == "source")
 533             {
 534                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 535                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 536                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 537                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 538                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 539                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 540                Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);
 541             }
 542             else
 543             {
 544                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 545                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 546                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 547                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 548                Itm.Arch = Arch;
 549                Itm.IncludeArchAll = IncludeArchAll;
 550                Itm.LongDesc = LongDesc;
 551                if (TransWriter == NULL && DTrans.empty() == false && LongDesc == false && DTrans != "/dev/null")
 552                {
 553                   string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 554                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 555                   string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 556                   TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 557                   TransList.push_back(TransWriter);
 558                }
 559                Itm.TransWriter = TransWriter;
 560                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 561                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 562                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 563                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 564             }
 565
 566             Itm.GetGeneral(Setup,Block);
 567             PkgList.push_back(Itm);
 568          }
 569       }
 570
 571       Top = Top->Next;
 572    }
 573 }
 574                                                                         /*}}}*/
 575 static void UnloadTree(std::vector<TranslationWriter*> const &Trans)            /*{{{*/
 576 {
 577    for (std::vector<TranslationWriter*>::const_reverse_iterator T = Trans.rbegin(); T != Trans.rend(); ++T)
 578       delete *T;
 579 }
 580                                                                         /*}}}*/
 581 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 582 // ---------------------------------------------------------------------
 583 /* */
 584 static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 585 {
 586    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 587
 588    // Process 'bindirectory' type sections
 589    const Configuration::Item *Top = Setup.Tree("bindirectory");
 590    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 591    {
 592       Configuration Block(Top);
 593
 594       PackageMap Itm;
 595       Itm.PkgFile = Block.Find("Packages");
 596       Itm.SrcFile = Block.Find("Sources");
 597       Itm.BinCacheDB = Block.Find("BinCacheDB");
 598       Itm.SrcCacheDB = Block.Find("SrcCacheDB");
 599       Itm.BinOverride = Block.Find("BinOverride");
 600       Itm.ExtraOverride = Block.Find("ExtraOverride");
 601       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 602       Itm.SrcOverride = Block.Find("SrcOverride");
 603       Itm.BaseDir = Top->Tag;
 604       Itm.FLFile = Block.Find("FileList");
 605       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 606       Itm.Contents = Block.Find("Contents");
 607       Itm.ContentsHead = Block.Find("Contents::Header");
 608       Itm.Permissions = Block.FindI("FileMode", Permissions);
 609
 610       Itm.GetGeneral(Setup,Block);
 611       PkgList.push_back(Itm);
 612
 613       Top = Top->Next;
 614    }
 615 }
 616                                                                         /*}}}*/
 617
 618 bool ShowHelp(CommandLine &)                                            /*{{{*/
 619 {
 620    std::cout <<
 621     _("Usage: apt-ftparchive [options] command\n"
 622       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 623       "          sources srcpath [overridefile [pathprefix]]\n"
 624       "          contents path\n"
 625       "          release path\n"
 626       "          generate config [groups]\n"
 627       "          clean config\n"
 628       "\n"
 629       "apt-ftparchive generates index files for Debian archives. It supports\n"
 630       "many styles of generation from fully automated to functional replacements\n"
 631       "for dpkg-scanpackages and dpkg-scansources\n"
 632       "\n"
 633       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 634       "Package file contains the contents of all the control fields from\n"
 635       "each package as well as the MD5 hash and filesize. An override file\n"
 636       "is supported to force the value of Priority and Section.\n"
 637       "\n"
 638       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 639       "The --source-override option can be used to specify a src override file\n"
 640       "\n"
 641       "The 'packages' and 'sources' command should be run in the root of the\n"
 642       "tree. BinaryPath should point to the base of the recursive search and \n"
 643       "override file should contain the override flags. Pathprefix is\n"
 644       "appended to the filename fields if present. Example usage from the \n"
 645       "Debian archive:\n"
 646       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 647       "               dists/potato/main/binary-i386/Packages\n"
 648       "\n"
 649       "Options:\n"
 650       "  -h    This help text\n"
 651       "  --md5 Control MD5 generation\n"
 652       "  -s=?  Source override file\n"
 653       "  -q    Quiet\n"
 654       "  -d=?  Select the optional caching database\n"
 655       "  --no-delink Enable delinking debug mode\n"
 656       "  --contents  Control contents file generation\n"
 657       "  -c=?  Read this configuration file\n"
 658       "  -o=?  Set an arbitrary configuration option") << endl;
 659    return true;
 660 }
 661                                                                         /*}}}*/
 662 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 663 // ---------------------------------------------------------------------
 664 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 665 static bool SimpleGenPackages(CommandLine &CmdL)
 666 {
 667    if (CmdL.FileSize() < 2)
 668       return ShowHelp(CmdL);
 669
 670    string Override;
 671    if (CmdL.FileSize() >= 3)
 672       Override = CmdL.FileList[2];
 673
 674    // Create a package writer object.
 675    PackagesWriter Packages(NULL, NULL, _config->Find("APT::FTPArchive::DB"),
 676                            Override, "", _config->Find("APT::FTPArchive::Architecture"),
 677                            _config->FindB("APT::FTPArchive::IncludeArchitectureAll", true));
 678    if (_error->PendingError() == true)
 679       return false;
 680
 681    if (CmdL.FileSize() >= 4)
 682       Packages.PathPrefix = CmdL.FileList[3];
 683
 684    // Do recursive directory searching
 685    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 686       return false;
 687
 688    // Give some stats if asked for
 689    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 690      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 691
 692    return true;
 693 }
 694                                                                         /*}}}*/
 695 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 696 // ---------------------------------------------------------------------
 697 /* */
 698 static bool SimpleGenContents(CommandLine &CmdL)
 699 {
 700    if (CmdL.FileSize() < 2)
 701       return ShowHelp(CmdL);
 702
 703    // Create a package writer object.
 704    ContentsWriter Contents(NULL, _config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 705    if (_error->PendingError() == true)
 706       return false;
 707
 708    // Do recursive directory searching
 709    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 710       return false;
 711
 712    Contents.Finish();
 713
 714    return true;
 715 }
 716                                                                         /*}}}*/
 717 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 718 // ---------------------------------------------------------------------
 719 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 720 static bool SimpleGenSources(CommandLine &CmdL)
 721 {
 722    if (CmdL.FileSize() < 2)
 723       return ShowHelp(CmdL);
 724
 725    string Override;
 726    if (CmdL.FileSize() >= 3)
 727       Override = CmdL.FileList[2];
 728
 729    string SOverride;
 730    if (Override.empty() == false)
 731       SOverride = Override + ".src";
 732
 733    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 734                              SOverride.c_str());
 735
 736    // Create a package writer object.
 737    SourcesWriter Sources(NULL, _config->Find("APT::FTPArchive::DB"),Override,SOverride);
 738    if (_error->PendingError() == true)
 739       return false;
 740
 741    if (CmdL.FileSize() >= 4)
 742       Sources.PathPrefix = CmdL.FileList[3];
 743
 744    // Do recursive directory searching
 745    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 746       return false;
 747
 748    // Give some stats if asked for
 749    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 750      c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl;
 751
 752    return true;
 753 }
 754                                                                         /*}}}*/
 755 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 756 // ---------------------------------------------------------------------
 757 static bool SimpleGenRelease(CommandLine &CmdL)
 758 {
 759    if (CmdL.FileSize() < 2)
 760       return ShowHelp(CmdL);
 761
 762    string Dir = CmdL.FileList[1];
 763
 764    ReleaseWriter Release(NULL, "");
 765    Release.DirStrip = Dir;
 766
 767    if (_error->PendingError() == true)
 768       return false;
 769
 770    if (Release.RecursiveScan(Dir) == false)
 771       return false;
 772
 773    Release.Finish();
 774
 775    return true;
 776 }
 777
 778                                                                         /*}}}*/
 779 // DoGeneratePackagesAndSources - Helper for Generate                   /*{{{*/
 780 // ---------------------------------------------------------------------
 781 static bool DoGeneratePackagesAndSources(Configuration &Setup,
 782                                          vector<PackageMap> &PkgList,
 783                                          struct CacheDB::Stats &SrcStats,
 784                                          struct CacheDB::Stats &Stats,
 785                                          CommandLine &CmdL)
 786 {
 787    if (CmdL.FileSize() <= 2)
 788    {
 789       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 790          if (I->GenPackages(Setup,Stats) == false)
 791             _error->DumpErrors();
 792       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 793          if (I->GenSources(Setup,SrcStats) == false)
 794             _error->DumpErrors();
 795    }
 796    else
 797    {
 798       // Make a choice list out of the package list..
 799       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 800       RxChoiceList *End = List;
 801       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 802       {
 803          End->UserData = &(*I);
 804          End->Str = I->BaseDir.c_str();
 805          End++;
 806
 807          End->UserData = &(*I);
 808          End->Str = I->Tag.c_str();
 809          End++;
 810       }
 811       End->Str = 0;
 812
 813       // Regex it
 814       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 815       {
 816          delete [] List;
 817          return _error->Error(_("No selections matched"));
 818       }
 819       _error->DumpErrors();
 820
 821       // Do the generation for Packages
 822       for (End = List; End->Str != 0; ++End)
 823       {
 824          if (End->Hit == false)
 825             continue;
 826
 827          PackageMap * const I = static_cast<PackageMap *>(End->UserData);
 828          if (I->PkgDone == true)
 829             continue;
 830          if (I->GenPackages(Setup,Stats) == false)
 831             _error->DumpErrors();
 832       }
 833
 834       // Do the generation for Sources
 835       for (End = List; End->Str != 0; ++End)
 836       {
 837          if (End->Hit == false)
 838             continue;
 839
 840          PackageMap * const I = static_cast<PackageMap *>(End->UserData);
 841          if (I->SrcDone == true)
 842             continue;
 843          if (I->GenSources(Setup,SrcStats) == false)
 844             _error->DumpErrors();
 845       }
 846
 847       delete [] List;
 848    }
 849    return true;
 850 }
 851
 852                                                                         /*}}}*/
 853 // DoGenerateContents - Helper for Generate to generate the Contents    /*{{{*/
 854 // ---------------------------------------------------------------------
 855 static bool DoGenerateContents(Configuration &Setup,
 856                                vector<PackageMap> &PkgList,
 857                                CommandLine &CmdL)
 858 {
 859    c1out << "Packages done, Starting contents." << endl;
 860
 861    // Sort the contents file list by date
 862    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 863    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 864    {
 865       struct stat A;
 866       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 867                                  I->CntCompress,A) == false)
 868          time(&I->ContentsMTime);
 869       else
 870          I->ContentsMTime = A.st_mtime;
 871    }
 872    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 873
 874    /* Now for Contents.. The process here is to do a make-like dependency
 875       check. Each contents file is verified to be newer than the package files
 876       that describe the debs it indexes. Since the package files contain
 877       hashes of the .debs this means they have not changed either so the
 878       contents must be up to date. */
 879    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",
 880                                                  std::numeric_limits<unsigned int>::max())*1024;
 881    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 882    {
 883       // This record is not relevant
 884       if (I->ContentsDone == true ||
 885           I->Contents.empty() == true)
 886          continue;
 887
 888       // Do not do everything if the user specified sections.
 889       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 890          continue;
 891
 892       struct stat A,B;
 893       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 894       {
 895          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 896          {
 897             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 898             continue;
 899          }
 900
 901          if (A.st_mtime > B.st_mtime)
 902             continue;
 903       }
 904
 905       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 906                          MaxContentsChange) == false)
 907          _error->DumpErrors();
 908
 909       // Hit the limit?
 910       if (MaxContentsChange == 0)
 911       {
 912          c1out << "Hit contents update byte limit" << endl;
 913          break;
 914       }
 915    }
 916
 917    return true;
 918 }
 919
 920                                                                         /*}}}*/
 921 // Generate - Full generate, using a config file                        /*{{{*/
 922 // ---------------------------------------------------------------------
 923 /* */
 924 static bool Generate(CommandLine &CmdL)
 925 {
 926    struct CacheDB::Stats SrcStats;
 927    if (CmdL.FileSize() < 2)
 928       return ShowHelp(CmdL);
 929
 930    struct timeval StartTime;
 931    gettimeofday(&StartTime,0);
 932    struct CacheDB::Stats Stats;
 933
 934    // Read the configuration file.
 935    Configuration Setup;
 936    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 937       return false;
 938
 939    vector<PackageMap> PkgList;
 940    std::vector<TranslationWriter*> TransList;
 941    LoadTree(PkgList, TransList, Setup);
 942    LoadBinDir(PkgList,Setup);
 943
 944    // Sort by cache DB to improve IO locality.
 945    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 946    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 947
 948    // Generate packages
 949    if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false)
 950    {
 951       if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false)
 952       {
 953          UnloadTree(TransList);
 954          return false;
 955       }
 956    } else {
 957       c1out << "Skipping Packages/Sources generation" << endl;
 958    }
 959
 960    // do Contents if needed
 961    if (_config->FindB("APT::FTPArchive::Contents", true) == true)
 962       if (DoGenerateContents(Setup, PkgList, CmdL) == false)
 963       {
 964          UnloadTree(TransList);
 965          return false;
 966       }
 967
 968    struct timeval NewTime;
 969    gettimeofday(&NewTime,0);
 970    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 971                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 972    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 973          << " archives. Took " << TimeToStr((long)Delta) << endl;
 974
 975    UnloadTree(TransList);
 976    return true;
 977 }
 978
 979                                                                         /*}}}*/
 980 // Clean - Clean out the databases                                      /*{{{*/
 981 // ---------------------------------------------------------------------
 982 /* */
 983 static bool Clean(CommandLine &CmdL)
 984 {
 985    if (CmdL.FileSize() != 2)
 986       return ShowHelp(CmdL);
 987
 988    // Read the configuration file.
 989    Configuration Setup;
 990    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 991       return false;
 992    // we don't need translation creation here
 993    Setup.Set("TreeDefault::Translation", "/dev/null");
 994
 995    vector<PackageMap> PkgList;
 996    std::vector<TranslationWriter*> TransList;
 997    LoadTree(PkgList, TransList, Setup);
 998    LoadBinDir(PkgList,Setup);
 999
1000    // Sort by cache DB to improve IO locality.
1001    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
1002    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
1003
1004    string CacheDir = Setup.FindDir("Dir::CacheDir");
1005
1006    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
1007    {
1008       if(I->BinCacheDB != "")
1009          c0out << I->BinCacheDB << endl;
1010       if(I->SrcCacheDB != "")
1011          c0out << I->SrcCacheDB << endl;
1012       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
1013       CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));
1014       if (DB.Clean() == false)
1015          _error->DumpErrors();
1016       if (DB_SRC.Clean() == false)
1017          _error->DumpErrors();
1018
1019       string CacheDB = I->BinCacheDB;
1020       string SrcCacheDB = I->SrcCacheDB;
1021       while(I != PkgList.end() &&
1022             I->BinCacheDB == CacheDB &&
1023             I->SrcCacheDB == SrcCacheDB)
1024          ++I;
1025    }
1026
1027
1028    return true;
1029 }
1030                                                                         /*}}}*/
1031
1032 std::vector<aptDispatchWithHelp> GetCommands()                          /*{{{*/
1033 {
1034    return {
1035       {"packages",&SimpleGenPackages, nullptr},
1036       {"contents",&SimpleGenContents, nullptr},
1037       {"sources",&SimpleGenSources, nullptr},
1038       {"release",&SimpleGenRelease, nullptr},
1039       {"generate",&Generate, nullptr},
1040       {"clean",&Clean, nullptr},
1041       {nullptr, nullptr, nullptr}
1042    };
1043 }
1044                                                                         /*}}}*/
1045 int main(int argc, const char *argv[])                                  /*{{{*/
1046 {
1047    InitLocale();
1048
1049    // Parse the command line and initialize the package library
1050    CommandLine CmdL;
1051    auto const Cmds = ParseCommandLine(CmdL, APT_CMD::APT_FTPARCHIVE, &_config, NULL, argc, argv, ShowHelp, &GetCommands);
1052
1053    _config->CndSet("quiet",0);
1054    Quiet = _config->FindI("quiet",0);
1055    InitOutput(clog.rdbuf());
1056
1057    return DispatchCommandLine(CmdL, Cmds);
1058 }
1059                                                                         /*}}}*/