ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <apt-pkg/fileutl.h>
  21
  22 #include <apt-private/private-cmndline.h>
  23 #include <apt-private/private-output.h>
  24 #include <apt-private/private-main.h>
  25
  26 #include <algorithm>
  27 #include <climits>
  28 #include <sys/time.h>
  29 #include <locale.h>
  30 #include <stdio.h>
  31 #include <sys/stat.h>
  32 #include <time.h>
  33 #include <functional>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37
  38 #include "cachedb.h"
  39 #include "override.h"
  40 #include "apt-ftparchive.h"
  41 #include "multicompress.h"
  42 #include "writer.h"
  43
  44 #include <apti18n.h>
  45                                                                         /*}}}*/
  46
  47 using namespace std;
  48 unsigned Quiet = 0;
  49
  50 // struct PackageMap - List of all package files in the config file     /*{{{*/
  51 // ---------------------------------------------------------------------
  52 /* */
  53 struct PackageMap
  54 {
  55    // General Stuff
  56    string BaseDir;
  57    string InternalPrefix;
  58    string FLFile;
  59    string PkgExt;
  60    string SrcExt;
  61
  62    // Stuff for the Package File
  63    string PkgFile;
  64    string BinCacheDB;
  65    string SrcCacheDB;
  66    string BinOverride;
  67    string ExtraOverride;
  68
  69    // We generate for this given arch
  70    string Arch;
  71
  72    // Stuff for the Source File
  73    string SrcFile;
  74    string SrcOverride;
  75    string SrcExtraOverride;
  76
  77    // Translation master file
  78    bool LongDesc;
  79    TranslationWriter *TransWriter;
  80
  81    // Contents
  82    string Contents;
  83    string ContentsHead;
  84
  85    // Random things
  86    string Tag;
  87    string PkgCompress;
  88    string CntCompress;
  89    string SrcCompress;
  90    string PathPrefix;
  91    unsigned int DeLinkLimit;
  92    mode_t Permissions;
  93
  94    bool ContentsDone;
  95    bool PkgDone;
  96    bool SrcDone;
  97    time_t ContentsMTime;
  98
  99    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
 100    {
 101       inline bool operator() (const PackageMap &x,const PackageMap &y)
 102       {return x.ContentsMTime < y.ContentsMTime;};
 103    };
 104
 105    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
 106    {
 107       inline bool operator() (const PackageMap &x,const PackageMap &y)
 108       {return x.BinCacheDB < y.BinCacheDB;};
 109    };
 110
 111    struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
 112    {
 113       inline bool operator() (const PackageMap &x,const PackageMap &y)
 114       {return x.SrcCacheDB < y.SrcCacheDB;};
 115    };
 116
 117    void GetGeneral(Configuration &Setup,Configuration &Block);
 118    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 119    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 120    bool GenContents(Configuration &Setup,
 121                     vector<PackageMap>::iterator Begin,
 122                     vector<PackageMap>::iterator End,
 123                     unsigned long &Left);
 124
 125    PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 126                   ContentsDone(false), PkgDone(false), SrcDone(false),
 127                   ContentsMTime(0) {};
 128 };
 129                                                                         /*}}}*/
 130
 131 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 132 // ---------------------------------------------------------------------
 133 /* */
 134 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 135 {
 136    PathPrefix = Block.Find("PathPrefix");
 137
 138    if (Block.FindB("External-Links",true) == false)
 139       DeLinkLimit = Setup.FindI("Default::DeLinkLimit", std::numeric_limits<unsigned int>::max());
 140    else
 141       DeLinkLimit = 0;
 142
 143    PkgCompress = Block.Find("Packages::Compress",
 144                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 145    CntCompress = Block.Find("Contents::Compress",
 146                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 147    SrcCompress = Block.Find("Sources::Compress",
 148                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 149
 150    SrcExt = Block.Find("Sources::Extensions",
 151                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 152    PkgExt = Block.Find("Packages::Extensions",
 153                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 154
 155    Permissions = Setup.FindI("Default::FileMode",0644);
 156
 157    if (FLFile.empty() == false)
 158       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 159
 160    if (Contents == " ")
 161       Contents= string();
 162 }
 163                                                                         /*}}}*/
 164 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 165 // ---------------------------------------------------------------------
 166 /* This generates the Package File described by this object. */
 167 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 168 {
 169    if (PkgFile.empty() == true)
 170       return true;
 171
 172    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 173    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 174    string CacheDir = Setup.FindDir("Dir::CacheDir");
 175
 176    struct timeval StartTime;
 177    gettimeofday(&StartTime,0);
 178
 179    PkgDone = true;
 180
 181    // Create a package writer object.
 182    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 183                       PkgCompress,Permissions);
 184    PackagesWriter Packages(&Comp.Input, TransWriter, flCombine(CacheDir,BinCacheDB),
 185                            flCombine(OverrideDir,BinOverride),
 186                            flCombine(OverrideDir,ExtraOverride),
 187                            Arch);
 188    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 189       return _error->Error(_("Package extension list is too long"));
 190    if (_error->PendingError() == true)
 191       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 192
 193    Packages.PathPrefix = PathPrefix;
 194    Packages.DirStrip = ArchiveDir;
 195    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 196
 197    Packages.LongDescription = LongDesc;
 198
 199    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 200    Packages.DeLinkLimit = DeLinkLimit;
 201
 202    if (_error->PendingError() == true)
 203       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 204
 205    c0out << ' ' << BaseDir << ":" << flush;
 206
 207    // Do recursive directory searching
 208    if (FLFile.empty() == true)
 209    {
 210       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 211          return false;
 212    }
 213    else
 214    {
 215       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 216          return false;
 217    }
 218
 219    Packages.Output = 0;      // Just in case
 220
 221    // Finish compressing
 222    unsigned long long Size;
 223    if (Comp.Finalize(Size) == false)
 224    {
 225       c0out << endl;
 226       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 227    }
 228
 229    if (Size != 0)
 230       c0out << " New "
 231              << SizeToStr(Size) << "B ";
 232    else
 233       c0out << ' ';
 234
 235    struct timeval NewTime;
 236    gettimeofday(&NewTime,0);
 237    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 238                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 239
 240    c0out << Packages.Stats.Packages << " files " <<
 241 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 242       SizeToStr(Packages.Stats.Bytes) << "B " <<
 243       TimeToStr((long)Delta) << endl;
 244
 245    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 246      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 247
 248    Stats.Add(Packages.Stats);
 249    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 250
 251    return !_error->PendingError();
 252 }
 253
 254                                                                         /*}}}*/
 255 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 256 // ---------------------------------------------------------------------
 257 /* This generates the Sources File described by this object. */
 258 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 259 {
 260    if (SrcFile.empty() == true)
 261       return true;
 262
 263    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 264    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 265    string CacheDir = Setup.FindDir("Dir::CacheDir");
 266
 267    struct timeval StartTime;
 268    gettimeofday(&StartTime,0);
 269
 270    SrcDone = true;
 271
 272    // Create a package writer object.
 273    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 274                       SrcCompress,Permissions);
 275    SourcesWriter Sources(&Comp.Input, flCombine(CacheDir, SrcCacheDB),
 276                          flCombine(OverrideDir,BinOverride),
 277                          flCombine(OverrideDir,SrcOverride),
 278                          flCombine(OverrideDir,SrcExtraOverride));
 279    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 280       return _error->Error(_("Source extension list is too long"));
 281    if (_error->PendingError() == true)
 282       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 283
 284    Sources.PathPrefix = PathPrefix;
 285    Sources.DirStrip = ArchiveDir;
 286    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 287
 288    Sources.DeLinkLimit = DeLinkLimit;
 289    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 290
 291    if (_error->PendingError() == true)
 292       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 293
 294    c0out << ' ' << BaseDir << ":" << flush;
 295
 296    // Do recursive directory searching
 297    if (FLFile.empty() == true)
 298    {
 299       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 300          return false;
 301    }
 302    else
 303    {
 304       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 305          return false;
 306    }
 307    Sources.Output = 0;      // Just in case
 308
 309    // Finish compressing
 310    unsigned long long Size;
 311    if (Comp.Finalize(Size) == false)
 312    {
 313       c0out << endl;
 314       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 315    }
 316
 317    if (Size != 0)
 318       c0out << " New "
 319              << SizeToStr(Size) << "B ";
 320    else
 321       c0out << ' ';
 322
 323    struct timeval NewTime;
 324    gettimeofday(&NewTime,0);
 325    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 326                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 327
 328    c0out << Sources.Stats.Packages << " pkgs in " <<
 329       TimeToStr((long)Delta) << endl;
 330
 331    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 332      c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
 333
 334    Stats.Add(Sources.Stats);
 335    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 336
 337    return !_error->PendingError();
 338 }
 339                                                                         /*}}}*/
 340 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 341 // ---------------------------------------------------------------------
 342 /* This generates the contents file partially described by this object.
 343    It searches the given iterator range for other package files that map
 344    into this contents file and includes their data as well when building. */
 345 bool PackageMap::GenContents(Configuration &Setup,
 346                              vector<PackageMap>::iterator Begin,
 347                              vector<PackageMap>::iterator End,
 348                              unsigned long &Left)
 349 {
 350    if (Contents.empty() == true)
 351       return true;
 352
 353    if (Left == 0)
 354       return true;
 355
 356    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 357    string CacheDir = Setup.FindDir("Dir::CacheDir");
 358    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 359
 360    struct timeval StartTime;
 361    gettimeofday(&StartTime,0);
 362
 363    // Create a package writer object.
 364    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 365                       CntCompress,Permissions);
 366    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 367    ContentsWriter Contents(&Comp.Input, "", Arch);
 368    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 369       return _error->Error(_("Package extension list is too long"));
 370    if (_error->PendingError() == true)
 371       return false;
 372
 373    if (_error->PendingError() == true)
 374       return false;
 375
 376    // Write the header out.
 377    if (ContentsHead.empty() == false)
 378    {
 379       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 380       if (_error->PendingError() == true)
 381          return false;
 382
 383       unsigned long long Size = Head.Size();
 384       unsigned char Buf[4096];
 385       while (Size != 0)
 386       {
 387          unsigned long long ToRead = Size;
 388          if (Size > sizeof(Buf))
 389             ToRead = sizeof(Buf);
 390
 391          if (Head.Read(Buf,ToRead) == false)
 392             return false;
 393
 394          if (Comp.Input.Write(Buf, ToRead) == false)
 395             return _error->Errno("fwrite",_("Error writing header to contents file"));
 396
 397          Size -= ToRead;
 398       }
 399    }
 400
 401    /* Go over all the package file records and parse all the package
 402       files associated with this contents file into one great big honking
 403       memory structure, then dump the sorted version */
 404    c0out << ' ' << this->Contents << ":" << flush;
 405    for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
 406    {
 407       if (I->Contents != this->Contents)
 408          continue;
 409
 410       Contents.Prefix = ArchiveDir;
 411       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 412       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 413                             I->PkgCompress);
 414
 415       I->ContentsDone = true;
 416    }
 417
 418    Contents.Finish();
 419
 420    // Finish compressing
 421    unsigned long long Size;
 422    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 423    {
 424       c0out << endl;
 425       return _error->Error(_("Error processing contents %s"),
 426                            this->Contents.c_str());
 427    }
 428
 429    if (Size != 0)
 430    {
 431       c0out << " New " << SizeToStr(Size) << "B ";
 432       if (Left > Size)
 433          Left -= Size;
 434       else
 435          Left = 0;
 436    }
 437    else
 438       c0out << ' ';
 439
 440    struct timeval NewTime;
 441    gettimeofday(&NewTime,0);
 442    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 443                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 444
 445    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 446      c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
 447
 448    c0out << Contents.Stats.Packages << " files " <<
 449       SizeToStr(Contents.Stats.Bytes) << "B " <<
 450       TimeToStr((long)Delta) << endl;
 451
 452    return true;
 453 }
 454                                                                         /*}}}*/
 455
 456 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 457 // ---------------------------------------------------------------------
 458 /* This populates the PkgList with all the possible permutations of the
 459    section/arch lists. */
 460 static void LoadTree(vector<PackageMap> &PkgList, std::vector<TranslationWriter*> &TransList, Configuration &Setup)
 461 {
 462    // Load the defaults
 463    string DDir = Setup.Find("TreeDefault::Directory",
 464                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 465    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 466                             "$(DIST)/$(SECTION)/source/");
 467    string DPkg = Setup.Find("TreeDefault::Packages",
 468                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 469    string DTrans = Setup.Find("TreeDefault::Translation",
 470                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 471    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 472                             "$(DIST)/$(SECTION)/");
 473    string DContents = Setup.Find("TreeDefault::Contents",
 474                             "$(DIST)/$(SECTION)/Contents-$(ARCH)");
 475    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 476    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 477                                "packages-$(ARCH).db");
 478    string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
 479                                "sources-$(SECTION).db");
 480    string DSources = Setup.Find("TreeDefault::Sources",
 481                                 "$(DIST)/$(SECTION)/source/Sources");
 482    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 483    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 484
 485    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 486
 487    bool const LongDescription = Setup.FindB("Default::LongDescription",
 488                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 489    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 490
 491    // Process 'tree' type sections
 492    const Configuration::Item *Top = Setup.Tree("tree");
 493    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 494    {
 495       Configuration Block(Top);
 496       string Dist = Top->Tag;
 497
 498       // Parse the sections
 499       string Tmp = Block.Find("Sections");
 500       const char *Sections = Tmp.c_str();
 501       string Section;
 502       while (ParseQuoteWord(Sections,Section) == true)
 503       {
 504          string Arch;
 505          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 506                                          {"$(SECTION)",&Section},
 507                                          {"$(ARCH)",&Arch},
 508                                          {NULL, NULL}};
 509          mode_t const Perms = Block.FindI("FileMode", Permissions);
 510          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 511          TranslationWriter *TransWriter = NULL;
 512
 513          string const Tmp2 = Block.Find("Architectures");
 514          const char *Archs = Tmp2.c_str();
 515          while (ParseQuoteWord(Archs,Arch) == true)
 516          {
 517             PackageMap Itm;
 518             Itm.Permissions = Perms;
 519             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 520             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 521
 522             if (stringcasecmp(Arch,"source") == 0)
 523             {
 524                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 525                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 526                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 527                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 528                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 529                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 530                Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);
 531             }
 532             else
 533             {
 534                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 535                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 536                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 537                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 538                Itm.Arch = Arch;
 539                Itm.LongDesc = LongDesc;
 540                if (TransWriter == NULL && DTrans.empty() == false && LongDesc == false && DTrans != "/dev/null")
 541                {
 542                   string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 543                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 544                   string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 545                   TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 546                   TransList.push_back(TransWriter);
 547                }
 548                Itm.TransWriter = TransWriter;
 549                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 550                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 551                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 552                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 553             }
 554
 555             Itm.GetGeneral(Setup,Block);
 556             PkgList.push_back(Itm);
 557          }
 558       }
 559
 560       Top = Top->Next;
 561    }
 562 }
 563                                                                         /*}}}*/
 564 static void UnloadTree(std::vector<TranslationWriter*> const &Trans)            /*{{{*/
 565 {
 566    for (std::vector<TranslationWriter*>::const_reverse_iterator T = Trans.rbegin(); T != Trans.rend(); ++T)
 567       delete *T;
 568 }
 569                                                                         /*}}}*/
 570 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 571 // ---------------------------------------------------------------------
 572 /* */
 573 static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 574 {
 575    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 576
 577    // Process 'bindirectory' type sections
 578    const Configuration::Item *Top = Setup.Tree("bindirectory");
 579    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 580    {
 581       Configuration Block(Top);
 582
 583       PackageMap Itm;
 584       Itm.PkgFile = Block.Find("Packages");
 585       Itm.SrcFile = Block.Find("Sources");
 586       Itm.BinCacheDB = Block.Find("BinCacheDB");
 587       Itm.SrcCacheDB = Block.Find("SrcCacheDB");
 588       Itm.BinOverride = Block.Find("BinOverride");
 589       Itm.ExtraOverride = Block.Find("ExtraOverride");
 590       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 591       Itm.SrcOverride = Block.Find("SrcOverride");
 592       Itm.BaseDir = Top->Tag;
 593       Itm.FLFile = Block.Find("FileList");
 594       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 595       Itm.Contents = Block.Find("Contents");
 596       Itm.ContentsHead = Block.Find("Contents::Header");
 597       Itm.Permissions = Block.FindI("FileMode", Permissions);
 598
 599       Itm.GetGeneral(Setup,Block);
 600       PkgList.push_back(Itm);
 601
 602       Top = Top->Next;
 603    }
 604 }
 605                                                                         /*}}}*/
 606
 607 bool ShowHelp(CommandLine &, aptDispatchWithHelp const *)               /*{{{*/
 608 {
 609    std::cout <<
 610     _("Usage: apt-ftparchive [options] command\n"
 611       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 612       "          sources srcpath [overridefile [pathprefix]]\n"
 613       "          contents path\n"
 614       "          release path\n"
 615       "          generate config [groups]\n"
 616       "          clean config\n"
 617       "\n"
 618       "apt-ftparchive generates index files for Debian archives. It supports\n"
 619       "many styles of generation from fully automated to functional replacements\n"
 620       "for dpkg-scanpackages and dpkg-scansources\n"
 621       "\n"
 622       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 623       "Package file contains the contents of all the control fields from\n"
 624       "each package as well as the MD5 hash and filesize. An override file\n"
 625       "is supported to force the value of Priority and Section.\n"
 626       "\n"
 627       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 628       "The --source-override option can be used to specify a src override file\n"
 629       "\n"
 630       "The 'packages' and 'sources' command should be run in the root of the\n"
 631       "tree. BinaryPath should point to the base of the recursive search and \n"
 632       "override file should contain the override flags. Pathprefix is\n"
 633       "appended to the filename fields if present. Example usage from the \n"
 634       "Debian archive:\n"
 635       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 636       "               dists/potato/main/binary-i386/Packages\n"
 637       "\n"
 638       "Options:\n"
 639       "  -h    This help text\n"
 640       "  --md5 Control MD5 generation\n"
 641       "  -s=?  Source override file\n"
 642       "  -q    Quiet\n"
 643       "  -d=?  Select the optional caching database\n"
 644       "  --no-delink Enable delinking debug mode\n"
 645       "  --contents  Control contents file generation\n"
 646       "  -c=?  Read this configuration file\n"
 647       "  -o=?  Set an arbitrary configuration option") << endl;
 648    return true;
 649 }
 650                                                                         /*}}}*/
 651 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 652 // ---------------------------------------------------------------------
 653 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 654 static bool SimpleGenPackages(CommandLine &CmdL)
 655 {
 656    if (CmdL.FileSize() < 2)
 657       return ShowHelp(CmdL, nullptr);
 658
 659    string Override;
 660    if (CmdL.FileSize() >= 3)
 661       Override = CmdL.FileList[2];
 662
 663    // Create a package writer object.
 664    PackagesWriter Packages(NULL, NULL, _config->Find("APT::FTPArchive::DB"),
 665                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 666    if (_error->PendingError() == true)
 667       return false;
 668
 669    if (CmdL.FileSize() >= 4)
 670       Packages.PathPrefix = CmdL.FileList[3];
 671
 672    // Do recursive directory searching
 673    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 674       return false;
 675
 676    // Give some stats if asked for
 677    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 678      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 679
 680    return true;
 681 }
 682                                                                         /*}}}*/
 683 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 684 // ---------------------------------------------------------------------
 685 /* */
 686 static bool SimpleGenContents(CommandLine &CmdL)
 687 {
 688    if (CmdL.FileSize() < 2)
 689       return ShowHelp(CmdL, nullptr);
 690
 691    // Create a package writer object.
 692    ContentsWriter Contents(NULL, _config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 693    if (_error->PendingError() == true)
 694       return false;
 695
 696    // Do recursive directory searching
 697    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 698       return false;
 699
 700    Contents.Finish();
 701
 702    return true;
 703 }
 704                                                                         /*}}}*/
 705 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 706 // ---------------------------------------------------------------------
 707 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 708 static bool SimpleGenSources(CommandLine &CmdL)
 709 {
 710    if (CmdL.FileSize() < 2)
 711       return ShowHelp(CmdL, nullptr);
 712
 713    string Override;
 714    if (CmdL.FileSize() >= 3)
 715       Override = CmdL.FileList[2];
 716
 717    string SOverride;
 718    if (Override.empty() == false)
 719       SOverride = Override + ".src";
 720
 721    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 722                              SOverride.c_str());
 723
 724    // Create a package writer object.
 725    SourcesWriter Sources(NULL, _config->Find("APT::FTPArchive::DB"),Override,SOverride);
 726    if (_error->PendingError() == true)
 727       return false;
 728
 729    if (CmdL.FileSize() >= 4)
 730       Sources.PathPrefix = CmdL.FileList[3];
 731
 732    // Do recursive directory searching
 733    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 734       return false;
 735
 736    // Give some stats if asked for
 737    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 738      c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl;
 739
 740    return true;
 741 }
 742                                                                         /*}}}*/
 743 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 744 // ---------------------------------------------------------------------
 745 static bool SimpleGenRelease(CommandLine &CmdL)
 746 {
 747    if (CmdL.FileSize() < 2)
 748       return ShowHelp(CmdL, nullptr);
 749
 750    string Dir = CmdL.FileList[1];
 751
 752    ReleaseWriter Release(NULL, "");
 753    Release.DirStrip = Dir;
 754
 755    if (_error->PendingError() == true)
 756       return false;
 757
 758    if (Release.RecursiveScan(Dir) == false)
 759       return false;
 760
 761    Release.Finish();
 762
 763    return true;
 764 }
 765
 766                                                                         /*}}}*/
 767 // DoGeneratePackagesAndSources - Helper for Generate                   /*{{{*/
 768 // ---------------------------------------------------------------------
 769 static bool DoGeneratePackagesAndSources(Configuration &Setup,
 770                                          vector<PackageMap> &PkgList,
 771                                          struct CacheDB::Stats &SrcStats,
 772                                          struct CacheDB::Stats &Stats,
 773                                          CommandLine &CmdL)
 774 {
 775    if (CmdL.FileSize() <= 2)
 776    {
 777       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 778          if (I->GenPackages(Setup,Stats) == false)
 779             _error->DumpErrors();
 780       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 781          if (I->GenSources(Setup,SrcStats) == false)
 782             _error->DumpErrors();
 783    }
 784    else
 785    {
 786       // Make a choice list out of the package list..
 787       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 788       RxChoiceList *End = List;
 789       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 790       {
 791          End->UserData = &(*I);
 792          End->Str = I->BaseDir.c_str();
 793          End++;
 794
 795          End->UserData = &(*I);
 796          End->Str = I->Tag.c_str();
 797          End++;
 798       }
 799       End->Str = 0;
 800
 801       // Regex it
 802       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 803       {
 804          delete [] List;
 805          return _error->Error(_("No selections matched"));
 806       }
 807       _error->DumpErrors();
 808
 809       // Do the generation for Packages
 810       for (End = List; End->Str != 0; End++)
 811       {
 812          if (End->Hit == false)
 813             continue;
 814
 815          PackageMap *I = (PackageMap *)End->UserData;
 816          if (I->PkgDone == true)
 817             continue;
 818          if (I->GenPackages(Setup,Stats) == false)
 819             _error->DumpErrors();
 820       }
 821
 822       // Do the generation for Sources
 823       for (End = List; End->Str != 0; End++)
 824       {
 825          if (End->Hit == false)
 826             continue;
 827
 828          PackageMap *I = (PackageMap *)End->UserData;
 829          if (I->SrcDone == true)
 830             continue;
 831          if (I->GenSources(Setup,SrcStats) == false)
 832             _error->DumpErrors();
 833       }
 834
 835       delete [] List;
 836    }
 837    return true;
 838 }
 839
 840                                                                         /*}}}*/
 841 // DoGenerateContents - Helper for Generate to generate the Contents    /*{{{*/
 842 // ---------------------------------------------------------------------
 843 static bool DoGenerateContents(Configuration &Setup,
 844                                vector<PackageMap> &PkgList,
 845                                CommandLine &CmdL)
 846 {
 847    c1out << "Packages done, Starting contents." << endl;
 848
 849    // Sort the contents file list by date
 850    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 851    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 852    {
 853       struct stat A;
 854       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 855                                  I->CntCompress,A) == false)
 856          time(&I->ContentsMTime);
 857       else
 858          I->ContentsMTime = A.st_mtime;
 859    }
 860    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 861
 862    /* Now for Contents.. The process here is to do a make-like dependency
 863       check. Each contents file is verified to be newer than the package files
 864       that describe the debs it indexes. Since the package files contain
 865       hashes of the .debs this means they have not changed either so the
 866       contents must be up to date. */
 867    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",
 868                                                  std::numeric_limits<unsigned int>::max())*1024;
 869    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 870    {
 871       // This record is not relevant
 872       if (I->ContentsDone == true ||
 873           I->Contents.empty() == true)
 874          continue;
 875
 876       // Do not do everything if the user specified sections.
 877       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 878          continue;
 879
 880       struct stat A,B;
 881       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 882       {
 883          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 884          {
 885             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 886             continue;
 887          }
 888
 889          if (A.st_mtime > B.st_mtime)
 890             continue;
 891       }
 892
 893       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 894                          MaxContentsChange) == false)
 895          _error->DumpErrors();
 896
 897       // Hit the limit?
 898       if (MaxContentsChange == 0)
 899       {
 900          c1out << "Hit contents update byte limit" << endl;
 901          break;
 902       }
 903    }
 904
 905    return true;
 906 }
 907
 908                                                                         /*}}}*/
 909 // Generate - Full generate, using a config file                        /*{{{*/
 910 // ---------------------------------------------------------------------
 911 /* */
 912 static bool Generate(CommandLine &CmdL)
 913 {
 914    struct CacheDB::Stats SrcStats;
 915    if (CmdL.FileSize() < 2)
 916       return ShowHelp(CmdL, nullptr);
 917
 918    struct timeval StartTime;
 919    gettimeofday(&StartTime,0);
 920    struct CacheDB::Stats Stats;
 921
 922    // Read the configuration file.
 923    Configuration Setup;
 924    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 925       return false;
 926
 927    vector<PackageMap> PkgList;
 928    std::vector<TranslationWriter*> TransList;
 929    LoadTree(PkgList, TransList, Setup);
 930    LoadBinDir(PkgList,Setup);
 931
 932    // Sort by cache DB to improve IO locality.
 933    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 934    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 935
 936    // Generate packages
 937    if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false)
 938    {
 939       if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false)
 940       {
 941          UnloadTree(TransList);
 942          return false;
 943       }
 944    } else {
 945       c1out << "Skipping Packages/Sources generation" << endl;
 946    }
 947
 948    // do Contents if needed
 949    if (_config->FindB("APT::FTPArchive::Contents", true) == true)
 950       if (DoGenerateContents(Setup, PkgList, CmdL) == false)
 951       {
 952          UnloadTree(TransList);
 953          return false;
 954       }
 955
 956    struct timeval NewTime;
 957    gettimeofday(&NewTime,0);
 958    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 959                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 960    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 961          << " archives. Took " << TimeToStr((long)Delta) << endl;
 962
 963    UnloadTree(TransList);
 964    return true;
 965 }
 966
 967                                                                         /*}}}*/
 968 // Clean - Clean out the databases                                      /*{{{*/
 969 // ---------------------------------------------------------------------
 970 /* */
 971 static bool Clean(CommandLine &CmdL)
 972 {
 973    if (CmdL.FileSize() != 2)
 974       return ShowHelp(CmdL, nullptr);
 975
 976    // Read the configuration file.
 977    Configuration Setup;
 978    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 979       return false;
 980    // we don't need translation creation here
 981    Setup.Set("TreeDefault::Translation", "/dev/null");
 982
 983    vector<PackageMap> PkgList;
 984    std::vector<TranslationWriter*> TransList;
 985    LoadTree(PkgList, TransList, Setup);
 986    LoadBinDir(PkgList,Setup);
 987
 988    // Sort by cache DB to improve IO locality.
 989    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 990    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 991
 992    string CacheDir = Setup.FindDir("Dir::CacheDir");
 993
 994    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 995    {
 996       if(I->BinCacheDB != "")
 997          c0out << I->BinCacheDB << endl;
 998       if(I->SrcCacheDB != "")
 999          c0out << I->SrcCacheDB << endl;
1000       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
1001       CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));
1002       if (DB.Clean() == false)
1003          _error->DumpErrors();
1004       if (DB_SRC.Clean() == false)
1005          _error->DumpErrors();
1006
1007       string CacheDB = I->BinCacheDB;
1008       string SrcCacheDB = I->SrcCacheDB;
1009       while(I != PkgList.end() &&
1010             I->BinCacheDB == CacheDB &&
1011             I->SrcCacheDB == SrcCacheDB)
1012          ++I;
1013    }
1014
1015
1016    return true;
1017 }
1018                                                                         /*}}}*/
1019
1020 std::vector<aptDispatchWithHelp> GetCommands()                          /*{{{*/
1021 {
1022    return {
1023       {"packages",&SimpleGenPackages, nullptr},
1024       {"contents",&SimpleGenContents, nullptr},
1025       {"sources",&SimpleGenSources, nullptr},
1026       {"release",&SimpleGenRelease, nullptr},
1027       {"generate",&Generate, nullptr},
1028       {"clean",&Clean, nullptr},
1029       {nullptr, nullptr, nullptr}
1030    };
1031 }
1032                                                                         /*}}}*/
1033 int main(int argc, const char *argv[])                                  /*{{{*/
1034 {
1035    InitLocale();
1036
1037    // Parse the command line and initialize the package library
1038    CommandLine CmdL;
1039    auto const Cmds = ParseCommandLine(CmdL, APT_CMD::APT_FTPARCHIVE, &_config, NULL, argc, argv);
1040
1041    _config->CndSet("quiet",0);
1042    Quiet = _config->FindI("quiet",0);
1043    InitOutput(clog.rdbuf());
1044
1045    return DispatchCommandLine(CmdL, Cmds);
1046 }
1047                                                                         /*}}}*/