ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <apt-pkg/fileutl.h>
  21
  22 #include <apt-private/private-cmndline.h>
  23 #include <apt-private/private-output.h>
  24 #include <apt-private/private-main.h>
  25
  26 #include <algorithm>
  27 #include <climits>
  28 #include <sys/time.h>
  29 #include <locale.h>
  30 #include <stdio.h>
  31 #include <sys/stat.h>
  32 #include <time.h>
  33 #include <functional>
  34 #include <iostream>
  35 #include <string>
  36 #include <vector>
  37
  38 #include "cachedb.h"
  39 #include "override.h"
  40 #include "apt-ftparchive.h"
  41 #include "multicompress.h"
  42 #include "writer.h"
  43
  44 #include <apti18n.h>
  45                                                                         /*}}}*/
  46
  47 using namespace std;
  48 unsigned Quiet = 0;
  49
  50 // struct PackageMap - List of all package files in the config file     /*{{{*/
  51 // ---------------------------------------------------------------------
  52 /* */
  53 struct PackageMap
  54 {
  55    // General Stuff
  56    string BaseDir;
  57    string InternalPrefix;
  58    string FLFile;
  59    string PkgExt;
  60    string SrcExt;
  61
  62    // Stuff for the Package File
  63    string PkgFile;
  64    string BinCacheDB;
  65    string SrcCacheDB;
  66    string BinOverride;
  67    string ExtraOverride;
  68
  69    // We generate for this given arch
  70    string Arch;
  71
  72    // Stuff for the Source File
  73    string SrcFile;
  74    string SrcOverride;
  75    string SrcExtraOverride;
  76
  77    // Translation master file
  78    bool LongDesc;
  79    TranslationWriter *TransWriter;
  80
  81    // Contents
  82    string Contents;
  83    string ContentsHead;
  84
  85    // Random things
  86    string Tag;
  87    string PkgCompress;
  88    string CntCompress;
  89    string SrcCompress;
  90    string PathPrefix;
  91    unsigned int DeLinkLimit;
  92    mode_t Permissions;
  93
  94    bool ContentsDone;
  95    bool PkgDone;
  96    bool SrcDone;
  97    time_t ContentsMTime;
  98
  99    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
 100    {
 101       inline bool operator() (const PackageMap &x,const PackageMap &y)
 102       {return x.ContentsMTime < y.ContentsMTime;};
 103    };
 104
 105    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
 106    {
 107       inline bool operator() (const PackageMap &x,const PackageMap &y)
 108       {return x.BinCacheDB < y.BinCacheDB;};
 109    };
 110
 111    struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
 112    {
 113       inline bool operator() (const PackageMap &x,const PackageMap &y)
 114       {return x.SrcCacheDB < y.SrcCacheDB;};
 115    };
 116
 117    void GetGeneral(Configuration &Setup,Configuration &Block);
 118    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 119    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 120    bool GenContents(Configuration &Setup,
 121                     vector<PackageMap>::iterator Begin,
 122                     vector<PackageMap>::iterator End,
 123                     unsigned long &Left);
 124
 125    PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 126                   ContentsDone(false), PkgDone(false), SrcDone(false),
 127                   ContentsMTime(0) {};
 128 };
 129                                                                         /*}}}*/
 130
 131 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 132 // ---------------------------------------------------------------------
 133 /* */
 134 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 135 {
 136    PathPrefix = Block.Find("PathPrefix");
 137
 138    if (Block.FindB("External-Links",true) == false)
 139       DeLinkLimit = Setup.FindI("Default::DeLinkLimit", std::numeric_limits<unsigned int>::max());
 140    else
 141       DeLinkLimit = 0;
 142
 143    PkgCompress = Block.Find("Packages::Compress",
 144                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 145    CntCompress = Block.Find("Contents::Compress",
 146                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 147    SrcCompress = Block.Find("Sources::Compress",
 148                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 149
 150    SrcExt = Block.Find("Sources::Extensions",
 151                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 152    PkgExt = Block.Find("Packages::Extensions",
 153                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 154
 155    Permissions = Setup.FindI("Default::FileMode",0644);
 156
 157    if (FLFile.empty() == false)
 158       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 159
 160    if (Contents == " ")
 161       Contents= string();
 162 }
 163                                                                         /*}}}*/
 164 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 165 // ---------------------------------------------------------------------
 166 /* This generates the Package File described by this object. */
 167 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 168 {
 169    if (PkgFile.empty() == true)
 170       return true;
 171
 172    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 173    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 174    string CacheDir = Setup.FindDir("Dir::CacheDir");
 175
 176    struct timeval StartTime;
 177    gettimeofday(&StartTime,0);
 178
 179    PkgDone = true;
 180
 181    // Create a package writer object.
 182    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 183                       PkgCompress,Permissions);
 184    PackagesWriter Packages(&Comp.Input, TransWriter, flCombine(CacheDir,BinCacheDB),
 185                            flCombine(OverrideDir,BinOverride),
 186                            flCombine(OverrideDir,ExtraOverride),
 187                            Arch);
 188    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 189       return _error->Error(_("Package extension list is too long"));
 190    if (_error->PendingError() == true)
 191       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 192
 193    Packages.PathPrefix = PathPrefix;
 194    Packages.DirStrip = ArchiveDir;
 195    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 196
 197    Packages.LongDescription = LongDesc;
 198
 199    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 200    Packages.DeLinkLimit = DeLinkLimit;
 201
 202    if (_error->PendingError() == true)
 203       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 204
 205    c0out << ' ' << BaseDir << ":" << flush;
 206
 207    // Do recursive directory searching
 208    if (FLFile.empty() == true)
 209    {
 210       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 211          return false;
 212    }
 213    else
 214    {
 215       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 216          return false;
 217    }
 218
 219    Packages.Output = 0;      // Just in case
 220
 221    // Finish compressing
 222    unsigned long long Size;
 223    if (Comp.Finalize(Size) == false)
 224    {
 225       c0out << endl;
 226       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 227    }
 228
 229    if (Size != 0)
 230       c0out << " New "
 231              << SizeToStr(Size) << "B ";
 232    else
 233       c0out << ' ';
 234
 235    struct timeval NewTime;
 236    gettimeofday(&NewTime,0);
 237    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 238                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 239
 240    c0out << Packages.Stats.Packages << " files " <<
 241 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 242       SizeToStr(Packages.Stats.Bytes) << "B " <<
 243       TimeToStr((long)Delta) << endl;
 244
 245    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 246      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 247
 248    Stats.Add(Packages.Stats);
 249    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 250
 251    return !_error->PendingError();
 252 }
 253
 254                                                                         /*}}}*/
 255 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 256 // ---------------------------------------------------------------------
 257 /* This generates the Sources File described by this object. */
 258 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 259 {
 260    if (SrcFile.empty() == true)
 261       return true;
 262
 263    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 264    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 265    string CacheDir = Setup.FindDir("Dir::CacheDir");
 266
 267    struct timeval StartTime;
 268    gettimeofday(&StartTime,0);
 269
 270    SrcDone = true;
 271
 272    // Create a package writer object.
 273    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 274                       SrcCompress,Permissions);
 275    SourcesWriter Sources(&Comp.Input, flCombine(CacheDir, SrcCacheDB),
 276                          flCombine(OverrideDir,BinOverride),
 277                          flCombine(OverrideDir,SrcOverride),
 278                          flCombine(OverrideDir,SrcExtraOverride));
 279    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 280       return _error->Error(_("Source extension list is too long"));
 281    if (_error->PendingError() == true)
 282       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 283
 284    Sources.PathPrefix = PathPrefix;
 285    Sources.DirStrip = ArchiveDir;
 286    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 287
 288    Sources.DeLinkLimit = DeLinkLimit;
 289    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 290
 291    if (_error->PendingError() == true)
 292       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 293
 294    c0out << ' ' << BaseDir << ":" << flush;
 295
 296    // Do recursive directory searching
 297    if (FLFile.empty() == true)
 298    {
 299       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 300          return false;
 301    }
 302    else
 303    {
 304       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 305          return false;
 306    }
 307    Sources.Output = 0;      // Just in case
 308
 309    // Finish compressing
 310    unsigned long long Size;
 311    if (Comp.Finalize(Size) == false)
 312    {
 313       c0out << endl;
 314       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 315    }
 316
 317    if (Size != 0)
 318       c0out << " New "
 319              << SizeToStr(Size) << "B ";
 320    else
 321       c0out << ' ';
 322
 323    struct timeval NewTime;
 324    gettimeofday(&NewTime,0);
 325    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 326                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 327
 328    c0out << Sources.Stats.Packages << " pkgs in " <<
 329       TimeToStr((long)Delta) << endl;
 330
 331    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 332      c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
 333
 334    Stats.Add(Sources.Stats);
 335    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 336
 337    return !_error->PendingError();
 338 }
 339                                                                         /*}}}*/
 340 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 341 // ---------------------------------------------------------------------
 342 /* This generates the contents file partially described by this object.
 343    It searches the given iterator range for other package files that map
 344    into this contents file and includes their data as well when building. */
 345 bool PackageMap::GenContents(Configuration &Setup,
 346                              vector<PackageMap>::iterator Begin,
 347                              vector<PackageMap>::iterator End,
 348                              unsigned long &Left)
 349 {
 350    if (Contents.empty() == true)
 351       return true;
 352
 353    if (Left == 0)
 354       return true;
 355
 356    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 357    string CacheDir = Setup.FindDir("Dir::CacheDir");
 358    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 359
 360    struct timeval StartTime;
 361    gettimeofday(&StartTime,0);
 362
 363    // Create a package writer object.
 364    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 365                       CntCompress,Permissions);
 366    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 367    ContentsWriter Contents(&Comp.Input, "", Arch);
 368    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 369       return _error->Error(_("Package extension list is too long"));
 370    if (_error->PendingError() == true)
 371       return false;
 372
 373    if (_error->PendingError() == true)
 374       return false;
 375
 376    // Write the header out.
 377    if (ContentsHead.empty() == false)
 378    {
 379       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 380       if (_error->PendingError() == true)
 381          return false;
 382
 383       unsigned long long Size = Head.Size();
 384       unsigned char Buf[4096];
 385       while (Size != 0)
 386       {
 387          unsigned long long ToRead = Size;
 388          if (Size > sizeof(Buf))
 389             ToRead = sizeof(Buf);
 390
 391          if (Head.Read(Buf,ToRead) == false)
 392             return false;
 393
 394          if (Comp.Input.Write(Buf, ToRead) == false)
 395             return _error->Errno("fwrite",_("Error writing header to contents file"));
 396
 397          Size -= ToRead;
 398       }
 399    }
 400
 401    /* Go over all the package file records and parse all the package
 402       files associated with this contents file into one great big honking
 403       memory structure, then dump the sorted version */
 404    c0out << ' ' << this->Contents << ":" << flush;
 405    for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
 406    {
 407       if (I->Contents != this->Contents)
 408          continue;
 409
 410       Contents.Prefix = ArchiveDir;
 411       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 412       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 413                             I->PkgCompress);
 414
 415       I->ContentsDone = true;
 416    }
 417
 418    Contents.Finish();
 419
 420    // Finish compressing
 421    unsigned long long Size;
 422    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 423    {
 424       c0out << endl;
 425       return _error->Error(_("Error processing contents %s"),
 426                            this->Contents.c_str());
 427    }
 428
 429    if (Size != 0)
 430    {
 431       c0out << " New " << SizeToStr(Size) << "B ";
 432       if (Left > Size)
 433          Left -= Size;
 434       else
 435          Left = 0;
 436    }
 437    else
 438       c0out << ' ';
 439
 440    struct timeval NewTime;
 441    gettimeofday(&NewTime,0);
 442    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 443                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 444
 445    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 446      c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
 447
 448    c0out << Contents.Stats.Packages << " files " <<
 449       SizeToStr(Contents.Stats.Bytes) << "B " <<
 450       TimeToStr((long)Delta) << endl;
 451
 452    return true;
 453 }
 454                                                                         /*}}}*/
 455
 456 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 457 // ---------------------------------------------------------------------
 458 /* This populates the PkgList with all the possible permutations of the
 459    section/arch lists. */
 460 static void LoadTree(vector<PackageMap> &PkgList, std::vector<TranslationWriter*> &TransList, Configuration &Setup)
 461 {
 462    // Load the defaults
 463    string DDir = Setup.Find("TreeDefault::Directory",
 464                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 465    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 466                             "$(DIST)/$(SECTION)/source/");
 467    string DPkg = Setup.Find("TreeDefault::Packages",
 468                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 469    string DTrans = Setup.Find("TreeDefault::Translation",
 470                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 471    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 472                             "$(DIST)/$(SECTION)/");
 473    string DContents = Setup.Find("TreeDefault::Contents",
 474                             "$(DIST)/$(SECTION)/Contents-$(ARCH)");
 475    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 476    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 477                                "packages-$(ARCH).db");
 478    string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
 479                                "sources-$(SECTION).db");
 480    string DSources = Setup.Find("TreeDefault::Sources",
 481                                 "$(DIST)/$(SECTION)/source/Sources");
 482    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 483    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 484
 485    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 486
 487    bool const LongDescription = Setup.FindB("Default::LongDescription",
 488                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 489    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 490
 491    // Process 'tree' type sections
 492    const Configuration::Item *Top = Setup.Tree("tree");
 493    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 494    {
 495       Configuration Block(Top);
 496       string Dist = Top->Tag;
 497
 498       // Parse the sections
 499       string Tmp = Block.Find("Sections");
 500       const char *Sections = Tmp.c_str();
 501       string Section;
 502       while (ParseQuoteWord(Sections,Section) == true)
 503       {
 504          string Arch;
 505          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 506                                          {"$(SECTION)",&Section},
 507                                          {"$(ARCH)",&Arch},
 508                                          {NULL, NULL}};
 509          mode_t const Perms = Block.FindI("FileMode", Permissions);
 510          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 511          TranslationWriter *TransWriter = NULL;
 512
 513          string const Tmp2 = Block.Find("Architectures");
 514          const char *Archs = Tmp2.c_str();
 515          while (ParseQuoteWord(Archs,Arch) == true)
 516          {
 517             PackageMap Itm;
 518             Itm.Permissions = Perms;
 519             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 520             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 521
 522             if (stringcasecmp(Arch,"source") == 0)
 523             {
 524                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 525                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 526                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 527                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 528                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 529                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 530                Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);
 531             }
 532             else
 533             {
 534                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 535                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 536                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 537                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 538                Itm.Arch = Arch;
 539                Itm.LongDesc = LongDesc;
 540                if (TransWriter == NULL && DTrans.empty() == false && LongDesc == false && DTrans != "/dev/null")
 541                {
 542                   string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 543                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 544                   string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 545                   TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 546                   TransList.push_back(TransWriter);
 547                }
 548                Itm.TransWriter = TransWriter;
 549                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 550                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 551                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 552                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 553             }
 554
 555             Itm.GetGeneral(Setup,Block);
 556             PkgList.push_back(Itm);
 557          }
 558       }
 559
 560       Top = Top->Next;
 561    }
 562 }
 563                                                                         /*}}}*/
 564 static void UnloadTree(std::vector<TranslationWriter*> const &Trans)            /*{{{*/
 565 {
 566    for (std::vector<TranslationWriter*>::const_reverse_iterator T = Trans.rbegin(); T != Trans.rend(); ++T)
 567       delete *T;
 568 }
 569                                                                         /*}}}*/
 570 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 571 // ---------------------------------------------------------------------
 572 /* */
 573 static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 574 {
 575    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 576
 577    // Process 'bindirectory' type sections
 578    const Configuration::Item *Top = Setup.Tree("bindirectory");
 579    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 580    {
 581       Configuration Block(Top);
 582
 583       PackageMap Itm;
 584       Itm.PkgFile = Block.Find("Packages");
 585       Itm.SrcFile = Block.Find("Sources");
 586       Itm.BinCacheDB = Block.Find("BinCacheDB");
 587       Itm.SrcCacheDB = Block.Find("SrcCacheDB");
 588       Itm.BinOverride = Block.Find("BinOverride");
 589       Itm.ExtraOverride = Block.Find("ExtraOverride");
 590       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 591       Itm.SrcOverride = Block.Find("SrcOverride");
 592       Itm.BaseDir = Top->Tag;
 593       Itm.FLFile = Block.Find("FileList");
 594       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 595       Itm.Contents = Block.Find("Contents");
 596       Itm.ContentsHead = Block.Find("Contents::Header");
 597       Itm.Permissions = Block.FindI("FileMode", Permissions);
 598
 599       Itm.GetGeneral(Setup,Block);
 600       PkgList.push_back(Itm);
 601
 602       Top = Top->Next;
 603    }
 604 }
 605                                                                         /*}}}*/
 606
 607 // ShowHelp - Show the help text                                        /*{{{*/
 608 // ---------------------------------------------------------------------
 609 /* */
 610 static bool ShowHelp(CommandLine &, CommandLine::DispatchWithHelp const *)
 611 {
 612    ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 613    if (_config->FindB("version") == true)
 614       return true;
 615
 616    cout <<
 617     _("Usage: apt-ftparchive [options] command\n"
 618       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 619       "          sources srcpath [overridefile [pathprefix]]\n"
 620       "          contents path\n"
 621       "          release path\n"
 622       "          generate config [groups]\n"
 623       "          clean config\n"
 624       "\n"
 625       "apt-ftparchive generates index files for Debian archives. It supports\n"
 626       "many styles of generation from fully automated to functional replacements\n"
 627       "for dpkg-scanpackages and dpkg-scansources\n"
 628       "\n"
 629       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 630       "Package file contains the contents of all the control fields from\n"
 631       "each package as well as the MD5 hash and filesize. An override file\n"
 632       "is supported to force the value of Priority and Section.\n"
 633       "\n"
 634       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 635       "The --source-override option can be used to specify a src override file\n"
 636       "\n"
 637       "The 'packages' and 'sources' command should be run in the root of the\n"
 638       "tree. BinaryPath should point to the base of the recursive search and \n"
 639       "override file should contain the override flags. Pathprefix is\n"
 640       "appended to the filename fields if present. Example usage from the \n"
 641       "Debian archive:\n"
 642       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 643       "               dists/potato/main/binary-i386/Packages\n"
 644       "\n"
 645       "Options:\n"
 646       "  -h    This help text\n"
 647       "  --md5 Control MD5 generation\n"
 648       "  -s=?  Source override file\n"
 649       "  -q    Quiet\n"
 650       "  -d=?  Select the optional caching database\n"
 651       "  --no-delink Enable delinking debug mode\n"
 652       "  --contents  Control contents file generation\n"
 653       "  -c=?  Read this configuration file\n"
 654       "  -o=?  Set an arbitrary configuration option") << endl;
 655
 656    return true;
 657 }
 658                                                                         /*}}}*/
 659 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 660 // ---------------------------------------------------------------------
 661 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 662 static bool SimpleGenPackages(CommandLine &CmdL)
 663 {
 664    if (CmdL.FileSize() < 2)
 665       return ShowHelp(CmdL, nullptr);
 666
 667    string Override;
 668    if (CmdL.FileSize() >= 3)
 669       Override = CmdL.FileList[2];
 670
 671    // Create a package writer object.
 672    PackagesWriter Packages(NULL, NULL, _config->Find("APT::FTPArchive::DB"),
 673                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 674    if (_error->PendingError() == true)
 675       return false;
 676
 677    if (CmdL.FileSize() >= 4)
 678       Packages.PathPrefix = CmdL.FileList[3];
 679
 680    // Do recursive directory searching
 681    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 682       return false;
 683
 684    // Give some stats if asked for
 685    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 686      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 687
 688    return true;
 689 }
 690                                                                         /*}}}*/
 691 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 692 // ---------------------------------------------------------------------
 693 /* */
 694 static bool SimpleGenContents(CommandLine &CmdL)
 695 {
 696    if (CmdL.FileSize() < 2)
 697       return ShowHelp(CmdL, nullptr);
 698
 699    // Create a package writer object.
 700    ContentsWriter Contents(NULL, _config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 701    if (_error->PendingError() == true)
 702       return false;
 703
 704    // Do recursive directory searching
 705    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 706       return false;
 707
 708    Contents.Finish();
 709
 710    return true;
 711 }
 712                                                                         /*}}}*/
 713 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 714 // ---------------------------------------------------------------------
 715 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 716 static bool SimpleGenSources(CommandLine &CmdL)
 717 {
 718    if (CmdL.FileSize() < 2)
 719       return ShowHelp(CmdL, nullptr);
 720
 721    string Override;
 722    if (CmdL.FileSize() >= 3)
 723       Override = CmdL.FileList[2];
 724
 725    string SOverride;
 726    if (Override.empty() == false)
 727       SOverride = Override + ".src";
 728
 729    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 730                              SOverride.c_str());
 731
 732    // Create a package writer object.
 733    SourcesWriter Sources(NULL, _config->Find("APT::FTPArchive::DB"),Override,SOverride);
 734    if (_error->PendingError() == true)
 735       return false;
 736
 737    if (CmdL.FileSize() >= 4)
 738       Sources.PathPrefix = CmdL.FileList[3];
 739
 740    // Do recursive directory searching
 741    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 742       return false;
 743
 744    // Give some stats if asked for
 745    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 746      c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl;
 747
 748    return true;
 749 }
 750                                                                         /*}}}*/
 751 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 752 // ---------------------------------------------------------------------
 753 static bool SimpleGenRelease(CommandLine &CmdL)
 754 {
 755    if (CmdL.FileSize() < 2)
 756       return ShowHelp(CmdL, nullptr);
 757
 758    string Dir = CmdL.FileList[1];
 759
 760    ReleaseWriter Release(NULL, "");
 761    Release.DirStrip = Dir;
 762
 763    if (_error->PendingError() == true)
 764       return false;
 765
 766    if (Release.RecursiveScan(Dir) == false)
 767       return false;
 768
 769    Release.Finish();
 770
 771    return true;
 772 }
 773
 774                                                                         /*}}}*/
 775 // DoGeneratePackagesAndSources - Helper for Generate                   /*{{{*/
 776 // ---------------------------------------------------------------------
 777 static bool DoGeneratePackagesAndSources(Configuration &Setup,
 778                                          vector<PackageMap> &PkgList,
 779                                          struct CacheDB::Stats &SrcStats,
 780                                          struct CacheDB::Stats &Stats,
 781                                          CommandLine &CmdL)
 782 {
 783    if (CmdL.FileSize() <= 2)
 784    {
 785       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 786          if (I->GenPackages(Setup,Stats) == false)
 787             _error->DumpErrors();
 788       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 789          if (I->GenSources(Setup,SrcStats) == false)
 790             _error->DumpErrors();
 791    }
 792    else
 793    {
 794       // Make a choice list out of the package list..
 795       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 796       RxChoiceList *End = List;
 797       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 798       {
 799          End->UserData = &(*I);
 800          End->Str = I->BaseDir.c_str();
 801          End++;
 802
 803          End->UserData = &(*I);
 804          End->Str = I->Tag.c_str();
 805          End++;
 806       }
 807       End->Str = 0;
 808
 809       // Regex it
 810       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 811       {
 812          delete [] List;
 813          return _error->Error(_("No selections matched"));
 814       }
 815       _error->DumpErrors();
 816
 817       // Do the generation for Packages
 818       for (End = List; End->Str != 0; End++)
 819       {
 820          if (End->Hit == false)
 821             continue;
 822
 823          PackageMap *I = (PackageMap *)End->UserData;
 824          if (I->PkgDone == true)
 825             continue;
 826          if (I->GenPackages(Setup,Stats) == false)
 827             _error->DumpErrors();
 828       }
 829
 830       // Do the generation for Sources
 831       for (End = List; End->Str != 0; End++)
 832       {
 833          if (End->Hit == false)
 834             continue;
 835
 836          PackageMap *I = (PackageMap *)End->UserData;
 837          if (I->SrcDone == true)
 838             continue;
 839          if (I->GenSources(Setup,SrcStats) == false)
 840             _error->DumpErrors();
 841       }
 842
 843       delete [] List;
 844    }
 845    return true;
 846 }
 847
 848                                                                         /*}}}*/
 849 // DoGenerateContents - Helper for Generate to generate the Contents    /*{{{*/
 850 // ---------------------------------------------------------------------
 851 static bool DoGenerateContents(Configuration &Setup,
 852                                vector<PackageMap> &PkgList,
 853                                CommandLine &CmdL)
 854 {
 855    c1out << "Packages done, Starting contents." << endl;
 856
 857    // Sort the contents file list by date
 858    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 859    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 860    {
 861       struct stat A;
 862       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 863                                  I->CntCompress,A) == false)
 864          time(&I->ContentsMTime);
 865       else
 866          I->ContentsMTime = A.st_mtime;
 867    }
 868    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 869
 870    /* Now for Contents.. The process here is to do a make-like dependency
 871       check. Each contents file is verified to be newer than the package files
 872       that describe the debs it indexes. Since the package files contain
 873       hashes of the .debs this means they have not changed either so the
 874       contents must be up to date. */
 875    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",
 876                                                  std::numeric_limits<unsigned int>::max())*1024;
 877    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 878    {
 879       // This record is not relevant
 880       if (I->ContentsDone == true ||
 881           I->Contents.empty() == true)
 882          continue;
 883
 884       // Do not do everything if the user specified sections.
 885       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 886          continue;
 887
 888       struct stat A,B;
 889       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 890       {
 891          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 892          {
 893             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 894             continue;
 895          }
 896
 897          if (A.st_mtime > B.st_mtime)
 898             continue;
 899       }
 900
 901       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 902                          MaxContentsChange) == false)
 903          _error->DumpErrors();
 904
 905       // Hit the limit?
 906       if (MaxContentsChange == 0)
 907       {
 908          c1out << "Hit contents update byte limit" << endl;
 909          break;
 910       }
 911    }
 912
 913    return true;
 914 }
 915
 916                                                                         /*}}}*/
 917 // Generate - Full generate, using a config file                        /*{{{*/
 918 // ---------------------------------------------------------------------
 919 /* */
 920 static bool Generate(CommandLine &CmdL)
 921 {
 922    struct CacheDB::Stats SrcStats;
 923    if (CmdL.FileSize() < 2)
 924       return ShowHelp(CmdL, nullptr);
 925
 926    struct timeval StartTime;
 927    gettimeofday(&StartTime,0);
 928    struct CacheDB::Stats Stats;
 929
 930    // Read the configuration file.
 931    Configuration Setup;
 932    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 933       return false;
 934
 935    vector<PackageMap> PkgList;
 936    std::vector<TranslationWriter*> TransList;
 937    LoadTree(PkgList, TransList, Setup);
 938    LoadBinDir(PkgList,Setup);
 939
 940    // Sort by cache DB to improve IO locality.
 941    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 942    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 943
 944    // Generate packages
 945    if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false)
 946    {
 947       if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false)
 948       {
 949          UnloadTree(TransList);
 950          return false;
 951       }
 952    } else {
 953       c1out << "Skipping Packages/Sources generation" << endl;
 954    }
 955
 956    // do Contents if needed
 957    if (_config->FindB("APT::FTPArchive::Contents", true) == true)
 958       if (DoGenerateContents(Setup, PkgList, CmdL) == false)
 959       {
 960          UnloadTree(TransList);
 961          return false;
 962       }
 963
 964    struct timeval NewTime;
 965    gettimeofday(&NewTime,0);
 966    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 967                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 968    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 969          << " archives. Took " << TimeToStr((long)Delta) << endl;
 970
 971    UnloadTree(TransList);
 972    return true;
 973 }
 974
 975                                                                         /*}}}*/
 976 // Clean - Clean out the databases                                      /*{{{*/
 977 // ---------------------------------------------------------------------
 978 /* */
 979 static bool Clean(CommandLine &CmdL)
 980 {
 981    if (CmdL.FileSize() != 2)
 982       return ShowHelp(CmdL, nullptr);
 983
 984    // Read the configuration file.
 985    Configuration Setup;
 986    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 987       return false;
 988    // we don't need translation creation here
 989    Setup.Set("TreeDefault::Translation", "/dev/null");
 990
 991    vector<PackageMap> PkgList;
 992    std::vector<TranslationWriter*> TransList;
 993    LoadTree(PkgList, TransList, Setup);
 994    LoadBinDir(PkgList,Setup);
 995
 996    // Sort by cache DB to improve IO locality.
 997    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 998    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 999
1000    string CacheDir = Setup.FindDir("Dir::CacheDir");
1001
1002    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
1003    {
1004       if(I->BinCacheDB != "")
1005          c0out << I->BinCacheDB << endl;
1006       if(I->SrcCacheDB != "")
1007          c0out << I->SrcCacheDB << endl;
1008       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
1009       CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));
1010       if (DB.Clean() == false)
1011          _error->DumpErrors();
1012       if (DB_SRC.Clean() == false)
1013          _error->DumpErrors();
1014
1015       string CacheDB = I->BinCacheDB;
1016       string SrcCacheDB = I->SrcCacheDB;
1017       while(I != PkgList.end() &&
1018             I->BinCacheDB == CacheDB &&
1019             I->SrcCacheDB == SrcCacheDB)
1020          ++I;
1021    }
1022
1023
1024    return true;
1025 }
1026                                                                         /*}}}*/
1027
1028 int main(int argc, const char *argv[])
1029 {
1030    InitLocale();
1031
1032    CommandLine::DispatchWithHelp Cmds[] = {
1033       {"packages",&SimpleGenPackages, nullptr},
1034       {"contents",&SimpleGenContents, nullptr},
1035       {"sources",&SimpleGenSources, nullptr},
1036       {"release",&SimpleGenRelease, nullptr},
1037       {"generate",&Generate, nullptr},
1038       {"clean",&Clean, nullptr},
1039       {nullptr, nullptr, nullptr}
1040    };
1041
1042    // Parse the command line and initialize the package library
1043    CommandLine CmdL;
1044    ParseCommandLine(CmdL, Cmds, "apt-ftparchive", &_config, NULL, argc, argv, ShowHelp);
1045
1046    _config->CndSet("quiet",0);
1047    Quiet = _config->FindI("quiet",0);
1048    InitOutput(clog.rdbuf());
1049
1050    return DispatchCommandLine(CmdL, Cmds);
1051 }