ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <apt-pkg/fileutl.h>
  21
  22 #include <apt-private/private-cmndline.h>
  23 #include <apt-private/private-output.h>
  24
  25 #include <algorithm>
  26 #include <climits>
  27 #include <sys/time.h>
  28 #include <locale.h>
  29 #include <stdio.h>
  30 #include <sys/stat.h>
  31 #include <time.h>
  32 #include <functional>
  33 #include <iostream>
  34 #include <string>
  35 #include <vector>
  36
  37 #include "cachedb.h"
  38 #include "override.h"
  39 #include "apt-ftparchive.h"
  40 #include "multicompress.h"
  41 #include "writer.h"
  42
  43 #include <apti18n.h>
  44                                                                         /*}}}*/
  45
  46 using namespace std;
  47 unsigned Quiet = 0;
  48
  49 // struct PackageMap - List of all package files in the config file     /*{{{*/
  50 // ---------------------------------------------------------------------
  51 /* */
  52 struct PackageMap
  53 {
  54    // General Stuff
  55    string BaseDir;
  56    string InternalPrefix;
  57    string FLFile;
  58    string PkgExt;
  59    string SrcExt;
  60
  61    // Stuff for the Package File
  62    string PkgFile;
  63    string BinCacheDB;
  64    string SrcCacheDB;
  65    string BinOverride;
  66    string ExtraOverride;
  67
  68    // We generate for this given arch
  69    string Arch;
  70
  71    // Stuff for the Source File
  72    string SrcFile;
  73    string SrcOverride;
  74    string SrcExtraOverride;
  75
  76    // Translation master file
  77    bool LongDesc;
  78    TranslationWriter *TransWriter;
  79
  80    // Contents
  81    string Contents;
  82    string ContentsHead;
  83
  84    // Random things
  85    string Tag;
  86    string PkgCompress;
  87    string CntCompress;
  88    string SrcCompress;
  89    string PathPrefix;
  90    unsigned int DeLinkLimit;
  91    mode_t Permissions;
  92
  93    bool ContentsDone;
  94    bool PkgDone;
  95    bool SrcDone;
  96    time_t ContentsMTime;
  97
  98    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  99    {
 100       inline bool operator() (const PackageMap &x,const PackageMap &y)
 101       {return x.ContentsMTime < y.ContentsMTime;};
 102    };
 103
 104    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
 105    {
 106       inline bool operator() (const PackageMap &x,const PackageMap &y)
 107       {return x.BinCacheDB < y.BinCacheDB;};
 108    };
 109
 110    struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
 111    {
 112       inline bool operator() (const PackageMap &x,const PackageMap &y)
 113       {return x.SrcCacheDB < y.SrcCacheDB;};
 114    };
 115
 116    void GetGeneral(Configuration &Setup,Configuration &Block);
 117    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 118    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 119    bool GenContents(Configuration &Setup,
 120                     vector<PackageMap>::iterator Begin,
 121                     vector<PackageMap>::iterator End,
 122                     unsigned long &Left);
 123
 124    PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 125                   ContentsDone(false), PkgDone(false), SrcDone(false),
 126                   ContentsMTime(0) {};
 127 };
 128                                                                         /*}}}*/
 129
 130 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 131 // ---------------------------------------------------------------------
 132 /* */
 133 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 134 {
 135    PathPrefix = Block.Find("PathPrefix");
 136
 137    if (Block.FindB("External-Links",true) == false)
 138       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 139    else
 140       DeLinkLimit = 0;
 141
 142    PkgCompress = Block.Find("Packages::Compress",
 143                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 144    CntCompress = Block.Find("Contents::Compress",
 145                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 146    SrcCompress = Block.Find("Sources::Compress",
 147                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 148
 149    SrcExt = Block.Find("Sources::Extensions",
 150                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 151    PkgExt = Block.Find("Packages::Extensions",
 152                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 153
 154    Permissions = Setup.FindI("Default::FileMode",0644);
 155
 156    if (FLFile.empty() == false)
 157       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 158
 159    if (Contents == " ")
 160       Contents= string();
 161 }
 162                                                                         /*}}}*/
 163 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 164 // ---------------------------------------------------------------------
 165 /* This generates the Package File described by this object. */
 166 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 167 {
 168    if (PkgFile.empty() == true)
 169       return true;
 170
 171    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 172    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 173    string CacheDir = Setup.FindDir("Dir::CacheDir");
 174
 175    struct timeval StartTime;
 176    gettimeofday(&StartTime,0);
 177
 178    PkgDone = true;
 179
 180    // Create a package writer object.
 181    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 182                       PkgCompress,Permissions);
 183    PackagesWriter Packages(&Comp.Input, flCombine(CacheDir,BinCacheDB),
 184                            flCombine(OverrideDir,BinOverride),
 185                            flCombine(OverrideDir,ExtraOverride),
 186                            Arch);
 187    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 188       return _error->Error(_("Package extension list is too long"));
 189    if (_error->PendingError() == true)
 190       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 191
 192    Packages.PathPrefix = PathPrefix;
 193    Packages.DirStrip = ArchiveDir;
 194    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 195
 196    Packages.TransWriter = TransWriter;
 197    Packages.LongDescription = LongDesc;
 198
 199    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 200    Packages.DeLinkLimit = DeLinkLimit;
 201
 202    if (_error->PendingError() == true)
 203       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 204
 205    c0out << ' ' << BaseDir << ":" << flush;
 206
 207    // Do recursive directory searching
 208    if (FLFile.empty() == true)
 209    {
 210       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 211          return false;
 212    }
 213    else
 214    {
 215       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 216          return false;
 217    }
 218
 219    Packages.Output = 0;      // Just in case
 220
 221    // Finish compressing
 222    unsigned long long Size;
 223    if (Comp.Finalize(Size) == false)
 224    {
 225       c0out << endl;
 226       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 227    }
 228
 229    if (Size != 0)
 230       c0out << " New "
 231              << SizeToStr(Size) << "B ";
 232    else
 233       c0out << ' ';
 234
 235    struct timeval NewTime;
 236    gettimeofday(&NewTime,0);
 237    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 238                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 239
 240    c0out << Packages.Stats.Packages << " files " <<
 241 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 242       SizeToStr(Packages.Stats.Bytes) << "B " <<
 243       TimeToStr((long)Delta) << endl;
 244
 245    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 246      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 247
 248    Stats.Add(Packages.Stats);
 249    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 250
 251    return !_error->PendingError();
 252 }
 253
 254                                                                         /*}}}*/
 255 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 256 // ---------------------------------------------------------------------
 257 /* This generates the Sources File described by this object. */
 258 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 259 {
 260    if (SrcFile.empty() == true)
 261       return true;
 262
 263    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 264    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 265    string CacheDir = Setup.FindDir("Dir::CacheDir");
 266
 267    struct timeval StartTime;
 268    gettimeofday(&StartTime,0);
 269
 270    SrcDone = true;
 271
 272    // Create a package writer object.
 273    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 274                       SrcCompress,Permissions);
 275    SourcesWriter Sources(&Comp.Input, flCombine(CacheDir, SrcCacheDB),
 276                          flCombine(OverrideDir,BinOverride),
 277                          flCombine(OverrideDir,SrcOverride),
 278                          flCombine(OverrideDir,SrcExtraOverride));
 279    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 280       return _error->Error(_("Source extension list is too long"));
 281    if (_error->PendingError() == true)
 282       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 283
 284    Sources.PathPrefix = PathPrefix;
 285    Sources.DirStrip = ArchiveDir;
 286    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 287
 288    Sources.DeLinkLimit = DeLinkLimit;
 289    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 290
 291    if (_error->PendingError() == true)
 292       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 293
 294    c0out << ' ' << BaseDir << ":" << flush;
 295
 296    // Do recursive directory searching
 297    if (FLFile.empty() == true)
 298    {
 299       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 300          return false;
 301    }
 302    else
 303    {
 304       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 305          return false;
 306    }
 307    Sources.Output = 0;      // Just in case
 308
 309    // Finish compressing
 310    unsigned long long Size;
 311    if (Comp.Finalize(Size) == false)
 312    {
 313       c0out << endl;
 314       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 315    }
 316
 317    if (Size != 0)
 318       c0out << " New "
 319              << SizeToStr(Size) << "B ";
 320    else
 321       c0out << ' ';
 322
 323    struct timeval NewTime;
 324    gettimeofday(&NewTime,0);
 325    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 326                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 327
 328    c0out << Sources.Stats.Packages << " pkgs in " <<
 329       TimeToStr((long)Delta) << endl;
 330
 331    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 332      c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
 333
 334    Stats.Add(Sources.Stats);
 335    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 336
 337    return !_error->PendingError();
 338 }
 339                                                                         /*}}}*/
 340 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 341 // ---------------------------------------------------------------------
 342 /* This generates the contents file partially described by this object.
 343    It searches the given iterator range for other package files that map
 344    into this contents file and includes their data as well when building. */
 345 bool PackageMap::GenContents(Configuration &Setup,
 346                              vector<PackageMap>::iterator Begin,
 347                              vector<PackageMap>::iterator End,
 348                              unsigned long &Left)
 349 {
 350    if (Contents.empty() == true)
 351       return true;
 352
 353    if (Left == 0)
 354       return true;
 355
 356    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 357    string CacheDir = Setup.FindDir("Dir::CacheDir");
 358    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 359
 360    struct timeval StartTime;
 361    gettimeofday(&StartTime,0);
 362
 363    // Create a package writer object.
 364    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 365                       CntCompress,Permissions);
 366    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 367    ContentsWriter Contents(&Comp.Input, "", Arch);
 368    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 369       return _error->Error(_("Package extension list is too long"));
 370    if (_error->PendingError() == true)
 371       return false;
 372
 373    if (_error->PendingError() == true)
 374       return false;
 375
 376    // Write the header out.
 377    if (ContentsHead.empty() == false)
 378    {
 379       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 380       if (_error->PendingError() == true)
 381          return false;
 382
 383       unsigned long long Size = Head.Size();
 384       unsigned char Buf[4096];
 385       while (Size != 0)
 386       {
 387          unsigned long long ToRead = Size;
 388          if (Size > sizeof(Buf))
 389             ToRead = sizeof(Buf);
 390
 391          if (Head.Read(Buf,ToRead) == false)
 392             return false;
 393
 394          if (Comp.Input.Write(Buf, ToRead) == false)
 395             return _error->Errno("fwrite",_("Error writing header to contents file"));
 396
 397          Size -= ToRead;
 398       }
 399    }
 400
 401    /* Go over all the package file records and parse all the package
 402       files associated with this contents file into one great big honking
 403       memory structure, then dump the sorted version */
 404    c0out << ' ' << this->Contents << ":" << flush;
 405    for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
 406    {
 407       if (I->Contents != this->Contents)
 408          continue;
 409
 410       Contents.Prefix = ArchiveDir;
 411       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 412       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 413                             I->PkgCompress);
 414
 415       I->ContentsDone = true;
 416    }
 417
 418    Contents.Finish();
 419
 420    // Finish compressing
 421    unsigned long long Size;
 422    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 423    {
 424       c0out << endl;
 425       return _error->Error(_("Error processing contents %s"),
 426                            this->Contents.c_str());
 427    }
 428
 429    if (Size != 0)
 430    {
 431       c0out << " New " << SizeToStr(Size) << "B ";
 432       if (Left > Size)
 433          Left -= Size;
 434       else
 435          Left = 0;
 436    }
 437    else
 438       c0out << ' ';
 439
 440    struct timeval NewTime;
 441    gettimeofday(&NewTime,0);
 442    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 443                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 444
 445    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 446      c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
 447
 448    c0out << Contents.Stats.Packages << " files " <<
 449       SizeToStr(Contents.Stats.Bytes) << "B " <<
 450       TimeToStr((long)Delta) << endl;
 451
 452    return true;
 453 }
 454                                                                         /*}}}*/
 455
 456 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 457 // ---------------------------------------------------------------------
 458 /* This populates the PkgList with all the possible permutations of the
 459    section/arch lists. */
 460 static void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 461 {
 462    // Load the defaults
 463    string DDir = Setup.Find("TreeDefault::Directory",
 464                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 465    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 466                             "$(DIST)/$(SECTION)/source/");
 467    string DPkg = Setup.Find("TreeDefault::Packages",
 468                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 469    string DTrans = Setup.Find("TreeDefault::Translation",
 470                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 471    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 472                             "$(DIST)/$(SECTION)/");
 473    string DContents = Setup.Find("TreeDefault::Contents",
 474                             "$(DIST)/$(SECTION)/Contents-$(ARCH)");
 475    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 476    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 477                                "packages-$(ARCH).db");
 478    string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
 479                                "sources-$(SECTION).db");
 480    string DSources = Setup.Find("TreeDefault::Sources",
 481                                 "$(DIST)/$(SECTION)/source/Sources");
 482    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 483    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 484
 485    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 486
 487    bool const LongDescription = Setup.FindB("Default::LongDescription",
 488                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 489    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 490
 491    // Process 'tree' type sections
 492    const Configuration::Item *Top = Setup.Tree("tree");
 493    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 494    {
 495       Configuration Block(Top);
 496       string Dist = Top->Tag;
 497
 498       // Parse the sections
 499       string Tmp = Block.Find("Sections");
 500       const char *Sections = Tmp.c_str();
 501       string Section;
 502       while (ParseQuoteWord(Sections,Section) == true)
 503       {
 504          string Arch;
 505          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 506                                          {"$(SECTION)",&Section},
 507                                          {"$(ARCH)",&Arch},
 508                                          {NULL, NULL}};
 509          mode_t const Perms = Block.FindI("FileMode", Permissions);
 510          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 511          TranslationWriter *TransWriter;
 512          if (DTrans.empty() == false && LongDesc == false)
 513          {
 514             string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 515                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 516             string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 517             TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 518          }
 519          else
 520             TransWriter = NULL;
 521
 522          string const Tmp2 = Block.Find("Architectures");
 523          const char *Archs = Tmp2.c_str();
 524          while (ParseQuoteWord(Archs,Arch) == true)
 525          {
 526             PackageMap Itm;
 527             Itm.Permissions = Perms;
 528             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 529             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 530
 531             if (stringcasecmp(Arch,"source") == 0)
 532             {
 533                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 534                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 535                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 536                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 537                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 538                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 539                Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);
 540             }
 541             else
 542             {
 543                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 544                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 545                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 546                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 547                Itm.Arch = Arch;
 548                Itm.LongDesc = LongDesc;
 549                if (TransWriter != NULL)
 550                {
 551                   TransWriter->IncreaseRefCounter();
 552                   Itm.TransWriter = TransWriter;
 553                }
 554                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 555                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 556                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 557                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 558             }
 559
 560             Itm.GetGeneral(Setup,Block);
 561             PkgList.push_back(Itm);
 562          }
 563          // we didn't use this TransWriter, so we can release it
 564          if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
 565             delete TransWriter;
 566       }
 567
 568       Top = Top->Next;
 569    }
 570 }
 571                                                                         /*}}}*/
 572 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 573 // ---------------------------------------------------------------------
 574 /* */
 575 static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 576 {
 577    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 578
 579    // Process 'bindirectory' type sections
 580    const Configuration::Item *Top = Setup.Tree("bindirectory");
 581    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 582    {
 583       Configuration Block(Top);
 584
 585       PackageMap Itm;
 586       Itm.PkgFile = Block.Find("Packages");
 587       Itm.SrcFile = Block.Find("Sources");
 588       Itm.BinCacheDB = Block.Find("BinCacheDB");
 589       Itm.SrcCacheDB = Block.Find("SrcCacheDB");
 590       Itm.BinOverride = Block.Find("BinOverride");
 591       Itm.ExtraOverride = Block.Find("ExtraOverride");
 592       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 593       Itm.SrcOverride = Block.Find("SrcOverride");
 594       Itm.BaseDir = Top->Tag;
 595       Itm.FLFile = Block.Find("FileList");
 596       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 597       Itm.Contents = Block.Find("Contents");
 598       Itm.ContentsHead = Block.Find("Contents::Header");
 599       Itm.Permissions = Block.FindI("FileMode", Permissions);
 600
 601       Itm.GetGeneral(Setup,Block);
 602       PkgList.push_back(Itm);
 603
 604       Top = Top->Next;
 605    }
 606 }
 607                                                                         /*}}}*/
 608
 609 // ShowHelp - Show the help text                                        /*{{{*/
 610 // ---------------------------------------------------------------------
 611 /* */
 612 static bool ShowHelp(CommandLine &)
 613 {
 614    ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 615    if (_config->FindB("version") == true)
 616       return true;
 617
 618    cout <<
 619     _("Usage: apt-ftparchive [options] command\n"
 620       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 621       "          sources srcpath [overridefile [pathprefix]]\n"
 622       "          contents path\n"
 623       "          release path\n"
 624       "          generate config [groups]\n"
 625       "          clean config\n"
 626       "\n"
 627       "apt-ftparchive generates index files for Debian archives. It supports\n"
 628       "many styles of generation from fully automated to functional replacements\n"
 629       "for dpkg-scanpackages and dpkg-scansources\n"
 630       "\n"
 631       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 632       "Package file contains the contents of all the control fields from\n"
 633       "each package as well as the MD5 hash and filesize. An override file\n"
 634       "is supported to force the value of Priority and Section.\n"
 635       "\n"
 636       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 637       "The --source-override option can be used to specify a src override file\n"
 638       "\n"
 639       "The 'packages' and 'sources' command should be run in the root of the\n"
 640       "tree. BinaryPath should point to the base of the recursive search and \n"
 641       "override file should contain the override flags. Pathprefix is\n"
 642       "appended to the filename fields if present. Example usage from the \n"
 643       "Debian archive:\n"
 644       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 645       "               dists/potato/main/binary-i386/Packages\n"
 646       "\n"
 647       "Options:\n"
 648       "  -h    This help text\n"
 649       "  --md5 Control MD5 generation\n"
 650       "  -s=?  Source override file\n"
 651       "  -q    Quiet\n"
 652       "  -d=?  Select the optional caching database\n"
 653       "  --no-delink Enable delinking debug mode\n"
 654       "  --contents  Control contents file generation\n"
 655       "  -c=?  Read this configuration file\n"
 656       "  -o=?  Set an arbitrary configuration option") << endl;
 657
 658    return true;
 659 }
 660                                                                         /*}}}*/
 661 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 662 // ---------------------------------------------------------------------
 663 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 664 static bool SimpleGenPackages(CommandLine &CmdL)
 665 {
 666    if (CmdL.FileSize() < 2)
 667       return ShowHelp(CmdL);
 668
 669    string Override;
 670    if (CmdL.FileSize() >= 3)
 671       Override = CmdL.FileList[2];
 672
 673    // Create a package writer object.
 674    PackagesWriter Packages(NULL, _config->Find("APT::FTPArchive::DB"),
 675                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 676    if (_error->PendingError() == true)
 677       return false;
 678
 679    if (CmdL.FileSize() >= 4)
 680       Packages.PathPrefix = CmdL.FileList[3];
 681
 682    // Do recursive directory searching
 683    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 684       return false;
 685
 686    // Give some stats if asked for
 687    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 688      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 689
 690    return true;
 691 }
 692                                                                         /*}}}*/
 693 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 694 // ---------------------------------------------------------------------
 695 /* */
 696 static bool SimpleGenContents(CommandLine &CmdL)
 697 {
 698    if (CmdL.FileSize() < 2)
 699       return ShowHelp(CmdL);
 700
 701    // Create a package writer object.
 702    ContentsWriter Contents(NULL, _config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 703    if (_error->PendingError() == true)
 704       return false;
 705
 706    // Do recursive directory searching
 707    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 708       return false;
 709
 710    Contents.Finish();
 711
 712    return true;
 713 }
 714                                                                         /*}}}*/
 715 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 716 // ---------------------------------------------------------------------
 717 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 718 static bool SimpleGenSources(CommandLine &CmdL)
 719 {
 720    if (CmdL.FileSize() < 2)
 721       return ShowHelp(CmdL);
 722
 723    string Override;
 724    if (CmdL.FileSize() >= 3)
 725       Override = CmdL.FileList[2];
 726
 727    string SOverride;
 728    if (Override.empty() == false)
 729       SOverride = Override + ".src";
 730
 731    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 732                              SOverride.c_str());
 733
 734    // Create a package writer object.
 735    SourcesWriter Sources(NULL, _config->Find("APT::FTPArchive::DB"),Override,SOverride);
 736    if (_error->PendingError() == true)
 737       return false;
 738
 739    if (CmdL.FileSize() >= 4)
 740       Sources.PathPrefix = CmdL.FileList[3];
 741
 742    // Do recursive directory searching
 743    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 744       return false;
 745
 746    // Give some stats if asked for
 747    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 748      c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl;
 749
 750    return true;
 751 }
 752                                                                         /*}}}*/
 753 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 754 // ---------------------------------------------------------------------
 755 static bool SimpleGenRelease(CommandLine &CmdL)
 756 {
 757    if (CmdL.FileSize() < 2)
 758       return ShowHelp(CmdL);
 759
 760    string Dir = CmdL.FileList[1];
 761
 762    ReleaseWriter Release(NULL, "");
 763    Release.DirStrip = Dir;
 764
 765    if (_error->PendingError() == true)
 766       return false;
 767
 768    if (Release.RecursiveScan(Dir) == false)
 769       return false;
 770
 771    Release.Finish();
 772
 773    return true;
 774 }
 775
 776                                                                         /*}}}*/
 777 // DoGeneratePackagesAndSources - Helper for Generate                   /*{{{*/
 778 // ---------------------------------------------------------------------
 779 static bool DoGeneratePackagesAndSources(Configuration &Setup,
 780                                          vector<PackageMap> &PkgList,
 781                                          struct CacheDB::Stats &SrcStats,
 782                                          struct CacheDB::Stats &Stats,
 783                                          CommandLine &CmdL)
 784 {
 785    if (CmdL.FileSize() <= 2)
 786    {
 787       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 788          if (I->GenPackages(Setup,Stats) == false)
 789             _error->DumpErrors();
 790       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 791          if (I->GenSources(Setup,SrcStats) == false)
 792             _error->DumpErrors();
 793    }
 794    else
 795    {
 796       // Make a choice list out of the package list..
 797       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 798       RxChoiceList *End = List;
 799       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 800       {
 801          End->UserData = &(*I);
 802          End->Str = I->BaseDir.c_str();
 803          End++;
 804
 805          End->UserData = &(*I);
 806          End->Str = I->Tag.c_str();
 807          End++;
 808       }
 809       End->Str = 0;
 810
 811       // Regex it
 812       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 813       {
 814          delete [] List;
 815          return _error->Error(_("No selections matched"));
 816       }
 817       _error->DumpErrors();
 818
 819       // Do the generation for Packages
 820       for (End = List; End->Str != 0; End++)
 821       {
 822          if (End->Hit == false)
 823             continue;
 824
 825          PackageMap *I = (PackageMap *)End->UserData;
 826          if (I->PkgDone == true)
 827             continue;
 828          if (I->GenPackages(Setup,Stats) == false)
 829             _error->DumpErrors();
 830       }
 831
 832       // Do the generation for Sources
 833       for (End = List; End->Str != 0; End++)
 834       {
 835          if (End->Hit == false)
 836             continue;
 837
 838          PackageMap *I = (PackageMap *)End->UserData;
 839          if (I->SrcDone == true)
 840             continue;
 841          if (I->GenSources(Setup,SrcStats) == false)
 842             _error->DumpErrors();
 843       }
 844
 845       delete [] List;
 846    }
 847
 848    // close the Translation master files
 849    for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I)
 850       if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
 851          delete I->TransWriter;
 852
 853    return true;
 854 }
 855
 856                                                                         /*}}}*/
 857 // DoGenerateContents - Helper for Generate to generate the Contents    /*{{{*/
 858 // ---------------------------------------------------------------------
 859 static bool DoGenerateContents(Configuration &Setup,
 860                                vector<PackageMap> &PkgList,
 861                                CommandLine &CmdL)
 862 {
 863    c1out << "Packages done, Starting contents." << endl;
 864
 865    // Sort the contents file list by date
 866    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 867    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 868    {
 869       struct stat A;
 870       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 871                                  I->CntCompress,A) == false)
 872          time(&I->ContentsMTime);
 873       else
 874          I->ContentsMTime = A.st_mtime;
 875    }
 876    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 877
 878    /* Now for Contents.. The process here is to do a make-like dependency
 879       check. Each contents file is verified to be newer than the package files
 880       that describe the debs it indexes. Since the package files contain
 881       hashes of the .debs this means they have not changed either so the
 882       contents must be up to date. */
 883    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 884    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 885    {
 886       // This record is not relevant
 887       if (I->ContentsDone == true ||
 888           I->Contents.empty() == true)
 889          continue;
 890
 891       // Do not do everything if the user specified sections.
 892       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 893          continue;
 894
 895       struct stat A,B;
 896       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 897       {
 898          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 899          {
 900             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 901             continue;
 902          }
 903
 904          if (A.st_mtime > B.st_mtime)
 905             continue;
 906       }
 907
 908       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 909                          MaxContentsChange) == false)
 910          _error->DumpErrors();
 911
 912       // Hit the limit?
 913       if (MaxContentsChange == 0)
 914       {
 915          c1out << "Hit contents update byte limit" << endl;
 916          break;
 917       }
 918    }
 919
 920    return true;
 921 }
 922
 923                                                                         /*}}}*/
 924 // Generate - Full generate, using a config file                        /*{{{*/
 925 // ---------------------------------------------------------------------
 926 /* */
 927 static bool Generate(CommandLine &CmdL)
 928 {
 929    struct CacheDB::Stats SrcStats;
 930    if (CmdL.FileSize() < 2)
 931       return ShowHelp(CmdL);
 932
 933    struct timeval StartTime;
 934    gettimeofday(&StartTime,0);
 935    struct CacheDB::Stats Stats;
 936
 937    // Read the configuration file.
 938    Configuration Setup;
 939    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 940       return false;
 941
 942    vector<PackageMap> PkgList;
 943    LoadTree(PkgList,Setup);
 944    LoadBinDir(PkgList,Setup);
 945
 946    // Sort by cache DB to improve IO locality.
 947    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 948    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 949
 950    // Generate packages
 951    if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false)
 952    {
 953       if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false)
 954          return false;
 955    } else {
 956       c1out << "Skipping Packages/Sources generation" << endl;
 957    }
 958
 959    // do Contents if needed
 960    if (_config->FindB("APT::FTPArchive::Contents", true) == true)
 961       if (DoGenerateContents(Setup, PkgList, CmdL) == false)
 962          return false;
 963
 964    struct timeval NewTime;
 965    gettimeofday(&NewTime,0);
 966    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 967                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 968    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 969          << " archives. Took " << TimeToStr((long)Delta) << endl;
 970
 971    return true;
 972 }
 973
 974                                                                         /*}}}*/
 975 // Clean - Clean out the databases                                      /*{{{*/
 976 // ---------------------------------------------------------------------
 977 /* */
 978 static bool Clean(CommandLine &CmdL)
 979 {
 980    if (CmdL.FileSize() != 2)
 981       return ShowHelp(CmdL);
 982
 983    // Read the configuration file.
 984    Configuration Setup;
 985    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 986       return false;
 987
 988    vector<PackageMap> PkgList;
 989    LoadTree(PkgList,Setup);
 990    LoadBinDir(PkgList,Setup);
 991
 992    // Sort by cache DB to improve IO locality.
 993    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 994    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 995
 996    string CacheDir = Setup.FindDir("Dir::CacheDir");
 997
 998    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 999    {
1000       if(I->BinCacheDB != "")
1001          c0out << I->BinCacheDB << endl;
1002       if(I->SrcCacheDB != "")
1003          c0out << I->SrcCacheDB << endl;
1004       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
1005       CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));
1006       if (DB.Clean() == false)
1007          _error->DumpErrors();
1008       if (DB_SRC.Clean() == false)
1009          _error->DumpErrors();
1010
1011       string CacheDB = I->BinCacheDB;
1012       string SrcCacheDB = I->SrcCacheDB;
1013       while(I != PkgList.end() &&
1014             I->BinCacheDB == CacheDB &&
1015             I->SrcCacheDB == SrcCacheDB)
1016          ++I;
1017
1018    }
1019
1020
1021    return true;
1022 }
1023                                                                         /*}}}*/
1024
1025 int main(int argc, const char *argv[])
1026 {
1027    setlocale(LC_ALL, "");
1028    CommandLine::Args Args[] = {
1029       {'h',"help","help",0},
1030       {0,"md5","APT::FTPArchive::MD5",0},
1031       {0,"sha1","APT::FTPArchive::SHA1",0},
1032       {0,"sha256","APT::FTPArchive::SHA256",0},
1033       {'v',"version","version",0},
1034       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
1035       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
1036       {'q',"quiet","quiet",CommandLine::IntLevel},
1037       {'q',"silent","quiet",CommandLine::IntLevel},
1038       {0,"delink","APT::FTPArchive::DeLinkAct",0},
1039       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
1040       {0,"contents","APT::FTPArchive::Contents",0},
1041       {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
1042       {'c',"config-file",0,CommandLine::ConfigFile},
1043       {'o',"option",0,CommandLine::ArbItem},
1044       {0,0,0,0}};
1045    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
1046                                    {"contents",&SimpleGenContents},
1047                                    {"sources",&SimpleGenSources},
1048                                    {"release",&SimpleGenRelease},
1049                                    {"generate",&Generate},
1050                                    {"clean",&Clean},
1051                                    {"help",&ShowHelp},
1052                                    {0,0}};
1053
1054    // Parse the command line and initialize the package library
1055    CommandLine CmdL(Args,_config);
1056    ParseCommandLine(CmdL, Cmds, Args, &_config, NULL, argc, argv, ShowHelp);
1057
1058    _config->CndSet("quiet",0);
1059    Quiet = _config->FindI("quiet",0);
1060    InitOutput(clog.rdbuf());
1061
1062    // Match the operation
1063    CmdL.DispatchArg(Cmds);
1064
1065    if (_error->empty() == false)
1066    {
1067       bool Errors = _error->PendingError();
1068       _error->DumpErrors();
1069       return Errors == true?100:0;
1070    }
1071    return 0;
1072 }