ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <apt-pkg/fileutl.h>
  21
  22 #include <algorithm>
  23 #include <climits>
  24 #include <sys/time.h>
  25 #include <locale.h>
  26 #include <stdio.h>
  27 #include <sys/stat.h>
  28 #include <time.h>
  29 #include <functional>
  30 #include <iostream>
  31 #include <string>
  32 #include <vector>
  33
  34 #include "cachedb.h"
  35 #include "override.h"
  36 #include "apt-ftparchive.h"
  37 #include "multicompress.h"
  38 #include "writer.h"
  39
  40 #include <apti18n.h>
  41                                                                         /*}}}*/
  42
  43 using namespace std;
  44 ostream c0out(0);
  45 ostream c1out(0);
  46 ostream c2out(0);
  47 ofstream devnull("/dev/null");
  48 unsigned Quiet = 0;
  49
  50 // struct PackageMap - List of all package files in the config file     /*{{{*/
  51 // ---------------------------------------------------------------------
  52 /* */
  53 struct PackageMap
  54 {
  55    // General Stuff
  56    string BaseDir;
  57    string InternalPrefix;
  58    string FLFile;
  59    string PkgExt;
  60    string SrcExt;
  61
  62    // Stuff for the Package File
  63    string PkgFile;
  64    string BinCacheDB;
  65    string SrcCacheDB;
  66    string BinOverride;
  67    string ExtraOverride;
  68
  69    // We generate for this given arch
  70    string Arch;
  71
  72    // Stuff for the Source File
  73    string SrcFile;
  74    string SrcOverride;
  75    string SrcExtraOverride;
  76
  77    // Translation master file
  78    bool LongDesc;
  79    TranslationWriter *TransWriter;
  80
  81    // Contents
  82    string Contents;
  83    string ContentsHead;
  84
  85    // Random things
  86    string Tag;
  87    string PkgCompress;
  88    string CntCompress;
  89    string SrcCompress;
  90    string PathPrefix;
  91    unsigned int DeLinkLimit;
  92    mode_t Permissions;
  93
  94    bool ContentsDone;
  95    bool PkgDone;
  96    bool SrcDone;
  97    time_t ContentsMTime;
  98
  99    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
 100    {
 101       inline bool operator() (const PackageMap &x,const PackageMap &y)
 102       {return x.ContentsMTime < y.ContentsMTime;};
 103    };
 104
 105    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
 106    {
 107       inline bool operator() (const PackageMap &x,const PackageMap &y)
 108       {return x.BinCacheDB < y.BinCacheDB;};
 109    };
 110
 111    struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
 112    {
 113       inline bool operator() (const PackageMap &x,const PackageMap &y)
 114       {return x.SrcCacheDB < y.SrcCacheDB;};
 115    };
 116
 117    void GetGeneral(Configuration &Setup,Configuration &Block);
 118    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 119    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 120    bool GenContents(Configuration &Setup,
 121                     vector<PackageMap>::iterator Begin,
 122                     vector<PackageMap>::iterator End,
 123                     unsigned long &Left);
 124
 125    PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 126                   ContentsDone(false), PkgDone(false), SrcDone(false),
 127                   ContentsMTime(0) {};
 128 };
 129                                                                         /*}}}*/
 130
 131 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 132 // ---------------------------------------------------------------------
 133 /* */
 134 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 135 {
 136    PathPrefix = Block.Find("PathPrefix");
 137
 138    if (Block.FindB("External-Links",true) == false)
 139       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 140    else
 141       DeLinkLimit = 0;
 142
 143    PkgCompress = Block.Find("Packages::Compress",
 144                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 145    CntCompress = Block.Find("Contents::Compress",
 146                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 147    SrcCompress = Block.Find("Sources::Compress",
 148                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 149
 150    SrcExt = Block.Find("Sources::Extensions",
 151                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 152    PkgExt = Block.Find("Packages::Extensions",
 153                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 154
 155    Permissions = Setup.FindI("Default::FileMode",0644);
 156
 157    if (FLFile.empty() == false)
 158       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 159
 160    if (Contents == " ")
 161       Contents= string();
 162 }
 163                                                                         /*}}}*/
 164 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 165 // ---------------------------------------------------------------------
 166 /* This generates the Package File described by this object. */
 167 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 168 {
 169    if (PkgFile.empty() == true)
 170       return true;
 171
 172    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 173    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 174    string CacheDir = Setup.FindDir("Dir::CacheDir");
 175
 176    struct timeval StartTime;
 177    gettimeofday(&StartTime,0);
 178
 179    PkgDone = true;
 180
 181    // Create a package writer object.
 182    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 183                            flCombine(OverrideDir,BinOverride),
 184                            flCombine(OverrideDir,ExtraOverride),
 185                            Arch);
 186    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 187       return _error->Error(_("Package extension list is too long"));
 188    if (_error->PendingError() == true)
 189       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 190
 191    Packages.PathPrefix = PathPrefix;
 192    Packages.DirStrip = ArchiveDir;
 193    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 194
 195    Packages.TransWriter = TransWriter;
 196    Packages.LongDescription = LongDesc;
 197
 198    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 199    Packages.DeLinkLimit = DeLinkLimit;
 200
 201    // Create a compressor object
 202    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 203                       PkgCompress,Permissions);
 204    Packages.Output = Comp.Input;
 205    if (_error->PendingError() == true)
 206       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 207
 208    c0out << ' ' << BaseDir << ":" << flush;
 209
 210    // Do recursive directory searching
 211    if (FLFile.empty() == true)
 212    {
 213       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 214          return false;
 215    }
 216    else
 217    {
 218       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 219          return false;
 220    }
 221
 222    Packages.Output = 0;      // Just in case
 223
 224    // Finish compressing
 225    unsigned long long Size;
 226    if (Comp.Finalize(Size) == false)
 227    {
 228       c0out << endl;
 229       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 230    }
 231
 232    if (Size != 0)
 233       c0out << " New "
 234              << SizeToStr(Size) << "B ";
 235    else
 236       c0out << ' ';
 237
 238    struct timeval NewTime;
 239    gettimeofday(&NewTime,0);
 240    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 241                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 242
 243    c0out << Packages.Stats.Packages << " files " <<
 244 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 245       SizeToStr(Packages.Stats.Bytes) << "B " <<
 246       TimeToStr((long)Delta) << endl;
 247
 248    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 249      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 250
 251    Stats.Add(Packages.Stats);
 252    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 253
 254    return !_error->PendingError();
 255 }
 256
 257                                                                         /*}}}*/
 258 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 259 // ---------------------------------------------------------------------
 260 /* This generates the Sources File described by this object. */
 261 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 262 {
 263    if (SrcFile.empty() == true)
 264       return true;
 265
 266    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 267    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 268    string CacheDir = Setup.FindDir("Dir::CacheDir");
 269
 270    struct timeval StartTime;
 271    gettimeofday(&StartTime,0);
 272
 273    SrcDone = true;
 274
 275    // Create a package writer object.
 276    SourcesWriter Sources(flCombine(CacheDir, SrcCacheDB),
 277                          flCombine(OverrideDir,BinOverride),
 278                          flCombine(OverrideDir,SrcOverride),
 279                          flCombine(OverrideDir,SrcExtraOverride));
 280    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 281       return _error->Error(_("Source extension list is too long"));
 282    if (_error->PendingError() == true)
 283       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 284
 285    Sources.PathPrefix = PathPrefix;
 286    Sources.DirStrip = ArchiveDir;
 287    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 288
 289    Sources.DeLinkLimit = DeLinkLimit;
 290    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 291
 292    // Create a compressor object
 293    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 294                       SrcCompress,Permissions);
 295    Sources.Output = Comp.Input;
 296    if (_error->PendingError() == true)
 297       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 298
 299    c0out << ' ' << BaseDir << ":" << flush;
 300
 301    // Do recursive directory searching
 302    if (FLFile.empty() == true)
 303    {
 304       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 305          return false;
 306    }
 307    else
 308    {
 309       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 310          return false;
 311    }
 312    Sources.Output = 0;      // Just in case
 313
 314    // Finish compressing
 315    unsigned long long Size;
 316    if (Comp.Finalize(Size) == false)
 317    {
 318       c0out << endl;
 319       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 320    }
 321
 322    if (Size != 0)
 323       c0out << " New "
 324              << SizeToStr(Size) << "B ";
 325    else
 326       c0out << ' ';
 327
 328    struct timeval NewTime;
 329    gettimeofday(&NewTime,0);
 330    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 331                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 332
 333    c0out << Sources.Stats.Packages << " pkgs in " <<
 334       TimeToStr((long)Delta) << endl;
 335
 336    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 337      c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
 338
 339    Stats.Add(Sources.Stats);
 340    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 341
 342    return !_error->PendingError();
 343 }
 344                                                                         /*}}}*/
 345 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 346 // ---------------------------------------------------------------------
 347 /* This generates the contents file partially described by this object.
 348    It searches the given iterator range for other package files that map
 349    into this contents file and includes their data as well when building. */
 350 bool PackageMap::GenContents(Configuration &Setup,
 351                              vector<PackageMap>::iterator Begin,
 352                              vector<PackageMap>::iterator End,
 353                              unsigned long &Left)
 354 {
 355    if (Contents.empty() == true)
 356       return true;
 357
 358    if (Left == 0)
 359       return true;
 360
 361    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 362    string CacheDir = Setup.FindDir("Dir::CacheDir");
 363    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 364
 365    struct timeval StartTime;
 366    gettimeofday(&StartTime,0);
 367
 368    // Create a package writer object.
 369    ContentsWriter Contents("", Arch);
 370    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 371       return _error->Error(_("Package extension list is too long"));
 372    if (_error->PendingError() == true)
 373       return false;
 374
 375    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 376                       CntCompress,Permissions);
 377    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 378    Contents.Output = Comp.Input;
 379    if (_error->PendingError() == true)
 380       return false;
 381
 382    // Write the header out.
 383    if (ContentsHead.empty() == false)
 384    {
 385       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 386       if (_error->PendingError() == true)
 387          return false;
 388
 389       unsigned long long Size = Head.Size();
 390       unsigned char Buf[4096];
 391       while (Size != 0)
 392       {
 393          unsigned long long ToRead = Size;
 394          if (Size > sizeof(Buf))
 395             ToRead = sizeof(Buf);
 396
 397          if (Head.Read(Buf,ToRead) == false)
 398             return false;
 399
 400          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 401             return _error->Errno("fwrite",_("Error writing header to contents file"));
 402
 403          Size -= ToRead;
 404       }
 405    }
 406
 407    /* Go over all the package file records and parse all the package
 408       files associated with this contents file into one great big honking
 409       memory structure, then dump the sorted version */
 410    c0out << ' ' << this->Contents << ":" << flush;
 411    for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
 412    {
 413       if (I->Contents != this->Contents)
 414          continue;
 415
 416       Contents.Prefix = ArchiveDir;
 417       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 418       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 419                             I->PkgCompress);
 420
 421       I->ContentsDone = true;
 422    }
 423
 424    Contents.Finish();
 425
 426    // Finish compressing
 427    unsigned long long Size;
 428    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 429    {
 430       c0out << endl;
 431       return _error->Error(_("Error processing contents %s"),
 432                            this->Contents.c_str());
 433    }
 434
 435    if (Size != 0)
 436    {
 437       c0out << " New " << SizeToStr(Size) << "B ";
 438       if (Left > Size)
 439          Left -= Size;
 440       else
 441          Left = 0;
 442    }
 443    else
 444       c0out << ' ';
 445
 446    struct timeval NewTime;
 447    gettimeofday(&NewTime,0);
 448    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 449                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 450
 451    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 452      c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
 453
 454    c0out << Contents.Stats.Packages << " files " <<
 455       SizeToStr(Contents.Stats.Bytes) << "B " <<
 456       TimeToStr((long)Delta) << endl;
 457
 458    return true;
 459 }
 460                                                                         /*}}}*/
 461
 462 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 463 // ---------------------------------------------------------------------
 464 /* This populates the PkgList with all the possible permutations of the
 465    section/arch lists. */
 466 static void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 467 {
 468    // Load the defaults
 469    string DDir = Setup.Find("TreeDefault::Directory",
 470                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 471    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 472                             "$(DIST)/$(SECTION)/source/");
 473    string DPkg = Setup.Find("TreeDefault::Packages",
 474                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 475    string DTrans = Setup.Find("TreeDefault::Translation",
 476                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 477    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 478                             "$(DIST)/$(SECTION)/");
 479    string DContents = Setup.Find("TreeDefault::Contents",
 480                             "$(DIST)/$(SECTION)/Contents-$(ARCH)");
 481    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 482    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 483                                "packages-$(ARCH).db");
 484    string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
 485                                "sources-$(SECTION).db");
 486    string DSources = Setup.Find("TreeDefault::Sources",
 487                                 "$(DIST)/$(SECTION)/source/Sources");
 488    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 489    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 490
 491    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 492
 493    bool const LongDescription = Setup.FindB("Default::LongDescription",
 494                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 495    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 496
 497    // Process 'tree' type sections
 498    const Configuration::Item *Top = Setup.Tree("tree");
 499    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 500    {
 501       Configuration Block(Top);
 502       string Dist = Top->Tag;
 503
 504       // Parse the sections
 505       string Tmp = Block.Find("Sections");
 506       const char *Sections = Tmp.c_str();
 507       string Section;
 508       while (ParseQuoteWord(Sections,Section) == true)
 509       {
 510          string Arch;
 511          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 512                                          {"$(SECTION)",&Section},
 513                                          {"$(ARCH)",&Arch},
 514                                          {NULL, NULL}};
 515          mode_t const Perms = Block.FindI("FileMode", Permissions);
 516          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 517          TranslationWriter *TransWriter;
 518          if (DTrans.empty() == false && LongDesc == false)
 519          {
 520             string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 521                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 522             string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 523             TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 524          }
 525          else
 526             TransWriter = NULL;
 527
 528          string const Tmp2 = Block.Find("Architectures");
 529          const char *Archs = Tmp2.c_str();
 530          while (ParseQuoteWord(Archs,Arch) == true)
 531          {
 532             PackageMap Itm;
 533             Itm.Permissions = Perms;
 534             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 535             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 536
 537             if (stringcasecmp(Arch,"source") == 0)
 538             {
 539                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 540                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 541                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 542                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 543                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 544                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 545                Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);
 546             }
 547             else
 548             {
 549                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 550                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 551                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 552                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 553                Itm.Arch = Arch;
 554                Itm.LongDesc = LongDesc;
 555                if (TransWriter != NULL)
 556                {
 557                   TransWriter->IncreaseRefCounter();
 558                   Itm.TransWriter = TransWriter;
 559                }
 560                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 561                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 562                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 563                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 564             }
 565
 566             Itm.GetGeneral(Setup,Block);
 567             PkgList.push_back(Itm);
 568          }
 569          // we didn't use this TransWriter, so we can release it
 570          if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
 571             delete TransWriter;
 572       }
 573
 574       Top = Top->Next;
 575    }
 576 }
 577                                                                         /*}}}*/
 578 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 579 // ---------------------------------------------------------------------
 580 /* */
 581 static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 582 {
 583    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 584
 585    // Process 'bindirectory' type sections
 586    const Configuration::Item *Top = Setup.Tree("bindirectory");
 587    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 588    {
 589       Configuration Block(Top);
 590
 591       PackageMap Itm;
 592       Itm.PkgFile = Block.Find("Packages");
 593       Itm.SrcFile = Block.Find("Sources");
 594       Itm.BinCacheDB = Block.Find("BinCacheDB");
 595       Itm.SrcCacheDB = Block.Find("SrcCacheDB");
 596       Itm.BinOverride = Block.Find("BinOverride");
 597       Itm.ExtraOverride = Block.Find("ExtraOverride");
 598       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 599       Itm.SrcOverride = Block.Find("SrcOverride");
 600       Itm.BaseDir = Top->Tag;
 601       Itm.FLFile = Block.Find("FileList");
 602       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 603       Itm.Contents = Block.Find("Contents");
 604       Itm.ContentsHead = Block.Find("Contents::Header");
 605       Itm.Permissions = Block.FindI("FileMode", Permissions);
 606
 607       Itm.GetGeneral(Setup,Block);
 608       PkgList.push_back(Itm);
 609
 610       Top = Top->Next;
 611    }
 612 }
 613                                                                         /*}}}*/
 614
 615 // ShowHelp - Show the help text                                        /*{{{*/
 616 // ---------------------------------------------------------------------
 617 /* */
 618 static bool ShowHelp(CommandLine &)
 619 {
 620    ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
 621             COMMON_ARCH,__DATE__,__TIME__);
 622    if (_config->FindB("version") == true)
 623       return true;
 624
 625    cout <<
 626     _("Usage: apt-ftparchive [options] command\n"
 627       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 628       "          sources srcpath [overridefile [pathprefix]]\n"
 629       "          contents path\n"
 630       "          release path\n"
 631       "          generate config [groups]\n"
 632       "          clean config\n"
 633       "\n"
 634       "apt-ftparchive generates index files for Debian archives. It supports\n"
 635       "many styles of generation from fully automated to functional replacements\n"
 636       "for dpkg-scanpackages and dpkg-scansources\n"
 637       "\n"
 638       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 639       "Package file contains the contents of all the control fields from\n"
 640       "each package as well as the MD5 hash and filesize. An override file\n"
 641       "is supported to force the value of Priority and Section.\n"
 642       "\n"
 643       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 644       "The --source-override option can be used to specify a src override file\n"
 645       "\n"
 646       "The 'packages' and 'sources' command should be run in the root of the\n"
 647       "tree. BinaryPath should point to the base of the recursive search and \n"
 648       "override file should contain the override flags. Pathprefix is\n"
 649       "appended to the filename fields if present. Example usage from the \n"
 650       "Debian archive:\n"
 651       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 652       "               dists/potato/main/binary-i386/Packages\n"
 653       "\n"
 654       "Options:\n"
 655       "  -h    This help text\n"
 656       "  --md5 Control MD5 generation\n"
 657       "  -s=?  Source override file\n"
 658       "  -q    Quiet\n"
 659       "  -d=?  Select the optional caching database\n"
 660       "  --no-delink Enable delinking debug mode\n"
 661       "  --contents  Control contents file generation\n"
 662       "  -c=?  Read this configuration file\n"
 663       "  -o=?  Set an arbitrary configuration option") << endl;
 664
 665    return true;
 666 }
 667                                                                         /*}}}*/
 668 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 669 // ---------------------------------------------------------------------
 670 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 671 static bool SimpleGenPackages(CommandLine &CmdL)
 672 {
 673    if (CmdL.FileSize() < 2)
 674       return ShowHelp(CmdL);
 675
 676    string Override;
 677    if (CmdL.FileSize() >= 3)
 678       Override = CmdL.FileList[2];
 679
 680    // Create a package writer object.
 681    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 682                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 683    if (_error->PendingError() == true)
 684       return false;
 685
 686    if (CmdL.FileSize() >= 4)
 687       Packages.PathPrefix = CmdL.FileList[3];
 688
 689    // Do recursive directory searching
 690    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 691       return false;
 692
 693    // Give some stats if asked for
 694    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 695      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 696
 697    return true;
 698 }
 699                                                                         /*}}}*/
 700 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 701 // ---------------------------------------------------------------------
 702 /* */
 703 static bool SimpleGenContents(CommandLine &CmdL)
 704 {
 705    if (CmdL.FileSize() < 2)
 706       return ShowHelp(CmdL);
 707
 708    // Create a package writer object.
 709    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 710    if (_error->PendingError() == true)
 711       return false;
 712
 713    // Do recursive directory searching
 714    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 715       return false;
 716
 717    Contents.Finish();
 718
 719    return true;
 720 }
 721                                                                         /*}}}*/
 722 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 723 // ---------------------------------------------------------------------
 724 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 725 static bool SimpleGenSources(CommandLine &CmdL)
 726 {
 727    if (CmdL.FileSize() < 2)
 728       return ShowHelp(CmdL);
 729
 730    string Override;
 731    if (CmdL.FileSize() >= 3)
 732       Override = CmdL.FileList[2];
 733
 734    string SOverride;
 735    if (Override.empty() == false)
 736       SOverride = Override + ".src";
 737
 738    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 739                              SOverride.c_str());
 740
 741    // Create a package writer object.
 742    SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),Override,SOverride);
 743    if (_error->PendingError() == true)
 744       return false;
 745
 746    if (CmdL.FileSize() >= 4)
 747       Sources.PathPrefix = CmdL.FileList[3];
 748
 749    // Do recursive directory searching
 750    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 751       return false;
 752
 753    // Give some stats if asked for
 754    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 755      c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl;
 756
 757    return true;
 758 }
 759                                                                         /*}}}*/
 760 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 761 // ---------------------------------------------------------------------
 762 static bool SimpleGenRelease(CommandLine &CmdL)
 763 {
 764    if (CmdL.FileSize() < 2)
 765       return ShowHelp(CmdL);
 766
 767    string Dir = CmdL.FileList[1];
 768
 769    ReleaseWriter Release("");
 770    Release.DirStrip = Dir;
 771
 772    if (_error->PendingError() == true)
 773       return false;
 774
 775    if (Release.RecursiveScan(Dir) == false)
 776       return false;
 777
 778    Release.Finish();
 779
 780    return true;
 781 }
 782
 783                                                                         /*}}}*/
 784 // DoGeneratePackagesAndSources - Helper for Generate                   /*{{{*/
 785 // ---------------------------------------------------------------------
 786 static bool DoGeneratePackagesAndSources(Configuration &Setup,
 787                                          vector<PackageMap> &PkgList,
 788                                          struct CacheDB::Stats &SrcStats,
 789                                          struct CacheDB::Stats &Stats,
 790                                          CommandLine &CmdL)
 791 {
 792    if (CmdL.FileSize() <= 2)
 793    {
 794       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 795          if (I->GenPackages(Setup,Stats) == false)
 796             _error->DumpErrors();
 797       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 798          if (I->GenSources(Setup,SrcStats) == false)
 799             _error->DumpErrors();
 800    }
 801    else
 802    {
 803       // Make a choice list out of the package list..
 804       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 805       RxChoiceList *End = List;
 806       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 807       {
 808          End->UserData = &(*I);
 809          End->Str = I->BaseDir.c_str();
 810          End++;
 811
 812          End->UserData = &(*I);
 813          End->Str = I->Tag.c_str();
 814          End++;
 815       }
 816       End->Str = 0;
 817
 818       // Regex it
 819       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 820       {
 821          delete [] List;
 822          return _error->Error(_("No selections matched"));
 823       }
 824       _error->DumpErrors();
 825
 826       // Do the generation for Packages
 827       for (End = List; End->Str != 0; End++)
 828       {
 829          if (End->Hit == false)
 830             continue;
 831
 832          PackageMap *I = (PackageMap *)End->UserData;
 833          if (I->PkgDone == true)
 834             continue;
 835          if (I->GenPackages(Setup,Stats) == false)
 836             _error->DumpErrors();
 837       }
 838
 839       // Do the generation for Sources
 840       for (End = List; End->Str != 0; End++)
 841       {
 842          if (End->Hit == false)
 843             continue;
 844
 845          PackageMap *I = (PackageMap *)End->UserData;
 846          if (I->SrcDone == true)
 847             continue;
 848          if (I->GenSources(Setup,SrcStats) == false)
 849             _error->DumpErrors();
 850       }
 851
 852       delete [] List;
 853    }
 854
 855    // close the Translation master files
 856    for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I)
 857       if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
 858          delete I->TransWriter;
 859
 860    return true;
 861 }
 862
 863                                                                         /*}}}*/
 864 // DoGenerateContents - Helper for Generate to generate the Contents    /*{{{*/
 865 // ---------------------------------------------------------------------
 866 static bool DoGenerateContents(Configuration &Setup,
 867                                vector<PackageMap> &PkgList,
 868                                CommandLine &CmdL)
 869 {
 870    c1out << "Packages done, Starting contents." << endl;
 871
 872    // Sort the contents file list by date
 873    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 874    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 875    {
 876       struct stat A;
 877       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 878                                  I->CntCompress,A) == false)
 879          time(&I->ContentsMTime);
 880       else
 881          I->ContentsMTime = A.st_mtime;
 882    }
 883    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 884
 885    /* Now for Contents.. The process here is to do a make-like dependency
 886       check. Each contents file is verified to be newer than the package files
 887       that describe the debs it indexes. Since the package files contain
 888       hashes of the .debs this means they have not changed either so the
 889       contents must be up to date. */
 890    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 891    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 892    {
 893       // This record is not relevant
 894       if (I->ContentsDone == true ||
 895           I->Contents.empty() == true)
 896          continue;
 897
 898       // Do not do everything if the user specified sections.
 899       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 900          continue;
 901
 902       struct stat A,B;
 903       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 904       {
 905          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 906          {
 907             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 908             continue;
 909          }
 910
 911          if (A.st_mtime > B.st_mtime)
 912             continue;
 913       }
 914
 915       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 916                          MaxContentsChange) == false)
 917          _error->DumpErrors();
 918
 919       // Hit the limit?
 920       if (MaxContentsChange == 0)
 921       {
 922          c1out << "Hit contents update byte limit" << endl;
 923          break;
 924       }
 925    }
 926
 927    return true;
 928 }
 929
 930                                                                         /*}}}*/
 931 // Generate - Full generate, using a config file                        /*{{{*/
 932 // ---------------------------------------------------------------------
 933 /* */
 934 static bool Generate(CommandLine &CmdL)
 935 {
 936    struct CacheDB::Stats SrcStats;
 937    if (CmdL.FileSize() < 2)
 938       return ShowHelp(CmdL);
 939
 940    struct timeval StartTime;
 941    gettimeofday(&StartTime,0);
 942    struct CacheDB::Stats Stats;
 943
 944    // Read the configuration file.
 945    Configuration Setup;
 946    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 947       return false;
 948
 949    vector<PackageMap> PkgList;
 950    LoadTree(PkgList,Setup);
 951    LoadBinDir(PkgList,Setup);
 952
 953    // Sort by cache DB to improve IO locality.
 954    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 955    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 956
 957    // Generate packages
 958    if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false)
 959    {
 960       if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false)
 961          return false;
 962    } else {
 963       c1out << "Skipping Packages/Sources generation" << endl;
 964    }
 965
 966    // do Contents if needed
 967    if (_config->FindB("APT::FTPArchive::Contents", true) == true)
 968       if (DoGenerateContents(Setup, PkgList, CmdL) == false)
 969          return false;
 970
 971    struct timeval NewTime;
 972    gettimeofday(&NewTime,0);
 973    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 974                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 975    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 976          << " archives. Took " << TimeToStr((long)Delta) << endl;
 977
 978    return true;
 979 }
 980
 981                                                                         /*}}}*/
 982 // Clean - Clean out the databases                                      /*{{{*/
 983 // ---------------------------------------------------------------------
 984 /* */
 985 static bool Clean(CommandLine &CmdL)
 986 {
 987    if (CmdL.FileSize() != 2)
 988       return ShowHelp(CmdL);
 989
 990    // Read the configuration file.
 991    Configuration Setup;
 992    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 993       return false;
 994
 995    vector<PackageMap> PkgList;
 996    LoadTree(PkgList,Setup);
 997    LoadBinDir(PkgList,Setup);
 998
 999    // Sort by cache DB to improve IO locality.
1000    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
1001    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
1002
1003    string CacheDir = Setup.FindDir("Dir::CacheDir");
1004
1005    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
1006    {
1007       if(I->BinCacheDB != "")
1008          c0out << I->BinCacheDB << endl;
1009       if(I->SrcCacheDB != "")
1010          c0out << I->SrcCacheDB << endl;
1011       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
1012       CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));
1013       if (DB.Clean() == false)
1014          _error->DumpErrors();
1015       if (DB_SRC.Clean() == false)
1016          _error->DumpErrors();
1017
1018       string CacheDB = I->BinCacheDB;
1019       string SrcCacheDB = I->SrcCacheDB;
1020       while(I != PkgList.end() &&
1021             I->BinCacheDB == CacheDB &&
1022             I->SrcCacheDB == SrcCacheDB)
1023          ++I;
1024
1025    }
1026
1027
1028    return true;
1029 }
1030                                                                         /*}}}*/
1031
1032 int main(int argc, const char *argv[])
1033 {
1034    setlocale(LC_ALL, "");
1035    CommandLine::Args Args[] = {
1036       {'h',"help","help",0},
1037       {0,"md5","APT::FTPArchive::MD5",0},
1038       {0,"sha1","APT::FTPArchive::SHA1",0},
1039       {0,"sha256","APT::FTPArchive::SHA256",0},
1040       {'v',"version","version",0},
1041       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
1042       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
1043       {'q',"quiet","quiet",CommandLine::IntLevel},
1044       {'q',"silent","quiet",CommandLine::IntLevel},
1045       {0,"delink","APT::FTPArchive::DeLinkAct",0},
1046       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
1047       {0,"contents","APT::FTPArchive::Contents",0},
1048       {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
1049       {'c',"config-file",0,CommandLine::ConfigFile},
1050       {'o',"option",0,CommandLine::ArbItem},
1051       {0,0,0,0}};
1052    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
1053                                    {"contents",&SimpleGenContents},
1054                                    {"sources",&SimpleGenSources},
1055                                    {"release",&SimpleGenRelease},
1056                                    {"generate",&Generate},
1057                                    {"clean",&Clean},
1058                                    {"help",&ShowHelp},
1059                                    {0,0}};
1060
1061    // Parse the command line and initialize the package library
1062    CommandLine CmdL(Args,_config);
1063    if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false)
1064    {
1065       _error->DumpErrors();
1066       return 100;
1067    }
1068
1069    // See if the help should be shown
1070    if (_config->FindB("help") == true ||
1071        _config->FindB("version") == true ||
1072        CmdL.FileSize() == 0)
1073    {
1074       ShowHelp(CmdL);
1075       return 0;
1076    }
1077
1078    // Setup the output streams
1079    c0out.rdbuf(clog.rdbuf());
1080    c1out.rdbuf(clog.rdbuf());
1081    c2out.rdbuf(clog.rdbuf());
1082    Quiet = _config->FindI("quiet",0);
1083    if (Quiet > 0)
1084       c0out.rdbuf(devnull.rdbuf());
1085    if (Quiet > 1)
1086       c1out.rdbuf(devnull.rdbuf());
1087
1088    // Match the operation
1089    CmdL.DispatchArg(Cmds);
1090
1091    if (_error->empty() == false)
1092    {
1093       bool Errors = _error->PendingError();
1094       _error->DumpErrors();
1095       return Errors == true?100:0;
1096    }
1097    return 0;
1098 }