ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <apt-pkg/fileutl.h>
  21
  22 #include <algorithm>
  23 #include <climits>
  24 #include <sys/time.h>
  25 #include <locale.h>
  26 #include <stdio.h>
  27 #include <sys/stat.h>
  28 #include <time.h>
  29 #include <functional>
  30 #include <iostream>
  31 #include <string>
  32 #include <vector>
  33
  34 #include "cachedb.h"
  35 #include "override.h"
  36 #include "apt-ftparchive.h"
  37 #include "multicompress.h"
  38 #include "writer.h"
  39
  40 #include <apti18n.h>
  41                                                                         /*}}}*/
  42
  43 using namespace std;
  44 ostream c0out(0);
  45 ostream c1out(0);
  46 ostream c2out(0);
  47 ofstream devnull("/dev/null");
  48 unsigned Quiet = 0;
  49
  50 // struct PackageMap - List of all package files in the config file     /*{{{*/
  51 // ---------------------------------------------------------------------
  52 /* */
  53 struct PackageMap
  54 {
  55    // General Stuff
  56    string BaseDir;
  57    string InternalPrefix;
  58    string FLFile;
  59    string PkgExt;
  60    string SrcExt;
  61
  62    // Stuff for the Package File
  63    string PkgFile;
  64    string BinCacheDB;
  65    string SrcCacheDB;
  66    string BinOverride;
  67    string ExtraOverride;
  68
  69    // We generate for this given arch
  70    string Arch;
  71
  72    // Stuff for the Source File
  73    string SrcFile;
  74    string SrcOverride;
  75    string SrcExtraOverride;
  76
  77    // Translation master file
  78    bool LongDesc;
  79    TranslationWriter *TransWriter;
  80
  81    // Contents
  82    string Contents;
  83    string ContentsHead;
  84
  85    // Random things
  86    string Tag;
  87    string PkgCompress;
  88    string CntCompress;
  89    string SrcCompress;
  90    string PathPrefix;
  91    unsigned int DeLinkLimit;
  92    mode_t Permissions;
  93
  94    bool ContentsDone;
  95    bool PkgDone;
  96    bool SrcDone;
  97    time_t ContentsMTime;
  98
  99    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
 100    {
 101       inline bool operator() (const PackageMap &x,const PackageMap &y)
 102       {return x.ContentsMTime < y.ContentsMTime;};
 103    };
 104
 105    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
 106    {
 107       inline bool operator() (const PackageMap &x,const PackageMap &y)
 108       {return x.BinCacheDB < y.BinCacheDB;};
 109    };
 110
 111    struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
 112    {
 113       inline bool operator() (const PackageMap &x,const PackageMap &y)
 114       {return x.SrcCacheDB < y.SrcCacheDB;};
 115    };
 116
 117    void GetGeneral(Configuration &Setup,Configuration &Block);
 118    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 119    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 120    bool GenContents(Configuration &Setup,
 121                     vector<PackageMap>::iterator Begin,
 122                     vector<PackageMap>::iterator End,
 123                     unsigned long &Left);
 124
 125    PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 126                   ContentsDone(false), PkgDone(false), SrcDone(false),
 127                   ContentsMTime(0) {};
 128 };
 129                                                                         /*}}}*/
 130
 131 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 132 // ---------------------------------------------------------------------
 133 /* */
 134 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 135 {
 136    PathPrefix = Block.Find("PathPrefix");
 137
 138    if (Block.FindB("External-Links",true) == false)
 139       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 140    else
 141       DeLinkLimit = 0;
 142
 143    PkgCompress = Block.Find("Packages::Compress",
 144                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 145    CntCompress = Block.Find("Contents::Compress",
 146                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 147    SrcCompress = Block.Find("Sources::Compress",
 148                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 149
 150    SrcExt = Block.Find("Sources::Extensions",
 151                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 152    PkgExt = Block.Find("Packages::Extensions",
 153                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 154
 155    Permissions = Setup.FindI("Default::FileMode",0644);
 156
 157    if (FLFile.empty() == false)
 158       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 159
 160    if (Contents == " ")
 161       Contents= string();
 162 }
 163                                                                         /*}}}*/
 164 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 165 // ---------------------------------------------------------------------
 166 /* This generates the Package File described by this object. */
 167 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 168 {
 169    if (PkgFile.empty() == true)
 170       return true;
 171
 172    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 173    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 174    string CacheDir = Setup.FindDir("Dir::CacheDir");
 175
 176    struct timeval StartTime;
 177    gettimeofday(&StartTime,0);
 178
 179    PkgDone = true;
 180
 181    // Create a package writer object.
 182    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 183                            flCombine(OverrideDir,BinOverride),
 184                            flCombine(OverrideDir,ExtraOverride),
 185                            Arch);
 186    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 187       return _error->Error(_("Package extension list is too long"));
 188    if (_error->PendingError() == true)
 189       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 190
 191    Packages.PathPrefix = PathPrefix;
 192    Packages.DirStrip = ArchiveDir;
 193    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 194
 195    Packages.TransWriter = TransWriter;
 196    Packages.LongDescription = LongDesc;
 197
 198    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 199    Packages.DeLinkLimit = DeLinkLimit;
 200
 201    // Create a compressor object
 202    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 203                       PkgCompress,Permissions);
 204    Packages.Output = Comp.Input;
 205    if (_error->PendingError() == true)
 206       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 207
 208    c0out << ' ' << BaseDir << ":" << flush;
 209
 210    // Do recursive directory searching
 211    if (FLFile.empty() == true)
 212    {
 213       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 214          return false;
 215    }
 216    else
 217    {
 218       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 219          return false;
 220    }
 221
 222    Packages.Output = 0;      // Just in case
 223
 224    // Finish compressing
 225    unsigned long long Size;
 226    if (Comp.Finalize(Size) == false)
 227    {
 228       c0out << endl;
 229       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 230    }
 231
 232    if (Size != 0)
 233       c0out << " New "
 234              << SizeToStr(Size) << "B ";
 235    else
 236       c0out << ' ';
 237
 238    struct timeval NewTime;
 239    gettimeofday(&NewTime,0);
 240    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 241                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 242
 243    c0out << Packages.Stats.Packages << " files " <<
 244 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 245       SizeToStr(Packages.Stats.Bytes) << "B " <<
 246       TimeToStr((long)Delta) << endl;
 247
 248    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 249      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 250
 251    Stats.Add(Packages.Stats);
 252    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 253
 254    return !_error->PendingError();
 255 }
 256
 257                                                                         /*}}}*/
 258 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 259 // ---------------------------------------------------------------------
 260 /* This generates the Sources File described by this object. */
 261 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 262 {
 263    if (SrcFile.empty() == true)
 264       return true;
 265
 266    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 267    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 268    string CacheDir = Setup.FindDir("Dir::CacheDir");
 269
 270    struct timeval StartTime;
 271    gettimeofday(&StartTime,0);
 272
 273    SrcDone = true;
 274
 275    // Create a package writer object.
 276    SourcesWriter Sources(flCombine(CacheDir, SrcCacheDB),
 277                          flCombine(OverrideDir,BinOverride),
 278                          flCombine(OverrideDir,SrcOverride),
 279                          flCombine(OverrideDir,SrcExtraOverride));
 280    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 281       return _error->Error(_("Source extension list is too long"));
 282    if (_error->PendingError() == true)
 283       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 284
 285    Sources.PathPrefix = PathPrefix;
 286    Sources.DirStrip = ArchiveDir;
 287    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 288
 289    Sources.DeLinkLimit = DeLinkLimit;
 290    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 291
 292    // Create a compressor object
 293    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 294                       SrcCompress,Permissions);
 295    Sources.Output = Comp.Input;
 296    if (_error->PendingError() == true)
 297       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 298
 299    c0out << ' ' << BaseDir << ":" << flush;
 300
 301    // Do recursive directory searching
 302    if (FLFile.empty() == true)
 303    {
 304       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 305          return false;
 306    }
 307    else
 308    {
 309       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 310          return false;
 311    }
 312    Sources.Output = 0;      // Just in case
 313
 314    // Finish compressing
 315    unsigned long long Size;
 316    if (Comp.Finalize(Size) == false)
 317    {
 318       c0out << endl;
 319       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 320    }
 321
 322    if (Size != 0)
 323       c0out << " New "
 324              << SizeToStr(Size) << "B ";
 325    else
 326       c0out << ' ';
 327
 328    struct timeval NewTime;
 329    gettimeofday(&NewTime,0);
 330    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 331                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 332
 333    c0out << Sources.Stats.Packages << " pkgs in " <<
 334       TimeToStr((long)Delta) << endl;
 335
 336    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 337      c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
 338
 339    Stats.Add(Sources.Stats);
 340    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 341
 342    return !_error->PendingError();
 343 }
 344                                                                         /*}}}*/
 345 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 346 // ---------------------------------------------------------------------
 347 /* This generates the contents file partially described by this object.
 348    It searches the given iterator range for other package files that map
 349    into this contents file and includes their data as well when building. */
 350 bool PackageMap::GenContents(Configuration &Setup,
 351                              vector<PackageMap>::iterator Begin,
 352                              vector<PackageMap>::iterator End,
 353                              unsigned long &Left)
 354 {
 355    if (Contents.empty() == true)
 356       return true;
 357
 358    if (Left == 0)
 359       return true;
 360
 361    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 362    string CacheDir = Setup.FindDir("Dir::CacheDir");
 363    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 364
 365    struct timeval StartTime;
 366    gettimeofday(&StartTime,0);
 367
 368    // Create a package writer object.
 369    ContentsWriter Contents("", Arch);
 370    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 371       return _error->Error(_("Package extension list is too long"));
 372    if (_error->PendingError() == true)
 373       return false;
 374
 375    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 376                       CntCompress,Permissions);
 377    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 378    Contents.Output = Comp.Input;
 379    if (_error->PendingError() == true)
 380       return false;
 381
 382    // Write the header out.
 383    if (ContentsHead.empty() == false)
 384    {
 385       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 386       if (_error->PendingError() == true)
 387          return false;
 388
 389       unsigned long long Size = Head.Size();
 390       unsigned char Buf[4096];
 391       while (Size != 0)
 392       {
 393          unsigned long long ToRead = Size;
 394          if (Size > sizeof(Buf))
 395             ToRead = sizeof(Buf);
 396
 397          if (Head.Read(Buf,ToRead) == false)
 398             return false;
 399
 400          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 401             return _error->Errno("fwrite",_("Error writing header to contents file"));
 402
 403          Size -= ToRead;
 404       }
 405    }
 406
 407    /* Go over all the package file records and parse all the package
 408       files associated with this contents file into one great big honking
 409       memory structure, then dump the sorted version */
 410    c0out << ' ' << this->Contents << ":" << flush;
 411    for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
 412    {
 413       if (I->Contents != this->Contents)
 414          continue;
 415
 416       Contents.Prefix = ArchiveDir;
 417       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 418       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 419                             I->PkgCompress);
 420
 421       I->ContentsDone = true;
 422    }
 423
 424    Contents.Finish();
 425
 426    // Finish compressing
 427    unsigned long long Size;
 428    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 429    {
 430       c0out << endl;
 431       return _error->Error(_("Error processing contents %s"),
 432                            this->Contents.c_str());
 433    }
 434
 435    if (Size != 0)
 436    {
 437       c0out << " New " << SizeToStr(Size) << "B ";
 438       if (Left > Size)
 439          Left -= Size;
 440       else
 441          Left = 0;
 442    }
 443    else
 444       c0out << ' ';
 445
 446    struct timeval NewTime;
 447    gettimeofday(&NewTime,0);
 448    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 449                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 450
 451    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 452      c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
 453
 454    c0out << Contents.Stats.Packages << " files " <<
 455       SizeToStr(Contents.Stats.Bytes) << "B " <<
 456       TimeToStr((long)Delta) << endl;
 457
 458    return true;
 459 }
 460                                                                         /*}}}*/
 461
 462 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 463 // ---------------------------------------------------------------------
 464 /* This populates the PkgList with all the possible permutations of the
 465    section/arch lists. */
 466 static void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 467 {
 468    // Load the defaults
 469    string DDir = Setup.Find("TreeDefault::Directory",
 470                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 471    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 472                             "$(DIST)/$(SECTION)/source/");
 473    string DPkg = Setup.Find("TreeDefault::Packages",
 474                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 475    string DTrans = Setup.Find("TreeDefault::Translation",
 476                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 477    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 478                             "$(DIST)/$(SECTION)/");
 479    string DContents = Setup.Find("TreeDefault::Contents",
 480                             "$(DIST)/$(SECTION)/Contents-$(ARCH)");
 481    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 482    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 483                                "packages-$(ARCH).db");
 484    string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
 485                                "sources-$(SECTION).db");
 486    string DSources = Setup.Find("TreeDefault::Sources",
 487                                 "$(DIST)/$(SECTION)/source/Sources");
 488    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 489    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 490
 491    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 492
 493    bool const LongDescription = Setup.FindB("Default::LongDescription",
 494                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 495    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 496
 497    // Process 'tree' type sections
 498    const Configuration::Item *Top = Setup.Tree("tree");
 499    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 500    {
 501       Configuration Block(Top);
 502       string Dist = Top->Tag;
 503
 504       // Parse the sections
 505       string Tmp = Block.Find("Sections");
 506       const char *Sections = Tmp.c_str();
 507       string Section;
 508       while (ParseQuoteWord(Sections,Section) == true)
 509       {
 510          string Arch;
 511          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 512                                          {"$(SECTION)",&Section},
 513                                          {"$(ARCH)",&Arch},
 514                                          {NULL, NULL}};
 515          mode_t const Perms = Block.FindI("FileMode", Permissions);
 516          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 517          TranslationWriter *TransWriter;
 518          if (DTrans.empty() == false && LongDesc == false)
 519          {
 520             string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 521                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 522             string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 523             TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 524          }
 525          else
 526             TransWriter = NULL;
 527
 528          string const Tmp2 = Block.Find("Architectures");
 529          const char *Archs = Tmp2.c_str();
 530          while (ParseQuoteWord(Archs,Arch) == true)
 531          {
 532             PackageMap Itm;
 533             Itm.Permissions = Perms;
 534             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 535             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 536
 537             if (stringcasecmp(Arch,"source") == 0)
 538             {
 539                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 540                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 541                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 542                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 543                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 544                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 545                Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);
 546             }
 547             else
 548             {
 549                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 550                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 551                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 552                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 553                Itm.Arch = Arch;
 554                Itm.LongDesc = LongDesc;
 555                if (TransWriter != NULL)
 556                {
 557                   TransWriter->IncreaseRefCounter();
 558                   Itm.TransWriter = TransWriter;
 559                }
 560                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 561                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 562                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 563                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 564             }
 565
 566             Itm.GetGeneral(Setup,Block);
 567             PkgList.push_back(Itm);
 568          }
 569          // we didn't use this TransWriter, so we can release it
 570          if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
 571             delete TransWriter;
 572       }
 573
 574       Top = Top->Next;
 575    }
 576 }
 577                                                                         /*}}}*/
 578 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 579 // ---------------------------------------------------------------------
 580 /* */
 581 static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 582 {
 583    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 584
 585    // Process 'bindirectory' type sections
 586    const Configuration::Item *Top = Setup.Tree("bindirectory");
 587    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 588    {
 589       Configuration Block(Top);
 590
 591       PackageMap Itm;
 592       Itm.PkgFile = Block.Find("Packages");
 593       Itm.SrcFile = Block.Find("Sources");
 594       Itm.BinCacheDB = Block.Find("BinCacheDB");
 595       Itm.SrcCacheDB = Block.Find("SrcCacheDB");
 596       Itm.BinOverride = Block.Find("BinOverride");
 597       Itm.ExtraOverride = Block.Find("ExtraOverride");
 598       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 599       Itm.SrcOverride = Block.Find("SrcOverride");
 600       Itm.BaseDir = Top->Tag;
 601       Itm.FLFile = Block.Find("FileList");
 602       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 603       Itm.Contents = Block.Find("Contents");
 604       Itm.ContentsHead = Block.Find("Contents::Header");
 605       Itm.Permissions = Block.FindI("FileMode", Permissions);
 606
 607       Itm.GetGeneral(Setup,Block);
 608       PkgList.push_back(Itm);
 609
 610       Top = Top->Next;
 611    }
 612 }
 613                                                                         /*}}}*/
 614
 615 // ShowHelp - Show the help text                                        /*{{{*/
 616 // ---------------------------------------------------------------------
 617 /* */
 618 static bool ShowHelp(CommandLine &)
 619 {
 620    ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
 621             COMMON_ARCH,__DATE__,__TIME__);
 622    if (_config->FindB("version") == true)
 623       return true;
 624
 625    cout <<
 626     _("Usage: apt-ftparchive [options] command\n"
 627       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 628       "          sources srcpath [overridefile [pathprefix]]\n"
 629       "          contents path\n"
 630       "          release path\n"
 631       "          generate config [groups]\n"
 632       "          clean config\n"
 633       "\n"
 634       "apt-ftparchive generates index files for Debian archives. It supports\n"
 635       "many styles of generation from fully automated to functional replacements\n"
 636       "for dpkg-scanpackages and dpkg-scansources\n"
 637       "\n"
 638       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 639       "Package file contains the contents of all the control fields from\n"
 640       "each package as well as the MD5 hash and filesize. An override file\n"
 641       "is supported to force the value of Priority and Section.\n"
 642       "\n"
 643       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 644       "The --source-override option can be used to specify a src override file\n"
 645       "\n"
 646       "The 'packages' and 'sources' command should be run in the root of the\n"
 647       "tree. BinaryPath should point to the base of the recursive search and \n"
 648       "override file should contain the override flags. Pathprefix is\n"
 649       "appended to the filename fields if present. Example usage from the \n"
 650       "Debian archive:\n"
 651       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 652       "               dists/potato/main/binary-i386/Packages\n"
 653       "\n"
 654       "Options:\n"
 655       "  -h    This help text\n"
 656       "  --md5 Control MD5 generation\n"
 657       "  -s=?  Source override file\n"
 658       "  -q    Quiet\n"
 659       "  -d=?  Select the optional caching database\n"
 660       "  --no-delink Enable delinking debug mode\n"
 661       "  --contents  Control contents file generation\n"
 662       "  -c=?  Read this configuration file\n"
 663       "  -o=?  Set an arbitrary configuration option") << endl;
 664
 665    return true;
 666 }
 667                                                                         /*}}}*/
 668 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 669 // ---------------------------------------------------------------------
 670 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 671 static bool SimpleGenPackages(CommandLine &CmdL)
 672 {
 673    if (CmdL.FileSize() < 2)
 674       return ShowHelp(CmdL);
 675
 676    string Override;
 677    if (CmdL.FileSize() >= 3)
 678       Override = CmdL.FileList[2];
 679
 680    // Create a package writer object.
 681    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 682                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 683    if (_error->PendingError() == true)
 684       return false;
 685
 686    if (CmdL.FileSize() >= 4)
 687       Packages.PathPrefix = CmdL.FileList[3];
 688
 689    // Do recursive directory searching
 690    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 691       return false;
 692
 693    // Give some stats if asked for
 694    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 695      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 696
 697    return true;
 698 }
 699                                                                         /*}}}*/
 700 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 701 // ---------------------------------------------------------------------
 702 /* */
 703 static bool SimpleGenContents(CommandLine &CmdL)
 704 {
 705    if (CmdL.FileSize() < 2)
 706       return ShowHelp(CmdL);
 707
 708    // Create a package writer object.
 709    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 710    if (_error->PendingError() == true)
 711       return false;
 712
 713    // Do recursive directory searching
 714    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 715       return false;
 716
 717    Contents.Finish();
 718
 719    return true;
 720 }
 721                                                                         /*}}}*/
 722 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 723 // ---------------------------------------------------------------------
 724 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 725 static bool SimpleGenSources(CommandLine &CmdL)
 726 {
 727    if (CmdL.FileSize() < 2)
 728       return ShowHelp(CmdL);
 729
 730    string Override;
 731    if (CmdL.FileSize() >= 3)
 732       Override = CmdL.FileList[2];
 733
 734    string SOverride;
 735    if (Override.empty() == false)
 736       SOverride = Override + ".src";
 737
 738    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 739                              SOverride.c_str());
 740
 741    // Create a package writer object.
 742    SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),Override,SOverride);
 743    if (_error->PendingError() == true)
 744       return false;
 745
 746    if (CmdL.FileSize() >= 4)
 747       Sources.PathPrefix = CmdL.FileList[3];
 748
 749    // Do recursive directory searching
 750    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 751       return false;
 752
 753    // Give some stats if asked for
 754    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 755      c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl;
 756
 757    return true;
 758 }
 759                                                                         /*}}}*/
 760 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 761 // ---------------------------------------------------------------------
 762 static bool SimpleGenRelease(CommandLine &CmdL)
 763 {
 764    if (CmdL.FileSize() < 2)
 765       return ShowHelp(CmdL);
 766
 767    string Dir = CmdL.FileList[1];
 768
 769    ReleaseWriter Release("");
 770    Release.DirStrip = Dir;
 771
 772    if (_error->PendingError() == true)
 773       return false;
 774
 775    if (Release.RecursiveScan(Dir) == false)
 776       return false;
 777
 778    Release.Finish();
 779
 780    return true;
 781 }
 782
 783                                                                         /*}}}*/
 784 // Generate - Full generate, using a config file                        /*{{{*/
 785 // ---------------------------------------------------------------------
 786 /* */
 787 static bool Generate(CommandLine &CmdL)
 788 {
 789    struct CacheDB::Stats SrcStats;
 790    if (CmdL.FileSize() < 2)
 791       return ShowHelp(CmdL);
 792
 793    struct timeval StartTime;
 794    gettimeofday(&StartTime,0);
 795    struct CacheDB::Stats Stats;
 796
 797    // Read the configuration file.
 798    Configuration Setup;
 799    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 800       return false;
 801
 802    vector<PackageMap> PkgList;
 803    LoadTree(PkgList,Setup);
 804    LoadBinDir(PkgList,Setup);
 805
 806    // Sort by cache DB to improve IO locality.
 807    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 808    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 809
 810    // Generate packages
 811    if (CmdL.FileSize() <= 2)
 812    {
 813       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 814          if (I->GenPackages(Setup,Stats) == false)
 815             _error->DumpErrors();
 816       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 817          if (I->GenSources(Setup,SrcStats) == false)
 818             _error->DumpErrors();
 819    }
 820    else
 821    {
 822       // Make a choice list out of the package list..
 823       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 824       RxChoiceList *End = List;
 825       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 826       {
 827          End->UserData = &(*I);
 828          End->Str = I->BaseDir.c_str();
 829          End++;
 830
 831          End->UserData = &(*I);
 832          End->Str = I->Tag.c_str();
 833          End++;
 834       }
 835       End->Str = 0;
 836
 837       // Regex it
 838       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 839       {
 840          delete [] List;
 841          return _error->Error(_("No selections matched"));
 842       }
 843       _error->DumpErrors();
 844
 845       // Do the generation for Packages
 846       for (End = List; End->Str != 0; End++)
 847       {
 848          if (End->Hit == false)
 849             continue;
 850
 851          PackageMap *I = (PackageMap *)End->UserData;
 852          if (I->PkgDone == true)
 853             continue;
 854          if (I->GenPackages(Setup,Stats) == false)
 855             _error->DumpErrors();
 856       }
 857
 858       // Do the generation for Sources
 859       for (End = List; End->Str != 0; End++)
 860       {
 861          if (End->Hit == false)
 862             continue;
 863
 864          PackageMap *I = (PackageMap *)End->UserData;
 865          if (I->SrcDone == true)
 866             continue;
 867          if (I->GenSources(Setup,SrcStats) == false)
 868             _error->DumpErrors();
 869       }
 870
 871       delete [] List;
 872    }
 873
 874    // close the Translation master files
 875    for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I)
 876       if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
 877          delete I->TransWriter;
 878
 879    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 880       return true;
 881
 882    c1out << "Packages done, Starting contents." << endl;
 883
 884    // Sort the contents file list by date
 885    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 886    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 887    {
 888       struct stat A;
 889       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 890                                  I->CntCompress,A) == false)
 891          time(&I->ContentsMTime);
 892       else
 893          I->ContentsMTime = A.st_mtime;
 894    }
 895    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 896
 897    /* Now for Contents.. The process here is to do a make-like dependency
 898       check. Each contents file is verified to be newer than the package files
 899       that describe the debs it indexes. Since the package files contain
 900       hashes of the .debs this means they have not changed either so the
 901       contents must be up to date. */
 902    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 903    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 904    {
 905       // This record is not relevant
 906       if (I->ContentsDone == true ||
 907           I->Contents.empty() == true)
 908          continue;
 909
 910       // Do not do everything if the user specified sections.
 911       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 912          continue;
 913
 914       struct stat A,B;
 915       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 916       {
 917          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 918          {
 919             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 920             continue;
 921          }
 922
 923          if (A.st_mtime > B.st_mtime)
 924             continue;
 925       }
 926
 927       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 928                          MaxContentsChange) == false)
 929          _error->DumpErrors();
 930
 931       // Hit the limit?
 932       if (MaxContentsChange == 0)
 933       {
 934          c1out << "Hit contents update byte limit" << endl;
 935          break;
 936       }
 937    }
 938
 939    struct timeval NewTime;
 940    gettimeofday(&NewTime,0);
 941    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 942                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 943    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 944          << " archives. Took " << TimeToStr((long)Delta) << endl;
 945
 946    return true;
 947 }
 948                                                                         /*}}}*/
 949 // Clean - Clean out the databases                                      /*{{{*/
 950 // ---------------------------------------------------------------------
 951 /* */
 952 static bool Clean(CommandLine &CmdL)
 953 {
 954    if (CmdL.FileSize() != 2)
 955       return ShowHelp(CmdL);
 956
 957    // Read the configuration file.
 958    Configuration Setup;
 959    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 960       return false;
 961
 962    vector<PackageMap> PkgList;
 963    LoadTree(PkgList,Setup);
 964    LoadBinDir(PkgList,Setup);
 965
 966    // Sort by cache DB to improve IO locality.
 967    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 968    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 969
 970    string CacheDir = Setup.FindDir("Dir::CacheDir");
 971
 972    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 973    {
 974       if(I->BinCacheDB != "")
 975          c0out << I->BinCacheDB << endl;
 976       if(I->SrcCacheDB != "")
 977          c0out << I->SrcCacheDB << endl;
 978       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 979       CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));
 980       if (DB.Clean() == false)
 981          _error->DumpErrors();
 982       if (DB_SRC.Clean() == false)
 983          _error->DumpErrors();
 984
 985       string CacheDB = I->BinCacheDB;
 986       string SrcCacheDB = I->SrcCacheDB;
 987       while(I != PkgList.end() &&
 988             I->BinCacheDB == CacheDB &&
 989             I->SrcCacheDB == SrcCacheDB)
 990          ++I;
 991
 992    }
 993
 994
 995    return true;
 996 }
 997                                                                         /*}}}*/
 998
 999 int main(int argc, const char *argv[])
1000 {
1001    setlocale(LC_ALL, "");
1002    CommandLine::Args Args[] = {
1003       {'h',"help","help",0},
1004       {0,"md5","APT::FTPArchive::MD5",0},
1005       {0,"sha1","APT::FTPArchive::SHA1",0},
1006       {0,"sha256","APT::FTPArchive::SHA256",0},
1007       {'v',"version","version",0},
1008       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
1009       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
1010       {'q',"quiet","quiet",CommandLine::IntLevel},
1011       {'q',"silent","quiet",CommandLine::IntLevel},
1012       {0,"delink","APT::FTPArchive::DeLinkAct",0},
1013       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
1014       {0,"contents","APT::FTPArchive::Contents",0},
1015       {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
1016       {'c',"config-file",0,CommandLine::ConfigFile},
1017       {'o',"option",0,CommandLine::ArbItem},
1018       {0,0,0,0}};
1019    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
1020                                    {"contents",&SimpleGenContents},
1021                                    {"sources",&SimpleGenSources},
1022                                    {"release",&SimpleGenRelease},
1023                                    {"generate",&Generate},
1024                                    {"clean",&Clean},
1025                                    {"help",&ShowHelp},
1026                                    {0,0}};
1027
1028    // Parse the command line and initialize the package library
1029    CommandLine CmdL(Args,_config);
1030    if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false)
1031    {
1032       _error->DumpErrors();
1033       return 100;
1034    }
1035
1036    // See if the help should be shown
1037    if (_config->FindB("help") == true ||
1038        _config->FindB("version") == true ||
1039        CmdL.FileSize() == 0)
1040    {
1041       ShowHelp(CmdL);
1042       return 0;
1043    }
1044
1045    // Setup the output streams
1046    c0out.rdbuf(clog.rdbuf());
1047    c1out.rdbuf(clog.rdbuf());
1048    c2out.rdbuf(clog.rdbuf());
1049    Quiet = _config->FindI("quiet",0);
1050    if (Quiet > 0)
1051       c0out.rdbuf(devnull.rdbuf());
1052    if (Quiet > 1)
1053       c1out.rdbuf(devnull.rdbuf());
1054
1055    // Match the operation
1056    CmdL.DispatchArg(Cmds);
1057
1058    if (_error->empty() == false)
1059    {
1060       bool Errors = _error->PendingError();
1061       _error->DumpErrors();
1062       return Errors == true?100:0;
1063    }
1064    return 0;
1065 }