ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <apt-pkg/fileutl.h>
  21
  22 #include <algorithm>
  23 #include <climits>
  24 #include <sys/time.h>
  25 #include <locale.h>
  26 #include <stdio.h>
  27 #include <sys/stat.h>
  28 #include <time.h>
  29 #include <functional>
  30 #include <iostream>
  31 #include <string>
  32 #include <vector>
  33
  34 #include "cachedb.h"
  35 #include "override.h"
  36 #include "apt-ftparchive.h"
  37 #include "multicompress.h"
  38 #include "writer.h"
  39
  40 #include <apti18n.h>
  41                                                                         /*}}}*/
  42
  43 using namespace std;
  44 ostream c0out(0);
  45 ostream c1out(0);
  46 ostream c2out(0);
  47 ofstream devnull("/dev/null");
  48 unsigned Quiet = 0;
  49
  50 // struct PackageMap - List of all package files in the config file     /*{{{*/
  51 // ---------------------------------------------------------------------
  52 /* */
  53 struct PackageMap
  54 {
  55    // General Stuff
  56    string BaseDir;
  57    string InternalPrefix;
  58    string FLFile;
  59    string PkgExt;
  60    string SrcExt;
  61
  62    // Stuff for the Package File
  63    string PkgFile;
  64    string BinCacheDB;
  65    string SrcCacheDB;
  66    string BinOverride;
  67    string ExtraOverride;
  68
  69    // We generate for this given arch
  70    string Arch;
  71
  72    // Stuff for the Source File
  73    string SrcFile;
  74    string SrcOverride;
  75    string SrcExtraOverride;
  76
  77    // Translation master file
  78    bool LongDesc;
  79    TranslationWriter *TransWriter;
  80
  81    // Contents
  82    string Contents;
  83    string ContentsHead;
  84
  85    // Random things
  86    string Tag;
  87    string PkgCompress;
  88    string CntCompress;
  89    string SrcCompress;
  90    string PathPrefix;
  91    unsigned int DeLinkLimit;
  92    mode_t Permissions;
  93
  94    bool ContentsDone;
  95    bool PkgDone;
  96    bool SrcDone;
  97    time_t ContentsMTime;
  98
  99    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
 100    {
 101       inline bool operator() (const PackageMap &x,const PackageMap &y)
 102       {return x.ContentsMTime < y.ContentsMTime;};
 103    };
 104
 105    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
 106    {
 107       inline bool operator() (const PackageMap &x,const PackageMap &y)
 108       {return x.BinCacheDB < y.BinCacheDB;};
 109    };
 110
 111    struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
 112    {
 113       inline bool operator() (const PackageMap &x,const PackageMap &y)
 114       {return x.SrcCacheDB < y.SrcCacheDB;};
 115    };
 116
 117    void GetGeneral(Configuration &Setup,Configuration &Block);
 118    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 119    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 120    bool GenContents(Configuration &Setup,
 121                     vector<PackageMap>::iterator Begin,
 122                     vector<PackageMap>::iterator End,
 123                     unsigned long &Left);
 124
 125    PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 126                   ContentsDone(false), PkgDone(false), SrcDone(false),
 127                   ContentsMTime(0) {};
 128 };
 129                                                                         /*}}}*/
 130
 131 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 132 // ---------------------------------------------------------------------
 133 /* */
 134 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 135 {
 136    PathPrefix = Block.Find("PathPrefix");
 137
 138    if (Block.FindB("External-Links",true) == false)
 139       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 140    else
 141       DeLinkLimit = 0;
 142
 143    PkgCompress = Block.Find("Packages::Compress",
 144                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 145    CntCompress = Block.Find("Contents::Compress",
 146                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 147    SrcCompress = Block.Find("Sources::Compress",
 148                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 149
 150    SrcExt = Block.Find("Sources::Extensions",
 151                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 152    PkgExt = Block.Find("Packages::Extensions",
 153                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 154
 155    Permissions = Setup.FindI("Default::FileMode",0644);
 156
 157    if (FLFile.empty() == false)
 158       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 159
 160    if (Contents == " ")
 161       Contents= string();
 162 }
 163                                                                         /*}}}*/
 164 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 165 // ---------------------------------------------------------------------
 166 /* This generates the Package File described by this object. */
 167 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 168 {
 169    if (PkgFile.empty() == true)
 170       return true;
 171
 172    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 173    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 174    string CacheDir = Setup.FindDir("Dir::CacheDir");
 175
 176    struct timeval StartTime;
 177    gettimeofday(&StartTime,0);
 178
 179    PkgDone = true;
 180
 181    // Create a package writer object.
 182    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 183                            flCombine(OverrideDir,BinOverride),
 184                            flCombine(OverrideDir,ExtraOverride),
 185                            Arch);
 186    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 187       return _error->Error(_("Package extension list is too long"));
 188    if (_error->PendingError() == true)
 189       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 190
 191    Packages.PathPrefix = PathPrefix;
 192    Packages.DirStrip = ArchiveDir;
 193    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 194
 195    Packages.TransWriter = TransWriter;
 196    Packages.LongDescription = LongDesc;
 197
 198    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 199    Packages.DeLinkLimit = DeLinkLimit;
 200
 201    // Create a compressor object
 202    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 203                       PkgCompress,Permissions);
 204    Packages.Output = Comp.Input;
 205    if (_error->PendingError() == true)
 206       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 207
 208    c0out << ' ' << BaseDir << ":" << flush;
 209
 210    // Do recursive directory searching
 211    if (FLFile.empty() == true)
 212    {
 213       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 214          return false;
 215    }
 216    else
 217    {
 218       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 219          return false;
 220    }
 221
 222    Packages.Output = 0;      // Just in case
 223
 224    // Finish compressing
 225    unsigned long long Size;
 226    if (Comp.Finalize(Size) == false)
 227    {
 228       c0out << endl;
 229       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 230    }
 231
 232    if (Size != 0)
 233       c0out << " New "
 234              << SizeToStr(Size) << "B ";
 235    else
 236       c0out << ' ';
 237
 238    struct timeval NewTime;
 239    gettimeofday(&NewTime,0);
 240    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 241                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 242
 243    c0out << Packages.Stats.Packages << " files " <<
 244 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 245       SizeToStr(Packages.Stats.Bytes) << "B " <<
 246       TimeToStr((long)Delta) << endl;
 247
 248    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 249      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 250
 251    Stats.Add(Packages.Stats);
 252    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 253
 254    return !_error->PendingError();
 255 }
 256
 257                                                                         /*}}}*/
 258 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 259 // ---------------------------------------------------------------------
 260 /* This generates the Sources File described by this object. */
 261 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 262 {
 263    if (SrcFile.empty() == true)
 264       return true;
 265
 266    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 267    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 268    string CacheDir = Setup.FindDir("Dir::CacheDir");
 269
 270    struct timeval StartTime;
 271    gettimeofday(&StartTime,0);
 272
 273    SrcDone = true;
 274
 275    // Create a package writer object.
 276    SourcesWriter Sources(flCombine(CacheDir, SrcCacheDB),
 277                          flCombine(OverrideDir,BinOverride),
 278                          flCombine(OverrideDir,SrcOverride),
 279                          flCombine(OverrideDir,SrcExtraOverride));
 280    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 281       return _error->Error(_("Source extension list is too long"));
 282    if (_error->PendingError() == true)
 283       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 284
 285    Sources.PathPrefix = PathPrefix;
 286    Sources.DirStrip = ArchiveDir;
 287    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 288
 289    Sources.DeLinkLimit = DeLinkLimit;
 290    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 291
 292    // Create a compressor object
 293    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 294                       SrcCompress,Permissions);
 295    Sources.Output = Comp.Input;
 296    if (_error->PendingError() == true)
 297       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 298
 299    c0out << ' ' << BaseDir << ":" << flush;
 300
 301    // Do recursive directory searching
 302    if (FLFile.empty() == true)
 303    {
 304       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 305          return false;
 306    }
 307    else
 308    {
 309       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 310          return false;
 311    }
 312    Sources.Output = 0;      // Just in case
 313
 314    // Finish compressing
 315    unsigned long long Size;
 316    if (Comp.Finalize(Size) == false)
 317    {
 318       c0out << endl;
 319       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 320    }
 321
 322    if (Size != 0)
 323       c0out << " New "
 324              << SizeToStr(Size) << "B ";
 325    else
 326       c0out << ' ';
 327
 328    struct timeval NewTime;
 329    gettimeofday(&NewTime,0);
 330    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 331                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 332
 333    c0out << Sources.Stats.Packages << " pkgs in " <<
 334       TimeToStr((long)Delta) << endl;
 335
 336    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 337      c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
 338
 339    Stats.Add(Sources.Stats);
 340    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 341
 342    return !_error->PendingError();
 343 }
 344                                                                         /*}}}*/
 345 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 346 // ---------------------------------------------------------------------
 347 /* This generates the contents file partially described by this object.
 348    It searches the given iterator range for other package files that map
 349    into this contents file and includes their data as well when building. */
 350 bool PackageMap::GenContents(Configuration &Setup,
 351                              vector<PackageMap>::iterator Begin,
 352                              vector<PackageMap>::iterator End,
 353                              unsigned long &Left)
 354 {
 355    if (Contents.empty() == true)
 356       return true;
 357
 358    if (Left == 0)
 359       return true;
 360
 361    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 362    string CacheDir = Setup.FindDir("Dir::CacheDir");
 363    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 364
 365    struct timeval StartTime;
 366    gettimeofday(&StartTime,0);
 367
 368    // Create a package writer object.
 369    ContentsWriter Contents("", Arch);
 370    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 371       return _error->Error(_("Package extension list is too long"));
 372    if (_error->PendingError() == true)
 373       return false;
 374
 375    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 376                       CntCompress,Permissions);
 377    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 378    Contents.Output = Comp.Input;
 379    if (_error->PendingError() == true)
 380       return false;
 381
 382    // Write the header out.
 383    if (ContentsHead.empty() == false)
 384    {
 385       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 386       if (_error->PendingError() == true)
 387          return false;
 388
 389       unsigned long long Size = Head.Size();
 390       unsigned char Buf[4096];
 391       while (Size != 0)
 392       {
 393          unsigned long long ToRead = Size;
 394          if (Size > sizeof(Buf))
 395             ToRead = sizeof(Buf);
 396
 397          if (Head.Read(Buf,ToRead) == false)
 398             return false;
 399
 400          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 401             return _error->Errno("fwrite",_("Error writing header to contents file"));
 402
 403          Size -= ToRead;
 404       }
 405    }
 406
 407    /* Go over all the package file records and parse all the package
 408       files associated with this contents file into one great big honking
 409       memory structure, then dump the sorted version */
 410    c0out << ' ' << this->Contents << ":" << flush;
 411    for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
 412    {
 413       if (I->Contents != this->Contents)
 414          continue;
 415
 416       Contents.Prefix = ArchiveDir;
 417       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 418       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 419                             I->PkgCompress);
 420
 421       I->ContentsDone = true;
 422    }
 423
 424    Contents.Finish();
 425
 426    // Finish compressing
 427    unsigned long long Size;
 428    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 429    {
 430       c0out << endl;
 431       return _error->Error(_("Error processing contents %s"),
 432                            this->Contents.c_str());
 433    }
 434
 435    if (Size != 0)
 436    {
 437       c0out << " New " << SizeToStr(Size) << "B ";
 438       if (Left > Size)
 439          Left -= Size;
 440       else
 441          Left = 0;
 442    }
 443    else
 444       c0out << ' ';
 445
 446    struct timeval NewTime;
 447    gettimeofday(&NewTime,0);
 448    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 449                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 450
 451    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 452      c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
 453
 454    c0out << Contents.Stats.Packages << " files " <<
 455       SizeToStr(Contents.Stats.Bytes) << "B " <<
 456       TimeToStr((long)Delta) << endl;
 457
 458    return true;
 459 }
 460                                                                         /*}}}*/
 461
 462 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 463 // ---------------------------------------------------------------------
 464 /* This populates the PkgList with all the possible permutations of the
 465    section/arch lists. */
 466 static void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 467 {
 468    // Load the defaults
 469    string DDir = Setup.Find("TreeDefault::Directory",
 470                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 471    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 472                             "$(DIST)/$(SECTION)/source/");
 473    string DPkg = Setup.Find("TreeDefault::Packages",
 474                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 475    string DTrans = Setup.Find("TreeDefault::Translation",
 476                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 477    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 478                             "$(DIST)/$(SECTION)/");
 479    string DContents = Setup.Find("TreeDefault::Contents",
 480                             "$(DIST)/$(SECTION)/Contents-$(ARCH)");
 481    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 482    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 483                                "packages-$(ARCH).db");
 484    string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
 485                                "sources-$(SECTION).db");
 486    string DSources = Setup.Find("TreeDefault::Sources",
 487                                 "$(DIST)/$(SECTION)/source/Sources");
 488    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 489    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 490
 491    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 492
 493    bool const LongDescription = Setup.FindB("Default::LongDescription",
 494                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 495    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 496
 497    // Process 'tree' type sections
 498    const Configuration::Item *Top = Setup.Tree("tree");
 499    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 500    {
 501       Configuration Block(Top);
 502       string Dist = Top->Tag;
 503
 504       // Parse the sections
 505       string Tmp = Block.Find("Sections");
 506       const char *Sections = Tmp.c_str();
 507       string Section;
 508       while (ParseQuoteWord(Sections,Section) == true)
 509       {
 510          string Arch;
 511          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 512                                          {"$(SECTION)",&Section},
 513                                          {"$(ARCH)",&Arch},
 514                                          {NULL, NULL}};
 515          mode_t const Perms = Block.FindI("FileMode", Permissions);
 516          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 517          TranslationWriter *TransWriter;
 518          if (DTrans.empty() == false && LongDesc == false)
 519          {
 520             string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 521                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 522             string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 523             TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 524          }
 525          else
 526             TransWriter = NULL;
 527
 528          string const Tmp2 = Block.Find("Architectures");
 529          const char *Archs = Tmp2.c_str();
 530          while (ParseQuoteWord(Archs,Arch) == true)
 531          {
 532             PackageMap Itm;
 533             Itm.Permissions = Perms;
 534             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 535             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 536
 537             if (stringcasecmp(Arch,"source") == 0)
 538             {
 539                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 540                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 541                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 542                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 543                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 544                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 545                Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);
 546             }
 547             else
 548             {
 549                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 550                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 551                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 552                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 553                Itm.Arch = Arch;
 554                Itm.LongDesc = LongDesc;
 555                if (TransWriter != NULL)
 556                {
 557                   TransWriter->IncreaseRefCounter();
 558                   Itm.TransWriter = TransWriter;
 559                }
 560                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 561                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 562                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 563                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 564             }
 565
 566             Itm.GetGeneral(Setup,Block);
 567             PkgList.push_back(Itm);
 568          }
 569          // we didn't use this TransWriter, so we can release it
 570          if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
 571             delete TransWriter;
 572       }
 573
 574       Top = Top->Next;
 575    }
 576 }
 577                                                                         /*}}}*/
 578 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 579 // ---------------------------------------------------------------------
 580 /* */
 581 static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 582 {
 583    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 584
 585    // Process 'bindirectory' type sections
 586    const Configuration::Item *Top = Setup.Tree("bindirectory");
 587    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 588    {
 589       Configuration Block(Top);
 590
 591       PackageMap Itm;
 592       Itm.PkgFile = Block.Find("Packages");
 593       Itm.SrcFile = Block.Find("Sources");
 594       Itm.BinCacheDB = Block.Find("BinCacheDB");
 595       Itm.SrcCacheDB = Block.Find("SrcCacheDB");
 596       Itm.BinOverride = Block.Find("BinOverride");
 597       Itm.ExtraOverride = Block.Find("ExtraOverride");
 598       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 599       Itm.SrcOverride = Block.Find("SrcOverride");
 600       Itm.BaseDir = Top->Tag;
 601       Itm.FLFile = Block.Find("FileList");
 602       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 603       Itm.Contents = Block.Find("Contents");
 604       Itm.ContentsHead = Block.Find("Contents::Header");
 605       Itm.Permissions = Block.FindI("FileMode", Permissions);
 606
 607       Itm.GetGeneral(Setup,Block);
 608       PkgList.push_back(Itm);
 609
 610       Top = Top->Next;
 611    }
 612 }
 613                                                                         /*}}}*/
 614
 615 // ShowHelp - Show the help text                                        /*{{{*/
 616 // ---------------------------------------------------------------------
 617 /* */
 618 static bool ShowHelp(CommandLine &)
 619 {
 620    ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
 621             COMMON_ARCH,__DATE__,__TIME__);
 622    if (_config->FindB("version") == true)
 623       return true;
 624
 625    cout <<
 626     _("Usage: apt-ftparchive [options] command\n"
 627       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 628       "          sources srcpath [overridefile [pathprefix]]\n"
 629       "          contents path\n"
 630       "          release path\n"
 631       "          generate config [groups]\n"
 632       "          clean config\n"
 633       "\n"
 634       "apt-ftparchive generates index files for Debian archives. It supports\n"
 635       "many styles of generation from fully automated to functional replacements\n"
 636       "for dpkg-scanpackages and dpkg-scansources\n"
 637       "\n"
 638       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 639       "Package file contains the contents of all the control fields from\n"
 640       "each package as well as the MD5 hash and filesize. An override file\n"
 641       "is supported to force the value of Priority and Section.\n"
 642       "\n"
 643       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 644       "The --source-override option can be used to specify a src override file\n"
 645       "\n"
 646       "The 'packages' and 'sources' command should be run in the root of the\n"
 647       "tree. BinaryPath should point to the base of the recursive search and \n"
 648       "override file should contain the override flags. Pathprefix is\n"
 649       "appended to the filename fields if present. Example usage from the \n"
 650       "Debian archive:\n"
 651       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 652       "               dists/potato/main/binary-i386/Packages\n"
 653       "\n"
 654       "Options:\n"
 655       "  -h    This help text\n"
 656       "  --md5 Control MD5 generation\n"
 657       "  -s=?  Source override file\n"
 658       "  -q    Quiet\n"
 659       "  -d=?  Select the optional caching database\n"
 660       "  --no-delink Enable delinking debug mode\n"
 661       "  --contents  Control contents file generation\n"
 662       "  -c=?  Read this configuration file\n"
 663       "  -o=?  Set an arbitrary configuration option") << endl;
 664
 665    return true;
 666 }
 667                                                                         /*}}}*/
 668 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 669 // ---------------------------------------------------------------------
 670 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 671 static bool SimpleGenPackages(CommandLine &CmdL)
 672 {
 673    if (CmdL.FileSize() < 2)
 674       return ShowHelp(CmdL);
 675
 676    string Override;
 677    if (CmdL.FileSize() >= 3)
 678       Override = CmdL.FileList[2];
 679
 680    // Create a package writer object.
 681    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 682                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 683    if (_error->PendingError() == true)
 684       return false;
 685
 686    if (CmdL.FileSize() >= 4)
 687       Packages.PathPrefix = CmdL.FileList[3];
 688
 689    // Do recursive directory searching
 690    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 691       return false;
 692
 693    return true;
 694 }
 695                                                                         /*}}}*/
 696 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 697 // ---------------------------------------------------------------------
 698 /* */
 699 static bool SimpleGenContents(CommandLine &CmdL)
 700 {
 701    if (CmdL.FileSize() < 2)
 702       return ShowHelp(CmdL);
 703
 704    // Create a package writer object.
 705    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 706    if (_error->PendingError() == true)
 707       return false;
 708
 709    // Do recursive directory searching
 710    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 711       return false;
 712
 713    Contents.Finish();
 714
 715    return true;
 716 }
 717                                                                         /*}}}*/
 718 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 719 // ---------------------------------------------------------------------
 720 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 721 static bool SimpleGenSources(CommandLine &CmdL)
 722 {
 723    if (CmdL.FileSize() < 2)
 724       return ShowHelp(CmdL);
 725
 726    string Override;
 727    if (CmdL.FileSize() >= 3)
 728       Override = CmdL.FileList[2];
 729
 730    string SOverride;
 731    if (Override.empty() == false)
 732       SOverride = Override + ".src";
 733
 734    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 735                              SOverride.c_str());
 736
 737    // Create a package writer object.
 738    SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),Override,SOverride);
 739    if (_error->PendingError() == true)
 740       return false;
 741
 742    if (CmdL.FileSize() >= 4)
 743       Sources.PathPrefix = CmdL.FileList[3];
 744
 745    // Do recursive directory searching
 746    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 747       return false;
 748
 749    return true;
 750 }
 751                                                                         /*}}}*/
 752 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 753 // ---------------------------------------------------------------------
 754 static bool SimpleGenRelease(CommandLine &CmdL)
 755 {
 756    if (CmdL.FileSize() < 2)
 757       return ShowHelp(CmdL);
 758
 759    string Dir = CmdL.FileList[1];
 760
 761    ReleaseWriter Release("");
 762    Release.DirStrip = Dir;
 763
 764    if (_error->PendingError() == true)
 765       return false;
 766
 767    if (Release.RecursiveScan(Dir) == false)
 768       return false;
 769
 770    Release.Finish();
 771
 772    return true;
 773 }
 774
 775                                                                         /*}}}*/
 776 // Generate - Full generate, using a config file                        /*{{{*/
 777 // ---------------------------------------------------------------------
 778 /* */
 779 static bool Generate(CommandLine &CmdL)
 780 {
 781    struct CacheDB::Stats SrcStats;
 782    if (CmdL.FileSize() < 2)
 783       return ShowHelp(CmdL);
 784
 785    struct timeval StartTime;
 786    gettimeofday(&StartTime,0);
 787    struct CacheDB::Stats Stats;
 788
 789    // Read the configuration file.
 790    Configuration Setup;
 791    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 792       return false;
 793
 794    vector<PackageMap> PkgList;
 795    LoadTree(PkgList,Setup);
 796    LoadBinDir(PkgList,Setup);
 797
 798    // Sort by cache DB to improve IO locality.
 799    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 800    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 801
 802    // Generate packages
 803    if (CmdL.FileSize() <= 2)
 804    {
 805       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 806          if (I->GenPackages(Setup,Stats) == false)
 807             _error->DumpErrors();
 808       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 809          if (I->GenSources(Setup,SrcStats) == false)
 810             _error->DumpErrors();
 811    }
 812    else
 813    {
 814       // Make a choice list out of the package list..
 815       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 816       RxChoiceList *End = List;
 817       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 818       {
 819          End->UserData = &(*I);
 820          End->Str = I->BaseDir.c_str();
 821          End++;
 822
 823          End->UserData = &(*I);
 824          End->Str = I->Tag.c_str();
 825          End++;
 826       }
 827       End->Str = 0;
 828
 829       // Regex it
 830       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 831       {
 832          delete [] List;
 833          return _error->Error(_("No selections matched"));
 834       }
 835       _error->DumpErrors();
 836
 837       // Do the generation for Packages
 838       for (End = List; End->Str != 0; End++)
 839       {
 840          if (End->Hit == false)
 841             continue;
 842
 843          PackageMap *I = (PackageMap *)End->UserData;
 844          if (I->PkgDone == true)
 845             continue;
 846          if (I->GenPackages(Setup,Stats) == false)
 847             _error->DumpErrors();
 848       }
 849
 850       // Do the generation for Sources
 851       for (End = List; End->Str != 0; End++)
 852       {
 853          if (End->Hit == false)
 854             continue;
 855
 856          PackageMap *I = (PackageMap *)End->UserData;
 857          if (I->SrcDone == true)
 858             continue;
 859          if (I->GenSources(Setup,SrcStats) == false)
 860             _error->DumpErrors();
 861       }
 862
 863       delete [] List;
 864    }
 865
 866    // close the Translation master files
 867    for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I)
 868       if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
 869          delete I->TransWriter;
 870
 871    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 872       return true;
 873
 874    c1out << "Packages done, Starting contents." << endl;
 875
 876    // Sort the contents file list by date
 877    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 878    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 879    {
 880       struct stat A;
 881       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 882                                  I->CntCompress,A) == false)
 883          time(&I->ContentsMTime);
 884       else
 885          I->ContentsMTime = A.st_mtime;
 886    }
 887    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 888
 889    /* Now for Contents.. The process here is to do a make-like dependency
 890       check. Each contents file is verified to be newer than the package files
 891       that describe the debs it indexes. Since the package files contain
 892       hashes of the .debs this means they have not changed either so the
 893       contents must be up to date. */
 894    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 895    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 896    {
 897       // This record is not relevant
 898       if (I->ContentsDone == true ||
 899           I->Contents.empty() == true)
 900          continue;
 901
 902       // Do not do everything if the user specified sections.
 903       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 904          continue;
 905
 906       struct stat A,B;
 907       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 908       {
 909          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 910          {
 911             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 912             continue;
 913          }
 914
 915          if (A.st_mtime > B.st_mtime)
 916             continue;
 917       }
 918
 919       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 920                          MaxContentsChange) == false)
 921          _error->DumpErrors();
 922
 923       // Hit the limit?
 924       if (MaxContentsChange == 0)
 925       {
 926          c1out << "Hit contents update byte limit" << endl;
 927          break;
 928       }
 929    }
 930
 931    struct timeval NewTime;
 932    gettimeofday(&NewTime,0);
 933    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 934                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 935    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 936          << " archives. Took " << TimeToStr((long)Delta) << endl;
 937
 938    return true;
 939 }
 940                                                                         /*}}}*/
 941 // Clean - Clean out the databases                                      /*{{{*/
 942 // ---------------------------------------------------------------------
 943 /* */
 944 static bool Clean(CommandLine &CmdL)
 945 {
 946    if (CmdL.FileSize() != 2)
 947       return ShowHelp(CmdL);
 948
 949    // Read the configuration file.
 950    Configuration Setup;
 951    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 952       return false;
 953
 954    vector<PackageMap> PkgList;
 955    LoadTree(PkgList,Setup);
 956    LoadBinDir(PkgList,Setup);
 957
 958    // Sort by cache DB to improve IO locality.
 959    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 960    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 961
 962    string CacheDir = Setup.FindDir("Dir::CacheDir");
 963
 964    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 965    {
 966       if(I->BinCacheDB != "")
 967          c0out << I->BinCacheDB << endl;
 968       if(I->SrcCacheDB != "")
 969          c0out << I->SrcCacheDB << endl;
 970       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 971       CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));
 972       if (DB.Clean() == false)
 973          _error->DumpErrors();
 974       if (DB_SRC.Clean() == false)
 975          _error->DumpErrors();
 976
 977       string CacheDB = I->BinCacheDB;
 978       string SrcCacheDB = I->SrcCacheDB;
 979       while(I != PkgList.end() &&
 980             I->BinCacheDB == CacheDB &&
 981             I->SrcCacheDB == SrcCacheDB)
 982          ++I;
 983
 984    }
 985
 986
 987    return true;
 988 }
 989                                                                         /*}}}*/
 990
 991 int main(int argc, const char *argv[])
 992 {
 993    setlocale(LC_ALL, "");
 994    CommandLine::Args Args[] = {
 995       {'h',"help","help",0},
 996       {0,"md5","APT::FTPArchive::MD5",0},
 997       {0,"sha1","APT::FTPArchive::SHA1",0},
 998       {0,"sha256","APT::FTPArchive::SHA256",0},
 999       {'v',"version","version",0},
1000       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
1001       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
1002       {'q',"quiet","quiet",CommandLine::IntLevel},
1003       {'q',"silent","quiet",CommandLine::IntLevel},
1004       {0,"delink","APT::FTPArchive::DeLinkAct",0},
1005       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
1006       {0,"contents","APT::FTPArchive::Contents",0},
1007       {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
1008       {'c',"config-file",0,CommandLine::ConfigFile},
1009       {'o',"option",0,CommandLine::ArbItem},
1010       {0,0,0,0}};
1011    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
1012                                    {"contents",&SimpleGenContents},
1013                                    {"sources",&SimpleGenSources},
1014                                    {"release",&SimpleGenRelease},
1015                                    {"generate",&Generate},
1016                                    {"clean",&Clean},
1017                                    {"help",&ShowHelp},
1018                                    {0,0}};
1019
1020    // Parse the command line and initialize the package library
1021    CommandLine CmdL(Args,_config);
1022    if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false)
1023    {
1024       _error->DumpErrors();
1025       return 100;
1026    }
1027
1028    // See if the help should be shown
1029    if (_config->FindB("help") == true ||
1030        _config->FindB("version") == true ||
1031        CmdL.FileSize() == 0)
1032    {
1033       ShowHelp(CmdL);
1034       return 0;
1035    }
1036
1037    // Setup the output streams
1038    c0out.rdbuf(clog.rdbuf());
1039    c1out.rdbuf(clog.rdbuf());
1040    c2out.rdbuf(clog.rdbuf());
1041    Quiet = _config->FindI("quiet",0);
1042    if (Quiet > 0)
1043       c0out.rdbuf(devnull.rdbuf());
1044    if (Quiet > 1)
1045       c1out.rdbuf(devnull.rdbuf());
1046
1047    // Match the operation
1048    CmdL.DispatchArg(Cmds);
1049
1050    if (_error->empty() == false)
1051    {
1052       bool Errors = _error->PendingError();
1053       _error->DumpErrors();
1054       return Errors == true?100:0;
1055    }
1056    return 0;
1057 }