ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <apt-pkg/fileutl.h>
  21
  22 #include <apt-private/private-cmndline.h>
  23 #include <apt-private/private-output.h>
  24
  25 #include <algorithm>
  26 #include <climits>
  27 #include <sys/time.h>
  28 #include <locale.h>
  29 #include <stdio.h>
  30 #include <sys/stat.h>
  31 #include <time.h>
  32 #include <functional>
  33 #include <iostream>
  34 #include <string>
  35 #include <vector>
  36
  37 #include "cachedb.h"
  38 #include "override.h"
  39 #include "apt-ftparchive.h"
  40 #include "multicompress.h"
  41 #include "writer.h"
  42
  43 #include <apti18n.h>
  44                                                                         /*}}}*/
  45
  46 using namespace std;
  47 unsigned Quiet = 0;
  48
  49 // struct PackageMap - List of all package files in the config file     /*{{{*/
  50 // ---------------------------------------------------------------------
  51 /* */
  52 struct PackageMap
  53 {
  54    // General Stuff
  55    string BaseDir;
  56    string InternalPrefix;
  57    string FLFile;
  58    string PkgExt;
  59    string SrcExt;
  60
  61    // Stuff for the Package File
  62    string PkgFile;
  63    string BinCacheDB;
  64    string SrcCacheDB;
  65    string BinOverride;
  66    string ExtraOverride;
  67
  68    // We generate for this given arch
  69    string Arch;
  70
  71    // Stuff for the Source File
  72    string SrcFile;
  73    string SrcOverride;
  74    string SrcExtraOverride;
  75
  76    // Translation master file
  77    bool LongDesc;
  78    TranslationWriter *TransWriter;
  79
  80    // Contents
  81    string Contents;
  82    string ContentsHead;
  83
  84    // Random things
  85    string Tag;
  86    string PkgCompress;
  87    string CntCompress;
  88    string SrcCompress;
  89    string PathPrefix;
  90    unsigned int DeLinkLimit;
  91    mode_t Permissions;
  92
  93    bool ContentsDone;
  94    bool PkgDone;
  95    bool SrcDone;
  96    time_t ContentsMTime;
  97
  98    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  99    {
 100       inline bool operator() (const PackageMap &x,const PackageMap &y)
 101       {return x.ContentsMTime < y.ContentsMTime;};
 102    };
 103
 104    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
 105    {
 106       inline bool operator() (const PackageMap &x,const PackageMap &y)
 107       {return x.BinCacheDB < y.BinCacheDB;};
 108    };
 109
 110    struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
 111    {
 112       inline bool operator() (const PackageMap &x,const PackageMap &y)
 113       {return x.SrcCacheDB < y.SrcCacheDB;};
 114    };
 115
 116    void GetGeneral(Configuration &Setup,Configuration &Block);
 117    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 118    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 119    bool GenContents(Configuration &Setup,
 120                     vector<PackageMap>::iterator Begin,
 121                     vector<PackageMap>::iterator End,
 122                     unsigned long &Left);
 123
 124    PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 125                   ContentsDone(false), PkgDone(false), SrcDone(false),
 126                   ContentsMTime(0) {};
 127 };
 128                                                                         /*}}}*/
 129
 130 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 131 // ---------------------------------------------------------------------
 132 /* */
 133 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 134 {
 135    PathPrefix = Block.Find("PathPrefix");
 136
 137    if (Block.FindB("External-Links",true) == false)
 138       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 139    else
 140       DeLinkLimit = 0;
 141
 142    PkgCompress = Block.Find("Packages::Compress",
 143                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 144    CntCompress = Block.Find("Contents::Compress",
 145                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 146    SrcCompress = Block.Find("Sources::Compress",
 147                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 148
 149    SrcExt = Block.Find("Sources::Extensions",
 150                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 151    PkgExt = Block.Find("Packages::Extensions",
 152                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 153
 154    Permissions = Setup.FindI("Default::FileMode",0644);
 155
 156    if (FLFile.empty() == false)
 157       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 158
 159    if (Contents == " ")
 160       Contents= string();
 161 }
 162                                                                         /*}}}*/
 163 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 164 // ---------------------------------------------------------------------
 165 /* This generates the Package File described by this object. */
 166 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 167 {
 168    if (PkgFile.empty() == true)
 169       return true;
 170
 171    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 172    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 173    string CacheDir = Setup.FindDir("Dir::CacheDir");
 174
 175    struct timeval StartTime;
 176    gettimeofday(&StartTime,0);
 177
 178    PkgDone = true;
 179
 180    // Create a package writer object.
 181    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 182                            flCombine(OverrideDir,BinOverride),
 183                            flCombine(OverrideDir,ExtraOverride),
 184                            Arch);
 185    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 186       return _error->Error(_("Package extension list is too long"));
 187    if (_error->PendingError() == true)
 188       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 189
 190    Packages.PathPrefix = PathPrefix;
 191    Packages.DirStrip = ArchiveDir;
 192    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 193
 194    Packages.TransWriter = TransWriter;
 195    Packages.LongDescription = LongDesc;
 196
 197    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 198    Packages.DeLinkLimit = DeLinkLimit;
 199
 200    // Create a compressor object
 201    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 202                       PkgCompress,Permissions);
 203    Packages.Output = Comp.Input;
 204    if (_error->PendingError() == true)
 205       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 206
 207    c0out << ' ' << BaseDir << ":" << flush;
 208
 209    // Do recursive directory searching
 210    if (FLFile.empty() == true)
 211    {
 212       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 213          return false;
 214    }
 215    else
 216    {
 217       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 218          return false;
 219    }
 220
 221    Packages.Output = 0;      // Just in case
 222
 223    // Finish compressing
 224    unsigned long long Size;
 225    if (Comp.Finalize(Size) == false)
 226    {
 227       c0out << endl;
 228       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 229    }
 230
 231    if (Size != 0)
 232       c0out << " New "
 233              << SizeToStr(Size) << "B ";
 234    else
 235       c0out << ' ';
 236
 237    struct timeval NewTime;
 238    gettimeofday(&NewTime,0);
 239    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 240                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 241
 242    c0out << Packages.Stats.Packages << " files " <<
 243 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 244       SizeToStr(Packages.Stats.Bytes) << "B " <<
 245       TimeToStr((long)Delta) << endl;
 246
 247    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 248      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 249
 250    Stats.Add(Packages.Stats);
 251    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 252
 253    return !_error->PendingError();
 254 }
 255
 256                                                                         /*}}}*/
 257 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 258 // ---------------------------------------------------------------------
 259 /* This generates the Sources File described by this object. */
 260 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 261 {
 262    if (SrcFile.empty() == true)
 263       return true;
 264
 265    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 266    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 267    string CacheDir = Setup.FindDir("Dir::CacheDir");
 268
 269    struct timeval StartTime;
 270    gettimeofday(&StartTime,0);
 271
 272    SrcDone = true;
 273
 274    // Create a package writer object.
 275    SourcesWriter Sources(flCombine(CacheDir, SrcCacheDB),
 276                          flCombine(OverrideDir,BinOverride),
 277                          flCombine(OverrideDir,SrcOverride),
 278                          flCombine(OverrideDir,SrcExtraOverride));
 279    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 280       return _error->Error(_("Source extension list is too long"));
 281    if (_error->PendingError() == true)
 282       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 283
 284    Sources.PathPrefix = PathPrefix;
 285    Sources.DirStrip = ArchiveDir;
 286    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 287
 288    Sources.DeLinkLimit = DeLinkLimit;
 289    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 290
 291    // Create a compressor object
 292    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 293                       SrcCompress,Permissions);
 294    Sources.Output = Comp.Input;
 295    if (_error->PendingError() == true)
 296       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 297
 298    c0out << ' ' << BaseDir << ":" << flush;
 299
 300    // Do recursive directory searching
 301    if (FLFile.empty() == true)
 302    {
 303       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 304          return false;
 305    }
 306    else
 307    {
 308       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 309          return false;
 310    }
 311    Sources.Output = 0;      // Just in case
 312
 313    // Finish compressing
 314    unsigned long long Size;
 315    if (Comp.Finalize(Size) == false)
 316    {
 317       c0out << endl;
 318       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 319    }
 320
 321    if (Size != 0)
 322       c0out << " New "
 323              << SizeToStr(Size) << "B ";
 324    else
 325       c0out << ' ';
 326
 327    struct timeval NewTime;
 328    gettimeofday(&NewTime,0);
 329    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 330                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 331
 332    c0out << Sources.Stats.Packages << " pkgs in " <<
 333       TimeToStr((long)Delta) << endl;
 334
 335    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 336      c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
 337
 338    Stats.Add(Sources.Stats);
 339    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 340
 341    return !_error->PendingError();
 342 }
 343                                                                         /*}}}*/
 344 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 345 // ---------------------------------------------------------------------
 346 /* This generates the contents file partially described by this object.
 347    It searches the given iterator range for other package files that map
 348    into this contents file and includes their data as well when building. */
 349 bool PackageMap::GenContents(Configuration &Setup,
 350                              vector<PackageMap>::iterator Begin,
 351                              vector<PackageMap>::iterator End,
 352                              unsigned long &Left)
 353 {
 354    if (Contents.empty() == true)
 355       return true;
 356
 357    if (Left == 0)
 358       return true;
 359
 360    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 361    string CacheDir = Setup.FindDir("Dir::CacheDir");
 362    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 363
 364    struct timeval StartTime;
 365    gettimeofday(&StartTime,0);
 366
 367    // Create a package writer object.
 368    ContentsWriter Contents("", Arch);
 369    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 370       return _error->Error(_("Package extension list is too long"));
 371    if (_error->PendingError() == true)
 372       return false;
 373
 374    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 375                       CntCompress,Permissions);
 376    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 377    Contents.Output = Comp.Input;
 378    if (_error->PendingError() == true)
 379       return false;
 380
 381    // Write the header out.
 382    if (ContentsHead.empty() == false)
 383    {
 384       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 385       if (_error->PendingError() == true)
 386          return false;
 387
 388       unsigned long long Size = Head.Size();
 389       unsigned char Buf[4096];
 390       while (Size != 0)
 391       {
 392          unsigned long long ToRead = Size;
 393          if (Size > sizeof(Buf))
 394             ToRead = sizeof(Buf);
 395
 396          if (Head.Read(Buf,ToRead) == false)
 397             return false;
 398
 399          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 400             return _error->Errno("fwrite",_("Error writing header to contents file"));
 401
 402          Size -= ToRead;
 403       }
 404    }
 405
 406    /* Go over all the package file records and parse all the package
 407       files associated with this contents file into one great big honking
 408       memory structure, then dump the sorted version */
 409    c0out << ' ' << this->Contents << ":" << flush;
 410    for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
 411    {
 412       if (I->Contents != this->Contents)
 413          continue;
 414
 415       Contents.Prefix = ArchiveDir;
 416       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 417       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 418                             I->PkgCompress);
 419
 420       I->ContentsDone = true;
 421    }
 422
 423    Contents.Finish();
 424
 425    // Finish compressing
 426    unsigned long long Size;
 427    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 428    {
 429       c0out << endl;
 430       return _error->Error(_("Error processing contents %s"),
 431                            this->Contents.c_str());
 432    }
 433
 434    if (Size != 0)
 435    {
 436       c0out << " New " << SizeToStr(Size) << "B ";
 437       if (Left > Size)
 438          Left -= Size;
 439       else
 440          Left = 0;
 441    }
 442    else
 443       c0out << ' ';
 444
 445    struct timeval NewTime;
 446    gettimeofday(&NewTime,0);
 447    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 448                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 449
 450    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 451      c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
 452
 453    c0out << Contents.Stats.Packages << " files " <<
 454       SizeToStr(Contents.Stats.Bytes) << "B " <<
 455       TimeToStr((long)Delta) << endl;
 456
 457    return true;
 458 }
 459                                                                         /*}}}*/
 460
 461 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 462 // ---------------------------------------------------------------------
 463 /* This populates the PkgList with all the possible permutations of the
 464    section/arch lists. */
 465 static void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 466 {
 467    // Load the defaults
 468    string DDir = Setup.Find("TreeDefault::Directory",
 469                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 470    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 471                             "$(DIST)/$(SECTION)/source/");
 472    string DPkg = Setup.Find("TreeDefault::Packages",
 473                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 474    string DTrans = Setup.Find("TreeDefault::Translation",
 475                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 476    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 477                             "$(DIST)/$(SECTION)/");
 478    string DContents = Setup.Find("TreeDefault::Contents",
 479                             "$(DIST)/$(SECTION)/Contents-$(ARCH)");
 480    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 481    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 482                                "packages-$(ARCH).db");
 483    string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
 484                                "sources-$(SECTION).db");
 485    string DSources = Setup.Find("TreeDefault::Sources",
 486                                 "$(DIST)/$(SECTION)/source/Sources");
 487    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 488    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 489
 490    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 491
 492    bool const LongDescription = Setup.FindB("Default::LongDescription",
 493                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 494    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 495
 496    // Process 'tree' type sections
 497    const Configuration::Item *Top = Setup.Tree("tree");
 498    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 499    {
 500       Configuration Block(Top);
 501       string Dist = Top->Tag;
 502
 503       // Parse the sections
 504       string Tmp = Block.Find("Sections");
 505       const char *Sections = Tmp.c_str();
 506       string Section;
 507       while (ParseQuoteWord(Sections,Section) == true)
 508       {
 509          string Arch;
 510          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 511                                          {"$(SECTION)",&Section},
 512                                          {"$(ARCH)",&Arch},
 513                                          {NULL, NULL}};
 514          mode_t const Perms = Block.FindI("FileMode", Permissions);
 515          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 516          TranslationWriter *TransWriter;
 517          if (DTrans.empty() == false && LongDesc == false)
 518          {
 519             string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 520                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 521             string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 522             TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 523          }
 524          else
 525             TransWriter = NULL;
 526
 527          string const Tmp2 = Block.Find("Architectures");
 528          const char *Archs = Tmp2.c_str();
 529          while (ParseQuoteWord(Archs,Arch) == true)
 530          {
 531             PackageMap Itm;
 532             Itm.Permissions = Perms;
 533             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 534             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 535
 536             if (stringcasecmp(Arch,"source") == 0)
 537             {
 538                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 539                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 540                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 541                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 542                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 543                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 544                Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);
 545             }
 546             else
 547             {
 548                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 549                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 550                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 551                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 552                Itm.Arch = Arch;
 553                Itm.LongDesc = LongDesc;
 554                if (TransWriter != NULL)
 555                {
 556                   TransWriter->IncreaseRefCounter();
 557                   Itm.TransWriter = TransWriter;
 558                }
 559                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 560                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 561                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 562                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 563             }
 564
 565             Itm.GetGeneral(Setup,Block);
 566             PkgList.push_back(Itm);
 567          }
 568          // we didn't use this TransWriter, so we can release it
 569          if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
 570             delete TransWriter;
 571       }
 572
 573       Top = Top->Next;
 574    }
 575 }
 576                                                                         /*}}}*/
 577 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 578 // ---------------------------------------------------------------------
 579 /* */
 580 static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 581 {
 582    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 583
 584    // Process 'bindirectory' type sections
 585    const Configuration::Item *Top = Setup.Tree("bindirectory");
 586    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 587    {
 588       Configuration Block(Top);
 589
 590       PackageMap Itm;
 591       Itm.PkgFile = Block.Find("Packages");
 592       Itm.SrcFile = Block.Find("Sources");
 593       Itm.BinCacheDB = Block.Find("BinCacheDB");
 594       Itm.SrcCacheDB = Block.Find("SrcCacheDB");
 595       Itm.BinOverride = Block.Find("BinOverride");
 596       Itm.ExtraOverride = Block.Find("ExtraOverride");
 597       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 598       Itm.SrcOverride = Block.Find("SrcOverride");
 599       Itm.BaseDir = Top->Tag;
 600       Itm.FLFile = Block.Find("FileList");
 601       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 602       Itm.Contents = Block.Find("Contents");
 603       Itm.ContentsHead = Block.Find("Contents::Header");
 604       Itm.Permissions = Block.FindI("FileMode", Permissions);
 605
 606       Itm.GetGeneral(Setup,Block);
 607       PkgList.push_back(Itm);
 608
 609       Top = Top->Next;
 610    }
 611 }
 612                                                                         /*}}}*/
 613
 614 // ShowHelp - Show the help text                                        /*{{{*/
 615 // ---------------------------------------------------------------------
 616 /* */
 617 static bool ShowHelp(CommandLine &)
 618 {
 619    ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
 620             COMMON_ARCH,__DATE__,__TIME__);
 621    if (_config->FindB("version") == true)
 622       return true;
 623
 624    cout <<
 625     _("Usage: apt-ftparchive [options] command\n"
 626       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 627       "          sources srcpath [overridefile [pathprefix]]\n"
 628       "          contents path\n"
 629       "          release path\n"
 630       "          generate config [groups]\n"
 631       "          clean config\n"
 632       "\n"
 633       "apt-ftparchive generates index files for Debian archives. It supports\n"
 634       "many styles of generation from fully automated to functional replacements\n"
 635       "for dpkg-scanpackages and dpkg-scansources\n"
 636       "\n"
 637       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 638       "Package file contains the contents of all the control fields from\n"
 639       "each package as well as the MD5 hash and filesize. An override file\n"
 640       "is supported to force the value of Priority and Section.\n"
 641       "\n"
 642       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 643       "The --source-override option can be used to specify a src override file\n"
 644       "\n"
 645       "The 'packages' and 'sources' command should be run in the root of the\n"
 646       "tree. BinaryPath should point to the base of the recursive search and \n"
 647       "override file should contain the override flags. Pathprefix is\n"
 648       "appended to the filename fields if present. Example usage from the \n"
 649       "Debian archive:\n"
 650       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 651       "               dists/potato/main/binary-i386/Packages\n"
 652       "\n"
 653       "Options:\n"
 654       "  -h    This help text\n"
 655       "  --md5 Control MD5 generation\n"
 656       "  -s=?  Source override file\n"
 657       "  -q    Quiet\n"
 658       "  -d=?  Select the optional caching database\n"
 659       "  --no-delink Enable delinking debug mode\n"
 660       "  --contents  Control contents file generation\n"
 661       "  -c=?  Read this configuration file\n"
 662       "  -o=?  Set an arbitrary configuration option") << endl;
 663
 664    return true;
 665 }
 666                                                                         /*}}}*/
 667 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 668 // ---------------------------------------------------------------------
 669 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 670 static bool SimpleGenPackages(CommandLine &CmdL)
 671 {
 672    if (CmdL.FileSize() < 2)
 673       return ShowHelp(CmdL);
 674
 675    string Override;
 676    if (CmdL.FileSize() >= 3)
 677       Override = CmdL.FileList[2];
 678
 679    // Create a package writer object.
 680    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 681                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 682    if (_error->PendingError() == true)
 683       return false;
 684
 685    if (CmdL.FileSize() >= 4)
 686       Packages.PathPrefix = CmdL.FileList[3];
 687
 688    // Do recursive directory searching
 689    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 690       return false;
 691
 692    // Give some stats if asked for
 693    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 694      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 695
 696    return true;
 697 }
 698                                                                         /*}}}*/
 699 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 700 // ---------------------------------------------------------------------
 701 /* */
 702 static bool SimpleGenContents(CommandLine &CmdL)
 703 {
 704    if (CmdL.FileSize() < 2)
 705       return ShowHelp(CmdL);
 706
 707    // Create a package writer object.
 708    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 709    if (_error->PendingError() == true)
 710       return false;
 711
 712    // Do recursive directory searching
 713    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 714       return false;
 715
 716    Contents.Finish();
 717
 718    return true;
 719 }
 720                                                                         /*}}}*/
 721 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 722 // ---------------------------------------------------------------------
 723 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 724 static bool SimpleGenSources(CommandLine &CmdL)
 725 {
 726    if (CmdL.FileSize() < 2)
 727       return ShowHelp(CmdL);
 728
 729    string Override;
 730    if (CmdL.FileSize() >= 3)
 731       Override = CmdL.FileList[2];
 732
 733    string SOverride;
 734    if (Override.empty() == false)
 735       SOverride = Override + ".src";
 736
 737    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 738                              SOverride.c_str());
 739
 740    // Create a package writer object.
 741    SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),Override,SOverride);
 742    if (_error->PendingError() == true)
 743       return false;
 744
 745    if (CmdL.FileSize() >= 4)
 746       Sources.PathPrefix = CmdL.FileList[3];
 747
 748    // Do recursive directory searching
 749    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 750       return false;
 751
 752    // Give some stats if asked for
 753    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 754      c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl;
 755
 756    return true;
 757 }
 758                                                                         /*}}}*/
 759 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 760 // ---------------------------------------------------------------------
 761 static bool SimpleGenRelease(CommandLine &CmdL)
 762 {
 763    if (CmdL.FileSize() < 2)
 764       return ShowHelp(CmdL);
 765
 766    string Dir = CmdL.FileList[1];
 767
 768    ReleaseWriter Release("");
 769    Release.DirStrip = Dir;
 770
 771    if (_error->PendingError() == true)
 772       return false;
 773
 774    if (Release.RecursiveScan(Dir) == false)
 775       return false;
 776
 777    Release.Finish();
 778
 779    return true;
 780 }
 781
 782                                                                         /*}}}*/
 783 // DoGeneratePackagesAndSources - Helper for Generate                   /*{{{*/
 784 // ---------------------------------------------------------------------
 785 static bool DoGeneratePackagesAndSources(Configuration &Setup,
 786                                          vector<PackageMap> &PkgList,
 787                                          struct CacheDB::Stats &SrcStats,
 788                                          struct CacheDB::Stats &Stats,
 789                                          CommandLine &CmdL)
 790 {
 791    if (CmdL.FileSize() <= 2)
 792    {
 793       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 794          if (I->GenPackages(Setup,Stats) == false)
 795             _error->DumpErrors();
 796       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 797          if (I->GenSources(Setup,SrcStats) == false)
 798             _error->DumpErrors();
 799    }
 800    else
 801    {
 802       // Make a choice list out of the package list..
 803       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 804       RxChoiceList *End = List;
 805       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 806       {
 807          End->UserData = &(*I);
 808          End->Str = I->BaseDir.c_str();
 809          End++;
 810
 811          End->UserData = &(*I);
 812          End->Str = I->Tag.c_str();
 813          End++;
 814       }
 815       End->Str = 0;
 816
 817       // Regex it
 818       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 819       {
 820          delete [] List;
 821          return _error->Error(_("No selections matched"));
 822       }
 823       _error->DumpErrors();
 824
 825       // Do the generation for Packages
 826       for (End = List; End->Str != 0; End++)
 827       {
 828          if (End->Hit == false)
 829             continue;
 830
 831          PackageMap *I = (PackageMap *)End->UserData;
 832          if (I->PkgDone == true)
 833             continue;
 834          if (I->GenPackages(Setup,Stats) == false)
 835             _error->DumpErrors();
 836       }
 837
 838       // Do the generation for Sources
 839       for (End = List; End->Str != 0; End++)
 840       {
 841          if (End->Hit == false)
 842             continue;
 843
 844          PackageMap *I = (PackageMap *)End->UserData;
 845          if (I->SrcDone == true)
 846             continue;
 847          if (I->GenSources(Setup,SrcStats) == false)
 848             _error->DumpErrors();
 849       }
 850
 851       delete [] List;
 852    }
 853
 854    // close the Translation master files
 855    for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I)
 856       if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
 857          delete I->TransWriter;
 858
 859    return true;
 860 }
 861
 862                                                                         /*}}}*/
 863 // DoGenerateContents - Helper for Generate to generate the Contents    /*{{{*/
 864 // ---------------------------------------------------------------------
 865 static bool DoGenerateContents(Configuration &Setup,
 866                                vector<PackageMap> &PkgList,
 867                                CommandLine &CmdL)
 868 {
 869    c1out << "Packages done, Starting contents." << endl;
 870
 871    // Sort the contents file list by date
 872    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 873    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 874    {
 875       struct stat A;
 876       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 877                                  I->CntCompress,A) == false)
 878          time(&I->ContentsMTime);
 879       else
 880          I->ContentsMTime = A.st_mtime;
 881    }
 882    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 883
 884    /* Now for Contents.. The process here is to do a make-like dependency
 885       check. Each contents file is verified to be newer than the package files
 886       that describe the debs it indexes. Since the package files contain
 887       hashes of the .debs this means they have not changed either so the
 888       contents must be up to date. */
 889    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 890    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 891    {
 892       // This record is not relevant
 893       if (I->ContentsDone == true ||
 894           I->Contents.empty() == true)
 895          continue;
 896
 897       // Do not do everything if the user specified sections.
 898       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 899          continue;
 900
 901       struct stat A,B;
 902       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 903       {
 904          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 905          {
 906             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 907             continue;
 908          }
 909
 910          if (A.st_mtime > B.st_mtime)
 911             continue;
 912       }
 913
 914       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 915                          MaxContentsChange) == false)
 916          _error->DumpErrors();
 917
 918       // Hit the limit?
 919       if (MaxContentsChange == 0)
 920       {
 921          c1out << "Hit contents update byte limit" << endl;
 922          break;
 923       }
 924    }
 925
 926    return true;
 927 }
 928
 929                                                                         /*}}}*/
 930 // Generate - Full generate, using a config file                        /*{{{*/
 931 // ---------------------------------------------------------------------
 932 /* */
 933 static bool Generate(CommandLine &CmdL)
 934 {
 935    struct CacheDB::Stats SrcStats;
 936    if (CmdL.FileSize() < 2)
 937       return ShowHelp(CmdL);
 938
 939    struct timeval StartTime;
 940    gettimeofday(&StartTime,0);
 941    struct CacheDB::Stats Stats;
 942
 943    // Read the configuration file.
 944    Configuration Setup;
 945    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 946       return false;
 947
 948    vector<PackageMap> PkgList;
 949    LoadTree(PkgList,Setup);
 950    LoadBinDir(PkgList,Setup);
 951
 952    // Sort by cache DB to improve IO locality.
 953    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 954    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 955
 956    // Generate packages
 957    if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false)
 958    {
 959       if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false)
 960          return false;
 961    } else {
 962       c1out << "Skipping Packages/Sources generation" << endl;
 963    }
 964
 965    // do Contents if needed
 966    if (_config->FindB("APT::FTPArchive::Contents", true) == true)
 967       if (DoGenerateContents(Setup, PkgList, CmdL) == false)
 968          return false;
 969
 970    struct timeval NewTime;
 971    gettimeofday(&NewTime,0);
 972    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 973                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 974    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 975          << " archives. Took " << TimeToStr((long)Delta) << endl;
 976
 977    return true;
 978 }
 979
 980                                                                         /*}}}*/
 981 // Clean - Clean out the databases                                      /*{{{*/
 982 // ---------------------------------------------------------------------
 983 /* */
 984 static bool Clean(CommandLine &CmdL)
 985 {
 986    if (CmdL.FileSize() != 2)
 987       return ShowHelp(CmdL);
 988
 989    // Read the configuration file.
 990    Configuration Setup;
 991    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 992       return false;
 993
 994    vector<PackageMap> PkgList;
 995    LoadTree(PkgList,Setup);
 996    LoadBinDir(PkgList,Setup);
 997
 998    // Sort by cache DB to improve IO locality.
 999    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
1000    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
1001
1002    string CacheDir = Setup.FindDir("Dir::CacheDir");
1003
1004    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
1005    {
1006       if(I->BinCacheDB != "")
1007          c0out << I->BinCacheDB << endl;
1008       if(I->SrcCacheDB != "")
1009          c0out << I->SrcCacheDB << endl;
1010       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
1011       CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));
1012       if (DB.Clean() == false)
1013          _error->DumpErrors();
1014       if (DB_SRC.Clean() == false)
1015          _error->DumpErrors();
1016
1017       string CacheDB = I->BinCacheDB;
1018       string SrcCacheDB = I->SrcCacheDB;
1019       while(I != PkgList.end() &&
1020             I->BinCacheDB == CacheDB &&
1021             I->SrcCacheDB == SrcCacheDB)
1022          ++I;
1023
1024    }
1025
1026
1027    return true;
1028 }
1029                                                                         /*}}}*/
1030
1031 int main(int argc, const char *argv[])
1032 {
1033    setlocale(LC_ALL, "");
1034    CommandLine::Args Args[] = {
1035       {'h',"help","help",0},
1036       {0,"md5","APT::FTPArchive::MD5",0},
1037       {0,"sha1","APT::FTPArchive::SHA1",0},
1038       {0,"sha256","APT::FTPArchive::SHA256",0},
1039       {'v',"version","version",0},
1040       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
1041       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
1042       {'q',"quiet","quiet",CommandLine::IntLevel},
1043       {'q',"silent","quiet",CommandLine::IntLevel},
1044       {0,"delink","APT::FTPArchive::DeLinkAct",0},
1045       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
1046       {0,"contents","APT::FTPArchive::Contents",0},
1047       {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
1048       {'c',"config-file",0,CommandLine::ConfigFile},
1049       {'o',"option",0,CommandLine::ArbItem},
1050       {0,0,0,0}};
1051    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
1052                                    {"contents",&SimpleGenContents},
1053                                    {"sources",&SimpleGenSources},
1054                                    {"release",&SimpleGenRelease},
1055                                    {"generate",&Generate},
1056                                    {"clean",&Clean},
1057                                    {"help",&ShowHelp},
1058                                    {0,0}};
1059
1060    // Parse the command line and initialize the package library
1061    CommandLine CmdL(Args,_config);
1062    ParseCommandLine(CmdL, Cmds, Args, &_config, NULL, argc, argv, ShowHelp);
1063
1064    _config->CndSet("quiet",0);
1065    Quiet = _config->FindI("quiet",0);
1066    InitOutput(clog.rdbuf());
1067
1068    // Match the operation
1069    CmdL.DispatchArg(Cmds);
1070
1071    if (_error->empty() == false)
1072    {
1073       bool Errors = _error->PendingError();
1074       _error->DumpErrors();
1075       return Errors == true?100:0;
1076    }
1077    return 0;
1078 }