ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <apt-pkg/fileutl.h>
  21
  22 #include <apt-private/private-cmndline.h>
  23 #include <apt-private/private-output.h>
  24
  25 #include <algorithm>
  26 #include <climits>
  27 #include <sys/time.h>
  28 #include <locale.h>
  29 #include <stdio.h>
  30 #include <sys/stat.h>
  31 #include <time.h>
  32 #include <functional>
  33 #include <iostream>
  34 #include <string>
  35 #include <vector>
  36
  37 #include "cachedb.h"
  38 #include "override.h"
  39 #include "apt-ftparchive.h"
  40 #include "multicompress.h"
  41 #include "writer.h"
  42
  43 #include <apti18n.h>
  44                                                                         /*}}}*/
  45
  46 using namespace std;
  47 unsigned Quiet = 0;
  48
  49 // struct PackageMap - List of all package files in the config file     /*{{{*/
  50 // ---------------------------------------------------------------------
  51 /* */
  52 struct PackageMap
  53 {
  54    // General Stuff
  55    string BaseDir;
  56    string InternalPrefix;
  57    string FLFile;
  58    string PkgExt;
  59    string SrcExt;
  60
  61    // Stuff for the Package File
  62    string PkgFile;
  63    string BinCacheDB;
  64    string SrcCacheDB;
  65    string BinOverride;
  66    string ExtraOverride;
  67
  68    // We generate for this given arch
  69    string Arch;
  70
  71    // Stuff for the Source File
  72    string SrcFile;
  73    string SrcOverride;
  74    string SrcExtraOverride;
  75
  76    // Translation master file
  77    bool LongDesc;
  78    TranslationWriter *TransWriter;
  79
  80    // Contents
  81    string Contents;
  82    string ContentsHead;
  83
  84    // Random things
  85    string Tag;
  86    string PkgCompress;
  87    string CntCompress;
  88    string SrcCompress;
  89    string PathPrefix;
  90    unsigned int DeLinkLimit;
  91    mode_t Permissions;
  92
  93    bool ContentsDone;
  94    bool PkgDone;
  95    bool SrcDone;
  96    time_t ContentsMTime;
  97
  98    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  99    {
 100       inline bool operator() (const PackageMap &x,const PackageMap &y)
 101       {return x.ContentsMTime < y.ContentsMTime;};
 102    };
 103
 104    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
 105    {
 106       inline bool operator() (const PackageMap &x,const PackageMap &y)
 107       {return x.BinCacheDB < y.BinCacheDB;};
 108    };
 109
 110    struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
 111    {
 112       inline bool operator() (const PackageMap &x,const PackageMap &y)
 113       {return x.SrcCacheDB < y.SrcCacheDB;};
 114    };
 115
 116    void GetGeneral(Configuration &Setup,Configuration &Block);
 117    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 118    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 119    bool GenContents(Configuration &Setup,
 120                     vector<PackageMap>::iterator Begin,
 121                     vector<PackageMap>::iterator End,
 122                     unsigned long &Left);
 123
 124    PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 125                   ContentsDone(false), PkgDone(false), SrcDone(false),
 126                   ContentsMTime(0) {};
 127 };
 128                                                                         /*}}}*/
 129
 130 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 131 // ---------------------------------------------------------------------
 132 /* */
 133 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 134 {
 135    PathPrefix = Block.Find("PathPrefix");
 136
 137    if (Block.FindB("External-Links",true) == false)
 138       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 139    else
 140       DeLinkLimit = 0;
 141
 142    PkgCompress = Block.Find("Packages::Compress",
 143                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 144    CntCompress = Block.Find("Contents::Compress",
 145                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 146    SrcCompress = Block.Find("Sources::Compress",
 147                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 148
 149    SrcExt = Block.Find("Sources::Extensions",
 150                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 151    PkgExt = Block.Find("Packages::Extensions",
 152                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 153
 154    Permissions = Setup.FindI("Default::FileMode",0644);
 155
 156    if (FLFile.empty() == false)
 157       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 158
 159    if (Contents == " ")
 160       Contents= string();
 161 }
 162                                                                         /*}}}*/
 163 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 164 // ---------------------------------------------------------------------
 165 /* This generates the Package File described by this object. */
 166 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 167 {
 168    if (PkgFile.empty() == true)
 169       return true;
 170
 171    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 172    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 173    string CacheDir = Setup.FindDir("Dir::CacheDir");
 174
 175    struct timeval StartTime;
 176    gettimeofday(&StartTime,0);
 177
 178    PkgDone = true;
 179
 180    // Create a package writer object.
 181    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 182                            flCombine(OverrideDir,BinOverride),
 183                            flCombine(OverrideDir,ExtraOverride),
 184                            Arch);
 185    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 186       return _error->Error(_("Package extension list is too long"));
 187    if (_error->PendingError() == true)
 188       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 189
 190    Packages.PathPrefix = PathPrefix;
 191    Packages.DirStrip = ArchiveDir;
 192    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 193
 194    Packages.TransWriter = TransWriter;
 195    Packages.LongDescription = LongDesc;
 196
 197    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 198    Packages.DeLinkLimit = DeLinkLimit;
 199
 200    // Create a compressor object
 201    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 202                       PkgCompress,Permissions);
 203    Packages.Output = Comp.Input;
 204    if (_error->PendingError() == true)
 205       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 206
 207    c0out << ' ' << BaseDir << ":" << flush;
 208
 209    // Do recursive directory searching
 210    if (FLFile.empty() == true)
 211    {
 212       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 213          return false;
 214    }
 215    else
 216    {
 217       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 218          return false;
 219    }
 220
 221    Packages.Output = 0;      // Just in case
 222
 223    // Finish compressing
 224    unsigned long long Size;
 225    if (Comp.Finalize(Size) == false)
 226    {
 227       c0out << endl;
 228       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 229    }
 230
 231    if (Size != 0)
 232       c0out << " New "
 233              << SizeToStr(Size) << "B ";
 234    else
 235       c0out << ' ';
 236
 237    struct timeval NewTime;
 238    gettimeofday(&NewTime,0);
 239    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 240                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 241
 242    c0out << Packages.Stats.Packages << " files " <<
 243 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 244       SizeToStr(Packages.Stats.Bytes) << "B " <<
 245       TimeToStr((long)Delta) << endl;
 246
 247    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 248      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 249
 250    Stats.Add(Packages.Stats);
 251    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 252
 253    return !_error->PendingError();
 254 }
 255
 256                                                                         /*}}}*/
 257 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 258 // ---------------------------------------------------------------------
 259 /* This generates the Sources File described by this object. */
 260 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 261 {
 262    if (SrcFile.empty() == true)
 263       return true;
 264
 265    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 266    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 267    string CacheDir = Setup.FindDir("Dir::CacheDir");
 268
 269    struct timeval StartTime;
 270    gettimeofday(&StartTime,0);
 271
 272    SrcDone = true;
 273
 274    // Create a package writer object.
 275    SourcesWriter Sources(flCombine(CacheDir, SrcCacheDB),
 276                          flCombine(OverrideDir,BinOverride),
 277                          flCombine(OverrideDir,SrcOverride),
 278                          flCombine(OverrideDir,SrcExtraOverride));
 279    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 280       return _error->Error(_("Source extension list is too long"));
 281    if (_error->PendingError() == true)
 282       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 283
 284    Sources.PathPrefix = PathPrefix;
 285    Sources.DirStrip = ArchiveDir;
 286    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 287
 288    Sources.DeLinkLimit = DeLinkLimit;
 289    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 290
 291    // Create a compressor object
 292    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 293                       SrcCompress,Permissions);
 294    Sources.Output = Comp.Input;
 295    if (_error->PendingError() == true)
 296       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 297
 298    c0out << ' ' << BaseDir << ":" << flush;
 299
 300    // Do recursive directory searching
 301    if (FLFile.empty() == true)
 302    {
 303       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 304          return false;
 305    }
 306    else
 307    {
 308       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 309          return false;
 310    }
 311    Sources.Output = 0;      // Just in case
 312
 313    // Finish compressing
 314    unsigned long long Size;
 315    if (Comp.Finalize(Size) == false)
 316    {
 317       c0out << endl;
 318       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 319    }
 320
 321    if (Size != 0)
 322       c0out << " New "
 323              << SizeToStr(Size) << "B ";
 324    else
 325       c0out << ' ';
 326
 327    struct timeval NewTime;
 328    gettimeofday(&NewTime,0);
 329    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 330                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 331
 332    c0out << Sources.Stats.Packages << " pkgs in " <<
 333       TimeToStr((long)Delta) << endl;
 334
 335    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 336      c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
 337
 338    Stats.Add(Sources.Stats);
 339    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 340
 341    return !_error->PendingError();
 342 }
 343                                                                         /*}}}*/
 344 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 345 // ---------------------------------------------------------------------
 346 /* This generates the contents file partially described by this object.
 347    It searches the given iterator range for other package files that map
 348    into this contents file and includes their data as well when building. */
 349 bool PackageMap::GenContents(Configuration &Setup,
 350                              vector<PackageMap>::iterator Begin,
 351                              vector<PackageMap>::iterator End,
 352                              unsigned long &Left)
 353 {
 354    if (Contents.empty() == true)
 355       return true;
 356
 357    if (Left == 0)
 358       return true;
 359
 360    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 361    string CacheDir = Setup.FindDir("Dir::CacheDir");
 362    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 363
 364    struct timeval StartTime;
 365    gettimeofday(&StartTime,0);
 366
 367    // Create a package writer object.
 368    ContentsWriter Contents("", Arch);
 369    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 370       return _error->Error(_("Package extension list is too long"));
 371    if (_error->PendingError() == true)
 372       return false;
 373
 374    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 375                       CntCompress,Permissions);
 376    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 377    Contents.Output = Comp.Input;
 378    if (_error->PendingError() == true)
 379       return false;
 380
 381    // Write the header out.
 382    if (ContentsHead.empty() == false)
 383    {
 384       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 385       if (_error->PendingError() == true)
 386          return false;
 387
 388       unsigned long long Size = Head.Size();
 389       unsigned char Buf[4096];
 390       while (Size != 0)
 391       {
 392          unsigned long long ToRead = Size;
 393          if (Size > sizeof(Buf))
 394             ToRead = sizeof(Buf);
 395
 396          if (Head.Read(Buf,ToRead) == false)
 397             return false;
 398
 399          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 400             return _error->Errno("fwrite",_("Error writing header to contents file"));
 401
 402          Size -= ToRead;
 403       }
 404    }
 405
 406    /* Go over all the package file records and parse all the package
 407       files associated with this contents file into one great big honking
 408       memory structure, then dump the sorted version */
 409    c0out << ' ' << this->Contents << ":" << flush;
 410    for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
 411    {
 412       if (I->Contents != this->Contents)
 413          continue;
 414
 415       Contents.Prefix = ArchiveDir;
 416       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 417       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 418                             I->PkgCompress);
 419
 420       I->ContentsDone = true;
 421    }
 422
 423    Contents.Finish();
 424
 425    // Finish compressing
 426    unsigned long long Size;
 427    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 428    {
 429       c0out << endl;
 430       return _error->Error(_("Error processing contents %s"),
 431                            this->Contents.c_str());
 432    }
 433
 434    if (Size != 0)
 435    {
 436       c0out << " New " << SizeToStr(Size) << "B ";
 437       if (Left > Size)
 438          Left -= Size;
 439       else
 440          Left = 0;
 441    }
 442    else
 443       c0out << ' ';
 444
 445    struct timeval NewTime;
 446    gettimeofday(&NewTime,0);
 447    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 448                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 449
 450    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 451      c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
 452
 453    c0out << Contents.Stats.Packages << " files " <<
 454       SizeToStr(Contents.Stats.Bytes) << "B " <<
 455       TimeToStr((long)Delta) << endl;
 456
 457    return true;
 458 }
 459                                                                         /*}}}*/
 460
 461 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 462 // ---------------------------------------------------------------------
 463 /* This populates the PkgList with all the possible permutations of the
 464    section/arch lists. */
 465 static void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 466 {
 467    // Load the defaults
 468    string DDir = Setup.Find("TreeDefault::Directory",
 469                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 470    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 471                             "$(DIST)/$(SECTION)/source/");
 472    string DPkg = Setup.Find("TreeDefault::Packages",
 473                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 474    string DTrans = Setup.Find("TreeDefault::Translation",
 475                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 476    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 477                             "$(DIST)/$(SECTION)/");
 478    string DContents = Setup.Find("TreeDefault::Contents",
 479                             "$(DIST)/$(SECTION)/Contents-$(ARCH)");
 480    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 481    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 482                                "packages-$(ARCH).db");
 483    string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
 484                                "sources-$(SECTION).db");
 485    string DSources = Setup.Find("TreeDefault::Sources",
 486                                 "$(DIST)/$(SECTION)/source/Sources");
 487    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 488    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 489
 490    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 491
 492    bool const LongDescription = Setup.FindB("Default::LongDescription",
 493                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 494    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 495
 496    // Process 'tree' type sections
 497    const Configuration::Item *Top = Setup.Tree("tree");
 498    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 499    {
 500       Configuration Block(Top);
 501       string Dist = Top->Tag;
 502
 503       // Parse the sections
 504       string Tmp = Block.Find("Sections");
 505       const char *Sections = Tmp.c_str();
 506       string Section;
 507       while (ParseQuoteWord(Sections,Section) == true)
 508       {
 509          string Arch;
 510          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 511                                          {"$(SECTION)",&Section},
 512                                          {"$(ARCH)",&Arch},
 513                                          {NULL, NULL}};
 514          mode_t const Perms = Block.FindI("FileMode", Permissions);
 515          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 516          TranslationWriter *TransWriter;
 517          if (DTrans.empty() == false && LongDesc == false)
 518          {
 519             string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 520                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 521             string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 522             TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 523          }
 524          else
 525             TransWriter = NULL;
 526
 527          string const Tmp2 = Block.Find("Architectures");
 528          const char *Archs = Tmp2.c_str();
 529          while (ParseQuoteWord(Archs,Arch) == true)
 530          {
 531             PackageMap Itm;
 532             Itm.Permissions = Perms;
 533             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 534             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 535
 536             if (stringcasecmp(Arch,"source") == 0)
 537             {
 538                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 539                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 540                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 541                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 542                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 543                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 544                Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);
 545             }
 546             else
 547             {
 548                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 549                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 550                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 551                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 552                Itm.Arch = Arch;
 553                Itm.LongDesc = LongDesc;
 554                if (TransWriter != NULL)
 555                {
 556                   TransWriter->IncreaseRefCounter();
 557                   Itm.TransWriter = TransWriter;
 558                }
 559                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 560                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 561                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 562                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 563             }
 564
 565             Itm.GetGeneral(Setup,Block);
 566             PkgList.push_back(Itm);
 567          }
 568          // we didn't use this TransWriter, so we can release it
 569          if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
 570             delete TransWriter;
 571       }
 572
 573       Top = Top->Next;
 574    }
 575 }
 576                                                                         /*}}}*/
 577 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 578 // ---------------------------------------------------------------------
 579 /* */
 580 static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 581 {
 582    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 583
 584    // Process 'bindirectory' type sections
 585    const Configuration::Item *Top = Setup.Tree("bindirectory");
 586    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 587    {
 588       Configuration Block(Top);
 589
 590       PackageMap Itm;
 591       Itm.PkgFile = Block.Find("Packages");
 592       Itm.SrcFile = Block.Find("Sources");
 593       Itm.BinCacheDB = Block.Find("BinCacheDB");
 594       Itm.SrcCacheDB = Block.Find("SrcCacheDB");
 595       Itm.BinOverride = Block.Find("BinOverride");
 596       Itm.ExtraOverride = Block.Find("ExtraOverride");
 597       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 598       Itm.SrcOverride = Block.Find("SrcOverride");
 599       Itm.BaseDir = Top->Tag;
 600       Itm.FLFile = Block.Find("FileList");
 601       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 602       Itm.Contents = Block.Find("Contents");
 603       Itm.ContentsHead = Block.Find("Contents::Header");
 604       Itm.Permissions = Block.FindI("FileMode", Permissions);
 605
 606       Itm.GetGeneral(Setup,Block);
 607       PkgList.push_back(Itm);
 608
 609       Top = Top->Next;
 610    }
 611 }
 612                                                                         /*}}}*/
 613
 614 // ShowHelp - Show the help text                                        /*{{{*/
 615 // ---------------------------------------------------------------------
 616 /* */
 617 static bool ShowHelp(CommandLine &)
 618 {
 619    ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 620    if (_config->FindB("version") == true)
 621       return true;
 622
 623    cout <<
 624     _("Usage: apt-ftparchive [options] command\n"
 625       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 626       "          sources srcpath [overridefile [pathprefix]]\n"
 627       "          contents path\n"
 628       "          release path\n"
 629       "          generate config [groups]\n"
 630       "          clean config\n"
 631       "\n"
 632       "apt-ftparchive generates index files for Debian archives. It supports\n"
 633       "many styles of generation from fully automated to functional replacements\n"
 634       "for dpkg-scanpackages and dpkg-scansources\n"
 635       "\n"
 636       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 637       "Package file contains the contents of all the control fields from\n"
 638       "each package as well as the MD5 hash and filesize. An override file\n"
 639       "is supported to force the value of Priority and Section.\n"
 640       "\n"
 641       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 642       "The --source-override option can be used to specify a src override file\n"
 643       "\n"
 644       "The 'packages' and 'sources' command should be run in the root of the\n"
 645       "tree. BinaryPath should point to the base of the recursive search and \n"
 646       "override file should contain the override flags. Pathprefix is\n"
 647       "appended to the filename fields if present. Example usage from the \n"
 648       "Debian archive:\n"
 649       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 650       "               dists/potato/main/binary-i386/Packages\n"
 651       "\n"
 652       "Options:\n"
 653       "  -h    This help text\n"
 654       "  --md5 Control MD5 generation\n"
 655       "  -s=?  Source override file\n"
 656       "  -q    Quiet\n"
 657       "  -d=?  Select the optional caching database\n"
 658       "  --no-delink Enable delinking debug mode\n"
 659       "  --contents  Control contents file generation\n"
 660       "  -c=?  Read this configuration file\n"
 661       "  -o=?  Set an arbitrary configuration option") << endl;
 662
 663    return true;
 664 }
 665                                                                         /*}}}*/
 666 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 667 // ---------------------------------------------------------------------
 668 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 669 static bool SimpleGenPackages(CommandLine &CmdL)
 670 {
 671    if (CmdL.FileSize() < 2)
 672       return ShowHelp(CmdL);
 673
 674    string Override;
 675    if (CmdL.FileSize() >= 3)
 676       Override = CmdL.FileList[2];
 677
 678    // Create a package writer object.
 679    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 680                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 681    if (_error->PendingError() == true)
 682       return false;
 683
 684    if (CmdL.FileSize() >= 4)
 685       Packages.PathPrefix = CmdL.FileList[3];
 686
 687    // Do recursive directory searching
 688    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 689       return false;
 690
 691    // Give some stats if asked for
 692    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 693      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 694
 695    return true;
 696 }
 697                                                                         /*}}}*/
 698 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 699 // ---------------------------------------------------------------------
 700 /* */
 701 static bool SimpleGenContents(CommandLine &CmdL)
 702 {
 703    if (CmdL.FileSize() < 2)
 704       return ShowHelp(CmdL);
 705
 706    // Create a package writer object.
 707    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 708    if (_error->PendingError() == true)
 709       return false;
 710
 711    // Do recursive directory searching
 712    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 713       return false;
 714
 715    Contents.Finish();
 716
 717    return true;
 718 }
 719                                                                         /*}}}*/
 720 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 721 // ---------------------------------------------------------------------
 722 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 723 static bool SimpleGenSources(CommandLine &CmdL)
 724 {
 725    if (CmdL.FileSize() < 2)
 726       return ShowHelp(CmdL);
 727
 728    string Override;
 729    if (CmdL.FileSize() >= 3)
 730       Override = CmdL.FileList[2];
 731
 732    string SOverride;
 733    if (Override.empty() == false)
 734       SOverride = Override + ".src";
 735
 736    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 737                              SOverride.c_str());
 738
 739    // Create a package writer object.
 740    SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),Override,SOverride);
 741    if (_error->PendingError() == true)
 742       return false;
 743
 744    if (CmdL.FileSize() >= 4)
 745       Sources.PathPrefix = CmdL.FileList[3];
 746
 747    // Do recursive directory searching
 748    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 749       return false;
 750
 751    // Give some stats if asked for
 752    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 753      c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl;
 754
 755    return true;
 756 }
 757                                                                         /*}}}*/
 758 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 759 // ---------------------------------------------------------------------
 760 static bool SimpleGenRelease(CommandLine &CmdL)
 761 {
 762    if (CmdL.FileSize() < 2)
 763       return ShowHelp(CmdL);
 764
 765    string Dir = CmdL.FileList[1];
 766
 767    ReleaseWriter Release("");
 768    Release.DirStrip = Dir;
 769
 770    if (_error->PendingError() == true)
 771       return false;
 772
 773    if (Release.RecursiveScan(Dir) == false)
 774       return false;
 775
 776    Release.Finish();
 777
 778    return true;
 779 }
 780
 781                                                                         /*}}}*/
 782 // DoGeneratePackagesAndSources - Helper for Generate                   /*{{{*/
 783 // ---------------------------------------------------------------------
 784 static bool DoGeneratePackagesAndSources(Configuration &Setup,
 785                                          vector<PackageMap> &PkgList,
 786                                          struct CacheDB::Stats &SrcStats,
 787                                          struct CacheDB::Stats &Stats,
 788                                          CommandLine &CmdL)
 789 {
 790    if (CmdL.FileSize() <= 2)
 791    {
 792       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 793          if (I->GenPackages(Setup,Stats) == false)
 794             _error->DumpErrors();
 795       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 796          if (I->GenSources(Setup,SrcStats) == false)
 797             _error->DumpErrors();
 798    }
 799    else
 800    {
 801       // Make a choice list out of the package list..
 802       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 803       RxChoiceList *End = List;
 804       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 805       {
 806          End->UserData = &(*I);
 807          End->Str = I->BaseDir.c_str();
 808          End++;
 809
 810          End->UserData = &(*I);
 811          End->Str = I->Tag.c_str();
 812          End++;
 813       }
 814       End->Str = 0;
 815
 816       // Regex it
 817       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 818       {
 819          delete [] List;
 820          return _error->Error(_("No selections matched"));
 821       }
 822       _error->DumpErrors();
 823
 824       // Do the generation for Packages
 825       for (End = List; End->Str != 0; End++)
 826       {
 827          if (End->Hit == false)
 828             continue;
 829
 830          PackageMap *I = (PackageMap *)End->UserData;
 831          if (I->PkgDone == true)
 832             continue;
 833          if (I->GenPackages(Setup,Stats) == false)
 834             _error->DumpErrors();
 835       }
 836
 837       // Do the generation for Sources
 838       for (End = List; End->Str != 0; End++)
 839       {
 840          if (End->Hit == false)
 841             continue;
 842
 843          PackageMap *I = (PackageMap *)End->UserData;
 844          if (I->SrcDone == true)
 845             continue;
 846          if (I->GenSources(Setup,SrcStats) == false)
 847             _error->DumpErrors();
 848       }
 849
 850       delete [] List;
 851    }
 852
 853    // close the Translation master files
 854    for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I)
 855       if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
 856          delete I->TransWriter;
 857
 858    return true;
 859 }
 860
 861                                                                         /*}}}*/
 862 // DoGenerateContents - Helper for Generate to generate the Contents    /*{{{*/
 863 // ---------------------------------------------------------------------
 864 static bool DoGenerateContents(Configuration &Setup,
 865                                vector<PackageMap> &PkgList,
 866                                CommandLine &CmdL)
 867 {
 868    c1out << "Packages done, Starting contents." << endl;
 869
 870    // Sort the contents file list by date
 871    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 872    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 873    {
 874       struct stat A;
 875       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 876                                  I->CntCompress,A) == false)
 877          time(&I->ContentsMTime);
 878       else
 879          I->ContentsMTime = A.st_mtime;
 880    }
 881    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 882
 883    /* Now for Contents.. The process here is to do a make-like dependency
 884       check. Each contents file is verified to be newer than the package files
 885       that describe the debs it indexes. Since the package files contain
 886       hashes of the .debs this means they have not changed either so the
 887       contents must be up to date. */
 888    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 889    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 890    {
 891       // This record is not relevant
 892       if (I->ContentsDone == true ||
 893           I->Contents.empty() == true)
 894          continue;
 895
 896       // Do not do everything if the user specified sections.
 897       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 898          continue;
 899
 900       struct stat A,B;
 901       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 902       {
 903          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 904          {
 905             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 906             continue;
 907          }
 908
 909          if (A.st_mtime > B.st_mtime)
 910             continue;
 911       }
 912
 913       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 914                          MaxContentsChange) == false)
 915          _error->DumpErrors();
 916
 917       // Hit the limit?
 918       if (MaxContentsChange == 0)
 919       {
 920          c1out << "Hit contents update byte limit" << endl;
 921          break;
 922       }
 923    }
 924
 925    return true;
 926 }
 927
 928                                                                         /*}}}*/
 929 // Generate - Full generate, using a config file                        /*{{{*/
 930 // ---------------------------------------------------------------------
 931 /* */
 932 static bool Generate(CommandLine &CmdL)
 933 {
 934    struct CacheDB::Stats SrcStats;
 935    if (CmdL.FileSize() < 2)
 936       return ShowHelp(CmdL);
 937
 938    struct timeval StartTime;
 939    gettimeofday(&StartTime,0);
 940    struct CacheDB::Stats Stats;
 941
 942    // Read the configuration file.
 943    Configuration Setup;
 944    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 945       return false;
 946
 947    vector<PackageMap> PkgList;
 948    LoadTree(PkgList,Setup);
 949    LoadBinDir(PkgList,Setup);
 950
 951    // Sort by cache DB to improve IO locality.
 952    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 953    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 954
 955    // Generate packages
 956    if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false)
 957    {
 958       if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false)
 959          return false;
 960    } else {
 961       c1out << "Skipping Packages/Sources generation" << endl;
 962    }
 963
 964    // do Contents if needed
 965    if (_config->FindB("APT::FTPArchive::Contents", true) == true)
 966       if (DoGenerateContents(Setup, PkgList, CmdL) == false)
 967          return false;
 968
 969    struct timeval NewTime;
 970    gettimeofday(&NewTime,0);
 971    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 972                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 973    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 974          << " archives. Took " << TimeToStr((long)Delta) << endl;
 975
 976    return true;
 977 }
 978
 979                                                                         /*}}}*/
 980 // Clean - Clean out the databases                                      /*{{{*/
 981 // ---------------------------------------------------------------------
 982 /* */
 983 static bool Clean(CommandLine &CmdL)
 984 {
 985    if (CmdL.FileSize() != 2)
 986       return ShowHelp(CmdL);
 987
 988    // Read the configuration file.
 989    Configuration Setup;
 990    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 991       return false;
 992
 993    vector<PackageMap> PkgList;
 994    LoadTree(PkgList,Setup);
 995    LoadBinDir(PkgList,Setup);
 996
 997    // Sort by cache DB to improve IO locality.
 998    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 999    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
1000
1001    string CacheDir = Setup.FindDir("Dir::CacheDir");
1002
1003    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
1004    {
1005       if(I->BinCacheDB != "")
1006          c0out << I->BinCacheDB << endl;
1007       if(I->SrcCacheDB != "")
1008          c0out << I->SrcCacheDB << endl;
1009       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
1010       CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));
1011       if (DB.Clean() == false)
1012          _error->DumpErrors();
1013       if (DB_SRC.Clean() == false)
1014          _error->DumpErrors();
1015
1016       string CacheDB = I->BinCacheDB;
1017       string SrcCacheDB = I->SrcCacheDB;
1018       while(I != PkgList.end() &&
1019             I->BinCacheDB == CacheDB &&
1020             I->SrcCacheDB == SrcCacheDB)
1021          ++I;
1022
1023    }
1024
1025
1026    return true;
1027 }
1028                                                                         /*}}}*/
1029
1030 int main(int argc, const char *argv[])
1031 {
1032    setlocale(LC_ALL, "");
1033    CommandLine::Args Args[] = {
1034       {'h',"help","help",0},
1035       {0,"md5","APT::FTPArchive::MD5",0},
1036       {0,"sha1","APT::FTPArchive::SHA1",0},
1037       {0,"sha256","APT::FTPArchive::SHA256",0},
1038       {'v',"version","version",0},
1039       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
1040       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
1041       {'q',"quiet","quiet",CommandLine::IntLevel},
1042       {'q',"silent","quiet",CommandLine::IntLevel},
1043       {0,"delink","APT::FTPArchive::DeLinkAct",0},
1044       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
1045       {0,"contents","APT::FTPArchive::Contents",0},
1046       {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
1047       {'c',"config-file",0,CommandLine::ConfigFile},
1048       {'o',"option",0,CommandLine::ArbItem},
1049       {0,0,0,0}};
1050    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
1051                                    {"contents",&SimpleGenContents},
1052                                    {"sources",&SimpleGenSources},
1053                                    {"release",&SimpleGenRelease},
1054                                    {"generate",&Generate},
1055                                    {"clean",&Clean},
1056                                    {"help",&ShowHelp},
1057                                    {0,0}};
1058
1059    // Parse the command line and initialize the package library
1060    CommandLine CmdL(Args,_config);
1061    ParseCommandLine(CmdL, Cmds, Args, &_config, NULL, argc, argv, ShowHelp);
1062
1063    _config->CndSet("quiet",0);
1064    Quiet = _config->FindI("quiet",0);
1065    InitOutput(clog.rdbuf());
1066
1067    // Match the operation
1068    CmdL.DispatchArg(Cmds);
1069
1070    if (_error->empty() == false)
1071    {
1072       bool Errors = _error->PendingError();
1073       _error->DumpErrors();
1074       return Errors == true?100:0;
1075    }
1076    return 0;
1077 }