ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <apt-pkg/fileutl.h>
  21
  22 #include <apt-private/private-cmndline.h>
  23 #include <apt-private/private-output.h>
  24
  25 #include <algorithm>
  26 #include <climits>
  27 #include <sys/time.h>
  28 #include <locale.h>
  29 #include <stdio.h>
  30 #include <sys/stat.h>
  31 #include <time.h>
  32 #include <functional>
  33 #include <iostream>
  34 #include <string>
  35 #include <vector>
  36
  37 #include "cachedb.h"
  38 #include "override.h"
  39 #include "apt-ftparchive.h"
  40 #include "multicompress.h"
  41 #include "writer.h"
  42
  43 #include <apti18n.h>
  44                                                                         /*}}}*/
  45
  46 using namespace std;
  47 unsigned Quiet = 0;
  48
  49 // struct PackageMap - List of all package files in the config file     /*{{{*/
  50 // ---------------------------------------------------------------------
  51 /* */
  52 struct PackageMap
  53 {
  54    // General Stuff
  55    string BaseDir;
  56    string InternalPrefix;
  57    string FLFile;
  58    string PkgExt;
  59    string SrcExt;
  60
  61    // Stuff for the Package File
  62    string PkgFile;
  63    string BinCacheDB;
  64    string SrcCacheDB;
  65    string BinOverride;
  66    string ExtraOverride;
  67
  68    // We generate for this given arch
  69    string Arch;
  70
  71    // Stuff for the Source File
  72    string SrcFile;
  73    string SrcOverride;
  74    string SrcExtraOverride;
  75
  76    // Translation master file
  77    bool LongDesc;
  78    TranslationWriter *TransWriter;
  79
  80    // Contents
  81    string Contents;
  82    string ContentsHead;
  83
  84    // Random things
  85    string Tag;
  86    string PkgCompress;
  87    string CntCompress;
  88    string SrcCompress;
  89    string PathPrefix;
  90    unsigned int DeLinkLimit;
  91    mode_t Permissions;
  92
  93    bool ContentsDone;
  94    bool PkgDone;
  95    bool SrcDone;
  96    time_t ContentsMTime;
  97
  98    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  99    {
 100       inline bool operator() (const PackageMap &x,const PackageMap &y)
 101       {return x.ContentsMTime < y.ContentsMTime;};
 102    };
 103
 104    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
 105    {
 106       inline bool operator() (const PackageMap &x,const PackageMap &y)
 107       {return x.BinCacheDB < y.BinCacheDB;};
 108    };
 109
 110    struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
 111    {
 112       inline bool operator() (const PackageMap &x,const PackageMap &y)
 113       {return x.SrcCacheDB < y.SrcCacheDB;};
 114    };
 115
 116    void GetGeneral(Configuration &Setup,Configuration &Block);
 117    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 118    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 119    bool GenContents(Configuration &Setup,
 120                     vector<PackageMap>::iterator Begin,
 121                     vector<PackageMap>::iterator End,
 122                     unsigned long &Left);
 123
 124    PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 125                   ContentsDone(false), PkgDone(false), SrcDone(false),
 126                   ContentsMTime(0) {};
 127 };
 128                                                                         /*}}}*/
 129
 130 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 131 // ---------------------------------------------------------------------
 132 /* */
 133 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 134 {
 135    PathPrefix = Block.Find("PathPrefix");
 136
 137    if (Block.FindB("External-Links",true) == false)
 138       DeLinkLimit = Setup.FindI("Default::DeLinkLimit", std::numeric_limits<unsigned int>::max());
 139    else
 140       DeLinkLimit = 0;
 141
 142    PkgCompress = Block.Find("Packages::Compress",
 143                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 144    CntCompress = Block.Find("Contents::Compress",
 145                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 146    SrcCompress = Block.Find("Sources::Compress",
 147                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 148
 149    SrcExt = Block.Find("Sources::Extensions",
 150                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 151    PkgExt = Block.Find("Packages::Extensions",
 152                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 153
 154    Permissions = Setup.FindI("Default::FileMode",0644);
 155
 156    if (FLFile.empty() == false)
 157       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 158
 159    if (Contents == " ")
 160       Contents= string();
 161 }
 162                                                                         /*}}}*/
 163 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 164 // ---------------------------------------------------------------------
 165 /* This generates the Package File described by this object. */
 166 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 167 {
 168    if (PkgFile.empty() == true)
 169       return true;
 170
 171    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 172    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 173    string CacheDir = Setup.FindDir("Dir::CacheDir");
 174
 175    struct timeval StartTime;
 176    gettimeofday(&StartTime,0);
 177
 178    PkgDone = true;
 179
 180    // Create a package writer object.
 181    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 182                       PkgCompress,Permissions);
 183    PackagesWriter Packages(&Comp.Input, TransWriter, flCombine(CacheDir,BinCacheDB),
 184                            flCombine(OverrideDir,BinOverride),
 185                            flCombine(OverrideDir,ExtraOverride),
 186                            Arch);
 187    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 188       return _error->Error(_("Package extension list is too long"));
 189    if (_error->PendingError() == true)
 190       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 191
 192    Packages.PathPrefix = PathPrefix;
 193    Packages.DirStrip = ArchiveDir;
 194    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 195
 196    Packages.LongDescription = LongDesc;
 197
 198    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 199    Packages.DeLinkLimit = DeLinkLimit;
 200
 201    if (_error->PendingError() == true)
 202       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 203
 204    c0out << ' ' << BaseDir << ":" << flush;
 205
 206    // Do recursive directory searching
 207    if (FLFile.empty() == true)
 208    {
 209       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 210          return false;
 211    }
 212    else
 213    {
 214       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 215          return false;
 216    }
 217
 218    Packages.Output = 0;      // Just in case
 219
 220    // Finish compressing
 221    unsigned long long Size;
 222    if (Comp.Finalize(Size) == false)
 223    {
 224       c0out << endl;
 225       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 226    }
 227
 228    if (Size != 0)
 229       c0out << " New "
 230              << SizeToStr(Size) << "B ";
 231    else
 232       c0out << ' ';
 233
 234    struct timeval NewTime;
 235    gettimeofday(&NewTime,0);
 236    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 237                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 238
 239    c0out << Packages.Stats.Packages << " files " <<
 240 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 241       SizeToStr(Packages.Stats.Bytes) << "B " <<
 242       TimeToStr((long)Delta) << endl;
 243
 244    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 245      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 246
 247    Stats.Add(Packages.Stats);
 248    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 249
 250    return !_error->PendingError();
 251 }
 252
 253                                                                         /*}}}*/
 254 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 255 // ---------------------------------------------------------------------
 256 /* This generates the Sources File described by this object. */
 257 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 258 {
 259    if (SrcFile.empty() == true)
 260       return true;
 261
 262    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 263    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 264    string CacheDir = Setup.FindDir("Dir::CacheDir");
 265
 266    struct timeval StartTime;
 267    gettimeofday(&StartTime,0);
 268
 269    SrcDone = true;
 270
 271    // Create a package writer object.
 272    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 273                       SrcCompress,Permissions);
 274    SourcesWriter Sources(&Comp.Input, flCombine(CacheDir, SrcCacheDB),
 275                          flCombine(OverrideDir,BinOverride),
 276                          flCombine(OverrideDir,SrcOverride),
 277                          flCombine(OverrideDir,SrcExtraOverride));
 278    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 279       return _error->Error(_("Source extension list is too long"));
 280    if (_error->PendingError() == true)
 281       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 282
 283    Sources.PathPrefix = PathPrefix;
 284    Sources.DirStrip = ArchiveDir;
 285    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 286
 287    Sources.DeLinkLimit = DeLinkLimit;
 288    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 289
 290    if (_error->PendingError() == true)
 291       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 292
 293    c0out << ' ' << BaseDir << ":" << flush;
 294
 295    // Do recursive directory searching
 296    if (FLFile.empty() == true)
 297    {
 298       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 299          return false;
 300    }
 301    else
 302    {
 303       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 304          return false;
 305    }
 306    Sources.Output = 0;      // Just in case
 307
 308    // Finish compressing
 309    unsigned long long Size;
 310    if (Comp.Finalize(Size) == false)
 311    {
 312       c0out << endl;
 313       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 314    }
 315
 316    if (Size != 0)
 317       c0out << " New "
 318              << SizeToStr(Size) << "B ";
 319    else
 320       c0out << ' ';
 321
 322    struct timeval NewTime;
 323    gettimeofday(&NewTime,0);
 324    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 325                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 326
 327    c0out << Sources.Stats.Packages << " pkgs in " <<
 328       TimeToStr((long)Delta) << endl;
 329
 330    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 331      c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
 332
 333    Stats.Add(Sources.Stats);
 334    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 335
 336    return !_error->PendingError();
 337 }
 338                                                                         /*}}}*/
 339 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 340 // ---------------------------------------------------------------------
 341 /* This generates the contents file partially described by this object.
 342    It searches the given iterator range for other package files that map
 343    into this contents file and includes their data as well when building. */
 344 bool PackageMap::GenContents(Configuration &Setup,
 345                              vector<PackageMap>::iterator Begin,
 346                              vector<PackageMap>::iterator End,
 347                              unsigned long &Left)
 348 {
 349    if (Contents.empty() == true)
 350       return true;
 351
 352    if (Left == 0)
 353       return true;
 354
 355    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 356    string CacheDir = Setup.FindDir("Dir::CacheDir");
 357    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 358
 359    struct timeval StartTime;
 360    gettimeofday(&StartTime,0);
 361
 362    // Create a package writer object.
 363    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 364                       CntCompress,Permissions);
 365    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 366    ContentsWriter Contents(&Comp.Input, "", Arch);
 367    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 368       return _error->Error(_("Package extension list is too long"));
 369    if (_error->PendingError() == true)
 370       return false;
 371
 372    if (_error->PendingError() == true)
 373       return false;
 374
 375    // Write the header out.
 376    if (ContentsHead.empty() == false)
 377    {
 378       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 379       if (_error->PendingError() == true)
 380          return false;
 381
 382       unsigned long long Size = Head.Size();
 383       unsigned char Buf[4096];
 384       while (Size != 0)
 385       {
 386          unsigned long long ToRead = Size;
 387          if (Size > sizeof(Buf))
 388             ToRead = sizeof(Buf);
 389
 390          if (Head.Read(Buf,ToRead) == false)
 391             return false;
 392
 393          if (Comp.Input.Write(Buf, ToRead) == false)
 394             return _error->Errno("fwrite",_("Error writing header to contents file"));
 395
 396          Size -= ToRead;
 397       }
 398    }
 399
 400    /* Go over all the package file records and parse all the package
 401       files associated with this contents file into one great big honking
 402       memory structure, then dump the sorted version */
 403    c0out << ' ' << this->Contents << ":" << flush;
 404    for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
 405    {
 406       if (I->Contents != this->Contents)
 407          continue;
 408
 409       Contents.Prefix = ArchiveDir;
 410       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 411       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 412                             I->PkgCompress);
 413
 414       I->ContentsDone = true;
 415    }
 416
 417    Contents.Finish();
 418
 419    // Finish compressing
 420    unsigned long long Size;
 421    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 422    {
 423       c0out << endl;
 424       return _error->Error(_("Error processing contents %s"),
 425                            this->Contents.c_str());
 426    }
 427
 428    if (Size != 0)
 429    {
 430       c0out << " New " << SizeToStr(Size) << "B ";
 431       if (Left > Size)
 432          Left -= Size;
 433       else
 434          Left = 0;
 435    }
 436    else
 437       c0out << ' ';
 438
 439    struct timeval NewTime;
 440    gettimeofday(&NewTime,0);
 441    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 442                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 443
 444    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 445      c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
 446
 447    c0out << Contents.Stats.Packages << " files " <<
 448       SizeToStr(Contents.Stats.Bytes) << "B " <<
 449       TimeToStr((long)Delta) << endl;
 450
 451    return true;
 452 }
 453                                                                         /*}}}*/
 454
 455 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 456 // ---------------------------------------------------------------------
 457 /* This populates the PkgList with all the possible permutations of the
 458    section/arch lists. */
 459 static void LoadTree(vector<PackageMap> &PkgList, std::vector<TranslationWriter*> &TransList, Configuration &Setup)
 460 {
 461    // Load the defaults
 462    string DDir = Setup.Find("TreeDefault::Directory",
 463                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 464    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 465                             "$(DIST)/$(SECTION)/source/");
 466    string DPkg = Setup.Find("TreeDefault::Packages",
 467                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 468    string DTrans = Setup.Find("TreeDefault::Translation",
 469                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 470    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 471                             "$(DIST)/$(SECTION)/");
 472    string DContents = Setup.Find("TreeDefault::Contents",
 473                             "$(DIST)/$(SECTION)/Contents-$(ARCH)");
 474    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 475    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 476                                "packages-$(ARCH).db");
 477    string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
 478                                "sources-$(SECTION).db");
 479    string DSources = Setup.Find("TreeDefault::Sources",
 480                                 "$(DIST)/$(SECTION)/source/Sources");
 481    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 482    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 483
 484    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 485
 486    bool const LongDescription = Setup.FindB("Default::LongDescription",
 487                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 488    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 489
 490    // Process 'tree' type sections
 491    const Configuration::Item *Top = Setup.Tree("tree");
 492    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 493    {
 494       Configuration Block(Top);
 495       string Dist = Top->Tag;
 496
 497       // Parse the sections
 498       string Tmp = Block.Find("Sections");
 499       const char *Sections = Tmp.c_str();
 500       string Section;
 501       while (ParseQuoteWord(Sections,Section) == true)
 502       {
 503          string Arch;
 504          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 505                                          {"$(SECTION)",&Section},
 506                                          {"$(ARCH)",&Arch},
 507                                          {NULL, NULL}};
 508          mode_t const Perms = Block.FindI("FileMode", Permissions);
 509          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 510          TranslationWriter *TransWriter = NULL;
 511
 512          string const Tmp2 = Block.Find("Architectures");
 513          const char *Archs = Tmp2.c_str();
 514          while (ParseQuoteWord(Archs,Arch) == true)
 515          {
 516             PackageMap Itm;
 517             Itm.Permissions = Perms;
 518             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 519             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 520
 521             if (stringcasecmp(Arch,"source") == 0)
 522             {
 523                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 524                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 525                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 526                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 527                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 528                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 529                Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);
 530             }
 531             else
 532             {
 533                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 534                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 535                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 536                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 537                Itm.Arch = Arch;
 538                Itm.LongDesc = LongDesc;
 539                if (TransWriter == NULL && DTrans.empty() == false && LongDesc == false && DTrans != "/dev/null")
 540                {
 541                   string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 542                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 543                   string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 544                   TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 545                   TransList.push_back(TransWriter);
 546                }
 547                Itm.TransWriter = TransWriter;
 548                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 549                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 550                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 551                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 552             }
 553
 554             Itm.GetGeneral(Setup,Block);
 555             PkgList.push_back(Itm);
 556          }
 557       }
 558
 559       Top = Top->Next;
 560    }
 561 }
 562                                                                         /*}}}*/
 563 static void UnloadTree(std::vector<TranslationWriter*> const &Trans)            /*{{{*/
 564 {
 565    for (std::vector<TranslationWriter*>::const_reverse_iterator T = Trans.rbegin(); T != Trans.rend(); ++T)
 566       delete *T;
 567 }
 568                                                                         /*}}}*/
 569 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 570 // ---------------------------------------------------------------------
 571 /* */
 572 static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 573 {
 574    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 575
 576    // Process 'bindirectory' type sections
 577    const Configuration::Item *Top = Setup.Tree("bindirectory");
 578    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 579    {
 580       Configuration Block(Top);
 581
 582       PackageMap Itm;
 583       Itm.PkgFile = Block.Find("Packages");
 584       Itm.SrcFile = Block.Find("Sources");
 585       Itm.BinCacheDB = Block.Find("BinCacheDB");
 586       Itm.SrcCacheDB = Block.Find("SrcCacheDB");
 587       Itm.BinOverride = Block.Find("BinOverride");
 588       Itm.ExtraOverride = Block.Find("ExtraOverride");
 589       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 590       Itm.SrcOverride = Block.Find("SrcOverride");
 591       Itm.BaseDir = Top->Tag;
 592       Itm.FLFile = Block.Find("FileList");
 593       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 594       Itm.Contents = Block.Find("Contents");
 595       Itm.ContentsHead = Block.Find("Contents::Header");
 596       Itm.Permissions = Block.FindI("FileMode", Permissions);
 597
 598       Itm.GetGeneral(Setup,Block);
 599       PkgList.push_back(Itm);
 600
 601       Top = Top->Next;
 602    }
 603 }
 604                                                                         /*}}}*/
 605
 606 // ShowHelp - Show the help text                                        /*{{{*/
 607 // ---------------------------------------------------------------------
 608 /* */
 609 static bool ShowHelp(CommandLine &)
 610 {
 611    ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 612    if (_config->FindB("version") == true)
 613       return true;
 614
 615    cout <<
 616     _("Usage: apt-ftparchive [options] command\n"
 617       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 618       "          sources srcpath [overridefile [pathprefix]]\n"
 619       "          contents path\n"
 620       "          release path\n"
 621       "          generate config [groups]\n"
 622       "          clean config\n"
 623       "\n"
 624       "apt-ftparchive generates index files for Debian archives. It supports\n"
 625       "many styles of generation from fully automated to functional replacements\n"
 626       "for dpkg-scanpackages and dpkg-scansources\n"
 627       "\n"
 628       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 629       "Package file contains the contents of all the control fields from\n"
 630       "each package as well as the MD5 hash and filesize. An override file\n"
 631       "is supported to force the value of Priority and Section.\n"
 632       "\n"
 633       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 634       "The --source-override option can be used to specify a src override file\n"
 635       "\n"
 636       "The 'packages' and 'sources' command should be run in the root of the\n"
 637       "tree. BinaryPath should point to the base of the recursive search and \n"
 638       "override file should contain the override flags. Pathprefix is\n"
 639       "appended to the filename fields if present. Example usage from the \n"
 640       "Debian archive:\n"
 641       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 642       "               dists/potato/main/binary-i386/Packages\n"
 643       "\n"
 644       "Options:\n"
 645       "  -h    This help text\n"
 646       "  --md5 Control MD5 generation\n"
 647       "  -s=?  Source override file\n"
 648       "  -q    Quiet\n"
 649       "  -d=?  Select the optional caching database\n"
 650       "  --no-delink Enable delinking debug mode\n"
 651       "  --contents  Control contents file generation\n"
 652       "  -c=?  Read this configuration file\n"
 653       "  -o=?  Set an arbitrary configuration option") << endl;
 654
 655    return true;
 656 }
 657                                                                         /*}}}*/
 658 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 659 // ---------------------------------------------------------------------
 660 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 661 static bool SimpleGenPackages(CommandLine &CmdL)
 662 {
 663    if (CmdL.FileSize() < 2)
 664       return ShowHelp(CmdL);
 665
 666    string Override;
 667    if (CmdL.FileSize() >= 3)
 668       Override = CmdL.FileList[2];
 669
 670    // Create a package writer object.
 671    PackagesWriter Packages(NULL, NULL, _config->Find("APT::FTPArchive::DB"),
 672                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 673    if (_error->PendingError() == true)
 674       return false;
 675
 676    if (CmdL.FileSize() >= 4)
 677       Packages.PathPrefix = CmdL.FileList[3];
 678
 679    // Do recursive directory searching
 680    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 681       return false;
 682
 683    // Give some stats if asked for
 684    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 685      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 686
 687    return true;
 688 }
 689                                                                         /*}}}*/
 690 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 691 // ---------------------------------------------------------------------
 692 /* */
 693 static bool SimpleGenContents(CommandLine &CmdL)
 694 {
 695    if (CmdL.FileSize() < 2)
 696       return ShowHelp(CmdL);
 697
 698    // Create a package writer object.
 699    ContentsWriter Contents(NULL, _config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 700    if (_error->PendingError() == true)
 701       return false;
 702
 703    // Do recursive directory searching
 704    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 705       return false;
 706
 707    Contents.Finish();
 708
 709    return true;
 710 }
 711                                                                         /*}}}*/
 712 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 713 // ---------------------------------------------------------------------
 714 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 715 static bool SimpleGenSources(CommandLine &CmdL)
 716 {
 717    if (CmdL.FileSize() < 2)
 718       return ShowHelp(CmdL);
 719
 720    string Override;
 721    if (CmdL.FileSize() >= 3)
 722       Override = CmdL.FileList[2];
 723
 724    string SOverride;
 725    if (Override.empty() == false)
 726       SOverride = Override + ".src";
 727
 728    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 729                              SOverride.c_str());
 730
 731    // Create a package writer object.
 732    SourcesWriter Sources(NULL, _config->Find("APT::FTPArchive::DB"),Override,SOverride);
 733    if (_error->PendingError() == true)
 734       return false;
 735
 736    if (CmdL.FileSize() >= 4)
 737       Sources.PathPrefix = CmdL.FileList[3];
 738
 739    // Do recursive directory searching
 740    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 741       return false;
 742
 743    // Give some stats if asked for
 744    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 745      c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl;
 746
 747    return true;
 748 }
 749                                                                         /*}}}*/
 750 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 751 // ---------------------------------------------------------------------
 752 static bool SimpleGenRelease(CommandLine &CmdL)
 753 {
 754    if (CmdL.FileSize() < 2)
 755       return ShowHelp(CmdL);
 756
 757    string Dir = CmdL.FileList[1];
 758
 759    ReleaseWriter Release(NULL, "");
 760    Release.DirStrip = Dir;
 761
 762    if (_error->PendingError() == true)
 763       return false;
 764
 765    if (Release.RecursiveScan(Dir) == false)
 766       return false;
 767
 768    Release.Finish();
 769
 770    return true;
 771 }
 772
 773                                                                         /*}}}*/
 774 // DoGeneratePackagesAndSources - Helper for Generate                   /*{{{*/
 775 // ---------------------------------------------------------------------
 776 static bool DoGeneratePackagesAndSources(Configuration &Setup,
 777                                          vector<PackageMap> &PkgList,
 778                                          struct CacheDB::Stats &SrcStats,
 779                                          struct CacheDB::Stats &Stats,
 780                                          CommandLine &CmdL)
 781 {
 782    if (CmdL.FileSize() <= 2)
 783    {
 784       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 785          if (I->GenPackages(Setup,Stats) == false)
 786             _error->DumpErrors();
 787       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 788          if (I->GenSources(Setup,SrcStats) == false)
 789             _error->DumpErrors();
 790    }
 791    else
 792    {
 793       // Make a choice list out of the package list..
 794       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 795       RxChoiceList *End = List;
 796       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 797       {
 798          End->UserData = &(*I);
 799          End->Str = I->BaseDir.c_str();
 800          End++;
 801
 802          End->UserData = &(*I);
 803          End->Str = I->Tag.c_str();
 804          End++;
 805       }
 806       End->Str = 0;
 807
 808       // Regex it
 809       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 810       {
 811          delete [] List;
 812          return _error->Error(_("No selections matched"));
 813       }
 814       _error->DumpErrors();
 815
 816       // Do the generation for Packages
 817       for (End = List; End->Str != 0; End++)
 818       {
 819          if (End->Hit == false)
 820             continue;
 821
 822          PackageMap *I = (PackageMap *)End->UserData;
 823          if (I->PkgDone == true)
 824             continue;
 825          if (I->GenPackages(Setup,Stats) == false)
 826             _error->DumpErrors();
 827       }
 828
 829       // Do the generation for Sources
 830       for (End = List; End->Str != 0; End++)
 831       {
 832          if (End->Hit == false)
 833             continue;
 834
 835          PackageMap *I = (PackageMap *)End->UserData;
 836          if (I->SrcDone == true)
 837             continue;
 838          if (I->GenSources(Setup,SrcStats) == false)
 839             _error->DumpErrors();
 840       }
 841
 842       delete [] List;
 843    }
 844    return true;
 845 }
 846
 847                                                                         /*}}}*/
 848 // DoGenerateContents - Helper for Generate to generate the Contents    /*{{{*/
 849 // ---------------------------------------------------------------------
 850 static bool DoGenerateContents(Configuration &Setup,
 851                                vector<PackageMap> &PkgList,
 852                                CommandLine &CmdL)
 853 {
 854    c1out << "Packages done, Starting contents." << endl;
 855
 856    // Sort the contents file list by date
 857    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 858    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 859    {
 860       struct stat A;
 861       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 862                                  I->CntCompress,A) == false)
 863          time(&I->ContentsMTime);
 864       else
 865          I->ContentsMTime = A.st_mtime;
 866    }
 867    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 868
 869    /* Now for Contents.. The process here is to do a make-like dependency
 870       check. Each contents file is verified to be newer than the package files
 871       that describe the debs it indexes. Since the package files contain
 872       hashes of the .debs this means they have not changed either so the
 873       contents must be up to date. */
 874    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",
 875                                                  std::numeric_limits<unsigned int>::max())*1024;
 876    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 877    {
 878       // This record is not relevant
 879       if (I->ContentsDone == true ||
 880           I->Contents.empty() == true)
 881          continue;
 882
 883       // Do not do everything if the user specified sections.
 884       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 885          continue;
 886
 887       struct stat A,B;
 888       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 889       {
 890          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 891          {
 892             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 893             continue;
 894          }
 895
 896          if (A.st_mtime > B.st_mtime)
 897             continue;
 898       }
 899
 900       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 901                          MaxContentsChange) == false)
 902          _error->DumpErrors();
 903
 904       // Hit the limit?
 905       if (MaxContentsChange == 0)
 906       {
 907          c1out << "Hit contents update byte limit" << endl;
 908          break;
 909       }
 910    }
 911
 912    return true;
 913 }
 914
 915                                                                         /*}}}*/
 916 // Generate - Full generate, using a config file                        /*{{{*/
 917 // ---------------------------------------------------------------------
 918 /* */
 919 static bool Generate(CommandLine &CmdL)
 920 {
 921    struct CacheDB::Stats SrcStats;
 922    if (CmdL.FileSize() < 2)
 923       return ShowHelp(CmdL);
 924
 925    struct timeval StartTime;
 926    gettimeofday(&StartTime,0);
 927    struct CacheDB::Stats Stats;
 928
 929    // Read the configuration file.
 930    Configuration Setup;
 931    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 932       return false;
 933
 934    vector<PackageMap> PkgList;
 935    std::vector<TranslationWriter*> TransList;
 936    LoadTree(PkgList, TransList, Setup);
 937    LoadBinDir(PkgList,Setup);
 938
 939    // Sort by cache DB to improve IO locality.
 940    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 941    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 942
 943    // Generate packages
 944    if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false)
 945    {
 946       if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false)
 947       {
 948          UnloadTree(TransList);
 949          return false;
 950       }
 951    } else {
 952       c1out << "Skipping Packages/Sources generation" << endl;
 953    }
 954
 955    // do Contents if needed
 956    if (_config->FindB("APT::FTPArchive::Contents", true) == true)
 957       if (DoGenerateContents(Setup, PkgList, CmdL) == false)
 958       {
 959          UnloadTree(TransList);
 960          return false;
 961       }
 962
 963    struct timeval NewTime;
 964    gettimeofday(&NewTime,0);
 965    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 966                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 967    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 968          << " archives. Took " << TimeToStr((long)Delta) << endl;
 969
 970    UnloadTree(TransList);
 971    return true;
 972 }
 973
 974                                                                         /*}}}*/
 975 // Clean - Clean out the databases                                      /*{{{*/
 976 // ---------------------------------------------------------------------
 977 /* */
 978 static bool Clean(CommandLine &CmdL)
 979 {
 980    if (CmdL.FileSize() != 2)
 981       return ShowHelp(CmdL);
 982
 983    // Read the configuration file.
 984    Configuration Setup;
 985    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 986       return false;
 987    // we don't need translation creation here
 988    Setup.Set("TreeDefault::Translation", "/dev/null");
 989
 990    vector<PackageMap> PkgList;
 991    std::vector<TranslationWriter*> TransList;
 992    LoadTree(PkgList, TransList, Setup);
 993    LoadBinDir(PkgList,Setup);
 994
 995    // Sort by cache DB to improve IO locality.
 996    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 997    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 998
 999    string CacheDir = Setup.FindDir("Dir::CacheDir");
1000
1001    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
1002    {
1003       if(I->BinCacheDB != "")
1004          c0out << I->BinCacheDB << endl;
1005       if(I->SrcCacheDB != "")
1006          c0out << I->SrcCacheDB << endl;
1007       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
1008       CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));
1009       if (DB.Clean() == false)
1010          _error->DumpErrors();
1011       if (DB_SRC.Clean() == false)
1012          _error->DumpErrors();
1013
1014       string CacheDB = I->BinCacheDB;
1015       string SrcCacheDB = I->SrcCacheDB;
1016       while(I != PkgList.end() &&
1017             I->BinCacheDB == CacheDB &&
1018             I->SrcCacheDB == SrcCacheDB)
1019          ++I;
1020    }
1021
1022
1023    return true;
1024 }
1025                                                                         /*}}}*/
1026
1027 int main(int argc, const char *argv[])
1028 {
1029    setlocale(LC_ALL, "");
1030    CommandLine::Args Args[] = {
1031       {'h',"help","help",0},
1032       {0,"md5","APT::FTPArchive::MD5",0},
1033       {0,"sha1","APT::FTPArchive::SHA1",0},
1034       {0,"sha256","APT::FTPArchive::SHA256",0},
1035       {0,"sha512","APT::FTPArchive::SHA512",0},
1036       {'v',"version","version",0},
1037       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
1038       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
1039       {'q',"quiet","quiet",CommandLine::IntLevel},
1040       {'q',"silent","quiet",CommandLine::IntLevel},
1041       {0,"delink","APT::FTPArchive::DeLinkAct",0},
1042       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
1043       {0,"contents","APT::FTPArchive::Contents",0},
1044       {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
1045       {'c',"config-file",0,CommandLine::ConfigFile},
1046       {'o',"option",0,CommandLine::ArbItem},
1047       {0,0,0,0}};
1048    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
1049                                    {"contents",&SimpleGenContents},
1050                                    {"sources",&SimpleGenSources},
1051                                    {"release",&SimpleGenRelease},
1052                                    {"generate",&Generate},
1053                                    {"clean",&Clean},
1054                                    {"help",&ShowHelp},
1055                                    {0,0}};
1056
1057    // Parse the command line and initialize the package library
1058    CommandLine CmdL(Args,_config);
1059    ParseCommandLine(CmdL, Cmds, Args, &_config, NULL, argc, argv, ShowHelp);
1060
1061    _config->CndSet("quiet",0);
1062    Quiet = _config->FindI("quiet",0);
1063    InitOutput(clog.rdbuf());
1064
1065    // Match the operation
1066    CmdL.DispatchArg(Cmds);
1067
1068    if (_error->empty() == false)
1069    {
1070       bool Errors = _error->PendingError();
1071       _error->DumpErrors();
1072       return Errors == true?100:0;
1073    }
1074    return 0;
1075 }