ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <apt-pkg/fileutl.h>
  21
  22 #include <apt-private/private-cmndline.h>
  23
  24 #include <algorithm>
  25 #include <climits>
  26 #include <sys/time.h>
  27 #include <locale.h>
  28 #include <stdio.h>
  29 #include <sys/stat.h>
  30 #include <time.h>
  31 #include <functional>
  32 #include <iostream>
  33 #include <string>
  34 #include <vector>
  35
  36 #include "cachedb.h"
  37 #include "override.h"
  38 #include "apt-ftparchive.h"
  39 #include "multicompress.h"
  40 #include "writer.h"
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44
  45 using namespace std;
  46 ostream c0out(0);
  47 ostream c1out(0);
  48 ostream c2out(0);
  49 ofstream devnull("/dev/null");
  50 unsigned Quiet = 0;
  51
  52 // struct PackageMap - List of all package files in the config file     /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* */
  55 struct PackageMap
  56 {
  57    // General Stuff
  58    string BaseDir;
  59    string InternalPrefix;
  60    string FLFile;
  61    string PkgExt;
  62    string SrcExt;
  63
  64    // Stuff for the Package File
  65    string PkgFile;
  66    string BinCacheDB;
  67    string SrcCacheDB;
  68    string BinOverride;
  69    string ExtraOverride;
  70
  71    // We generate for this given arch
  72    string Arch;
  73
  74    // Stuff for the Source File
  75    string SrcFile;
  76    string SrcOverride;
  77    string SrcExtraOverride;
  78
  79    // Translation master file
  80    bool LongDesc;
  81    TranslationWriter *TransWriter;
  82
  83    // Contents
  84    string Contents;
  85    string ContentsHead;
  86
  87    // Random things
  88    string Tag;
  89    string PkgCompress;
  90    string CntCompress;
  91    string SrcCompress;
  92    string PathPrefix;
  93    unsigned int DeLinkLimit;
  94    mode_t Permissions;
  95
  96    bool ContentsDone;
  97    bool PkgDone;
  98    bool SrcDone;
  99    time_t ContentsMTime;
 100
 101    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
 102    {
 103       inline bool operator() (const PackageMap &x,const PackageMap &y)
 104       {return x.ContentsMTime < y.ContentsMTime;};
 105    };
 106
 107    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
 108    {
 109       inline bool operator() (const PackageMap &x,const PackageMap &y)
 110       {return x.BinCacheDB < y.BinCacheDB;};
 111    };
 112
 113    struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
 114    {
 115       inline bool operator() (const PackageMap &x,const PackageMap &y)
 116       {return x.SrcCacheDB < y.SrcCacheDB;};
 117    };
 118
 119    void GetGeneral(Configuration &Setup,Configuration &Block);
 120    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 121    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 122    bool GenContents(Configuration &Setup,
 123                     vector<PackageMap>::iterator Begin,
 124                     vector<PackageMap>::iterator End,
 125                     unsigned long &Left);
 126
 127    PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 128                   ContentsDone(false), PkgDone(false), SrcDone(false),
 129                   ContentsMTime(0) {};
 130 };
 131                                                                         /*}}}*/
 132
 133 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 134 // ---------------------------------------------------------------------
 135 /* */
 136 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 137 {
 138    PathPrefix = Block.Find("PathPrefix");
 139
 140    if (Block.FindB("External-Links",true) == false)
 141       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 142    else
 143       DeLinkLimit = 0;
 144
 145    PkgCompress = Block.Find("Packages::Compress",
 146                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 147    CntCompress = Block.Find("Contents::Compress",
 148                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 149    SrcCompress = Block.Find("Sources::Compress",
 150                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 151
 152    SrcExt = Block.Find("Sources::Extensions",
 153                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 154    PkgExt = Block.Find("Packages::Extensions",
 155                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 156
 157    Permissions = Setup.FindI("Default::FileMode",0644);
 158
 159    if (FLFile.empty() == false)
 160       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 161
 162    if (Contents == " ")
 163       Contents= string();
 164 }
 165                                                                         /*}}}*/
 166 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 167 // ---------------------------------------------------------------------
 168 /* This generates the Package File described by this object. */
 169 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 170 {
 171    if (PkgFile.empty() == true)
 172       return true;
 173
 174    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 175    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 176    string CacheDir = Setup.FindDir("Dir::CacheDir");
 177
 178    struct timeval StartTime;
 179    gettimeofday(&StartTime,0);
 180
 181    PkgDone = true;
 182
 183    // Create a package writer object.
 184    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 185                            flCombine(OverrideDir,BinOverride),
 186                            flCombine(OverrideDir,ExtraOverride),
 187                            Arch);
 188    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 189       return _error->Error(_("Package extension list is too long"));
 190    if (_error->PendingError() == true)
 191       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 192
 193    Packages.PathPrefix = PathPrefix;
 194    Packages.DirStrip = ArchiveDir;
 195    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 196
 197    Packages.TransWriter = TransWriter;
 198    Packages.LongDescription = LongDesc;
 199
 200    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 201    Packages.DeLinkLimit = DeLinkLimit;
 202
 203    // Create a compressor object
 204    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 205                       PkgCompress,Permissions);
 206    Packages.Output = Comp.Input;
 207    if (_error->PendingError() == true)
 208       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 209
 210    c0out << ' ' << BaseDir << ":" << flush;
 211
 212    // Do recursive directory searching
 213    if (FLFile.empty() == true)
 214    {
 215       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 216          return false;
 217    }
 218    else
 219    {
 220       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 221          return false;
 222    }
 223
 224    Packages.Output = 0;      // Just in case
 225
 226    // Finish compressing
 227    unsigned long long Size;
 228    if (Comp.Finalize(Size) == false)
 229    {
 230       c0out << endl;
 231       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 232    }
 233
 234    if (Size != 0)
 235       c0out << " New "
 236              << SizeToStr(Size) << "B ";
 237    else
 238       c0out << ' ';
 239
 240    struct timeval NewTime;
 241    gettimeofday(&NewTime,0);
 242    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 243                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 244
 245    c0out << Packages.Stats.Packages << " files " <<
 246 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 247       SizeToStr(Packages.Stats.Bytes) << "B " <<
 248       TimeToStr((long)Delta) << endl;
 249
 250    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 251      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 252
 253    Stats.Add(Packages.Stats);
 254    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 255
 256    return !_error->PendingError();
 257 }
 258
 259                                                                         /*}}}*/
 260 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 261 // ---------------------------------------------------------------------
 262 /* This generates the Sources File described by this object. */
 263 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 264 {
 265    if (SrcFile.empty() == true)
 266       return true;
 267
 268    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 269    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 270    string CacheDir = Setup.FindDir("Dir::CacheDir");
 271
 272    struct timeval StartTime;
 273    gettimeofday(&StartTime,0);
 274
 275    SrcDone = true;
 276
 277    // Create a package writer object.
 278    SourcesWriter Sources(flCombine(CacheDir, SrcCacheDB),
 279                          flCombine(OverrideDir,BinOverride),
 280                          flCombine(OverrideDir,SrcOverride),
 281                          flCombine(OverrideDir,SrcExtraOverride));
 282    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 283       return _error->Error(_("Source extension list is too long"));
 284    if (_error->PendingError() == true)
 285       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 286
 287    Sources.PathPrefix = PathPrefix;
 288    Sources.DirStrip = ArchiveDir;
 289    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 290
 291    Sources.DeLinkLimit = DeLinkLimit;
 292    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 293
 294    // Create a compressor object
 295    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 296                       SrcCompress,Permissions);
 297    Sources.Output = Comp.Input;
 298    if (_error->PendingError() == true)
 299       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 300
 301    c0out << ' ' << BaseDir << ":" << flush;
 302
 303    // Do recursive directory searching
 304    if (FLFile.empty() == true)
 305    {
 306       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 307          return false;
 308    }
 309    else
 310    {
 311       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 312          return false;
 313    }
 314    Sources.Output = 0;      // Just in case
 315
 316    // Finish compressing
 317    unsigned long long Size;
 318    if (Comp.Finalize(Size) == false)
 319    {
 320       c0out << endl;
 321       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 322    }
 323
 324    if (Size != 0)
 325       c0out << " New "
 326              << SizeToStr(Size) << "B ";
 327    else
 328       c0out << ' ';
 329
 330    struct timeval NewTime;
 331    gettimeofday(&NewTime,0);
 332    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 333                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 334
 335    c0out << Sources.Stats.Packages << " pkgs in " <<
 336       TimeToStr((long)Delta) << endl;
 337
 338    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 339      c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
 340
 341    Stats.Add(Sources.Stats);
 342    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 343
 344    return !_error->PendingError();
 345 }
 346                                                                         /*}}}*/
 347 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 348 // ---------------------------------------------------------------------
 349 /* This generates the contents file partially described by this object.
 350    It searches the given iterator range for other package files that map
 351    into this contents file and includes their data as well when building. */
 352 bool PackageMap::GenContents(Configuration &Setup,
 353                              vector<PackageMap>::iterator Begin,
 354                              vector<PackageMap>::iterator End,
 355                              unsigned long &Left)
 356 {
 357    if (Contents.empty() == true)
 358       return true;
 359
 360    if (Left == 0)
 361       return true;
 362
 363    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 364    string CacheDir = Setup.FindDir("Dir::CacheDir");
 365    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 366
 367    struct timeval StartTime;
 368    gettimeofday(&StartTime,0);
 369
 370    // Create a package writer object.
 371    ContentsWriter Contents("", Arch);
 372    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 373       return _error->Error(_("Package extension list is too long"));
 374    if (_error->PendingError() == true)
 375       return false;
 376
 377    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 378                       CntCompress,Permissions);
 379    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 380    Contents.Output = Comp.Input;
 381    if (_error->PendingError() == true)
 382       return false;
 383
 384    // Write the header out.
 385    if (ContentsHead.empty() == false)
 386    {
 387       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 388       if (_error->PendingError() == true)
 389          return false;
 390
 391       unsigned long long Size = Head.Size();
 392       unsigned char Buf[4096];
 393       while (Size != 0)
 394       {
 395          unsigned long long ToRead = Size;
 396          if (Size > sizeof(Buf))
 397             ToRead = sizeof(Buf);
 398
 399          if (Head.Read(Buf,ToRead) == false)
 400             return false;
 401
 402          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 403             return _error->Errno("fwrite",_("Error writing header to contents file"));
 404
 405          Size -= ToRead;
 406       }
 407    }
 408
 409    /* Go over all the package file records and parse all the package
 410       files associated with this contents file into one great big honking
 411       memory structure, then dump the sorted version */
 412    c0out << ' ' << this->Contents << ":" << flush;
 413    for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
 414    {
 415       if (I->Contents != this->Contents)
 416          continue;
 417
 418       Contents.Prefix = ArchiveDir;
 419       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 420       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 421                             I->PkgCompress);
 422
 423       I->ContentsDone = true;
 424    }
 425
 426    Contents.Finish();
 427
 428    // Finish compressing
 429    unsigned long long Size;
 430    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 431    {
 432       c0out << endl;
 433       return _error->Error(_("Error processing contents %s"),
 434                            this->Contents.c_str());
 435    }
 436
 437    if (Size != 0)
 438    {
 439       c0out << " New " << SizeToStr(Size) << "B ";
 440       if (Left > Size)
 441          Left -= Size;
 442       else
 443          Left = 0;
 444    }
 445    else
 446       c0out << ' ';
 447
 448    struct timeval NewTime;
 449    gettimeofday(&NewTime,0);
 450    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 451                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 452
 453    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 454      c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
 455
 456    c0out << Contents.Stats.Packages << " files " <<
 457       SizeToStr(Contents.Stats.Bytes) << "B " <<
 458       TimeToStr((long)Delta) << endl;
 459
 460    return true;
 461 }
 462                                                                         /*}}}*/
 463
 464 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 465 // ---------------------------------------------------------------------
 466 /* This populates the PkgList with all the possible permutations of the
 467    section/arch lists. */
 468 static void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 469 {
 470    // Load the defaults
 471    string DDir = Setup.Find("TreeDefault::Directory",
 472                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 473    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 474                             "$(DIST)/$(SECTION)/source/");
 475    string DPkg = Setup.Find("TreeDefault::Packages",
 476                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 477    string DTrans = Setup.Find("TreeDefault::Translation",
 478                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 479    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 480                             "$(DIST)/$(SECTION)/");
 481    string DContents = Setup.Find("TreeDefault::Contents",
 482                             "$(DIST)/$(SECTION)/Contents-$(ARCH)");
 483    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 484    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 485                                "packages-$(ARCH).db");
 486    string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
 487                                "sources-$(SECTION).db");
 488    string DSources = Setup.Find("TreeDefault::Sources",
 489                                 "$(DIST)/$(SECTION)/source/Sources");
 490    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 491    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 492
 493    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 494
 495    bool const LongDescription = Setup.FindB("Default::LongDescription",
 496                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 497    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 498
 499    // Process 'tree' type sections
 500    const Configuration::Item *Top = Setup.Tree("tree");
 501    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 502    {
 503       Configuration Block(Top);
 504       string Dist = Top->Tag;
 505
 506       // Parse the sections
 507       string Tmp = Block.Find("Sections");
 508       const char *Sections = Tmp.c_str();
 509       string Section;
 510       while (ParseQuoteWord(Sections,Section) == true)
 511       {
 512          string Arch;
 513          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 514                                          {"$(SECTION)",&Section},
 515                                          {"$(ARCH)",&Arch},
 516                                          {NULL, NULL}};
 517          mode_t const Perms = Block.FindI("FileMode", Permissions);
 518          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 519          TranslationWriter *TransWriter;
 520          if (DTrans.empty() == false && LongDesc == false)
 521          {
 522             string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 523                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 524             string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 525             TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 526          }
 527          else
 528             TransWriter = NULL;
 529
 530          string const Tmp2 = Block.Find("Architectures");
 531          const char *Archs = Tmp2.c_str();
 532          while (ParseQuoteWord(Archs,Arch) == true)
 533          {
 534             PackageMap Itm;
 535             Itm.Permissions = Perms;
 536             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 537             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 538
 539             if (stringcasecmp(Arch,"source") == 0)
 540             {
 541                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 542                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 543                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 544                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 545                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 546                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 547                Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);
 548             }
 549             else
 550             {
 551                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 552                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 553                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 554                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 555                Itm.Arch = Arch;
 556                Itm.LongDesc = LongDesc;
 557                if (TransWriter != NULL)
 558                {
 559                   TransWriter->IncreaseRefCounter();
 560                   Itm.TransWriter = TransWriter;
 561                }
 562                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 563                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 564                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 565                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 566             }
 567
 568             Itm.GetGeneral(Setup,Block);
 569             PkgList.push_back(Itm);
 570          }
 571          // we didn't use this TransWriter, so we can release it
 572          if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
 573             delete TransWriter;
 574       }
 575
 576       Top = Top->Next;
 577    }
 578 }
 579                                                                         /*}}}*/
 580 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 581 // ---------------------------------------------------------------------
 582 /* */
 583 static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 584 {
 585    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 586
 587    // Process 'bindirectory' type sections
 588    const Configuration::Item *Top = Setup.Tree("bindirectory");
 589    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 590    {
 591       Configuration Block(Top);
 592
 593       PackageMap Itm;
 594       Itm.PkgFile = Block.Find("Packages");
 595       Itm.SrcFile = Block.Find("Sources");
 596       Itm.BinCacheDB = Block.Find("BinCacheDB");
 597       Itm.SrcCacheDB = Block.Find("SrcCacheDB");
 598       Itm.BinOverride = Block.Find("BinOverride");
 599       Itm.ExtraOverride = Block.Find("ExtraOverride");
 600       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 601       Itm.SrcOverride = Block.Find("SrcOverride");
 602       Itm.BaseDir = Top->Tag;
 603       Itm.FLFile = Block.Find("FileList");
 604       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 605       Itm.Contents = Block.Find("Contents");
 606       Itm.ContentsHead = Block.Find("Contents::Header");
 607       Itm.Permissions = Block.FindI("FileMode", Permissions);
 608
 609       Itm.GetGeneral(Setup,Block);
 610       PkgList.push_back(Itm);
 611
 612       Top = Top->Next;
 613    }
 614 }
 615                                                                         /*}}}*/
 616
 617 // ShowHelp - Show the help text                                        /*{{{*/
 618 // ---------------------------------------------------------------------
 619 /* */
 620 static bool ShowHelp(CommandLine &)
 621 {
 622    ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
 623             COMMON_ARCH,__DATE__,__TIME__);
 624    if (_config->FindB("version") == true)
 625       return true;
 626
 627    cout <<
 628     _("Usage: apt-ftparchive [options] command\n"
 629       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 630       "          sources srcpath [overridefile [pathprefix]]\n"
 631       "          contents path\n"
 632       "          release path\n"
 633       "          generate config [groups]\n"
 634       "          clean config\n"
 635       "\n"
 636       "apt-ftparchive generates index files for Debian archives. It supports\n"
 637       "many styles of generation from fully automated to functional replacements\n"
 638       "for dpkg-scanpackages and dpkg-scansources\n"
 639       "\n"
 640       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 641       "Package file contains the contents of all the control fields from\n"
 642       "each package as well as the MD5 hash and filesize. An override file\n"
 643       "is supported to force the value of Priority and Section.\n"
 644       "\n"
 645       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 646       "The --source-override option can be used to specify a src override file\n"
 647       "\n"
 648       "The 'packages' and 'sources' command should be run in the root of the\n"
 649       "tree. BinaryPath should point to the base of the recursive search and \n"
 650       "override file should contain the override flags. Pathprefix is\n"
 651       "appended to the filename fields if present. Example usage from the \n"
 652       "Debian archive:\n"
 653       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 654       "               dists/potato/main/binary-i386/Packages\n"
 655       "\n"
 656       "Options:\n"
 657       "  -h    This help text\n"
 658       "  --md5 Control MD5 generation\n"
 659       "  -s=?  Source override file\n"
 660       "  -q    Quiet\n"
 661       "  -d=?  Select the optional caching database\n"
 662       "  --no-delink Enable delinking debug mode\n"
 663       "  --contents  Control contents file generation\n"
 664       "  -c=?  Read this configuration file\n"
 665       "  -o=?  Set an arbitrary configuration option") << endl;
 666
 667    return true;
 668 }
 669                                                                         /*}}}*/
 670 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 671 // ---------------------------------------------------------------------
 672 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 673 static bool SimpleGenPackages(CommandLine &CmdL)
 674 {
 675    if (CmdL.FileSize() < 2)
 676       return ShowHelp(CmdL);
 677
 678    string Override;
 679    if (CmdL.FileSize() >= 3)
 680       Override = CmdL.FileList[2];
 681
 682    // Create a package writer object.
 683    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 684                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 685    if (_error->PendingError() == true)
 686       return false;
 687
 688    if (CmdL.FileSize() >= 4)
 689       Packages.PathPrefix = CmdL.FileList[3];
 690
 691    // Do recursive directory searching
 692    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 693       return false;
 694
 695    // Give some stats if asked for
 696    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 697      c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
 698
 699    return true;
 700 }
 701                                                                         /*}}}*/
 702 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 703 // ---------------------------------------------------------------------
 704 /* */
 705 static bool SimpleGenContents(CommandLine &CmdL)
 706 {
 707    if (CmdL.FileSize() < 2)
 708       return ShowHelp(CmdL);
 709
 710    // Create a package writer object.
 711    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 712    if (_error->PendingError() == true)
 713       return false;
 714
 715    // Do recursive directory searching
 716    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 717       return false;
 718
 719    Contents.Finish();
 720
 721    return true;
 722 }
 723                                                                         /*}}}*/
 724 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 725 // ---------------------------------------------------------------------
 726 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 727 static bool SimpleGenSources(CommandLine &CmdL)
 728 {
 729    if (CmdL.FileSize() < 2)
 730       return ShowHelp(CmdL);
 731
 732    string Override;
 733    if (CmdL.FileSize() >= 3)
 734       Override = CmdL.FileList[2];
 735
 736    string SOverride;
 737    if (Override.empty() == false)
 738       SOverride = Override + ".src";
 739
 740    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 741                              SOverride.c_str());
 742
 743    // Create a package writer object.
 744    SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),Override,SOverride);
 745    if (_error->PendingError() == true)
 746       return false;
 747
 748    if (CmdL.FileSize() >= 4)
 749       Sources.PathPrefix = CmdL.FileList[3];
 750
 751    // Do recursive directory searching
 752    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 753       return false;
 754
 755    // Give some stats if asked for
 756    if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
 757      c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl;
 758
 759    return true;
 760 }
 761                                                                         /*}}}*/
 762 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 763 // ---------------------------------------------------------------------
 764 static bool SimpleGenRelease(CommandLine &CmdL)
 765 {
 766    if (CmdL.FileSize() < 2)
 767       return ShowHelp(CmdL);
 768
 769    string Dir = CmdL.FileList[1];
 770
 771    ReleaseWriter Release("");
 772    Release.DirStrip = Dir;
 773
 774    if (_error->PendingError() == true)
 775       return false;
 776
 777    if (Release.RecursiveScan(Dir) == false)
 778       return false;
 779
 780    Release.Finish();
 781
 782    return true;
 783 }
 784
 785                                                                         /*}}}*/
 786 // DoGeneratePackagesAndSources - Helper for Generate                   /*{{{*/
 787 // ---------------------------------------------------------------------
 788 static bool DoGeneratePackagesAndSources(Configuration &Setup,
 789                                          vector<PackageMap> &PkgList,
 790                                          struct CacheDB::Stats &SrcStats,
 791                                          struct CacheDB::Stats &Stats,
 792                                          CommandLine &CmdL)
 793 {
 794    if (CmdL.FileSize() <= 2)
 795    {
 796       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 797          if (I->GenPackages(Setup,Stats) == false)
 798             _error->DumpErrors();
 799       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 800          if (I->GenSources(Setup,SrcStats) == false)
 801             _error->DumpErrors();
 802    }
 803    else
 804    {
 805       // Make a choice list out of the package list..
 806       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 807       RxChoiceList *End = List;
 808       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 809       {
 810          End->UserData = &(*I);
 811          End->Str = I->BaseDir.c_str();
 812          End++;
 813
 814          End->UserData = &(*I);
 815          End->Str = I->Tag.c_str();
 816          End++;
 817       }
 818       End->Str = 0;
 819
 820       // Regex it
 821       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 822       {
 823          delete [] List;
 824          return _error->Error(_("No selections matched"));
 825       }
 826       _error->DumpErrors();
 827
 828       // Do the generation for Packages
 829       for (End = List; End->Str != 0; End++)
 830       {
 831          if (End->Hit == false)
 832             continue;
 833
 834          PackageMap *I = (PackageMap *)End->UserData;
 835          if (I->PkgDone == true)
 836             continue;
 837          if (I->GenPackages(Setup,Stats) == false)
 838             _error->DumpErrors();
 839       }
 840
 841       // Do the generation for Sources
 842       for (End = List; End->Str != 0; End++)
 843       {
 844          if (End->Hit == false)
 845             continue;
 846
 847          PackageMap *I = (PackageMap *)End->UserData;
 848          if (I->SrcDone == true)
 849             continue;
 850          if (I->GenSources(Setup,SrcStats) == false)
 851             _error->DumpErrors();
 852       }
 853
 854       delete [] List;
 855    }
 856
 857    // close the Translation master files
 858    for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I)
 859       if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
 860          delete I->TransWriter;
 861
 862    return true;
 863 }
 864
 865                                                                         /*}}}*/
 866 // DoGenerateContents - Helper for Generate to generate the Contents    /*{{{*/
 867 // ---------------------------------------------------------------------
 868 static bool DoGenerateContents(Configuration &Setup,
 869                                vector<PackageMap> &PkgList,
 870                                CommandLine &CmdL)
 871 {
 872    c1out << "Packages done, Starting contents." << endl;
 873
 874    // Sort the contents file list by date
 875    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 876    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 877    {
 878       struct stat A;
 879       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 880                                  I->CntCompress,A) == false)
 881          time(&I->ContentsMTime);
 882       else
 883          I->ContentsMTime = A.st_mtime;
 884    }
 885    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 886
 887    /* Now for Contents.. The process here is to do a make-like dependency
 888       check. Each contents file is verified to be newer than the package files
 889       that describe the debs it indexes. Since the package files contain
 890       hashes of the .debs this means they have not changed either so the
 891       contents must be up to date. */
 892    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 893    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 894    {
 895       // This record is not relevant
 896       if (I->ContentsDone == true ||
 897           I->Contents.empty() == true)
 898          continue;
 899
 900       // Do not do everything if the user specified sections.
 901       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 902          continue;
 903
 904       struct stat A,B;
 905       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 906       {
 907          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 908          {
 909             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 910             continue;
 911          }
 912
 913          if (A.st_mtime > B.st_mtime)
 914             continue;
 915       }
 916
 917       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 918                          MaxContentsChange) == false)
 919          _error->DumpErrors();
 920
 921       // Hit the limit?
 922       if (MaxContentsChange == 0)
 923       {
 924          c1out << "Hit contents update byte limit" << endl;
 925          break;
 926       }
 927    }
 928
 929    return true;
 930 }
 931
 932                                                                         /*}}}*/
 933 // Generate - Full generate, using a config file                        /*{{{*/
 934 // ---------------------------------------------------------------------
 935 /* */
 936 static bool Generate(CommandLine &CmdL)
 937 {
 938    struct CacheDB::Stats SrcStats;
 939    if (CmdL.FileSize() < 2)
 940       return ShowHelp(CmdL);
 941
 942    struct timeval StartTime;
 943    gettimeofday(&StartTime,0);
 944    struct CacheDB::Stats Stats;
 945
 946    // Read the configuration file.
 947    Configuration Setup;
 948    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 949       return false;
 950
 951    vector<PackageMap> PkgList;
 952    LoadTree(PkgList,Setup);
 953    LoadBinDir(PkgList,Setup);
 954
 955    // Sort by cache DB to improve IO locality.
 956    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 957    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
 958
 959    // Generate packages
 960    if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false)
 961    {
 962       if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false)
 963          return false;
 964    } else {
 965       c1out << "Skipping Packages/Sources generation" << endl;
 966    }
 967
 968    // do Contents if needed
 969    if (_config->FindB("APT::FTPArchive::Contents", true) == true)
 970       if (DoGenerateContents(Setup, PkgList, CmdL) == false)
 971          return false;
 972
 973    struct timeval NewTime;
 974    gettimeofday(&NewTime,0);
 975    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 976                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 977    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 978          << " archives. Took " << TimeToStr((long)Delta) << endl;
 979
 980    return true;
 981 }
 982
 983                                                                         /*}}}*/
 984 // Clean - Clean out the databases                                      /*{{{*/
 985 // ---------------------------------------------------------------------
 986 /* */
 987 static bool Clean(CommandLine &CmdL)
 988 {
 989    if (CmdL.FileSize() != 2)
 990       return ShowHelp(CmdL);
 991
 992    // Read the configuration file.
 993    Configuration Setup;
 994    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 995       return false;
 996
 997    vector<PackageMap> PkgList;
 998    LoadTree(PkgList,Setup);
 999    LoadBinDir(PkgList,Setup);
1000
1001    // Sort by cache DB to improve IO locality.
1002    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
1003    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
1004
1005    string CacheDir = Setup.FindDir("Dir::CacheDir");
1006
1007    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
1008    {
1009       if(I->BinCacheDB != "")
1010          c0out << I->BinCacheDB << endl;
1011       if(I->SrcCacheDB != "")
1012          c0out << I->SrcCacheDB << endl;
1013       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
1014       CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));
1015       if (DB.Clean() == false)
1016          _error->DumpErrors();
1017       if (DB_SRC.Clean() == false)
1018          _error->DumpErrors();
1019
1020       string CacheDB = I->BinCacheDB;
1021       string SrcCacheDB = I->SrcCacheDB;
1022       while(I != PkgList.end() &&
1023             I->BinCacheDB == CacheDB &&
1024             I->SrcCacheDB == SrcCacheDB)
1025          ++I;
1026
1027    }
1028
1029
1030    return true;
1031 }
1032                                                                         /*}}}*/
1033
1034 int main(int argc, const char *argv[])
1035 {
1036    setlocale(LC_ALL, "");
1037    CommandLine::Args Args[] = {
1038       {'h',"help","help",0},
1039       {0,"md5","APT::FTPArchive::MD5",0},
1040       {0,"sha1","APT::FTPArchive::SHA1",0},
1041       {0,"sha256","APT::FTPArchive::SHA256",0},
1042       {'v',"version","version",0},
1043       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
1044       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
1045       {'q',"quiet","quiet",CommandLine::IntLevel},
1046       {'q',"silent","quiet",CommandLine::IntLevel},
1047       {0,"delink","APT::FTPArchive::DeLinkAct",0},
1048       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
1049       {0,"contents","APT::FTPArchive::Contents",0},
1050       {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
1051       {'c',"config-file",0,CommandLine::ConfigFile},
1052       {'o',"option",0,CommandLine::ArbItem},
1053       {0,0,0,0}};
1054    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
1055                                    {"contents",&SimpleGenContents},
1056                                    {"sources",&SimpleGenSources},
1057                                    {"release",&SimpleGenRelease},
1058                                    {"generate",&Generate},
1059                                    {"clean",&Clean},
1060                                    {"help",&ShowHelp},
1061                                    {0,0}};
1062
1063    // Parse the command line and initialize the package library
1064    CommandLine CmdL(Args,_config);
1065    ParseCommandLine(CmdL, Cmds, Args, &_config, NULL, argc, argv, ShowHelp);
1066
1067    // Setup the output streams
1068    c0out.rdbuf(clog.rdbuf());
1069    c1out.rdbuf(clog.rdbuf());
1070    c2out.rdbuf(clog.rdbuf());
1071    Quiet = _config->FindI("quiet",0);
1072    if (Quiet > 0)
1073       c0out.rdbuf(devnull.rdbuf());
1074    if (Quiet > 1)
1075       c1out.rdbuf(devnull.rdbuf());
1076
1077    // Match the operation
1078    CmdL.DispatchArg(Cmds);
1079
1080    if (_error->empty() == false)
1081    {
1082       bool Errors = _error->PendingError();
1083       _error->DumpErrors();
1084       return Errors == true?100:0;
1085    }
1086    return 0;
1087 }