ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <apt-pkg/fileutl.h>
  21
  22 #include <algorithm>
  23 #include <climits>
  24 #include <sys/time.h>
  25 #include <locale.h>
  26 #include <stdio.h>
  27 #include <sys/stat.h>
  28 #include <time.h>
  29 #include <functional>
  30 #include <iostream>
  31 #include <string>
  32 #include <vector>
  33
  34 #include "cachedb.h"
  35 #include "override.h"
  36 #include "apt-ftparchive.h"
  37 #include "multicompress.h"
  38 #include "writer.h"
  39
  40 #include <apti18n.h>
  41                                                                         /*}}}*/
  42
  43 using namespace std;
  44 ostream c0out(0);
  45 ostream c1out(0);
  46 ostream c2out(0);
  47 ofstream devnull("/dev/null");
  48 unsigned Quiet = 0;
  49
  50 // struct PackageMap - List of all package files in the config file     /*{{{*/
  51 // ---------------------------------------------------------------------
  52 /* */
  53 struct PackageMap
  54 {
  55    // General Stuff
  56    string BaseDir;
  57    string InternalPrefix;
  58    string FLFile;
  59    string PkgExt;
  60    string SrcExt;
  61
  62    // Stuff for the Package File
  63    string PkgFile;
  64    string BinCacheDB;
  65    string BinOverride;
  66    string ExtraOverride;
  67
  68    // We generate for this given arch
  69    string Arch;
  70
  71    // Stuff for the Source File
  72    string SrcFile;
  73    string SrcOverride;
  74    string SrcExtraOverride;
  75
  76    // Translation master file
  77    bool LongDesc;
  78    TranslationWriter *TransWriter;
  79
  80    // Contents
  81    string Contents;
  82    string ContentsHead;
  83
  84    // Random things
  85    string Tag;
  86    string PkgCompress;
  87    string CntCompress;
  88    string SrcCompress;
  89    string PathPrefix;
  90    unsigned int DeLinkLimit;
  91    mode_t Permissions;
  92
  93    bool ContentsDone;
  94    bool PkgDone;
  95    bool SrcDone;
  96    time_t ContentsMTime;
  97
  98    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  99    {
 100       inline bool operator() (const PackageMap &x,const PackageMap &y)
 101       {return x.ContentsMTime < y.ContentsMTime;};
 102    };
 103
 104    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
 105    {
 106       inline bool operator() (const PackageMap &x,const PackageMap &y)
 107       {return x.BinCacheDB < y.BinCacheDB;};
 108    };
 109
 110    void GetGeneral(Configuration &Setup,Configuration &Block);
 111    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 112    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 113    bool GenContents(Configuration &Setup,
 114                     vector<PackageMap>::iterator Begin,
 115                     vector<PackageMap>::iterator End,
 116                     unsigned long &Left);
 117
 118    PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 119                   ContentsDone(false), PkgDone(false), SrcDone(false),
 120                   ContentsMTime(0) {};
 121 };
 122                                                                         /*}}}*/
 123
 124 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 125 // ---------------------------------------------------------------------
 126 /* */
 127 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 128 {
 129    PathPrefix = Block.Find("PathPrefix");
 130
 131    if (Block.FindB("External-Links",true) == false)
 132       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 133    else
 134       DeLinkLimit = 0;
 135
 136    PkgCompress = Block.Find("Packages::Compress",
 137                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 138    CntCompress = Block.Find("Contents::Compress",
 139                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 140    SrcCompress = Block.Find("Sources::Compress",
 141                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 142
 143    SrcExt = Block.Find("Sources::Extensions",
 144                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 145    PkgExt = Block.Find("Packages::Extensions",
 146                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 147
 148    Permissions = Setup.FindI("Default::FileMode",0644);
 149
 150    if (FLFile.empty() == false)
 151       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 152
 153    if (Contents == " ")
 154       Contents= string();
 155 }
 156                                                                         /*}}}*/
 157 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 158 // ---------------------------------------------------------------------
 159 /* This generates the Package File described by this object. */
 160 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 161 {
 162    if (PkgFile.empty() == true)
 163       return true;
 164
 165    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 166    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 167    string CacheDir = Setup.FindDir("Dir::CacheDir");
 168
 169    struct timeval StartTime;
 170    gettimeofday(&StartTime,0);
 171
 172    PkgDone = true;
 173
 174    // Create a package writer object.
 175    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 176                            flCombine(OverrideDir,BinOverride),
 177                            flCombine(OverrideDir,ExtraOverride),
 178                            Arch);
 179    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 180       return _error->Error(_("Package extension list is too long"));
 181    if (_error->PendingError() == true)
 182       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 183
 184    Packages.PathPrefix = PathPrefix;
 185    Packages.DirStrip = ArchiveDir;
 186    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 187
 188    Packages.TransWriter = TransWriter;
 189    Packages.LongDescription = LongDesc;
 190
 191    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 192    Packages.DeLinkLimit = DeLinkLimit;
 193
 194    // Create a compressor object
 195    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 196                       PkgCompress,Permissions);
 197    Packages.Output = Comp.Input;
 198    if (_error->PendingError() == true)
 199       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 200
 201    c0out << ' ' << BaseDir << ":" << flush;
 202
 203    // Do recursive directory searching
 204    if (FLFile.empty() == true)
 205    {
 206       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 207          return false;
 208    }
 209    else
 210    {
 211       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 212          return false;
 213    }
 214
 215    Packages.Output = 0;      // Just in case
 216
 217    // Finish compressing
 218    unsigned long long Size;
 219    if (Comp.Finalize(Size) == false)
 220    {
 221       c0out << endl;
 222       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 223    }
 224
 225    if (Size != 0)
 226       c0out << " New "
 227              << SizeToStr(Size) << "B ";
 228    else
 229       c0out << ' ';
 230
 231    struct timeval NewTime;
 232    gettimeofday(&NewTime,0);
 233    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 234                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 235
 236    c0out << Packages.Stats.Packages << " files " <<
 237 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 238       SizeToStr(Packages.Stats.Bytes) << "B " <<
 239       TimeToStr((long)Delta) << endl;
 240
 241    Stats.Add(Packages.Stats);
 242    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 243
 244    return !_error->PendingError();
 245 }
 246
 247                                                                         /*}}}*/
 248 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 249 // ---------------------------------------------------------------------
 250 /* This generates the Sources File described by this object. */
 251 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 252 {
 253    if (SrcFile.empty() == true)
 254       return true;
 255
 256    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 257    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 258    string CacheDir = Setup.FindDir("Dir::CacheDir");
 259
 260    struct timeval StartTime;
 261    gettimeofday(&StartTime,0);
 262
 263    SrcDone = true;
 264
 265    // Create a package writer object.
 266    SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),
 267                          flCombine(OverrideDir,BinOverride),
 268                          flCombine(OverrideDir,SrcOverride),
 269                          flCombine(OverrideDir,SrcExtraOverride));
 270    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 271       return _error->Error(_("Source extension list is too long"));
 272    if (_error->PendingError() == true)
 273       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 274
 275    Sources.PathPrefix = PathPrefix;
 276    Sources.DirStrip = ArchiveDir;
 277    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 278
 279    Sources.DeLinkLimit = DeLinkLimit;
 280    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 281
 282    // Create a compressor object
 283    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 284                       SrcCompress,Permissions);
 285    Sources.Output = Comp.Input;
 286    if (_error->PendingError() == true)
 287       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 288
 289    c0out << ' ' << BaseDir << ":" << flush;
 290
 291    // Do recursive directory searching
 292    if (FLFile.empty() == true)
 293    {
 294       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 295          return false;
 296    }
 297    else
 298    {
 299       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 300          return false;
 301    }
 302    Sources.Output = 0;      // Just in case
 303
 304    // Finish compressing
 305    unsigned long long Size;
 306    if (Comp.Finalize(Size) == false)
 307    {
 308       c0out << endl;
 309       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 310    }
 311
 312    if (Size != 0)
 313       c0out << " New "
 314              << SizeToStr(Size) << "B ";
 315    else
 316       c0out << ' ';
 317
 318    struct timeval NewTime;
 319    gettimeofday(&NewTime,0);
 320    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 321                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 322
 323    c0out << Sources.Stats.Packages << " pkgs in " <<
 324       TimeToStr((long)Delta) << endl;
 325
 326    Stats.Add(Sources.Stats);
 327    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 328
 329    return !_error->PendingError();
 330 }
 331                                                                         /*}}}*/
 332 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 333 // ---------------------------------------------------------------------
 334 /* This generates the contents file partially described by this object.
 335    It searches the given iterator range for other package files that map
 336    into this contents file and includes their data as well when building. */
 337 bool PackageMap::GenContents(Configuration &Setup,
 338                              vector<PackageMap>::iterator Begin,
 339                              vector<PackageMap>::iterator End,
 340                              unsigned long &Left)
 341 {
 342    if (Contents.empty() == true)
 343       return true;
 344
 345    if (Left == 0)
 346       return true;
 347
 348    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 349    string CacheDir = Setup.FindDir("Dir::CacheDir");
 350    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 351
 352    struct timeval StartTime;
 353    gettimeofday(&StartTime,0);
 354
 355    // Create a package writer object.
 356    ContentsWriter Contents("", Arch);
 357    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 358       return _error->Error(_("Package extension list is too long"));
 359    if (_error->PendingError() == true)
 360       return false;
 361
 362    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 363                       CntCompress,Permissions);
 364    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 365    Contents.Output = Comp.Input;
 366    if (_error->PendingError() == true)
 367       return false;
 368
 369    // Write the header out.
 370    if (ContentsHead.empty() == false)
 371    {
 372       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 373       if (_error->PendingError() == true)
 374          return false;
 375
 376       unsigned long long Size = Head.Size();
 377       unsigned char Buf[4096];
 378       while (Size != 0)
 379       {
 380          unsigned long long ToRead = Size;
 381          if (Size > sizeof(Buf))
 382             ToRead = sizeof(Buf);
 383
 384          if (Head.Read(Buf,ToRead) == false)
 385             return false;
 386
 387          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 388             return _error->Errno("fwrite",_("Error writing header to contents file"));
 389
 390          Size -= ToRead;
 391       }
 392    }
 393
 394    /* Go over all the package file records and parse all the package
 395       files associated with this contents file into one great big honking
 396       memory structure, then dump the sorted version */
 397    c0out << ' ' << this->Contents << ":" << flush;
 398    for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
 399    {
 400       if (I->Contents != this->Contents)
 401          continue;
 402
 403       Contents.Prefix = ArchiveDir;
 404       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 405       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 406                             I->PkgCompress);
 407
 408       I->ContentsDone = true;
 409    }
 410
 411    Contents.Finish();
 412
 413    // Finish compressing
 414    unsigned long long Size;
 415    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 416    {
 417       c0out << endl;
 418       return _error->Error(_("Error processing contents %s"),
 419                            this->Contents.c_str());
 420    }
 421
 422    if (Size != 0)
 423    {
 424       c0out << " New " << SizeToStr(Size) << "B ";
 425       if (Left > Size)
 426          Left -= Size;
 427       else
 428          Left = 0;
 429    }
 430    else
 431       c0out << ' ';
 432
 433    struct timeval NewTime;
 434    gettimeofday(&NewTime,0);
 435    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 436                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 437
 438    c0out << Contents.Stats.Packages << " files " <<
 439       SizeToStr(Contents.Stats.Bytes) << "B " <<
 440       TimeToStr((long)Delta) << endl;
 441
 442    return true;
 443 }
 444                                                                         /*}}}*/
 445
 446 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 447 // ---------------------------------------------------------------------
 448 /* This populates the PkgList with all the possible permutations of the
 449    section/arch lists. */
 450 static void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 451 {
 452    // Load the defaults
 453    string DDir = Setup.Find("TreeDefault::Directory",
 454                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 455    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 456                             "$(DIST)/$(SECTION)/source/");
 457    string DPkg = Setup.Find("TreeDefault::Packages",
 458                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 459    string DTrans = Setup.Find("TreeDefault::Translation",
 460                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 461    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 462                             "$(DIST)/$(SECTION)/");
 463    string DContents = Setup.Find("TreeDefault::Contents",
 464                             "$(DIST)/$(SECTION)/Contents-$(ARCH)");
 465    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 466    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 467                                "packages-$(ARCH).db");
 468    string DSources = Setup.Find("TreeDefault::Sources",
 469                                 "$(DIST)/$(SECTION)/source/Sources");
 470    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 471    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 472
 473    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 474
 475    bool const LongDescription = Setup.FindB("Default::LongDescription",
 476                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 477    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 478
 479    // Process 'tree' type sections
 480    const Configuration::Item *Top = Setup.Tree("tree");
 481    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 482    {
 483       Configuration Block(Top);
 484       string Dist = Top->Tag;
 485
 486       // Parse the sections
 487       string Tmp = Block.Find("Sections");
 488       const char *Sections = Tmp.c_str();
 489       string Section;
 490       while (ParseQuoteWord(Sections,Section) == true)
 491       {
 492          string Arch;
 493          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 494                                          {"$(SECTION)",&Section},
 495                                          {"$(ARCH)",&Arch},
 496                                          {NULL, NULL}};
 497          mode_t const Perms = Block.FindI("FileMode", Permissions);
 498          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 499          TranslationWriter *TransWriter;
 500          if (DTrans.empty() == false && LongDesc == false)
 501          {
 502             string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 503                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 504             string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 505             TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 506          }
 507          else
 508             TransWriter = NULL;
 509
 510          string const Tmp2 = Block.Find("Architectures");
 511          const char *Archs = Tmp2.c_str();
 512          while (ParseQuoteWord(Archs,Arch) == true)
 513          {
 514             PackageMap Itm;
 515             Itm.Permissions = Perms;
 516             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 517             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 518
 519             if (stringcasecmp(Arch,"source") == 0)
 520             {
 521                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 522                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 523                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 524                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 525                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 526                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 527             }
 528             else
 529             {
 530                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 531                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 532                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 533                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 534                Itm.Arch = Arch;
 535                Itm.LongDesc = LongDesc;
 536                if (TransWriter != NULL)
 537                {
 538                   TransWriter->IncreaseRefCounter();
 539                   Itm.TransWriter = TransWriter;
 540                }
 541                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 542                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 543                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 544                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 545             }
 546
 547             Itm.GetGeneral(Setup,Block);
 548             PkgList.push_back(Itm);
 549          }
 550          // we didn't use this TransWriter, so we can release it
 551          if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
 552             delete TransWriter;
 553       }
 554
 555       Top = Top->Next;
 556    }
 557 }
 558                                                                         /*}}}*/
 559 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 560 // ---------------------------------------------------------------------
 561 /* */
 562 static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 563 {
 564    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 565
 566    // Process 'bindirectory' type sections
 567    const Configuration::Item *Top = Setup.Tree("bindirectory");
 568    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 569    {
 570       Configuration Block(Top);
 571
 572       PackageMap Itm;
 573       Itm.PkgFile = Block.Find("Packages");
 574       Itm.SrcFile = Block.Find("Sources");
 575       Itm.BinCacheDB = Block.Find("BinCacheDB");
 576       Itm.BinOverride = Block.Find("BinOverride");
 577       Itm.ExtraOverride = Block.Find("ExtraOverride");
 578       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 579       Itm.SrcOverride = Block.Find("SrcOverride");
 580       Itm.BaseDir = Top->Tag;
 581       Itm.FLFile = Block.Find("FileList");
 582       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 583       Itm.Contents = Block.Find("Contents");
 584       Itm.ContentsHead = Block.Find("Contents::Header");
 585       Itm.Permissions = Block.FindI("FileMode", Permissions);
 586
 587       Itm.GetGeneral(Setup,Block);
 588       PkgList.push_back(Itm);
 589
 590       Top = Top->Next;
 591    }
 592 }
 593                                                                         /*}}}*/
 594
 595 // ShowHelp - Show the help text                                        /*{{{*/
 596 // ---------------------------------------------------------------------
 597 /* */
 598 static bool ShowHelp(CommandLine &)
 599 {
 600    ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
 601             COMMON_ARCH,__DATE__,__TIME__);
 602    if (_config->FindB("version") == true)
 603       return true;
 604
 605    cout <<
 606     _("Usage: apt-ftparchive [options] command\n"
 607       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 608       "          sources srcpath [overridefile [pathprefix]]\n"
 609       "          contents path\n"
 610       "          release path\n"
 611       "          generate config [groups]\n"
 612       "          clean config\n"
 613       "\n"
 614       "apt-ftparchive generates index files for Debian archives. It supports\n"
 615       "many styles of generation from fully automated to functional replacements\n"
 616       "for dpkg-scanpackages and dpkg-scansources\n"
 617       "\n"
 618       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 619       "Package file contains the contents of all the control fields from\n"
 620       "each package as well as the MD5 hash and filesize. An override file\n"
 621       "is supported to force the value of Priority and Section.\n"
 622       "\n"
 623       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 624       "The --source-override option can be used to specify a src override file\n"
 625       "\n"
 626       "The 'packages' and 'sources' command should be run in the root of the\n"
 627       "tree. BinaryPath should point to the base of the recursive search and \n"
 628       "override file should contain the override flags. Pathprefix is\n"
 629       "appended to the filename fields if present. Example usage from the \n"
 630       "Debian archive:\n"
 631       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 632       "               dists/potato/main/binary-i386/Packages\n"
 633       "\n"
 634       "Options:\n"
 635       "  -h    This help text\n"
 636       "  --md5 Control MD5 generation\n"
 637       "  -s=?  Source override file\n"
 638       "  -q    Quiet\n"
 639       "  -d=?  Select the optional caching database\n"
 640       "  --no-delink Enable delinking debug mode\n"
 641       "  --contents  Control contents file generation\n"
 642       "  -c=?  Read this configuration file\n"
 643       "  -o=?  Set an arbitrary configuration option") << endl;
 644
 645    return true;
 646 }
 647                                                                         /*}}}*/
 648 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 649 // ---------------------------------------------------------------------
 650 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 651 static bool SimpleGenPackages(CommandLine &CmdL)
 652 {
 653    if (CmdL.FileSize() < 2)
 654       return ShowHelp(CmdL);
 655
 656    string Override;
 657    if (CmdL.FileSize() >= 3)
 658       Override = CmdL.FileList[2];
 659
 660    // Create a package writer object.
 661    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 662                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 663    if (_error->PendingError() == true)
 664       return false;
 665
 666    if (CmdL.FileSize() >= 4)
 667       Packages.PathPrefix = CmdL.FileList[3];
 668
 669    // Do recursive directory searching
 670    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 671       return false;
 672
 673    return true;
 674 }
 675                                                                         /*}}}*/
 676 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 677 // ---------------------------------------------------------------------
 678 /* */
 679 static bool SimpleGenContents(CommandLine &CmdL)
 680 {
 681    if (CmdL.FileSize() < 2)
 682       return ShowHelp(CmdL);
 683
 684    // Create a package writer object.
 685    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 686    if (_error->PendingError() == true)
 687       return false;
 688
 689    // Do recursive directory searching
 690    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 691       return false;
 692
 693    Contents.Finish();
 694
 695    return true;
 696 }
 697                                                                         /*}}}*/
 698 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 699 // ---------------------------------------------------------------------
 700 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 701 static bool SimpleGenSources(CommandLine &CmdL)
 702 {
 703    if (CmdL.FileSize() < 2)
 704       return ShowHelp(CmdL);
 705
 706    string Override;
 707    if (CmdL.FileSize() >= 3)
 708       Override = CmdL.FileList[2];
 709
 710    string SOverride;
 711    if (Override.empty() == false)
 712       SOverride = Override + ".src";
 713
 714    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 715                              SOverride.c_str());
 716
 717    // Create a package writer object.
 718    SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),Override,SOverride);
 719    if (_error->PendingError() == true)
 720       return false;
 721
 722    if (CmdL.FileSize() >= 4)
 723       Sources.PathPrefix = CmdL.FileList[3];
 724
 725    // Do recursive directory searching
 726    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 727       return false;
 728
 729    return true;
 730 }
 731                                                                         /*}}}*/
 732 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 733 // ---------------------------------------------------------------------
 734 static bool SimpleGenRelease(CommandLine &CmdL)
 735 {
 736    if (CmdL.FileSize() < 2)
 737       return ShowHelp(CmdL);
 738
 739    string Dir = CmdL.FileList[1];
 740
 741    ReleaseWriter Release("");
 742    Release.DirStrip = Dir;
 743
 744    if (_error->PendingError() == true)
 745       return false;
 746
 747    if (Release.RecursiveScan(Dir) == false)
 748       return false;
 749
 750    Release.Finish();
 751
 752    return true;
 753 }
 754
 755                                                                         /*}}}*/
 756 // Generate - Full generate, using a config file                        /*{{{*/
 757 // ---------------------------------------------------------------------
 758 /* */
 759 static bool Generate(CommandLine &CmdL)
 760 {
 761    struct CacheDB::Stats SrcStats;
 762    if (CmdL.FileSize() < 2)
 763       return ShowHelp(CmdL);
 764
 765    struct timeval StartTime;
 766    gettimeofday(&StartTime,0);
 767    struct CacheDB::Stats Stats;
 768
 769    // Read the configuration file.
 770    Configuration Setup;
 771    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 772       return false;
 773
 774    vector<PackageMap> PkgList;
 775    LoadTree(PkgList,Setup);
 776    LoadBinDir(PkgList,Setup);
 777
 778    // Sort by cache DB to improve IO locality.
 779    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 780
 781    // Generate packages
 782    if (CmdL.FileSize() <= 2)
 783    {
 784       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 785          if (I->GenPackages(Setup,Stats) == false)
 786             _error->DumpErrors();
 787       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 788          if (I->GenSources(Setup,SrcStats) == false)
 789             _error->DumpErrors();
 790    }
 791    else
 792    {
 793       // Make a choice list out of the package list..
 794       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 795       RxChoiceList *End = List;
 796       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 797       {
 798          End->UserData = &(*I);
 799          End->Str = I->BaseDir.c_str();
 800          End++;
 801
 802          End->UserData = &(*I);
 803          End->Str = I->Tag.c_str();
 804          End++;
 805       }
 806       End->Str = 0;
 807
 808       // Regex it
 809       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 810       {
 811          delete [] List;
 812          return _error->Error(_("No selections matched"));
 813       }
 814       _error->DumpErrors();
 815
 816       // Do the generation for Packages
 817       for (End = List; End->Str != 0; End++)
 818       {
 819          if (End->Hit == false)
 820             continue;
 821
 822          PackageMap *I = (PackageMap *)End->UserData;
 823          if (I->PkgDone == true)
 824             continue;
 825          if (I->GenPackages(Setup,Stats) == false)
 826             _error->DumpErrors();
 827       }
 828
 829       // Do the generation for Sources
 830       for (End = List; End->Str != 0; End++)
 831       {
 832          if (End->Hit == false)
 833             continue;
 834
 835          PackageMap *I = (PackageMap *)End->UserData;
 836          if (I->SrcDone == true)
 837             continue;
 838          if (I->GenSources(Setup,SrcStats) == false)
 839             _error->DumpErrors();
 840       }
 841
 842       delete [] List;
 843    }
 844
 845    // close the Translation master files
 846    for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I)
 847       if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
 848          delete I->TransWriter;
 849
 850    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 851       return true;
 852
 853    c1out << "Packages done, Starting contents." << endl;
 854
 855    // Sort the contents file list by date
 856    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 857    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 858    {
 859       struct stat A;
 860       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 861                                  I->CntCompress,A) == false)
 862          time(&I->ContentsMTime);
 863       else
 864          I->ContentsMTime = A.st_mtime;
 865    }
 866    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 867
 868    /* Now for Contents.. The process here is to do a make-like dependency
 869       check. Each contents file is verified to be newer than the package files
 870       that describe the debs it indexes. Since the package files contain
 871       hashes of the .debs this means they have not changed either so the
 872       contents must be up to date. */
 873    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 874    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 875    {
 876       // This record is not relevant
 877       if (I->ContentsDone == true ||
 878           I->Contents.empty() == true)
 879          continue;
 880
 881       // Do not do everything if the user specified sections.
 882       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 883          continue;
 884
 885       struct stat A,B;
 886       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 887       {
 888          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 889          {
 890             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 891             continue;
 892          }
 893
 894          if (A.st_mtime > B.st_mtime)
 895             continue;
 896       }
 897
 898       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 899                          MaxContentsChange) == false)
 900          _error->DumpErrors();
 901
 902       // Hit the limit?
 903       if (MaxContentsChange == 0)
 904       {
 905          c1out << "Hit contents update byte limit" << endl;
 906          break;
 907       }
 908    }
 909
 910    struct timeval NewTime;
 911    gettimeofday(&NewTime,0);
 912    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 913                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 914    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 915          << " archives. Took " << TimeToStr((long)Delta) << endl;
 916
 917    return true;
 918 }
 919                                                                         /*}}}*/
 920 // Clean - Clean out the databases                                      /*{{{*/
 921 // ---------------------------------------------------------------------
 922 /* */
 923 static bool Clean(CommandLine &CmdL)
 924 {
 925    if (CmdL.FileSize() != 2)
 926       return ShowHelp(CmdL);
 927
 928    // Read the configuration file.
 929    Configuration Setup;
 930    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 931       return false;
 932
 933    vector<PackageMap> PkgList;
 934    LoadTree(PkgList,Setup);
 935    LoadBinDir(PkgList,Setup);
 936
 937    // Sort by cache DB to improve IO locality.
 938    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 939
 940    string CacheDir = Setup.FindDir("Dir::CacheDir");
 941
 942    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 943    {
 944       c0out << I->BinCacheDB << endl;
 945       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 946       if (DB.Clean() == false)
 947          _error->DumpErrors();
 948
 949       string CacheDB = I->BinCacheDB;
 950       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; ++I);
 951    }
 952
 953    return true;
 954 }
 955                                                                         /*}}}*/
 956
 957 int main(int argc, const char *argv[])
 958 {
 959    setlocale(LC_ALL, "");
 960    CommandLine::Args Args[] = {
 961       {'h',"help","help",0},
 962       {0,"md5","APT::FTPArchive::MD5",0},
 963       {0,"sha1","APT::FTPArchive::SHA1",0},
 964       {0,"sha256","APT::FTPArchive::SHA256",0},
 965       {'v',"version","version",0},
 966       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 967       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 968       {'q',"quiet","quiet",CommandLine::IntLevel},
 969       {'q',"silent","quiet",CommandLine::IntLevel},
 970       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 971       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 972       {0,"contents","APT::FTPArchive::Contents",0},
 973       {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
 974       {'c',"config-file",0,CommandLine::ConfigFile},
 975       {'o',"option",0,CommandLine::ArbItem},
 976       {0,0,0,0}};
 977    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 978                                    {"contents",&SimpleGenContents},
 979                                    {"sources",&SimpleGenSources},
 980                                    {"release",&SimpleGenRelease},
 981                                    {"generate",&Generate},
 982                                    {"clean",&Clean},
 983                                    {"help",&ShowHelp},
 984                                    {0,0}};
 985
 986    // Parse the command line and initialize the package library
 987    CommandLine CmdL(Args,_config);
 988    if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false)
 989    {
 990       _error->DumpErrors();
 991       return 100;
 992    }
 993
 994    // See if the help should be shown
 995    if (_config->FindB("help") == true ||
 996        _config->FindB("version") == true ||
 997        CmdL.FileSize() == 0)
 998    {
 999       ShowHelp(CmdL);
1000       return 0;
1001    }
1002
1003    // Setup the output streams
1004    c0out.rdbuf(clog.rdbuf());
1005    c1out.rdbuf(clog.rdbuf());
1006    c2out.rdbuf(clog.rdbuf());
1007    Quiet = _config->FindI("quiet",0);
1008    if (Quiet > 0)
1009       c0out.rdbuf(devnull.rdbuf());
1010    if (Quiet > 1)
1011       c1out.rdbuf(devnull.rdbuf());
1012
1013    // Match the operation
1014    CmdL.DispatchArg(Cmds);
1015
1016    if (_error->empty() == false)
1017    {
1018       bool Errors = _error->PendingError();
1019       _error->DumpErrors();
1020       return Errors == true?100:0;
1021    }
1022    return 0;
1023 }