ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "apt-ftparchive.h"
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <config.h>
  21 #include <apti18n.h>
  22 #include <algorithm>
  23
  24 #include <climits>
  25 #include <sys/time.h>
  26 #include <regex.h>
  27
  28 #include "contents.h"
  29 #include "multicompress.h"
  30 #include "writer.h"
  31                                                                         /*}}}*/
  32
  33 using namespace std;
  34 ostream c0out(0);
  35 ostream c1out(0);
  36 ostream c2out(0);
  37 ofstream devnull("/dev/null");
  38 unsigned Quiet = 0;
  39
  40 // struct PackageMap - List of all package files in the config file     /*{{{*/
  41 // ---------------------------------------------------------------------
  42 /* */
  43 struct PackageMap
  44 {
  45    // General Stuff
  46    string BaseDir;
  47    string InternalPrefix;
  48    string FLFile;
  49    string PkgExt;
  50    string SrcExt;
  51
  52    // Stuff for the Package File
  53    string PkgFile;
  54    string BinCacheDB;
  55    string BinOverride;
  56    string ExtraOverride;
  57
  58    // We generate for this given arch
  59    string Arch;
  60
  61    // Stuff for the Source File
  62    string SrcFile;
  63    string SrcOverride;
  64    string SrcExtraOverride;
  65
  66    // Contents
  67    string Contents;
  68    string ContentsHead;
  69
  70    // Random things
  71    string Tag;
  72    string PkgCompress;
  73    string CntCompress;
  74    string SrcCompress;
  75    string PathPrefix;
  76    unsigned int DeLinkLimit;
  77    mode_t Permissions;
  78
  79    bool ContentsDone;
  80    bool PkgDone;
  81    bool SrcDone;
  82    time_t ContentsMTime;
  83
  84    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  85    {
  86       inline bool operator() (const PackageMap &x,const PackageMap &y)
  87       {return x.ContentsMTime < y.ContentsMTime;};
  88    };
  89
  90    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  91    {
  92       inline bool operator() (const PackageMap &x,const PackageMap &y)
  93       {return x.BinCacheDB < y.BinCacheDB;};
  94    };
  95
  96    void GetGeneral(Configuration &Setup,Configuration &Block);
  97    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
  98    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
  99    bool GenContents(Configuration &Setup,
 100                     vector<PackageMap>::iterator Begin,
 101                     vector<PackageMap>::iterator End,
 102                     unsigned long &Left);
 103
 104    PackageMap() : DeLinkLimit(0), Permissions(1), ContentsDone(false),
 105         PkgDone(false), SrcDone(false), ContentsMTime(0) {};
 106 };
 107                                                                         /*}}}*/
 108
 109 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 110 // ---------------------------------------------------------------------
 111 /* */
 112 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 113 {
 114    PathPrefix = Block.Find("PathPrefix");
 115
 116    if (Block.FindB("External-Links",true) == false)
 117       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 118    else
 119       DeLinkLimit = 0;
 120
 121    PkgCompress = Block.Find("Packages::Compress",
 122                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 123    CntCompress = Block.Find("Contents::Compress",
 124                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 125    SrcCompress = Block.Find("Sources::Compress",
 126                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 127
 128    SrcExt = Block.Find("Sources::Extensions",
 129                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 130    PkgExt = Block.Find("Packages::Extensions",
 131                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 132
 133    Permissions = Setup.FindI("Default::FileMode",0644);
 134
 135    if (FLFile.empty() == false)
 136       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 137
 138    if (Contents == " ")
 139       Contents= string();
 140 }
 141                                                                         /*}}}*/
 142 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 143 // ---------------------------------------------------------------------
 144 /* This generates the Package File described by this object. */
 145 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 146 {
 147    if (PkgFile.empty() == true)
 148       return true;
 149
 150    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 151    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 152    string CacheDir = Setup.FindDir("Dir::CacheDir");
 153
 154    struct timeval StartTime;
 155    gettimeofday(&StartTime,0);
 156
 157    PkgDone = true;
 158
 159    // Create a package writer object.
 160    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 161                            flCombine(OverrideDir,BinOverride),
 162                            flCombine(OverrideDir,ExtraOverride),
 163                            Arch);
 164    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 165       return _error->Error(_("Package extension list is too long"));
 166    if (_error->PendingError() == true)
 167       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 168
 169    Packages.PathPrefix = PathPrefix;
 170    Packages.DirStrip = ArchiveDir;
 171    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 172
 173    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 174    Packages.DeLinkLimit = DeLinkLimit;
 175
 176    // Create a compressor object
 177    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 178                       PkgCompress,Permissions);
 179    Packages.Output = Comp.Input;
 180    if (_error->PendingError() == true)
 181       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 182
 183    c0out << ' ' << BaseDir << ":" << flush;
 184
 185    // Do recursive directory searching
 186    if (FLFile.empty() == true)
 187    {
 188       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 189          return false;
 190    }
 191    else
 192    {
 193       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 194          return false;
 195    }
 196
 197    Packages.Output = 0;      // Just in case
 198
 199    // Finish compressing
 200    unsigned long Size;
 201    if (Comp.Finalize(Size) == false)
 202    {
 203       c0out << endl;
 204       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 205    }
 206
 207    if (Size != 0)
 208       c0out << " New "
 209              << SizeToStr(Size) << "B ";
 210    else
 211       c0out << ' ';
 212
 213    struct timeval NewTime;
 214    gettimeofday(&NewTime,0);
 215    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 216                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 217
 218    c0out << Packages.Stats.Packages << " files " <<
 219 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 220       SizeToStr(Packages.Stats.Bytes) << "B " <<
 221       TimeToStr((long)Delta) << endl;
 222
 223    Stats.Add(Packages.Stats);
 224    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 225
 226    return !_error->PendingError();
 227 }
 228
 229                                                                         /*}}}*/
 230 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 231 // ---------------------------------------------------------------------
 232 /* This generates the Sources File described by this object. */
 233 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 234 {
 235    if (SrcFile.empty() == true)
 236       return true;
 237
 238    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 239    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 240    string CacheDir = Setup.FindDir("Dir::CacheDir");
 241
 242    struct timeval StartTime;
 243    gettimeofday(&StartTime,0);
 244
 245    SrcDone = true;
 246
 247    // Create a package writer object.
 248    SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
 249                          flCombine(OverrideDir,SrcOverride),
 250                          flCombine(OverrideDir,SrcExtraOverride));
 251    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 252       return _error->Error(_("Source extension list is too long"));
 253    if (_error->PendingError() == true)
 254       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 255
 256    Sources.PathPrefix = PathPrefix;
 257    Sources.DirStrip = ArchiveDir;
 258    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 259
 260    Sources.DeLinkLimit = DeLinkLimit;
 261    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 262
 263    // Create a compressor object
 264    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 265                       SrcCompress,Permissions);
 266    Sources.Output = Comp.Input;
 267    if (_error->PendingError() == true)
 268       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 269
 270    c0out << ' ' << BaseDir << ":" << flush;
 271
 272    // Do recursive directory searching
 273    if (FLFile.empty() == true)
 274    {
 275       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 276          return false;
 277    }
 278    else
 279    {
 280       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 281          return false;
 282    }
 283    Sources.Output = 0;      // Just in case
 284
 285    // Finish compressing
 286    unsigned long Size;
 287    if (Comp.Finalize(Size) == false)
 288    {
 289       c0out << endl;
 290       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 291    }
 292
 293    if (Size != 0)
 294       c0out << " New "
 295              << SizeToStr(Size) << "B ";
 296    else
 297       c0out << ' ';
 298
 299    struct timeval NewTime;
 300    gettimeofday(&NewTime,0);
 301    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 302                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 303
 304    c0out << Sources.Stats.Packages << " pkgs in " <<
 305       TimeToStr((long)Delta) << endl;
 306
 307    Stats.Add(Sources.Stats);
 308    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 309
 310    return !_error->PendingError();
 311 }
 312                                                                         /*}}}*/
 313 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 314 // ---------------------------------------------------------------------
 315 /* This generates the contents file partially described by this object.
 316    It searches the given iterator range for other package files that map
 317    into this contents file and includes their data as well when building. */
 318 bool PackageMap::GenContents(Configuration &Setup,
 319                              vector<PackageMap>::iterator Begin,
 320                              vector<PackageMap>::iterator End,
 321                              unsigned long &Left)
 322 {
 323    if (Contents.empty() == true)
 324       return true;
 325
 326    if (Left == 0)
 327       return true;
 328
 329    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 330    string CacheDir = Setup.FindDir("Dir::CacheDir");
 331    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 332
 333    struct timeval StartTime;
 334    gettimeofday(&StartTime,0);
 335
 336    // Create a package writer object.
 337    ContentsWriter Contents("", Arch);
 338    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 339       return _error->Error(_("Package extension list is too long"));
 340    if (_error->PendingError() == true)
 341       return false;
 342
 343    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 344                       CntCompress,Permissions);
 345    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 346    Contents.Output = Comp.Input;
 347    if (_error->PendingError() == true)
 348       return false;
 349
 350    // Write the header out.
 351    if (ContentsHead.empty() == false)
 352    {
 353       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 354       if (_error->PendingError() == true)
 355          return false;
 356
 357       unsigned long Size = Head.Size();
 358       unsigned char Buf[4096];
 359       while (Size != 0)
 360       {
 361          unsigned long ToRead = Size;
 362          if (Size > sizeof(Buf))
 363             ToRead = sizeof(Buf);
 364
 365          if (Head.Read(Buf,ToRead) == false)
 366             return false;
 367
 368          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 369             return _error->Errno("fwrite",_("Error writing header to contents file"));
 370
 371          Size -= ToRead;
 372       }
 373    }
 374
 375    /* Go over all the package file records and parse all the package
 376       files associated with this contents file into one great big honking
 377       memory structure, then dump the sorted version */
 378    c0out << ' ' << this->Contents << ":" << flush;
 379    for (vector<PackageMap>::iterator I = Begin; I != End; I++)
 380    {
 381       if (I->Contents != this->Contents)
 382          continue;
 383
 384       Contents.Prefix = ArchiveDir;
 385       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 386       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 387                             I->PkgCompress);
 388
 389       I->ContentsDone = true;
 390    }
 391
 392    Contents.Finish();
 393
 394    // Finish compressing
 395    unsigned long Size;
 396    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 397    {
 398       c0out << endl;
 399       return _error->Error(_("Error processing contents %s"),
 400                            this->Contents.c_str());
 401    }
 402
 403    if (Size != 0)
 404    {
 405       c0out << " New " << SizeToStr(Size) << "B ";
 406       if (Left > Size)
 407          Left -= Size;
 408       else
 409          Left = 0;
 410    }
 411    else
 412       c0out << ' ';
 413
 414    struct timeval NewTime;
 415    gettimeofday(&NewTime,0);
 416    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 417                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 418
 419    c0out << Contents.Stats.Packages << " files " <<
 420       SizeToStr(Contents.Stats.Bytes) << "B " <<
 421       TimeToStr((long)Delta) << endl;
 422
 423    return true;
 424 }
 425                                                                         /*}}}*/
 426
 427 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 428 // ---------------------------------------------------------------------
 429 /* This populates the PkgList with all the possible permutations of the
 430    section/arch lists. */
 431 void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 432 {
 433    // Load the defaults
 434    string DDir = Setup.Find("TreeDefault::Directory",
 435                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 436    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 437                             "$(DIST)/$(SECTION)/source/");
 438    string DPkg = Setup.Find("TreeDefault::Packages",
 439                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 440    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 441                             "$(DIST)/$(SECTION)/");
 442    string DContents = Setup.Find("TreeDefault::Contents",
 443                             "$(DIST)/Contents-$(ARCH)");
 444    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 445    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 446                                "packages-$(ARCH).db");
 447    string DSources = Setup.Find("TreeDefault::Sources",
 448                                 "$(DIST)/$(SECTION)/source/Sources");
 449    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 450    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 451
 452    // Process 'tree' type sections
 453    const Configuration::Item *Top = Setup.Tree("tree");
 454    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 455    {
 456       Configuration Block(Top);
 457       string Dist = Top->Tag;
 458
 459       // Parse the sections
 460       string Tmp = Block.Find("Sections");
 461       const char *Sections = Tmp.c_str();
 462       string Section;
 463       while (ParseQuoteWord(Sections,Section) == true)
 464       {
 465          string Tmp2 = Block.Find("Architectures");
 466          string Arch;
 467          const char *Archs = Tmp2.c_str();
 468          while (ParseQuoteWord(Archs,Arch) == true)
 469          {
 470             struct SubstVar Vars[] = {{"$(DIST)",&Dist},
 471                                       {"$(SECTION)",&Section},
 472                                       {"$(ARCH)",&Arch},
 473                                       {}};
 474             PackageMap Itm;
 475
 476             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 477             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 478
 479             if (stringcasecmp(Arch,"source") == 0)
 480             {
 481                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 482                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 483                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 484                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 485                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 486                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 487             }
 488             else
 489             {
 490                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 491                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 492                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 493                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 494                Itm.Arch = Arch;
 495                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 496                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 497                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 498                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 499             }
 500
 501             Itm.GetGeneral(Setup,Block);
 502             PkgList.push_back(Itm);
 503          }
 504       }
 505
 506       Top = Top->Next;
 507    }
 508 }
 509                                                                         /*}}}*/
 510 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 511 // ---------------------------------------------------------------------
 512 /* */
 513 void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 514 {
 515    // Process 'bindirectory' type sections
 516    const Configuration::Item *Top = Setup.Tree("bindirectory");
 517    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 518    {
 519       Configuration Block(Top);
 520
 521       PackageMap Itm;
 522       Itm.PkgFile = Block.Find("Packages");
 523       Itm.SrcFile = Block.Find("Sources");
 524       Itm.BinCacheDB = Block.Find("BinCacheDB");
 525       Itm.BinOverride = Block.Find("BinOverride");
 526       Itm.ExtraOverride = Block.Find("ExtraOverride");
 527       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 528       Itm.SrcOverride = Block.Find("SrcOverride");
 529       Itm.BaseDir = Top->Tag;
 530       Itm.FLFile = Block.Find("FileList");
 531       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 532       Itm.Contents = Block.Find("Contents");
 533       Itm.ContentsHead = Block.Find("Contents::Header");
 534
 535       Itm.GetGeneral(Setup,Block);
 536       PkgList.push_back(Itm);
 537
 538       Top = Top->Next;
 539    }
 540 }
 541                                                                         /*}}}*/
 542
 543 // ShowHelp - Show the help text                                        /*{{{*/
 544 // ---------------------------------------------------------------------
 545 /* */
 546 bool ShowHelp(CommandLine &CmdL)
 547 {
 548    ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,VERSION,
 549             COMMON_ARCH,__DATE__,__TIME__);
 550    if (_config->FindB("version") == true)
 551       return true;
 552
 553    cout <<
 554     _("Usage: apt-ftparchive [options] command\n"
 555       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 556       "          sources srcpath [overridefile [pathprefix]]\n"
 557       "          contents path\n"
 558       "          release path\n"
 559       "          generate config [groups]\n"
 560       "          clean config\n"
 561       "\n"
 562       "apt-ftparchive generates index files for Debian archives. It supports\n"
 563       "many styles of generation from fully automated to functional replacements\n"
 564       "for dpkg-scanpackages and dpkg-scansources\n"
 565       "\n"
 566       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 567       "Package file contains the contents of all the control fields from\n"
 568       "each package as well as the MD5 hash and filesize. An override file\n"
 569       "is supported to force the value of Priority and Section.\n"
 570       "\n"
 571       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 572       "The --source-override option can be used to specify a src override file\n"
 573       "\n"
 574       "The 'packages' and 'sources' command should be run in the root of the\n"
 575       "tree. BinaryPath should point to the base of the recursive search and \n"
 576       "override file should contain the override flags. Pathprefix is\n"
 577       "appended to the filename fields if present. Example usage from the \n"
 578       "Debian archive:\n"
 579       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 580       "               dists/potato/main/binary-i386/Packages\n"
 581       "\n"
 582       "Options:\n"
 583       "  -h    This help text\n"
 584       "  --md5 Control MD5 generation\n"
 585       "  -s=?  Source override file\n"
 586       "  -q    Quiet\n"
 587       "  -d=?  Select the optional caching database\n"
 588       "  --no-delink Enable delinking debug mode\n"
 589       "  --contents  Control contents file generation\n"
 590       "  -c=?  Read this configuration file\n"
 591       "  -o=?  Set an arbitrary configuration option") << endl;
 592
 593    return true;
 594 }
 595                                                                         /*}}}*/
 596 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 597 // ---------------------------------------------------------------------
 598 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 599 bool SimpleGenPackages(CommandLine &CmdL)
 600 {
 601    if (CmdL.FileSize() < 2)
 602       return ShowHelp(CmdL);
 603
 604    string Override;
 605    if (CmdL.FileSize() >= 3)
 606       Override = CmdL.FileList[2];
 607
 608    // Create a package writer object.
 609    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 610                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 611    if (_error->PendingError() == true)
 612       return false;
 613
 614    if (CmdL.FileSize() >= 4)
 615       Packages.PathPrefix = CmdL.FileList[3];
 616
 617    // Do recursive directory searching
 618    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 619       return false;
 620
 621    return true;
 622 }
 623                                                                         /*}}}*/
 624 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 625 // ---------------------------------------------------------------------
 626 /* */
 627 bool SimpleGenContents(CommandLine &CmdL)
 628 {
 629    if (CmdL.FileSize() < 2)
 630       return ShowHelp(CmdL);
 631
 632    // Create a package writer object.
 633    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 634    if (_error->PendingError() == true)
 635       return false;
 636
 637    // Do recursive directory searching
 638    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 639       return false;
 640
 641    Contents.Finish();
 642
 643    return true;
 644 }
 645                                                                         /*}}}*/
 646 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 647 // ---------------------------------------------------------------------
 648 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 649 bool SimpleGenSources(CommandLine &CmdL)
 650 {
 651    if (CmdL.FileSize() < 2)
 652       return ShowHelp(CmdL);
 653
 654    string Override;
 655    if (CmdL.FileSize() >= 3)
 656       Override = CmdL.FileList[2];
 657
 658    string SOverride;
 659    if (Override.empty() == false)
 660       SOverride = Override + ".src";
 661
 662    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 663                              SOverride.c_str());
 664
 665    // Create a package writer object.
 666    SourcesWriter Sources(Override,SOverride);
 667    if (_error->PendingError() == true)
 668       return false;
 669
 670    if (CmdL.FileSize() >= 4)
 671       Sources.PathPrefix = CmdL.FileList[3];
 672
 673    // Do recursive directory searching
 674    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 675       return false;
 676
 677    return true;
 678 }
 679                                                                         /*}}}*/
 680 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 681 // ---------------------------------------------------------------------
 682 bool SimpleGenRelease(CommandLine &CmdL)
 683 {
 684    if (CmdL.FileSize() < 2)
 685       return ShowHelp(CmdL);
 686
 687    string Dir = CmdL.FileList[1];
 688
 689    ReleaseWriter Release("");
 690    Release.DirStrip = Dir;
 691
 692    if (_error->PendingError() == true)
 693       return false;
 694
 695    if (Release.RecursiveScan(Dir) == false)
 696       return false;
 697
 698    Release.Finish();
 699
 700    return true;
 701 }
 702
 703                                                                         /*}}}*/
 704 // Generate - Full generate, using a config file                        /*{{{*/
 705 // ---------------------------------------------------------------------
 706 /* */
 707 bool Generate(CommandLine &CmdL)
 708 {
 709    struct CacheDB::Stats SrcStats;
 710    if (CmdL.FileSize() < 2)
 711       return ShowHelp(CmdL);
 712
 713    struct timeval StartTime;
 714    gettimeofday(&StartTime,0);
 715    struct CacheDB::Stats Stats;
 716
 717    // Read the configuration file.
 718    Configuration Setup;
 719    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 720       return false;
 721
 722    vector<PackageMap> PkgList;
 723    LoadTree(PkgList,Setup);
 724    LoadBinDir(PkgList,Setup);
 725
 726    // Sort by cache DB to improve IO locality.
 727    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 728
 729    // Generate packages
 730    if (CmdL.FileSize() <= 2)
 731    {
 732       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 733          if (I->GenPackages(Setup,Stats) == false)
 734             _error->DumpErrors();
 735       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 736          if (I->GenSources(Setup,SrcStats) == false)
 737             _error->DumpErrors();
 738    }
 739    else
 740    {
 741       // Make a choice list out of the package list..
 742       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 743       RxChoiceList *End = List;
 744       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 745       {
 746          End->UserData = &(*I);
 747          End->Str = I->BaseDir.c_str();
 748          End++;
 749
 750          End->UserData = &(*I);
 751          End->Str = I->Tag.c_str();
 752          End++;
 753       }
 754       End->Str = 0;
 755
 756       // Regex it
 757       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 758       {
 759          delete [] List;
 760          return _error->Error(_("No selections matched"));
 761       }
 762       _error->DumpErrors();
 763
 764       // Do the generation for Packages
 765       for (End = List; End->Str != 0; End++)
 766       {
 767          if (End->Hit == false)
 768             continue;
 769
 770          PackageMap *I = (PackageMap *)End->UserData;
 771          if (I->PkgDone == true)
 772             continue;
 773          if (I->GenPackages(Setup,Stats) == false)
 774             _error->DumpErrors();
 775       }
 776
 777       // Do the generation for Sources
 778       for (End = List; End->Str != 0; End++)
 779       {
 780          if (End->Hit == false)
 781             continue;
 782
 783          PackageMap *I = (PackageMap *)End->UserData;
 784          if (I->SrcDone == true)
 785             continue;
 786          if (I->GenSources(Setup,SrcStats) == false)
 787             _error->DumpErrors();
 788       }
 789
 790       delete [] List;
 791    }
 792
 793    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 794       return true;
 795
 796    c1out << "Packages done, Starting contents." << endl;
 797
 798    // Sort the contents file list by date
 799    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 800    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 801    {
 802       struct stat A;
 803       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 804                                  I->CntCompress,A) == false)
 805          time(&I->ContentsMTime);
 806       else
 807          I->ContentsMTime = A.st_mtime;
 808    }
 809    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 810
 811    /* Now for Contents.. The process here is to do a make-like dependency
 812       check. Each contents file is verified to be newer than the package files
 813       that describe the debs it indexes. Since the package files contain
 814       hashes of the .debs this means they have not changed either so the
 815       contents must be up to date. */
 816    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 817    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 818    {
 819       // This record is not relevent
 820       if (I->ContentsDone == true ||
 821           I->Contents.empty() == true)
 822          continue;
 823
 824       // Do not do everything if the user specified sections.
 825       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 826          continue;
 827
 828       struct stat A,B;
 829       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 830       {
 831          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 832          {
 833             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 834             continue;
 835          }
 836
 837          if (A.st_mtime > B.st_mtime)
 838             continue;
 839       }
 840
 841       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 842                          MaxContentsChange) == false)
 843          _error->DumpErrors();
 844
 845       // Hit the limit?
 846       if (MaxContentsChange == 0)
 847       {
 848          c1out << "Hit contents update byte limit" << endl;
 849          break;
 850       }
 851    }
 852
 853    struct timeval NewTime;
 854    gettimeofday(&NewTime,0);
 855    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 856                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 857    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 858          << " archives. Took " << TimeToStr((long)Delta) << endl;
 859
 860    return true;
 861 }
 862                                                                         /*}}}*/
 863 // Clean - Clean out the databases                                      /*{{{*/
 864 // ---------------------------------------------------------------------
 865 /* */
 866 bool Clean(CommandLine &CmdL)
 867 {
 868    if (CmdL.FileSize() != 2)
 869       return ShowHelp(CmdL);
 870
 871    // Read the configuration file.
 872    Configuration Setup;
 873    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 874       return false;
 875
 876    vector<PackageMap> PkgList;
 877    LoadTree(PkgList,Setup);
 878    LoadBinDir(PkgList,Setup);
 879
 880    // Sort by cache DB to improve IO locality.
 881    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 882
 883    string CacheDir = Setup.FindDir("Dir::CacheDir");
 884
 885    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 886    {
 887       c0out << I->BinCacheDB << endl;
 888       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 889       if (DB.Clean() == false)
 890          _error->DumpErrors();
 891
 892       string CacheDB = I->BinCacheDB;
 893       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
 894    }
 895
 896    return true;
 897 }
 898                                                                         /*}}}*/
 899
 900 int main(int argc, const char *argv[])
 901 {
 902    setlocale(LC_ALL, "");
 903    CommandLine::Args Args[] = {
 904       {'h',"help","help",0},
 905       {0,"md5","APT::FTPArchive::MD5",0},
 906       {'v',"version","version",0},
 907       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 908       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 909       {'q',"quiet","quiet",CommandLine::IntLevel},
 910       {'q',"silent","quiet",CommandLine::IntLevel},
 911       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 912       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 913       {0,"contents","APT::FTPArchive::Contents",0},
 914       {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
 915       {'c',"config-file",0,CommandLine::ConfigFile},
 916       {'o',"option",0,CommandLine::ArbItem},
 917       {0,0,0,0}};
 918    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 919                                    {"contents",&SimpleGenContents},
 920                                    {"sources",&SimpleGenSources},
 921                                    {"release",&SimpleGenRelease},
 922                                    {"generate",&Generate},
 923                                    {"clean",&Clean},
 924                                    {"help",&ShowHelp},
 925                                    {0,0}};
 926
 927    // Parse the command line and initialize the package library
 928    CommandLine CmdL(Args,_config);
 929    if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false)
 930    {
 931       _error->DumpErrors();
 932       return 100;
 933    }
 934
 935    // See if the help should be shown
 936    if (_config->FindB("help") == true ||
 937        _config->FindB("version") == true ||
 938        CmdL.FileSize() == 0)
 939    {
 940       ShowHelp(CmdL);
 941       return 0;
 942    }
 943
 944    // Setup the output streams
 945    c0out.rdbuf(clog.rdbuf());
 946    c1out.rdbuf(clog.rdbuf());
 947    c2out.rdbuf(clog.rdbuf());
 948    Quiet = _config->FindI("quiet",0);
 949    if (Quiet > 0)
 950       c0out.rdbuf(devnull.rdbuf());
 951    if (Quiet > 1)
 952       c1out.rdbuf(devnull.rdbuf());
 953
 954    // Match the operation
 955    CmdL.DispatchArg(Cmds);
 956
 957    if (_error->empty() == false)
 958    {
 959       bool Errors = _error->PendingError();
 960       _error->DumpErrors();
 961       return Errors == true?100:0;
 962    }
 963    return 0;
 964 }