ftparchive/apt-ftparchive.cc

   1 extern "C" {
   2     #include <mach-o/nlist.h>
   3 }
   4
   5 // -*- mode: cpp; mode: fold -*-
   6 // Description                                                          /*{{{*/
   7 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   8 /* ######################################################################
   9
  10    apt-scanpackages - Efficient work-alike for dpkg-scanpackages
  11
  12    Let contents be disabled from the conf
  13
  14    ##################################################################### */
  15                                                                         /*}}}*/
  16 // Include Files                                                        /*{{{*/
  17 #ifdef __GNUG__
  18 #pragma implementation "apt-ftparchive.h"
  19 #endif
  20
  21 #include "apt-ftparchive.h"
  22
  23 #include <apt-pkg/error.h>
  24 #include <apt-pkg/configuration.h>
  25 #include <apt-pkg/cmndline.h>
  26 #include <apt-pkg/strutl.h>
  27 #include <config.h>
  28 #include <apti18n.h>
  29 #include <algorithm>
  30
  31 #include <sys/time.h>
  32 #include <regex.h>
  33
  34 #include "contents.h"
  35 #include "multicompress.h"
  36 #include "writer.h"
  37                                                                         /*}}}*/
  38
  39 using namespace std;
  40 ostream c0out(0);
  41 ostream c1out(0);
  42 ostream c2out(0);
  43 ofstream devnull("/dev/null");
  44 unsigned Quiet = 0;
  45
  46 // struct PackageMap - List of all package files in the config file     /*{{{*/
  47 // ---------------------------------------------------------------------
  48 /* */
  49 struct PackageMap
  50 {
  51    // General Stuff
  52    string BaseDir;
  53    string InternalPrefix;
  54    string FLFile;
  55    string PkgExt;
  56    string SrcExt;
  57
  58    // Stuff for the Package File
  59    string PkgFile;
  60    string BinCacheDB;
  61    string BinOverride;
  62    string ExtraOverride;
  63
  64    // We generate for this given arch
  65    string Arch;
  66
  67    // Stuff for the Source File
  68    string SrcFile;
  69    string SrcOverride;
  70    string SrcExtraOverride;
  71
  72    // Contents
  73    string Contents;
  74    string ContentsHead;
  75
  76    // Random things
  77    string Tag;
  78    string PkgCompress;
  79    string CntCompress;
  80    string SrcCompress;
  81    string PathPrefix;
  82    unsigned int DeLinkLimit;
  83    mode_t Permissions;
  84
  85    bool ContentsDone;
  86    bool PkgDone;
  87    bool SrcDone;
  88    time_t ContentsMTime;
  89
  90    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  91    {
  92       inline bool operator() (const PackageMap &x,const PackageMap &y)
  93       {return x.ContentsMTime < y.ContentsMTime;};
  94    };
  95
  96    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  97    {
  98       inline bool operator() (const PackageMap &x,const PackageMap &y)
  99       {return x.BinCacheDB < y.BinCacheDB;};
 100    };
 101
 102    void GetGeneral(Configuration &Setup,Configuration &Block);
 103    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 104    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 105    bool GenContents(Configuration &Setup,
 106                     vector<PackageMap>::iterator Begin,
 107                     vector<PackageMap>::iterator End,
 108                     unsigned long &Left);
 109
 110    PackageMap() : DeLinkLimit(0), Permissions(1), ContentsDone(false),
 111         PkgDone(false), SrcDone(false), ContentsMTime(0) {};
 112 };
 113                                                                         /*}}}*/
 114
 115 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 116 // ---------------------------------------------------------------------
 117 /* */
 118 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 119 {
 120    PathPrefix = Block.Find("PathPrefix");
 121
 122    if (Block.FindB("External-Links",true) == false)
 123       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 124    else
 125       DeLinkLimit = 0;
 126
 127    PkgCompress = Block.Find("Packages::Compress",
 128                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 129    CntCompress = Block.Find("Contents::Compress",
 130                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 131    SrcCompress = Block.Find("Sources::Compress",
 132                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 133
 134    SrcExt = Block.Find("Sources::Extensions",
 135                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 136    PkgExt = Block.Find("Packages::Extensions",
 137                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 138
 139    Permissions = Setup.FindI("Default::FileMode",0644);
 140
 141    if (FLFile.empty() == false)
 142       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 143
 144    if (Contents == " ")
 145       Contents= string();
 146 }
 147                                                                         /*}}}*/
 148 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 149 // ---------------------------------------------------------------------
 150 /* This generates the Package File described by this object. */
 151 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 152 {
 153    if (PkgFile.empty() == true)
 154       return true;
 155
 156    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 157    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 158    string CacheDir = Setup.FindDir("Dir::CacheDir");
 159
 160    struct timeval StartTime;
 161    gettimeofday(&StartTime,0);
 162
 163    PkgDone = true;
 164
 165    // Create a package writer object.
 166    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 167                            flCombine(OverrideDir,BinOverride),
 168                            flCombine(OverrideDir,ExtraOverride),
 169                            Arch);
 170    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 171       return _error->Error(_("Package extension list is too long"));
 172    if (_error->PendingError() == true)
 173       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 174
 175    Packages.PathPrefix = PathPrefix;
 176    Packages.DirStrip = ArchiveDir;
 177    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 178
 179    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 180    Packages.DeLinkLimit = DeLinkLimit;
 181
 182    // Create a compressor object
 183    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 184                       PkgCompress,Permissions);
 185    Packages.Output = Comp.Input;
 186    if (_error->PendingError() == true)
 187       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 188
 189    c0out << ' ' << BaseDir << ":" << flush;
 190
 191    // Do recursive directory searching
 192    if (FLFile.empty() == true)
 193    {
 194       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 195          return false;
 196    }
 197    else
 198    {
 199       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 200          return false;
 201    }
 202
 203    Packages.Output = 0;      // Just in case
 204
 205    // Finish compressing
 206    unsigned long Size;
 207    if (Comp.Finalize(Size) == false)
 208    {
 209       c0out << endl;
 210       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 211    }
 212
 213    if (Size != 0)
 214       c0out << " New "
 215              << SizeToStr(Size) << "B ";
 216    else
 217       c0out << ' ';
 218
 219    struct timeval NewTime;
 220    gettimeofday(&NewTime,0);
 221    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 222                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 223
 224    c0out << Packages.Stats.Packages << " files " <<
 225 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 226       SizeToStr(Packages.Stats.Bytes) << "B " <<
 227       TimeToStr((long)Delta) << endl;
 228
 229    Stats.Add(Packages.Stats);
 230    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 231
 232    return !_error->PendingError();
 233 }
 234
 235                                                                         /*}}}*/
 236 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 237 // ---------------------------------------------------------------------
 238 /* This generates the Sources File described by this object. */
 239 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 240 {
 241    if (SrcFile.empty() == true)
 242       return true;
 243
 244    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 245    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 246    string CacheDir = Setup.FindDir("Dir::CacheDir");
 247
 248    struct timeval StartTime;
 249    gettimeofday(&StartTime,0);
 250
 251    SrcDone = true;
 252
 253    // Create a package writer object.
 254    SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
 255                          flCombine(OverrideDir,SrcOverride),
 256                          flCombine(OverrideDir,SrcExtraOverride));
 257    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 258       return _error->Error(_("Source extension list is too long"));
 259    if (_error->PendingError() == true)
 260       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 261
 262    Sources.PathPrefix = PathPrefix;
 263    Sources.DirStrip = ArchiveDir;
 264    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 265
 266    Sources.DeLinkLimit = DeLinkLimit;
 267    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 268
 269    // Create a compressor object
 270    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 271                       SrcCompress,Permissions);
 272    Sources.Output = Comp.Input;
 273    if (_error->PendingError() == true)
 274       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 275
 276    c0out << ' ' << BaseDir << ":" << flush;
 277
 278    // Do recursive directory searching
 279    if (FLFile.empty() == true)
 280    {
 281       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 282          return false;
 283    }
 284    else
 285    {
 286       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 287          return false;
 288    }
 289    Sources.Output = 0;      // Just in case
 290
 291    // Finish compressing
 292    unsigned long Size;
 293    if (Comp.Finalize(Size) == false)
 294    {
 295       c0out << endl;
 296       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 297    }
 298
 299    if (Size != 0)
 300       c0out << " New "
 301              << SizeToStr(Size) << "B ";
 302    else
 303       c0out << ' ';
 304
 305    struct timeval NewTime;
 306    gettimeofday(&NewTime,0);
 307    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 308                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 309
 310    c0out << Sources.Stats.Packages << " pkgs in " <<
 311       TimeToStr((long)Delta) << endl;
 312
 313    Stats.Add(Sources.Stats);
 314    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 315
 316    return !_error->PendingError();
 317 }
 318                                                                         /*}}}*/
 319 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 320 // ---------------------------------------------------------------------
 321 /* This generates the contents file partially described by this object.
 322    It searches the given iterator range for other package files that map
 323    into this contents file and includes their data as well when building. */
 324 bool PackageMap::GenContents(Configuration &Setup,
 325                              vector<PackageMap>::iterator Begin,
 326                              vector<PackageMap>::iterator End,
 327                              unsigned long &Left)
 328 {
 329    if (Contents.empty() == true)
 330       return true;
 331
 332    if (Left == 0)
 333       return true;
 334
 335    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 336    string CacheDir = Setup.FindDir("Dir::CacheDir");
 337    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 338
 339    struct timeval StartTime;
 340    gettimeofday(&StartTime,0);
 341
 342    // Create a package writer object.
 343    ContentsWriter Contents("");
 344    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 345       return _error->Error(_("Package extension list is too long"));
 346    if (_error->PendingError() == true)
 347       return false;
 348
 349    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 350                       CntCompress,Permissions);
 351    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 352    Contents.Output = Comp.Input;
 353    if (_error->PendingError() == true)
 354       return false;
 355
 356    // Write the header out.
 357    if (ContentsHead.empty() == false)
 358    {
 359       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 360       if (_error->PendingError() == true)
 361          return false;
 362
 363       unsigned long Size = Head.Size();
 364       unsigned char Buf[4096];
 365       while (Size != 0)
 366       {
 367          unsigned long ToRead = Size;
 368          if (Size > sizeof(Buf))
 369             ToRead = sizeof(Buf);
 370
 371          if (Head.Read(Buf,ToRead) == false)
 372             return false;
 373
 374          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 375             return _error->Errno("fwrite",_("Error writing header to contents file"));
 376
 377          Size -= ToRead;
 378       }
 379    }
 380
 381    /* Go over all the package file records and parse all the package
 382       files associated with this contents file into one great big honking
 383       memory structure, then dump the sorted version */
 384    c0out << ' ' << this->Contents << ":" << flush;
 385    for (vector<PackageMap>::iterator I = Begin; I != End; I++)
 386    {
 387       if (I->Contents != this->Contents)
 388          continue;
 389
 390       Contents.Prefix = ArchiveDir;
 391       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 392       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 393                             I->PkgCompress);
 394
 395       I->ContentsDone = true;
 396    }
 397
 398    Contents.Finish();
 399
 400    // Finish compressing
 401    unsigned long Size;
 402    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 403    {
 404       c0out << endl;
 405       return _error->Error(_("Error processing contents %s"),
 406                            this->Contents.c_str());
 407    }
 408
 409    if (Size != 0)
 410    {
 411       c0out << " New " << SizeToStr(Size) << "B ";
 412       if (Left > Size)
 413          Left -= Size;
 414       else
 415          Left = 0;
 416    }
 417    else
 418       c0out << ' ';
 419
 420    struct timeval NewTime;
 421    gettimeofday(&NewTime,0);
 422    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 423                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 424
 425    c0out << Contents.Stats.Packages << " files " <<
 426       SizeToStr(Contents.Stats.Bytes) << "B " <<
 427       TimeToStr((long)Delta) << endl;
 428
 429    return true;
 430 }
 431                                                                         /*}}}*/
 432
 433 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 434 // ---------------------------------------------------------------------
 435 /* This populates the PkgList with all the possible permutations of the
 436    section/arch lists. */
 437 void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 438 {
 439    // Load the defaults
 440    string DDir = Setup.Find("TreeDefault::Directory",
 441                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 442    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 443                             "$(DIST)/$(SECTION)/source/");
 444    string DPkg = Setup.Find("TreeDefault::Packages",
 445                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 446    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 447                             "$(DIST)/$(SECTION)/");
 448    string DContents = Setup.Find("TreeDefault::Contents",
 449                             "$(DIST)/Contents-$(ARCH)");
 450    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 451    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 452                                "packages-$(ARCH).db");
 453    string DSources = Setup.Find("TreeDefault::Sources",
 454                                 "$(DIST)/$(SECTION)/source/Sources");
 455    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 456    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 457
 458    // Process 'tree' type sections
 459    const Configuration::Item *Top = Setup.Tree("tree");
 460    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 461    {
 462       Configuration Block(Top);
 463       string Dist = Top->Tag;
 464
 465       // Parse the sections
 466       string Tmp = Block.Find("Sections");
 467       const char *Sections = Tmp.c_str();
 468       string Section;
 469       while (ParseQuoteWord(Sections,Section) == true)
 470       {
 471          string Tmp2 = Block.Find("Architectures");
 472          string Arch;
 473          const char *Archs = Tmp2.c_str();
 474          while (ParseQuoteWord(Archs,Arch) == true)
 475          {
 476             struct SubstVar Vars[] = {{"$(DIST)",&Dist},
 477                                       {"$(SECTION)",&Section},
 478                                       {"$(ARCH)",&Arch},
 479                                       {}};
 480             PackageMap Itm;
 481
 482             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 483             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 484
 485             if (stringcasecmp(Arch,"source") == 0)
 486             {
 487                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 488                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 489                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 490                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 491                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 492                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 493             }
 494             else
 495             {
 496                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 497                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 498                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 499                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 500                Itm.Arch = Arch;
 501                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 502                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 503                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 504                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 505             }
 506
 507             Itm.GetGeneral(Setup,Block);
 508             PkgList.push_back(Itm);
 509          }
 510       }
 511
 512       Top = Top->Next;
 513    }
 514 }
 515                                                                         /*}}}*/
 516 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 517 // ---------------------------------------------------------------------
 518 /* */
 519 void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 520 {
 521    // Process 'bindirectory' type sections
 522    const Configuration::Item *Top = Setup.Tree("bindirectory");
 523    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 524    {
 525       Configuration Block(Top);
 526
 527       PackageMap Itm;
 528       Itm.PkgFile = Block.Find("Packages");
 529       Itm.SrcFile = Block.Find("Sources");
 530       Itm.BinCacheDB = Block.Find("BinCacheDB");
 531       Itm.BinOverride = Block.Find("BinOverride");
 532       Itm.ExtraOverride = Block.Find("ExtraOverride");
 533       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 534       Itm.SrcOverride = Block.Find("SrcOverride");
 535       Itm.BaseDir = Top->Tag;
 536       Itm.FLFile = Block.Find("FileList");
 537       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 538       Itm.Contents = Block.Find("Contents");
 539       Itm.ContentsHead = Block.Find("Contents::Header");
 540
 541       Itm.GetGeneral(Setup,Block);
 542       PkgList.push_back(Itm);
 543
 544       Top = Top->Next;
 545    }
 546 }
 547                                                                         /*}}}*/
 548
 549 // ShowHelp - Show the help text                                        /*{{{*/
 550 // ---------------------------------------------------------------------
 551 /* */
 552 bool ShowHelp(CommandLine &CmdL)
 553 {
 554    ioprintf(cout,_("%s %s for %s %s\n"),PACKAGE,VERSION,
 555             COMMON_OS,COMMON_CPU);
 556    if (_config->FindB("version") == true)
 557       return true;
 558
 559    cout <<
 560     _("Usage: apt-ftparchive [options] command\n"
 561       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 562       "          sources srcpath [overridefile [pathprefix]]\n"
 563       "          contents path\n"
 564       "          release path\n"
 565       "          generate config [groups]\n"
 566       "          clean config\n"
 567       "\n"
 568       "apt-ftparchive generates index files for Debian archives. It supports\n"
 569       "many styles of generation from fully automated to functional replacements\n"
 570       "for dpkg-scanpackages and dpkg-scansources\n"
 571       "\n"
 572       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 573       "Package file contains the contents of all the control fields from\n"
 574       "each package as well as the MD5 hash and filesize. An override file\n"
 575       "is supported to force the value of Priority and Section.\n"
 576       "\n"
 577       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 578       "The --source-override option can be used to specify a src override file\n"
 579       "\n"
 580       "The 'packages' and 'sources' command should be run in the root of the\n"
 581       "tree. BinaryPath should point to the base of the recursive search and \n"
 582       "override file should contain the override flags. Pathprefix is\n"
 583       "appended to the filename fields if present. Example usage from the \n"
 584       "Debian archive:\n"
 585       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 586       "               dists/potato/main/binary-i386/Packages\n"
 587       "\n"
 588       "Options:\n"
 589       "  -h    This help text\n"
 590       "  --md5 Control MD5 generation\n"
 591       "  -s=?  Source override file\n"
 592       "  -q    Quiet\n"
 593       "  -d=?  Select the optional caching database\n"
 594       "  --no-delink Enable delinking debug mode\n"
 595       "  --contents  Control contents file generation\n"
 596       "  -c=?  Read this configuration file\n"
 597       "  -o=?  Set an arbitrary configuration option") << endl;
 598
 599    return true;
 600 }
 601                                                                         /*}}}*/
 602 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 603 // ---------------------------------------------------------------------
 604 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 605 bool SimpleGenPackages(CommandLine &CmdL)
 606 {
 607    if (CmdL.FileSize() < 2)
 608       return ShowHelp(CmdL);
 609
 610    string Override;
 611    if (CmdL.FileSize() >= 3)
 612       Override = CmdL.FileList[2];
 613
 614    // Create a package writer object.
 615    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 616                            Override, "");
 617    if (_error->PendingError() == true)
 618       return false;
 619
 620    if (CmdL.FileSize() >= 4)
 621       Packages.PathPrefix = CmdL.FileList[3];
 622
 623    // Do recursive directory searching
 624    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 625       return false;
 626
 627    return true;
 628 }
 629                                                                         /*}}}*/
 630 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 631 // ---------------------------------------------------------------------
 632 /* */
 633 bool SimpleGenContents(CommandLine &CmdL)
 634 {
 635    if (CmdL.FileSize() < 2)
 636       return ShowHelp(CmdL);
 637
 638    // Create a package writer object.
 639    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"));
 640    if (_error->PendingError() == true)
 641       return false;
 642
 643    // Do recursive directory searching
 644    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 645       return false;
 646
 647    Contents.Finish();
 648
 649    return true;
 650 }
 651                                                                         /*}}}*/
 652 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 653 // ---------------------------------------------------------------------
 654 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 655 bool SimpleGenSources(CommandLine &CmdL)
 656 {
 657    if (CmdL.FileSize() < 2)
 658       return ShowHelp(CmdL);
 659
 660    string Override;
 661    if (CmdL.FileSize() >= 3)
 662       Override = CmdL.FileList[2];
 663
 664    string SOverride;
 665    if (Override.empty() == false)
 666       SOverride = Override + ".src";
 667
 668    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 669                              SOverride.c_str());
 670
 671    // Create a package writer object.
 672    SourcesWriter Sources(Override,SOverride);
 673    if (_error->PendingError() == true)
 674       return false;
 675
 676    if (CmdL.FileSize() >= 4)
 677       Sources.PathPrefix = CmdL.FileList[3];
 678
 679    // Do recursive directory searching
 680    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 681       return false;
 682
 683    return true;
 684 }
 685                                                                         /*}}}*/
 686 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 687 // ---------------------------------------------------------------------
 688 bool SimpleGenRelease(CommandLine &CmdL)
 689 {
 690    if (CmdL.FileSize() < 2)
 691       return ShowHelp(CmdL);
 692
 693    string Dir = CmdL.FileList[1];
 694
 695    ReleaseWriter Release("");
 696    Release.DirStrip = Dir;
 697
 698    if (_error->PendingError() == true)
 699       return false;
 700
 701    if (Release.RecursiveScan(Dir) == false)
 702       return false;
 703
 704    Release.Finish();
 705
 706    return true;
 707 }
 708
 709                                                                         /*}}}*/
 710 // Generate - Full generate, using a config file                        /*{{{*/
 711 // ---------------------------------------------------------------------
 712 /* */
 713 bool Generate(CommandLine &CmdL)
 714 {
 715    struct CacheDB::Stats SrcStats;
 716    if (CmdL.FileSize() < 2)
 717       return ShowHelp(CmdL);
 718
 719    struct timeval StartTime;
 720    gettimeofday(&StartTime,0);
 721    struct CacheDB::Stats Stats;
 722
 723    // Read the configuration file.
 724    Configuration Setup;
 725    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 726       return false;
 727
 728    vector<PackageMap> PkgList;
 729    LoadTree(PkgList,Setup);
 730    LoadBinDir(PkgList,Setup);
 731
 732    // Sort by cache DB to improve IO locality.
 733    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 734
 735    // Generate packages
 736    if (CmdL.FileSize() <= 2)
 737    {
 738       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 739          if (I->GenPackages(Setup,Stats) == false)
 740             _error->DumpErrors();
 741       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 742          if (I->GenSources(Setup,SrcStats) == false)
 743             _error->DumpErrors();
 744    }
 745    else
 746    {
 747       // Make a choice list out of the package list..
 748       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 749       RxChoiceList *End = List;
 750       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 751       {
 752          End->UserData = &(*I);
 753          End->Str = I->BaseDir.c_str();
 754          End++;
 755
 756          End->UserData = &(*I);
 757          End->Str = I->Tag.c_str();
 758          End++;
 759       }
 760       End->Str = 0;
 761
 762       // Regex it
 763       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 764       {
 765          delete [] List;
 766          return _error->Error(_("No selections matched"));
 767       }
 768       _error->DumpErrors();
 769
 770       // Do the generation for Packages
 771       for (End = List; End->Str != 0; End++)
 772       {
 773          if (End->Hit == false)
 774             continue;
 775
 776          PackageMap *I = (PackageMap *)End->UserData;
 777          if (I->PkgDone == true)
 778             continue;
 779          if (I->GenPackages(Setup,Stats) == false)
 780             _error->DumpErrors();
 781       }
 782
 783       // Do the generation for Sources
 784       for (End = List; End->Str != 0; End++)
 785       {
 786          if (End->Hit == false)
 787             continue;
 788
 789          PackageMap *I = (PackageMap *)End->UserData;
 790          if (I->SrcDone == true)
 791             continue;
 792          if (I->GenSources(Setup,SrcStats) == false)
 793             _error->DumpErrors();
 794       }
 795
 796       delete [] List;
 797    }
 798
 799    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 800       return true;
 801
 802    c1out << "Done Packages, Starting contents." << endl;
 803
 804    // Sort the contents file list by date
 805    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 806    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 807    {
 808       struct stat A;
 809       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 810                                  I->CntCompress,A) == false)
 811          time(&I->ContentsMTime);
 812       else
 813          I->ContentsMTime = A.st_mtime;
 814    }
 815    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 816
 817    /* Now for Contents.. The process here is to do a make-like dependency
 818       check. Each contents file is verified to be newer than the package files
 819       that describe the debs it indexes. Since the package files contain
 820       hashes of the .debs this means they have not changed either so the
 821       contents must be up to date. */
 822    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 823    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 824    {
 825       // This record is not relevent
 826       if (I->ContentsDone == true ||
 827           I->Contents.empty() == true)
 828          continue;
 829
 830       // Do not do everything if the user specified sections.
 831       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 832          continue;
 833
 834       struct stat A,B;
 835       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 836       {
 837          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 838          {
 839             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 840             continue;
 841          }
 842
 843          if (A.st_mtime > B.st_mtime)
 844             continue;
 845       }
 846
 847       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 848                          MaxContentsChange) == false)
 849          _error->DumpErrors();
 850
 851       // Hit the limit?
 852       if (MaxContentsChange == 0)
 853       {
 854          c1out << "Hit contents update byte limit" << endl;
 855          break;
 856       }
 857    }
 858
 859    struct timeval NewTime;
 860    gettimeofday(&NewTime,0);
 861    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 862                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 863    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 864          << " archives. Took " << TimeToStr((long)Delta) << endl;
 865
 866    return true;
 867 }
 868                                                                         /*}}}*/
 869 // Clean - Clean out the databases                                      /*{{{*/
 870 // ---------------------------------------------------------------------
 871 /* */
 872 bool Clean(CommandLine &CmdL)
 873 {
 874    if (CmdL.FileSize() != 2)
 875       return ShowHelp(CmdL);
 876
 877    // Read the configuration file.
 878    Configuration Setup;
 879    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 880       return false;
 881
 882    vector<PackageMap> PkgList;
 883    LoadTree(PkgList,Setup);
 884    LoadBinDir(PkgList,Setup);
 885
 886    // Sort by cache DB to improve IO locality.
 887    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 888
 889    string CacheDir = Setup.FindDir("Dir::CacheDir");
 890
 891    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 892    {
 893       c0out << I->BinCacheDB << endl;
 894       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 895       if (DB.Clean() == false)
 896          _error->DumpErrors();
 897
 898       string CacheDB = I->BinCacheDB;
 899       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
 900    }
 901
 902    return true;
 903 }
 904                                                                         /*}}}*/
 905
 906 int main(int argc, const char *argv[])
 907 {
 908 #if !defined(__ENVIRONMENT_ASPEN_VERSION_MIN_REQUIRED__) || __ENVIRONMENT_ASPEN_VERSION_MIN_REQUIRED__ < 10200
 909    struct nlist nl[2];
 910    memset(nl, 0, sizeof(nl));
 911    nl[0].n_un.n_name = (char *) "_useMDNSResponder";
 912    nlist("/usr/lib/libc.dylib", nl);
 913    if (nl[0].n_type != N_UNDF)
 914        *(int *) nl[0].n_value = 0;
 915 #endif
 916
 917    CommandLine::Args Args[] = {
 918       {'h',"help","help",0},
 919       {0,"md5","APT::FTPArchive::MD5",0},
 920       {'v',"version","version",0},
 921       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 922       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 923       {'q',"quiet","quiet",CommandLine::IntLevel},
 924       {'q',"silent","quiet",CommandLine::IntLevel},
 925       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 926       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 927       {0,"contents","APT::FTPArchive::Contents",0},
 928       {'c',"config-file",0,CommandLine::ConfigFile},
 929       {'o',"option",0,CommandLine::ArbItem},
 930       {0,0,0,0}};
 931    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 932                                    {"contents",&SimpleGenContents},
 933                                    {"sources",&SimpleGenSources},
 934                                    {"release",&SimpleGenRelease},
 935                                    {"generate",&Generate},
 936                                    {"clean",&Clean},
 937                                    {"help",&ShowHelp},
 938                                    {0,0}};
 939
 940    // Parse the command line and initialize the package library
 941    CommandLine CmdL(Args,_config);
 942    if (CmdL.Parse(argc,argv) == false)
 943    {
 944       _error->DumpErrors();
 945       return 100;
 946    }
 947
 948    // See if the help should be shown
 949    if (_config->FindB("help") == true ||
 950        _config->FindB("version") == true ||
 951        CmdL.FileSize() == 0)
 952    {
 953       ShowHelp(CmdL);
 954       return 0;
 955    }
 956
 957    // Setup the output streams
 958    c0out.rdbuf(clog.rdbuf());
 959    c1out.rdbuf(clog.rdbuf());
 960    c2out.rdbuf(clog.rdbuf());
 961    Quiet = _config->FindI("quiet",0);
 962    if (Quiet > 0)
 963       c0out.rdbuf(devnull.rdbuf());
 964    if (Quiet > 1)
 965       c1out.rdbuf(devnull.rdbuf());
 966
 967    // Match the operation
 968    CmdL.DispatchArg(Cmds);
 969
 970    if (_error->empty() == false)
 971    {
 972       bool Errors = _error->PendingError();
 973       _error->DumpErrors();
 974       return Errors == true?100:0;
 975    }
 976    return 0;
 977 }