1 // -*- mode: cpp; mode: fold -*-
3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
4 /* ######################################################################
6 apt-ftparchive - Efficient work-alike for dpkg-scanpackages
8 Let contents be disabled from the conf
10 ##################################################################### */
12 // Include Files /*{{{*/
15 #include <apt-pkg/error.h>
16 #include <apt-pkg/configuration.h>
17 #include <apt-pkg/cmndline.h>
18 #include <apt-pkg/strutl.h>
19 #include <apt-pkg/init.h>
20 #include <apt-pkg/fileutl.h>
36 #include "apt-ftparchive.h"
37 #include "multicompress.h"
47 ofstream
devnull("/dev/null");
50 // struct PackageMap - List of all package files in the config file /*{{{*/
51 // ---------------------------------------------------------------------
57 string InternalPrefix
;
62 // Stuff for the Package File
69 // We generate for this given arch
72 // Stuff for the Source File
75 string SrcExtraOverride
;
77 // Translation master file
79 TranslationWriter
*TransWriter
;
91 unsigned int DeLinkLimit
;
99 struct ContentsCompare
: public binary_function
<PackageMap
,PackageMap
,bool>
101 inline bool operator() (const PackageMap
&x
,const PackageMap
&y
)
102 {return x
.ContentsMTime
< y
.ContentsMTime
;};
105 struct DBCompare
: public binary_function
<PackageMap
,PackageMap
,bool>
107 inline bool operator() (const PackageMap
&x
,const PackageMap
&y
)
108 {return x
.BinCacheDB
< y
.BinCacheDB
;};
111 struct SrcDBCompare
: public binary_function
<PackageMap
,PackageMap
,bool>
113 inline bool operator() (const PackageMap
&x
,const PackageMap
&y
)
114 {return x
.SrcCacheDB
< y
.SrcCacheDB
;};
117 void GetGeneral(Configuration
&Setup
,Configuration
&Block
);
118 bool GenPackages(Configuration
&Setup
,struct CacheDB::Stats
&Stats
);
119 bool GenSources(Configuration
&Setup
,struct CacheDB::Stats
&Stats
);
120 bool GenContents(Configuration
&Setup
,
121 vector
<PackageMap
>::iterator Begin
,
122 vector
<PackageMap
>::iterator End
,
123 unsigned long &Left
);
125 PackageMap() : LongDesc(true), TransWriter(NULL
), DeLinkLimit(0), Permissions(1),
126 ContentsDone(false), PkgDone(false), SrcDone(false),
131 // PackageMap::GetGeneral - Common per-section definitions /*{{{*/
132 // ---------------------------------------------------------------------
134 void PackageMap::GetGeneral(Configuration
&Setup
,Configuration
&Block
)
136 PathPrefix
= Block
.Find("PathPrefix");
138 if (Block
.FindB("External-Links",true) == false)
139 DeLinkLimit
= Setup
.FindI("Default::DeLinkLimit",UINT_MAX
);
143 PkgCompress
= Block
.Find("Packages::Compress",
144 Setup
.Find("Default::Packages::Compress",". gzip").c_str());
145 CntCompress
= Block
.Find("Contents::Compress",
146 Setup
.Find("Default::Contents::Compress",". gzip").c_str());
147 SrcCompress
= Block
.Find("Sources::Compress",
148 Setup
.Find("Default::Sources::Compress",". gzip").c_str());
150 SrcExt
= Block
.Find("Sources::Extensions",
151 Setup
.Find("Default::Sources::Extensions",".dsc").c_str());
152 PkgExt
= Block
.Find("Packages::Extensions",
153 Setup
.Find("Default::Packages::Extensions",".deb").c_str());
155 Permissions
= Setup
.FindI("Default::FileMode",0644);
157 if (FLFile
.empty() == false)
158 FLFile
= flCombine(Setup
.Find("Dir::FileListDir"),FLFile
);
164 // PackageMap::GenPackages - Actually generate a Package file /*{{{*/
165 // ---------------------------------------------------------------------
166 /* This generates the Package File described by this object. */
167 bool PackageMap::GenPackages(Configuration
&Setup
,struct CacheDB::Stats
&Stats
)
169 if (PkgFile
.empty() == true)
172 string ArchiveDir
= Setup
.FindDir("Dir::ArchiveDir");
173 string OverrideDir
= Setup
.FindDir("Dir::OverrideDir");
174 string CacheDir
= Setup
.FindDir("Dir::CacheDir");
176 struct timeval StartTime
;
177 gettimeofday(&StartTime
,0);
181 // Create a package writer object.
182 PackagesWriter
Packages(flCombine(CacheDir
,BinCacheDB
),
183 flCombine(OverrideDir
,BinOverride
),
184 flCombine(OverrideDir
,ExtraOverride
),
186 if (PkgExt
.empty() == false && Packages
.SetExts(PkgExt
) == false)
187 return _error
->Error(_("Package extension list is too long"));
188 if (_error
->PendingError() == true)
189 return _error
->Error(_("Error processing directory %s"),BaseDir
.c_str());
191 Packages
.PathPrefix
= PathPrefix
;
192 Packages
.DirStrip
= ArchiveDir
;
193 Packages
.InternalPrefix
= flCombine(ArchiveDir
,InternalPrefix
);
195 Packages
.TransWriter
= TransWriter
;
196 Packages
.LongDescription
= LongDesc
;
198 Packages
.Stats
.DeLinkBytes
= Stats
.DeLinkBytes
;
199 Packages
.DeLinkLimit
= DeLinkLimit
;
201 // Create a compressor object
202 MultiCompress
Comp(flCombine(ArchiveDir
,PkgFile
),
203 PkgCompress
,Permissions
);
204 Packages
.Output
= Comp
.Input
;
205 if (_error
->PendingError() == true)
206 return _error
->Error(_("Error processing directory %s"),BaseDir
.c_str());
208 c0out
<< ' ' << BaseDir
<< ":" << flush
;
210 // Do recursive directory searching
211 if (FLFile
.empty() == true)
213 if (Packages
.RecursiveScan(flCombine(ArchiveDir
,BaseDir
)) == false)
218 if (Packages
.LoadFileList(ArchiveDir
,FLFile
) == false)
222 Packages
.Output
= 0; // Just in case
224 // Finish compressing
225 unsigned long long Size
;
226 if (Comp
.Finalize(Size
) == false)
229 return _error
->Error(_("Error processing directory %s"),BaseDir
.c_str());
234 << SizeToStr(Size
) << "B ";
238 struct timeval NewTime
;
239 gettimeofday(&NewTime
,0);
240 double Delta
= NewTime
.tv_sec
- StartTime
.tv_sec
+
241 (NewTime
.tv_usec
- StartTime
.tv_usec
)/1000000.0;
243 c0out
<< Packages
.Stats
.Packages
<< " files " <<
244 /* SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
245 SizeToStr(Packages
.Stats
.Bytes
) << "B " <<
246 TimeToStr((long)Delta
) << endl
;
248 if(_config
->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
249 c0out
<< " Misses in Cache: " << Packages
.Stats
.Misses
<< endl
;
251 Stats
.Add(Packages
.Stats
);
252 Stats
.DeLinkBytes
= Packages
.Stats
.DeLinkBytes
;
254 return !_error
->PendingError();
258 // PackageMap::GenSources - Actually generate a Source file /*{{{*/
259 // ---------------------------------------------------------------------
260 /* This generates the Sources File described by this object. */
261 bool PackageMap::GenSources(Configuration
&Setup
,struct CacheDB::Stats
&Stats
)
263 if (SrcFile
.empty() == true)
266 string ArchiveDir
= Setup
.FindDir("Dir::ArchiveDir");
267 string OverrideDir
= Setup
.FindDir("Dir::OverrideDir");
268 string CacheDir
= Setup
.FindDir("Dir::CacheDir");
270 struct timeval StartTime
;
271 gettimeofday(&StartTime
,0);
275 // Create a package writer object.
276 SourcesWriter
Sources(flCombine(CacheDir
, SrcCacheDB
),
277 flCombine(OverrideDir
,BinOverride
),
278 flCombine(OverrideDir
,SrcOverride
),
279 flCombine(OverrideDir
,SrcExtraOverride
));
280 if (SrcExt
.empty() == false && Sources
.SetExts(SrcExt
) == false)
281 return _error
->Error(_("Source extension list is too long"));
282 if (_error
->PendingError() == true)
283 return _error
->Error(_("Error processing directory %s"),BaseDir
.c_str());
285 Sources
.PathPrefix
= PathPrefix
;
286 Sources
.DirStrip
= ArchiveDir
;
287 Sources
.InternalPrefix
= flCombine(ArchiveDir
,InternalPrefix
);
289 Sources
.DeLinkLimit
= DeLinkLimit
;
290 Sources
.Stats
.DeLinkBytes
= Stats
.DeLinkBytes
;
292 // Create a compressor object
293 MultiCompress
Comp(flCombine(ArchiveDir
,SrcFile
),
294 SrcCompress
,Permissions
);
295 Sources
.Output
= Comp
.Input
;
296 if (_error
->PendingError() == true)
297 return _error
->Error(_("Error processing directory %s"),BaseDir
.c_str());
299 c0out
<< ' ' << BaseDir
<< ":" << flush
;
301 // Do recursive directory searching
302 if (FLFile
.empty() == true)
304 if (Sources
.RecursiveScan(flCombine(ArchiveDir
,BaseDir
))== false)
309 if (Sources
.LoadFileList(ArchiveDir
,FLFile
) == false)
312 Sources
.Output
= 0; // Just in case
314 // Finish compressing
315 unsigned long long Size
;
316 if (Comp
.Finalize(Size
) == false)
319 return _error
->Error(_("Error processing directory %s"),BaseDir
.c_str());
324 << SizeToStr(Size
) << "B ";
328 struct timeval NewTime
;
329 gettimeofday(&NewTime
,0);
330 double Delta
= NewTime
.tv_sec
- StartTime
.tv_sec
+
331 (NewTime
.tv_usec
- StartTime
.tv_usec
)/1000000.0;
333 c0out
<< Sources
.Stats
.Packages
<< " pkgs in " <<
334 TimeToStr((long)Delta
) << endl
;
336 if(_config
->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
337 c0out
<< " Misses in Cache: " << Sources
.Stats
.Misses
<< endl
;
339 Stats
.Add(Sources
.Stats
);
340 Stats
.DeLinkBytes
= Sources
.Stats
.DeLinkBytes
;
342 return !_error
->PendingError();
345 // PackageMap::GenContents - Actually generate a Contents file /*{{{*/
346 // ---------------------------------------------------------------------
347 /* This generates the contents file partially described by this object.
348 It searches the given iterator range for other package files that map
349 into this contents file and includes their data as well when building. */
350 bool PackageMap::GenContents(Configuration
&Setup
,
351 vector
<PackageMap
>::iterator Begin
,
352 vector
<PackageMap
>::iterator End
,
355 if (Contents
.empty() == true)
361 string ArchiveDir
= Setup
.FindDir("Dir::ArchiveDir");
362 string CacheDir
= Setup
.FindDir("Dir::CacheDir");
363 string OverrideDir
= Setup
.FindDir("Dir::OverrideDir");
365 struct timeval StartTime
;
366 gettimeofday(&StartTime
,0);
368 // Create a package writer object.
369 ContentsWriter
Contents("", Arch
);
370 if (PkgExt
.empty() == false && Contents
.SetExts(PkgExt
) == false)
371 return _error
->Error(_("Package extension list is too long"));
372 if (_error
->PendingError() == true)
375 MultiCompress
Comp(flCombine(ArchiveDir
,this->Contents
),
376 CntCompress
,Permissions
);
377 Comp
.UpdateMTime
= Setup
.FindI("Default::ContentsAge",10)*24*60*60;
378 Contents
.Output
= Comp
.Input
;
379 if (_error
->PendingError() == true)
382 // Write the header out.
383 if (ContentsHead
.empty() == false)
385 FileFd
Head(flCombine(OverrideDir
,ContentsHead
),FileFd::ReadOnly
);
386 if (_error
->PendingError() == true)
389 unsigned long long Size
= Head
.Size();
390 unsigned char Buf
[4096];
393 unsigned long long ToRead
= Size
;
394 if (Size
> sizeof(Buf
))
395 ToRead
= sizeof(Buf
);
397 if (Head
.Read(Buf
,ToRead
) == false)
400 if (fwrite(Buf
,1,ToRead
,Comp
.Input
) != ToRead
)
401 return _error
->Errno("fwrite",_("Error writing header to contents file"));
407 /* Go over all the package file records and parse all the package
408 files associated with this contents file into one great big honking
409 memory structure, then dump the sorted version */
410 c0out
<< ' ' << this->Contents
<< ":" << flush
;
411 for (vector
<PackageMap
>::iterator I
= Begin
; I
!= End
; ++I
)
413 if (I
->Contents
!= this->Contents
)
416 Contents
.Prefix
= ArchiveDir
;
417 Contents
.ReadyDB(flCombine(CacheDir
,I
->BinCacheDB
));
418 Contents
.ReadFromPkgs(flCombine(ArchiveDir
,I
->PkgFile
),
421 I
->ContentsDone
= true;
426 // Finish compressing
427 unsigned long long Size
;
428 if (Comp
.Finalize(Size
) == false || _error
->PendingError() == true)
431 return _error
->Error(_("Error processing contents %s"),
432 this->Contents
.c_str());
437 c0out
<< " New " << SizeToStr(Size
) << "B ";
446 struct timeval NewTime
;
447 gettimeofday(&NewTime
,0);
448 double Delta
= NewTime
.tv_sec
- StartTime
.tv_sec
+
449 (NewTime
.tv_usec
- StartTime
.tv_usec
)/1000000.0;
451 if(_config
->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
452 c0out
<< " Misses in Cache: " << Contents
.Stats
.Misses
<< endl
;
454 c0out
<< Contents
.Stats
.Packages
<< " files " <<
455 SizeToStr(Contents
.Stats
.Bytes
) << "B " <<
456 TimeToStr((long)Delta
) << endl
;
462 // LoadTree - Load a 'tree' section from the Generate Config /*{{{*/
463 // ---------------------------------------------------------------------
464 /* This populates the PkgList with all the possible permutations of the
465 section/arch lists. */
466 static void LoadTree(vector
<PackageMap
> &PkgList
,Configuration
&Setup
)
469 string DDir
= Setup
.Find("TreeDefault::Directory",
470 "$(DIST)/$(SECTION)/binary-$(ARCH)/");
471 string DSDir
= Setup
.Find("TreeDefault::SrcDirectory",
472 "$(DIST)/$(SECTION)/source/");
473 string DPkg
= Setup
.Find("TreeDefault::Packages",
474 "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
475 string DTrans
= Setup
.Find("TreeDefault::Translation",
476 "$(DIST)/$(SECTION)/i18n/Translation-en");
477 string DIPrfx
= Setup
.Find("TreeDefault::InternalPrefix",
478 "$(DIST)/$(SECTION)/");
479 string DContents
= Setup
.Find("TreeDefault::Contents",
480 "$(DIST)/$(SECTION)/Contents-$(ARCH)");
481 string DContentsH
= Setup
.Find("TreeDefault::Contents::Header","");
482 string DBCache
= Setup
.Find("TreeDefault::BinCacheDB",
483 "packages-$(ARCH).db");
484 string SrcDBCache
= Setup
.Find("TreeDefault::SrcCacheDB",
485 "sources-$(SECTION).db");
486 string DSources
= Setup
.Find("TreeDefault::Sources",
487 "$(DIST)/$(SECTION)/source/Sources");
488 string DFLFile
= Setup
.Find("TreeDefault::FileList", "");
489 string DSFLFile
= Setup
.Find("TreeDefault::SourceFileList", "");
491 mode_t
const Permissions
= Setup
.FindI("Default::FileMode",0644);
493 bool const LongDescription
= Setup
.FindB("Default::LongDescription",
494 _config
->FindB("APT::FTPArchive::LongDescription", true));
495 string
const TranslationCompress
= Setup
.Find("Default::Translation::Compress",". gzip").c_str();
497 // Process 'tree' type sections
498 const Configuration::Item
*Top
= Setup
.Tree("tree");
499 for (Top
= (Top
== 0?0:Top
->Child
); Top
!= 0;)
501 Configuration
Block(Top
);
502 string Dist
= Top
->Tag
;
504 // Parse the sections
505 string Tmp
= Block
.Find("Sections");
506 const char *Sections
= Tmp
.c_str();
508 while (ParseQuoteWord(Sections
,Section
) == true)
511 struct SubstVar
const Vars
[] = {{"$(DIST)",&Dist
},
512 {"$(SECTION)",&Section
},
515 mode_t
const Perms
= Block
.FindI("FileMode", Permissions
);
516 bool const LongDesc
= Block
.FindB("LongDescription", LongDescription
);
517 TranslationWriter
*TransWriter
;
518 if (DTrans
.empty() == false && LongDesc
== false)
520 string
const TranslationFile
= flCombine(Setup
.FindDir("Dir::ArchiveDir"),
521 SubstVar(Block
.Find("Translation", DTrans
.c_str()), Vars
));
522 string
const TransCompress
= Block
.Find("Translation::Compress", TranslationCompress
);
523 TransWriter
= new TranslationWriter(TranslationFile
, TransCompress
, Perms
);
528 string
const Tmp2
= Block
.Find("Architectures");
529 const char *Archs
= Tmp2
.c_str();
530 while (ParseQuoteWord(Archs
,Arch
) == true)
533 Itm
.Permissions
= Perms
;
534 Itm
.BinOverride
= SubstVar(Block
.Find("BinOverride"),Vars
);
535 Itm
.InternalPrefix
= SubstVar(Block
.Find("InternalPrefix",DIPrfx
.c_str()),Vars
);
537 if (stringcasecmp(Arch
,"source") == 0)
539 Itm
.SrcOverride
= SubstVar(Block
.Find("SrcOverride"),Vars
);
540 Itm
.BaseDir
= SubstVar(Block
.Find("SrcDirectory",DSDir
.c_str()),Vars
);
541 Itm
.SrcFile
= SubstVar(Block
.Find("Sources",DSources
.c_str()),Vars
);
542 Itm
.Tag
= SubstVar("$(DIST)/$(SECTION)/source",Vars
);
543 Itm
.FLFile
= SubstVar(Block
.Find("SourceFileList",DSFLFile
.c_str()),Vars
);
544 Itm
.SrcExtraOverride
= SubstVar(Block
.Find("SrcExtraOverride"),Vars
);
545 Itm
.SrcCacheDB
= SubstVar(Block
.Find("SrcCacheDB",SrcDBCache
.c_str()),Vars
);
549 Itm
.BinCacheDB
= SubstVar(Block
.Find("BinCacheDB",DBCache
.c_str()),Vars
);
550 Itm
.BaseDir
= SubstVar(Block
.Find("Directory",DDir
.c_str()),Vars
);
551 Itm
.PkgFile
= SubstVar(Block
.Find("Packages",DPkg
.c_str()),Vars
);
552 Itm
.Tag
= SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars
);
554 Itm
.LongDesc
= LongDesc
;
555 if (TransWriter
!= NULL
)
557 TransWriter
->IncreaseRefCounter();
558 Itm
.TransWriter
= TransWriter
;
560 Itm
.Contents
= SubstVar(Block
.Find("Contents",DContents
.c_str()),Vars
);
561 Itm
.ContentsHead
= SubstVar(Block
.Find("Contents::Header",DContentsH
.c_str()),Vars
);
562 Itm
.FLFile
= SubstVar(Block
.Find("FileList",DFLFile
.c_str()),Vars
);
563 Itm
.ExtraOverride
= SubstVar(Block
.Find("ExtraOverride"),Vars
);
566 Itm
.GetGeneral(Setup
,Block
);
567 PkgList
.push_back(Itm
);
569 // we didn't use this TransWriter, so we can release it
570 if (TransWriter
!= NULL
&& TransWriter
->GetRefCounter() == 0)
578 // LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/
579 // ---------------------------------------------------------------------
581 static void LoadBinDir(vector
<PackageMap
> &PkgList
,Configuration
&Setup
)
583 mode_t
const Permissions
= Setup
.FindI("Default::FileMode",0644);
585 // Process 'bindirectory' type sections
586 const Configuration::Item
*Top
= Setup
.Tree("bindirectory");
587 for (Top
= (Top
== 0?0:Top
->Child
); Top
!= 0;)
589 Configuration
Block(Top
);
592 Itm
.PkgFile
= Block
.Find("Packages");
593 Itm
.SrcFile
= Block
.Find("Sources");
594 Itm
.BinCacheDB
= Block
.Find("BinCacheDB");
595 Itm
.SrcCacheDB
= Block
.Find("SrcCacheDB");
596 Itm
.BinOverride
= Block
.Find("BinOverride");
597 Itm
.ExtraOverride
= Block
.Find("ExtraOverride");
598 Itm
.SrcExtraOverride
= Block
.Find("SrcExtraOverride");
599 Itm
.SrcOverride
= Block
.Find("SrcOverride");
600 Itm
.BaseDir
= Top
->Tag
;
601 Itm
.FLFile
= Block
.Find("FileList");
602 Itm
.InternalPrefix
= Block
.Find("InternalPrefix",Top
->Tag
.c_str());
603 Itm
.Contents
= Block
.Find("Contents");
604 Itm
.ContentsHead
= Block
.Find("Contents::Header");
605 Itm
.Permissions
= Block
.FindI("FileMode", Permissions
);
607 Itm
.GetGeneral(Setup
,Block
);
608 PkgList
.push_back(Itm
);
615 // ShowHelp - Show the help text /*{{{*/
616 // ---------------------------------------------------------------------
618 static bool ShowHelp(CommandLine
&)
620 ioprintf(cout
,_("%s %s for %s compiled on %s %s\n"),PACKAGE
,PACKAGE_VERSION
,
621 COMMON_ARCH
,__DATE__
,__TIME__
);
622 if (_config
->FindB("version") == true)
626 _("Usage: apt-ftparchive [options] command\n"
627 "Commands: packages binarypath [overridefile [pathprefix]]\n"
628 " sources srcpath [overridefile [pathprefix]]\n"
631 " generate config [groups]\n"
634 "apt-ftparchive generates index files for Debian archives. It supports\n"
635 "many styles of generation from fully automated to functional replacements\n"
636 "for dpkg-scanpackages and dpkg-scansources\n"
638 "apt-ftparchive generates Package files from a tree of .debs. The\n"
639 "Package file contains the contents of all the control fields from\n"
640 "each package as well as the MD5 hash and filesize. An override file\n"
641 "is supported to force the value of Priority and Section.\n"
643 "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
644 "The --source-override option can be used to specify a src override file\n"
646 "The 'packages' and 'sources' command should be run in the root of the\n"
647 "tree. BinaryPath should point to the base of the recursive search and \n"
648 "override file should contain the override flags. Pathprefix is\n"
649 "appended to the filename fields if present. Example usage from the \n"
651 " apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
652 " dists/potato/main/binary-i386/Packages\n"
655 " -h This help text\n"
656 " --md5 Control MD5 generation\n"
657 " -s=? Source override file\n"
659 " -d=? Select the optional caching database\n"
660 " --no-delink Enable delinking debug mode\n"
661 " --contents Control contents file generation\n"
662 " -c=? Read this configuration file\n"
663 " -o=? Set an arbitrary configuration option") << endl
;
668 // SimpleGenPackages - Generate a Packages file for a directory tree /*{{{*/
669 // ---------------------------------------------------------------------
670 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
671 static bool SimpleGenPackages(CommandLine
&CmdL
)
673 if (CmdL
.FileSize() < 2)
674 return ShowHelp(CmdL
);
677 if (CmdL
.FileSize() >= 3)
678 Override
= CmdL
.FileList
[2];
680 // Create a package writer object.
681 PackagesWriter
Packages(_config
->Find("APT::FTPArchive::DB"),
682 Override
, "", _config
->Find("APT::FTPArchive::Architecture"));
683 if (_error
->PendingError() == true)
686 if (CmdL
.FileSize() >= 4)
687 Packages
.PathPrefix
= CmdL
.FileList
[3];
689 // Do recursive directory searching
690 if (Packages
.RecursiveScan(CmdL
.FileList
[1]) == false)
693 // Give some stats if asked for
694 if(_config
->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
695 c0out
<< " Misses in Cache: " << Packages
.Stats
.Misses
<< endl
;
700 // SimpleGenContents - Generate a Contents listing /*{{{*/
701 // ---------------------------------------------------------------------
703 static bool SimpleGenContents(CommandLine
&CmdL
)
705 if (CmdL
.FileSize() < 2)
706 return ShowHelp(CmdL
);
708 // Create a package writer object.
709 ContentsWriter
Contents(_config
->Find("APT::FTPArchive::DB"), _config
->Find("APT::FTPArchive::Architecture"));
710 if (_error
->PendingError() == true)
713 // Do recursive directory searching
714 if (Contents
.RecursiveScan(CmdL
.FileList
[1]) == false)
722 // SimpleGenSources - Generate a Sources file for a directory tree /*{{{*/
723 // ---------------------------------------------------------------------
724 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
725 static bool SimpleGenSources(CommandLine
&CmdL
)
727 if (CmdL
.FileSize() < 2)
728 return ShowHelp(CmdL
);
731 if (CmdL
.FileSize() >= 3)
732 Override
= CmdL
.FileList
[2];
735 if (Override
.empty() == false)
736 SOverride
= Override
+ ".src";
738 SOverride
= _config
->Find("APT::FTPArchive::SourceOverride",
741 // Create a package writer object.
742 SourcesWriter
Sources(_config
->Find("APT::FTPArchive::DB"),Override
,SOverride
);
743 if (_error
->PendingError() == true)
746 if (CmdL
.FileSize() >= 4)
747 Sources
.PathPrefix
= CmdL
.FileList
[3];
749 // Do recursive directory searching
750 if (Sources
.RecursiveScan(CmdL
.FileList
[1]) == false)
753 // Give some stats if asked for
754 if(_config
->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
755 c0out
<< " Misses in Cache: " << Sources
.Stats
.Misses
<< endl
;
760 // SimpleGenRelease - Generate a Release file for a directory tree /*{{{*/
761 // ---------------------------------------------------------------------
762 static bool SimpleGenRelease(CommandLine
&CmdL
)
764 if (CmdL
.FileSize() < 2)
765 return ShowHelp(CmdL
);
767 string Dir
= CmdL
.FileList
[1];
769 ReleaseWriter
Release("");
770 Release
.DirStrip
= Dir
;
772 if (_error
->PendingError() == true)
775 if (Release
.RecursiveScan(Dir
) == false)
784 // DoGeneratePackagesAndSources - Helper for Generate /*{{{*/
785 // ---------------------------------------------------------------------
786 static bool DoGeneratePackagesAndSources(Configuration
&Setup
,
787 vector
<PackageMap
> &PkgList
,
788 struct CacheDB::Stats
&SrcStats
,
789 struct CacheDB::Stats
&Stats
,
792 if (CmdL
.FileSize() <= 2)
794 for (vector
<PackageMap
>::iterator I
= PkgList
.begin(); I
!= PkgList
.end(); ++I
)
795 if (I
->GenPackages(Setup
,Stats
) == false)
796 _error
->DumpErrors();
797 for (vector
<PackageMap
>::iterator I
= PkgList
.begin(); I
!= PkgList
.end(); ++I
)
798 if (I
->GenSources(Setup
,SrcStats
) == false)
799 _error
->DumpErrors();
803 // Make a choice list out of the package list..
804 RxChoiceList
*List
= new RxChoiceList
[2*PkgList
.size()+1];
805 RxChoiceList
*End
= List
;
806 for (vector
<PackageMap
>::iterator I
= PkgList
.begin(); I
!= PkgList
.end(); ++I
)
808 End
->UserData
= &(*I
);
809 End
->Str
= I
->BaseDir
.c_str();
812 End
->UserData
= &(*I
);
813 End
->Str
= I
->Tag
.c_str();
819 if (RegexChoice(List
,CmdL
.FileList
+ 2,CmdL
.FileList
+ CmdL
.FileSize()) == 0)
822 return _error
->Error(_("No selections matched"));
824 _error
->DumpErrors();
826 // Do the generation for Packages
827 for (End
= List
; End
->Str
!= 0; End
++)
829 if (End
->Hit
== false)
832 PackageMap
*I
= (PackageMap
*)End
->UserData
;
833 if (I
->PkgDone
== true)
835 if (I
->GenPackages(Setup
,Stats
) == false)
836 _error
->DumpErrors();
839 // Do the generation for Sources
840 for (End
= List
; End
->Str
!= 0; End
++)
842 if (End
->Hit
== false)
845 PackageMap
*I
= (PackageMap
*)End
->UserData
;
846 if (I
->SrcDone
== true)
848 if (I
->GenSources(Setup
,SrcStats
) == false)
849 _error
->DumpErrors();
855 // close the Translation master files
856 for (vector
<PackageMap
>::reverse_iterator I
= PkgList
.rbegin(); I
!= PkgList
.rend(); ++I
)
857 if (I
->TransWriter
!= NULL
&& I
->TransWriter
->DecreaseRefCounter() == 0)
858 delete I
->TransWriter
;
864 // DoGenerateContents - Helper for Generate to generate the Contents /*{{{*/
865 // ---------------------------------------------------------------------
866 static bool DoGenerateContents(Configuration
&Setup
,
867 vector
<PackageMap
> &PkgList
,
870 c1out
<< "Packages done, Starting contents." << endl
;
872 // Sort the contents file list by date
873 string ArchiveDir
= Setup
.FindDir("Dir::ArchiveDir");
874 for (vector
<PackageMap
>::iterator I
= PkgList
.begin(); I
!= PkgList
.end(); ++I
)
877 if (MultiCompress::GetStat(flCombine(ArchiveDir
,I
->Contents
),
878 I
->CntCompress
,A
) == false)
879 time(&I
->ContentsMTime
);
881 I
->ContentsMTime
= A
.st_mtime
;
883 stable_sort(PkgList
.begin(),PkgList
.end(),PackageMap::ContentsCompare());
885 /* Now for Contents.. The process here is to do a make-like dependency
886 check. Each contents file is verified to be newer than the package files
887 that describe the debs it indexes. Since the package files contain
888 hashes of the .debs this means they have not changed either so the
889 contents must be up to date. */
890 unsigned long MaxContentsChange
= Setup
.FindI("Default::MaxContentsChange",UINT_MAX
)*1024;
891 for (vector
<PackageMap
>::iterator I
= PkgList
.begin(); I
!= PkgList
.end(); ++I
)
893 // This record is not relevant
894 if (I
->ContentsDone
== true ||
895 I
->Contents
.empty() == true)
898 // Do not do everything if the user specified sections.
899 if (CmdL
.FileSize() > 2 && I
->PkgDone
== false)
903 if (MultiCompress::GetStat(flCombine(ArchiveDir
,I
->Contents
),I
->CntCompress
,A
) == true)
905 if (MultiCompress::GetStat(flCombine(ArchiveDir
,I
->PkgFile
),I
->PkgCompress
,B
) == false)
907 _error
->Warning(_("Some files are missing in the package file group `%s'"),I
->PkgFile
.c_str());
911 if (A
.st_mtime
> B
.st_mtime
)
915 if (I
->GenContents(Setup
,PkgList
.begin(),PkgList
.end(),
916 MaxContentsChange
) == false)
917 _error
->DumpErrors();
920 if (MaxContentsChange
== 0)
922 c1out
<< "Hit contents update byte limit" << endl
;
931 // Generate - Full generate, using a config file /*{{{*/
932 // ---------------------------------------------------------------------
934 static bool Generate(CommandLine
&CmdL
)
936 struct CacheDB::Stats SrcStats
;
937 if (CmdL
.FileSize() < 2)
938 return ShowHelp(CmdL
);
940 struct timeval StartTime
;
941 gettimeofday(&StartTime
,0);
942 struct CacheDB::Stats Stats
;
944 // Read the configuration file.
946 if (ReadConfigFile(Setup
,CmdL
.FileList
[1],true) == false)
949 vector
<PackageMap
> PkgList
;
950 LoadTree(PkgList
,Setup
);
951 LoadBinDir(PkgList
,Setup
);
953 // Sort by cache DB to improve IO locality.
954 stable_sort(PkgList
.begin(),PkgList
.end(),PackageMap::DBCompare());
955 stable_sort(PkgList
.begin(),PkgList
.end(),PackageMap::SrcDBCompare());
958 if (_config
->FindB("APT::FTPArchive::ContentsOnly", false) == false)
960 if(DoGeneratePackagesAndSources(Setup
, PkgList
, SrcStats
, Stats
, CmdL
) == false)
963 c1out
<< "Skipping Packages/Sources generation" << endl
;
966 // do Contents if needed
967 if (_config
->FindB("APT::FTPArchive::Contents", true) == true)
968 if (DoGenerateContents(Setup
, PkgList
, CmdL
) == false)
971 struct timeval NewTime
;
972 gettimeofday(&NewTime
,0);
973 double Delta
= NewTime
.tv_sec
- StartTime
.tv_sec
+
974 (NewTime
.tv_usec
- StartTime
.tv_usec
)/1000000.0;
975 c1out
<< "Done. " << SizeToStr(Stats
.Bytes
) << "B in " << Stats
.Packages
976 << " archives. Took " << TimeToStr((long)Delta
) << endl
;
982 // Clean - Clean out the databases /*{{{*/
983 // ---------------------------------------------------------------------
985 static bool Clean(CommandLine
&CmdL
)
987 if (CmdL
.FileSize() != 2)
988 return ShowHelp(CmdL
);
990 // Read the configuration file.
992 if (ReadConfigFile(Setup
,CmdL
.FileList
[1],true) == false)
995 vector
<PackageMap
> PkgList
;
996 LoadTree(PkgList
,Setup
);
997 LoadBinDir(PkgList
,Setup
);
999 // Sort by cache DB to improve IO locality.
1000 stable_sort(PkgList
.begin(),PkgList
.end(),PackageMap::DBCompare());
1001 stable_sort(PkgList
.begin(),PkgList
.end(),PackageMap::SrcDBCompare());
1003 string CacheDir
= Setup
.FindDir("Dir::CacheDir");
1005 for (vector
<PackageMap
>::iterator I
= PkgList
.begin(); I
!= PkgList
.end(); )
1007 if(I
->BinCacheDB
!= "")
1008 c0out
<< I
->BinCacheDB
<< endl
;
1009 if(I
->SrcCacheDB
!= "")
1010 c0out
<< I
->SrcCacheDB
<< endl
;
1011 CacheDB
DB(flCombine(CacheDir
,I
->BinCacheDB
));
1012 CacheDB
DB_SRC(flCombine(CacheDir
,I
->SrcCacheDB
));
1013 if (DB
.Clean() == false)
1014 _error
->DumpErrors();
1015 if (DB_SRC
.Clean() == false)
1016 _error
->DumpErrors();
1018 string CacheDB
= I
->BinCacheDB
;
1019 string SrcCacheDB
= I
->SrcCacheDB
;
1020 while(I
!= PkgList
.end() &&
1021 I
->BinCacheDB
== CacheDB
&&
1022 I
->SrcCacheDB
== SrcCacheDB
)
1032 int main(int argc
, const char *argv
[])
1034 setlocale(LC_ALL
, "");
1035 CommandLine::Args Args
[] = {
1036 {'h',"help","help",0},
1037 {0,"md5","APT::FTPArchive::MD5",0},
1038 {0,"sha1","APT::FTPArchive::SHA1",0},
1039 {0,"sha256","APT::FTPArchive::SHA256",0},
1040 {'v',"version","version",0},
1041 {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg
},
1042 {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg
},
1043 {'q',"quiet","quiet",CommandLine::IntLevel
},
1044 {'q',"silent","quiet",CommandLine::IntLevel
},
1045 {0,"delink","APT::FTPArchive::DeLinkAct",0},
1046 {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
1047 {0,"contents","APT::FTPArchive::Contents",0},
1048 {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg
},
1049 {'c',"config-file",0,CommandLine::ConfigFile
},
1050 {'o',"option",0,CommandLine::ArbItem
},
1052 CommandLine::Dispatch Cmds
[] = {{"packages",&SimpleGenPackages
},
1053 {"contents",&SimpleGenContents
},
1054 {"sources",&SimpleGenSources
},
1055 {"release",&SimpleGenRelease
},
1056 {"generate",&Generate
},
1061 // Parse the command line and initialize the package library
1062 CommandLine
CmdL(Args
,_config
);
1063 if (pkgInitConfig(*_config
) == false || CmdL
.Parse(argc
,argv
) == false)
1065 _error
->DumpErrors();
1069 // See if the help should be shown
1070 if (_config
->FindB("help") == true ||
1071 _config
->FindB("version") == true ||
1072 CmdL
.FileSize() == 0)
1078 // Setup the output streams
1079 c0out
.rdbuf(clog
.rdbuf());
1080 c1out
.rdbuf(clog
.rdbuf());
1081 c2out
.rdbuf(clog
.rdbuf());
1082 Quiet
= _config
->FindI("quiet",0);
1084 c0out
.rdbuf(devnull
.rdbuf());
1086 c1out
.rdbuf(devnull
.rdbuf());
1088 // Match the operation
1089 CmdL
.DispatchArg(Cmds
);
1091 if (_error
->empty() == false)
1093 bool Errors
= _error
->PendingError();
1094 _error
->DumpErrors();
1095 return Errors
== true?100:0;