git.saurik.com Git - apt-legacy.git/blame - ftparchive/apt-ftparchive.cc

Commit	Line	Data
da6ee469 JF	1	// -- mode: cpp; mode: fold --
	2	// Description /{{{/
	3	// $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
	4	/* ######################################################################
	5
0e5943eb	6	apt-ftparchive - Efficient work-alike for dpkg-scanpackages
da6ee469 JF	7
	8	Let contents be disabled from the conf
	9
	10	##################################################################### */
	11	/}}}/
	12	// Include Files /{{{/
da6ee469 JF	13	#include "apt-ftparchive.h"
	14
	15	#include <apt-pkg/error.h>
	16	#include <apt-pkg/configuration.h>
	17	#include <apt-pkg/cmndline.h>
	18	#include <apt-pkg/strutl.h>
	19	#include <config.h>
	20	#include <apti18n.h>
	21	#include <algorithm>
	22
00ec24d0	23	#include <climits>
da6ee469 JF	24	#include <sys/time.h>
	25	#include <regex.h>
	26
	27	#include "contents.h"
	28	#include "multicompress.h"
	29	#include "writer.h"
	30	/}}}/
	31
	32	using namespace std;
	33	ostream c0out(0);
	34	ostream c1out(0);
	35	ostream c2out(0);
	36	ofstream devnull("/dev/null");
	37	unsigned Quiet = 0;
	38
	39	// struct PackageMap - List of all package files in the config file /{{{/
	40	// ---------------------------------------------------------------------
	41	/* */
	42	struct PackageMap
	43	{
	44	// General Stuff
	45	string BaseDir;
	46	string InternalPrefix;
	47	string FLFile;
	48	string PkgExt;
	49	string SrcExt;
	50
	51	// Stuff for the Package File
	52	string PkgFile;
	53	string BinCacheDB;
	54	string BinOverride;
	55	string ExtraOverride;
	56
	57	// We generate for this given arch
	58	string Arch;
	59
	60	// Stuff for the Source File
	61	string SrcFile;
	62	string SrcOverride;
	63	string SrcExtraOverride;
	64
	65	// Contents
	66	string Contents;
	67	string ContentsHead;
	68
	69	// Random things
	70	string Tag;
	71	string PkgCompress;
	72	string CntCompress;
	73	string SrcCompress;
	74	string PathPrefix;
	75	unsigned int DeLinkLimit;
	76	mode_t Permissions;
	77
	78	bool ContentsDone;
	79	bool PkgDone;
	80	bool SrcDone;
	81	time_t ContentsMTime;
	82
	83	struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
	84	{
	85	inline bool operator() (const PackageMap &x,const PackageMap &y)
	86	{return x.ContentsMTime < y.ContentsMTime;};
	87	};
88
89	struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
90	{
91	inline bool operator() (const PackageMap &x,const PackageMap &y)
92	{return x.BinCacheDB < y.BinCacheDB;};
93	};
94
95	void GetGeneral(Configuration &Setup,Configuration &Block);
96	bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
97	bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
98	bool GenContents(Configuration &Setup,
99	vector<PackageMap>::iterator Begin,
100	vector<PackageMap>::iterator End,
101	unsigned long &Left);
102
103	PackageMap() : DeLinkLimit(0), Permissions(1), ContentsDone(false),
104	PkgDone(false), SrcDone(false), ContentsMTime(0) {};
105	};
106	/}}}/
107
108	// PackageMap::GetGeneral - Common per-section definitions /{{{/
109	// ---------------------------------------------------------------------
110	/* */
111	void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
112	{
113	PathPrefix = Block.Find("PathPrefix");
114
115	if (Block.FindB("External-Links",true) == false)
116	DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
117	else
118	DeLinkLimit = 0;
119
120	PkgCompress = Block.Find("Packages::Compress",
121	Setup.Find("Default::Packages::Compress",". gzip").c_str());
122	CntCompress = Block.Find("Contents::Compress",
123	Setup.Find("Default::Contents::Compress",". gzip").c_str());
124	SrcCompress = Block.Find("Sources::Compress",
125	Setup.Find("Default::Sources::Compress",". gzip").c_str());
126
127	SrcExt = Block.Find("Sources::Extensions",
128	Setup.Find("Default::Sources::Extensions",".dsc").c_str());
129	PkgExt = Block.Find("Packages::Extensions",
130	Setup.Find("Default::Packages::Extensions",".deb").c_str());
131
132	Permissions = Setup.FindI("Default::FileMode",0644);
133
134	if (FLFile.empty() == false)
135	FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
136
137	if (Contents == " ")
138	Contents= string();
139	}
140	/}}}/
141	// PackageMap::GenPackages - Actually generate a Package file /{{{/
142	// ---------------------------------------------------------------------
143	/* This generates the Package File described by this object. */
144	bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
145	{
146	if (PkgFile.empty() == true)
147	return true;
148
149	string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
150	string OverrideDir = Setup.FindDir("Dir::OverrideDir");
151	string CacheDir = Setup.FindDir("Dir::CacheDir");
152
153	struct timeval StartTime;
154	gettimeofday(&StartTime,0);
155
156	PkgDone = true;
157
158	// Create a package writer object.
159	PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
160	flCombine(OverrideDir,BinOverride),
161	flCombine(OverrideDir,ExtraOverride),
162	Arch);
163	if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
164	return _error->Error(_("Package extension list is too long"));
165	if (_error->PendingError() == true)
166	return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
167
168	Packages.PathPrefix = PathPrefix;
169	Packages.DirStrip = ArchiveDir;
170	Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
171
172	Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
173	Packages.DeLinkLimit = DeLinkLimit;
174
175	// Create a compressor object
176	MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
177	PkgCompress,Permissions);
178	Packages.Output = Comp.Input;
179	if (_error->PendingError() == true)
180	return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
181
182	c0out << ' ' << BaseDir << ":" << flush;
183
184	// Do recursive directory searching
185	if (FLFile.empty() == true)
186	{
187	if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
188	return false;
189	}
190	else
191	{
192	if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
193	return false;
194	}
195
196	Packages.Output = 0; // Just in case
197
198	// Finish compressing
199	unsigned long Size;
200	if (Comp.Finalize(Size) == false)
201	{
202	c0out << endl;
203	return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
204	}
205
206	if (Size != 0)
207	c0out << " New "
208	<< SizeToStr(Size) << "B ";
209	else
210	c0out << ' ';
211
212	struct timeval NewTime;
213	gettimeofday(&NewTime,0);
214	double Delta = NewTime.tv_sec - StartTime.tv_sec +
215	(NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
216
217	c0out << Packages.Stats.Packages << " files " <<
218	/* SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
219	SizeToStr(Packages.Stats.Bytes) << "B " <<
220	TimeToStr((long)Delta) << endl;
221
222	Stats.Add(Packages.Stats);
223	Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
224
225	return !_error->PendingError();
226	}
227
228	/}}}/
229	// PackageMap::GenSources - Actually generate a Source file /{{{/
230	// ---------------------------------------------------------------------
231	/* This generates the Sources File described by this object. */
232	bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
233	{
234	if (SrcFile.empty() == true)
235	return true;
236
237	string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
238	string OverrideDir = Setup.FindDir("Dir::OverrideDir");
239	string CacheDir = Setup.FindDir("Dir::CacheDir");
240
241	struct timeval StartTime;
242	gettimeofday(&StartTime,0);
243
244	SrcDone = true;
245
246	// Create a package writer object.
247	SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
248	flCombine(OverrideDir,SrcOverride),
249	flCombine(OverrideDir,SrcExtraOverride));
250	if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
251	return _error->Error(_("Source extension list is too long"));
252	if (_error->PendingError() == true)
253	return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
254
255	Sources.PathPrefix = PathPrefix;
256	Sources.DirStrip = ArchiveDir;
257	Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
258
259	Sources.DeLinkLimit = DeLinkLimit;
260	Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
261
262	// Create a compressor object
263	MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
264	SrcCompress,Permissions);
265	Sources.Output = Comp.Input;
266	if (_error->PendingError() == true)
267	return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
268
269	c0out << ' ' << BaseDir << ":" << flush;
270
271	// Do recursive directory searching
272	if (FLFile.empty() == true)
273	{
274	if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
275	return false;
276	}
277	else
278	{
279	if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
280	return false;
281	}
282	Sources.Output = 0; // Just in case
283
284	// Finish compressing
285	unsigned long Size;
286	if (Comp.Finalize(Size) == false)
287	{
288	c0out << endl;
289	return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
290	}
291
292	if (Size != 0)
293	c0out << " New "
294	<< SizeToStr(Size) << "B ";
295	else
296	c0out << ' ';
297
298	struct timeval NewTime;
299	gettimeofday(&NewTime,0);
300	double Delta = NewTime.tv_sec - StartTime.tv_sec +
301	(NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
302
303	c0out << Sources.Stats.Packages << " pkgs in " <<
304	TimeToStr((long)Delta) << endl;
305
306	Stats.Add(Sources.Stats);
307	Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
308
309	return !_error->PendingError();
310	}
311	/}}}/
312	// PackageMap::GenContents - Actually generate a Contents file /{{{/
313	// ---------------------------------------------------------------------
314	/* This generates the contents file partially described by this object.
315	It searches the given iterator range for other package files that map
316	into this contents file and includes their data as well when building. */
317	bool PackageMap::GenContents(Configuration &Setup,
318	vector<PackageMap>::iterator Begin,
319	vector<PackageMap>::iterator End,
320	unsigned long &Left)
321	{
322	if (Contents.empty() == true)
323	return true;
324
325	if (Left == 0)
326	return true;
327
328	string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
329	string CacheDir = Setup.FindDir("Dir::CacheDir");
330	string OverrideDir = Setup.FindDir("Dir::OverrideDir");
331
332	struct timeval StartTime;
333	gettimeofday(&StartTime,0);
334
335	// Create a package writer object.
336	ContentsWriter Contents("");
337	if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
338	return _error->Error(_("Package extension list is too long"));
339	if (_error->PendingError() == true)
340	return false;
341
342	MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
343	CntCompress,Permissions);
344	Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)2460*60;
345	Contents.Output = Comp.Input;
346	if (_error->PendingError() == true)
347	return false;
348
349	// Write the header out.
350	if (ContentsHead.empty() == false)
351	{
352	FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
353	if (_error->PendingError() == true)
354	return false;
355
356	unsigned long Size = Head.Size();
357	unsigned char Buf[4096];
358	while (Size != 0)
359	{
360	unsigned long ToRead = Size;
361	if (Size > sizeof(Buf))
362	ToRead = sizeof(Buf);
363
364	if (Head.Read(Buf,ToRead) == false)
365	return false;
366
367	if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
368	return _error->Errno("fwrite",_("Error writing header to contents file"));
369
370	Size -= ToRead;
371	}
372	}
373
374	/* Go over all the package file records and parse all the package
375	files associated with this contents file into one great big honking
376	memory structure, then dump the sorted version */
377	c0out << ' ' << this->Contents << ":" << flush;
378	for (vector<PackageMap>::iterator I = Begin; I != End; I++)
379	{
380	if (I->Contents != this->Contents)
381	continue;
382
383	Contents.Prefix = ArchiveDir;
384	Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
385	Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
386	I->PkgCompress);
387
388	I->ContentsDone = true;
389	}
390
391	Contents.Finish();
392
393	// Finish compressing
394	unsigned long Size;
395	if (Comp.Finalize(Size) == false \|\| _error->PendingError() == true)
396	{
397	c0out << endl;
398	return _error->Error(_("Error processing contents %s"),
399	this->Contents.c_str());
400	}
401
402	if (Size != 0)
403	{
404	c0out << " New " << SizeToStr(Size) << "B ";
405	if (Left > Size)
406	Left -= Size;
407	else
408	Left = 0;
409	}
410	else
411	c0out << ' ';
412
413	struct timeval NewTime;
414	gettimeofday(&NewTime,0);
415	double Delta = NewTime.tv_sec - StartTime.tv_sec +
416	(NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
417
418	c0out << Contents.Stats.Packages << " files " <<
419	SizeToStr(Contents.Stats.Bytes) << "B " <<
420	TimeToStr((long)Delta) << endl;
421
422	return true;
423	}
424	/}}}/
425
426	// LoadTree - Load a 'tree' section from the Generate Config /{{{/
427	// ---------------------------------------------------------------------
428	/* This populates the PkgList with all the possible permutations of the
429	section/arch lists. */
430	void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
431	{
432	// Load the defaults
433	string DDir = Setup.Find("TreeDefault::Directory",
434	"$(DIST)/$(SECTION)/binary-$(ARCH)/");
435	string DSDir = Setup.Find("TreeDefault::SrcDirectory",
436	"$(DIST)/$(SECTION)/source/");
437	string DPkg = Setup.Find("TreeDefault::Packages",
438	"$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
439	string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
440	"$(DIST)/$(SECTION)/");
441	string DContents = Setup.Find("TreeDefault::Contents",
442	"$(DIST)/Contents-$(ARCH)");
443	string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
444	string DBCache = Setup.Find("TreeDefault::BinCacheDB",
445	"packages-$(ARCH).db");
446	string DSources = Setup.Find("TreeDefault::Sources",
447	"$(DIST)/$(SECTION)/source/Sources");
448	string DFLFile = Setup.Find("TreeDefault::FileList", "");
449	string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
450
451	// Process 'tree' type sections
452	const Configuration::Item *Top = Setup.Tree("tree");
453	for (Top = (Top == 0?0:Top->Child); Top != 0;)
454	{
455	Configuration Block(Top);
456	string Dist = Top->Tag;
457
458	// Parse the sections
459	string Tmp = Block.Find("Sections");
460	const char *Sections = Tmp.c_str();
461	string Section;
462	while (ParseQuoteWord(Sections,Section) == true)
463	{
464	string Tmp2 = Block.Find("Architectures");
465	string Arch;
466	const char *Archs = Tmp2.c_str();
467	while (ParseQuoteWord(Archs,Arch) == true)
468	{
469	struct SubstVar Vars[] = {{"$(DIST)",&Dist},
470	{"$(SECTION)",&Section},
471	{"$(ARCH)",&Arch},
472	{}};
473	PackageMap Itm;
474
475	Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
476	Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
477
478	if (stringcasecmp(Arch,"source") == 0)
479	{
480	Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
481	Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
482	Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
483	Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
484	Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
485	Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
486	}
487	else
488	{
489	Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
490	Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
491	Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
492	Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
493	Itm.Arch = Arch;
494	Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
495	Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
496	Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
497	Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
498	}
499
500	Itm.GetGeneral(Setup,Block);
501	PkgList.push_back(Itm);
502	}
503	}
504
505	Top = Top->Next;
506	}
507	}
508	/}}}/
509	// LoadBinDir - Load a 'bindirectory' section from the Generate Config /{{{/
510	// ---------------------------------------------------------------------
511	/* */
512	void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
513	{
514	// Process 'bindirectory' type sections
515	const Configuration::Item *Top = Setup.Tree("bindirectory");
516	for (Top = (Top == 0?0:Top->Child); Top != 0;)
517	{
518	Configuration Block(Top);
519
520	PackageMap Itm;
521	Itm.PkgFile = Block.Find("Packages");
522	Itm.SrcFile = Block.Find("Sources");
523	Itm.BinCacheDB = Block.Find("BinCacheDB");
524	Itm.BinOverride = Block.Find("BinOverride");
525	Itm.ExtraOverride = Block.Find("ExtraOverride");
526	Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
527	Itm.SrcOverride = Block.Find("SrcOverride");
528	Itm.BaseDir = Top->Tag;
529	Itm.FLFile = Block.Find("FileList");
530	Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
531	Itm.Contents = Block.Find("Contents");
532	Itm.ContentsHead = Block.Find("Contents::Header");
533
534	Itm.GetGeneral(Setup,Block);
535	PkgList.push_back(Itm);
536
537	Top = Top->Next;
538	}
539	}
540	/}}}/
541
542	// ShowHelp - Show the help text /{{{/
543	// ---------------------------------------------------------------------
544	/* */
545	bool ShowHelp(CommandLine &CmdL)
546	{
00ec24d0 JF	547	ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,VERSION,
00ec24d0 JF	548	COMMON_ARCH,__DATE__,__TIME__);
da6ee469 JF	549	if (_config->FindB("version") == true)
	550	return true;
	551
	552	cout <<
	553	_("Usage: apt-ftparchive [options] command\n"
	554	"Commands: packages binarypath [overridefile [pathprefix]]\n"
	555	" sources srcpath [overridefile [pathprefix]]\n"
	556	" contents path\n"
	557	" release path\n"
	558	" generate config [groups]\n"
	559	" clean config\n"
	560	"\n"
	561	"apt-ftparchive generates index files for Debian archives. It supports\n"
	562	"many styles of generation from fully automated to functional replacements\n"
	563	"for dpkg-scanpackages and dpkg-scansources\n"
	564	"\n"
	565	"apt-ftparchive generates Package files from a tree of .debs. The\n"
	566	"Package file contains the contents of all the control fields from\n"
	567	"each package as well as the MD5 hash and filesize. An override file\n"
	568	"is supported to force the value of Priority and Section.\n"
	569	"\n"
	570	"Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
	571	"The --source-override option can be used to specify a src override file\n"
	572	"\n"
	573	"The 'packages' and 'sources' command should be run in the root of the\n"
	574	"tree. BinaryPath should point to the base of the recursive search and \n"
	575	"override file should contain the override flags. Pathprefix is\n"
	576	"appended to the filename fields if present. Example usage from the \n"
	577	"Debian archive:\n"
	578	" apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
	579	" dists/potato/main/binary-i386/Packages\n"
	580	"\n"
	581	"Options:\n"
	582	" -h This help text\n"
	583	" --md5 Control MD5 generation\n"
	584	" -s=? Source override file\n"
	585	" -q Quiet\n"
	586	" -d=? Select the optional caching database\n"
	587	" --no-delink Enable delinking debug mode\n"
	588	" --contents Control contents file generation\n"
	589	" -c=? Read this configuration file\n"
	590	" -o=? Set an arbitrary configuration option") << endl;
	591
	592	return true;
	593	}
	594	/}}}/
	595	// SimpleGenPackages - Generate a Packages file for a directory tree /{{{/
	596	// ---------------------------------------------------------------------
	597	/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
	598	bool SimpleGenPackages(CommandLine &CmdL)
	599	{
	600	if (CmdL.FileSize() < 2)
	601	return ShowHelp(CmdL);
	602
	603	string Override;
	604	if (CmdL.FileSize() >= 3)
	605	Override = CmdL.FileList[2];
	606
	607	// Create a package writer object.
	608	PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
	609	Override, "");
	610	if (_error->PendingError() == true)
	611	return false;
	612
613	if (CmdL.FileSize() >= 4)
614	Packages.PathPrefix = CmdL.FileList[3];
615
616	// Do recursive directory searching
617	if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
618	return false;
619
620	return true;
621	}
622	/}}}/
623	// SimpleGenContents - Generate a Contents listing /{{{/
624	// ---------------------------------------------------------------------
625	/* */
626	bool SimpleGenContents(CommandLine &CmdL)
627	{
628	if (CmdL.FileSize() < 2)
629	return ShowHelp(CmdL);
630
631	// Create a package writer object.
632	ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"));
633	if (_error->PendingError() == true)
634	return false;
635
636	// Do recursive directory searching
637	if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
638	return false;
639
640	Contents.Finish();
641
642	return true;
643	}
644	/}}}/
645	// SimpleGenSources - Generate a Sources file for a directory tree /{{{/
646	// ---------------------------------------------------------------------
647	/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
648	bool SimpleGenSources(CommandLine &CmdL)
649	{
650	if (CmdL.FileSize() < 2)
651	return ShowHelp(CmdL);
652
653	string Override;
654	if (CmdL.FileSize() >= 3)
655	Override = CmdL.FileList[2];
656
657	string SOverride;
658	if (Override.empty() == false)
659	SOverride = Override + ".src";
660
661	SOverride = _config->Find("APT::FTPArchive::SourceOverride",
662	SOverride.c_str());
663
664	// Create a package writer object.
665	SourcesWriter Sources(Override,SOverride);
666	if (_error->PendingError() == true)
667	return false;
668
669	if (CmdL.FileSize() >= 4)
670	Sources.PathPrefix = CmdL.FileList[3];
671
672	// Do recursive directory searching
673	if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
674	return false;
675
676	return true;
677	}
678	/}}}/
679	// SimpleGenRelease - Generate a Release file for a directory tree /{{{/
680	// ---------------------------------------------------------------------
681	bool SimpleGenRelease(CommandLine &CmdL)
682	{
683	if (CmdL.FileSize() < 2)
684	return ShowHelp(CmdL);
685
686	string Dir = CmdL.FileList[1];
687
688	ReleaseWriter Release("");
689	Release.DirStrip = Dir;
690
691	if (_error->PendingError() == true)
692	return false;
693
694	if (Release.RecursiveScan(Dir) == false)
695	return false;
696
697	Release.Finish();
698
699	return true;
700	}
701
702	/}}}/
703	// Generate - Full generate, using a config file /{{{/
704	// ---------------------------------------------------------------------
705	/* */
706	bool Generate(CommandLine &CmdL)
707	{
708	struct CacheDB::Stats SrcStats;
709	if (CmdL.FileSize() < 2)
710	return ShowHelp(CmdL);
711
712	struct timeval StartTime;
713	gettimeofday(&StartTime,0);
714	struct CacheDB::Stats Stats;
715
716	// Read the configuration file.
717	Configuration Setup;
718	if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
719	return false;
720
721	vector<PackageMap> PkgList;
722	LoadTree(PkgList,Setup);
723	LoadBinDir(PkgList,Setup);
724
725	// Sort by cache DB to improve IO locality.
726	stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
727
728	// Generate packages
729	if (CmdL.FileSize() <= 2)
730	{
731	for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
732	if (I->GenPackages(Setup,Stats) == false)
733	_error->DumpErrors();
734	for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
735	if (I->GenSources(Setup,SrcStats) == false)
736	_error->DumpErrors();
737	}
738	else
739	{
740	// Make a choice list out of the package list..
741	RxChoiceList List = new RxChoiceList[2PkgList.size()+1];
742	RxChoiceList *End = List;
743	for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
744	{
745	End->UserData = &(*I);
746	End->Str = I->BaseDir.c_str();
747	End++;
748
749	End->UserData = &(*I);
750	End->Str = I->Tag.c_str();
751	End++;
752	}
753	End->Str = 0;
754
755	// Regex it
756	if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
757	{
758	delete [] List;
759	return _error->Error(_("No selections matched"));
760	}
761	_error->DumpErrors();
762
763	// Do the generation for Packages
764	for (End = List; End->Str != 0; End++)
765	{
766	if (End->Hit == false)
767	continue;
768
769	PackageMap I = (PackageMap )End->UserData;
770	if (I->PkgDone == true)
771	continue;
772	if (I->GenPackages(Setup,Stats) == false)
773	_error->DumpErrors();
774	}
775
776	// Do the generation for Sources
777	for (End = List; End->Str != 0; End++)
778	{
779	if (End->Hit == false)
780	continue;
781
782	PackageMap I = (PackageMap )End->UserData;
783	if (I->SrcDone == true)
784	continue;
785	if (I->GenSources(Setup,SrcStats) == false)
786	_error->DumpErrors();
787	}
788
789	delete [] List;
790	}
791
792	if (_config->FindB("APT::FTPArchive::Contents",true) == false)
793	return true;
794
0e5943eb	795	c1out << "Packages done, Starting contents." << endl;
da6ee469 JF	796
	797	// Sort the contents file list by date
	798	string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
	799	for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
	800	{
	801	struct stat A;
	802	if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
	803	I->CntCompress,A) == false)
	804	time(&I->ContentsMTime);
	805	else
	806	I->ContentsMTime = A.st_mtime;
	807	}
	808	stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
	809
	810	/* Now for Contents.. The process here is to do a make-like dependency
	811	check. Each contents file is verified to be newer than the package files
	812	that describe the debs it indexes. Since the package files contain
	813	hashes of the .debs this means they have not changed either so the
	814	contents must be up to date. */
	815	unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
	816	for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
	817	{
	818	// This record is not relevent
	819	if (I->ContentsDone == true \|\|
	820	I->Contents.empty() == true)
	821	continue;
	822
	823	// Do not do everything if the user specified sections.
	824	if (CmdL.FileSize() > 2 && I->PkgDone == false)
	825	continue;
	826
	827	struct stat A,B;
	828	if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
	829	{
	830	if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
	831	{
	832	_error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
	833	continue;
	834	}
	835
	836	if (A.st_mtime > B.st_mtime)
	837	continue;
	838	}
	839
	840	if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
	841	MaxContentsChange) == false)
	842	_error->DumpErrors();
	843
	844	// Hit the limit?
	845	if (MaxContentsChange == 0)
	846	{
	847	c1out << "Hit contents update byte limit" << endl;
	848	break;
	849	}
	850	}
	851
	852	struct timeval NewTime;
	853	gettimeofday(&NewTime,0);
	854	double Delta = NewTime.tv_sec - StartTime.tv_sec +
	855	(NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
	856	c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
	857	<< " archives. Took " << TimeToStr((long)Delta) << endl;
	858
	859	return true;
860	}
861	/}}}/
862	// Clean - Clean out the databases /{{{/
863	// ---------------------------------------------------------------------
864	/* */
865	bool Clean(CommandLine &CmdL)
866	{
867	if (CmdL.FileSize() != 2)
868	return ShowHelp(CmdL);
869
870	// Read the configuration file.
871	Configuration Setup;
872	if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
873	return false;
874
875	vector<PackageMap> PkgList;
876	LoadTree(PkgList,Setup);
877	LoadBinDir(PkgList,Setup);
878
879	// Sort by cache DB to improve IO locality.
880	stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
881
882	string CacheDir = Setup.FindDir("Dir::CacheDir");
883
884	for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
885	{
886	c0out << I->BinCacheDB << endl;
887	CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
888	if (DB.Clean() == false)
889	_error->DumpErrors();
890
891	string CacheDB = I->BinCacheDB;
892	for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
893	}
894
895	return true;
896	}
897	/}}}/
898
899	int main(int argc, const char *argv[])
900	{
0e5943eb	901	setlocale(LC_ALL, "");
da6ee469 JF	902	CommandLine::Args Args[] = {
	903	{'h',"help","help",0},
	904	{0,"md5","APT::FTPArchive::MD5",0},
	905	{'v',"version","version",0},
	906	{'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
	907	{'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
	908	{'q',"quiet","quiet",CommandLine::IntLevel},
	909	{'q',"silent","quiet",CommandLine::IntLevel},
	910	{0,"delink","APT::FTPArchive::DeLinkAct",0},
	911	{0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
	912	{0,"contents","APT::FTPArchive::Contents",0},
	913	{'c',"config-file",0,CommandLine::ConfigFile},
	914	{'o',"option",0,CommandLine::ArbItem},
	915	{0,0,0,0}};
	916	CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
	917	{"contents",&SimpleGenContents},
	918	{"sources",&SimpleGenSources},
	919	{"release",&SimpleGenRelease},
	920	{"generate",&Generate},
	921	{"clean",&Clean},
	922	{"help",&ShowHelp},
	923	{0,0}};
	924
	925	// Parse the command line and initialize the package library
	926	CommandLine CmdL(Args,_config);
	927	if (CmdL.Parse(argc,argv) == false)
	928	{
	929	_error->DumpErrors();
	930	return 100;
	931	}
	932
	933	// See if the help should be shown
	934	if (_config->FindB("help") == true \|\|
	935	_config->FindB("version") == true \|\|
	936	CmdL.FileSize() == 0)
	937	{
	938	ShowHelp(CmdL);
	939	return 0;
	940	}
	941
	942	// Setup the output streams
	943	c0out.rdbuf(clog.rdbuf());
	944	c1out.rdbuf(clog.rdbuf());
	945	c2out.rdbuf(clog.rdbuf());
	946	Quiet = _config->FindI("quiet",0);
	947	if (Quiet > 0)
	948	c0out.rdbuf(devnull.rdbuf());
	949	if (Quiet > 1)
	950	c1out.rdbuf(devnull.rdbuf());
	951
	952	// Match the operation
	953	CmdL.DispatchArg(Cmds);
	954
	955	if (_error->empty() == false)
	956	{
	957	bool Errors = _error->PendingError();
	958	_error->DumpErrors();
	959	return Errors == true?100:0;
	960	}
	961	return 0;
	962	}