]> git.saurik.com Git - apt.git/blame - ftparchive/apt-ftparchive.cc
projects / apt.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
* remove all the remaining #pragma implementation
[apt.git] / ftparchive / apt-ftparchive.cc
CommitLineData
b2e465d6
AL		 1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
7db98ffc 3// $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
b2e465d6
AL		 4/* ######################################################################
5
6 apt-scanpackages - Efficient work-alike for dpkg-scanpackages
7
8 Let contents be disabled from the conf
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
b2e465d6
AL		 13#include "apt-ftparchive.h"
14
15#include <apt-pkg/error.h>
16#include <apt-pkg/configuration.h>
17#include <apt-pkg/cmndline.h>
18#include <apt-pkg/strutl.h>
19#include <config.h>
20#include <apti18n.h>
21#include <algorithm>
22
23#include <sys/time.h>
24#include <regex.h>
25
26#include "contents.h"
27#include "multicompress.h"
28#include "writer.h"
29 /*}}}*/
30
d935bb17
AL		 31using namespace std;
32ostream c0out(0);
33ostream c1out(0);
34ostream c2out(0);
b2e465d6
AL		 35ofstream devnull("/dev/null");
36unsigned Quiet = 0;
37
38// struct PackageMap - List of all package files in the config file /*{{{*/
39// ---------------------------------------------------------------------
40/* */
41struct PackageMap
42{
43 // General Stuff
44 string BaseDir;
45 string InternalPrefix;
46 string FLFile;
47 string PkgExt;
48 string SrcExt;
49
50 // Stuff for the Package File
51 string PkgFile;
52 string BinCacheDB;
53 string BinOverride;
64177f17 54 string ExtraOverride;
0b41e0e7
MV		 55
56 // We generate for this given arch
57 string Arch;
b2e465d6
AL		 58
59 // Stuff for the Source File
60 string SrcFile;
61 string SrcOverride;
64177f17 62 string SrcExtraOverride;
b2e465d6
AL		 63
64 // Contents
65 string Contents;
66 string ContentsHead;
67
68 // Random things
69 string Tag;
70 string PkgCompress;
71 string CntCompress;
72 string SrcCompress;
73 string PathPrefix;
74 unsigned int DeLinkLimit;
75 mode_t Permissions;
76
77 bool ContentsDone;
78 bool PkgDone;
79 bool SrcDone;
80 time_t ContentsMTime;
81
82 struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
83 {
84 inline bool operator() (const PackageMap &x,const PackageMap &y)
85 {return x.ContentsMTime < y.ContentsMTime;};
86 };
87
88 struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
89 {
90 inline bool operator() (const PackageMap &x,const PackageMap &y)
91 {return x.BinCacheDB < y.BinCacheDB;};
92 };
93
94 void GetGeneral(Configuration &Setup,Configuration &Block);
95 bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
96 bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
97 bool GenContents(Configuration &Setup,
d935bb17
AL		 98 vector<PackageMap>::iterator Begin,
99 vector<PackageMap>::iterator End,
b2e465d6
AL		 100 unsigned long &Left);
101
102 PackageMap() : DeLinkLimit(0), Permissions(1), ContentsDone(false),
103 PkgDone(false), SrcDone(false), ContentsMTime(0) {};
104};
105 /*}}}*/
106
107// PackageMap::GetGeneral - Common per-section definitions /*{{{*/
108// ---------------------------------------------------------------------
109/* */
110void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
111{
112 PathPrefix = Block.Find("PathPrefix");
113
114 if (Block.FindB("External-Links",true) == false)
115 DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
116 else
117 DeLinkLimit = 0;
118
119 PkgCompress = Block.Find("Packages::Compress",
120 Setup.Find("Default::Packages::Compress",". gzip").c_str());
121 CntCompress = Block.Find("Contents::Compress",
122 Setup.Find("Default::Contents::Compress",". gzip").c_str());
123 SrcCompress = Block.Find("Sources::Compress",
124 Setup.Find("Default::Sources::Compress",". gzip").c_str());
125
126 SrcExt = Block.Find("Sources::Extensions",
127 Setup.Find("Default::Sources::Extensions",".dsc").c_str());
128 PkgExt = Block.Find("Packages::Extensions",
129 Setup.Find("Default::Packages::Extensions",".deb").c_str());
130
131 Permissions = Setup.FindI("Default::FileMode",0644);
132
133 if (FLFile.empty() == false)
134 FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
135
136 if (Contents == " ")
137 Contents= string();
138}
139 /*}}}*/
140// PackageMap::GenPackages - Actually generate a Package file /*{{{*/
141// ---------------------------------------------------------------------
142/* This generates the Package File described by this object. */
143bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
144{
145 if (PkgFile.empty() == true)
146 return true;
147
148 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
149 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
150 string CacheDir = Setup.FindDir("Dir::CacheDir");
151
152 struct timeval StartTime;
153 gettimeofday(&StartTime,0);
154
155 PkgDone = true;
156
157 // Create a package writer object.
158 PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
64177f17 159 flCombine(OverrideDir,BinOverride),
0b41e0e7
MV		 160 flCombine(OverrideDir,ExtraOverride),
161 Arch);
b2e465d6 162 if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
dc738e7a 163 return _error->Error(_("Package extension list is too long"));
b2e465d6 164 if (_error->PendingError() == true)
db0db9fe 165 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 166
167 Packages.PathPrefix = PathPrefix;
168 Packages.DirStrip = ArchiveDir;
169 Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
170
171 Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
172 Packages.DeLinkLimit = DeLinkLimit;
64177f17 173
b2e465d6
AL		 174 // Create a compressor object
175 MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
176 PkgCompress,Permissions);
177 Packages.Output = Comp.Input;
178 if (_error->PendingError() == true)
db0db9fe 179 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 180
181 c0out << ' ' << BaseDir << ":" << flush;
182
183 // Do recursive directory searching
184 if (FLFile.empty() == true)
185 {
186 if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
187 return false;
188 }
189 else
190 {
191 if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
192 return false;
193 }
194
195 Packages.Output = 0; // Just in case
196
197 // Finish compressing
198 unsigned long Size;
199 if (Comp.Finalize(Size) == false)
200 {
201 c0out << endl;
db0db9fe 202 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 203 }
204
205 if (Size != 0)
206 c0out << " New "
207 << SizeToStr(Size) << "B ";
208 else
209 c0out << ' ';
210
211 struct timeval NewTime;
212 gettimeofday(&NewTime,0);
213 double Delta = NewTime.tv_sec - StartTime.tv_sec +
214 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
215
216 c0out << Packages.Stats.Packages << " files " <<
217/* SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
218 SizeToStr(Packages.Stats.Bytes) << "B " <<
219 TimeToStr((long)Delta) << endl;
220
221 Stats.Add(Packages.Stats);
222 Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
223
224 return !_error->PendingError();
225}
98953965 226
b2e465d6 227 /*}}}*/
64177f17 228// PackageMap::GenSources - Actually generate a Source file /*{{{*/
b2e465d6
AL		 229// ---------------------------------------------------------------------
230/* This generates the Sources File described by this object. */
231bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
232{
233 if (SrcFile.empty() == true)
234 return true;
235
236 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
237 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
238 string CacheDir = Setup.FindDir("Dir::CacheDir");
239
240 struct timeval StartTime;
241 gettimeofday(&StartTime,0);
242
243 SrcDone = true;
244
245 // Create a package writer object.
246 SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
64177f17
AL		 247 flCombine(OverrideDir,SrcOverride),
248 flCombine(OverrideDir,SrcExtraOverride));
b2e465d6 249 if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
dc738e7a 250 return _error->Error(_("Source extension list is too long"));
b2e465d6 251 if (_error->PendingError() == true)
db0db9fe 252 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 253
254 Sources.PathPrefix = PathPrefix;
255 Sources.DirStrip = ArchiveDir;
256 Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
257
258 Sources.DeLinkLimit = DeLinkLimit;
259 Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
260
261 // Create a compressor object
262 MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
263 SrcCompress,Permissions);
264 Sources.Output = Comp.Input;
265 if (_error->PendingError() == true)
db0db9fe 266 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 267
268 c0out << ' ' << BaseDir << ":" << flush;
269
270 // Do recursive directory searching
271 if (FLFile.empty() == true)
272 {
273 if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
274 return false;
275 }
276 else
277 {
278 if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
279 return false;
280 }
281 Sources.Output = 0; // Just in case
282
283 // Finish compressing
284 unsigned long Size;
285 if (Comp.Finalize(Size) == false)
286 {
287 c0out << endl;
db0db9fe 288 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 289 }
290
291 if (Size != 0)
292 c0out << " New "
293 << SizeToStr(Size) << "B ";
294 else
295 c0out << ' ';
296
297 struct timeval NewTime;
298 gettimeofday(&NewTime,0);
299 double Delta = NewTime.tv_sec - StartTime.tv_sec +
300 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
301
302 c0out << Sources.Stats.Packages << " pkgs in " <<
303 TimeToStr((long)Delta) << endl;
304
305 Stats.Add(Sources.Stats);
306 Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
307
308 return !_error->PendingError();
309}
310 /*}}}*/
311// PackageMap::GenContents - Actually generate a Contents file /*{{{*/
312// ---------------------------------------------------------------------
313/* This generates the contents file partially described by this object.
314 It searches the given iterator range for other package files that map
315 into this contents file and includes their data as well when building. */
316bool PackageMap::GenContents(Configuration &Setup,
d935bb17
AL		 317 vector<PackageMap>::iterator Begin,
318 vector<PackageMap>::iterator End,
319 unsigned long &Left)
b2e465d6
AL		 320{
321 if (Contents.empty() == true)
322 return true;
323
324 if (Left == 0)
325 return true;
326
327 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
328 string CacheDir = Setup.FindDir("Dir::CacheDir");
329 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
330
331 struct timeval StartTime;
332 gettimeofday(&StartTime,0);
333
334 // Create a package writer object.
335 ContentsWriter Contents("");
336 if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
dc738e7a 337 return _error->Error(_("Package extension list is too long"));
b2e465d6
AL		 338 if (_error->PendingError() == true)
339 return false;
340
341 MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
342 CntCompress,Permissions);
343 Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
344 Contents.Output = Comp.Input;
345 if (_error->PendingError() == true)
346 return false;
347
348 // Write the header out.
349 if (ContentsHead.empty() == false)
350 {
351 FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
352 if (_error->PendingError() == true)
353 return false;
354
355 unsigned long Size = Head.Size();
356 unsigned char Buf[4096];
357 while (Size != 0)
358 {
359 unsigned long ToRead = Size;
360 if (Size > sizeof(Buf))
361 ToRead = sizeof(Buf);
362
363 if (Head.Read(Buf,ToRead) == false)
364 return false;
365
366 if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
dc738e7a 367 return _error->Errno("fwrite",_("Error writing header to contents file"));
b2e465d6
AL		 368
369 Size -= ToRead;
370 }
371 }
372
373 /* Go over all the package file records and parse all the package
374 files associated with this contents file into one great big honking
375 memory structure, then dump the sorted version */
376 c0out << ' ' << this->Contents << ":" << flush;
d935bb17 377 for (vector<PackageMap>::iterator I = Begin; I != End; I++)
b2e465d6
AL		 378 {
379 if (I->Contents != this->Contents)
380 continue;
381
382 Contents.Prefix = ArchiveDir;
383 Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
384 Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
385 I->PkgCompress);
386
387 I->ContentsDone = true;
388 }
389
390 Contents.Finish();
391
392 // Finish compressing
393 unsigned long Size;
394 if (Comp.Finalize(Size) == false || _error->PendingError() == true)
395 {
396 c0out << endl;
db0db9fe 397 return _error->Error(_("Error processing contents %s"),
b2e465d6
AL		 398 this->Contents.c_str());
399 }
400
401 if (Size != 0)
402 {
403 c0out << " New " << SizeToStr(Size) << "B ";
404 if (Left > Size)
405 Left -= Size;
406 else
407 Left = 0;
408 }
409 else
410 c0out << ' ';
411
412 struct timeval NewTime;
413 gettimeofday(&NewTime,0);
414 double Delta = NewTime.tv_sec - StartTime.tv_sec +
415 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
416
417 c0out << Contents.Stats.Packages << " files " <<
418 SizeToStr(Contents.Stats.Bytes) << "B " <<
419 TimeToStr((long)Delta) << endl;
420
421 return true;
422}
423 /*}}}*/
424
425// LoadTree - Load a 'tree' section from the Generate Config /*{{{*/
426// ---------------------------------------------------------------------
427/* This populates the PkgList with all the possible permutations of the
428 section/arch lists. */
429void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
430{
431 // Load the defaults
432 string DDir = Setup.Find("TreeDefault::Directory",
433 "$(DIST)/$(SECTION)/binary-$(ARCH)/");
434 string DSDir = Setup.Find("TreeDefault::SrcDirectory",
435 "$(DIST)/$(SECTION)/source/");
436 string DPkg = Setup.Find("TreeDefault::Packages",
437 "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
438 string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
439 "$(DIST)/$(SECTION)/");
440 string DContents = Setup.Find("TreeDefault::Contents",
441 "$(DIST)/Contents-$(ARCH)");
442 string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
443 string DBCache = Setup.Find("TreeDefault::BinCacheDB",
444 "packages-$(ARCH).db");
445 string DSources = Setup.Find("TreeDefault::Sources",
446 "$(DIST)/$(SECTION)/source/Sources");
447 string DFLFile = Setup.Find("TreeDefault::FileList", "");
448 string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
449
450 // Process 'tree' type sections
451 const Configuration::Item *Top = Setup.Tree("tree");
452 for (Top = (Top == 0?0:Top->Child); Top != 0;)
453 {
454 Configuration Block(Top);
455 string Dist = Top->Tag;
456
457 // Parse the sections
d935bb17
AL		 458 string Tmp = Block.Find("Sections");
459 const char *Sections = Tmp.c_str();
b2e465d6
AL		 460 string Section;
461 while (ParseQuoteWord(Sections,Section) == true)
462 {
d935bb17 463 string Tmp2 = Block.Find("Architectures");
b2e465d6 464 string Arch;
d935bb17 465 const char *Archs = Tmp2.c_str();
b2e465d6
AL		 466 while (ParseQuoteWord(Archs,Arch) == true)
467 {
468 struct SubstVar Vars[] = {{"$(DIST)",&Dist},
469 {"$(SECTION)",&Section},
470 {"$(ARCH)",&Arch},
471 {}};
472 PackageMap Itm;
473
474 Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
475 Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
476
477 if (stringcasecmp(Arch,"source") == 0)
478 {
479 Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
480 Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
481 Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
482 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
483 Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
64177f17 484 Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
b2e465d6
AL		 485 }
486 else
487 {
488 Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
489 Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
490 Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
491 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
0b41e0e7 492 Itm.Arch = Arch;
b2e465d6
AL		 493 Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
494 Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
495 Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
64177f17 496 Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
b2e465d6
AL		 497 }
498
499 Itm.GetGeneral(Setup,Block);
500 PkgList.push_back(Itm);
501 }
502 }
503
504 Top = Top->Next;
505 }
506}
507 /*}}}*/
508// LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/
509// ---------------------------------------------------------------------
510/* */
511void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
512{
513 // Process 'bindirectory' type sections
514 const Configuration::Item *Top = Setup.Tree("bindirectory");
515 for (Top = (Top == 0?0:Top->Child); Top != 0;)
516 {
517 Configuration Block(Top);
518
519 PackageMap Itm;
520 Itm.PkgFile = Block.Find("Packages");
521 Itm.SrcFile = Block.Find("Sources");
522 Itm.BinCacheDB = Block.Find("BinCacheDB");
523 Itm.BinOverride = Block.Find("BinOverride");
64177f17
AL		 524 Itm.ExtraOverride = Block.Find("ExtraOverride");
525 Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
b2e465d6
AL		 526 Itm.SrcOverride = Block.Find("SrcOverride");
527 Itm.BaseDir = Top->Tag;
528 Itm.FLFile = Block.Find("FileList");
529 Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
530 Itm.Contents = Block.Find("Contents");
531 Itm.ContentsHead = Block.Find("Contents::Header");
532
533 Itm.GetGeneral(Setup,Block);
534 PkgList.push_back(Itm);
535
536 Top = Top->Next;
537 }
538}
539 /*}}}*/
540
541// ShowHelp - Show the help text /*{{{*/
542// ---------------------------------------------------------------------
543/* */
544bool ShowHelp(CommandLine &CmdL)
545{
546 ioprintf(cout,_("%s %s for %s %s compiled on %s %s\n"),PACKAGE,VERSION,
547 COMMON_OS,COMMON_CPU,__DATE__,__TIME__);
548 if (_config->FindB("version") == true)
549 return true;
550
551 cout <<
dc738e7a 552 _("Usage: apt-ftparchive [options] command\n"
7a3e9e0a 553 "Commands: packages binarypath [overridefile [pathprefix]]\n"
b2e465d6
AL		 554 " sources srcpath [overridefile [pathprefix]]\n"
555 " contents path\n"
0dfd2728 556 " release path\n"
b2e465d6
AL		 557 " generate config [groups]\n"
558 " clean config\n"
559 "\n"
560 "apt-ftparchive generates index files for Debian archives. It supports\n"
561 "many styles of generation from fully automated to functional replacements\n"
562 "for dpkg-scanpackages and dpkg-scansources\n"
563 "\n"
564 "apt-ftparchive generates Package files from a tree of .debs. The\n"
565 "Package file contains the contents of all the control fields from\n"
566 "each package as well as the MD5 hash and filesize. An override file\n"
567 "is supported to force the value of Priority and Section.\n"
568 "\n"
569 "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
570 "The --source-override option can be used to specify a src override file\n"
571 "\n"
572 "The 'packages' and 'sources' command should be run in the root of the\n"
573 "tree. BinaryPath should point to the base of the recursive search and \n"
75224826 574 "override file should contain the override flags. Pathprefix is\n"
b2e465d6 575 "appended to the filename fields if present. Example usage from the \n"
75224826 576 "Debian archive:\n"
b2e465d6
AL		 577 " apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
578 " dists/potato/main/binary-i386/Packages\n"
579 "\n"
580 "Options:\n"
581 " -h This help text\n"
582 " --md5 Control MD5 generation\n"
583 " -s=? Source override file\n"
584 " -q Quiet\n"
585 " -d=? Select the optional caching database\n"
586 " --no-delink Enable delinking debug mode\n"
587 " --contents Control contents file generation\n"
588 " -c=? Read this configuration file\n"
8a2d7db6 589 " -o=? Set an arbitrary configuration option") << endl;
b2e465d6
AL		 590
591 return true;
592}
593 /*}}}*/
594// SimpleGenPackages - Generate a Packages file for a directory tree /*{{{*/
595// ---------------------------------------------------------------------
596/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
597bool SimpleGenPackages(CommandLine &CmdL)
598{
599 if (CmdL.FileSize() < 2)
600 return ShowHelp(CmdL);
601
602 string Override;
603 if (CmdL.FileSize() >= 3)
604 Override = CmdL.FileList[2];
605
606 // Create a package writer object.
607 PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
64177f17 608 Override, "");
b2e465d6
AL		 609 if (_error->PendingError() == true)
610 return false;
611
612 if (CmdL.FileSize() >= 4)
613 Packages.PathPrefix = CmdL.FileList[3];
614
615 // Do recursive directory searching
616 if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
617 return false;
618
619 return true;
620}
621 /*}}}*/
622// SimpleGenContents - Generate a Contents listing /*{{{*/
623// ---------------------------------------------------------------------
624/* */
625bool SimpleGenContents(CommandLine &CmdL)
626{
627 if (CmdL.FileSize() < 2)
628 return ShowHelp(CmdL);
629
630 // Create a package writer object.
631 ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"));
632 if (_error->PendingError() == true)
633 return false;
634
635 // Do recursive directory searching
636 if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
637 return false;
638
639 Contents.Finish();
640
641 return true;
642}
643 /*}}}*/
644// SimpleGenSources - Generate a Sources file for a directory tree /*{{{*/
645// ---------------------------------------------------------------------
646/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
647bool SimpleGenSources(CommandLine &CmdL)
648{
649 if (CmdL.FileSize() < 2)
650 return ShowHelp(CmdL);
651
652 string Override;
653 if (CmdL.FileSize() >= 3)
654 Override = CmdL.FileList[2];
655
656 string SOverride;
657 if (Override.empty() == false)
658 SOverride = Override + ".src";
659
660 SOverride = _config->Find("APT::FTPArchive::SourceOverride",
661 SOverride.c_str());
662
663 // Create a package writer object.
664 SourcesWriter Sources(Override,SOverride);
665 if (_error->PendingError() == true)
666 return false;
667
668 if (CmdL.FileSize() >= 4)
669 Sources.PathPrefix = CmdL.FileList[3];
670
671 // Do recursive directory searching
672 if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
673 return false;
674
675 return true;
676}
98953965
AL		 677 /*}}}*/
678// SimpleGenRelease - Generate a Release file for a directory tree /*{{{*/
679// ---------------------------------------------------------------------
680bool SimpleGenRelease(CommandLine &CmdL)
681{
0dfd2728
AL		 682 if (CmdL.FileSize() < 2)
683 return ShowHelp(CmdL);
684
c0eb6bc6
AL		 685 string Dir = CmdL.FileList[1];
686
98953965 687 ReleaseWriter Release("");
c0eb6bc6
AL		 688 Release.DirStrip = Dir;
689
98953965
AL		 690 if (_error->PendingError() == true)
691 return false;
692
c0eb6bc6 693 if (Release.RecursiveScan(Dir) == false)
98953965
AL		 694 return false;
695
f7291f62
AL		 696 Release.Finish();
697
98953965
AL		 698 return true;
699}
700
b2e465d6
AL		 701 /*}}}*/
702// Generate - Full generate, using a config file /*{{{*/
703// ---------------------------------------------------------------------
704/* */
705bool Generate(CommandLine &CmdL)
706{
707 struct CacheDB::Stats SrcStats;
708 if (CmdL.FileSize() < 2)
709 return ShowHelp(CmdL);
710
711 struct timeval StartTime;
712 gettimeofday(&StartTime,0);
713 struct CacheDB::Stats Stats;
714
715 // Read the configuration file.
716 Configuration Setup;
717 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
718 return false;
719
720 vector<PackageMap> PkgList;
721 LoadTree(PkgList,Setup);
722 LoadBinDir(PkgList,Setup);
723
724 // Sort by cache DB to improve IO locality.
725 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
726
727 // Generate packages
728 if (CmdL.FileSize() <= 2)
729 {
d935bb17 730 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 731 if (I->GenPackages(Setup,Stats) == false)
732 _error->DumpErrors();
d935bb17 733 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 734 if (I->GenSources(Setup,SrcStats) == false)
735 _error->DumpErrors();
736 }
737 else
738 {
739 // Make a choice list out of the package list..
740 RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
741 RxChoiceList *End = List;
d935bb17 742 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6 743 {
d935bb17 744 End->UserData = &(*I);
b2e465d6
AL		 745 End->Str = I->BaseDir.c_str();
746 End++;
747
d935bb17 748 End->UserData = &(*I);
b2e465d6
AL		 749 End->Str = I->Tag.c_str();
750 End++;
751 }
752 End->Str = 0;
753
754 // Regex it
755 if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
756 {
757 delete [] List;
dc738e7a 758 return _error->Error(_("No selections matched"));
b2e465d6
AL		 759 }
760 _error->DumpErrors();
761
762 // Do the generation for Packages
763 for (End = List; End->Str != 0; End++)
764 {
765 if (End->Hit == false)
766 continue;
767
768 PackageMap *I = (PackageMap *)End->UserData;
769 if (I->PkgDone == true)
770 continue;
771 if (I->GenPackages(Setup,Stats) == false)
772 _error->DumpErrors();
773 }
774
775 // Do the generation for Sources
776 for (End = List; End->Str != 0; End++)
777 {
778 if (End->Hit == false)
779 continue;
780
781 PackageMap *I = (PackageMap *)End->UserData;
782 if (I->SrcDone == true)
783 continue;
784 if (I->GenSources(Setup,SrcStats) == false)
785 _error->DumpErrors();
786 }
787
788 delete [] List;
789 }
790
791 if (_config->FindB("APT::FTPArchive::Contents",true) == false)
792 return true;
793
794 c1out << "Done Packages, Starting contents." << endl;
795
796 // Sort the contents file list by date
797 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
d935bb17 798 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 799 {
800 struct stat A;
801 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
802 I->CntCompress,A) == false)
803 time(&I->ContentsMTime);
804 else
805 I->ContentsMTime = A.st_mtime;
806 }
807 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
808
809 /* Now for Contents.. The process here is to do a make-like dependency
810 check. Each contents file is verified to be newer than the package files
811 that describe the debs it indexes. Since the package files contain
812 hashes of the .debs this means they have not changed either so the
813 contents must be up to date. */
814 unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
d935bb17 815 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 816 {
817 // This record is not relevent
818 if (I->ContentsDone == true ||
819 I->Contents.empty() == true)
820 continue;
821
822 // Do not do everything if the user specified sections.
823 if (CmdL.FileSize() > 2 && I->PkgDone == false)
824 continue;
825
826 struct stat A,B;
827 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
828 {
829 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
830 {
dc738e7a 831 _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
b2e465d6
AL		 832 continue;
833 }
834
835 if (A.st_mtime > B.st_mtime)
836 continue;
837 }
838
839 if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
840 MaxContentsChange) == false)
841 _error->DumpErrors();
842
843 // Hit the limit?
844 if (MaxContentsChange == 0)
845 {
846 c1out << "Hit contents update byte limit" << endl;
847 break;
848 }
849 }
850
851 struct timeval NewTime;
852 gettimeofday(&NewTime,0);
853 double Delta = NewTime.tv_sec - StartTime.tv_sec +
854 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
855 c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
856 << " archives. Took " << TimeToStr((long)Delta) << endl;
857
858 return true;
859}
860 /*}}}*/
861// Clean - Clean out the databases /*{{{*/
862// ---------------------------------------------------------------------
863/* */
864bool Clean(CommandLine &CmdL)
865{
866 if (CmdL.FileSize() != 2)
867 return ShowHelp(CmdL);
868
869 // Read the configuration file.
870 Configuration Setup;
871 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
872 return false;
873
874 vector<PackageMap> PkgList;
875 LoadTree(PkgList,Setup);
876 LoadBinDir(PkgList,Setup);
877
878 // Sort by cache DB to improve IO locality.
879 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
880
881 string CacheDir = Setup.FindDir("Dir::CacheDir");
882
d935bb17 883 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
b2e465d6
AL		 884 {
885 c0out << I->BinCacheDB << endl;
886 CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
887 if (DB.Clean() == false)
888 _error->DumpErrors();
889
890 string CacheDB = I->BinCacheDB;
891 for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
892 }
893
894 return true;
895}
896 /*}}}*/
897
898int main(int argc, const char *argv[])
899{
900 CommandLine::Args Args[] = {
901 {'h',"help","help",0},
902 {0,"md5","APT::FTPArchive::MD5",0},
903 {'v',"version","version",0},
904 {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
905 {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
906 {'q',"quiet","quiet",CommandLine::IntLevel},
907 {'q',"silent","quiet",CommandLine::IntLevel},
908 {0,"delink","APT::FTPArchive::DeLinkAct",0},
909 {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
910 {0,"contents","APT::FTPArchive::Contents",0},
911 {'c',"config-file",0,CommandLine::ConfigFile},
912 {'o',"option",0,CommandLine::ArbItem},
913 {0,0,0,0}};
914 CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
915 {"contents",&SimpleGenContents},
916 {"sources",&SimpleGenSources},
98953965 917 {"release",&SimpleGenRelease},
b2e465d6
AL		 918 {"generate",&Generate},
919 {"clean",&Clean},
920 {"help",&ShowHelp},
921 {0,0}};
922
923 // Parse the command line and initialize the package library
924 CommandLine CmdL(Args,_config);
925 if (CmdL.Parse(argc,argv) == false)
926 {
927 _error->DumpErrors();
928 return 100;
929 }
930
931 // See if the help should be shown
932 if (_config->FindB("help") == true ||
933 _config->FindB("version") == true ||
934 CmdL.FileSize() == 0)
935 {
936 ShowHelp(CmdL);
937 return 0;
938 }
939
940 // Setup the output streams
1809f8d2
AL		 941 c0out.rdbuf(clog.rdbuf());
942 c1out.rdbuf(clog.rdbuf());
943 c2out.rdbuf(clog.rdbuf());
b2e465d6
AL		 944 Quiet = _config->FindI("quiet",0);
945 if (Quiet > 0)
946 c0out.rdbuf(devnull.rdbuf());
947 if (Quiet > 1)
948 c1out.rdbuf(devnull.rdbuf());
949
950 // Match the operation
951 CmdL.DispatchArg(Cmds);
952
953 if (_error->empty() == false)
954 {
955 bool Errors = _error->PendingError();
956 _error->DumpErrors();
957 return Errors == true?100:0;
958 }
959 return 0;
960}
APT (for Cydia)