]> git.saurik.com Git - apt.git/blame - ftparchive/apt-ftparchive.cc
projects / apt.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
cherry pick test/integration/Packages-hashsum-verification addition from lp:~mvo...
[apt.git] / ftparchive / apt-ftparchive.cc
CommitLineData
b2e465d6
AL		 1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
7db98ffc 3// $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
b2e465d6
AL		 4/* ######################################################################
5
c6474fb6 6 apt-ftparchive - Efficient work-alike for dpkg-scanpackages
b2e465d6
AL		 7
8 Let contents be disabled from the conf
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
b2e465d6
AL		 13#include "apt-ftparchive.h"
14
15#include <apt-pkg/error.h>
16#include <apt-pkg/configuration.h>
17#include <apt-pkg/cmndline.h>
18#include <apt-pkg/strutl.h>
3b1fffc3 19#include <apt-pkg/init.h>
b2e465d6
AL		 20#include <config.h>
21#include <apti18n.h>
22#include <algorithm>
23
152ab79e 24#include <climits>
b2e465d6
AL		 25#include <sys/time.h>
26#include <regex.h>
27
28#include "contents.h"
29#include "multicompress.h"
30#include "writer.h"
31 /*}}}*/
32
d935bb17
AL		 33using namespace std;
34ostream c0out(0);
35ostream c1out(0);
36ostream c2out(0);
b2e465d6
AL		 37ofstream devnull("/dev/null");
38unsigned Quiet = 0;
39
40// struct PackageMap - List of all package files in the config file /*{{{*/
41// ---------------------------------------------------------------------
42/* */
43struct PackageMap
44{
45 // General Stuff
46 string BaseDir;
47 string InternalPrefix;
48 string FLFile;
49 string PkgExt;
50 string SrcExt;
51
52 // Stuff for the Package File
53 string PkgFile;
54 string BinCacheDB;
55 string BinOverride;
64177f17 56 string ExtraOverride;
0b41e0e7
MV		 57
58 // We generate for this given arch
59 string Arch;
b2e465d6
AL		 60
61 // Stuff for the Source File
62 string SrcFile;
63 string SrcOverride;
64177f17 64 string SrcExtraOverride;
b2e465d6 65
66905344 66 // Translation master file
4e794c50 67 bool LongDesc;
66905344
DK		 68 TranslationWriter *TransWriter;
69
b2e465d6
AL		 70 // Contents
71 string Contents;
72 string ContentsHead;
73
74 // Random things
75 string Tag;
76 string PkgCompress;
77 string CntCompress;
78 string SrcCompress;
79 string PathPrefix;
80 unsigned int DeLinkLimit;
81 mode_t Permissions;
82
83 bool ContentsDone;
84 bool PkgDone;
85 bool SrcDone;
86 time_t ContentsMTime;
87
88 struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
89 {
90 inline bool operator() (const PackageMap &x,const PackageMap &y)
91 {return x.ContentsMTime < y.ContentsMTime;};
92 };
93
94 struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
95 {
96 inline bool operator() (const PackageMap &x,const PackageMap &y)
97 {return x.BinCacheDB < y.BinCacheDB;};
98 };
99
100 void GetGeneral(Configuration &Setup,Configuration &Block);
101 bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
102 bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
103 bool GenContents(Configuration &Setup,
d935bb17
AL		 104 vector<PackageMap>::iterator Begin,
105 vector<PackageMap>::iterator End,
b2e465d6
AL		 106 unsigned long &Left);
107
4e794c50 108 PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
66905344
DK		 109 ContentsDone(false), PkgDone(false), SrcDone(false),
110 ContentsMTime(0) {};
b2e465d6
AL		 111};
112 /*}}}*/
113
114// PackageMap::GetGeneral - Common per-section definitions /*{{{*/
115// ---------------------------------------------------------------------
116/* */
117void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
118{
119 PathPrefix = Block.Find("PathPrefix");
120
121 if (Block.FindB("External-Links",true) == false)
122 DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
123 else
124 DeLinkLimit = 0;
125
126 PkgCompress = Block.Find("Packages::Compress",
127 Setup.Find("Default::Packages::Compress",". gzip").c_str());
128 CntCompress = Block.Find("Contents::Compress",
129 Setup.Find("Default::Contents::Compress",". gzip").c_str());
130 SrcCompress = Block.Find("Sources::Compress",
131 Setup.Find("Default::Sources::Compress",". gzip").c_str());
132
133 SrcExt = Block.Find("Sources::Extensions",
134 Setup.Find("Default::Sources::Extensions",".dsc").c_str());
135 PkgExt = Block.Find("Packages::Extensions",
136 Setup.Find("Default::Packages::Extensions",".deb").c_str());
137
2db5b414
DK		 138 Permissions = Setup.FindI("Default::FileMode",0644);
139
b2e465d6
AL		 140 if (FLFile.empty() == false)
141 FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
142
143 if (Contents == " ")
144 Contents= string();
145}
146 /*}}}*/
147// PackageMap::GenPackages - Actually generate a Package file /*{{{*/
148// ---------------------------------------------------------------------
149/* This generates the Package File described by this object. */
150bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
151{
152 if (PkgFile.empty() == true)
153 return true;
154
155 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
156 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
157 string CacheDir = Setup.FindDir("Dir::CacheDir");
158
159 struct timeval StartTime;
160 gettimeofday(&StartTime,0);
161
162 PkgDone = true;
163
164 // Create a package writer object.
165 PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
64177f17 166 flCombine(OverrideDir,BinOverride),
0b41e0e7
MV		 167 flCombine(OverrideDir,ExtraOverride),
168 Arch);
b2e465d6 169 if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
dc738e7a 170 return _error->Error(_("Package extension list is too long"));
b2e465d6 171 if (_error->PendingError() == true)
db0db9fe 172 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 173
174 Packages.PathPrefix = PathPrefix;
175 Packages.DirStrip = ArchiveDir;
176 Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
177
66905344 178 Packages.TransWriter = TransWriter;
4e794c50 179 Packages.LongDescription = LongDesc;
66905344 180
b2e465d6
AL		 181 Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
182 Packages.DeLinkLimit = DeLinkLimit;
64177f17 183
b2e465d6
AL		 184 // Create a compressor object
185 MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
186 PkgCompress,Permissions);
187 Packages.Output = Comp.Input;
188 if (_error->PendingError() == true)
db0db9fe 189 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 190
191 c0out << ' ' << BaseDir << ":" << flush;
192
193 // Do recursive directory searching
194 if (FLFile.empty() == true)
195 {
196 if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
197 return false;
198 }
199 else
200 {
201 if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
202 return false;
203 }
204
205 Packages.Output = 0; // Just in case
206
207 // Finish compressing
208 unsigned long Size;
209 if (Comp.Finalize(Size) == false)
210 {
211 c0out << endl;
db0db9fe 212 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 213 }
214
215 if (Size != 0)
216 c0out << " New "
217 << SizeToStr(Size) << "B ";
218 else
219 c0out << ' ';
220
221 struct timeval NewTime;
222 gettimeofday(&NewTime,0);
223 double Delta = NewTime.tv_sec - StartTime.tv_sec +
224 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
225
226 c0out << Packages.Stats.Packages << " files " <<
227/* SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
228 SizeToStr(Packages.Stats.Bytes) << "B " <<
229 TimeToStr((long)Delta) << endl;
230
231 Stats.Add(Packages.Stats);
232 Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
233
234 return !_error->PendingError();
235}
98953965 236
b2e465d6 237 /*}}}*/
64177f17 238// PackageMap::GenSources - Actually generate a Source file /*{{{*/
b2e465d6
AL		 239// ---------------------------------------------------------------------
240/* This generates the Sources File described by this object. */
241bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
242{
243 if (SrcFile.empty() == true)
244 return true;
245
246 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
247 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
248 string CacheDir = Setup.FindDir("Dir::CacheDir");
249
250 struct timeval StartTime;
251 gettimeofday(&StartTime,0);
252
253 SrcDone = true;
254
255 // Create a package writer object.
256 SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
64177f17
AL		 257 flCombine(OverrideDir,SrcOverride),
258 flCombine(OverrideDir,SrcExtraOverride));
b2e465d6 259 if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
dc738e7a 260 return _error->Error(_("Source extension list is too long"));
b2e465d6 261 if (_error->PendingError() == true)
db0db9fe 262 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 263
264 Sources.PathPrefix = PathPrefix;
265 Sources.DirStrip = ArchiveDir;
266 Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
267
268 Sources.DeLinkLimit = DeLinkLimit;
269 Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
270
271 // Create a compressor object
272 MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
273 SrcCompress,Permissions);
274 Sources.Output = Comp.Input;
275 if (_error->PendingError() == true)
db0db9fe 276 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 277
278 c0out << ' ' << BaseDir << ":" << flush;
279
280 // Do recursive directory searching
281 if (FLFile.empty() == true)
282 {
283 if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
284 return false;
285 }
286 else
287 {
288 if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
289 return false;
290 }
291 Sources.Output = 0; // Just in case
292
293 // Finish compressing
294 unsigned long Size;
295 if (Comp.Finalize(Size) == false)
296 {
297 c0out << endl;
db0db9fe 298 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 299 }
300
301 if (Size != 0)
302 c0out << " New "
303 << SizeToStr(Size) << "B ";
304 else
305 c0out << ' ';
306
307 struct timeval NewTime;
308 gettimeofday(&NewTime,0);
309 double Delta = NewTime.tv_sec - StartTime.tv_sec +
310 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
311
312 c0out << Sources.Stats.Packages << " pkgs in " <<
313 TimeToStr((long)Delta) << endl;
314
315 Stats.Add(Sources.Stats);
316 Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
317
318 return !_error->PendingError();
319}
320 /*}}}*/
321// PackageMap::GenContents - Actually generate a Contents file /*{{{*/
322// ---------------------------------------------------------------------
323/* This generates the contents file partially described by this object.
324 It searches the given iterator range for other package files that map
325 into this contents file and includes their data as well when building. */
326bool PackageMap::GenContents(Configuration &Setup,
d935bb17
AL		 327 vector<PackageMap>::iterator Begin,
328 vector<PackageMap>::iterator End,
329 unsigned long &Left)
b2e465d6
AL		 330{
331 if (Contents.empty() == true)
332 return true;
333
334 if (Left == 0)
335 return true;
336
337 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
338 string CacheDir = Setup.FindDir("Dir::CacheDir");
339 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
340
341 struct timeval StartTime;
342 gettimeofday(&StartTime,0);
343
344 // Create a package writer object.
31981076 345 ContentsWriter Contents("", Arch);
b2e465d6 346 if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
dc738e7a 347 return _error->Error(_("Package extension list is too long"));
b2e465d6
AL		 348 if (_error->PendingError() == true)
349 return false;
350
351 MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
352 CntCompress,Permissions);
353 Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
354 Contents.Output = Comp.Input;
355 if (_error->PendingError() == true)
356 return false;
357
358 // Write the header out.
359 if (ContentsHead.empty() == false)
360 {
361 FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
362 if (_error->PendingError() == true)
363 return false;
364
365 unsigned long Size = Head.Size();
366 unsigned char Buf[4096];
367 while (Size != 0)
368 {
369 unsigned long ToRead = Size;
370 if (Size > sizeof(Buf))
371 ToRead = sizeof(Buf);
372
373 if (Head.Read(Buf,ToRead) == false)
374 return false;
375
376 if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
dc738e7a 377 return _error->Errno("fwrite",_("Error writing header to contents file"));
b2e465d6
AL		 378
379 Size -= ToRead;
380 }
381 }
382
383 /* Go over all the package file records and parse all the package
384 files associated with this contents file into one great big honking
385 memory structure, then dump the sorted version */
386 c0out << ' ' << this->Contents << ":" << flush;
d935bb17 387 for (vector<PackageMap>::iterator I = Begin; I != End; I++)
b2e465d6
AL		 388 {
389 if (I->Contents != this->Contents)
390 continue;
391
392 Contents.Prefix = ArchiveDir;
393 Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
394 Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
395 I->PkgCompress);
396
397 I->ContentsDone = true;
398 }
399
400 Contents.Finish();
401
402 // Finish compressing
403 unsigned long Size;
404 if (Comp.Finalize(Size) == false || _error->PendingError() == true)
405 {
406 c0out << endl;
db0db9fe 407 return _error->Error(_("Error processing contents %s"),
b2e465d6
AL		 408 this->Contents.c_str());
409 }
410
411 if (Size != 0)
412 {
413 c0out << " New " << SizeToStr(Size) << "B ";
414 if (Left > Size)
415 Left -= Size;
416 else
417 Left = 0;
418 }
419 else
420 c0out << ' ';
421
422 struct timeval NewTime;
423 gettimeofday(&NewTime,0);
424 double Delta = NewTime.tv_sec - StartTime.tv_sec +
425 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
426
427 c0out << Contents.Stats.Packages << " files " <<
428 SizeToStr(Contents.Stats.Bytes) << "B " <<
429 TimeToStr((long)Delta) << endl;
430
431 return true;
432}
433 /*}}}*/
434
435// LoadTree - Load a 'tree' section from the Generate Config /*{{{*/
436// ---------------------------------------------------------------------
437/* This populates the PkgList with all the possible permutations of the
438 section/arch lists. */
439void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
440{
441 // Load the defaults
442 string DDir = Setup.Find("TreeDefault::Directory",
443 "$(DIST)/$(SECTION)/binary-$(ARCH)/");
444 string DSDir = Setup.Find("TreeDefault::SrcDirectory",
445 "$(DIST)/$(SECTION)/source/");
446 string DPkg = Setup.Find("TreeDefault::Packages",
447 "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
66905344
DK		 448 string DTrans = Setup.Find("TreeDefault::Translation",
449 "$(DIST)/$(SECTION)/i18n/Translation-en");
b2e465d6
AL		 450 string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
451 "$(DIST)/$(SECTION)/");
452 string DContents = Setup.Find("TreeDefault::Contents",
453 "$(DIST)/Contents-$(ARCH)");
454 string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
455 string DBCache = Setup.Find("TreeDefault::BinCacheDB",
456 "packages-$(ARCH).db");
457 string DSources = Setup.Find("TreeDefault::Sources",
458 "$(DIST)/$(SECTION)/source/Sources");
459 string DFLFile = Setup.Find("TreeDefault::FileList", "");
460 string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
461
2db5b414 462 mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
34f1d96c
DK		 463
464 bool const LongDescription = Setup.FindB("Default::LongDescription",
4e794c50 465 _config->FindB("APT::FTPArchive::LongDescription", true));
34f1d96c 466 string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
4e794c50 467
b2e465d6
AL		 468 // Process 'tree' type sections
469 const Configuration::Item *Top = Setup.Tree("tree");
470 for (Top = (Top == 0?0:Top->Child); Top != 0;)
471 {
472 Configuration Block(Top);
473 string Dist = Top->Tag;
474
475 // Parse the sections
d935bb17
AL		 476 string Tmp = Block.Find("Sections");
477 const char *Sections = Tmp.c_str();
b2e465d6
AL		 478 string Section;
479 while (ParseQuoteWord(Sections,Section) == true)
480 {
b2e465d6 481 string Arch;
66905344
DK		 482 struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
483 {"$(SECTION)",&Section},
484 {"$(ARCH)",&Arch},
485 {}};
34f1d96c 486 mode_t const Perms = Block.FindI("FileMode", Permissions);
4e794c50 487 bool const LongDesc = Block.FindB("LongDescription", LongDescription);
66905344 488 TranslationWriter *TransWriter;
4e794c50 489 if (DTrans.empty() == false && LongDesc == false)
66905344
DK		 490 {
491 string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
492 SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
34f1d96c
DK		 493 string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
494 TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
66905344
DK		 495 }
496 else
497 TransWriter = NULL;
498
499 string const Tmp2 = Block.Find("Architectures");
d935bb17 500 const char *Archs = Tmp2.c_str();
b2e465d6
AL		 501 while (ParseQuoteWord(Archs,Arch) == true)
502 {
b2e465d6 503 PackageMap Itm;
34f1d96c 504 Itm.Permissions = Perms;
b2e465d6
AL		 505 Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
506 Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
507
508 if (stringcasecmp(Arch,"source") == 0)
509 {
510 Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
511 Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
512 Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
513 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
514 Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
64177f17 515 Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
b2e465d6
AL		 516 }
517 else
518 {
519 Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
520 Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
521 Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
522 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
0b41e0e7 523 Itm.Arch = Arch;
4e794c50 524 Itm.LongDesc = LongDesc;
66905344
DK		 525 if (TransWriter != NULL)
526 {
527 TransWriter->IncreaseRefCounter();
528 Itm.TransWriter = TransWriter;
529 }
b2e465d6
AL		 530 Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
531 Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
532 Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
64177f17 533 Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
b2e465d6
AL		 534 }
535
536 Itm.GetGeneral(Setup,Block);
537 PkgList.push_back(Itm);
538 }
66905344
DK		 539 // we didn't use this TransWriter, so we can release it
540 if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
541 delete TransWriter;
b2e465d6
AL		 542 }
543
544 Top = Top->Next;
545 }
546}
547 /*}}}*/
548// LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/
549// ---------------------------------------------------------------------
550/* */
551void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
552{
2db5b414
DK		 553 mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
554
b2e465d6
AL		 555 // Process 'bindirectory' type sections
556 const Configuration::Item *Top = Setup.Tree("bindirectory");
557 for (Top = (Top == 0?0:Top->Child); Top != 0;)
558 {
559 Configuration Block(Top);
560
561 PackageMap Itm;
562 Itm.PkgFile = Block.Find("Packages");
563 Itm.SrcFile = Block.Find("Sources");
564 Itm.BinCacheDB = Block.Find("BinCacheDB");
565 Itm.BinOverride = Block.Find("BinOverride");
64177f17
AL		 566 Itm.ExtraOverride = Block.Find("ExtraOverride");
567 Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
b2e465d6
AL		 568 Itm.SrcOverride = Block.Find("SrcOverride");
569 Itm.BaseDir = Top->Tag;
570 Itm.FLFile = Block.Find("FileList");
571 Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
572 Itm.Contents = Block.Find("Contents");
573 Itm.ContentsHead = Block.Find("Contents::Header");
2db5b414 574 Itm.Permissions = Block.FindI("FileMode", Permissions);
b2e465d6
AL		 575
576 Itm.GetGeneral(Setup,Block);
577 PkgList.push_back(Itm);
578
579 Top = Top->Next;
580 }
581}
582 /*}}}*/
583
584// ShowHelp - Show the help text /*{{{*/
585// ---------------------------------------------------------------------
586/* */
587bool ShowHelp(CommandLine &CmdL)
588{
5b28c804
OS		 589 ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,VERSION,
590 COMMON_ARCH,__DATE__,__TIME__);
b2e465d6
AL		 591 if (_config->FindB("version") == true)
592 return true;
593
594 cout <<
dc738e7a 595 _("Usage: apt-ftparchive [options] command\n"
7a3e9e0a 596 "Commands: packages binarypath [overridefile [pathprefix]]\n"
b2e465d6
AL		 597 " sources srcpath [overridefile [pathprefix]]\n"
598 " contents path\n"
0dfd2728 599 " release path\n"
b2e465d6
AL		 600 " generate config [groups]\n"
601 " clean config\n"
602 "\n"
603 "apt-ftparchive generates index files for Debian archives. It supports\n"
604 "many styles of generation from fully automated to functional replacements\n"
605 "for dpkg-scanpackages and dpkg-scansources\n"
606 "\n"
607 "apt-ftparchive generates Package files from a tree of .debs. The\n"
608 "Package file contains the contents of all the control fields from\n"
609 "each package as well as the MD5 hash and filesize. An override file\n"
610 "is supported to force the value of Priority and Section.\n"
611 "\n"
612 "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
613 "The --source-override option can be used to specify a src override file\n"
614 "\n"
615 "The 'packages' and 'sources' command should be run in the root of the\n"
616 "tree. BinaryPath should point to the base of the recursive search and \n"
75224826 617 "override file should contain the override flags. Pathprefix is\n"
b2e465d6 618 "appended to the filename fields if present. Example usage from the \n"
75224826 619 "Debian archive:\n"
b2e465d6
AL		 620 " apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
621 " dists/potato/main/binary-i386/Packages\n"
622 "\n"
623 "Options:\n"
624 " -h This help text\n"
625 " --md5 Control MD5 generation\n"
626 " -s=? Source override file\n"
627 " -q Quiet\n"
628 " -d=? Select the optional caching database\n"
629 " --no-delink Enable delinking debug mode\n"
630 " --contents Control contents file generation\n"
631 " -c=? Read this configuration file\n"
8a2d7db6 632 " -o=? Set an arbitrary configuration option") << endl;
b2e465d6
AL		 633
634 return true;
635}
636 /*}}}*/
637// SimpleGenPackages - Generate a Packages file for a directory tree /*{{{*/
638// ---------------------------------------------------------------------
639/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
640bool SimpleGenPackages(CommandLine &CmdL)
641{
642 if (CmdL.FileSize() < 2)
643 return ShowHelp(CmdL);
644
645 string Override;
646 if (CmdL.FileSize() >= 3)
647 Override = CmdL.FileList[2];
648
649 // Create a package writer object.
650 PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
31981076 651 Override, "", _config->Find("APT::FTPArchive::Architecture"));
b2e465d6
AL		 652 if (_error->PendingError() == true)
653 return false;
654
655 if (CmdL.FileSize() >= 4)
656 Packages.PathPrefix = CmdL.FileList[3];
657
658 // Do recursive directory searching
659 if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
660 return false;
661
662 return true;
663}
664 /*}}}*/
665// SimpleGenContents - Generate a Contents listing /*{{{*/
666// ---------------------------------------------------------------------
667/* */
668bool SimpleGenContents(CommandLine &CmdL)
669{
670 if (CmdL.FileSize() < 2)
671 return ShowHelp(CmdL);
672
673 // Create a package writer object.
31981076 674 ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
b2e465d6
AL		 675 if (_error->PendingError() == true)
676 return false;
677
678 // Do recursive directory searching
679 if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
680 return false;
681
682 Contents.Finish();
683
684 return true;
685}
686 /*}}}*/
687// SimpleGenSources - Generate a Sources file for a directory tree /*{{{*/
688// ---------------------------------------------------------------------
689/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
690bool SimpleGenSources(CommandLine &CmdL)
691{
692 if (CmdL.FileSize() < 2)
693 return ShowHelp(CmdL);
694
695 string Override;
696 if (CmdL.FileSize() >= 3)
697 Override = CmdL.FileList[2];
698
699 string SOverride;
700 if (Override.empty() == false)
701 SOverride = Override + ".src";
702
703 SOverride = _config->Find("APT::FTPArchive::SourceOverride",
704 SOverride.c_str());
705
706 // Create a package writer object.
707 SourcesWriter Sources(Override,SOverride);
708 if (_error->PendingError() == true)
709 return false;
710
711 if (CmdL.FileSize() >= 4)
712 Sources.PathPrefix = CmdL.FileList[3];
713
714 // Do recursive directory searching
715 if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
716 return false;
717
718 return true;
719}
98953965
AL		 720 /*}}}*/
721// SimpleGenRelease - Generate a Release file for a directory tree /*{{{*/
722// ---------------------------------------------------------------------
723bool SimpleGenRelease(CommandLine &CmdL)
724{
0dfd2728
AL		 725 if (CmdL.FileSize() < 2)
726 return ShowHelp(CmdL);
727
c0eb6bc6
AL		 728 string Dir = CmdL.FileList[1];
729
98953965 730 ReleaseWriter Release("");
c0eb6bc6
AL		 731 Release.DirStrip = Dir;
732
98953965
AL		 733 if (_error->PendingError() == true)
734 return false;
735
c0eb6bc6 736 if (Release.RecursiveScan(Dir) == false)
98953965
AL		 737 return false;
738
f7291f62
AL		 739 Release.Finish();
740
98953965
AL		 741 return true;
742}
743
b2e465d6
AL		 744 /*}}}*/
745// Generate - Full generate, using a config file /*{{{*/
746// ---------------------------------------------------------------------
747/* */
748bool Generate(CommandLine &CmdL)
749{
750 struct CacheDB::Stats SrcStats;
751 if (CmdL.FileSize() < 2)
752 return ShowHelp(CmdL);
753
754 struct timeval StartTime;
755 gettimeofday(&StartTime,0);
756 struct CacheDB::Stats Stats;
757
758 // Read the configuration file.
759 Configuration Setup;
760 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
761 return false;
762
763 vector<PackageMap> PkgList;
764 LoadTree(PkgList,Setup);
765 LoadBinDir(PkgList,Setup);
766
767 // Sort by cache DB to improve IO locality.
768 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
769
770 // Generate packages
771 if (CmdL.FileSize() <= 2)
772 {
d935bb17 773 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 774 if (I->GenPackages(Setup,Stats) == false)
775 _error->DumpErrors();
d935bb17 776 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 777 if (I->GenSources(Setup,SrcStats) == false)
778 _error->DumpErrors();
779 }
780 else
781 {
782 // Make a choice list out of the package list..
783 RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
784 RxChoiceList *End = List;
d935bb17 785 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6 786 {
d935bb17 787 End->UserData = &(*I);
b2e465d6
AL		 788 End->Str = I->BaseDir.c_str();
789 End++;
790
d935bb17 791 End->UserData = &(*I);
b2e465d6
AL		 792 End->Str = I->Tag.c_str();
793 End++;
794 }
795 End->Str = 0;
796
797 // Regex it
798 if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
799 {
800 delete [] List;
dc738e7a 801 return _error->Error(_("No selections matched"));
b2e465d6
AL		 802 }
803 _error->DumpErrors();
804
805 // Do the generation for Packages
806 for (End = List; End->Str != 0; End++)
807 {
808 if (End->Hit == false)
809 continue;
810
811 PackageMap *I = (PackageMap *)End->UserData;
812 if (I->PkgDone == true)
813 continue;
814 if (I->GenPackages(Setup,Stats) == false)
815 _error->DumpErrors();
816 }
817
818 // Do the generation for Sources
819 for (End = List; End->Str != 0; End++)
820 {
821 if (End->Hit == false)
822 continue;
823
824 PackageMap *I = (PackageMap *)End->UserData;
825 if (I->SrcDone == true)
826 continue;
827 if (I->GenSources(Setup,SrcStats) == false)
828 _error->DumpErrors();
829 }
830
831 delete [] List;
832 }
66905344
DK		 833
834 // close the Translation master files
f92800f8 835 for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); I++)
66905344
DK		 836 if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
837 delete I->TransWriter;
838
b2e465d6
AL		 839 if (_config->FindB("APT::FTPArchive::Contents",true) == false)
840 return true;
841
c6474fb6 842 c1out << "Packages done, Starting contents." << endl;
b2e465d6
AL		 843
844 // Sort the contents file list by date
845 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
d935bb17 846 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 847 {
848 struct stat A;
849 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
850 I->CntCompress,A) == false)
851 time(&I->ContentsMTime);
852 else
853 I->ContentsMTime = A.st_mtime;
854 }
855 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
856
857 /* Now for Contents.. The process here is to do a make-like dependency
858 check. Each contents file is verified to be newer than the package files
859 that describe the debs it indexes. Since the package files contain
860 hashes of the .debs this means they have not changed either so the
861 contents must be up to date. */
862 unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
d935bb17 863 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 864 {
865 // This record is not relevent
866 if (I->ContentsDone == true ||
867 I->Contents.empty() == true)
868 continue;
869
870 // Do not do everything if the user specified sections.
871 if (CmdL.FileSize() > 2 && I->PkgDone == false)
872 continue;
873
874 struct stat A,B;
875 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
876 {
877 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
878 {
dc738e7a 879 _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
b2e465d6
AL		 880 continue;
881 }
882
883 if (A.st_mtime > B.st_mtime)
884 continue;
885 }
886
887 if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
888 MaxContentsChange) == false)
889 _error->DumpErrors();
890
891 // Hit the limit?
892 if (MaxContentsChange == 0)
893 {
894 c1out << "Hit contents update byte limit" << endl;
895 break;
896 }
897 }
898
899 struct timeval NewTime;
900 gettimeofday(&NewTime,0);
901 double Delta = NewTime.tv_sec - StartTime.tv_sec +
902 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
903 c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
904 << " archives. Took " << TimeToStr((long)Delta) << endl;
905
906 return true;
907}
908 /*}}}*/
909// Clean - Clean out the databases /*{{{*/
910// ---------------------------------------------------------------------
911/* */
912bool Clean(CommandLine &CmdL)
913{
914 if (CmdL.FileSize() != 2)
915 return ShowHelp(CmdL);
916
917 // Read the configuration file.
918 Configuration Setup;
919 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
920 return false;
921
922 vector<PackageMap> PkgList;
923 LoadTree(PkgList,Setup);
924 LoadBinDir(PkgList,Setup);
925
926 // Sort by cache DB to improve IO locality.
927 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
928
929 string CacheDir = Setup.FindDir("Dir::CacheDir");
930
d935bb17 931 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
b2e465d6
AL		 932 {
933 c0out << I->BinCacheDB << endl;
934 CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
935 if (DB.Clean() == false)
936 _error->DumpErrors();
937
938 string CacheDB = I->BinCacheDB;
939 for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
940 }
941
942 return true;
943}
944 /*}}}*/
945
946int main(int argc, const char *argv[])
947{
1692a2c7 948 setlocale(LC_ALL, "");
b2e465d6
AL		 949 CommandLine::Args Args[] = {
950 {'h',"help","help",0},
951 {0,"md5","APT::FTPArchive::MD5",0},
3c54407f
DK		 952 {0,"sha1","APT::FTPArchive::SHA1",0},
953 {0,"sha256","APT::FTPArchive::SHA256",0},
b2e465d6
AL		 954 {'v',"version","version",0},
955 {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
956 {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
957 {'q',"quiet","quiet",CommandLine::IntLevel},
958 {'q',"silent","quiet",CommandLine::IntLevel},
959 {0,"delink","APT::FTPArchive::DeLinkAct",0},
960 {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
961 {0,"contents","APT::FTPArchive::Contents",0},
31981076 962 {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
b2e465d6
AL		 963 {'c',"config-file",0,CommandLine::ConfigFile},
964 {'o',"option",0,CommandLine::ArbItem},
965 {0,0,0,0}};
966 CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
967 {"contents",&SimpleGenContents},
968 {"sources",&SimpleGenSources},
98953965 969 {"release",&SimpleGenRelease},
b2e465d6
AL		 970 {"generate",&Generate},
971 {"clean",&Clean},
972 {"help",&ShowHelp},
973 {0,0}};
974
975 // Parse the command line and initialize the package library
976 CommandLine CmdL(Args,_config);
b34d4b47 977 if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false)
b2e465d6
AL		 978 {
979 _error->DumpErrors();
980 return 100;
981 }
982
983 // See if the help should be shown
984 if (_config->FindB("help") == true ||
985 _config->FindB("version") == true ||
986 CmdL.FileSize() == 0)
987 {
988 ShowHelp(CmdL);
989 return 0;
990 }
991
992 // Setup the output streams
1809f8d2
AL		 993 c0out.rdbuf(clog.rdbuf());
994 c1out.rdbuf(clog.rdbuf());
995 c2out.rdbuf(clog.rdbuf());
b2e465d6
AL		 996 Quiet = _config->FindI("quiet",0);
997 if (Quiet > 0)
998 c0out.rdbuf(devnull.rdbuf());
999 if (Quiet > 1)
1000 c1out.rdbuf(devnull.rdbuf());
1001
1002 // Match the operation
1003 CmdL.DispatchArg(Cmds);
1004
1005 if (_error->empty() == false)
1006 {
1007 bool Errors = _error->PendingError();
1008 _error->DumpErrors();
1009 return Errors == true?100:0;
1010 }
1011 return 0;
1012}
APT (for Cydia)