]> git.saurik.com Git - apt.git/blame - ftparchive/apt-ftparchive.cc
projects / apt.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
improve test for #731853 and comments
[apt.git] / ftparchive / apt-ftparchive.cc
CommitLineData
b2e465d6
AL		 1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
7db98ffc 3// $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
b2e465d6
AL		 4/* ######################################################################
5
c6474fb6 6 apt-ftparchive - Efficient work-alike for dpkg-scanpackages
b2e465d6
AL		 7
8 Let contents be disabled from the conf
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
ea542140
DK		 13#include <config.h>
14
b2e465d6
AL		 15#include <apt-pkg/error.h>
16#include <apt-pkg/configuration.h>
17#include <apt-pkg/cmndline.h>
18#include <apt-pkg/strutl.h>
3b1fffc3 19#include <apt-pkg/init.h>
b2e465d6
AL		 20#include <algorithm>
21
152ab79e 22#include <climits>
b2e465d6
AL		 23#include <sys/time.h>
24#include <regex.h>
25
ea542140 26#include "apt-ftparchive.h"
b2e465d6
AL		 27#include "contents.h"
28#include "multicompress.h"
ea542140
DK		 29#include "writer.h"
30
31#include <apti18n.h>
b2e465d6
AL		 32 /*}}}*/
33
d935bb17
AL		 34using namespace std;
35ostream c0out(0);
36ostream c1out(0);
37ostream c2out(0);
b2e465d6
AL		 38ofstream devnull("/dev/null");
39unsigned Quiet = 0;
40
41// struct PackageMap - List of all package files in the config file /*{{{*/
42// ---------------------------------------------------------------------
43/* */
44struct PackageMap
45{
46 // General Stuff
47 string BaseDir;
48 string InternalPrefix;
49 string FLFile;
50 string PkgExt;
51 string SrcExt;
52
53 // Stuff for the Package File
54 string PkgFile;
55 string BinCacheDB;
56 string BinOverride;
64177f17 57 string ExtraOverride;
0b41e0e7
MV		 58
59 // We generate for this given arch
60 string Arch;
b2e465d6
AL		 61
62 // Stuff for the Source File
63 string SrcFile;
64 string SrcOverride;
64177f17 65 string SrcExtraOverride;
b2e465d6 66
66905344 67 // Translation master file
4e794c50 68 bool LongDesc;
66905344
DK		 69 TranslationWriter *TransWriter;
70
b2e465d6
AL		 71 // Contents
72 string Contents;
73 string ContentsHead;
74
75 // Random things
76 string Tag;
77 string PkgCompress;
78 string CntCompress;
79 string SrcCompress;
80 string PathPrefix;
81 unsigned int DeLinkLimit;
82 mode_t Permissions;
83
84 bool ContentsDone;
85 bool PkgDone;
86 bool SrcDone;
87 time_t ContentsMTime;
88
89 struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
90 {
91 inline bool operator() (const PackageMap &x,const PackageMap &y)
92 {return x.ContentsMTime < y.ContentsMTime;};
93 };
94
95 struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
96 {
97 inline bool operator() (const PackageMap &x,const PackageMap &y)
98 {return x.BinCacheDB < y.BinCacheDB;};
99 };
100
101 void GetGeneral(Configuration &Setup,Configuration &Block);
102 bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
103 bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
104 bool GenContents(Configuration &Setup,
d935bb17
AL		 105 vector<PackageMap>::iterator Begin,
106 vector<PackageMap>::iterator End,
b2e465d6
AL		 107 unsigned long &Left);
108
4e794c50 109 PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
66905344
DK		 110 ContentsDone(false), PkgDone(false), SrcDone(false),
111 ContentsMTime(0) {};
b2e465d6
AL		 112};
113 /*}}}*/
114
115// PackageMap::GetGeneral - Common per-section definitions /*{{{*/
116// ---------------------------------------------------------------------
117/* */
118void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
119{
120 PathPrefix = Block.Find("PathPrefix");
121
122 if (Block.FindB("External-Links",true) == false)
123 DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
124 else
125 DeLinkLimit = 0;
126
127 PkgCompress = Block.Find("Packages::Compress",
128 Setup.Find("Default::Packages::Compress",". gzip").c_str());
129 CntCompress = Block.Find("Contents::Compress",
130 Setup.Find("Default::Contents::Compress",". gzip").c_str());
131 SrcCompress = Block.Find("Sources::Compress",
132 Setup.Find("Default::Sources::Compress",". gzip").c_str());
133
134 SrcExt = Block.Find("Sources::Extensions",
135 Setup.Find("Default::Sources::Extensions",".dsc").c_str());
136 PkgExt = Block.Find("Packages::Extensions",
137 Setup.Find("Default::Packages::Extensions",".deb").c_str());
138
2db5b414
DK		 139 Permissions = Setup.FindI("Default::FileMode",0644);
140
b2e465d6
AL		 141 if (FLFile.empty() == false)
142 FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
143
144 if (Contents == " ")
145 Contents= string();
146}
147 /*}}}*/
148// PackageMap::GenPackages - Actually generate a Package file /*{{{*/
149// ---------------------------------------------------------------------
150/* This generates the Package File described by this object. */
151bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
152{
153 if (PkgFile.empty() == true)
154 return true;
155
156 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
157 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
158 string CacheDir = Setup.FindDir("Dir::CacheDir");
159
160 struct timeval StartTime;
161 gettimeofday(&StartTime,0);
162
163 PkgDone = true;
164
165 // Create a package writer object.
166 PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
64177f17 167 flCombine(OverrideDir,BinOverride),
0b41e0e7
MV		 168 flCombine(OverrideDir,ExtraOverride),
169 Arch);
b2e465d6 170 if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
dc738e7a 171 return _error->Error(_("Package extension list is too long"));
b2e465d6 172 if (_error->PendingError() == true)
db0db9fe 173 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 174
175 Packages.PathPrefix = PathPrefix;
176 Packages.DirStrip = ArchiveDir;
177 Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
178
66905344 179 Packages.TransWriter = TransWriter;
4e794c50 180 Packages.LongDescription = LongDesc;
66905344 181
b2e465d6
AL		 182 Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
183 Packages.DeLinkLimit = DeLinkLimit;
64177f17 184
b2e465d6
AL		 185 // Create a compressor object
186 MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
187 PkgCompress,Permissions);
188 Packages.Output = Comp.Input;
189 if (_error->PendingError() == true)
db0db9fe 190 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 191
192 c0out << ' ' << BaseDir << ":" << flush;
193
194 // Do recursive directory searching
195 if (FLFile.empty() == true)
196 {
197 if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
198 return false;
199 }
200 else
201 {
202 if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
203 return false;
204 }
205
206 Packages.Output = 0; // Just in case
207
208 // Finish compressing
650faab0 209 unsigned long long Size;
b2e465d6
AL		 210 if (Comp.Finalize(Size) == false)
211 {
212 c0out << endl;
db0db9fe 213 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 214 }
215
216 if (Size != 0)
217 c0out << " New "
218 << SizeToStr(Size) << "B ";
219 else
220 c0out << ' ';
221
222 struct timeval NewTime;
223 gettimeofday(&NewTime,0);
224 double Delta = NewTime.tv_sec - StartTime.tv_sec +
225 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
226
227 c0out << Packages.Stats.Packages << " files " <<
228/* SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
229 SizeToStr(Packages.Stats.Bytes) << "B " <<
230 TimeToStr((long)Delta) << endl;
231
232 Stats.Add(Packages.Stats);
233 Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
234
235 return !_error->PendingError();
236}
98953965 237
b2e465d6 238 /*}}}*/
64177f17 239// PackageMap::GenSources - Actually generate a Source file /*{{{*/
b2e465d6
AL		 240// ---------------------------------------------------------------------
241/* This generates the Sources File described by this object. */
242bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
243{
244 if (SrcFile.empty() == true)
245 return true;
246
247 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
248 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
249 string CacheDir = Setup.FindDir("Dir::CacheDir");
250
251 struct timeval StartTime;
252 gettimeofday(&StartTime,0);
253
254 SrcDone = true;
255
256 // Create a package writer object.
f6f06a8f
MV		 257 SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),
258 flCombine(OverrideDir,BinOverride),
64177f17
AL		 259 flCombine(OverrideDir,SrcOverride),
260 flCombine(OverrideDir,SrcExtraOverride));
b2e465d6 261 if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
dc738e7a 262 return _error->Error(_("Source extension list is too long"));
b2e465d6 263 if (_error->PendingError() == true)
db0db9fe 264 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 265
266 Sources.PathPrefix = PathPrefix;
267 Sources.DirStrip = ArchiveDir;
268 Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
269
270 Sources.DeLinkLimit = DeLinkLimit;
271 Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
272
273 // Create a compressor object
274 MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
275 SrcCompress,Permissions);
276 Sources.Output = Comp.Input;
277 if (_error->PendingError() == true)
db0db9fe 278 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 279
280 c0out << ' ' << BaseDir << ":" << flush;
281
282 // Do recursive directory searching
283 if (FLFile.empty() == true)
284 {
285 if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
286 return false;
287 }
288 else
289 {
290 if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
291 return false;
292 }
293 Sources.Output = 0; // Just in case
294
295 // Finish compressing
650faab0 296 unsigned long long Size;
b2e465d6
AL		 297 if (Comp.Finalize(Size) == false)
298 {
299 c0out << endl;
db0db9fe 300 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 301 }
302
303 if (Size != 0)
304 c0out << " New "
305 << SizeToStr(Size) << "B ";
306 else
307 c0out << ' ';
308
309 struct timeval NewTime;
310 gettimeofday(&NewTime,0);
311 double Delta = NewTime.tv_sec - StartTime.tv_sec +
312 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
313
314 c0out << Sources.Stats.Packages << " pkgs in " <<
315 TimeToStr((long)Delta) << endl;
316
317 Stats.Add(Sources.Stats);
318 Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
319
320 return !_error->PendingError();
321}
322 /*}}}*/
323// PackageMap::GenContents - Actually generate a Contents file /*{{{*/
324// ---------------------------------------------------------------------
325/* This generates the contents file partially described by this object.
326 It searches the given iterator range for other package files that map
327 into this contents file and includes their data as well when building. */
328bool PackageMap::GenContents(Configuration &Setup,
d935bb17
AL		 329 vector<PackageMap>::iterator Begin,
330 vector<PackageMap>::iterator End,
331 unsigned long &Left)
b2e465d6
AL		 332{
333 if (Contents.empty() == true)
334 return true;
335
336 if (Left == 0)
337 return true;
338
339 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
340 string CacheDir = Setup.FindDir("Dir::CacheDir");
341 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
342
343 struct timeval StartTime;
344 gettimeofday(&StartTime,0);
345
346 // Create a package writer object.
31981076 347 ContentsWriter Contents("", Arch);
b2e465d6 348 if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
dc738e7a 349 return _error->Error(_("Package extension list is too long"));
b2e465d6
AL		 350 if (_error->PendingError() == true)
351 return false;
352
353 MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
354 CntCompress,Permissions);
355 Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
356 Contents.Output = Comp.Input;
357 if (_error->PendingError() == true)
358 return false;
359
360 // Write the header out.
361 if (ContentsHead.empty() == false)
362 {
363 FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
364 if (_error->PendingError() == true)
365 return false;
366
650faab0 367 unsigned long long Size = Head.Size();
b2e465d6
AL		 368 unsigned char Buf[4096];
369 while (Size != 0)
370 {
650faab0 371 unsigned long long ToRead = Size;
b2e465d6
AL		 372 if (Size > sizeof(Buf))
373 ToRead = sizeof(Buf);
374
375 if (Head.Read(Buf,ToRead) == false)
376 return false;
377
378 if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
dc738e7a 379 return _error->Errno("fwrite",_("Error writing header to contents file"));
b2e465d6
AL		 380
381 Size -= ToRead;
382 }
383 }
384
385 /* Go over all the package file records and parse all the package
386 files associated with this contents file into one great big honking
387 memory structure, then dump the sorted version */
388 c0out << ' ' << this->Contents << ":" << flush;
f7f0d6c7 389 for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
b2e465d6
AL		 390 {
391 if (I->Contents != this->Contents)
392 continue;
393
394 Contents.Prefix = ArchiveDir;
395 Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
396 Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
397 I->PkgCompress);
398
399 I->ContentsDone = true;
400 }
401
402 Contents.Finish();
403
404 // Finish compressing
650faab0 405 unsigned long long Size;
b2e465d6
AL		 406 if (Comp.Finalize(Size) == false || _error->PendingError() == true)
407 {
408 c0out << endl;
db0db9fe 409 return _error->Error(_("Error processing contents %s"),
b2e465d6
AL		 410 this->Contents.c_str());
411 }
412
413 if (Size != 0)
414 {
415 c0out << " New " << SizeToStr(Size) << "B ";
416 if (Left > Size)
417 Left -= Size;
418 else
419 Left = 0;
420 }
421 else
422 c0out << ' ';
423
424 struct timeval NewTime;
425 gettimeofday(&NewTime,0);
426 double Delta = NewTime.tv_sec - StartTime.tv_sec +
427 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
428
429 c0out << Contents.Stats.Packages << " files " <<
430 SizeToStr(Contents.Stats.Bytes) << "B " <<
431 TimeToStr((long)Delta) << endl;
432
433 return true;
434}
435 /*}}}*/
436
437// LoadTree - Load a 'tree' section from the Generate Config /*{{{*/
438// ---------------------------------------------------------------------
439/* This populates the PkgList with all the possible permutations of the
440 section/arch lists. */
441void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
442{
443 // Load the defaults
444 string DDir = Setup.Find("TreeDefault::Directory",
445 "$(DIST)/$(SECTION)/binary-$(ARCH)/");
446 string DSDir = Setup.Find("TreeDefault::SrcDirectory",
447 "$(DIST)/$(SECTION)/source/");
448 string DPkg = Setup.Find("TreeDefault::Packages",
449 "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
66905344
DK		 450 string DTrans = Setup.Find("TreeDefault::Translation",
451 "$(DIST)/$(SECTION)/i18n/Translation-en");
b2e465d6
AL		 452 string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
453 "$(DIST)/$(SECTION)/");
454 string DContents = Setup.Find("TreeDefault::Contents",
3adddfa8 455 "$(DIST)/$(SECTION)/Contents-$(ARCH)");
b2e465d6
AL		 456 string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
457 string DBCache = Setup.Find("TreeDefault::BinCacheDB",
458 "packages-$(ARCH).db");
459 string DSources = Setup.Find("TreeDefault::Sources",
460 "$(DIST)/$(SECTION)/source/Sources");
461 string DFLFile = Setup.Find("TreeDefault::FileList", "");
462 string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
463
2db5b414 464 mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
34f1d96c
DK		 465
466 bool const LongDescription = Setup.FindB("Default::LongDescription",
4e794c50 467 _config->FindB("APT::FTPArchive::LongDescription", true));
34f1d96c 468 string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
4e794c50 469
b2e465d6
AL		 470 // Process 'tree' type sections
471 const Configuration::Item *Top = Setup.Tree("tree");
472 for (Top = (Top == 0?0:Top->Child); Top != 0;)
473 {
474 Configuration Block(Top);
475 string Dist = Top->Tag;
476
477 // Parse the sections
d935bb17
AL		 478 string Tmp = Block.Find("Sections");
479 const char *Sections = Tmp.c_str();
b2e465d6
AL		 480 string Section;
481 while (ParseQuoteWord(Sections,Section) == true)
482 {
b2e465d6 483 string Arch;
66905344
DK		 484 struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
485 {"$(SECTION)",&Section},
486 {"$(ARCH)",&Arch},
487 {}};
34f1d96c 488 mode_t const Perms = Block.FindI("FileMode", Permissions);
4e794c50 489 bool const LongDesc = Block.FindB("LongDescription", LongDescription);
66905344 490 TranslationWriter *TransWriter;
4e794c50 491 if (DTrans.empty() == false && LongDesc == false)
66905344
DK		 492 {
493 string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
494 SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
34f1d96c
DK		 495 string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
496 TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
66905344
DK		 497 }
498 else
499 TransWriter = NULL;
500
501 string const Tmp2 = Block.Find("Architectures");
d935bb17 502 const char *Archs = Tmp2.c_str();
b2e465d6
AL		 503 while (ParseQuoteWord(Archs,Arch) == true)
504 {
b2e465d6 505 PackageMap Itm;
34f1d96c 506 Itm.Permissions = Perms;
b2e465d6
AL		 507 Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
508 Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
509
510 if (stringcasecmp(Arch,"source") == 0)
511 {
512 Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
513 Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
514 Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
515 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
516 Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
64177f17 517 Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
b2e465d6
AL		 518 }
519 else
520 {
521 Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
522 Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
523 Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
524 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
0b41e0e7 525 Itm.Arch = Arch;
4e794c50 526 Itm.LongDesc = LongDesc;
66905344
DK		 527 if (TransWriter != NULL)
528 {
529 TransWriter->IncreaseRefCounter();
530 Itm.TransWriter = TransWriter;
531 }
b2e465d6
AL		 532 Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
533 Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
534 Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
64177f17 535 Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
b2e465d6
AL		 536 }
537
538 Itm.GetGeneral(Setup,Block);
539 PkgList.push_back(Itm);
540 }
66905344
DK		 541 // we didn't use this TransWriter, so we can release it
542 if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
543 delete TransWriter;
b2e465d6
AL		 544 }
545
546 Top = Top->Next;
547 }
548}
549 /*}}}*/
550// LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/
551// ---------------------------------------------------------------------
552/* */
553void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
554{
2db5b414
DK		 555 mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
556
b2e465d6
AL		 557 // Process 'bindirectory' type sections
558 const Configuration::Item *Top = Setup.Tree("bindirectory");
559 for (Top = (Top == 0?0:Top->Child); Top != 0;)
560 {
561 Configuration Block(Top);
562
563 PackageMap Itm;
564 Itm.PkgFile = Block.Find("Packages");
565 Itm.SrcFile = Block.Find("Sources");
566 Itm.BinCacheDB = Block.Find("BinCacheDB");
567 Itm.BinOverride = Block.Find("BinOverride");
64177f17
AL		 568 Itm.ExtraOverride = Block.Find("ExtraOverride");
569 Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
b2e465d6
AL		 570 Itm.SrcOverride = Block.Find("SrcOverride");
571 Itm.BaseDir = Top->Tag;
572 Itm.FLFile = Block.Find("FileList");
573 Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
574 Itm.Contents = Block.Find("Contents");
575 Itm.ContentsHead = Block.Find("Contents::Header");
2db5b414 576 Itm.Permissions = Block.FindI("FileMode", Permissions);
b2e465d6
AL		 577
578 Itm.GetGeneral(Setup,Block);
579 PkgList.push_back(Itm);
580
581 Top = Top->Next;
582 }
583}
584 /*}}}*/
585
586// ShowHelp - Show the help text /*{{{*/
587// ---------------------------------------------------------------------
588/* */
589bool ShowHelp(CommandLine &CmdL)
590{
9179f697 591 ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
5b28c804 592 COMMON_ARCH,__DATE__,__TIME__);
b2e465d6
AL		 593 if (_config->FindB("version") == true)
594 return true;
595
596 cout <<
dc738e7a 597 _("Usage: apt-ftparchive [options] command\n"
7a3e9e0a 598 "Commands: packages binarypath [overridefile [pathprefix]]\n"
b2e465d6
AL		 599 " sources srcpath [overridefile [pathprefix]]\n"
600 " contents path\n"
0dfd2728 601 " release path\n"
b2e465d6
AL		 602 " generate config [groups]\n"
603 " clean config\n"
604 "\n"
605 "apt-ftparchive generates index files for Debian archives. It supports\n"
606 "many styles of generation from fully automated to functional replacements\n"
607 "for dpkg-scanpackages and dpkg-scansources\n"
608 "\n"
609 "apt-ftparchive generates Package files from a tree of .debs. The\n"
610 "Package file contains the contents of all the control fields from\n"
611 "each package as well as the MD5 hash and filesize. An override file\n"
612 "is supported to force the value of Priority and Section.\n"
613 "\n"
614 "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
615 "The --source-override option can be used to specify a src override file\n"
616 "\n"
617 "The 'packages' and 'sources' command should be run in the root of the\n"
618 "tree. BinaryPath should point to the base of the recursive search and \n"
75224826 619 "override file should contain the override flags. Pathprefix is\n"
b2e465d6 620 "appended to the filename fields if present. Example usage from the \n"
75224826 621 "Debian archive:\n"
b2e465d6
AL		 622 " apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
623 " dists/potato/main/binary-i386/Packages\n"
624 "\n"
625 "Options:\n"
626 " -h This help text\n"
627 " --md5 Control MD5 generation\n"
628 " -s=? Source override file\n"
629 " -q Quiet\n"
630 " -d=? Select the optional caching database\n"
631 " --no-delink Enable delinking debug mode\n"
632 " --contents Control contents file generation\n"
633 " -c=? Read this configuration file\n"
8a2d7db6 634 " -o=? Set an arbitrary configuration option") << endl;
b2e465d6
AL		 635
636 return true;
637}
638 /*}}}*/
639// SimpleGenPackages - Generate a Packages file for a directory tree /*{{{*/
640// ---------------------------------------------------------------------
641/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
642bool SimpleGenPackages(CommandLine &CmdL)
643{
644 if (CmdL.FileSize() < 2)
645 return ShowHelp(CmdL);
646
647 string Override;
648 if (CmdL.FileSize() >= 3)
649 Override = CmdL.FileList[2];
650
651 // Create a package writer object.
652 PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
31981076 653 Override, "", _config->Find("APT::FTPArchive::Architecture"));
b2e465d6
AL		 654 if (_error->PendingError() == true)
655 return false;
656
657 if (CmdL.FileSize() >= 4)
658 Packages.PathPrefix = CmdL.FileList[3];
659
660 // Do recursive directory searching
661 if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
662 return false;
663
664 return true;
665}
666 /*}}}*/
667// SimpleGenContents - Generate a Contents listing /*{{{*/
668// ---------------------------------------------------------------------
669/* */
670bool SimpleGenContents(CommandLine &CmdL)
671{
672 if (CmdL.FileSize() < 2)
673 return ShowHelp(CmdL);
674
675 // Create a package writer object.
31981076 676 ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
b2e465d6
AL		 677 if (_error->PendingError() == true)
678 return false;
679
680 // Do recursive directory searching
681 if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
682 return false;
683
684 Contents.Finish();
685
686 return true;
687}
688 /*}}}*/
689// SimpleGenSources - Generate a Sources file for a directory tree /*{{{*/
690// ---------------------------------------------------------------------
691/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
692bool SimpleGenSources(CommandLine &CmdL)
693{
694 if (CmdL.FileSize() < 2)
695 return ShowHelp(CmdL);
696
697 string Override;
698 if (CmdL.FileSize() >= 3)
699 Override = CmdL.FileList[2];
700
701 string SOverride;
702 if (Override.empty() == false)
703 SOverride = Override + ".src";
704
705 SOverride = _config->Find("APT::FTPArchive::SourceOverride",
706 SOverride.c_str());
707
708 // Create a package writer object.
f6f06a8f 709 SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),Override,SOverride);
b2e465d6
AL		 710 if (_error->PendingError() == true)
711 return false;
712
713 if (CmdL.FileSize() >= 4)
714 Sources.PathPrefix = CmdL.FileList[3];
715
716 // Do recursive directory searching
717 if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
718 return false;
719
720 return true;
721}
98953965
AL		 722 /*}}}*/
723// SimpleGenRelease - Generate a Release file for a directory tree /*{{{*/
724// ---------------------------------------------------------------------
725bool SimpleGenRelease(CommandLine &CmdL)
726{
0dfd2728
AL		 727 if (CmdL.FileSize() < 2)
728 return ShowHelp(CmdL);
729
c0eb6bc6
AL		 730 string Dir = CmdL.FileList[1];
731
98953965 732 ReleaseWriter Release("");
c0eb6bc6
AL		 733 Release.DirStrip = Dir;
734
98953965
AL		 735 if (_error->PendingError() == true)
736 return false;
737
c0eb6bc6 738 if (Release.RecursiveScan(Dir) == false)
98953965
AL		 739 return false;
740
f7291f62
AL		 741 Release.Finish();
742
98953965
AL		 743 return true;
744}
745
b2e465d6
AL		 746 /*}}}*/
747// Generate - Full generate, using a config file /*{{{*/
748// ---------------------------------------------------------------------
749/* */
750bool Generate(CommandLine &CmdL)
751{
752 struct CacheDB::Stats SrcStats;
753 if (CmdL.FileSize() < 2)
754 return ShowHelp(CmdL);
755
756 struct timeval StartTime;
757 gettimeofday(&StartTime,0);
758 struct CacheDB::Stats Stats;
759
760 // Read the configuration file.
761 Configuration Setup;
762 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
763 return false;
764
765 vector<PackageMap> PkgList;
766 LoadTree(PkgList,Setup);
767 LoadBinDir(PkgList,Setup);
768
769 // Sort by cache DB to improve IO locality.
770 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
771
772 // Generate packages
773 if (CmdL.FileSize() <= 2)
774 {
f7f0d6c7 775 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6
AL		 776 if (I->GenPackages(Setup,Stats) == false)
777 _error->DumpErrors();
f7f0d6c7 778 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6
AL		 779 if (I->GenSources(Setup,SrcStats) == false)
780 _error->DumpErrors();
781 }
782 else
783 {
784 // Make a choice list out of the package list..
785 RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
786 RxChoiceList *End = List;
f7f0d6c7 787 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6 788 {
d935bb17 789 End->UserData = &(*I);
b2e465d6
AL		 790 End->Str = I->BaseDir.c_str();
791 End++;
792
d935bb17 793 End->UserData = &(*I);
b2e465d6
AL		 794 End->Str = I->Tag.c_str();
795 End++;
796 }
797 End->Str = 0;
798
799 // Regex it
800 if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
801 {
802 delete [] List;
dc738e7a 803 return _error->Error(_("No selections matched"));
b2e465d6
AL		 804 }
805 _error->DumpErrors();
806
807 // Do the generation for Packages
808 for (End = List; End->Str != 0; End++)
809 {
810 if (End->Hit == false)
811 continue;
812
813 PackageMap *I = (PackageMap *)End->UserData;
814 if (I->PkgDone == true)
815 continue;
816 if (I->GenPackages(Setup,Stats) == false)
817 _error->DumpErrors();
818 }
819
820 // Do the generation for Sources
821 for (End = List; End->Str != 0; End++)
822 {
823 if (End->Hit == false)
824 continue;
825
826 PackageMap *I = (PackageMap *)End->UserData;
827 if (I->SrcDone == true)
828 continue;
829 if (I->GenSources(Setup,SrcStats) == false)
830 _error->DumpErrors();
831 }
832
833 delete [] List;
834 }
66905344
DK		 835
836 // close the Translation master files
f7f0d6c7 837 for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I)
66905344
DK		 838 if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
839 delete I->TransWriter;
840
b2e465d6
AL		 841 if (_config->FindB("APT::FTPArchive::Contents",true) == false)
842 return true;
843
c6474fb6 844 c1out << "Packages done, Starting contents." << endl;
b2e465d6
AL		 845
846 // Sort the contents file list by date
847 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
f7f0d6c7 848 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6
AL		 849 {
850 struct stat A;
851 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
852 I->CntCompress,A) == false)
853 time(&I->ContentsMTime);
854 else
855 I->ContentsMTime = A.st_mtime;
856 }
857 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
858
859 /* Now for Contents.. The process here is to do a make-like dependency
860 check. Each contents file is verified to be newer than the package files
861 that describe the debs it indexes. Since the package files contain
862 hashes of the .debs this means they have not changed either so the
863 contents must be up to date. */
864 unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
f7f0d6c7 865 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6
AL		 866 {
867 // This record is not relevent
868 if (I->ContentsDone == true ||
869 I->Contents.empty() == true)
870 continue;
871
872 // Do not do everything if the user specified sections.
873 if (CmdL.FileSize() > 2 && I->PkgDone == false)
874 continue;
875
876 struct stat A,B;
877 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
878 {
879 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
880 {
dc738e7a 881 _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
b2e465d6
AL		 882 continue;
883 }
884
885 if (A.st_mtime > B.st_mtime)
886 continue;
887 }
888
889 if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
890 MaxContentsChange) == false)
891 _error->DumpErrors();
892
893 // Hit the limit?
894 if (MaxContentsChange == 0)
895 {
896 c1out << "Hit contents update byte limit" << endl;
897 break;
898 }
899 }
900
901 struct timeval NewTime;
902 gettimeofday(&NewTime,0);
903 double Delta = NewTime.tv_sec - StartTime.tv_sec +
904 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
905 c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
906 << " archives. Took " << TimeToStr((long)Delta) << endl;
907
908 return true;
909}
910 /*}}}*/
911// Clean - Clean out the databases /*{{{*/
912// ---------------------------------------------------------------------
913/* */
914bool Clean(CommandLine &CmdL)
915{
916 if (CmdL.FileSize() != 2)
917 return ShowHelp(CmdL);
918
919 // Read the configuration file.
920 Configuration Setup;
921 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
922 return false;
923
924 vector<PackageMap> PkgList;
925 LoadTree(PkgList,Setup);
926 LoadBinDir(PkgList,Setup);
927
928 // Sort by cache DB to improve IO locality.
929 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
930
931 string CacheDir = Setup.FindDir("Dir::CacheDir");
932
d935bb17 933 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
b2e465d6
AL		 934 {
935 c0out << I->BinCacheDB << endl;
936 CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
937 if (DB.Clean() == false)
938 _error->DumpErrors();
939
940 string CacheDB = I->BinCacheDB;
f7f0d6c7 941 for (; I != PkgList.end() && I->BinCacheDB == CacheDB; ++I);
b2e465d6
AL		 942 }
943
944 return true;
945}
946 /*}}}*/
947
948int main(int argc, const char *argv[])
949{
1692a2c7 950 setlocale(LC_ALL, "");
b2e465d6
AL		 951 CommandLine::Args Args[] = {
952 {'h',"help","help",0},
953 {0,"md5","APT::FTPArchive::MD5",0},
3c54407f
DK		 954 {0,"sha1","APT::FTPArchive::SHA1",0},
955 {0,"sha256","APT::FTPArchive::SHA256",0},
b2e465d6
AL		 956 {'v',"version","version",0},
957 {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
958 {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
959 {'q',"quiet","quiet",CommandLine::IntLevel},
960 {'q',"silent","quiet",CommandLine::IntLevel},
961 {0,"delink","APT::FTPArchive::DeLinkAct",0},
962 {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
963 {0,"contents","APT::FTPArchive::Contents",0},
31981076 964 {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
b2e465d6
AL		 965 {'c',"config-file",0,CommandLine::ConfigFile},
966 {'o',"option",0,CommandLine::ArbItem},
967 {0,0,0,0}};
968 CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
969 {"contents",&SimpleGenContents},
970 {"sources",&SimpleGenSources},
98953965 971 {"release",&SimpleGenRelease},
b2e465d6
AL		 972 {"generate",&Generate},
973 {"clean",&Clean},
974 {"help",&ShowHelp},
975 {0,0}};
976
977 // Parse the command line and initialize the package library
978 CommandLine CmdL(Args,_config);
b34d4b47 979 if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false)
b2e465d6
AL		 980 {
981 _error->DumpErrors();
982 return 100;
983 }
984
985 // See if the help should be shown
986 if (_config->FindB("help") == true ||
987 _config->FindB("version") == true ||
988 CmdL.FileSize() == 0)
989 {
990 ShowHelp(CmdL);
991 return 0;
992 }
993
994 // Setup the output streams
1809f8d2
AL		 995 c0out.rdbuf(clog.rdbuf());
996 c1out.rdbuf(clog.rdbuf());
997 c2out.rdbuf(clog.rdbuf());
b2e465d6
AL		 998 Quiet = _config->FindI("quiet",0);
999 if (Quiet > 0)
1000 c0out.rdbuf(devnull.rdbuf());
1001 if (Quiet > 1)
1002 c1out.rdbuf(devnull.rdbuf());
1003
1004 // Match the operation
1005 CmdL.DispatchArg(Cmds);
1006
1007 if (_error->empty() == false)
1008 {
1009 bool Errors = _error->PendingError();
1010 _error->DumpErrors();
1011 return Errors == true?100:0;
1012 }
1013 return 0;
1014}
APT (for Cydia)