]> git.saurik.com Git - apt.git/blame - ftparchive/apt-ftparchive.cc
projects / apt.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
test framework: Unset http proxy variables in setupenvironment
[apt.git] / ftparchive / apt-ftparchive.cc
CommitLineData
b2e465d6
AL		 1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
7db98ffc 3// $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
b2e465d6
AL		 4/* ######################################################################
5
c6474fb6 6 apt-ftparchive - Efficient work-alike for dpkg-scanpackages
b2e465d6
AL		 7
8 Let contents be disabled from the conf
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
ea542140
DK		 13#include <config.h>
14
b2e465d6
AL		 15#include <apt-pkg/error.h>
16#include <apt-pkg/configuration.h>
17#include <apt-pkg/cmndline.h>
18#include <apt-pkg/strutl.h>
3b1fffc3 19#include <apt-pkg/init.h>
453b82a3 20#include <apt-pkg/fileutl.h>
b2e465d6 21
ad7e0941 22#include <apt-private/private-cmndline.h>
d9e518c6 23#include <apt-private/private-output.h>
e7e10e47 24#include <apt-private/private-main.h>
ad7e0941 25
453b82a3 26#include <algorithm>
152ab79e 27#include <climits>
b2e465d6 28#include <sys/time.h>
453b82a3
DK		 29#include <locale.h>
30#include <stdio.h>
31#include <sys/stat.h>
32#include <time.h>
33#include <functional>
34#include <iostream>
35#include <string>
36#include <vector>
b2e465d6 37
453b82a3
DK		 38#include "cachedb.h"
39#include "override.h"
ea542140 40#include "apt-ftparchive.h"
b2e465d6 41#include "multicompress.h"
ea542140
DK		 42#include "writer.h"
43
44#include <apti18n.h>
b2e465d6
AL		 45 /*}}}*/
46
d9e518c6 47using namespace std;
b2e465d6
AL		 48unsigned Quiet = 0;
49
50// struct PackageMap - List of all package files in the config file /*{{{*/
51// ---------------------------------------------------------------------
52/* */
53struct PackageMap
54{
55 // General Stuff
56 string BaseDir;
57 string InternalPrefix;
58 string FLFile;
59 string PkgExt;
60 string SrcExt;
61
62 // Stuff for the Package File
63 string PkgFile;
64 string BinCacheDB;
ce928105 65 string SrcCacheDB;
b2e465d6 66 string BinOverride;
64177f17 67 string ExtraOverride;
0b41e0e7
MV		 68
69 // We generate for this given arch
70 string Arch;
1dd20368 71 bool IncludeArchAll;
b2e465d6
AL		 72
73 // Stuff for the Source File
74 string SrcFile;
75 string SrcOverride;
64177f17 76 string SrcExtraOverride;
b2e465d6 77
66905344 78 // Translation master file
4e794c50 79 bool LongDesc;
66905344
DK		 80 TranslationWriter *TransWriter;
81
b2e465d6
AL		 82 // Contents
83 string Contents;
84 string ContentsHead;
85
86 // Random things
87 string Tag;
88 string PkgCompress;
89 string CntCompress;
90 string SrcCompress;
91 string PathPrefix;
92 unsigned int DeLinkLimit;
93 mode_t Permissions;
94
95 bool ContentsDone;
96 bool PkgDone;
97 bool SrcDone;
98 time_t ContentsMTime;
99
100 struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
101 {
102 inline bool operator() (const PackageMap &x,const PackageMap &y)
103 {return x.ContentsMTime < y.ContentsMTime;};
104 };
105
106 struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
107 {
108 inline bool operator() (const PackageMap &x,const PackageMap &y)
109 {return x.BinCacheDB < y.BinCacheDB;};
110 };
ce928105
MV		 111
112 struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
113 {
114 inline bool operator() (const PackageMap &x,const PackageMap &y)
115 {return x.SrcCacheDB < y.SrcCacheDB;};
116 };
b2e465d6
AL		 117
118 void GetGeneral(Configuration &Setup,Configuration &Block);
119 bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
120 bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
121 bool GenContents(Configuration &Setup,
d935bb17
AL		 122 vector<PackageMap>::iterator Begin,
123 vector<PackageMap>::iterator End,
b2e465d6
AL		 124 unsigned long &Left);
125
1dd20368
DK		 126 PackageMap() : IncludeArchAll(true), LongDesc(true), TransWriter(NULL),
127 DeLinkLimit(0), Permissions(1), ContentsDone(false),
128 PkgDone(false), SrcDone(false), ContentsMTime(0) {};
b2e465d6
AL		 129};
130 /*}}}*/
131
132// PackageMap::GetGeneral - Common per-section definitions /*{{{*/
133// ---------------------------------------------------------------------
134/* */
135void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
136{
137 PathPrefix = Block.Find("PathPrefix");
138
139 if (Block.FindB("External-Links",true) == false)
adfa4116 140 DeLinkLimit = Setup.FindI("Default::DeLinkLimit", std::numeric_limits<unsigned int>::max());
b2e465d6
AL		 141 else
142 DeLinkLimit = 0;
143
144 PkgCompress = Block.Find("Packages::Compress",
145 Setup.Find("Default::Packages::Compress",". gzip").c_str());
146 CntCompress = Block.Find("Contents::Compress",
147 Setup.Find("Default::Contents::Compress",". gzip").c_str());
148 SrcCompress = Block.Find("Sources::Compress",
149 Setup.Find("Default::Sources::Compress",". gzip").c_str());
150
151 SrcExt = Block.Find("Sources::Extensions",
152 Setup.Find("Default::Sources::Extensions",".dsc").c_str());
153 PkgExt = Block.Find("Packages::Extensions",
154 Setup.Find("Default::Packages::Extensions",".deb").c_str());
155
2db5b414
DK		 156 Permissions = Setup.FindI("Default::FileMode",0644);
157
b2e465d6
AL		 158 if (FLFile.empty() == false)
159 FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
160
161 if (Contents == " ")
162 Contents= string();
163}
164 /*}}}*/
165// PackageMap::GenPackages - Actually generate a Package file /*{{{*/
166// ---------------------------------------------------------------------
167/* This generates the Package File described by this object. */
168bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
169{
170 if (PkgFile.empty() == true)
171 return true;
172
173 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
174 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
175 string CacheDir = Setup.FindDir("Dir::CacheDir");
176
177 struct timeval StartTime;
178 gettimeofday(&StartTime,0);
179
180 PkgDone = true;
181
182 // Create a package writer object.
88593886
DK		 183 MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
184 PkgCompress,Permissions);
3d8232bf 185 PackagesWriter Packages(&Comp.Input, TransWriter, flCombine(CacheDir,BinCacheDB),
64177f17 186 flCombine(OverrideDir,BinOverride),
0b41e0e7 187 flCombine(OverrideDir,ExtraOverride),
1dd20368 188 Arch, IncludeArchAll);
b2e465d6 189 if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
dc738e7a 190 return _error->Error(_("Package extension list is too long"));
b2e465d6 191 if (_error->PendingError() == true)
db0db9fe 192 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 193
194 Packages.PathPrefix = PathPrefix;
195 Packages.DirStrip = ArchiveDir;
196 Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
197
4e794c50 198 Packages.LongDescription = LongDesc;
66905344 199
b2e465d6
AL		 200 Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
201 Packages.DeLinkLimit = DeLinkLimit;
64177f17 202
b2e465d6 203 if (_error->PendingError() == true)
db0db9fe 204 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 205
206 c0out << ' ' << BaseDir << ":" << flush;
207
208 // Do recursive directory searching
209 if (FLFile.empty() == true)
210 {
211 if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
212 return false;
213 }
214 else
215 {
216 if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
217 return false;
218 }
219
220 Packages.Output = 0; // Just in case
221
222 // Finish compressing
650faab0 223 unsigned long long Size;
b2e465d6
AL		 224 if (Comp.Finalize(Size) == false)
225 {
226 c0out << endl;
db0db9fe 227 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 228 }
229
230 if (Size != 0)
231 c0out << " New "
232 << SizeToStr(Size) << "B ";
233 else
234 c0out << ' ';
235
236 struct timeval NewTime;
237 gettimeofday(&NewTime,0);
238 double Delta = NewTime.tv_sec - StartTime.tv_sec +
239 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
ce928105 240
b2e465d6
AL		 241 c0out << Packages.Stats.Packages << " files " <<
242/* SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
243 SizeToStr(Packages.Stats.Bytes) << "B " <<
244 TimeToStr((long)Delta) << endl;
ce928105
MV		 245
246 if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
247 c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
b2e465d6
AL		 248
249 Stats.Add(Packages.Stats);
250 Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
251
252 return !_error->PendingError();
253}
98953965 254
b2e465d6 255 /*}}}*/
64177f17 256// PackageMap::GenSources - Actually generate a Source file /*{{{*/
b2e465d6
AL		 257// ---------------------------------------------------------------------
258/* This generates the Sources File described by this object. */
259bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
260{
261 if (SrcFile.empty() == true)
262 return true;
263
264 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
265 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
266 string CacheDir = Setup.FindDir("Dir::CacheDir");
267
268 struct timeval StartTime;
269 gettimeofday(&StartTime,0);
270
271 SrcDone = true;
272
273 // Create a package writer object.
88593886
DK		 274 MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
275 SrcCompress,Permissions);
276 SourcesWriter Sources(&Comp.Input, flCombine(CacheDir, SrcCacheDB),
f6f06a8f 277 flCombine(OverrideDir,BinOverride),
64177f17
AL		 278 flCombine(OverrideDir,SrcOverride),
279 flCombine(OverrideDir,SrcExtraOverride));
b2e465d6 280 if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
dc738e7a 281 return _error->Error(_("Source extension list is too long"));
b2e465d6 282 if (_error->PendingError() == true)
db0db9fe 283 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 284
285 Sources.PathPrefix = PathPrefix;
286 Sources.DirStrip = ArchiveDir;
287 Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
288
289 Sources.DeLinkLimit = DeLinkLimit;
290 Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
88593886 291
b2e465d6 292 if (_error->PendingError() == true)
db0db9fe 293 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 294
295 c0out << ' ' << BaseDir << ":" << flush;
296
297 // Do recursive directory searching
298 if (FLFile.empty() == true)
299 {
300 if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
301 return false;
302 }
303 else
304 {
305 if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
306 return false;
307 }
308 Sources.Output = 0; // Just in case
309
310 // Finish compressing
650faab0 311 unsigned long long Size;
b2e465d6
AL		 312 if (Comp.Finalize(Size) == false)
313 {
314 c0out << endl;
db0db9fe 315 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 316 }
317
318 if (Size != 0)
319 c0out << " New "
320 << SizeToStr(Size) << "B ";
321 else
322 c0out << ' ';
323
324 struct timeval NewTime;
325 gettimeofday(&NewTime,0);
326 double Delta = NewTime.tv_sec - StartTime.tv_sec +
327 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
328
329 c0out << Sources.Stats.Packages << " pkgs in " <<
330 TimeToStr((long)Delta) << endl;
331
ce928105
MV		 332 if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
333 c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
334
b2e465d6
AL		 335 Stats.Add(Sources.Stats);
336 Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
337
338 return !_error->PendingError();
339}
340 /*}}}*/
341// PackageMap::GenContents - Actually generate a Contents file /*{{{*/
342// ---------------------------------------------------------------------
343/* This generates the contents file partially described by this object.
344 It searches the given iterator range for other package files that map
345 into this contents file and includes their data as well when building. */
346bool PackageMap::GenContents(Configuration &Setup,
d935bb17
AL		 347 vector<PackageMap>::iterator Begin,
348 vector<PackageMap>::iterator End,
349 unsigned long &Left)
b2e465d6
AL		 350{
351 if (Contents.empty() == true)
352 return true;
353
354 if (Left == 0)
355 return true;
356
357 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
358 string CacheDir = Setup.FindDir("Dir::CacheDir");
359 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
360
361 struct timeval StartTime;
362 gettimeofday(&StartTime,0);
363
364 // Create a package writer object.
88593886
DK		 365 MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
366 CntCompress,Permissions);
367 Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
1dd20368 368 ContentsWriter Contents(&Comp.Input, "", Arch, IncludeArchAll);
b2e465d6 369 if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
dc738e7a 370 return _error->Error(_("Package extension list is too long"));
b2e465d6
AL		 371 if (_error->PendingError() == true)
372 return false;
373
b2e465d6
AL		 374 if (_error->PendingError() == true)
375 return false;
376
377 // Write the header out.
378 if (ContentsHead.empty() == false)
379 {
380 FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
381 if (_error->PendingError() == true)
382 return false;
88593886 383
650faab0 384 unsigned long long Size = Head.Size();
b2e465d6
AL		 385 unsigned char Buf[4096];
386 while (Size != 0)
387 {
650faab0 388 unsigned long long ToRead = Size;
b2e465d6
AL		 389 if (Size > sizeof(Buf))
390 ToRead = sizeof(Buf);
88593886 391
b2e465d6
AL		 392 if (Head.Read(Buf,ToRead) == false)
393 return false;
88593886
DK		 394
395 if (Comp.Input.Write(Buf, ToRead) == false)
dc738e7a 396 return _error->Errno("fwrite",_("Error writing header to contents file"));
88593886 397
b2e465d6 398 Size -= ToRead;
88593886
DK		 399 }
400 }
401
b2e465d6
AL		 402 /* Go over all the package file records and parse all the package
403 files associated with this contents file into one great big honking
404 memory structure, then dump the sorted version */
405 c0out << ' ' << this->Contents << ":" << flush;
f7f0d6c7 406 for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
b2e465d6
AL		 407 {
408 if (I->Contents != this->Contents)
409 continue;
410
411 Contents.Prefix = ArchiveDir;
412 Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
413 Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
414 I->PkgCompress);
415
416 I->ContentsDone = true;
417 }
418
419 Contents.Finish();
420
421 // Finish compressing
650faab0 422 unsigned long long Size;
b2e465d6
AL		 423 if (Comp.Finalize(Size) == false || _error->PendingError() == true)
424 {
425 c0out << endl;
db0db9fe 426 return _error->Error(_("Error processing contents %s"),
b2e465d6
AL		 427 this->Contents.c_str());
428 }
429
430 if (Size != 0)
431 {
432 c0out << " New " << SizeToStr(Size) << "B ";
433 if (Left > Size)
434 Left -= Size;
435 else
436 Left = 0;
437 }
438 else
439 c0out << ' ';
440
441 struct timeval NewTime;
442 gettimeofday(&NewTime,0);
443 double Delta = NewTime.tv_sec - StartTime.tv_sec +
444 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
445
ce928105
MV		 446 if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
447 c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
448
b2e465d6
AL		 449 c0out << Contents.Stats.Packages << " files " <<
450 SizeToStr(Contents.Stats.Bytes) << "B " <<
451 TimeToStr((long)Delta) << endl;
452
453 return true;
454}
455 /*}}}*/
456
457// LoadTree - Load a 'tree' section from the Generate Config /*{{{*/
458// ---------------------------------------------------------------------
459/* This populates the PkgList with all the possible permutations of the
460 section/arch lists. */
3d8232bf 461static void LoadTree(vector<PackageMap> &PkgList, std::vector<TranslationWriter*> &TransList, Configuration &Setup)
b2e465d6
AL		 462{
463 // Load the defaults
464 string DDir = Setup.Find("TreeDefault::Directory",
465 "$(DIST)/$(SECTION)/binary-$(ARCH)/");
466 string DSDir = Setup.Find("TreeDefault::SrcDirectory",
467 "$(DIST)/$(SECTION)/source/");
468 string DPkg = Setup.Find("TreeDefault::Packages",
469 "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
66905344
DK		 470 string DTrans = Setup.Find("TreeDefault::Translation",
471 "$(DIST)/$(SECTION)/i18n/Translation-en");
b2e465d6
AL		 472 string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
473 "$(DIST)/$(SECTION)/");
474 string DContents = Setup.Find("TreeDefault::Contents",
3adddfa8 475 "$(DIST)/$(SECTION)/Contents-$(ARCH)");
b2e465d6
AL		 476 string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
477 string DBCache = Setup.Find("TreeDefault::BinCacheDB",
478 "packages-$(ARCH).db");
ce928105
MV		 479 string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
480 "sources-$(SECTION).db");
b2e465d6
AL		 481 string DSources = Setup.Find("TreeDefault::Sources",
482 "$(DIST)/$(SECTION)/source/Sources");
483 string DFLFile = Setup.Find("TreeDefault::FileList", "");
484 string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
485
2db5b414 486 mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
34f1d96c
DK		 487
488 bool const LongDescription = Setup.FindB("Default::LongDescription",
4e794c50 489 _config->FindB("APT::FTPArchive::LongDescription", true));
34f1d96c 490 string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
1dd20368
DK		 491 bool const ConfIncludeArchAllExists = _config->Exists("APT::FTPArchive::IncludeArchitectureAll");
492 bool const ConfIncludeArchAll = _config->FindB("APT::FTPArchive::IncludeArchitectureAll", true);
4e794c50 493
b2e465d6
AL		 494 // Process 'tree' type sections
495 const Configuration::Item *Top = Setup.Tree("tree");
496 for (Top = (Top == 0?0:Top->Child); Top != 0;)
497 {
498 Configuration Block(Top);
499 string Dist = Top->Tag;
500
501 // Parse the sections
d935bb17
AL		 502 string Tmp = Block.Find("Sections");
503 const char *Sections = Tmp.c_str();
b2e465d6
AL		 504 string Section;
505 while (ParseQuoteWord(Sections,Section) == true)
506 {
1dd20368 507 struct SubstVar Vars[] = {{"$(DIST)",&Dist},
66905344 508 {"$(SECTION)",&Section},
1dd20368
DK		 509 {"$(ARCH)",nullptr},
510 {nullptr, nullptr}};
34f1d96c 511 mode_t const Perms = Block.FindI("FileMode", Permissions);
4e794c50 512 bool const LongDesc = Block.FindB("LongDescription", LongDescription);
1dd20368
DK		 513 TranslationWriter *TransWriter = nullptr;
514
515 std::string Tmp2 = Block.Find("Architectures");
516 std::transform(Tmp2.begin(), Tmp2.end(), Tmp2.begin(), ::tolower);
517 std::vector<std::string> const Archs = VectorizeString(Tmp2, ' ');
518 bool IncludeArchAll;
519 if (ConfIncludeArchAllExists == true)
520 IncludeArchAll = ConfIncludeArchAll;
521 else
522 IncludeArchAll = std::find(Archs.begin(), Archs.end(), "all") == Archs.end();
523 for (auto const& Arch: Archs)
b2e465d6 524 {
1dd20368
DK		 525 if (Arch.empty()) continue;
526 Vars[2].Contents = &Arch;
b2e465d6 527 PackageMap Itm;
34f1d96c 528 Itm.Permissions = Perms;
b2e465d6
AL		 529 Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
530 Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
531
1dd20368 532 if (Arch == "source")
b2e465d6
AL		 533 {
534 Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
535 Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
536 Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
537 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
538 Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
64177f17 539 Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
ce928105 540 Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);
b2e465d6
AL		 541 }
542 else
543 {
544 Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
545 Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
546 Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
547 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
0b41e0e7 548 Itm.Arch = Arch;
1dd20368 549 Itm.IncludeArchAll = IncludeArchAll;
4e794c50 550 Itm.LongDesc = LongDesc;
3d8232bf 551 if (TransWriter == NULL && DTrans.empty() == false && LongDesc == false && DTrans != "/dev/null")
66905344 552 {
3d8232bf
DK		 553 string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
554 SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
555 string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
556 TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
557 TransList.push_back(TransWriter);
66905344 558 }
3d8232bf 559 Itm.TransWriter = TransWriter;
b2e465d6
AL		 560 Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
561 Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
562 Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
64177f17 563 Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
b2e465d6
AL		 564 }
565
3d8232bf 566 Itm.GetGeneral(Setup,Block);
b2e465d6
AL		 567 PkgList.push_back(Itm);
568 }
569 }
3d8232bf 570
b2e465d6 571 Top = Top->Next;
3d8232bf
DK		 572 }
573}
574 /*}}}*/
575static void UnloadTree(std::vector<TranslationWriter*> const &Trans) /*{{{*/
576{
577 for (std::vector<TranslationWriter*>::const_reverse_iterator T = Trans.rbegin(); T != Trans.rend(); ++T)
578 delete *T;
b2e465d6
AL		 579}
580 /*}}}*/
581// LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/
582// ---------------------------------------------------------------------
583/* */
c3ccac92 584static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
b2e465d6 585{
2db5b414
DK		 586 mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
587
b2e465d6
AL		 588 // Process 'bindirectory' type sections
589 const Configuration::Item *Top = Setup.Tree("bindirectory");
590 for (Top = (Top == 0?0:Top->Child); Top != 0;)
591 {
592 Configuration Block(Top);
593
594 PackageMap Itm;
595 Itm.PkgFile = Block.Find("Packages");
596 Itm.SrcFile = Block.Find("Sources");
597 Itm.BinCacheDB = Block.Find("BinCacheDB");
ce928105 598 Itm.SrcCacheDB = Block.Find("SrcCacheDB");
b2e465d6 599 Itm.BinOverride = Block.Find("BinOverride");
64177f17
AL		 600 Itm.ExtraOverride = Block.Find("ExtraOverride");
601 Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
b2e465d6
AL		 602 Itm.SrcOverride = Block.Find("SrcOverride");
603 Itm.BaseDir = Top->Tag;
604 Itm.FLFile = Block.Find("FileList");
605 Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
606 Itm.Contents = Block.Find("Contents");
607 Itm.ContentsHead = Block.Find("Contents::Header");
2db5b414 608 Itm.Permissions = Block.FindI("FileMode", Permissions);
b2e465d6
AL		 609
610 Itm.GetGeneral(Setup,Block);
611 PkgList.push_back(Itm);
612
613 Top = Top->Next;
614 }
615}
616 /*}}}*/
617
f6777222 618static bool ShowHelp(CommandLine &) /*{{{*/
b2e465d6 619{
41d39345 620 std::cout <<
dc738e7a 621 _("Usage: apt-ftparchive [options] command\n"
7a3e9e0a 622 "Commands: packages binarypath [overridefile [pathprefix]]\n"
b2e465d6
AL		 623 " sources srcpath [overridefile [pathprefix]]\n"
624 " contents path\n"
0dfd2728 625 " release path\n"
b2e465d6
AL		 626 " generate config [groups]\n"
627 " clean config\n"
628 "\n"
629 "apt-ftparchive generates index files for Debian archives. It supports\n"
630 "many styles of generation from fully automated to functional replacements\n"
631 "for dpkg-scanpackages and dpkg-scansources\n"
632 "\n"
633 "apt-ftparchive generates Package files from a tree of .debs. The\n"
634 "Package file contains the contents of all the control fields from\n"
635 "each package as well as the MD5 hash and filesize. An override file\n"
636 "is supported to force the value of Priority and Section.\n"
637 "\n"
638 "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
639 "The --source-override option can be used to specify a src override file\n"
640 "\n"
641 "The 'packages' and 'sources' command should be run in the root of the\n"
642 "tree. BinaryPath should point to the base of the recursive search and \n"
75224826 643 "override file should contain the override flags. Pathprefix is\n"
b2e465d6 644 "appended to the filename fields if present. Example usage from the \n"
75224826 645 "Debian archive:\n"
b2e465d6
AL		 646 " apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
647 " dists/potato/main/binary-i386/Packages\n"
648 "\n"
649 "Options:\n"
650 " -h This help text\n"
651 " --md5 Control MD5 generation\n"
652 " -s=? Source override file\n"
653 " -q Quiet\n"
654 " -d=? Select the optional caching database\n"
655 " --no-delink Enable delinking debug mode\n"
656 " --contents Control contents file generation\n"
657 " -c=? Read this configuration file\n"
8a2d7db6 658 " -o=? Set an arbitrary configuration option") << endl;
b2e465d6
AL		 659 return true;
660}
661 /*}}}*/
662// SimpleGenPackages - Generate a Packages file for a directory tree /*{{{*/
663// ---------------------------------------------------------------------
664/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
c3ccac92 665static bool SimpleGenPackages(CommandLine &CmdL)
b2e465d6
AL		 666{
667 if (CmdL.FileSize() < 2)
8561c2fe 668 return ShowHelp(CmdL);
b2e465d6
AL		 669
670 string Override;
671 if (CmdL.FileSize() >= 3)
672 Override = CmdL.FileList[2];
673
674 // Create a package writer object.
3d8232bf 675 PackagesWriter Packages(NULL, NULL, _config->Find("APT::FTPArchive::DB"),
1dd20368
DK		 676 Override, "", _config->Find("APT::FTPArchive::Architecture"),
677 _config->FindB("APT::FTPArchive::IncludeArchitectureAll", true));
b2e465d6
AL		 678 if (_error->PendingError() == true)
679 return false;
680
681 if (CmdL.FileSize() >= 4)
682 Packages.PathPrefix = CmdL.FileList[3];
683
684 // Do recursive directory searching
685 if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
686 return false;
687
cf6bbca0
MV		 688 // Give some stats if asked for
689 if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
690 c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
691
b2e465d6
AL		 692 return true;
693}
694 /*}}}*/
695// SimpleGenContents - Generate a Contents listing /*{{{*/
696// ---------------------------------------------------------------------
697/* */
c3ccac92 698static bool SimpleGenContents(CommandLine &CmdL)
b2e465d6
AL		 699{
700 if (CmdL.FileSize() < 2)
8561c2fe 701 return ShowHelp(CmdL);
b2e465d6
AL		 702
703 // Create a package writer object.
88593886 704 ContentsWriter Contents(NULL, _config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
b2e465d6
AL		 705 if (_error->PendingError() == true)
706 return false;
707
708 // Do recursive directory searching
709 if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
710 return false;
711
712 Contents.Finish();
713
714 return true;
715}
716 /*}}}*/
717// SimpleGenSources - Generate a Sources file for a directory tree /*{{{*/
718// ---------------------------------------------------------------------
719/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
c3ccac92 720static bool SimpleGenSources(CommandLine &CmdL)
b2e465d6
AL		 721{
722 if (CmdL.FileSize() < 2)
8561c2fe 723 return ShowHelp(CmdL);
b2e465d6
AL		 724
725 string Override;
726 if (CmdL.FileSize() >= 3)
727 Override = CmdL.FileList[2];
728
729 string SOverride;
730 if (Override.empty() == false)
731 SOverride = Override + ".src";
732
733 SOverride = _config->Find("APT::FTPArchive::SourceOverride",
734 SOverride.c_str());
735
736 // Create a package writer object.
88593886 737 SourcesWriter Sources(NULL, _config->Find("APT::FTPArchive::DB"),Override,SOverride);
b2e465d6
AL		 738 if (_error->PendingError() == true)
739 return false;
740
741 if (CmdL.FileSize() >= 4)
742 Sources.PathPrefix = CmdL.FileList[3];
743
744 // Do recursive directory searching
745 if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
746 return false;
747
cf6bbca0
MV		 748 // Give some stats if asked for
749 if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
750 c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl;
751
b2e465d6
AL		 752 return true;
753}
98953965
AL		 754 /*}}}*/
755// SimpleGenRelease - Generate a Release file for a directory tree /*{{{*/
756// ---------------------------------------------------------------------
c3ccac92 757static bool SimpleGenRelease(CommandLine &CmdL)
98953965 758{
0dfd2728 759 if (CmdL.FileSize() < 2)
8561c2fe 760 return ShowHelp(CmdL);
0dfd2728 761
c0eb6bc6
AL		 762 string Dir = CmdL.FileList[1];
763
88593886 764 ReleaseWriter Release(NULL, "");
c0eb6bc6
AL		 765 Release.DirStrip = Dir;
766
98953965
AL		 767 if (_error->PendingError() == true)
768 return false;
769
c0eb6bc6 770 if (Release.RecursiveScan(Dir) == false)
98953965
AL		 771 return false;
772
f7291f62
AL		 773 Release.Finish();
774
98953965
AL		 775 return true;
776}
777
b2e465d6 778 /*}}}*/
308b6fc9 779// DoGeneratePackagesAndSources - Helper for Generate /*{{{*/
b2e465d6 780// ---------------------------------------------------------------------
308b6fc9
DJL		 781static bool DoGeneratePackagesAndSources(Configuration &Setup,
782 vector<PackageMap> &PkgList,
783 struct CacheDB::Stats &SrcStats,
784 struct CacheDB::Stats &Stats,
785 CommandLine &CmdL)
b2e465d6 786{
b2e465d6
AL		 787 if (CmdL.FileSize() <= 2)
788 {
f7f0d6c7 789 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6
AL		 790 if (I->GenPackages(Setup,Stats) == false)
791 _error->DumpErrors();
f7f0d6c7 792 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6
AL		 793 if (I->GenSources(Setup,SrcStats) == false)
794 _error->DumpErrors();
795 }
796 else
797 {
798 // Make a choice list out of the package list..
799 RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
800 RxChoiceList *End = List;
f7f0d6c7 801 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6 802 {
d935bb17 803 End->UserData = &(*I);
b2e465d6
AL		 804 End->Str = I->BaseDir.c_str();
805 End++;
806
d935bb17 807 End->UserData = &(*I);
b2e465d6
AL		 808 End->Str = I->Tag.c_str();
809 End++;
810 }
811 End->Str = 0;
812
813 // Regex it
814 if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
815 {
816 delete [] List;
dc738e7a 817 return _error->Error(_("No selections matched"));
b2e465d6
AL		 818 }
819 _error->DumpErrors();
820
821 // Do the generation for Packages
258b9e51 822 for (End = List; End->Str != 0; ++End)
b2e465d6
AL		 823 {
824 if (End->Hit == false)
825 continue;
826
258b9e51 827 PackageMap * const I = static_cast<PackageMap *>(End->UserData);
b2e465d6
AL		 828 if (I->PkgDone == true)
829 continue;
830 if (I->GenPackages(Setup,Stats) == false)
831 _error->DumpErrors();
832 }
833
834 // Do the generation for Sources
258b9e51 835 for (End = List; End->Str != 0; ++End)
b2e465d6
AL		 836 {
837 if (End->Hit == false)
838 continue;
839
258b9e51 840 PackageMap * const I = static_cast<PackageMap *>(End->UserData);
b2e465d6
AL		 841 if (I->SrcDone == true)
842 continue;
843 if (I->GenSources(Setup,SrcStats) == false)
844 _error->DumpErrors();
845 }
846
847 delete [] List;
848 }
308b6fc9
DJL		 849 return true;
850}
851
852 /*}}}*/
853// DoGenerateContents - Helper for Generate to generate the Contents /*{{{*/
854// ---------------------------------------------------------------------
855static bool DoGenerateContents(Configuration &Setup,
856 vector<PackageMap> &PkgList,
857 CommandLine &CmdL)
858{
c6474fb6 859 c1out << "Packages done, Starting contents." << endl;
b2e465d6
AL		 860
861 // Sort the contents file list by date
862 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
f7f0d6c7 863 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6
AL		 864 {
865 struct stat A;
866 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
867 I->CntCompress,A) == false)
868 time(&I->ContentsMTime);
869 else
870 I->ContentsMTime = A.st_mtime;
871 }
872 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
873
874 /* Now for Contents.. The process here is to do a make-like dependency
875 check. Each contents file is verified to be newer than the package files
876 that describe the debs it indexes. Since the package files contain
877 hashes of the .debs this means they have not changed either so the
878 contents must be up to date. */
adfa4116
JAK		 879 unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",
880 std::numeric_limits<unsigned int>::max())*1024;
f7f0d6c7 881 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6 882 {
1e3f4083 883 // This record is not relevant
b2e465d6
AL		 884 if (I->ContentsDone == true ||
885 I->Contents.empty() == true)
886 continue;
887
888 // Do not do everything if the user specified sections.
889 if (CmdL.FileSize() > 2 && I->PkgDone == false)
890 continue;
891
892 struct stat A,B;
893 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
894 {
895 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
896 {
dc738e7a 897 _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
b2e465d6
AL		 898 continue;
899 }
900
901 if (A.st_mtime > B.st_mtime)
902 continue;
903 }
904
905 if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
906 MaxContentsChange) == false)
907 _error->DumpErrors();
908
909 // Hit the limit?
910 if (MaxContentsChange == 0)
911 {
912 c1out << "Hit contents update byte limit" << endl;
913 break;
914 }
915 }
308b6fc9
DJL		 916
917 return true;
918}
919
920 /*}}}*/
921// Generate - Full generate, using a config file /*{{{*/
922// ---------------------------------------------------------------------
923/* */
924static bool Generate(CommandLine &CmdL)
925{
926 struct CacheDB::Stats SrcStats;
927 if (CmdL.FileSize() < 2)
8561c2fe 928 return ShowHelp(CmdL);
308b6fc9
DJL		 929
930 struct timeval StartTime;
931 gettimeofday(&StartTime,0);
932 struct CacheDB::Stats Stats;
b2e465d6 933
308b6fc9
DJL		 934 // Read the configuration file.
935 Configuration Setup;
936 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
937 return false;
938
939 vector<PackageMap> PkgList;
3d8232bf
DK		 940 std::vector<TranslationWriter*> TransList;
941 LoadTree(PkgList, TransList, Setup);
308b6fc9
DJL		 942 LoadBinDir(PkgList,Setup);
943
944 // Sort by cache DB to improve IO locality.
945 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
946 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
947
948 // Generate packages
949 if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false)
950 {
951 if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false)
3d8232bf
DK		 952 {
953 UnloadTree(TransList);
308b6fc9 954 return false;
3d8232bf 955 }
308b6fc9
DJL		 956 } else {
957 c1out << "Skipping Packages/Sources generation" << endl;
958 }
959
960 // do Contents if needed
961 if (_config->FindB("APT::FTPArchive::Contents", true) == true)
962 if (DoGenerateContents(Setup, PkgList, CmdL) == false)
3d8232bf
DK		 963 {
964 UnloadTree(TransList);
965 return false;
966 }
308b6fc9 967
b2e465d6 968 struct timeval NewTime;
308b6fc9
DJL		 969 gettimeofday(&NewTime,0);
970 double Delta = NewTime.tv_sec - StartTime.tv_sec +
b2e465d6 971 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
308b6fc9 972 c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
b2e465d6 973 << " archives. Took " << TimeToStr((long)Delta) << endl;
308b6fc9 974
3d8232bf 975 UnloadTree(TransList);
b2e465d6
AL		 976 return true;
977}
308b6fc9
DJL		 978
979 /*}}}*/
b2e465d6
AL		 980// Clean - Clean out the databases /*{{{*/
981// ---------------------------------------------------------------------
982/* */
c3ccac92 983static bool Clean(CommandLine &CmdL)
b2e465d6
AL		 984{
985 if (CmdL.FileSize() != 2)
8561c2fe 986 return ShowHelp(CmdL);
b2e465d6
AL		 987
988 // Read the configuration file.
989 Configuration Setup;
990 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
991 return false;
3d8232bf
DK		 992 // we don't need translation creation here
993 Setup.Set("TreeDefault::Translation", "/dev/null");
b2e465d6
AL		 994
995 vector<PackageMap> PkgList;
3d8232bf
DK		 996 std::vector<TranslationWriter*> TransList;
997 LoadTree(PkgList, TransList, Setup);
b2e465d6
AL		 998 LoadBinDir(PkgList,Setup);
999
1000 // Sort by cache DB to improve IO locality.
1001 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
53ba4e2c 1002 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
b2e465d6
AL		 1003
1004 string CacheDir = Setup.FindDir("Dir::CacheDir");
1005
d935bb17 1006 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
b2e465d6 1007 {
53ba4e2c
MV		 1008 if(I->BinCacheDB != "")
1009 c0out << I->BinCacheDB << endl;
1010 if(I->SrcCacheDB != "")
1011 c0out << I->SrcCacheDB << endl;
b2e465d6 1012 CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
53ba4e2c 1013 CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));
b2e465d6
AL		 1014 if (DB.Clean() == false)
1015 _error->DumpErrors();
53ba4e2c
MV		 1016 if (DB_SRC.Clean() == false)
1017 _error->DumpErrors();
3d8232bf 1018
b2e465d6 1019 string CacheDB = I->BinCacheDB;
53ba4e2c
MV		 1020 string SrcCacheDB = I->SrcCacheDB;
1021 while(I != PkgList.end() &&
1022 I->BinCacheDB == CacheDB &&
1023 I->SrcCacheDB == SrcCacheDB)
1024 ++I;
b2e465d6 1025 }
ce928105 1026
53ba4e2c 1027
b2e465d6
AL		 1028 return true;
1029}
1030 /*}}}*/
1031
f6777222 1032static std::vector<aptDispatchWithHelp> GetCommands() /*{{{*/
b2e465d6 1033{
011188e3 1034 return {
cbbee23e
DK		 1035 {"packages",&SimpleGenPackages, nullptr},
1036 {"contents",&SimpleGenContents, nullptr},
1037 {"sources",&SimpleGenSources, nullptr},
1038 {"release",&SimpleGenRelease, nullptr},
1039 {"generate",&Generate, nullptr},
1040 {"clean",&Clean, nullptr},
1041 {nullptr, nullptr, nullptr}
1042 };
011188e3
DK		 1043}
1044 /*}}}*/
1045int main(int argc, const char *argv[]) /*{{{*/
1046{
1047 InitLocale();
b2e465d6
AL		 1048
1049 // Parse the command line and initialize the package library
e7e10e47 1050 CommandLine CmdL;
90986d4d 1051 auto const Cmds = ParseCommandLine(CmdL, APT_CMD::APT_FTPARCHIVE, &_config, NULL, argc, argv, ShowHelp, &GetCommands);
ad7e0941 1052
d9e518c6 1053 _config->CndSet("quiet",0);
b2e465d6 1054 Quiet = _config->FindI("quiet",0);
d9e518c6
DK		 1055 InitOutput(clog.rdbuf());
1056
e7e10e47 1057 return DispatchCommandLine(CmdL, Cmds);
b2e465d6 1058}
011188e3 1059 /*}}}*/
APT (for Cydia)