]>
Commit | Line | Data |
---|---|---|
1 | // -*- mode: cpp; mode: fold -*- | |
2 | // Description /*{{{*/ | |
3 | // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $ | |
4 | /* ###################################################################### | |
5 | ||
6 | apt-ftparchive - Efficient work-alike for dpkg-scanpackages | |
7 | ||
8 | Let contents be disabled from the conf | |
9 | ||
10 | ##################################################################### */ | |
11 | /*}}}*/ | |
12 | // Include Files /*{{{*/ | |
13 | #include <config.h> | |
14 | ||
15 | #include <apt-pkg/error.h> | |
16 | #include <apt-pkg/configuration.h> | |
17 | #include <apt-pkg/cmndline.h> | |
18 | #include <apt-pkg/strutl.h> | |
19 | #include <apt-pkg/init.h> | |
20 | #include <algorithm> | |
21 | ||
22 | #include <climits> | |
23 | #include <sys/time.h> | |
24 | #include <regex.h> | |
25 | ||
26 | #include "apt-ftparchive.h" | |
27 | #include "contents.h" | |
28 | #include "multicompress.h" | |
29 | #include "writer.h" | |
30 | ||
31 | #include <apti18n.h> | |
32 | /*}}}*/ | |
33 | ||
34 | using namespace std; | |
35 | ostream c0out(0); | |
36 | ostream c1out(0); | |
37 | ostream c2out(0); | |
38 | ofstream devnull("/dev/null"); | |
39 | unsigned Quiet = 0; | |
40 | ||
41 | // struct PackageMap - List of all package files in the config file /*{{{*/ | |
42 | // --------------------------------------------------------------------- | |
43 | /* */ | |
44 | struct PackageMap | |
45 | { | |
46 | // General Stuff | |
47 | string BaseDir; | |
48 | string InternalPrefix; | |
49 | string FLFile; | |
50 | string PkgExt; | |
51 | string SrcExt; | |
52 | ||
53 | // Stuff for the Package File | |
54 | string PkgFile; | |
55 | string BinCacheDB; | |
56 | string BinOverride; | |
57 | string ExtraOverride; | |
58 | ||
59 | // We generate for this given arch | |
60 | string Arch; | |
61 | ||
62 | // Stuff for the Source File | |
63 | string SrcFile; | |
64 | string SrcOverride; | |
65 | string SrcExtraOverride; | |
66 | ||
67 | // Translation master file | |
68 | bool LongDesc; | |
69 | TranslationWriter *TransWriter; | |
70 | ||
71 | // Contents | |
72 | string Contents; | |
73 | string ContentsHead; | |
74 | ||
75 | // Random things | |
76 | string Tag; | |
77 | string PkgCompress; | |
78 | string CntCompress; | |
79 | string SrcCompress; | |
80 | string PathPrefix; | |
81 | unsigned int DeLinkLimit; | |
82 | mode_t Permissions; | |
83 | ||
84 | bool ContentsDone; | |
85 | bool PkgDone; | |
86 | bool SrcDone; | |
87 | time_t ContentsMTime; | |
88 | ||
89 | struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool> | |
90 | { | |
91 | inline bool operator() (const PackageMap &x,const PackageMap &y) | |
92 | {return x.ContentsMTime < y.ContentsMTime;}; | |
93 | }; | |
94 | ||
95 | struct DBCompare : public binary_function<PackageMap,PackageMap,bool> | |
96 | { | |
97 | inline bool operator() (const PackageMap &x,const PackageMap &y) | |
98 | {return x.BinCacheDB < y.BinCacheDB;}; | |
99 | }; | |
100 | ||
101 | void GetGeneral(Configuration &Setup,Configuration &Block); | |
102 | bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats); | |
103 | bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats); | |
104 | bool GenContents(Configuration &Setup, | |
105 | vector<PackageMap>::iterator Begin, | |
106 | vector<PackageMap>::iterator End, | |
107 | unsigned long &Left); | |
108 | ||
109 | PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1), | |
110 | ContentsDone(false), PkgDone(false), SrcDone(false), | |
111 | ContentsMTime(0) {}; | |
112 | }; | |
113 | /*}}}*/ | |
114 | ||
115 | // PackageMap::GetGeneral - Common per-section definitions /*{{{*/ | |
116 | // --------------------------------------------------------------------- | |
117 | /* */ | |
118 | void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block) | |
119 | { | |
120 | PathPrefix = Block.Find("PathPrefix"); | |
121 | ||
122 | if (Block.FindB("External-Links",true) == false) | |
123 | DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX); | |
124 | else | |
125 | DeLinkLimit = 0; | |
126 | ||
127 | PkgCompress = Block.Find("Packages::Compress", | |
128 | Setup.Find("Default::Packages::Compress",". gzip").c_str()); | |
129 | CntCompress = Block.Find("Contents::Compress", | |
130 | Setup.Find("Default::Contents::Compress",". gzip").c_str()); | |
131 | SrcCompress = Block.Find("Sources::Compress", | |
132 | Setup.Find("Default::Sources::Compress",". gzip").c_str()); | |
133 | ||
134 | SrcExt = Block.Find("Sources::Extensions", | |
135 | Setup.Find("Default::Sources::Extensions",".dsc").c_str()); | |
136 | PkgExt = Block.Find("Packages::Extensions", | |
137 | Setup.Find("Default::Packages::Extensions",".deb").c_str()); | |
138 | ||
139 | Permissions = Setup.FindI("Default::FileMode",0644); | |
140 | ||
141 | if (FLFile.empty() == false) | |
142 | FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile); | |
143 | ||
144 | if (Contents == " ") | |
145 | Contents= string(); | |
146 | } | |
147 | /*}}}*/ | |
148 | // PackageMap::GenPackages - Actually generate a Package file /*{{{*/ | |
149 | // --------------------------------------------------------------------- | |
150 | /* This generates the Package File described by this object. */ | |
151 | bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats) | |
152 | { | |
153 | if (PkgFile.empty() == true) | |
154 | return true; | |
155 | ||
156 | string ArchiveDir = Setup.FindDir("Dir::ArchiveDir"); | |
157 | string OverrideDir = Setup.FindDir("Dir::OverrideDir"); | |
158 | string CacheDir = Setup.FindDir("Dir::CacheDir"); | |
159 | ||
160 | struct timeval StartTime; | |
161 | gettimeofday(&StartTime,0); | |
162 | ||
163 | PkgDone = true; | |
164 | ||
165 | // Create a package writer object. | |
166 | PackagesWriter Packages(flCombine(CacheDir,BinCacheDB), | |
167 | flCombine(OverrideDir,BinOverride), | |
168 | flCombine(OverrideDir,ExtraOverride), | |
169 | Arch); | |
170 | if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false) | |
171 | return _error->Error(_("Package extension list is too long")); | |
172 | if (_error->PendingError() == true) | |
173 | return _error->Error(_("Error processing directory %s"),BaseDir.c_str()); | |
174 | ||
175 | Packages.PathPrefix = PathPrefix; | |
176 | Packages.DirStrip = ArchiveDir; | |
177 | Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix); | |
178 | ||
179 | Packages.TransWriter = TransWriter; | |
180 | Packages.LongDescription = LongDesc; | |
181 | ||
182 | Packages.Stats.DeLinkBytes = Stats.DeLinkBytes; | |
183 | Packages.DeLinkLimit = DeLinkLimit; | |
184 | ||
185 | // Create a compressor object | |
186 | MultiCompress Comp(flCombine(ArchiveDir,PkgFile), | |
187 | PkgCompress,Permissions); | |
188 | Packages.Output = Comp.Input; | |
189 | if (_error->PendingError() == true) | |
190 | return _error->Error(_("Error processing directory %s"),BaseDir.c_str()); | |
191 | ||
192 | c0out << ' ' << BaseDir << ":" << flush; | |
193 | ||
194 | // Do recursive directory searching | |
195 | if (FLFile.empty() == true) | |
196 | { | |
197 | if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false) | |
198 | return false; | |
199 | } | |
200 | else | |
201 | { | |
202 | if (Packages.LoadFileList(ArchiveDir,FLFile) == false) | |
203 | return false; | |
204 | } | |
205 | ||
206 | Packages.Output = 0; // Just in case | |
207 | ||
208 | // Finish compressing | |
209 | unsigned long long Size; | |
210 | if (Comp.Finalize(Size) == false) | |
211 | { | |
212 | c0out << endl; | |
213 | return _error->Error(_("Error processing directory %s"),BaseDir.c_str()); | |
214 | } | |
215 | ||
216 | if (Size != 0) | |
217 | c0out << " New " | |
218 | << SizeToStr(Size) << "B "; | |
219 | else | |
220 | c0out << ' '; | |
221 | ||
222 | struct timeval NewTime; | |
223 | gettimeofday(&NewTime,0); | |
224 | double Delta = NewTime.tv_sec - StartTime.tv_sec + | |
225 | (NewTime.tv_usec - StartTime.tv_usec)/1000000.0; | |
226 | ||
227 | c0out << Packages.Stats.Packages << " files " << | |
228 | /* SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */ | |
229 | SizeToStr(Packages.Stats.Bytes) << "B " << | |
230 | TimeToStr((long)Delta) << endl; | |
231 | ||
232 | Stats.Add(Packages.Stats); | |
233 | Stats.DeLinkBytes = Packages.Stats.DeLinkBytes; | |
234 | ||
235 | return !_error->PendingError(); | |
236 | } | |
237 | ||
238 | /*}}}*/ | |
239 | // PackageMap::GenSources - Actually generate a Source file /*{{{*/ | |
240 | // --------------------------------------------------------------------- | |
241 | /* This generates the Sources File described by this object. */ | |
242 | bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats) | |
243 | { | |
244 | if (SrcFile.empty() == true) | |
245 | return true; | |
246 | ||
247 | string ArchiveDir = Setup.FindDir("Dir::ArchiveDir"); | |
248 | string OverrideDir = Setup.FindDir("Dir::OverrideDir"); | |
249 | string CacheDir = Setup.FindDir("Dir::CacheDir"); | |
250 | ||
251 | struct timeval StartTime; | |
252 | gettimeofday(&StartTime,0); | |
253 | ||
254 | SrcDone = true; | |
255 | ||
256 | // Create a package writer object. | |
257 | SourcesWriter Sources(flCombine(OverrideDir,BinOverride), | |
258 | flCombine(OverrideDir,SrcOverride), | |
259 | flCombine(OverrideDir,SrcExtraOverride)); | |
260 | if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false) | |
261 | return _error->Error(_("Source extension list is too long")); | |
262 | if (_error->PendingError() == true) | |
263 | return _error->Error(_("Error processing directory %s"),BaseDir.c_str()); | |
264 | ||
265 | Sources.PathPrefix = PathPrefix; | |
266 | Sources.DirStrip = ArchiveDir; | |
267 | Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix); | |
268 | ||
269 | Sources.DeLinkLimit = DeLinkLimit; | |
270 | Sources.Stats.DeLinkBytes = Stats.DeLinkBytes; | |
271 | ||
272 | // Create a compressor object | |
273 | MultiCompress Comp(flCombine(ArchiveDir,SrcFile), | |
274 | SrcCompress,Permissions); | |
275 | Sources.Output = Comp.Input; | |
276 | if (_error->PendingError() == true) | |
277 | return _error->Error(_("Error processing directory %s"),BaseDir.c_str()); | |
278 | ||
279 | c0out << ' ' << BaseDir << ":" << flush; | |
280 | ||
281 | // Do recursive directory searching | |
282 | if (FLFile.empty() == true) | |
283 | { | |
284 | if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false) | |
285 | return false; | |
286 | } | |
287 | else | |
288 | { | |
289 | if (Sources.LoadFileList(ArchiveDir,FLFile) == false) | |
290 | return false; | |
291 | } | |
292 | Sources.Output = 0; // Just in case | |
293 | ||
294 | // Finish compressing | |
295 | unsigned long long Size; | |
296 | if (Comp.Finalize(Size) == false) | |
297 | { | |
298 | c0out << endl; | |
299 | return _error->Error(_("Error processing directory %s"),BaseDir.c_str()); | |
300 | } | |
301 | ||
302 | if (Size != 0) | |
303 | c0out << " New " | |
304 | << SizeToStr(Size) << "B "; | |
305 | else | |
306 | c0out << ' '; | |
307 | ||
308 | struct timeval NewTime; | |
309 | gettimeofday(&NewTime,0); | |
310 | double Delta = NewTime.tv_sec - StartTime.tv_sec + | |
311 | (NewTime.tv_usec - StartTime.tv_usec)/1000000.0; | |
312 | ||
313 | c0out << Sources.Stats.Packages << " pkgs in " << | |
314 | TimeToStr((long)Delta) << endl; | |
315 | ||
316 | Stats.Add(Sources.Stats); | |
317 | Stats.DeLinkBytes = Sources.Stats.DeLinkBytes; | |
318 | ||
319 | return !_error->PendingError(); | |
320 | } | |
321 | /*}}}*/ | |
322 | // PackageMap::GenContents - Actually generate a Contents file /*{{{*/ | |
323 | // --------------------------------------------------------------------- | |
324 | /* This generates the contents file partially described by this object. | |
325 | It searches the given iterator range for other package files that map | |
326 | into this contents file and includes their data as well when building. */ | |
327 | bool PackageMap::GenContents(Configuration &Setup, | |
328 | vector<PackageMap>::iterator Begin, | |
329 | vector<PackageMap>::iterator End, | |
330 | unsigned long &Left) | |
331 | { | |
332 | if (Contents.empty() == true) | |
333 | return true; | |
334 | ||
335 | if (Left == 0) | |
336 | return true; | |
337 | ||
338 | string ArchiveDir = Setup.FindDir("Dir::ArchiveDir"); | |
339 | string CacheDir = Setup.FindDir("Dir::CacheDir"); | |
340 | string OverrideDir = Setup.FindDir("Dir::OverrideDir"); | |
341 | ||
342 | struct timeval StartTime; | |
343 | gettimeofday(&StartTime,0); | |
344 | ||
345 | // Create a package writer object. | |
346 | ContentsWriter Contents("", Arch); | |
347 | if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false) | |
348 | return _error->Error(_("Package extension list is too long")); | |
349 | if (_error->PendingError() == true) | |
350 | return false; | |
351 | ||
352 | MultiCompress Comp(flCombine(ArchiveDir,this->Contents), | |
353 | CntCompress,Permissions); | |
354 | Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60; | |
355 | Contents.Output = Comp.Input; | |
356 | if (_error->PendingError() == true) | |
357 | return false; | |
358 | ||
359 | // Write the header out. | |
360 | if (ContentsHead.empty() == false) | |
361 | { | |
362 | FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly); | |
363 | if (_error->PendingError() == true) | |
364 | return false; | |
365 | ||
366 | unsigned long long Size = Head.Size(); | |
367 | unsigned char Buf[4096]; | |
368 | while (Size != 0) | |
369 | { | |
370 | unsigned long long ToRead = Size; | |
371 | if (Size > sizeof(Buf)) | |
372 | ToRead = sizeof(Buf); | |
373 | ||
374 | if (Head.Read(Buf,ToRead) == false) | |
375 | return false; | |
376 | ||
377 | if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead) | |
378 | return _error->Errno("fwrite",_("Error writing header to contents file")); | |
379 | ||
380 | Size -= ToRead; | |
381 | } | |
382 | } | |
383 | ||
384 | /* Go over all the package file records and parse all the package | |
385 | files associated with this contents file into one great big honking | |
386 | memory structure, then dump the sorted version */ | |
387 | c0out << ' ' << this->Contents << ":" << flush; | |
388 | for (vector<PackageMap>::iterator I = Begin; I != End; ++I) | |
389 | { | |
390 | if (I->Contents != this->Contents) | |
391 | continue; | |
392 | ||
393 | Contents.Prefix = ArchiveDir; | |
394 | Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB)); | |
395 | Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile), | |
396 | I->PkgCompress); | |
397 | ||
398 | I->ContentsDone = true; | |
399 | } | |
400 | ||
401 | Contents.Finish(); | |
402 | ||
403 | // Finish compressing | |
404 | unsigned long long Size; | |
405 | if (Comp.Finalize(Size) == false || _error->PendingError() == true) | |
406 | { | |
407 | c0out << endl; | |
408 | return _error->Error(_("Error processing contents %s"), | |
409 | this->Contents.c_str()); | |
410 | } | |
411 | ||
412 | if (Size != 0) | |
413 | { | |
414 | c0out << " New " << SizeToStr(Size) << "B "; | |
415 | if (Left > Size) | |
416 | Left -= Size; | |
417 | else | |
418 | Left = 0; | |
419 | } | |
420 | else | |
421 | c0out << ' '; | |
422 | ||
423 | struct timeval NewTime; | |
424 | gettimeofday(&NewTime,0); | |
425 | double Delta = NewTime.tv_sec - StartTime.tv_sec + | |
426 | (NewTime.tv_usec - StartTime.tv_usec)/1000000.0; | |
427 | ||
428 | c0out << Contents.Stats.Packages << " files " << | |
429 | SizeToStr(Contents.Stats.Bytes) << "B " << | |
430 | TimeToStr((long)Delta) << endl; | |
431 | ||
432 | return true; | |
433 | } | |
434 | /*}}}*/ | |
435 | ||
436 | // LoadTree - Load a 'tree' section from the Generate Config /*{{{*/ | |
437 | // --------------------------------------------------------------------- | |
438 | /* This populates the PkgList with all the possible permutations of the | |
439 | section/arch lists. */ | |
440 | void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup) | |
441 | { | |
442 | // Load the defaults | |
443 | string DDir = Setup.Find("TreeDefault::Directory", | |
444 | "$(DIST)/$(SECTION)/binary-$(ARCH)/"); | |
445 | string DSDir = Setup.Find("TreeDefault::SrcDirectory", | |
446 | "$(DIST)/$(SECTION)/source/"); | |
447 | string DPkg = Setup.Find("TreeDefault::Packages", | |
448 | "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages"); | |
449 | string DTrans = Setup.Find("TreeDefault::Translation", | |
450 | "$(DIST)/$(SECTION)/i18n/Translation-en"); | |
451 | string DIPrfx = Setup.Find("TreeDefault::InternalPrefix", | |
452 | "$(DIST)/$(SECTION)/"); | |
453 | string DContents = Setup.Find("TreeDefault::Contents", | |
454 | "$(DIST)/$(SECTION)/Contents-$(ARCH)"); | |
455 | string DContentsH = Setup.Find("TreeDefault::Contents::Header",""); | |
456 | string DBCache = Setup.Find("TreeDefault::BinCacheDB", | |
457 | "packages-$(ARCH).db"); | |
458 | string DSources = Setup.Find("TreeDefault::Sources", | |
459 | "$(DIST)/$(SECTION)/source/Sources"); | |
460 | string DFLFile = Setup.Find("TreeDefault::FileList", ""); | |
461 | string DSFLFile = Setup.Find("TreeDefault::SourceFileList", ""); | |
462 | ||
463 | mode_t const Permissions = Setup.FindI("Default::FileMode",0644); | |
464 | ||
465 | bool const LongDescription = Setup.FindB("Default::LongDescription", | |
466 | _config->FindB("APT::FTPArchive::LongDescription", true)); | |
467 | string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str(); | |
468 | ||
469 | // Process 'tree' type sections | |
470 | const Configuration::Item *Top = Setup.Tree("tree"); | |
471 | for (Top = (Top == 0?0:Top->Child); Top != 0;) | |
472 | { | |
473 | Configuration Block(Top); | |
474 | string Dist = Top->Tag; | |
475 | ||
476 | // Parse the sections | |
477 | string Tmp = Block.Find("Sections"); | |
478 | const char *Sections = Tmp.c_str(); | |
479 | string Section; | |
480 | while (ParseQuoteWord(Sections,Section) == true) | |
481 | { | |
482 | string Arch; | |
483 | struct SubstVar const Vars[] = {{"$(DIST)",&Dist}, | |
484 | {"$(SECTION)",&Section}, | |
485 | {"$(ARCH)",&Arch}, | |
486 | {}}; | |
487 | mode_t const Perms = Block.FindI("FileMode", Permissions); | |
488 | bool const LongDesc = Block.FindB("LongDescription", LongDescription); | |
489 | TranslationWriter *TransWriter; | |
490 | if (DTrans.empty() == false && LongDesc == false) | |
491 | { | |
492 | string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"), | |
493 | SubstVar(Block.Find("Translation", DTrans.c_str()), Vars)); | |
494 | string const TransCompress = Block.Find("Translation::Compress", TranslationCompress); | |
495 | TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms); | |
496 | } | |
497 | else | |
498 | TransWriter = NULL; | |
499 | ||
500 | string const Tmp2 = Block.Find("Architectures"); | |
501 | const char *Archs = Tmp2.c_str(); | |
502 | while (ParseQuoteWord(Archs,Arch) == true) | |
503 | { | |
504 | PackageMap Itm; | |
505 | Itm.Permissions = Perms; | |
506 | Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars); | |
507 | Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars); | |
508 | ||
509 | if (stringcasecmp(Arch,"source") == 0) | |
510 | { | |
511 | Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars); | |
512 | Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars); | |
513 | Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars); | |
514 | Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars); | |
515 | Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars); | |
516 | Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars); | |
517 | } | |
518 | else | |
519 | { | |
520 | Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars); | |
521 | Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars); | |
522 | Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars); | |
523 | Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars); | |
524 | Itm.Arch = Arch; | |
525 | Itm.LongDesc = LongDesc; | |
526 | if (TransWriter != NULL) | |
527 | { | |
528 | TransWriter->IncreaseRefCounter(); | |
529 | Itm.TransWriter = TransWriter; | |
530 | } | |
531 | Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars); | |
532 | Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars); | |
533 | Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars); | |
534 | Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars); | |
535 | } | |
536 | ||
537 | Itm.GetGeneral(Setup,Block); | |
538 | PkgList.push_back(Itm); | |
539 | } | |
540 | // we didn't use this TransWriter, so we can release it | |
541 | if (TransWriter != NULL && TransWriter->GetRefCounter() == 0) | |
542 | delete TransWriter; | |
543 | } | |
544 | ||
545 | Top = Top->Next; | |
546 | } | |
547 | } | |
548 | /*}}}*/ | |
549 | // LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/ | |
550 | // --------------------------------------------------------------------- | |
551 | /* */ | |
552 | void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup) | |
553 | { | |
554 | mode_t const Permissions = Setup.FindI("Default::FileMode",0644); | |
555 | ||
556 | // Process 'bindirectory' type sections | |
557 | const Configuration::Item *Top = Setup.Tree("bindirectory"); | |
558 | for (Top = (Top == 0?0:Top->Child); Top != 0;) | |
559 | { | |
560 | Configuration Block(Top); | |
561 | ||
562 | PackageMap Itm; | |
563 | Itm.PkgFile = Block.Find("Packages"); | |
564 | Itm.SrcFile = Block.Find("Sources"); | |
565 | Itm.BinCacheDB = Block.Find("BinCacheDB"); | |
566 | Itm.BinOverride = Block.Find("BinOverride"); | |
567 | Itm.ExtraOverride = Block.Find("ExtraOverride"); | |
568 | Itm.SrcExtraOverride = Block.Find("SrcExtraOverride"); | |
569 | Itm.SrcOverride = Block.Find("SrcOverride"); | |
570 | Itm.BaseDir = Top->Tag; | |
571 | Itm.FLFile = Block.Find("FileList"); | |
572 | Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str()); | |
573 | Itm.Contents = Block.Find("Contents"); | |
574 | Itm.ContentsHead = Block.Find("Contents::Header"); | |
575 | Itm.Permissions = Block.FindI("FileMode", Permissions); | |
576 | ||
577 | Itm.GetGeneral(Setup,Block); | |
578 | PkgList.push_back(Itm); | |
579 | ||
580 | Top = Top->Next; | |
581 | } | |
582 | } | |
583 | /*}}}*/ | |
584 | ||
585 | // ShowHelp - Show the help text /*{{{*/ | |
586 | // --------------------------------------------------------------------- | |
587 | /* */ | |
588 | bool ShowHelp(CommandLine &CmdL) | |
589 | { | |
590 | ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION, | |
591 | COMMON_ARCH,__DATE__,__TIME__); | |
592 | if (_config->FindB("version") == true) | |
593 | return true; | |
594 | ||
595 | cout << | |
596 | _("Usage: apt-ftparchive [options] command\n" | |
597 | "Commands: packages binarypath [overridefile [pathprefix]]\n" | |
598 | " sources srcpath [overridefile [pathprefix]]\n" | |
599 | " contents path\n" | |
600 | " release path\n" | |
601 | " generate config [groups]\n" | |
602 | " clean config\n" | |
603 | "\n" | |
604 | "apt-ftparchive generates index files for Debian archives. It supports\n" | |
605 | "many styles of generation from fully automated to functional replacements\n" | |
606 | "for dpkg-scanpackages and dpkg-scansources\n" | |
607 | "\n" | |
608 | "apt-ftparchive generates Package files from a tree of .debs. The\n" | |
609 | "Package file contains the contents of all the control fields from\n" | |
610 | "each package as well as the MD5 hash and filesize. An override file\n" | |
611 | "is supported to force the value of Priority and Section.\n" | |
612 | "\n" | |
613 | "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n" | |
614 | "The --source-override option can be used to specify a src override file\n" | |
615 | "\n" | |
616 | "The 'packages' and 'sources' command should be run in the root of the\n" | |
617 | "tree. BinaryPath should point to the base of the recursive search and \n" | |
618 | "override file should contain the override flags. Pathprefix is\n" | |
619 | "appended to the filename fields if present. Example usage from the \n" | |
620 | "Debian archive:\n" | |
621 | " apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n" | |
622 | " dists/potato/main/binary-i386/Packages\n" | |
623 | "\n" | |
624 | "Options:\n" | |
625 | " -h This help text\n" | |
626 | " --md5 Control MD5 generation\n" | |
627 | " -s=? Source override file\n" | |
628 | " -q Quiet\n" | |
629 | " -d=? Select the optional caching database\n" | |
630 | " --no-delink Enable delinking debug mode\n" | |
631 | " --contents Control contents file generation\n" | |
632 | " -c=? Read this configuration file\n" | |
633 | " -o=? Set an arbitrary configuration option") << endl; | |
634 | ||
635 | return true; | |
636 | } | |
637 | /*}}}*/ | |
638 | // SimpleGenPackages - Generate a Packages file for a directory tree /*{{{*/ | |
639 | // --------------------------------------------------------------------- | |
640 | /* This emulates dpkg-scanpackages's command line interface. 'mostly' */ | |
641 | bool SimpleGenPackages(CommandLine &CmdL) | |
642 | { | |
643 | if (CmdL.FileSize() < 2) | |
644 | return ShowHelp(CmdL); | |
645 | ||
646 | string Override; | |
647 | if (CmdL.FileSize() >= 3) | |
648 | Override = CmdL.FileList[2]; | |
649 | ||
650 | // Create a package writer object. | |
651 | PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"), | |
652 | Override, "", _config->Find("APT::FTPArchive::Architecture")); | |
653 | if (_error->PendingError() == true) | |
654 | return false; | |
655 | ||
656 | if (CmdL.FileSize() >= 4) | |
657 | Packages.PathPrefix = CmdL.FileList[3]; | |
658 | ||
659 | // Do recursive directory searching | |
660 | if (Packages.RecursiveScan(CmdL.FileList[1]) == false) | |
661 | return false; | |
662 | ||
663 | return true; | |
664 | } | |
665 | /*}}}*/ | |
666 | // SimpleGenContents - Generate a Contents listing /*{{{*/ | |
667 | // --------------------------------------------------------------------- | |
668 | /* */ | |
669 | bool SimpleGenContents(CommandLine &CmdL) | |
670 | { | |
671 | if (CmdL.FileSize() < 2) | |
672 | return ShowHelp(CmdL); | |
673 | ||
674 | // Create a package writer object. | |
675 | ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture")); | |
676 | if (_error->PendingError() == true) | |
677 | return false; | |
678 | ||
679 | // Do recursive directory searching | |
680 | if (Contents.RecursiveScan(CmdL.FileList[1]) == false) | |
681 | return false; | |
682 | ||
683 | Contents.Finish(); | |
684 | ||
685 | return true; | |
686 | } | |
687 | /*}}}*/ | |
688 | // SimpleGenSources - Generate a Sources file for a directory tree /*{{{*/ | |
689 | // --------------------------------------------------------------------- | |
690 | /* This emulates dpkg-scanpackages's command line interface. 'mostly' */ | |
691 | bool SimpleGenSources(CommandLine &CmdL) | |
692 | { | |
693 | if (CmdL.FileSize() < 2) | |
694 | return ShowHelp(CmdL); | |
695 | ||
696 | string Override; | |
697 | if (CmdL.FileSize() >= 3) | |
698 | Override = CmdL.FileList[2]; | |
699 | ||
700 | string SOverride; | |
701 | if (Override.empty() == false) | |
702 | SOverride = Override + ".src"; | |
703 | ||
704 | SOverride = _config->Find("APT::FTPArchive::SourceOverride", | |
705 | SOverride.c_str()); | |
706 | ||
707 | // Create a package writer object. | |
708 | SourcesWriter Sources(Override,SOverride); | |
709 | if (_error->PendingError() == true) | |
710 | return false; | |
711 | ||
712 | if (CmdL.FileSize() >= 4) | |
713 | Sources.PathPrefix = CmdL.FileList[3]; | |
714 | ||
715 | // Do recursive directory searching | |
716 | if (Sources.RecursiveScan(CmdL.FileList[1]) == false) | |
717 | return false; | |
718 | ||
719 | return true; | |
720 | } | |
721 | /*}}}*/ | |
722 | // SimpleGenRelease - Generate a Release file for a directory tree /*{{{*/ | |
723 | // --------------------------------------------------------------------- | |
724 | bool SimpleGenRelease(CommandLine &CmdL) | |
725 | { | |
726 | if (CmdL.FileSize() < 2) | |
727 | return ShowHelp(CmdL); | |
728 | ||
729 | string Dir = CmdL.FileList[1]; | |
730 | ||
731 | ReleaseWriter Release(""); | |
732 | Release.DirStrip = Dir; | |
733 | ||
734 | if (_error->PendingError() == true) | |
735 | return false; | |
736 | ||
737 | if (Release.RecursiveScan(Dir) == false) | |
738 | return false; | |
739 | ||
740 | Release.Finish(); | |
741 | ||
742 | return true; | |
743 | } | |
744 | ||
745 | /*}}}*/ | |
746 | // Generate - Full generate, using a config file /*{{{*/ | |
747 | // --------------------------------------------------------------------- | |
748 | /* */ | |
749 | bool Generate(CommandLine &CmdL) | |
750 | { | |
751 | struct CacheDB::Stats SrcStats; | |
752 | if (CmdL.FileSize() < 2) | |
753 | return ShowHelp(CmdL); | |
754 | ||
755 | struct timeval StartTime; | |
756 | gettimeofday(&StartTime,0); | |
757 | struct CacheDB::Stats Stats; | |
758 | ||
759 | // Read the configuration file. | |
760 | Configuration Setup; | |
761 | if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false) | |
762 | return false; | |
763 | ||
764 | vector<PackageMap> PkgList; | |
765 | LoadTree(PkgList,Setup); | |
766 | LoadBinDir(PkgList,Setup); | |
767 | ||
768 | // Sort by cache DB to improve IO locality. | |
769 | stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare()); | |
770 | ||
771 | // Generate packages | |
772 | if (CmdL.FileSize() <= 2) | |
773 | { | |
774 | for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I) | |
775 | if (I->GenPackages(Setup,Stats) == false) | |
776 | _error->DumpErrors(); | |
777 | for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I) | |
778 | if (I->GenSources(Setup,SrcStats) == false) | |
779 | _error->DumpErrors(); | |
780 | } | |
781 | else | |
782 | { | |
783 | // Make a choice list out of the package list.. | |
784 | RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1]; | |
785 | RxChoiceList *End = List; | |
786 | for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I) | |
787 | { | |
788 | End->UserData = &(*I); | |
789 | End->Str = I->BaseDir.c_str(); | |
790 | End++; | |
791 | ||
792 | End->UserData = &(*I); | |
793 | End->Str = I->Tag.c_str(); | |
794 | End++; | |
795 | } | |
796 | End->Str = 0; | |
797 | ||
798 | // Regex it | |
799 | if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0) | |
800 | { | |
801 | delete [] List; | |
802 | return _error->Error(_("No selections matched")); | |
803 | } | |
804 | _error->DumpErrors(); | |
805 | ||
806 | // Do the generation for Packages | |
807 | for (End = List; End->Str != 0; End++) | |
808 | { | |
809 | if (End->Hit == false) | |
810 | continue; | |
811 | ||
812 | PackageMap *I = (PackageMap *)End->UserData; | |
813 | if (I->PkgDone == true) | |
814 | continue; | |
815 | if (I->GenPackages(Setup,Stats) == false) | |
816 | _error->DumpErrors(); | |
817 | } | |
818 | ||
819 | // Do the generation for Sources | |
820 | for (End = List; End->Str != 0; End++) | |
821 | { | |
822 | if (End->Hit == false) | |
823 | continue; | |
824 | ||
825 | PackageMap *I = (PackageMap *)End->UserData; | |
826 | if (I->SrcDone == true) | |
827 | continue; | |
828 | if (I->GenSources(Setup,SrcStats) == false) | |
829 | _error->DumpErrors(); | |
830 | } | |
831 | ||
832 | delete [] List; | |
833 | } | |
834 | ||
835 | // close the Translation master files | |
836 | for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I) | |
837 | if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0) | |
838 | delete I->TransWriter; | |
839 | ||
840 | if (_config->FindB("APT::FTPArchive::Contents",true) == false) | |
841 | return true; | |
842 | ||
843 | c1out << "Packages done, Starting contents." << endl; | |
844 | ||
845 | // Sort the contents file list by date | |
846 | string ArchiveDir = Setup.FindDir("Dir::ArchiveDir"); | |
847 | for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I) | |
848 | { | |
849 | struct stat A; | |
850 | if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents), | |
851 | I->CntCompress,A) == false) | |
852 | time(&I->ContentsMTime); | |
853 | else | |
854 | I->ContentsMTime = A.st_mtime; | |
855 | } | |
856 | stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare()); | |
857 | ||
858 | /* Now for Contents.. The process here is to do a make-like dependency | |
859 | check. Each contents file is verified to be newer than the package files | |
860 | that describe the debs it indexes. Since the package files contain | |
861 | hashes of the .debs this means they have not changed either so the | |
862 | contents must be up to date. */ | |
863 | unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024; | |
864 | for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I) | |
865 | { | |
866 | // This record is not relevent | |
867 | if (I->ContentsDone == true || | |
868 | I->Contents.empty() == true) | |
869 | continue; | |
870 | ||
871 | // Do not do everything if the user specified sections. | |
872 | if (CmdL.FileSize() > 2 && I->PkgDone == false) | |
873 | continue; | |
874 | ||
875 | struct stat A,B; | |
876 | if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true) | |
877 | { | |
878 | if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false) | |
879 | { | |
880 | _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str()); | |
881 | continue; | |
882 | } | |
883 | ||
884 | if (A.st_mtime > B.st_mtime) | |
885 | continue; | |
886 | } | |
887 | ||
888 | if (I->GenContents(Setup,PkgList.begin(),PkgList.end(), | |
889 | MaxContentsChange) == false) | |
890 | _error->DumpErrors(); | |
891 | ||
892 | // Hit the limit? | |
893 | if (MaxContentsChange == 0) | |
894 | { | |
895 | c1out << "Hit contents update byte limit" << endl; | |
896 | break; | |
897 | } | |
898 | } | |
899 | ||
900 | struct timeval NewTime; | |
901 | gettimeofday(&NewTime,0); | |
902 | double Delta = NewTime.tv_sec - StartTime.tv_sec + | |
903 | (NewTime.tv_usec - StartTime.tv_usec)/1000000.0; | |
904 | c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages | |
905 | << " archives. Took " << TimeToStr((long)Delta) << endl; | |
906 | ||
907 | return true; | |
908 | } | |
909 | /*}}}*/ | |
910 | // Clean - Clean out the databases /*{{{*/ | |
911 | // --------------------------------------------------------------------- | |
912 | /* */ | |
913 | bool Clean(CommandLine &CmdL) | |
914 | { | |
915 | if (CmdL.FileSize() != 2) | |
916 | return ShowHelp(CmdL); | |
917 | ||
918 | // Read the configuration file. | |
919 | Configuration Setup; | |
920 | if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false) | |
921 | return false; | |
922 | ||
923 | vector<PackageMap> PkgList; | |
924 | LoadTree(PkgList,Setup); | |
925 | LoadBinDir(PkgList,Setup); | |
926 | ||
927 | // Sort by cache DB to improve IO locality. | |
928 | stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare()); | |
929 | ||
930 | string CacheDir = Setup.FindDir("Dir::CacheDir"); | |
931 | ||
932 | for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ) | |
933 | { | |
934 | c0out << I->BinCacheDB << endl; | |
935 | CacheDB DB(flCombine(CacheDir,I->BinCacheDB)); | |
936 | if (DB.Clean() == false) | |
937 | _error->DumpErrors(); | |
938 | ||
939 | string CacheDB = I->BinCacheDB; | |
940 | for (; I != PkgList.end() && I->BinCacheDB == CacheDB; ++I); | |
941 | } | |
942 | ||
943 | return true; | |
944 | } | |
945 | /*}}}*/ | |
946 | ||
947 | int main(int argc, const char *argv[]) | |
948 | { | |
949 | setlocale(LC_ALL, ""); | |
950 | CommandLine::Args Args[] = { | |
951 | {'h',"help","help",0}, | |
952 | {0,"md5","APT::FTPArchive::MD5",0}, | |
953 | {0,"sha1","APT::FTPArchive::SHA1",0}, | |
954 | {0,"sha256","APT::FTPArchive::SHA256",0}, | |
955 | {'v',"version","version",0}, | |
956 | {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg}, | |
957 | {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg}, | |
958 | {'q',"quiet","quiet",CommandLine::IntLevel}, | |
959 | {'q',"silent","quiet",CommandLine::IntLevel}, | |
960 | {0,"delink","APT::FTPArchive::DeLinkAct",0}, | |
961 | {0,"readonly","APT::FTPArchive::ReadOnlyDB",0}, | |
962 | {0,"contents","APT::FTPArchive::Contents",0}, | |
963 | {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg}, | |
964 | {'c',"config-file",0,CommandLine::ConfigFile}, | |
965 | {'o',"option",0,CommandLine::ArbItem}, | |
966 | {0,0,0,0}}; | |
967 | CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages}, | |
968 | {"contents",&SimpleGenContents}, | |
969 | {"sources",&SimpleGenSources}, | |
970 | {"release",&SimpleGenRelease}, | |
971 | {"generate",&Generate}, | |
972 | {"clean",&Clean}, | |
973 | {"help",&ShowHelp}, | |
974 | {0,0}}; | |
975 | ||
976 | // Parse the command line and initialize the package library | |
977 | CommandLine CmdL(Args,_config); | |
978 | if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false) | |
979 | { | |
980 | _error->DumpErrors(); | |
981 | return 100; | |
982 | } | |
983 | ||
984 | // See if the help should be shown | |
985 | if (_config->FindB("help") == true || | |
986 | _config->FindB("version") == true || | |
987 | CmdL.FileSize() == 0) | |
988 | { | |
989 | ShowHelp(CmdL); | |
990 | return 0; | |
991 | } | |
992 | ||
993 | // Setup the output streams | |
994 | c0out.rdbuf(clog.rdbuf()); | |
995 | c1out.rdbuf(clog.rdbuf()); | |
996 | c2out.rdbuf(clog.rdbuf()); | |
997 | Quiet = _config->FindI("quiet",0); | |
998 | if (Quiet > 0) | |
999 | c0out.rdbuf(devnull.rdbuf()); | |
1000 | if (Quiet > 1) | |
1001 | c1out.rdbuf(devnull.rdbuf()); | |
1002 | ||
1003 | // Match the operation | |
1004 | CmdL.DispatchArg(Cmds); | |
1005 | ||
1006 | if (_error->empty() == false) | |
1007 | { | |
1008 | bool Errors = _error->PendingError(); | |
1009 | _error->DumpErrors(); | |
1010 | return Errors == true?100:0; | |
1011 | } | |
1012 | return 0; | |
1013 | } |