]>
git.saurik.com Git - apt.git/blob - ftparchive/writer.cc
d4b46042f81631a3232c4a96ee6fda8fc22e8470
1 // -*- mode: cpp; mode: fold -*-
3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
4 /* ######################################################################
8 The file writer classes. These write various types of output, sources,
11 ##################################################################### */
13 // Include Files /*{{{*/
16 #include <apt-pkg/configuration.h>
17 #include <apt-pkg/deblistparser.h>
18 #include <apt-pkg/error.h>
19 #include <apt-pkg/fileutl.h>
20 #include <apt-pkg/gpgv.h>
21 #include <apt-pkg/hashes.h>
22 #include <apt-pkg/md5.h>
23 #include <apt-pkg/strutl.h>
24 #include <apt-pkg/debfile.h>
25 #include <apt-pkg/pkgcache.h>
26 #include <apt-pkg/sha1.h>
27 #include <apt-pkg/sha2.h>
28 #include <apt-pkg/tagfile.h>
36 #include <sys/types.h>
45 #include "apt-ftparchive.h"
48 #include "multicompress.h"
54 FTWScanner
*FTWScanner::Owner
;
56 // ConfigToDoHashes - which hashes to generate /*{{{*/
57 static void SingleConfigToDoHashes(unsigned int &DoHashes
, std::string
const &Conf
, unsigned int const Flag
)
59 if (_config
->FindB(Conf
, (DoHashes
& Flag
) == Flag
) == true)
64 static void ConfigToDoHashes(unsigned int &DoHashes
, std::string
const &Conf
)
66 SingleConfigToDoHashes(DoHashes
, Conf
+ "::MD5", Hashes::MD5SUM
);
67 SingleConfigToDoHashes(DoHashes
, Conf
+ "::SHA1", Hashes::SHA1SUM
);
68 SingleConfigToDoHashes(DoHashes
, Conf
+ "::SHA256", Hashes::SHA256SUM
);
69 SingleConfigToDoHashes(DoHashes
, Conf
+ "::SHA512", Hashes::SHA512SUM
);
73 // FTWScanner::FTWScanner - Constructor /*{{{*/
74 FTWScanner::FTWScanner(FileFd
* const GivenOutput
, string
const &Arch
, bool const IncludeArchAll
)
75 : Arch(Arch
), IncludeArchAll(IncludeArchAll
), DoHashes(~0)
77 if (GivenOutput
== NULL
)
81 Output
->OpenDescriptor(STDOUT_FILENO
, FileFd::WriteOnly
, false);
89 NoLinkAct
= !_config
->FindB("APT::FTPArchive::DeLinkAct",true);
90 ConfigToDoHashes(DoHashes
, "APT::FTPArchive");
93 FTWScanner::~FTWScanner()
95 if (Output
!= NULL
&& OwnsOutput
)
98 // FTWScanner::Scanner - FTW Scanner /*{{{*/
99 // ---------------------------------------------------------------------
100 /* This is the FTW scanner, it processes each directory element in the
102 int FTWScanner::ScannerFTW(const char *File
,const struct stat
* /*sb*/,int Flag
)
107 ioprintf(c1out
, _("W: Unable to read directory %s\n"), File
);
112 ioprintf(c1out
, _("W: Unable to stat %s\n"), File
);
117 return ScannerFile(File
, true);
120 // FTWScanner::ScannerFile - File Scanner /*{{{*/
121 // ---------------------------------------------------------------------
123 int FTWScanner::ScannerFile(const char *File
, bool const &ReadLink
)
125 const char *LastComponent
= strrchr(File
, '/');
126 char *RealPath
= NULL
;
128 if (LastComponent
== NULL
)
129 LastComponent
= File
;
133 vector
<string
>::const_iterator I
;
134 for(I
= Owner
->Patterns
.begin(); I
!= Owner
->Patterns
.end(); ++I
)
136 if (fnmatch((*I
).c_str(), LastComponent
, 0) == 0)
139 if (I
== Owner
->Patterns
.end())
142 /* Process it. If the file is a link then resolve it into an absolute
143 name.. This works best if the directory components the scanner are
144 given are not links themselves. */
146 Owner
->OriginalPath
= File
;
148 readlink(File
,Jnk
,sizeof(Jnk
)) != -1 &&
149 (RealPath
= realpath(File
,NULL
)) != 0)
151 Owner
->DoPackage(RealPath
);
155 Owner
->DoPackage(File
);
157 if (_error
->empty() == false)
159 // Print any errors or warnings found
161 bool SeenPath
= false;
162 while (_error
->empty() == false)
166 bool const Type
= _error
->PopMessage(Err
);
168 cerr
<< _("E: ") << Err
<< endl
;
170 cerr
<< _("W: ") << Err
<< endl
;
172 if (Err
.find(File
) != string::npos
)
176 if (SeenPath
== false)
177 cerr
<< _("E: Errors apply to file ") << "'" << File
<< "'" << endl
;
184 // FTWScanner::RecursiveScan - Just scan a directory tree /*{{{*/
185 // ---------------------------------------------------------------------
187 bool FTWScanner::RecursiveScan(string
const &Dir
)
189 char *RealPath
= NULL
;
190 /* If noprefix is set then jam the scan root in, so we don't generate
191 link followed paths out of control */
192 if (InternalPrefix
.empty() == true)
194 if ((RealPath
= realpath(Dir
.c_str(),NULL
)) == 0)
195 return _error
->Errno("realpath",_("Failed to resolve %s"),Dir
.c_str());
196 InternalPrefix
= RealPath
;
200 // Do recursive directory searching
202 int const Res
= ftw(Dir
.c_str(),ScannerFTW
,30);
204 // Error treewalking?
207 if (_error
->PendingError() == false)
208 _error
->Errno("ftw",_("Tree walking failed"));
215 // FTWScanner::LoadFileList - Load the file list from a file /*{{{*/
216 // ---------------------------------------------------------------------
217 /* This is an alternative to using FTW to locate files, it reads the list
218 of files from another file. */
219 bool FTWScanner::LoadFileList(string
const &Dir
, string
const &File
)
221 char *RealPath
= NULL
;
222 /* If noprefix is set then jam the scan root in, so we don't generate
223 link followed paths out of control */
224 if (InternalPrefix
.empty() == true)
226 if ((RealPath
= realpath(Dir
.c_str(),NULL
)) == 0)
227 return _error
->Errno("realpath",_("Failed to resolve %s"),Dir
.c_str());
228 InternalPrefix
= RealPath
;
233 FILE *List
= fopen(File
.c_str(),"r");
235 return _error
->Errno("fopen",_("Failed to open %s"),File
.c_str());
237 /* We are a tad tricky here.. We prefix the buffer with the directory
238 name, that way if we need a full path with just use line.. Sneaky and
242 if (Dir
.empty() == true || Dir
.end()[-1] != '/')
243 FileStart
= Line
+ snprintf(Line
,sizeof(Line
),"%s/",Dir
.c_str());
245 FileStart
= Line
+ snprintf(Line
,sizeof(Line
),"%s",Dir
.c_str());
246 while (fgets(FileStart
,sizeof(Line
) - (FileStart
- Line
),List
) != 0)
248 char *FileName
= _strstrip(FileStart
);
249 if (FileName
[0] == 0)
252 if (FileName
[0] != '/')
254 if (FileName
!= FileStart
)
255 memmove(FileStart
,FileName
,strlen(FileStart
));
262 if (stat(FileName
,&St
) != 0)
266 if (ScannerFile(FileName
, false) != 0)
274 // FTWScanner::Delink - Delink symlinks /*{{{*/
275 // ---------------------------------------------------------------------
277 bool FTWScanner::Delink(string
&FileName
,const char *OriginalPath
,
278 unsigned long long &DeLinkBytes
,
279 unsigned long long const &FileSize
)
281 // See if this isn't an internaly prefix'd file name.
282 if (InternalPrefix
.empty() == false &&
283 InternalPrefix
.length() < FileName
.length() &&
284 stringcmp(FileName
.begin(),FileName
.begin() + InternalPrefix
.length(),
285 InternalPrefix
.begin(),InternalPrefix
.end()) != 0)
287 if (DeLinkLimit
!= 0 && DeLinkBytes
/1024 < DeLinkLimit
)
289 // Tidy up the display
290 if (DeLinkBytes
== 0)
294 ioprintf(c1out
, _(" DeLink %s [%s]\n"), (OriginalPath
+ InternalPrefix
.length()),
295 SizeToStr(FileSize
).c_str());
298 if (NoLinkAct
== false)
301 if (readlink(OriginalPath
,OldLink
,sizeof(OldLink
)) == -1)
302 _error
->Errno("readlink",_("Failed to readlink %s"),OriginalPath
);
305 if (RemoveFile("FTWScanner::Delink", OriginalPath
))
307 if (link(FileName
.c_str(),OriginalPath
) != 0)
309 // Panic! Restore the symlink
310 if (symlink(OldLink
,OriginalPath
) != 0)
311 _error
->Errno("symlink", "failed to restore symlink");
312 return _error
->Errno("link",_("*** Failed to link %s to %s"),
320 DeLinkBytes
+= FileSize
;
321 if (DeLinkBytes
/1024 >= DeLinkLimit
)
322 ioprintf(c1out
, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes
).c_str());
325 FileName
= OriginalPath
;
331 // FTWScanner::SetExts - Set extensions to support /*{{{*/
332 // ---------------------------------------------------------------------
334 bool FTWScanner::SetExts(string
const &Vals
)
337 string::size_type Start
= 0;
338 while (Start
<= Vals
.length()-1)
340 string::size_type
const Space
= Vals
.find(' ',Start
);
341 string::size_type
const Length
= ((Space
== string::npos
) ? Vals
.length() : Space
) - Start
;
342 if ( Arch
.empty() == false )
344 AddPattern(string("*_") + Arch
+ Vals
.substr(Start
, Length
));
345 if (IncludeArchAll
== true && Arch
!= "all")
346 AddPattern(string("*_all") + Vals
.substr(Start
, Length
));
349 AddPattern(string("*") + Vals
.substr(Start
, Length
));
358 // PackagesWriter::PackagesWriter - Constructor /*{{{*/
359 // ---------------------------------------------------------------------
361 PackagesWriter::PackagesWriter(FileFd
* const GivenOutput
, TranslationWriter
* const transWriter
,
362 string
const &DB
,string
const &Overrides
,string
const &ExtOverrides
,
363 string
const &Arch
, bool const IncludeArchAll
) :
364 FTWScanner(GivenOutput
, Arch
, IncludeArchAll
), Db(DB
), Stats(Db
.Stats
), TransWriter(transWriter
)
366 SetExts(".deb .udeb");
369 // Process the command line options
370 ConfigToDoHashes(DoHashes
, "APT::FTPArchive::Packages");
371 DoAlwaysStat
= _config
->FindB("APT::FTPArchive::AlwaysStat", false);
372 DoContents
= _config
->FindB("APT::FTPArchive::Contents",true);
373 NoOverride
= _config
->FindB("APT::FTPArchive::NoOverrideMsg",false);
374 LongDescription
= _config
->FindB("APT::FTPArchive::LongDescription",true);
376 if (Db
.Loaded() == false)
379 // Read the override file
380 if (Overrides
.empty() == false && Over
.ReadOverride(Overrides
) == false)
385 if (ExtOverrides
.empty() == false)
386 Over
.ReadExtraOverride(ExtOverrides
);
388 _error
->DumpErrors();
391 // PackagesWriter::DoPackage - Process a single package /*{{{*/
392 // ---------------------------------------------------------------------
393 /* This method takes a package and gets its control information and
394 MD5, SHA1 and SHA256 then writes out a control record with the proper fields
395 rewritten and the path/size/hash appended. */
396 bool PackagesWriter::DoPackage(string FileName
)
398 // Pull all the data we need form the DB
399 if (Db
.GetFileInfo(FileName
,
400 true, /* DoControl */
402 true, /* GenContentsOnly */
403 false, /* DoSource */
404 DoHashes
, DoAlwaysStat
) == false)
409 unsigned long long FileSize
= Db
.GetFileSize();
410 if (Delink(FileName
,OriginalPath
,Stats
.DeLinkBytes
,FileSize
) == false)
413 // Lookup the overide information
414 pkgTagSection
&Tags
= Db
.Control
.Section
;
415 string Package
= Tags
.FindS("Package");
417 // if we generate a Packages file for a given arch, we use it to
418 // look for overrides. if we run in "simple" mode without the
419 // "Architecures" variable in the config we use the architecure value
424 Architecture
= Tags
.FindS("Architecture");
425 unique_ptr
<Override::Item
> OverItem(Over
.GetItem(Package
,Architecture
));
427 if (Package
.empty() == true)
428 return _error
->Error(_("Archive had no package field"));
430 // If we need to do any rewriting of the header do it now..
431 if (OverItem
.get() == 0)
433 if (NoOverride
== false)
436 ioprintf(c1out
, _(" %s has no override entry\n"), Package
.c_str());
439 OverItem
= unique_ptr
<Override::Item
>(new Override::Item
);
440 OverItem
->FieldOverride
["Section"] = Tags
.FindS("Section");
441 OverItem
->Priority
= Tags
.FindS("Priority");
444 // Strip the DirStrip prefix from the FileName and add the PathPrefix
446 if (DirStrip
.empty() == false &&
447 FileName
.length() > DirStrip
.length() &&
448 stringcmp(FileName
.begin(),FileName
.begin() + DirStrip
.length(),
449 DirStrip
.begin(),DirStrip
.end()) == 0)
450 NewFileName
= string(FileName
.begin() + DirStrip
.length(),FileName
.end());
452 NewFileName
= FileName
;
453 if (PathPrefix
.empty() == false)
454 NewFileName
= flCombine(PathPrefix
,NewFileName
);
456 /* Configuration says we don't want to include the long Description
457 in the package file - instead we want to ship a separated file */
459 if (LongDescription
== false) {
460 desc
= Tags
.FindS("Description").append("\n");
461 OverItem
->FieldOverride
["Description"] = desc
.substr(0, desc
.find('\n')).c_str();
464 // This lists all the changes to the fields we are going to make.
465 std::vector
<pkgTagSection::Tag
> Changes
;
468 strprintf(Size
, "%llu", (unsigned long long) FileSize
);
469 Changes
.push_back(pkgTagSection::Tag::Rewrite("Size", Size
));
471 for (HashStringList::const_iterator hs
= Db
.HashesList
.begin(); hs
!= Db
.HashesList
.end(); ++hs
)
473 if (hs
->HashType() == "MD5Sum")
474 Changes
.push_back(pkgTagSection::Tag::Rewrite("MD5sum", hs
->HashValue()));
475 else if (hs
->HashType() == "Checksum-FileSize")
478 Changes
.push_back(pkgTagSection::Tag::Rewrite(hs
->HashType(), hs
->HashValue()));
480 Changes
.push_back(pkgTagSection::Tag::Rewrite("Filename", NewFileName
));
481 Changes
.push_back(pkgTagSection::Tag::Rewrite("Priority", OverItem
->Priority
));
482 Changes
.push_back(pkgTagSection::Tag::Remove("Status"));
483 Changes
.push_back(pkgTagSection::Tag::Remove("Optional"));
485 string DescriptionMd5
;
486 if (LongDescription
== false) {
487 MD5Summation descmd5
;
488 descmd5
.Add(desc
.c_str());
489 DescriptionMd5
= descmd5
.Result().Value();
490 Changes
.push_back(pkgTagSection::Tag::Rewrite("Description-md5", DescriptionMd5
));
491 if (TransWriter
!= NULL
)
492 TransWriter
->DoPackage(Package
, desc
, DescriptionMd5
);
495 // Rewrite the maintainer field if necessary
497 string NewMaint
= OverItem
->SwapMaint(Tags
.FindS("Maintainer"),MaintFailed
);
498 if (MaintFailed
== true)
500 if (NoOverride
== false)
503 ioprintf(c1out
, _(" %s maintainer is %s not %s\n"),
504 Package
.c_str(), Tags
.FindS("Maintainer").c_str(), OverItem
->OldMaint
.c_str());
508 if (NewMaint
.empty() == false)
509 Changes
.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint
));
511 /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
512 dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
513 but dpkg does this append bit. So we do the append bit, at least that way the
514 status file and package file will remain similar. There are other transforms
515 but optional is the only legacy one still in use for some lazy reason. */
516 string OptionalStr
= Tags
.FindS("Optional");
517 if (OptionalStr
.empty() == false)
519 if (Tags
.FindS("Suggests").empty() == false)
520 OptionalStr
= Tags
.FindS("Suggests") + ", " + OptionalStr
;
521 Changes
.push_back(pkgTagSection::Tag::Rewrite("Suggests", OptionalStr
));
524 for (map
<string
,string
>::const_iterator I
= OverItem
->FieldOverride
.begin();
525 I
!= OverItem
->FieldOverride
.end(); ++I
)
526 Changes
.push_back(pkgTagSection::Tag::Rewrite(I
->first
, I
->second
));
528 // Rewrite and store the fields.
529 if (Tags
.Write(*Output
, TFRewritePackageOrder
, Changes
) == false ||
530 Output
->Write("\n", 1) == false)
536 PackagesWriter::~PackagesWriter() /*{{{*/
541 // TranslationWriter::TranslationWriter - Constructor /*{{{*/
542 // ---------------------------------------------------------------------
543 /* Create a Translation-Master file for this Packages file */
544 TranslationWriter::TranslationWriter(string
const &File
, string
const &TransCompress
,
545 mode_t
const &Permissions
) : Comp(NULL
), Output(NULL
)
547 if (File
.empty() == true)
550 Comp
= new MultiCompress(File
, TransCompress
, Permissions
);
551 Output
= &Comp
->Input
;
554 // TranslationWriter::DoPackage - Process a single package /*{{{*/
555 // ---------------------------------------------------------------------
556 /* Create a Translation-Master file for this Packages file */
557 bool TranslationWriter::DoPackage(string
const &Pkg
, string
const &Desc
,
563 // Different archs can include different versions and therefore
564 // different descriptions - so we need to check for both name and md5.
565 string
const Record
= Pkg
+ ":" + MD5
;
567 if (Included
.find(Record
) != Included
.end())
571 strprintf(out
, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
572 Pkg
.c_str(), MD5
.c_str(), Desc
.c_str());
573 Output
->Write(out
.c_str(), out
.length());
575 Included
.insert(Record
);
579 // TranslationWriter::~TranslationWriter - Destructor /*{{{*/
580 // ---------------------------------------------------------------------
582 TranslationWriter::~TranslationWriter()
589 // SourcesWriter::SourcesWriter - Constructor /*{{{*/
590 // ---------------------------------------------------------------------
592 SourcesWriter::SourcesWriter(FileFd
* const GivenOutput
, string
const &DB
, string
const &BOverrides
,string
const &SOverrides
,
593 string
const &ExtOverrides
) :
594 FTWScanner(GivenOutput
), Db(DB
), Stats(Db
.Stats
)
601 // Process the command line options
602 ConfigToDoHashes(DoHashes
, "APT::FTPArchive::Sources");
603 NoOverride
= _config
->FindB("APT::FTPArchive::NoOverrideMsg",false);
604 DoAlwaysStat
= _config
->FindB("APT::FTPArchive::AlwaysStat", false);
606 // Read the override file
607 if (BOverrides
.empty() == false && BOver
.ReadOverride(BOverrides
) == false)
612 // WTF?? The logic above: if we can't read binary overrides, don't even try
613 // reading source overrides. if we can read binary overrides, then say there
614 // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
616 if (ExtOverrides
.empty() == false)
617 SOver
.ReadExtraOverride(ExtOverrides
);
619 if (SOverrides
.empty() == false && FileExists(SOverrides
) == true)
620 SOver
.ReadOverride(SOverrides
,true);
623 // SourcesWriter::DoPackage - Process a single package /*{{{*/
624 static std::string
getDscHash(unsigned int const DoHashes
,
625 Hashes::SupportedHashes
const DoIt
, pkgTagSection
&Tags
, char const * const FieldName
,
626 HashString
const * const Hash
, unsigned long long Size
, std::string FileName
)
628 if ((DoHashes
& DoIt
) != DoIt
|| Tags
.Exists(FieldName
) == false || Hash
== NULL
)
630 std::ostringstream out
;
631 out
<< "\n " << Hash
->HashValue() << " " << Size
<< " " << FileName
632 << "\n " << Tags
.FindS(FieldName
);
635 bool SourcesWriter::DoPackage(string FileName
)
637 // Pull all the data we need form the DB
638 if (Db
.GetFileInfo(FileName
,
639 false, /* DoControl */
640 false, /* DoContents */
641 false, /* GenContentsOnly */
643 DoHashes
, DoAlwaysStat
) == false)
648 // we need to perform a "write" here (this is what finish is doing)
649 // because the call to Db.GetFileInfo() in the loop will change
654 if (Tags
.Scan(Db
.Dsc
.Data
.c_str(), Db
.Dsc
.Data
.length()) == false)
655 return _error
->Error("Could not find a record in the DSC '%s'",FileName
.c_str());
657 if (Tags
.Exists("Source") == false)
658 return _error
->Error("Could not find a Source entry in the DSC '%s'",FileName
.c_str());
661 // Lookup the overide information, finding first the best priority.
663 string Bins
= Tags
.FindS("Binary");
664 char Buffer
[Bins
.length() + 1];
665 unique_ptr
<Override::Item
> OverItem(nullptr);
666 if (Bins
.empty() == false)
668 strcpy(Buffer
,Bins
.c_str());
670 // Ignore too-long errors.
672 TokSplitString(',',Buffer
,BinList
,sizeof(BinList
)/sizeof(BinList
[0]));
674 // Look at all the binaries
675 unsigned char BestPrioV
= pkgCache::State::Extra
;
676 for (unsigned I
= 0; BinList
[I
] != 0; I
++)
678 unique_ptr
<Override::Item
> Itm(BOver
.GetItem(BinList
[I
]));
682 unsigned char NewPrioV
= debListParser::GetPrio(Itm
->Priority
);
683 if (NewPrioV
< BestPrioV
|| BestPrio
.empty() == true)
685 BestPrioV
= NewPrioV
;
686 BestPrio
= Itm
->Priority
;
689 if (OverItem
.get() == 0)
690 OverItem
= std::move(Itm
);
694 // If we need to do any rewriting of the header do it now..
695 if (OverItem
.get() == 0)
697 if (NoOverride
== false)
700 ioprintf(c1out
, _(" %s has no override entry\n"), Tags
.FindS("Source").c_str());
703 OverItem
.reset(new Override::Item
);
707 if (stat(FileName
.c_str(), &St
) != 0)
708 return _error
->Errno("fstat","Failed to stat %s",FileName
.c_str());
710 unique_ptr
<Override::Item
> SOverItem(SOver
.GetItem(Tags
.FindS("Source")));
711 // const unique_ptr<Override::Item> autoSOverItem(SOverItem);
712 if (SOverItem
.get() == 0)
714 ioprintf(c1out
, _(" %s has no source override entry\n"), Tags
.FindS("Source").c_str());
715 SOverItem
= unique_ptr
<Override::Item
>(BOver
.GetItem(Tags
.FindS("Source")));
716 if (SOverItem
.get() == 0)
718 ioprintf(c1out
, _(" %s has no binary override entry either\n"), Tags
.FindS("Source").c_str());
719 SOverItem
= unique_ptr
<Override::Item
>(new Override::Item
);
720 *SOverItem
= *OverItem
;
724 // Add the dsc to the files hash list
725 string
const strippedName
= flNotDir(FileName
);
726 std::string
const Files
= getDscHash(DoHashes
, Hashes::MD5SUM
, Tags
, "Files", Db
.HashesList
.find("MD5Sum"), St
.st_size
, strippedName
);
727 std::string ChecksumsSha1
= getDscHash(DoHashes
, Hashes::SHA1SUM
, Tags
, "Checksums-Sha1", Db
.HashesList
.find("SHA1"), St
.st_size
, strippedName
);
728 std::string ChecksumsSha256
= getDscHash(DoHashes
, Hashes::SHA256SUM
, Tags
, "Checksums-Sha256", Db
.HashesList
.find("SHA256"), St
.st_size
, strippedName
);
729 std::string ChecksumsSha512
= getDscHash(DoHashes
, Hashes::SHA512SUM
, Tags
, "Checksums-Sha512", Db
.HashesList
.find("SHA512"), St
.st_size
, strippedName
);
731 // Strip the DirStrip prefix from the FileName and add the PathPrefix
733 if (DirStrip
.empty() == false &&
734 FileName
.length() > DirStrip
.length() &&
735 stringcmp(DirStrip
,OriginalPath
,OriginalPath
+ DirStrip
.length()) == 0)
736 NewFileName
= string(OriginalPath
+ DirStrip
.length());
738 NewFileName
= OriginalPath
;
739 if (PathPrefix
.empty() == false)
740 NewFileName
= flCombine(PathPrefix
,NewFileName
);
742 string Directory
= flNotFile(OriginalPath
);
743 string Package
= Tags
.FindS("Source");
745 // Perform operation over all of the files
747 const char *C
= Files
.c_str();
748 char *RealPath
= NULL
;
749 for (;isspace(*C
); C
++);
752 // Parse each of the elements
753 if (ParseQuoteWord(C
,ParseJnk
) == false ||
754 ParseQuoteWord(C
,ParseJnk
) == false ||
755 ParseQuoteWord(C
,ParseJnk
) == false)
756 return _error
->Error("Error parsing file record");
758 string OriginalPath
= Directory
+ ParseJnk
;
760 // Add missing hashes to source files
761 if (((DoHashes
& Hashes::SHA1SUM
) == Hashes::SHA1SUM
&& !Tags
.Exists("Checksums-Sha1")) ||
762 ((DoHashes
& Hashes::SHA256SUM
) == Hashes::SHA256SUM
&& !Tags
.Exists("Checksums-Sha256")) ||
763 ((DoHashes
& Hashes::SHA512SUM
) == Hashes::SHA512SUM
&& !Tags
.Exists("Checksums-Sha512")))
765 if (Db
.GetFileInfo(OriginalPath
,
766 false, /* DoControl */
767 false, /* DoContents */
768 false, /* GenContentsOnly */
769 false, /* DoSource */
771 DoAlwaysStat
) == false)
773 return _error
->Error("Error getting file info");
776 for (HashStringList::const_iterator hs
= Db
.HashesList
.begin(); hs
!= Db
.HashesList
.end(); ++hs
)
778 if (hs
->HashType() == "MD5Sum" || hs
->HashType() == "Checksum-FileSize")
780 char const * fieldname
;
782 if (hs
->HashType() == "SHA1")
784 fieldname
= "Checksums-Sha1";
785 out
= &ChecksumsSha1
;
787 else if (hs
->HashType() == "SHA256")
789 fieldname
= "Checksums-Sha256";
790 out
= &ChecksumsSha256
;
792 else if (hs
->HashType() == "SHA512")
794 fieldname
= "Checksums-Sha512";
795 out
= &ChecksumsSha512
;
799 _error
->Warning("Ignoring unknown Checksumtype %s in SourcesWriter::DoPackages", hs
->HashType().c_str());
802 if (Tags
.Exists(fieldname
) == true)
804 std::ostringstream streamout
;
805 streamout
<< "\n " << hs
->HashValue() << " " << Db
.GetFileSize() << " " << ParseJnk
;
806 out
->append(streamout
.str());
809 // write back the GetFileInfo() stats data
813 // Perform the delinking operation
816 if (readlink(OriginalPath
.c_str(),Jnk
,sizeof(Jnk
)) != -1 &&
817 (RealPath
= realpath(OriginalPath
.c_str(),NULL
)) != 0)
819 string RP
= RealPath
;
821 if (Delink(RP
,OriginalPath
.c_str(),Stats
.DeLinkBytes
,St
.st_size
) == false)
826 Directory
= flNotFile(NewFileName
);
827 if (Directory
.length() > 2)
828 Directory
.erase(Directory
.end()-1);
830 // This lists all the changes to the fields we are going to make.
831 // (5 hardcoded + checksums + maintainer + end marker)
832 std::vector
<pkgTagSection::Tag
> Changes
;
834 Changes
.push_back(pkgTagSection::Tag::Remove("Source"));
835 Changes
.push_back(pkgTagSection::Tag::Rewrite("Package", Package
));
836 if (Files
.empty() == false)
837 Changes
.push_back(pkgTagSection::Tag::Rewrite("Files", Files
));
838 if (ChecksumsSha1
.empty() == false)
839 Changes
.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha1", ChecksumsSha1
));
840 if (ChecksumsSha256
.empty() == false)
841 Changes
.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha256", ChecksumsSha256
));
842 if (ChecksumsSha512
.empty() == false)
843 Changes
.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha512", ChecksumsSha512
));
844 if (Directory
!= "./")
845 Changes
.push_back(pkgTagSection::Tag::Rewrite("Directory", Directory
));
846 Changes
.push_back(pkgTagSection::Tag::Rewrite("Priority", BestPrio
));
847 Changes
.push_back(pkgTagSection::Tag::Remove("Status"));
849 // Rewrite the maintainer field if necessary
851 string NewMaint
= OverItem
->SwapMaint(Tags
.FindS("Maintainer"), MaintFailed
);
852 if (MaintFailed
== true)
854 if (NoOverride
== false)
857 ioprintf(c1out
, _(" %s maintainer is %s not %s\n"), Package
.c_str(),
858 Tags
.FindS("Maintainer").c_str(), OverItem
->OldMaint
.c_str());
861 if (NewMaint
.empty() == false)
862 Changes
.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint
.c_str()));
864 for (map
<string
,string
>::const_iterator I
= SOverItem
->FieldOverride
.begin();
865 I
!= SOverItem
->FieldOverride
.end(); ++I
)
866 Changes
.push_back(pkgTagSection::Tag::Rewrite(I
->first
, I
->second
));
868 // Rewrite and store the fields.
869 if (Tags
.Write(*Output
, TFRewriteSourceOrder
, Changes
) == false ||
870 Output
->Write("\n", 1) == false)
879 // ContentsWriter::ContentsWriter - Constructor /*{{{*/
880 // ---------------------------------------------------------------------
882 ContentsWriter::ContentsWriter(FileFd
* const GivenOutput
, string
const &DB
,
883 string
const &Arch
, bool const IncludeArchAll
) :
884 FTWScanner(GivenOutput
, Arch
, IncludeArchAll
), Db(DB
), Stats(Db
.Stats
)
890 // ContentsWriter::DoPackage - Process a single package /*{{{*/
891 // ---------------------------------------------------------------------
892 /* If Package is the empty string the control record will be parsed to
893 determine what the package name is. */
894 bool ContentsWriter::DoPackage(string FileName
, string Package
)
896 if (!Db
.GetFileInfo(FileName
,
897 Package
.empty(), /* DoControl */
898 true, /* DoContents */
899 false, /* GenContentsOnly */
900 false, /* DoSource */
902 false /* checkMtime */))
907 // Parse the package name
908 if (Package
.empty() == true)
910 Package
= Db
.Control
.Section
.FindS("Package");
913 Db
.Contents
.Add(Gen
,Package
);
918 // ContentsWriter::ReadFromPkgs - Read from a packages file /*{{{*/
919 // ---------------------------------------------------------------------
921 bool ContentsWriter::ReadFromPkgs(string
const &PkgFile
,string
const &PkgCompress
)
923 MultiCompress
Pkgs(PkgFile
,PkgCompress
,0,false);
924 if (_error
->PendingError() == true)
927 // Open the package file
929 if (Pkgs
.OpenOld(Fd
) == false)
932 pkgTagFile
Tags(&Fd
);
933 if (_error
->PendingError() == true)
937 pkgTagSection Section
;
938 while (Tags
.Step(Section
) == true)
940 string File
= flCombine(Prefix
,Section
.FindS("FileName"));
941 string Package
= Section
.FindS("Section");
942 if (Package
.empty() == false && Package
.end()[-1] != '/')
945 Package
+= Section
.FindS("Package");
948 Package
+= Section
.FindS("Package");
950 DoPackage(File
,Package
);
951 if (_error
->empty() == false)
953 _error
->Error("Errors apply to file '%s'",File
.c_str());
954 _error
->DumpErrors();
958 // Tidy the compressor
966 // ReleaseWriter::ReleaseWriter - Constructor /*{{{*/
967 // ---------------------------------------------------------------------
969 ReleaseWriter::ReleaseWriter(FileFd
* const GivenOutput
, string
const &/*DB*/) : FTWScanner(GivenOutput
)
971 if (_config
->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
973 AddPattern("Packages");
974 AddPattern("Packages.*");
975 AddPattern("Translation-*");
976 AddPattern("Sources");
977 AddPattern("Sources.*");
978 AddPattern("Release");
979 AddPattern("Contents-*");
981 AddPattern("icons-*.tar");
982 AddPattern("icons-*.tar.*");
983 AddPattern("Components-*.yml");
984 AddPattern("Components-*.yml.*");
985 AddPattern("md5sum.txt");
987 AddPatterns(_config
->FindVector("APT::FTPArchive::Release::Patterns"));
989 time_t const now
= time(NULL
);
991 setlocale(LC_TIME
, "C");
994 if (strftime(datestr
, sizeof(datestr
), "%a, %d %b %Y %H:%M:%S UTC",
1000 time_t const validuntil
= now
+ _config
->FindI("APT::FTPArchive::Release::ValidTime", 0);
1002 if (now
== validuntil
||
1003 strftime(validstr
, sizeof(validstr
), "%a, %d %b %Y %H:%M:%S UTC",
1004 gmtime(&validuntil
)) == 0)
1009 setlocale(LC_TIME
, "");
1011 map
<string
,string
> Fields
;
1012 Fields
["Origin"] = "";
1013 Fields
["Label"] = "";
1014 Fields
["Suite"] = "";
1015 Fields
["Version"] = "";
1016 Fields
["Codename"] = "";
1017 Fields
["Date"] = datestr
;
1018 Fields
["Valid-Until"] = validstr
;
1019 Fields
["Architectures"] = "";
1020 Fields
["Components"] = "";
1021 Fields
["Description"] = "";
1022 Fields
["Signed-By"] = "";
1023 if (_config
->FindB("APT::FTPArchive::DoByHash", false) == true)
1024 Fields
["Acquire-By-Hash"] = "true";
1026 for(map
<string
,string
>::const_iterator I
= Fields
.begin();
1030 string Config
= string("APT::FTPArchive::Release::") + (*I
).first
;
1031 string Value
= _config
->Find(Config
, (*I
).second
.c_str());
1035 std::string
const out
= I
->first
+ ": " + Value
+ "\n";
1036 Output
->Write(out
.c_str(), out
.length());
1039 ConfigToDoHashes(DoHashes
, "APT::FTPArchive::Release");
1042 // ReleaseWriter::DoPackage - Process a single package /*{{{*/
1043 // ---------------------------------------------------------------------
1044 bool ReleaseWriter::DoPackage(string FileName
)
1046 // Strip the DirStrip prefix from the FileName and add the PathPrefix
1048 if (DirStrip
.empty() == false &&
1049 FileName
.length() > DirStrip
.length() &&
1050 stringcmp(FileName
.begin(),FileName
.begin() + DirStrip
.length(),
1051 DirStrip
.begin(),DirStrip
.end()) == 0)
1053 NewFileName
= string(FileName
.begin() + DirStrip
.length(),FileName
.end());
1054 while (NewFileName
[0] == '/')
1055 NewFileName
= string(NewFileName
.begin() + 1,NewFileName
.end());
1058 NewFileName
= FileName
;
1060 if (PathPrefix
.empty() == false)
1061 NewFileName
= flCombine(PathPrefix
,NewFileName
);
1063 FileFd
fd(FileName
, FileFd::ReadOnly
);
1070 CheckSums
[NewFileName
].size
= fd
.Size();
1072 Hashes
hs(DoHashes
);
1074 CheckSums
[NewFileName
].Hashes
= hs
.GetHashStringList();
1077 // FIXME: wrong layer in the code(?)
1078 // FIXME2: symlink instead of create a copy
1079 if (_config
->FindB("APT::FTPArchive::DoByHash", false) == true)
1081 std::string Input
= FileName
;
1082 HashStringList hsl
= hs
.GetHashStringList();
1083 for(HashStringList::const_iterator h
= hsl
.begin();
1084 h
!= hsl
.end(); ++h
)
1088 if (flNotDir(FileName
) == "Release" || flNotDir(FileName
) == "InRelease")
1091 std::string ByHashOutputFile
= GenByHashFilename(Input
, *h
);
1092 std::string ByHashOutputDir
= flNotFile(ByHashOutputFile
);
1093 if(!CreateDirectory(flNotFile(Input
), ByHashOutputDir
))
1094 return _error
->Warning("can not create dir %s", flNotFile(ByHashOutputFile
).c_str());
1097 FileFd
In(Input
, FileFd::ReadOnly
);
1098 FileFd
Out(ByHashOutputFile
, FileFd::WriteEmpty
);
1099 if(!CopyFile(In
, Out
))
1100 return _error
->Warning("failed to copy %s %s", Input
.c_str(), ByHashOutputFile
.c_str());
1108 // ReleaseWriter::Finish - Output the checksums /*{{{*/
1109 // ---------------------------------------------------------------------
1110 static void printChecksumTypeRecord(FileFd
&Output
, char const * const Type
, map
<string
, ReleaseWriter::CheckSum
> const &CheckSums
)
1114 strprintf(out
, "%s:\n", Type
);
1115 Output
.Write(out
.c_str(), out
.length());
1117 for(map
<string
,ReleaseWriter::CheckSum
>::const_iterator I
= CheckSums
.begin();
1118 I
!= CheckSums
.end(); ++I
)
1120 HashString
const * const hs
= I
->second
.Hashes
.find(Type
);
1124 strprintf(out
, " %s %16llu %s\n",
1125 hs
->HashValue().c_str(),
1127 (*I
).first
.c_str());
1128 Output
.Write(out
.c_str(), out
.length());
1131 void ReleaseWriter::Finish()
1133 if ((DoHashes
& Hashes::MD5SUM
) == Hashes::MD5SUM
)
1134 printChecksumTypeRecord(*Output
, "MD5Sum", CheckSums
);
1135 if ((DoHashes
& Hashes::SHA1SUM
) == Hashes::SHA1SUM
)
1136 printChecksumTypeRecord(*Output
, "SHA1", CheckSums
);
1137 if ((DoHashes
& Hashes::SHA256SUM
) == Hashes::SHA256SUM
)
1138 printChecksumTypeRecord(*Output
, "SHA256", CheckSums
);
1139 if ((DoHashes
& Hashes::SHA512SUM
) == Hashes::SHA512SUM
)
1140 printChecksumTypeRecord(*Output
, "SHA512", CheckSums
);
1142 // go by-hash cleanup
1143 map
<string
,ReleaseWriter::CheckSum
>::const_iterator prev
= CheckSums
.begin();
1144 if (_config
->FindB("APT::FTPArchive::DoByHash", false) == true)
1146 for(map
<string
,ReleaseWriter::CheckSum
>::const_iterator I
= CheckSums
.begin();
1147 I
!= CheckSums
.end(); ++I
)
1149 if (I
->first
== "Release" || I
->first
== "InRelease")
1152 // keep iterating until we find a new subdir
1153 if(flNotFile(I
->first
) == flNotFile(prev
->first
))
1156 // clean that subdir up
1157 int keepFiles
= _config
->FindI("APT::FTPArchive::By-Hash-Keep", 3);
1158 // calculate how many compressors are used (the amount of files
1159 // in that subdir generated for this run)
1160 keepFiles
*= std::distance(prev
, I
);
1163 HashStringList hsl
= prev
->second
.Hashes
;
1164 for(HashStringList::const_iterator h
= hsl
.begin();
1165 h
!= hsl
.end(); ++h
)
1171 std::string RealFilename
= DirStrip
+"/"+prev
->first
;
1172 std::string ByHashOutputFile
= GenByHashFilename(RealFilename
, *h
);
1173 DeleteAllButMostRecent(flNotFile(ByHashOutputFile
), keepFiles
);