X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/3c54407f8783d5e27363eabf41dbc3d031526ffe..453b82a388013e522b3a1b9fcd6ed0810dab1f4f:/ftparchive/writer.cc diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc index d3a19ca28..153c4fb42 100644 --- a/ftparchive/writer.cc +++ b/ftparchive/writer.cc @@ -11,29 +11,42 @@ ##################################################################### */ /*}}}*/ // Include Files /*{{{*/ -#include "writer.h" - -#include -#include -#include +#include + #include +#include +#include +#include +#include +#include #include +#include +#include +#include #include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include -#include -#include #include #include #include - -#include "cachedb.h" +#include + #include "apt-ftparchive.h" +#include "writer.h" +#include "cachedb.h" #include "multicompress.h" + +#include /*}}}*/ using namespace std; FTWScanner *FTWScanner::Owner; @@ -63,13 +76,14 @@ FTWScanner::FTWScanner(string const &Arch): Arch(Arch) DoMD5 = _config->FindB("APT::FTPArchive::MD5",true); DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true); DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true); + DoSHA512 = _config->FindB("APT::FTPArchive::SHA512",true); } /*}}}*/ // FTWScanner::Scanner - FTW Scanner /*{{{*/ // --------------------------------------------------------------------- -/* This is the FTW scanner, it processes each directory element in the +/* This is the FTW scanner, it processes each directory element in the directory tree. */ -int FTWScanner::ScannerFTW(const char *File,const struct stat *sb,int Flag) +int FTWScanner::ScannerFTW(const char *File,const struct stat * /*sb*/,int Flag) { if (Flag == FTW_DNR) { @@ -245,8 +259,8 @@ bool FTWScanner::LoadFileList(string const &Dir, string const &File) // --------------------------------------------------------------------- /* */ bool FTWScanner::Delink(string &FileName,const char *OriginalPath, - unsigned long &DeLinkBytes, - off_t const &FileSize) + unsigned long long &DeLinkBytes, + unsigned long long const &FileSize) { // See if this isn't an internaly prefix'd file name. if (InternalPrefix.empty() == false && @@ -279,7 +293,8 @@ bool FTWScanner::Delink(string &FileName,const char *OriginalPath, if (link(FileName.c_str(),OriginalPath) != 0) { // Panic! Restore the symlink - symlink(OldLink,OriginalPath); + if (symlink(OldLink,OriginalPath) != 0) + _error->Errno("symlink", "failed to restore symlink"); return _error->Errno("link",_("*** Failed to link %s to %s"), FileName.c_str(), OriginalPath); @@ -315,6 +330,7 @@ PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string c DoMD5 = _config->FindB("APT::FTPArchive::Packages::MD5",DoMD5); DoSHA1 = _config->FindB("APT::FTPArchive::Packages::SHA1",DoSHA1); DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA256",DoSHA256); + DoSHA512 = _config->FindB("APT::FTPArchive::Packages::SHA512",DoSHA512); DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false); DoContents = _config->FindB("APT::FTPArchive::Contents",true); NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false); @@ -369,13 +385,13 @@ bool FTWScanner::SetExts(string const &Vals) bool PackagesWriter::DoPackage(string FileName) { // Pull all the data we need form the DB - if (Db.GetFileInfo(FileName, true, DoContents, true, DoMD5, DoSHA1, DoSHA256, DoAlwaysStat) + if (Db.GetFileInfo(FileName, true, DoContents, true, DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat) == false) { return false; } - off_t FileSize = Db.GetFileSize(); + unsigned long long FileSize = Db.GetFileSize(); if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false) return false; @@ -411,7 +427,7 @@ bool PackagesWriter::DoPackage(string FileName) } char Size[40]; - sprintf(Size,"%lu", (unsigned long) FileSize); + sprintf(Size,"%llu", (unsigned long long) FileSize); // Strip the DirStrip prefix from the FileName and add the PathPrefix string NewFileName; @@ -445,6 +461,8 @@ bool PackagesWriter::DoPackage(string FileName) SetTFRewriteData(Changes[End++], "SHA1", Db.SHA1Res.c_str()); if (DoSHA256 == true) SetTFRewriteData(Changes[End++], "SHA256", Db.SHA256Res.c_str()); + if (DoSHA512 == true) + SetTFRewriteData(Changes[End++], "SHA512", Db.SHA512Res.c_str()); SetTFRewriteData(Changes[End++], "Filename", NewFileName.c_str()); SetTFRewriteData(Changes[End++], "Priority", OverItem->Priority.c_str()); SetTFRewriteData(Changes[End++], "Status", 0); @@ -490,7 +508,7 @@ bool PackagesWriter::DoPackage(string FileName) } for (map::const_iterator I = OverItem->FieldOverride.begin(); - I != OverItem->FieldOverride.end(); I++) + I != OverItem->FieldOverride.end(); ++I) SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str()); SetTFRewriteData(Changes[End++], 0, 0); @@ -556,8 +574,9 @@ TranslationWriter::~TranslationWriter() // SourcesWriter::SourcesWriter - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -SourcesWriter::SourcesWriter(string const &BOverrides,string const &SOverrides, - string const &ExtOverrides) +SourcesWriter::SourcesWriter(string const &DB, string const &BOverrides,string const &SOverrides, + string const &ExtOverrides) : + Db(DB), Stats(Db.Stats) { Output = stdout; AddPattern("*.dsc"); @@ -569,7 +588,9 @@ SourcesWriter::SourcesWriter(string const &BOverrides,string const &SOverrides, DoMD5 = _config->FindB("APT::FTPArchive::Sources::MD5",DoMD5); DoSHA1 = _config->FindB("APT::FTPArchive::Sources::SHA1",DoSHA1); DoSHA256 = _config->FindB("APT::FTPArchive::Sources::SHA256",DoSHA256); + DoSHA512 = _config->FindB("APT::FTPArchive::Sources::SHA512",DoSHA512); NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false); + DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false); // Read the override file if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false) @@ -592,74 +613,62 @@ SourcesWriter::SourcesWriter(string const &BOverrides,string const &SOverrides, // --------------------------------------------------------------------- /* */ bool SourcesWriter::DoPackage(string FileName) -{ +{ // Open the archive - FileFd F(FileName,FileFd::ReadOnly); - if (_error->PendingError() == true) + FileFd F; + if (OpenMaybeClearSignedFile(FileName, F) == false) return false; - - // Stat the file for later - struct stat St; - if (fstat(F.Fd(),&St) != 0) - return _error->Errno("fstat","Failed to stat %s",FileName.c_str()); - if (St.st_size > 128*1024) + unsigned long long const FSize = F.FileSize(); + //FIXME: do we really need to enforce a maximum size of the dsc file? + if (FSize > 128*1024) return _error->Error("DSC file '%s' is too large!",FileName.c_str()); - - if (BufSize < (unsigned)St.st_size+1) + + if (BufSize < FSize + 2) { - BufSize = St.st_size+1; - Buffer = (char *)realloc(Buffer,St.st_size+1); + BufSize = FSize + 2; + Buffer = (char *)realloc(Buffer , BufSize); } - - if (F.Read(Buffer,St.st_size) == false) + + if (F.Read(Buffer, FSize) == false) return false; + // Stat the file for later (F might be clearsigned, so not F.FileSize()) + struct stat St; + if (stat(FileName.c_str(), &St) != 0) + return _error->Errno("fstat","Failed to stat %s",FileName.c_str()); + // Hash the file char *Start = Buffer; - char *BlkEnd = Buffer + St.st_size; - - MD5Summation MD5; - SHA1Summation SHA1; - SHA256Summation SHA256; - - if (DoMD5 == true) - MD5.Add((unsigned char *)Start,BlkEnd - Start); - if (DoSHA1 == true) - SHA1.Add((unsigned char *)Start,BlkEnd - Start); - if (DoSHA256 == true) - SHA256.Add((unsigned char *)Start,BlkEnd - Start); + char *BlkEnd = Buffer + FSize; - // Add an extra \n to the end, just in case - *BlkEnd++ = '\n'; - - /* Remove the PGP trailer. Some .dsc's have this without a blank line - before */ - const char *Key = "-----BEGIN PGP SIGNATURE-----"; - for (char *MsgEnd = Start; MsgEnd < BlkEnd - strlen(Key) -1; MsgEnd++) + Hashes DscHashes; + if (FSize == (unsigned long long) St.st_size) { - if (*MsgEnd == '\n' && strncmp(MsgEnd+1,Key,strlen(Key)) == 0) - { - MsgEnd[1] = '\n'; - break; - } + if (DoMD5 == true) + DscHashes.MD5.Add((unsigned char *)Start,BlkEnd - Start); + if (DoSHA1 == true) + DscHashes.SHA1.Add((unsigned char *)Start,BlkEnd - Start); + if (DoSHA256 == true) + DscHashes.SHA256.Add((unsigned char *)Start,BlkEnd - Start); + if (DoSHA512 == true) + DscHashes.SHA512.Add((unsigned char *)Start,BlkEnd - Start); } - - /* Read records until we locate the Source record. This neatly skips the - GPG header (which is RFC822 formed) without any trouble. */ - pkgTagSection Tags; - do + else { - unsigned Pos; - if (Tags.Scan(Start,BlkEnd - Start) == false) - return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str()); - if (Tags.Find("Source",Pos) == true) - break; - Start += Tags.size(); + FileFd DscFile(FileName, FileFd::ReadOnly); + DscHashes.AddFD(DscFile, St.st_size, DoMD5, DoSHA1, DoSHA256, DoSHA512); } - while (1); + + // Add extra \n to the end, just in case (as in clearsigned they are missing) + *BlkEnd++ = '\n'; + *BlkEnd++ = '\n'; + + pkgTagSection Tags; + if (Tags.Scan(Start,BlkEnd - Start) == false || Tags.Exists("Source") == false) + return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str()); Tags.Trim(); - + // Lookup the overide information, finding first the best priority. string BestPrio; string Bins = Tags.FindS("Binary"); @@ -723,21 +732,24 @@ bool SourcesWriter::DoPackage(string FileName) string const strippedName = flNotDir(FileName); std::ostringstream ostreamFiles; if (DoMD5 == true && Tags.Exists("Files")) - ostreamFiles << "\n " << string(MD5.Result()) << " " << St.st_size << " " + ostreamFiles << "\n " << string(DscHashes.MD5.Result()) << " " << St.st_size << " " << strippedName << "\n " << Tags.FindS("Files"); string const Files = ostreamFiles.str(); std::ostringstream ostreamSha1; if (DoSHA1 == true && Tags.Exists("Checksums-Sha1")) - ostreamSha1 << "\n " << string(SHA1.Result()) << " " << St.st_size << " " + ostreamSha1 << "\n " << string(DscHashes.SHA1.Result()) << " " << St.st_size << " " << strippedName << "\n " << Tags.FindS("Checksums-Sha1"); - string const ChecksumsSha1 = ostreamSha1.str(); std::ostringstream ostreamSha256; if (DoSHA256 == true && Tags.Exists("Checksums-Sha256")) - ostreamSha256 << "\n " << string(SHA256.Result()) << " " << St.st_size << " " + ostreamSha256 << "\n " << string(DscHashes.SHA256.Result()) << " " << St.st_size << " " << strippedName << "\n " << Tags.FindS("Checksums-Sha256"); - string const ChecksumsSha256 = ostreamSha256.str(); + + std::ostringstream ostreamSha512; + if (DoSHA512 == true && Tags.Exists("Checksums-Sha512")) + ostreamSha512 << "\n " << string(DscHashes.SHA512.Result()) << " " << St.st_size << " " + << strippedName << "\n " << Tags.FindS("Checksums-Sha512"); // Strip the DirStrip prefix from the FileName and add the PathPrefix string NewFileName; @@ -753,7 +765,7 @@ bool SourcesWriter::DoPackage(string FileName) string Directory = flNotFile(OriginalPath); string Package = Tags.FindS("Source"); - // Perform the delinking operation over all of the files + // Perform operation over all of the files string ParseJnk; const char *C = Files.c_str(); char *RealPath = NULL; @@ -765,9 +777,36 @@ bool SourcesWriter::DoPackage(string FileName) ParseQuoteWord(C,ParseJnk) == false || ParseQuoteWord(C,ParseJnk) == false) return _error->Error("Error parsing file record"); - - char Jnk[2]; + string OriginalPath = Directory + ParseJnk; + + // Add missing hashes to source files + if ((DoSHA1 == true && !Tags.Exists("Checksums-Sha1")) || + (DoSHA256 == true && !Tags.Exists("Checksums-Sha256")) || + (DoSHA512 == true && !Tags.Exists("Checksums-Sha512"))) + { + if (Db.GetFileInfo(OriginalPath, false, false, false, DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat) + == false) + { + return _error->Error("Error getting file info"); + } + + if (DoSHA1 == true && !Tags.Exists("Checksums-Sha1")) + ostreamSha1 << "\n " << string(Db.SHA1Res) << " " + << Db.GetFileSize() << " " << ParseJnk; + + if (DoSHA256 == true && !Tags.Exists("Checksums-Sha256")) + ostreamSha256 << "\n " << string(Db.SHA256Res) << " " + << Db.GetFileSize() << " " << ParseJnk; + + if (DoSHA512 == true && !Tags.Exists("Checksums-Sha512")) + ostreamSha512 << "\n " << string(Db.SHA512Res) << " " + << Db.GetFileSize() << " " << ParseJnk; + } + + // Perform the delinking operation + char Jnk[2]; + if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 && (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0) { @@ -782,6 +821,10 @@ bool SourcesWriter::DoPackage(string FileName) if (Directory.length() > 2) Directory.erase(Directory.end()-1); + string const ChecksumsSha1 = ostreamSha1.str(); + string const ChecksumsSha256 = ostreamSha256.str(); + string const ChecksumsSha512 = ostreamSha512.str(); + // This lists all the changes to the fields we are going to make. // (5 hardcoded + checksums + maintainer + end marker) TFRewriteData Changes[5+2+1+SOverItem->FieldOverride.size()+1]; @@ -794,6 +837,8 @@ bool SourcesWriter::DoPackage(string FileName) SetTFRewriteData(Changes[End++],"Checksums-Sha1",ChecksumsSha1.c_str()); if (ChecksumsSha256.empty() == false) SetTFRewriteData(Changes[End++],"Checksums-Sha256",ChecksumsSha256.c_str()); + if (ChecksumsSha512.empty() == false) + SetTFRewriteData(Changes[End++],"Checksums-Sha512",ChecksumsSha512.c_str()); if (Directory != "./") SetTFRewriteData(Changes[End++],"Directory",Directory.c_str()); SetTFRewriteData(Changes[End++],"Priority",BestPrio.c_str()); @@ -815,7 +860,7 @@ bool SourcesWriter::DoPackage(string FileName) SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str()); for (map::const_iterator I = SOverItem->FieldOverride.begin(); - I != SOverItem->FieldOverride.end(); I++) + I != SOverItem->FieldOverride.end(); ++I) SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str()); SetTFRewriteData(Changes[End++], 0, 0); @@ -827,7 +872,7 @@ bool SourcesWriter::DoPackage(string FileName) Stats.Packages++; - return true; + return Db.Finish(); } /*}}}*/ @@ -872,22 +917,16 @@ bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompres MultiCompress Pkgs(PkgFile,PkgCompress,0,false); if (_error->PendingError() == true) return false; - + // Open the package file - int CompFd = -1; - pid_t Proc = -1; - if (Pkgs.OpenOld(CompFd,Proc) == false) + FileFd Fd; + if (Pkgs.OpenOld(Fd) == false) return false; - - // No auto-close FD - FileFd Fd(CompFd,false); + pkgTagFile Tags(&Fd); if (_error->PendingError() == true) - { - Pkgs.CloseOld(CompFd,Proc); return false; - } - + // Parse. pkgTagSection Section; while (Tags.Step(Section) == true) @@ -909,11 +948,10 @@ bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompres _error->DumpErrors(); } } - + // Tidy the compressor - if (Pkgs.CloseOld(CompFd,Proc) == false) - return false; - + Fd.Close(); + return true; } @@ -922,7 +960,7 @@ bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompres // ReleaseWriter::ReleaseWriter - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -ReleaseWriter::ReleaseWriter(string const &DB) +ReleaseWriter::ReleaseWriter(string const &/*DB*/) { if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true) { @@ -930,11 +968,15 @@ ReleaseWriter::ReleaseWriter(string const &DB) AddPattern("Packages.gz"); AddPattern("Packages.bz2"); AddPattern("Packages.lzma"); + AddPattern("Packages.xz"); + AddPattern("Translation-*"); AddPattern("Sources"); AddPattern("Sources.gz"); AddPattern("Sources.bz2"); AddPattern("Sources.lzma"); + AddPattern("Sources.xz"); AddPattern("Release"); + AddPattern("Contents-*"); AddPattern("Index"); AddPattern("md5sum.txt"); } @@ -1022,29 +1064,18 @@ bool ReleaseWriter::DoPackage(string FileName) CheckSums[NewFileName].size = fd.Size(); + Hashes hs; + hs.AddFD(fd, 0, DoMD5, DoSHA1, DoSHA256, DoSHA512); if (DoMD5 == true) - { - MD5Summation MD5; - MD5.AddFD(fd.Fd(), fd.Size()); - CheckSums[NewFileName].MD5 = MD5.Result(); - fd.Seek(0); - } + CheckSums[NewFileName].MD5 = hs.MD5.Result(); if (DoSHA1 == true) - { - SHA1Summation SHA1; - SHA1.AddFD(fd.Fd(), fd.Size()); - CheckSums[NewFileName].SHA1 = SHA1.Result(); - fd.Seek(0); - } + CheckSums[NewFileName].SHA1 = hs.SHA1.Result(); if (DoSHA256 == true) - { - SHA256Summation SHA256; - SHA256.AddFD(fd.Fd(), fd.Size()); - CheckSums[NewFileName].SHA256 = SHA256.Result(); - } - + CheckSums[NewFileName].SHA256 = hs.SHA256.Result(); + if (DoSHA512 == true) + CheckSums[NewFileName].SHA512 = hs.SHA512.Result(); fd.Close(); - + return true; } @@ -1059,7 +1090,7 @@ void ReleaseWriter::Finish() for(map::const_iterator I = CheckSums.begin(); I != CheckSums.end(); ++I) { - fprintf(Output, " %s %16ld %s\n", + fprintf(Output, " %s %16llu %s\n", (*I).second.MD5.c_str(), (*I).second.size, (*I).first.c_str()); @@ -1071,7 +1102,7 @@ void ReleaseWriter::Finish() for(map::const_iterator I = CheckSums.begin(); I != CheckSums.end(); ++I) { - fprintf(Output, " %s %16ld %s\n", + fprintf(Output, " %s %16llu %s\n", (*I).second.SHA1.c_str(), (*I).second.size, (*I).first.c_str()); @@ -1083,10 +1114,22 @@ void ReleaseWriter::Finish() for(map::const_iterator I = CheckSums.begin(); I != CheckSums.end(); ++I) { - fprintf(Output, " %s %16ld %s\n", + fprintf(Output, " %s %16llu %s\n", (*I).second.SHA256.c_str(), (*I).second.size, (*I).first.c_str()); } } + + fprintf(Output, "SHA512:\n"); + for(map::const_iterator I = CheckSums.begin(); + I != CheckSums.end(); + ++I) + { + fprintf(Output, " %s %16llu %s\n", + (*I).second.SHA512.c_str(), + (*I).second.size, + (*I).first.c_str()); + } + }