X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/8501516a89c1f333f7bc3866b05b5ce7fbf4e177..83b880c6505a20247239d897b7387bba37942993:/apt-pkg/indexcopy.cc?ds=sidebyside diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc index 901001891..854ba1bd7 100644 --- a/apt-pkg/indexcopy.cc +++ b/apt-pkg/indexcopy.cc @@ -10,24 +10,30 @@ ##################################################################### */ /*}}}*/ // Include Files /*{{{*/ -#include "indexcopy.h" +#include<config.h> #include <apt-pkg/error.h> #include <apt-pkg/progress.h> #include <apt-pkg/strutl.h> #include <apt-pkg/fileutl.h> +#include <apt-pkg/aptconfiguration.h> #include <apt-pkg/configuration.h> #include <apt-pkg/tagfile.h> #include <apt-pkg/indexrecords.h> -#include <apt-pkg/md5.h> #include <apt-pkg/cdrom.h> -#include <apti18n.h> +#include <apt-pkg/gpgv.h> +#include <apt-pkg/hashes.h> #include <iostream> #include <sstream> #include <unistd.h> #include <sys/stat.h> #include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "indexcopy.h" +#include <apti18n.h> /*}}}*/ using namespace std; @@ -39,7 +45,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List, pkgCdromStatus *log) { OpProgress *Progress = NULL; - if (List.size() == 0) + if (List.empty() == true) return true; if(log) @@ -49,75 +55,39 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List, bool Debug = _config->FindB("Debug::aptcdrom",false); // Prepare the progress indicator - unsigned long TotalSize = 0; - for (vector<string>::iterator I = List.begin(); I != List.end(); I++) + off_t TotalSize = 0; + std::vector<APT::Configuration::Compressor> const compressor = APT::Configuration::getCompressors(); + for (vector<string>::iterator I = List.begin(); I != List.end(); ++I) { struct stat Buf; - if (stat(string(*I + GetFileName()).c_str(),&Buf) != 0 && - stat(string(*I + GetFileName() + ".gz").c_str(),&Buf) != 0) - return _error->Errno("stat","Stat failed for %s", - string(*I + GetFileName()).c_str()); + bool found = false; + std::string file = std::string(*I).append(GetFileName()); + for (std::vector<APT::Configuration::Compressor>::const_iterator c = compressor.begin(); + c != compressor.end(); ++c) + { + if (stat((file + c->Extension).c_str(), &Buf) != 0) + continue; + found = true; + break; + } + + if (found == false) + return _error->Errno("stat", "Stat failed for %s", file.c_str()); TotalSize += Buf.st_size; - } + } - unsigned long CurrentSize = 0; + off_t CurrentSize = 0; unsigned int NotFound = 0; unsigned int WrongSize = 0; unsigned int Packages = 0; - for (vector<string>::iterator I = List.begin(); I != List.end(); I++) + for (vector<string>::iterator I = List.begin(); I != List.end(); ++I) { string OrigPath = string(*I,CDROM.length()); - unsigned long FileSize = 0; // Open the package file - FileFd Pkg; - if (FileExists(*I + GetFileName()) == true) - { - Pkg.Open(*I + GetFileName(),FileFd::ReadOnly); - FileSize = Pkg.Size(); - } - else - { - FileFd From(*I + GetFileName() + ".gz",FileFd::ReadOnly); - if (_error->PendingError() == true) - return false; - FileSize = From.Size(); - - // Get a temp file - FILE *tmp = tmpfile(); - if (tmp == 0) - return _error->Errno("tmpfile","Unable to create a tmp file"); - Pkg.Fd(dup(fileno(tmp))); - fclose(tmp); - - // Fork gzip - pid_t Process = fork(); - if (Process < 0) - return _error->Errno("fork","Couldn't fork gzip"); - - // The child - if (Process == 0) - { - dup2(From.Fd(),STDIN_FILENO); - dup2(Pkg.Fd(),STDOUT_FILENO); - SetCloseExec(STDIN_FILENO,false); - SetCloseExec(STDOUT_FILENO,false); - - const char *Args[3]; - string Tmp = _config->Find("Dir::bin::gzip","gzip"); - Args[0] = Tmp.c_str(); - Args[1] = "-d"; - Args[2] = 0; - execvp(Args[0],(char **)Args); - exit(100); - } - - // Wait for gzip to finish - if (ExecWait(Process,_config->Find("Dir::bin::gzip","gzip").c_str(),false) == false) - return _error->Error("gzip failed, perhaps the disk is full."); - - Pkg.Seek(0); - } + FileFd Pkg(*I + GetFileName(), FileFd::ReadOnly, FileFd::Auto); + off_t const FileSize = Pkg.Size(); + pkgTagFile Parser(&Pkg); if (_error->PendingError() == true) return false; @@ -128,12 +98,17 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List, (*I).c_str() + CDROM.length(),GetFileName()); string TargetF = _config->FindDir("Dir::State::lists") + "partial/"; TargetF += URItoFileName(S); + FileFd Target; if (_config->FindB("APT::CDROM::NoAct",false) == true) + { TargetF = "/dev/null"; - FileFd Target(TargetF,FileFd::WriteEmpty); - FILE *TargetFl = fdopen(dup(Target.Fd()),"w"); + Target.Open(TargetF,FileFd::WriteExists); + } else { + Target.Open(TargetF,FileFd::WriteAtomic); + } if (_error->PendingError() == true) return false; + FILE *TargetFl = fdopen(dup(Target.Fd()),"w"); if (TargetFl == 0) return _error->Errno("fdopen","Failed to reopen fd"); @@ -155,7 +130,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List, if(Progress) Progress->Progress(Parser.Offset()); string File; - unsigned long Size; + unsigned long long Size; if (GetFile(File,Size) == false) { fclose(TargetFl); @@ -166,7 +141,6 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List, File = OrigPath + ChopDirs(File,Chop); // See if the file exists - bool Mangled = false; if (NoStat == false || Hits < 10) { // Attempt to fix broken structure @@ -186,9 +160,10 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List, // Get the size struct stat Buf; - if (stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0 || + if (stat((CDROM + Prefix + File).c_str(),&Buf) != 0 || Buf.st_size == 0) { + bool Mangled = false; // Attempt to fix busted symlink support for one instance string OrigFile = File; string::size_type Start = File.find("binary-"); @@ -200,7 +175,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List, } if (Mangled == false || - stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0) + stat((CDROM + Prefix + File).c_str(),&Buf) != 0) { if (Debug == true) clog << "Missed(2): " << OrigFile << endl; @@ -210,7 +185,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List, } // Size match - if ((unsigned)Buf.st_size != Size) + if ((unsigned long long)Buf.st_size != Size) { if (Debug == true) clog << "Wrong Size: " << File << endl; @@ -263,17 +238,17 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List, ioprintf(msg, _("Wrote %i records with %i missing files.\n"), Packages, NotFound); else if (NotFound == 0 && WrongSize != 0) - ioprintf(msg, _("Wrote %i records with %i mismachted files\n"), + ioprintf(msg, _("Wrote %i records with %i mismatched files\n"), Packages, WrongSize); if (NotFound != 0 && WrongSize != 0) - ioprintf(msg, _("Wrote %i records with %i missing files and %i mismachted files\n"), Packages, NotFound, WrongSize); + ioprintf(msg, _("Wrote %i records with %i missing files and %i mismatched files\n"), Packages, NotFound, WrongSize); } if (Packages == 0) _error->Warning("No valid records were found."); if (NotFound + WrongSize > 10) - _error->Warning("Alot of entries were discarded, something may be wrong.\n"); + _error->Warning("A lot of entries were discarded, something may be wrong.\n"); return true; @@ -311,7 +286,7 @@ bool IndexCopy::ReconstructPrefix(string &Prefix,string OrigPath,string CD, while (1) { struct stat Buf; - if (stat(string(CD + MyPrefix + File).c_str(),&Buf) != 0) + if (stat((CD + MyPrefix + File).c_str(),&Buf) != 0) { if (Debug == true) cout << "Failed, " << CD + MyPrefix + File << endl; @@ -340,7 +315,7 @@ bool IndexCopy::ReconstructChop(unsigned long &Chop,string Dir,string File) while (1) { struct stat Buf; - if (stat(string(Dir + File).c_str(),&Buf) != 0) + if (stat((Dir + File).c_str(),&Buf) != 0) { File = ChopDirs(File,1); Depth++; @@ -375,9 +350,6 @@ bool IndexCopy::ReconstructChop(unsigned long &Chop,string Dir,string File) */ void IndexCopy::ConvertToSourceList(string CD,string &Path) { - char S[300]; - snprintf(S,sizeof(S),"binary-%s",_config->Find("Apt::Architecture").c_str()); - // Strip the cdrom base path Path = string(Path,CD.length()); if (Path.empty() == true) @@ -413,7 +385,13 @@ void IndexCopy::ConvertToSourceList(string CD,string &Path) return; string Binary = string(Path,Slash+1,BinSlash - Slash-1); - if (Binary != S && Binary != "source") + if (strncmp(Binary.c_str(), "binary-", strlen("binary-")) == 0) + { + Binary.erase(0, strlen("binary-")); + if (APT::Configuration::checkArchitecture(Binary) == false) + continue; + } + else if (Binary != "source") continue; Path = Dist + ' ' + Comp; @@ -444,7 +422,7 @@ bool IndexCopy::GrabFirst(string Path,string &To,unsigned int Depth) // PackageCopy::GetFile - Get the file information from the section /*{{{*/ // --------------------------------------------------------------------- /* */ -bool PackageCopy::GetFile(string &File,unsigned long &Size) +bool PackageCopy::GetFile(string &File,unsigned long long &Size) { File = Section->FindS("Filename"); Size = Section->FindI("Size"); @@ -458,8 +436,8 @@ bool PackageCopy::GetFile(string &File,unsigned long &Size) /* */ bool PackageCopy::RewriteEntry(FILE *Target,string File) { - TFRewriteData Changes[] = {{"Filename",File.c_str()}, - {}}; + TFRewriteData Changes[] = {{ "Filename", File.c_str(), NULL }, + { NULL, NULL, NULL }}; if (TFRewrite(Target,*Section,TFRewritePackageOrder,Changes) == false) return false; @@ -470,7 +448,7 @@ bool PackageCopy::RewriteEntry(FILE *Target,string File) // SourceCopy::GetFile - Get the file information from the section /*{{{*/ // --------------------------------------------------------------------- /* */ -bool SourceCopy::GetFile(string &File,unsigned long &Size) +bool SourceCopy::GetFile(string &File,unsigned long long &Size) { string Files = Section->FindS("Files"); if (Files.empty() == true) @@ -493,7 +471,7 @@ bool SourceCopy::GetFile(string &File,unsigned long &Size) return _error->Error("Error parsing file record"); // Parse the size and append the directory - Size = atoi(sSize.c_str()); + Size = strtoull(sSize.c_str(), NULL, 10); File = Base + File; return true; } @@ -504,51 +482,56 @@ bool SourceCopy::GetFile(string &File,unsigned long &Size) bool SourceCopy::RewriteEntry(FILE *Target,string File) { string Dir(File,0,File.rfind('/')); - TFRewriteData Changes[] = {{"Directory",Dir.c_str()}, - {}}; + TFRewriteData Changes[] = {{ "Directory", Dir.c_str(), NULL }, + { NULL, NULL, NULL }}; if (TFRewrite(Target,*Section,TFRewriteSourceOrder,Changes) == false) return false; fputc('\n',Target); return true; } - - /*}}}*/ - +// SigVerify::Verify - Verify a files md5sum against its metaindex /*{{{*/ +// --------------------------------------------------------------------- +/* */ bool SigVerify::Verify(string prefix, string file, indexRecords *MetaIndex) { const indexRecords::checkSum *Record = MetaIndex->Lookup(file); + bool const Debug = _config->FindB("Debug::aptcdrom",false); - if (!Record) + // we skip non-existing files in the verifcation of the Release file + // as non-existing files do not harm, but a warning scares people and + // makes it hard to strip unneeded files from an ISO like uncompressed + // indexes as it is done on the mirrors (see also LP: #255545 ) + if(!RealFileExists(prefix+file)) { - _error->Warning("Can't find authentication record for: %s",file.c_str()); + if (Debug == true) + cout << "Skipping nonexistent in " << prefix << " file " << file << std::endl; + return true; + } + + if (!Record) + { + _error->Warning(_("Can't find authentication record for: %s"), file.c_str()); return false; } - MD5Summation sum; - FileFd Fd(prefix+file, FileFd::ReadOnly); - sum.AddFD(Fd.Fd(), Fd.Size()); - Fd.Close(); - string MD5 = (string)sum.Result(); - - if (Record->MD5Hash != MD5) + if (!Record->Hash.VerifyFile(prefix+file)) { - _error->Warning("MD5 mismatch for: %s",file.c_str()); + _error->Warning(_("Hash mismatch for: %s"),file.c_str()); return false; } - if(_config->FindB("Debug::aptcdrom",false)) + if(Debug == true) { cout << "File: " << prefix+file << endl; - cout << "Expected MD5sum: " << Record->MD5Hash << endl; - cout << "got: " << MD5 << endl << endl; + cout << "Expected Hash " << Record->Hash.toStr() << endl; } return true; } - -bool SigVerify::CopyMetaIndex(string CDROM, string CDName, + /*}}}*/ +bool SigVerify::CopyMetaIndex(string CDROM, string CDName, /*{{{*/ string prefix, string file) { char S[400]; @@ -559,26 +542,24 @@ bool SigVerify::CopyMetaIndex(string CDROM, string CDName, FileFd Target; FileFd Rel; - Target.Open(TargetF,FileFd::WriteEmpty); + Target.Open(TargetF,FileFd::WriteAtomic); Rel.Open(prefix + file,FileFd::ReadOnly); - if (_error->PendingError() == true) - return false; if (CopyFile(Rel,Target) == false) - return false; - + return _error->Error("Copying of '%s' for '%s' from '%s' failed", file.c_str(), CDName.c_str(), prefix.c_str()); + return true; } - -bool SigVerify::CopyAndVerify(string CDROM,string Name,vector<string> &SigList, - vector<string> PkgList,vector<string> SrcList) + /*}}}*/ +bool SigVerify::CopyAndVerify(string CDROM,string Name,vector<string> &SigList, /*{{{*/ + vector<string> /*PkgList*/,vector<string> /*SrcList*/) { - if (SigList.size() == 0) + if (SigList.empty() == true) return true; bool Debug = _config->FindB("Debug::aptcdrom",false); // Read all Release files - for (vector<string>::iterator I = SigList.begin(); I != SigList.end(); I++) + for (vector<string>::iterator I = SigList.begin(); I != SigList.end(); ++I) { if(Debug) cout << "Signature verify for: " << *I << endl; @@ -586,48 +567,60 @@ bool SigVerify::CopyAndVerify(string CDROM,string Name,vector<string> &SigList, indexRecords *MetaIndex = new indexRecords; string prefix = *I; + string const releasegpg = *I+"Release.gpg"; + string const release = *I+"Release"; + string const inrelease = *I+"InRelease"; + bool useInRelease = true; + // a Release.gpg without a Release should never happen - if(!FileExists(*I+"Release")) + if (RealFileExists(inrelease) == true) + ; + else if(RealFileExists(release) == false || RealFileExists(releasegpg) == false) + { + delete MetaIndex; continue; + } + else + useInRelease = false; - - // verify the gpg signature of "Release" - // gpg --verify "*I+Release.gpg", "*I+Release" - string gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv"); - string pubringpath = _config->Find("Apt::GPGV::TrustedKeyring", "/etc/apt/trusted.gpg"); pid_t pid = ExecFork(); if(pid < 0) { _error->Error("Fork failed"); return false; } - if(pid == 0) { - execlp(gpgvpath.c_str(), gpgvpath.c_str(), "--keyring", - pubringpath.c_str(), string(*I+"Release.gpg").c_str(), - string(*I+"Release").c_str(), NULL); + if(pid == 0) + { + if (useInRelease == true) + ExecGPGV(inrelease, inrelease); + else + ExecGPGV(release, releasegpg); } + if(!ExecWait(pid, "gpgv")) { _error->Warning("Signature verification failed for: %s", - string(*I+"Release.gpg").c_str()); + (useInRelease ? inrelease.c_str() : releasegpg.c_str())); // something went wrong, don't copy the Release.gpg // FIXME: delete any existing gpg file? + delete MetaIndex; continue; } // Open the Release file and add it to the MetaIndex - if(!MetaIndex->Load(*I+"Release")) + if(!MetaIndex->Load(release)) { - _error->Error(MetaIndex->ErrorText.c_str()); + _error->Error("%s",MetaIndex->ErrorText.c_str()); return false; } // go over the Indexfiles and see if they verify // if so, remove them from our copy of the lists vector<string> keys = MetaIndex->MetaKeys(); - for (vector<string>::iterator I = keys.begin(); I != keys.end(); I++) + for (vector<string>::iterator I = keys.begin(); I != keys.end(); ++I) { if(!Verify(prefix,*I, MetaIndex)) { // something went wrong, don't copy the Release.gpg // FIXME: delete any existing gpg file? + _error->Discard(); continue; } } @@ -636,9 +629,165 @@ bool SigVerify::CopyAndVerify(string CDROM,string Name,vector<string> &SigList, delete MetaIndex; // everything was fine, copy the Release and Release.gpg file - CopyMetaIndex(CDROM, Name, prefix, "Release"); - CopyMetaIndex(CDROM, Name, prefix, "Release.gpg"); + if (useInRelease == true) + CopyMetaIndex(CDROM, Name, prefix, "InRelease"); + else + { + CopyMetaIndex(CDROM, Name, prefix, "Release"); + CopyMetaIndex(CDROM, Name, prefix, "Release.gpg"); + } } return true; } + /*}}}*/ +// SigVerify::RunGPGV - deprecated wrapper calling ExecGPGV /*{{{*/ +APT_NORETURN bool SigVerify::RunGPGV(std::string const &File, std::string const &FileOut, + int const &statusfd, int fd[2]) { + ExecGPGV(File, FileOut, statusfd, fd); +} +APT_NORETURN bool SigVerify::RunGPGV(std::string const &File, std::string const &FileOut, + int const &statusfd) { + ExecGPGV(File, FileOut, statusfd); +} + /*}}}*/ +bool TranslationsCopy::CopyTranslations(string CDROM,string Name, /*{{{*/ + vector<string> &List, pkgCdromStatus *log) +{ + OpProgress *Progress = NULL; + if (List.empty() == true) + return true; + + if(log) + Progress = log->GetOpProgress(); + + bool Debug = _config->FindB("Debug::aptcdrom",false); + + // Prepare the progress indicator + off_t TotalSize = 0; + std::vector<APT::Configuration::Compressor> const compressor = APT::Configuration::getCompressors(); + for (vector<string>::iterator I = List.begin(); I != List.end(); ++I) + { + struct stat Buf; + bool found = false; + std::string file = *I; + for (std::vector<APT::Configuration::Compressor>::const_iterator c = compressor.begin(); + c != compressor.end(); ++c) + { + if (stat((file + c->Extension).c_str(), &Buf) != 0) + continue; + found = true; + break; + } + + if (found == false) + return _error->Errno("stat", "Stat failed for %s", file.c_str()); + TotalSize += Buf.st_size; + } + + off_t CurrentSize = 0; + unsigned int NotFound = 0; + unsigned int WrongSize = 0; + unsigned int Packages = 0; + for (vector<string>::iterator I = List.begin(); I != List.end(); ++I) + { + // Open the package file + FileFd Pkg(*I, FileFd::ReadOnly, FileFd::Auto); + off_t const FileSize = Pkg.Size(); + + pkgTagFile Parser(&Pkg); + if (_error->PendingError() == true) + return false; + + // Open the output file + char S[400]; + snprintf(S,sizeof(S),"cdrom:[%s]/%s",Name.c_str(), + (*I).c_str() + CDROM.length()); + string TargetF = _config->FindDir("Dir::State::lists") + "partial/"; + TargetF += URItoFileName(S); + FileFd Target; + if (_config->FindB("APT::CDROM::NoAct",false) == true) + { + TargetF = "/dev/null"; + Target.Open(TargetF,FileFd::WriteExists); + } else { + Target.Open(TargetF,FileFd::WriteAtomic); + } + if (_error->PendingError() == true) + return false; + FILE *TargetFl = fdopen(dup(Target.Fd()),"w"); + if (TargetFl == 0) + return _error->Errno("fdopen","Failed to reopen fd"); + + // Setup the progress meter + if(Progress) + Progress->OverallProgress(CurrentSize,TotalSize,FileSize, + string("Reading Translation Indexes")); + + // Parse + if(Progress) + Progress->SubProgress(Pkg.Size()); + pkgTagSection Section; + this->Section = &Section; + string Prefix; + unsigned long Hits = 0; + while (Parser.Step(Section) == true) + { + if(Progress) + Progress->Progress(Parser.Offset()); + + const char *Start; + const char *Stop; + Section.GetSection(Start,Stop); + fwrite(Start,Stop-Start, 1, TargetFl); + fputc('\n',TargetFl); + + Packages++; + Hits++; + } + fclose(TargetFl); + + if (Debug == true) + cout << " Processed by using Prefix '" << Prefix << "' and chop " << endl; + + if (_config->FindB("APT::CDROM::NoAct",false) == false) + { + // Move out of the partial directory + Target.Close(); + string FinalF = _config->FindDir("Dir::State::lists"); + FinalF += URItoFileName(S); + if (rename(TargetF.c_str(),FinalF.c_str()) != 0) + return _error->Errno("rename","Failed to rename"); + } + + + CurrentSize += FileSize; + } + if(Progress) + Progress->Done(); + + // Some stats + if(log) { + stringstream msg; + if(NotFound == 0 && WrongSize == 0) + ioprintf(msg, _("Wrote %i records.\n"), Packages); + else if (NotFound != 0 && WrongSize == 0) + ioprintf(msg, _("Wrote %i records with %i missing files.\n"), + Packages, NotFound); + else if (NotFound == 0 && WrongSize != 0) + ioprintf(msg, _("Wrote %i records with %i mismatched files\n"), + Packages, WrongSize); + if (NotFound != 0 && WrongSize != 0) + ioprintf(msg, _("Wrote %i records with %i missing files and %i mismatched files\n"), Packages, NotFound, WrongSize); + } + + if (Packages == 0) + _error->Warning("No valid records were found."); + + if (NotFound + WrongSize > 10) + _error->Warning("A lot of entries were discarded, something may be wrong.\n"); + + + return true; +} + /*}}}*/