X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/30b683f4f3021cd191ffef04bfaf2deb65820a52..1d970e6ce97385ed719a1ca169ec5cc7bfb82fea:/apt-pkg/acquire-item.cc diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index eee1097e9..3c66369cf 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -64,16 +64,25 @@ static void printHashSumComparision(std::string const &URI, HashStringList const /*}}}*/ // Acquire::Item::Item - Constructor /*{{{*/ +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif pkgAcquire::Item::Item(pkgAcquire *Owner, HashStringList const &ExpectedHashes, - unsigned long TransactionID) + pkgAcqMetaBase *TransactionManager) : Owner(Owner), FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false), - Local(false), QueueCounter(0), TransactionID(TransactionID), + Local(false), QueueCounter(0), TransactionManager(TransactionManager), ExpectedAdditionalItems(0), ExpectedHashes(ExpectedHashes) { Owner->Add(this); Status = StatIdle; + if(TransactionManager != NULL) + TransactionManager->Add(this); } +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif /*}}}*/ // Acquire::Item::~Item - Destructor /*{{{*/ // --------------------------------------------------------------------- @@ -155,7 +164,7 @@ void pkgAcquire::Item::Done(string Message,unsigned long long Size,HashStringLis // --------------------------------------------------------------------- /* This helper function is used by a lot of item methods as their final step */ -void pkgAcquire::Item::Rename(string From,string To) +bool pkgAcquire::Item::Rename(string From,string To) { if (rename(From.c_str(),To.c_str()) != 0) { @@ -163,8 +172,10 @@ void pkgAcquire::Item::Rename(string From,string To) snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno), From.c_str(),To.c_str()); Status = StatError; - ErrorText = S; + ErrorText += S; + return false; } + return true; } /*}}}*/ bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/ @@ -189,6 +200,14 @@ bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const Status = StatError; // do not report as usually its not the mirrors fault, but Portal/Proxy break; + case SignatureError: + ErrorText = _("Signature error"); + Status = StatError; + break; + case NotClearsigned: + ErrorText = _("Does not start with a cleartext signature"); + Status = StatError; + break; } return false; } @@ -237,120 +256,6 @@ void pkgAcquire::Item::ReportMirrorFailure(string FailCode) } } /*}}}*/ -// AcqSubIndex::AcqSubIndex - Constructor /*{{{*/ -// --------------------------------------------------------------------- -/* Get a sub-index file based on checksums from a 'master' file and - possibly query additional files */ -pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, - unsigned long TransactionID, - string const &URI, - string const &URIDesc, string const &ShortDesc, - HashStringList const &ExpectedHashes) - : Item(Owner, ExpectedHashes, TransactionID) -{ - /* XXX: Beware: Currently this class does nothing (of value) anymore ! */ - Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false); - - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(URI); - - Desc.URI = URI; - Desc.Description = URIDesc; - Desc.Owner = this; - Desc.ShortDesc = ShortDesc; - - QueueURI(Desc); - - if(Debug) - std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl; -} - /*}}}*/ -// AcqSubIndex::Custom600Headers - Insert custom request headers /*{{{*/ -// --------------------------------------------------------------------- -/* The only header we use is the last-modified header. */ -string pkgAcqSubIndex::Custom600Headers() const -{ - string Final = _config->FindDir("Dir::State::lists"); - Final += URItoFileName(Desc.URI); - - struct stat Buf; - if (stat(Final.c_str(),&Buf) != 0) - return "\nIndex-File: true\nFail-Ignore: true\n"; - return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); -} - /*}}}*/ -void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/ -{ - if(Debug) - std::clog << "pkgAcqSubIndex failed: " << Desc.URI << " with " << Message << std::endl; - - Complete = false; - Status = StatDone; - Dequeue(); - - // No good Index is provided -} - /*}}}*/ -void pkgAcqSubIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ - pkgAcquire::MethodConfig *Cnf) -{ - if(Debug) - std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl; - - string FileName = LookupTag(Message,"Filename"); - if (FileName.empty() == true) - { - Status = StatError; - ErrorText = "Method gave a blank filename"; - return; - } - - if (FileName != DestFile) - { - Local = true; - Desc.URI = "copy:" + FileName; - QueueURI(Desc); - return; - } - - Item::Done(Message, Size, Hashes, Cnf); - - string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI); - - /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */ - indexRecords SubIndexParser; - if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) { - Status = StatError; - ErrorText = SubIndexParser.ErrorText; - return; - } - - // success in downloading the index - // rename the index - if(Debug) - std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl; - Rename(DestFile,FinalFile); - chmod(FinalFile.c_str(),0644); - DestFile = FinalFile; - - if(ParseIndex(DestFile) == false) - return Failed("", NULL); - - Complete = true; - Status = StatDone; - Dequeue(); - return; -} - /*}}}*/ -bool pkgAcqSubIndex::ParseIndex(string const &IndexFile) /*{{{*/ -{ - indexRecords SubIndexParser; - if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false) - return false; - // so something with the downloaded index - return true; -} - /*}}}*/ // AcqDiffIndex::AcqDiffIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* Get the DiffIndex file first and see if there are patches available @@ -359,12 +264,12 @@ bool pkgAcqSubIndex::ParseIndex(string const &IndexFile) /*{{{*/ * the original packages file */ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) - : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHashes, - MetaIndexParser) + : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, + MetaIndexParser), PackagesFileReadyInPartial(false) { Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); @@ -460,9 +365,13 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ // we have the same sha1 as the server so we are done here if(Debug) std::clog << "Package file is up-to-date" << std::endl; + // ensure we have no leftovers from previous runs + std::string Partial = _config->FindDir("Dir::State::lists"); + Partial += "partial/" + URItoFileName(RealURI); + unlink(Partial.c_str()); // list cleanup needs to know that this file as well as the already // present index is ours, so we create an empty diff to save it for us - new pkgAcqIndexDiffs(Owner, TransactionID, Target, + new pkgAcqIndexDiffs(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser, ServerSha1, available_patches); return true; @@ -530,6 +439,21 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ // we have something, queue the next diff if(found) { + // FIXME: make this use the method + PackagesFileReadyInPartial = true; + std::string Partial = _config->FindDir("Dir::State::lists"); + Partial += "partial/" + URItoFileName(RealURI); + + FileFd From(CurrentPackagesFile, FileFd::ReadOnly); + FileFd To(Partial, FileFd::WriteEmpty); + if(CopyFile(From, To) == false) + return _error->Errno("CopyFile", "failed to copy"); + + if(Debug) + std::cerr << "Done copying " << CurrentPackagesFile + << " -> " << Partial + << std::endl; + // queue the diffs string::size_type const last_space = Description.rfind(" "); if(last_space != string::npos) @@ -550,7 +474,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ if (pdiff_merge == false) { - new pkgAcqIndexDiffs(Owner, TransactionID, Target, ExpectedHashes, + new pkgAcqIndexDiffs(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser, ServerSha1, available_patches); } @@ -559,7 +483,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ std::vector *diffs = new std::vector(available_patches.size()); for(size_t i = 0; i < available_patches.size(); ++i) (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, - TransactionID, + TransactionManager, Target, ExpectedHashes, MetaIndexParser, @@ -567,10 +491,10 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ diffs); } - Complete = false; - Status = StatDone; - Dequeue(); - return true; + Complete = false; + Status = StatDone; + Dequeue(); + return true; } } @@ -588,7 +512,7 @@ void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/ std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser); + new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser); Complete = false; Status = StatDone; @@ -603,6 +527,21 @@ void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList Item::Done(Message, Size, Hashes, Cnf); + // verify the index target + if(Target && Target->MetaKey != "" && MetaIndexParser && Hashes.usable()) + { + std::string IndexMetaKey = Target->MetaKey + ".diff/Index"; + indexRecords::checkSum *Record = MetaIndexParser->Lookup(IndexMetaKey); + if(Record && Record->Hashes.usable() && Hashes != Record->Hashes) + { + RenameOnError(HashSumMismatch); + printHashSumComparision(RealURI, Record->Hashes, Hashes); + Failed(Message, Cnf); + return; + } + + } + string FinalFile; FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); @@ -631,13 +570,13 @@ void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList * for each diff and the index */ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser, string ServerSha1, vector diffs) - : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser), + : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser), available_patches(diffs), ServerSha1(ServerSha1) { @@ -653,7 +592,9 @@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, if(available_patches.empty() == true) { - // we are done (yeah!) + // we are done (yeah!), check hashes against the final file + DestFile = _config->FindDir("Dir::State::lists"); + DestFile += URItoFileName(Target->URI); Finish(true); } else @@ -669,7 +610,7 @@ void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/) if(Debug) std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser); + new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser); Finish(); } /*}}}*/ @@ -685,13 +626,6 @@ void pkgAcqIndexDiffs::Finish(bool allDone) // the file will be cleaned if(allDone) { - DestFile = _config->FindDir("Dir::State::lists"); - DestFile += URItoFileName(RealURI); - - // FIXME: we want the rred stuff to use the real transactional update - // this is just a workaround - PartialFile = DestFile; - if(HashSums().usable() && !HashSums().VerifyFile(DestFile)) { RenameOnError(HashSumMismatch); @@ -699,6 +633,18 @@ void pkgAcqIndexDiffs::Finish(bool allDone) return; } + // queue for copy + PartialFile = _config->FindDir("Dir::State::lists")+"partial/"+URItoFileName(RealURI); + + DestFile = _config->FindDir("Dir::State::lists"); + DestFile += URItoFileName(RealURI); + + // this happens if we have a up-to-date indexfile + if(!FileExists(PartialFile)) + PartialFile = DestFile; + + TransactionManager->TransactionStageCopy(this, PartialFile, DestFile); + // this is for the "real" finish Complete = true; Status = StatDone; @@ -718,10 +664,15 @@ void pkgAcqIndexDiffs::Finish(bool allDone) /*}}}*/ bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ { - // calc sha1 of the just patched file string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); + FinalFile += "partial/" + URItoFileName(RealURI); + + if(!FileExists(FinalFile)) + { + Failed("No FinalFile " + FinalFile + " available", NULL); + return false; + } FileFd fd(FinalFile, FileFd::ReadOnly); SHA1Summation SHA1; @@ -731,6 +682,7 @@ bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ std::clog << "QueueNextDiff: " << FinalFile << " (" << local_sha1 << ")"<FindDir("Dir::State::lists")+URItoFileName(RealURI); + FinalFile = _config->FindDir("Dir::State::lists")+"partial/"+URItoFileName(RealURI); // success in downloading a diff, enter ApplyDiff state if(State == StateFetchDiff) @@ -795,7 +749,15 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringLi Local = true; Desc.URI = "rred:" + FinalFile; QueueURI(Desc); + ActiveSubprocess = "rred"; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif Mode = "rred"; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif return; } @@ -818,24 +780,26 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringLi // see if there is more to download if(available_patches.empty() == false) { - new pkgAcqIndexDiffs(Owner, TransactionID, Target, + new pkgAcqIndexDiffs(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser, ServerSha1, available_patches); return Finish(); } else + // update + DestFile = FinalFile; return Finish(true); } } /*}}}*/ // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser, DiffInfo const &patch, std::vector const * const allPatches) - : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser), + : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser), patch(patch), allPatches(allPatches), State(StateFetchDiff) { @@ -878,7 +842,7 @@ void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*C // first failure means we should fallback State = StateErrorDiff; std::clog << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser); + new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser); } /*}}}*/ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ @@ -889,7 +853,9 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri Item::Done(Message,Size,Hashes,Cnf); - string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); + // FIXME: verify download before feeding it to rred + + string const FinalFile = _config->FindDir("Dir::State::lists") + "partial/" + URItoFileName(RealURI); if (State == StateFetchDiff) { @@ -916,7 +882,15 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri Local = true; Desc.URI = "rred:" + FinalFile; QueueURI(Desc); + ActiveSubprocess = "rred"; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif Mode = "rred"; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif return; } // success in download/apply all diffs, clean up @@ -929,23 +903,26 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri return; } + + std::string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + // move the result into place if(Debug) - std::clog << "Moving patched file in place: " << std::endl + std::clog << "Queue patched file in place: " << std::endl << DestFile << " -> " << FinalFile << std::endl; - Rename(DestFile, FinalFile); - chmod(FinalFile.c_str(), 0644); - // otherwise lists cleanup will eat the file - DestFile = FinalFile; - // FIXME: make the merged rred code really transactional - PartialFile = FinalFile; + // queue for copy by the transaction manager + TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); // ensure the ed's are gone regardless of list-cleanup for (std::vector::const_iterator I = allPatches->begin(); I != allPatches->end(); ++I) { - std::string patch = FinalFile + ".ed." + (*I)->patch.file + ".gz"; + std::string PartialFile = _config->FindDir("Dir::State::lists"); + PartialFile += "partial/" + URItoFileName(RealURI); + std::string patch = PartialFile + ".ed." + (*I)->patch.file + ".gz"; + std::cerr << patch << std::endl; unlink(patch.c_str()); } @@ -956,40 +933,61 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri } } /*}}}*/ + +// AcqBaseIndex::VerifyHashByMetaKey - verify hash for the given metakey /*{{{*/ +bool pkgAcqBaseIndex::VerifyHashByMetaKey(HashStringList const &Hashes) +{ + if(MetaKey != "" && Hashes.usable()) + { + indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); + if(Record && Record->Hashes.usable() && Hashes != Record->Hashes) + { + printHashSumComparision(RealURI, Record->Hashes, Hashes); + return false; + } + } + return true; +} + + // AcqIndex::AcqIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The package file is added to the queue and a second class is instantiated to fetch the revision file */ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, - HashStringList const &ExpectedHash, string comprExt) - : pkgAcqBaseIndex(Owner, 0, NULL, ExpectedHash, NULL), RealURI(URI) + HashStringList const &ExpectedHash) + : pkgAcqBaseIndex(Owner, 0, NULL, ExpectedHash, NULL) { + RealURI = URI; + AutoSelectCompression(); Init(URI, URIDesc, ShortDesc); if(_config->FindB("Debug::Acquire::Transaction", false) == true) - std::clog << "New pkgIndex with TransactionID " - << TransactionID << std::endl; + std::clog << "New pkgIndex with TransactionManager " + << TransactionManager << std::endl; } /*}}}*/ // AcqIndex::AcqIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, IndexTarget const *Target, HashStringList const &ExpectedHash, indexRecords *MetaIndexParser) - : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHash, - MetaIndexParser), RealURI(Target->URI) + : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHash, + MetaIndexParser) { + RealURI = Target->URI; + // autoselect the compression method AutoSelectCompression(); Init(Target->URI, Target->Description, Target->ShortDesc); if(_config->FindB("Debug::Acquire::Transaction", false) == true) - std::clog << "New pkgIndex with TransactionID " - << TransactionID << std::endl; + std::clog << "New pkgIndex with TransactionManager " + << TransactionManager << std::endl; } /*}}}*/ // AcqIndex::AutoSelectCompression - Select compression /*{{{*/ @@ -997,31 +995,38 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, void pkgAcqIndex::AutoSelectCompression() { std::vector types = APT::Configuration::getCompressionTypes(); - CompressionExtension = ""; + CompressionExtensions = ""; if (ExpectedHashes.usable()) { - for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) - if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true) - CompressionExtension.append(*t).append(" "); + for (std::vector::const_iterator t = types.begin(); + t != types.end(); ++t) + { + std::string CompressedMetaKey = string(Target->MetaKey).append(".").append(*t); + if (*t == "uncompressed" || + MetaIndexParser->Exists(CompressedMetaKey) == true) + CompressionExtensions.append(*t).append(" "); + } } else { for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) - CompressionExtension.append(*t).append(" "); + CompressionExtensions.append(*t).append(" "); } - if (CompressionExtension.empty() == false) - CompressionExtension.erase(CompressionExtension.end()-1); + if (CompressionExtensions.empty() == false) + CompressionExtensions.erase(CompressionExtensions.end()-1); } // AcqIndex::Init - defered Constructor /*{{{*/ -void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) { - Decompression = false; - Erase = false; +// --------------------------------------------------------------------- +void pkgAcqIndex::Init(string const &URI, string const &URIDesc, + string const &ShortDesc) +{ + Stage = STAGE_DOWNLOAD; DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); - std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); - if (comprExt == "uncompressed") + CurrentCompressionExtension = CompressionExtensions.substr(0, CompressionExtensions.find(' ')); + if (CurrentCompressionExtension == "uncompressed") { Desc.URI = URI; if(Target) @@ -1029,9 +1034,10 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S } else { - Desc.URI = URI + '.' + comprExt; + Desc.URI = URI + '.' + CurrentCompressionExtension; + DestFile = DestFile + '.' + CurrentCompressionExtension; if(Target) - MetaKey = string(Target->MetaKey) + '.' + comprExt; + MetaKey = string(Target->MetaKey) + '.' + CurrentCompressionExtension; } // load the filesize @@ -1088,16 +1094,9 @@ void pkgAcqIndex::InitByHashIfNeeded(const std::string MetaKey) /* The only header we use is the last-modified header. */ string pkgAcqIndex::Custom600Headers() const { - string Final = _config->FindDir("Dir::State::lists"); - Final += URItoFileName(RealURI); - if (_config->FindB("Acquire::GzipIndexes",false)) - Final += ".gz"; + string Final = GetFinalFilename(); string msg = "\nIndex-File: true"; - // FIXME: this really should use "IndexTarget::IsOptional()" but that - // seems to be difficult without breaking ABI - if (ShortDesc().find("Translation") != 0) - msg += "\nFail-Ignore: true"; struct stat Buf; if (stat(Final.c_str(),&Buf) == 0) msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); @@ -1110,56 +1109,88 @@ string pkgAcqIndex::Custom600Headers() const /* */ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ { - size_t const nextExt = CompressionExtension.find(' '); + size_t const nextExt = CompressionExtensions.find(' '); if (nextExt != std::string::npos) { - CompressionExtension = CompressionExtension.substr(nextExt+1); + CompressionExtensions = CompressionExtensions.substr(nextExt+1); Init(RealURI, Desc.Description, Desc.ShortDesc); return; } // on decompression failure, remove bad versions in partial/ - if (Decompression && Erase) { - string s = _config->FindDir("Dir::State::lists") + "partial/"; - s.append(URItoFileName(RealURI)); - unlink(s.c_str()); + if (Stage == STAGE_DECOMPRESS_AND_VERIFY) + { + unlink(EraseFileName.c_str()); } Item::Failed(Message,Cnf); /// cancel the entire transaction - Owner->AbortTransaction(TransactionID); + TransactionManager->AbortTransaction(); } /*}}}*/ // pkgAcqIndex::GetFinalFilename - Return the full final file path /*{{{*/ // --------------------------------------------------------------------- /* */ -std::string pkgAcqIndex::GetFinalFilename(std::string const &URI, - std::string const &compExt) +std::string pkgAcqIndex::GetFinalFilename() const { std::string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(URI); - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") - FinalFile += ".gz"; + FinalFile += URItoFileName(RealURI); + if (_config->FindB("Acquire::GzipIndexes",false) == true) + FinalFile += '.' + CurrentCompressionExtension; return FinalFile; } /*}}}*/ // AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName) +void pkgAcqIndex::ReverifyAfterIMS() { - std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") - DestFile += ".gz"; + // update destfile to *not* include the compression extension when doing + // a reverify (as its uncompressed on disk already) + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI); + + // adjust DestFile if its compressed on disk + if (_config->FindB("Acquire::GzipIndexes",false) == true) + DestFile += '.' + CurrentCompressionExtension; // copy FinalFile into partial/ so that we check the hash again - string FinalFile = GetFinalFilename(RealURI, compExt); - Decompression = true; + string FinalFile = GetFinalFilename(); + Stage = STAGE_DECOMPRESS_AND_VERIFY; Desc.URI = "copy:" + FinalFile; QueueURI(Desc); } /*}}}*/ + +// AcqIndex::ValidateFile - Validate the content of the downloaded file /*{{{*/ +// -------------------------------------------------------------------------- +bool pkgAcqIndex::ValidateFile(const std::string &FileName) +{ + // FIXME: this can go away once we only ever download stuff that + // has a valid hash and we never do GET based probing + // FIXME2: this also leaks debian-isms into the code and should go therefore + + /* Always validate the index file for correctness (all indexes must + * have a Package field) (LP: #346386) (Closes: #627642) + */ + FileFd fd(FileName, FileFd::ReadOnly, FileFd::Extension); + // Only test for correctness if the content of the file is not empty + // (empty is ok) + if (fd.Size() > 0) + { + pkgTagSection sec; + pkgTagFile tag(&fd); + + // all our current indexes have a field 'Package' in each section + if (_error->PendingError() == true || + tag.Step(sec) == false || + sec.Exists("Package") == false) + return false; + } + return true; +} + /*}}}*/ // AcqIndex::Done - Finished a fetch /*{{{*/ // --------------------------------------------------------------------- /* This goes through a number of states.. On the initial fetch the @@ -1167,92 +1198,58 @@ void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName) to the uncompressed version of the file. If this is so the file is copied into the partial directory. In all other cases the file is decompressed with a compressed uri. */ -void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, +void pkgAcqIndex::Done(string Message, + unsigned long long Size, + HashStringList const &Hashes, pkgAcquire::MethodConfig *Cfg) { Item::Done(Message,Size,Hashes,Cfg); - std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); - if (Decompression == true) + switch(Stage) { - if (ExpectedHashes.usable() && ExpectedHashes != Hashes) - { - RenameOnError(HashSumMismatch); - printHashSumComparision(RealURI, ExpectedHashes, Hashes); - Failed(Message, Cfg); - return; - } - - // FIXME: this can go away once we only ever download stuff that - // has a valid hash and we never do GET based probing - // - /* Always verify the index file for correctness (all indexes must - * have a Package field) (LP: #346386) (Closes: #627642) - */ - FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Extension); - // Only test for correctness if the content of the file is not empty - // (empty is ok) - if (fd.Size() > 0) - { - pkgTagSection sec; - pkgTagFile tag(&fd); - - // all our current indexes have a field 'Package' in each section - if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false) - { - RenameOnError(InvalidFormat); - Failed(Message, Cfg); - return; - } - } - - // FIXME: can we void the "Erase" bool here as its very non-local? - std::string CompressedFile = _config->FindDir("Dir::State::lists") + "partial/"; - CompressedFile += URItoFileName(RealURI); - // Remove the compressed version. - if (Erase == true) - unlink(CompressedFile.c_str()); - - // Done, queue for rename on transaction finished - PartialFile = DestFile; - DestFile = GetFinalFilename(RealURI, compExt); - - return; + case STAGE_DOWNLOAD: + StageDownloadDone(Message, Hashes, Cfg); + break; + case STAGE_DECOMPRESS_AND_VERIFY: + StageDecompressDone(Message, Hashes, Cfg); + break; } - - // FIXME: use the same method to find - // check the compressed hash too - if(MetaKey != "" && Hashes.size() > 0) +} + +// AcqIndex::StageDownloadDone - Queue for decompress and verify /*{{{*/ +void pkgAcqIndex::StageDownloadDone(string Message, + HashStringList const &Hashes, + pkgAcquire::MethodConfig *Cfg) +{ + // First check if the calculcated Hash of the (compressed) downloaded + // file matches the hash we have in the MetaIndexRecords for this file + if(VerifyHashByMetaKey(Hashes) == false) { - indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); - if(Record && Record->Hashes.usable() && Hashes != Record->Hashes) - { - RenameOnError(HashSumMismatch); - printHashSumComparision(RealURI, Record->Hashes, Hashes); - Failed(Message, Cfg); - return; - } + RenameOnError(HashSumMismatch); + Failed(Message, Cfg); + return; } - Erase = false; Complete = true; // Handle the unzipd case string FileName = LookupTag(Message,"Alt-Filename"); if (FileName.empty() == false) { - // The files timestamp matches - if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true) - { - ReverifyAfterIMS(FileName); - return; - } - Decompression = true; + Stage = STAGE_DECOMPRESS_AND_VERIFY; Local = true; DestFile += ".decomp"; Desc.URI = "copy:" + FileName; QueueURI(Desc); + ActiveSubprocess = "copy"; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif Mode = "copy"; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif return; } @@ -1263,62 +1260,116 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con ErrorText = "Method gave a blank filename"; } - // The files timestamp matches - if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + // Methods like e.g. "file:" will give us a (compressed) FileName that is + // not the "DestFile" we set, in this case we uncompress from the local file + if (FileName != DestFile) + Local = true; + else + EraseFileName = FileName; + + // we need to verify the file against the current Release file again + // on if-modfied-since hit to avoid a stale attack against us + if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) { - ReverifyAfterIMS(FileName); + // do not reverify cdrom sources as apt-cdrom may rewrite the Packages + // file when its doing the indexcopy + if (RealURI.substr(0,6) == "cdrom:") + return; + + // The files timestamp matches, reverify by copy into partial/ + EraseFileName = ""; + ReverifyAfterIMS(); return; - } + } - if (FileName == DestFile) - Erase = true; - else - Local = true; - - string decompProg; + // If we have compressed indexes enabled, queue for hash verification + if (_config->FindB("Acquire::GzipIndexes",false)) + { + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI) + '.' + CurrentCompressionExtension; + EraseFileName = ""; + Stage = STAGE_DECOMPRESS_AND_VERIFY; + Desc.URI = "copy:" + FileName; + QueueURI(Desc); - // If we enable compressed indexes and already have gzip, keep it - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) { - // Done, queue for rename on transaction finished - PartialFile = DestFile; - DestFile = GetFinalFilename(RealURI, compExt); return; } // get the binary name for your used compression type - decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),""); - if(decompProg.empty() == false); - else if(compExt == "uncompressed") + string decompProg; + if(CurrentCompressionExtension == "uncompressed") decompProg = "copy"; - else { - _error->Error("Unsupported extension: %s", compExt.c_str()); + else + decompProg = _config->Find(string("Acquire::CompressionTypes::").append(CurrentCompressionExtension),""); + if(decompProg.empty() == true) + { + _error->Error("Unsupported extension: %s", CurrentCompressionExtension.c_str()); return; } - Decompression = true; + // queue uri for the next stage + Stage = STAGE_DECOMPRESS_AND_VERIFY; DestFile += ".decomp"; Desc.URI = decompProg + ":" + FileName; QueueURI(Desc); - // FIXME: this points to a c++ string that goes out of scope - Mode = decompProg.c_str(); + ActiveSubprocess = decompProg; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + Mode = ActiveSubprocess.c_str(); +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif } + /*}}}*/ +// pkgAcqIndex::StageDecompressDone - Final verification /*{{{*/ +void pkgAcqIndex::StageDecompressDone(string Message, + HashStringList const &Hashes, + pkgAcquire::MethodConfig *Cfg) +{ + if (ExpectedHashes.usable() && ExpectedHashes != Hashes) + { + Desc.URI = RealURI; + RenameOnError(HashSumMismatch); + printHashSumComparision(RealURI, ExpectedHashes, Hashes); + Failed(Message, Cfg); + return; + } + + if(!ValidateFile(DestFile)) + { + RenameOnError(InvalidFormat); + Failed(Message, Cfg); + return; + } + + // remove the compressed version of the file + unlink(EraseFileName.c_str()); + + // Done, queue for rename on transaction finished + TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename()); + + return; +} + /*}}}*/ /*}}}*/ // AcqIndexTrans::pkgAcqIndexTrans - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The Translation file is added to the queue */ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc) - : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashStringList(), "") + : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashStringList()) { } /*}}}*/ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) - : pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser) + : pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser) { // load the filesize indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey)); @@ -1330,8 +1381,7 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, // --------------------------------------------------------------------- string pkgAcqIndexTrans::Custom600Headers() const { - string Final = _config->FindDir("Dir::State::lists"); - Final += URItoFileName(RealURI); + string Final = GetFinalFilename(); struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) @@ -1344,10 +1394,10 @@ string pkgAcqIndexTrans::Custom600Headers() const /* */ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { - size_t const nextExt = CompressionExtension.find(' '); + size_t const nextExt = CompressionExtensions.find(' '); if (nextExt != std::string::npos) { - CompressionExtension = CompressionExtension.substr(nextExt+1); + CompressionExtensions = CompressionExtensions.substr(nextExt+1); Init(RealURI, Desc.Description, Desc.ShortDesc); Status = StatIdle; return; @@ -1367,16 +1417,119 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Item::Failed(Message,Cnf); } /*}}}*/ +// AcqMetaBase::Add - Add a item to the current Transaction /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqMetaBase::Add(Item *I) +{ + Transaction.push_back(I); +} + /*}}}*/ +// AcqMetaBase::AbortTransaction - Abort the current Transaction /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqMetaBase::AbortTransaction() +{ + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "AbortTransaction: " << TransactionManager << std::endl; -pkgAcqMetaSigBase::pkgAcqMetaSigBase(pkgAcquire *Owner, - HashStringList const &ExpectedHashes, - unsigned long TransactionID) - : Item(Owner, ExpectedHashes, TransactionID) + // ensure the toplevel is in error state too + for (std::vector::iterator I = Transaction.begin(); + I != Transaction.end(); ++I) + { + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << " Cancel: " << (*I)->DestFile << std::endl; + // the transaction will abort, so stop anything that is idle + if ((*I)->Status == pkgAcquire::Item::StatIdle) + (*I)->Status = pkgAcquire::Item::StatDone; + + // kill files in partial + string PartialFile = _config->FindDir("Dir::State::lists"); + PartialFile += "partial/"; + PartialFile += flNotDir((*I)->DestFile); + if(FileExists(PartialFile)) + Rename(PartialFile, PartialFile + ".FAILED"); + } +} + /*}}}*/ +// AcqMetaBase::TransactionHasError - Check for errors in Transaction /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgAcqMetaBase::TransactionHasError() +{ + for (pkgAcquire::ItemIterator I = Transaction.begin(); + I != Transaction.end(); ++I) + if((*I)->Status != pkgAcquire::Item::StatDone && + (*I)->Status != pkgAcquire::Item::StatIdle) + return true; + + return false; +} + /*}}}*/ +// AcqMetaBase::CommitTransaction - Commit a transaction /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqMetaBase::CommitTransaction() +{ + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "CommitTransaction: " << this << std::endl; + + // move new files into place *and* remove files that are not + // part of the transaction but are still on disk + for (std::vector::iterator I = Transaction.begin(); + I != Transaction.end(); ++I) + { + if((*I)->PartialFile != "") + { + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "mv " + << (*I)->PartialFile << " -> " + << (*I)->DestFile << " " + << (*I)->DescURI() + << std::endl; + Rename((*I)->PartialFile, (*I)->DestFile); + chmod((*I)->DestFile.c_str(),0644); + } else { + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "rm " + << (*I)->DestFile + << " " + << (*I)->DescURI() + << std::endl; + unlink((*I)->DestFile.c_str()); + } + // mark that this transaction is finished + (*I)->TransactionManager = 0; + } +} + /*}}}*/ +// AcqMetaBase::CommitTransaction - Commit a transaction /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqMetaBase::TransactionStageCopy(Item *I, + const std::string &From, + const std::string &To) { + I->PartialFile = From; + I->DestFile = To; } + /*}}}*/ +// AcqMetaBase::CommitTransaction - Commit a transaction /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqMetaBase::TransactionStageRemoval(Item *I, + const std::string &FinalFile) +{ + I->PartialFile = ""; + I->DestFile = FinalFile; +} + /*}}}*/ /*{{{*/ -bool pkgAcqMetaSigBase::GenerateAuthWarning(const std::string &RealURI, - const std::string &Message) +// AcqMetaBase::GenerateAuthWarning - Check gpg authentication error /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgAcqMetaBase::GenerateAuthWarning(const std::string &RealURI, + const std::string &Message) { string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); @@ -1408,30 +1561,32 @@ bool pkgAcqMetaSigBase::GenerateAuthWarning(const std::string &RealURI, return false; } /*}}}*/ - - -pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ - unsigned long TransactionID, +// AcqMetaSig::AcqMetaSig - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, + pkgAcqMetaBase *TransactionManager, string URI,string URIDesc,string ShortDesc, string MetaIndexFile, const vector* IndexTargets, indexRecords* MetaIndexParser) : - pkgAcqMetaSigBase(Owner, HashStringList(), TransactionID), RealURI(URI), - MetaIndexParser(MetaIndexParser), MetaIndexFile(MetaIndexFile), - IndexTargets(IndexTargets), AuthPass(false), IMSHit(false) + pkgAcqMetaBase(Owner, IndexTargets, MetaIndexParser, + HashStringList(), TransactionManager), + RealURI(URI), MetaIndexFile(MetaIndexFile), URIDesc(URIDesc), + ShortDesc(ShortDesc) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(URI); + DestFile += URItoFileName(RealURI); // remove any partial downloaded sig-file in partial/. // it may confuse proxies and is too small to warrant a // partial download anyway unlink(DestFile.c_str()); - // set the TransactionID + // set the TransactionManager if(_config->FindB("Debug::Acquire::Transaction", false) == true) - std::clog << "New pkgAcqMetaSig with TransactionID " - << TransactionID << std::endl; + std::clog << "New pkgAcqMetaSig with TransactionManager " + << TransactionManager << std::endl; // Create the item Desc.Description = URIDesc; @@ -1460,76 +1615,81 @@ string pkgAcqMetaSig::Custom600Headers() const return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } - -void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList const &Hashes, + /*}}}*/ +// pkgAcqMetaSig::Done - The signature was downloaded/verified /*{{{*/ +// --------------------------------------------------------------------- +/* The only header we use is the last-modified header. */ +void pkgAcqMetaSig::Done(string Message,unsigned long long Size, + HashStringList const &Hashes, pkgAcquire::MethodConfig *Cfg) { Item::Done(Message, Size, Hashes, Cfg); - string FileName = LookupTag(Message,"Filename"); - if (FileName.empty() == true) + if(AuthPass == false) { - Status = StatError; - ErrorText = "Method gave a blank filename"; - return; - } + // queue for verify, note that we change DestFile here to point to + // the file we want to verify (needed to make gpgv work) - if (FileName != DestFile) - { - // We have to copy it into place - Local = true; - Desc.URI = "copy:" + FileName; - QueueURI(Desc); - return; - } + string FileName = LookupTag(Message,"Filename"); + if (FileName.empty() == true) + { + Status = StatError; + ErrorText = "Method gave a blank filename"; + return; + } - if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) - IMSHit = true; + if (FileName != DestFile) + { + // We have to copy it into place + Local = true; + Desc.URI = "copy:" + FileName; + QueueURI(Desc); + return; + } - // adjust paths if its a ims-hit - if(IMSHit) - { - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); - - DestFile = PartialFile = FinalFile; - } + if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + { + IMSHit = true; + // adjust DestFile on i-m-s hit to the one we already have on disk + DestFile = _config->FindDir("Dir::State::lists"); + DestFile += URItoFileName(RealURI); + } + + // this is the file we verify from + MetaIndexFileSignature = DestFile; - // queue for verify - if(AuthPass == false) - { AuthPass = true; - Desc.URI = "gpgv:" + DestFile; + Desc.URI = "gpgv:" + MetaIndexFileSignature; DestFile = MetaIndexFile; QueueURI(Desc); + ActiveSubprocess = "gpgv"; return; } - - // queue to copy the file in place if it was not a ims hit, on ims - // hit the file is already at the right place - if(IMSHit == false) + else { - PartialFile = _config->FindDir("Dir::State::lists") + "partial/"; - PartialFile += URItoFileName(RealURI); - - DestFile = _config->FindDir("Dir::State::lists"); - DestFile += URItoFileName(RealURI); - } + // verify was successful - Complete = true; + // we parse the MetaIndexFile here (and not right after getting + // the pkgAcqMetaIndex) because at this point we can trust the data + // + // load indexes and queue further downloads + MetaIndexParser->Load(MetaIndexFile); + QueueIndexes(true); + + // DestFile points to the the MetaIndeFile at this point, make it + // point back to the Release.gpg file + std::string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + TransactionManager->TransactionStageCopy(this, MetaIndexFileSignature, FinalFile); + Complete = true; + } } /*}}}*/ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ { string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); - // this ensures that any file in the lists/ dir is removed by the - // transaction - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(RealURI); - PartialFile = ""; - // FIXME: duplicated code from pkgAcqMetaIndex if (AuthPass == true) { @@ -1538,6 +1698,53 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ return; } + // FIXME: meh, this is not really elegant + string InReleaseURI = RealURI.replace(RealURI.rfind("Release.gpg"), 12, + "InRelease"); + string FinalInRelease = _config->FindDir("Dir::State::lists") + URItoFileName(InReleaseURI); + + if (RealFileExists(Final) || RealFileExists(FinalInRelease)) + { + std::string downgrade_msg; + strprintf(downgrade_msg, _("The repository '%s' is no longer signed."), + URIDesc.c_str()); + if(_config->FindB("Acquire::AllowDowngradeToInsecureRepositories")) + { + // meh, the users wants to take risks (we still mark the packages + // from this repository as unauthenticated) + _error->Warning("%s", downgrade_msg.c_str()); + _error->Warning(_("This is normally not allowed, but the option " + "Acquire::AllowDowngradeToInsecureRepositories was " + "given to override it.")); + + } else { + _error->Error("%s", downgrade_msg.c_str()); + Rename(MetaIndexFile, MetaIndexFile+".FAILED"); + Status = pkgAcquire::Item::StatError; + TransactionManager->AbortTransaction(); + return; + } + } + + // this ensures that any file in the lists/ dir is removed by the + // transaction + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI); + TransactionManager->TransactionStageRemoval(this, DestFile); + + // only allow going further if the users explicitely wants it + if(_config->FindB("Acquire::AllowInsecureRepositories") == true) + { + // we parse the indexes here because at this point the user wanted + // a repository that may potentially harm him + MetaIndexParser->Load(MetaIndexFile); + QueueIndexes(true); + } + else + { + _error->Warning("Use --allow-insecure-repositories to force the update"); + } + // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) @@ -1552,22 +1759,27 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ } /*}}}*/ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ - unsigned long TransactionID, + pkgAcqMetaBase *TransactionManager, string URI,string URIDesc,string ShortDesc, string MetaIndexSigURI,string MetaIndexSigURIDesc, string MetaIndexSigShortDesc, const vector* IndexTargets, indexRecords* MetaIndexParser) : - pkgAcqMetaSigBase(Owner, HashStringList(), TransactionID), RealURI(URI), IndexTargets(IndexTargets), - MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false), + pkgAcqMetaBase(Owner, IndexTargets, MetaIndexParser, HashStringList(), + TransactionManager), + RealURI(URI), URIDesc(URIDesc), ShortDesc(ShortDesc), MetaIndexSigURI(MetaIndexSigURI), MetaIndexSigURIDesc(MetaIndexSigURIDesc), MetaIndexSigShortDesc(MetaIndexSigShortDesc) { - if(TransactionID == 0) - this->TransactionID = (unsigned long)this; + if(TransactionManager == NULL) + { + this->TransactionManager = this; + this->TransactionManager->Add(this); + } if(_config->FindB("Debug::Acquire::Transaction", false) == true) - std::clog << "New pkgAcqMetaIndex with TransactionID " - << TransactionID << std::endl; + std::clog << "New pkgAcqMetaIndex with TransactionManager " + << this->TransactionManager << std::endl; + Init(URIDesc, ShortDesc); } @@ -1626,13 +1838,7 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList // Still more retrieving to do return; - if (SigFile == "") - { - // load indexes, the signature will downloaded afterwards - MetaIndexParser->Load(DestFile); - QueueIndexes(true); - } - else + if (SigFile != "") { // There was a signature file, so pass it to gpgv for // verification @@ -1642,7 +1848,15 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList AuthPass = true; Desc.URI = "gpgv:" + SigFile; QueueURI(Desc); - Mode = "gpgv"; + ActiveSubprocess = "gpgv"; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + Mode = "gpgv"; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif return; } } @@ -1653,9 +1867,9 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList FinalFile += URItoFileName(RealURI); if (SigFile == DestFile) SigFile = FinalFile; + // queue for copy in place - PartialFile = DestFile; - DestFile = FinalFile; + TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); } } /*}}}*/ @@ -1702,7 +1916,7 @@ void pkgAcqMetaIndex::RetrievalDone(string Message) /*{{{*/ // queue a signature if(SigFile != DestFile) - new pkgAcqMetaSig(Owner, TransactionID, + new pkgAcqMetaSig(Owner, TransactionManager, MetaIndexSigURI, MetaIndexSigURIDesc, MetaIndexSigShortDesc, DestFile, IndexTargets, MetaIndexParser); @@ -1733,7 +1947,41 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ std::cerr << "Signature verification succeeded: " << DestFile << std::endl; - // Download further indexes with verification +// we ensure this by other means +#if 0 + // do not trust any previously unverified content that we may have + string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI)); + if (DestFile != SigFile) + LastGoodSigFile.append(".gpg"); + LastGoodSigFile.append(".reverify"); + if(IMSHit == false && RealFileExists(LastGoodSigFile) == false) + { + for (vector ::const_iterator Target = IndexTargets->begin(); + Target != IndexTargets->end(); + ++Target) + { + // remove old indexes + std::string index = _config->FindDir("Dir::State::lists") + + URItoFileName((*Target)->URI); + unlink(index.c_str()); + // and also old gzipindexes + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + index += '.' + (*t); + unlink(index.c_str()); + } + } + } +#endif + + // Download further indexes with verification + // + // it would be really nice if we could simply do + // if (IMSHit == false) QueueIndexes(true) + // and skip the download if the Release file has not changed + // - but right now the list cleaner will needs to be tricked + // to not delete all our packages/source indexes in this case QueueIndexes(true); #if 0 @@ -1749,7 +1997,7 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ #endif } /*}}}*/ -void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ +void pkgAcqMetaBase::QueueIndexes(bool verify) /*{{{*/ { bool transInRelease = false; { @@ -1812,17 +2060,13 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ if ((*Target)->IsOptional() == true) { - if ((*Target)->IsSubIndex() == true) - new pkgAcqSubIndex(Owner, TransactionID, - (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, ExpectedIndexHashes); - else if (transInRelease == false || Record != NULL || compressedAvailable == true) + if (transInRelease == false || Record != NULL || compressedAvailable == true) { if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true && MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true) - new pkgAcqDiffIndex(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqDiffIndex(Owner, TransactionManager, *Target, ExpectedIndexHashes, MetaIndexParser); else - new pkgAcqIndexTrans(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqIndexTrans(Owner, TransactionManager, *Target, ExpectedIndexHashes, MetaIndexParser); } continue; } @@ -1833,9 +2077,9 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ instead, but passing the required info to it is to much hassle */ if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false || MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)) - new pkgAcqDiffIndex(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqDiffIndex(Owner, TransactionManager, *Target, ExpectedIndexHashes, MetaIndexParser); else - new pkgAcqIndex(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqIndex(Owner, TransactionManager, *Target, ExpectedIndexHashes, MetaIndexParser); } } /*}}}*/ @@ -1931,29 +2175,44 @@ void pkgAcqMetaIndex::Failed(string Message, return; } - /* Always move the meta index, even if gpgv failed. This ensures - * that PackageFile objects are correctly filled in */ - if (FileExists(DestFile)) { - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); - /* InRelease files become Release files, otherwise - * they would be considered as trusted later on */ - if (SigFile == DestFile) { - RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9, - "Release"); - FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9, - "Release"); - SigFile = FinalFile; - } - - // Done, queue for rename on transaction finished - PartialFile = DestFile; - DestFile = FinalFile; - } + _error->Warning(_("The data from '%s' is not signed. Packages " + "from that repository can not be authenticated."), + URIDesc.c_str()); // No Release file was present, or verification failed, so fall // back to queueing Packages files without verification - QueueIndexes(false); + // only allow going further if the users explicitely wants it + if(_config->FindB("Acquire::AllowInsecureRepositories") == true) + { + /* Always move the meta index, even if gpgv failed. This ensures + * that PackageFile objects are correctly filled in */ + if (FileExists(DestFile)) + { + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + /* InRelease files become Release files, otherwise + * they would be considered as trusted later on */ + if (SigFile == DestFile) { + RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9, + "Release"); + FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9, + "Release"); + SigFile = FinalFile; + } + + // Done, queue for rename on transaction finished + TransactionManager->TransactionStageCopy(this, DestFile, FinalFile); + } + + QueueIndexes(false); + } else { + // warn if the repository is unsinged + _error->Warning("Use --allow-insecure-repositories to force the update"); + TransactionManager->AbortTransaction(); + Status = StatError; + return; + } + } /*}}}*/ @@ -1961,9 +2220,9 @@ void pkgAcqMetaIndex::Finished() { if(_config->FindB("Debug::Acquire::Transaction", false) == true) std::clog << "Finished: " << DestFile <TransactionHasError(TransactionID) == false && - TransactionID > 0) - Owner->CommitTransaction(TransactionID); + if(TransactionManager != NULL && + TransactionManager->TransactionHasError() == false) + TransactionManager->CommitTransaction(); } @@ -1973,7 +2232,7 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/ string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc, const vector* IndexTargets, indexRecords* MetaIndexParser) : - pkgAcqMetaIndex(Owner, (unsigned long)this, URI, URIDesc, ShortDesc, MetaSigURI, MetaSigURIDesc,MetaSigShortDesc, IndexTargets, MetaIndexParser), + pkgAcqMetaIndex(Owner, NULL, URI, URIDesc, ShortDesc, MetaSigURI, MetaSigURIDesc,MetaSigShortDesc, IndexTargets, MetaIndexParser), MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc) { @@ -2037,7 +2296,8 @@ void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long Size, if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile)) { pkgAcquire::Item::Failed(Message, Cnf); - ErrorText = _("Does not start with a cleartext signature"); + RenameOnError(NotClearsigned); + TransactionManager->AbortTransaction(); return; } pkgAcqMetaIndex::Done(Message, Size, Hashes, Cnf); @@ -2055,10 +2315,9 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* // impression (CVE-2012-0214) string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile.append(URItoFileName(RealURI)); - PartialFile = ""; - DestFile = FinalFile; + TransactionManager->TransactionStageRemoval(this, FinalFile); - new pkgAcqMetaIndex(Owner, TransactionID, + new pkgAcqMetaIndex(Owner, TransactionManager, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, MetaSigURI, MetaSigURIDesc, MetaSigShortDesc, IndexTargets, MetaIndexParser);