X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/8d89cda7d66b6f125c457f36beeb84abb0df07f1..38f8704e419ed93f433129e20df5611df6652620:/apt-pkg/acquire-item.cc diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 2cf6b60a8..b2e578629 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -121,6 +121,16 @@ static std::string GetExistingFilename(std::string const &File) /*{{{*/ return ""; } /*}}}*/ +static std::string GetDiffIndexFileName(std::string const &Name) /*{{{*/ +{ + return Name + ".diff/Index"; +} + /*}}}*/ +static std::string GetDiffIndexURI(IndexTarget const &Target) /*{{{*/ +{ + return Target.URI + ".diff/Index"; +} + /*}}}*/ static bool MessageInsecureRepository(bool const isError, std::string const &msg)/*{{{*/ { @@ -206,12 +216,12 @@ HashStringList pkgAcqMetaBase::GetExpectedHashes() const APT_CONST bool pkgAcqIndexDiffs::HashesRequired() const { - /* We don't always have the diff of the downloaded pdiff file. - What we have for sure is hashes for the uncompressed file, - but rred uncompresses them on the fly while parsing, so not handled here. - Hashes are (also) checked while searching for (next) patch to apply. */ + /* We can't check hashes of rred result as we don't know what the + hash of the file will be. We just know the hash of the patch(es), + the hash of the file they will apply on and the hash of the resulting + file. */ if (State == StateFetchDiff) - return available_patches[0].download_hashes.empty() == false; + return true; return false; } HashStringList pkgAcqIndexDiffs::GetExpectedHashes() const @@ -227,7 +237,7 @@ APT_CONST bool pkgAcqIndexMergeDiffs::HashesRequired() const we can check the rred result after all patches are applied as we know the expected result rather than potentially apply more patches */ if (State == StateFetchDiff) - return patch.download_hashes.empty() == false; + return true; return State == StateApplyDiff; } HashStringList pkgAcqIndexMergeDiffs::GetExpectedHashes() const @@ -271,6 +281,12 @@ bool pkgAcquire::Item::QueueURI(pkgAcquire::ItemDesc &Item) for a hashsum mismatch to happen which helps nobody) */ bool pkgAcqTransactionItem::QueueURI(pkgAcquire::ItemDesc &Item) { + if (TransactionManager->State != TransactionStarted) + { + if (_config->FindB("Debug::Acquire::Transaction", false)) + std::clog << "Skip " << Target.URI << " as transaction was already dealt with!" << std::endl; + return false; + } std::string const FinalFile = GetFinalFilename(); if (TransactionManager != NULL && TransactionManager->IMSHit == true && FileExists(FinalFile) == true) @@ -301,12 +317,12 @@ bool pkgAcqDiffIndex::QueueURI(pkgAcquire::ItemDesc &Item) // Acquire::Item::GetFinalFilename and specialisations for child classes /*{{{*/ std::string pkgAcquire::Item::GetFinalFilename() const { + // Beware: Desc.URI is modified by redirections return GetFinalFileNameFromURI(Desc.URI); } std::string pkgAcqDiffIndex::GetFinalFilename() const { - // the logic we inherent from pkgAcqBaseIndex isn't what we need here - return pkgAcquire::Item::GetFinalFilename(); + return GetFinalFileNameFromURI(GetDiffIndexURI(Target)); } std::string pkgAcqIndex::GetFinalFilename() const { @@ -343,7 +359,7 @@ std::string pkgAcqIndex::GetMetaKey() const } std::string pkgAcqDiffIndex::GetMetaKey() const { - return Target.MetaKey + ".diff/Index"; + return GetDiffIndexFileName(Target.MetaKey); } /*}}}*/ //pkgAcqTransactionItem::TransactionState and specialisations for child classes /*{{{*/ @@ -352,6 +368,7 @@ bool pkgAcqTransactionItem::TransactionState(TransactionStates const state) bool const Debug = _config->FindB("Debug::Acquire::Transaction", false); switch(state) { + case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break; case TransactionAbort: if(Debug == true) std::clog << " Cancel: " << DestFile << std::endl; @@ -441,6 +458,7 @@ bool pkgAcqIndex::TransactionState(TransactionStates const state) switch (state) { + case TransactionStarted: _error->Fatal("AcqIndex %s changed to invalid transaction start state!", Target.URI.c_str()); break; case TransactionAbort: if (Stage == STAGE_DECOMPRESS_AND_VERIFY) { @@ -464,6 +482,7 @@ bool pkgAcqDiffIndex::TransactionState(TransactionStates const state) switch (state) { + case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break; case TransactionCommit: break; case TransactionAbort: @@ -792,6 +811,32 @@ HashStringList pkgAcqTransactionItem::GetExpectedHashesFor(std::string const &Me } /*}}}*/ +static void LoadLastMetaIndexParser(pkgAcqMetaClearSig * const TransactionManager, std::string const &FinalRelease, std::string const &FinalInRelease)/*{{{*/ +{ + if (TransactionManager->IMSHit == true) + return; + if (RealFileExists(FinalInRelease) || RealFileExists(FinalRelease)) + { + TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone(); + if (TransactionManager->LastMetaIndexParser != NULL) + { + _error->PushToStack(); + if (RealFileExists(FinalInRelease)) + TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL); + else + TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL); + // its unlikely to happen, but if what we have is bad ignore it + if (_error->PendingError()) + { + delete TransactionManager->LastMetaIndexParser; + TransactionManager->LastMetaIndexParser = NULL; + } + _error->RevertToStack(); + } + } +} + /*}}}*/ + // AcqMetaBase - Constructor /*{{{*/ pkgAcqMetaBase::pkgAcqMetaBase(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, @@ -799,7 +844,7 @@ pkgAcqMetaBase::pkgAcqMetaBase(pkgAcquire * const Owner, IndexTarget const &DataTarget) : pkgAcqTransactionItem(Owner, TransactionManager, DataTarget), d(NULL), IndexTargets(IndexTargets), - AuthPass(false), IMSHit(false) + AuthPass(false), IMSHit(false), State(TransactionStarted) { } /*}}}*/ @@ -815,10 +860,20 @@ void pkgAcqMetaBase::AbortTransaction() if(_config->FindB("Debug::Acquire::Transaction", false) == true) std::clog << "AbortTransaction: " << TransactionManager << std::endl; + switch (TransactionManager->State) + { + case TransactionStarted: break; + case TransactionAbort: _error->Fatal("Transaction %s was already aborted and is aborted again", TransactionManager->Target.URI.c_str()); return; + case TransactionCommit: _error->Fatal("Transaction %s was already aborted and is now commited", TransactionManager->Target.URI.c_str()); return; + } + TransactionManager->State = TransactionAbort; + // ensure the toplevel is in error state too for (std::vector::iterator I = Transaction.begin(); I != Transaction.end(); ++I) { + if ((*I)->Status != pkgAcquire::Item::StatFetching) + Owner->Dequeue(*I); (*I)->TransactionState(TransactionAbort); } Transaction.clear(); @@ -848,6 +903,14 @@ void pkgAcqMetaBase::CommitTransaction() if(_config->FindB("Debug::Acquire::Transaction", false) == true) std::clog << "CommitTransaction: " << this << std::endl; + switch (TransactionManager->State) + { + case TransactionStarted: break; + case TransactionAbort: _error->Fatal("Transaction %s was already commited and is now aborted", TransactionManager->Target.URI.c_str()); return; + case TransactionCommit: _error->Fatal("Transaction %s was already commited and is again commited", TransactionManager->Target.URI.c_str()); return; + } + TransactionManager->State = TransactionCommit; + // move new files into place *and* remove files that are not // part of the transaction but are still on disk for (std::vector::iterator I = Transaction.begin(); @@ -1004,25 +1067,7 @@ bool pkgAcqMetaBase::CheckAuthDone(string const &Message) /*{{{*/ FinalInRelease = FinalFile.substr(0, FinalFile.length() - strlen("Release")) + "InRelease"; FinalRelease = FinalFile; } - if (RealFileExists(FinalInRelease) || RealFileExists(FinalRelease)) - { - TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone(); - if (TransactionManager->LastMetaIndexParser != NULL) - { - _error->PushToStack(); - if (RealFileExists(FinalInRelease)) - TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL); - else - TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL); - // its unlikely to happen, but if what we have is bad ignore it - if (_error->PendingError()) - { - delete TransactionManager->LastMetaIndexParser; - TransactionManager->LastMetaIndexParser = NULL; - } - _error->RevertToStack(); - } - } + LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease); } if (TransactionManager->MetaIndexParser->Load(DestFile, &ErrorText) == false) @@ -1166,14 +1211,14 @@ void pkgAcqMetaBase::QueueIndexes(bool const verify) /*{{{*/ if (filename.empty() == false) { new NoActionItem(Owner, *Target, filename); - std::string const idxfilename = GetFinalFileNameFromURI(Target->URI + ".diff/Index"); + std::string const idxfilename = GetFinalFileNameFromURI(GetDiffIndexURI(*Target)); if (FileExists(idxfilename)) new NoActionItem(Owner, *Target, idxfilename); continue; } // check if we have patches available - trypdiff &= TransactionManager->MetaIndexParser->Exists(Target->MetaKey + ".diff/Index"); + trypdiff &= TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target->MetaKey)); } else { @@ -1332,6 +1377,15 @@ string pkgAcqMetaClearSig::Custom600Headers() const return Header; } /*}}}*/ +void pkgAcqMetaClearSig::Finished() /*{{{*/ +{ + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "Finished: " << DestFile <State == TransactionStarted && + TransactionManager->TransactionHasError() == false) + TransactionManager->CommitTransaction(); +} + /*}}}*/ bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message, /*{{{*/ pkgAcquire::MethodConfig const * const Cnf) { @@ -1418,30 +1472,7 @@ void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig c string const FinalInRelease = GetFinalFilename(); Rename(DestFile, PartialRelease); TransactionManager->TransactionStageCopy(this, PartialRelease, FinalRelease); - - if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease)) - { - // open the last Release if we have it - if (TransactionManager->IMSHit == false) - { - TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone(); - if (TransactionManager->LastMetaIndexParser != NULL) - { - _error->PushToStack(); - if (RealFileExists(FinalInRelease)) - TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL); - else - TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL); - // its unlikely to happen, but if what we have is bad ignore it - if (_error->PendingError()) - { - delete TransactionManager->LastMetaIndexParser; - TransactionManager->LastMetaIndexParser = NULL; - } - _error->RevertToStack(); - } - } - } + LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease); // we parse the indexes here because at this point the user wanted // a repository that may potentially harm him @@ -1514,15 +1545,6 @@ void pkgAcqMetaIndex::Failed(string const &Message, } } /*}}}*/ -void pkgAcqMetaIndex::Finished() /*{{{*/ -{ - if(_config->FindB("Debug::Acquire::Transaction", false) == true) - std::clog << "Finished: " << DestFile <TransactionHasError() == false) - TransactionManager->CommitTransaction(); -} - /*}}}*/ std::string pkgAcqMetaIndex::DescURI() const /*{{{*/ { return Target.URI; @@ -1657,29 +1679,7 @@ void pkgAcqMetaSig::Failed(string const &Message,pkgAcquire::MethodConfig const // only allow going further if the user explicitly wants it if (AllowInsecureRepositories(_("The repository '%s' is not signed."), MetaIndex->Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true) { - if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease)) - { - // open the last Release if we have it - if (TransactionManager->IMSHit == false) - { - TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone(); - if (TransactionManager->LastMetaIndexParser != NULL) - { - _error->PushToStack(); - if (RealFileExists(FinalInRelease)) - TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL); - else - TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL); - // its unlikely to happen, but if what we have is bad ignore it - if (_error->PendingError()) - { - delete TransactionManager->LastMetaIndexParser; - TransactionManager->LastMetaIndexParser = NULL; - } - _error->RevertToStack(); - } - } - } + LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease); // we parse the indexes here because at this point the user wanted // a repository that may potentially harm him @@ -1728,9 +1728,9 @@ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner, Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); Desc.Owner = this; - Desc.Description = Target.Description + ".diff/Index"; + Desc.Description = GetDiffIndexFileName(Target.Description); Desc.ShortDesc = Target.ShortDesc; - Desc.URI = Target.URI + ".diff/Index"; + Desc.URI = GetDiffIndexURI(Target); DestFile = GetPartialFileNameFromURI(Desc.URI); @@ -2022,6 +2022,17 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ return false; } + for (auto const &patch: available_patches) + if (patch.result_hashes.usable() == false || + patch.patch_hashes.usable() == false || + patch.download_hashes.usable() == false) + { + if (Debug) + std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": provides no usable hashes for " << patch.file + << " so fallback to complete download" << std::endl; + return false; + } + // patching with too many files is rather slow compared to a fast download unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0); if (fileLimit != 0 && fileLimit < available_patches.size()) @@ -2036,7 +2047,6 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100); if (sizeLimitPercent > 0 && TransactionManager->MetaIndexParser != nullptr) { - // compressed case unsigned long long downloadSize = std::accumulate(available_patches.begin(), available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) { return T + I.download_hashes.FileSize(); @@ -2065,23 +2075,6 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ return false; } } - // uncompressed case - downloadSize = std::accumulate(available_patches.begin(), - available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) { - return T + I.patch_hashes.FileSize(); - }); - if (downloadSize != 0) - { - unsigned long long const downloadSizeIdx = ServerSize; - unsigned long long const sizeLimit = downloadSizeIdx * sizeLimitPercent; - if ((sizeLimit/100) < downloadSize) - { - if (Debug) - std::clog << "Need " << downloadSize << " uncompressed bytes (Limit is " << (sizeLimit/100) << ", " - << "original is " << downloadSizeIdx << ") so fallback to complete download" << std::endl; - return false; - } - } } // we have something, queue the diffs @@ -2451,16 +2444,18 @@ void pkgAcqIndexMergeDiffs::Failed(string const &Message,pkgAcquire::MethodConfi for (std::vector::const_iterator I = allPatches->begin(); I != allPatches->end(); ++I) if ((*I)->State == StateErrorDiff) + { + State = StateErrorDiff; return; + } // first failure means we should fallback State = StateErrorDiff; if (Debug) std::clog << "Falling back to normal index file acquire" << std::endl; RenameOnError(PDiffError); - std::string const patchname = GetPartialFileNameFromURI(Desc.URI); - if (RealFileExists(patchname)) - Rename(patchname, patchname + ".FAILED"); + if (RealFileExists(DestFile)) + Rename(DestFile, DestFile + ".FAILED"); std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI)); if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile)) Rename(UnpatchedFile, UnpatchedFile + ".FAILED"); @@ -2481,6 +2476,7 @@ void pkgAcqIndexMergeDiffs::Done(string const &Message, HashStringList const &Ha { if(Debug) std::clog << "Another patch failed already, no point in processing this one." << std::endl; + State = StateErrorDiff; return; } @@ -2488,7 +2484,8 @@ void pkgAcqIndexMergeDiffs::Done(string const &Message, HashStringList const &Ha std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile); if (UnpatchedFile.empty()) { - _error->Fatal("Unpatched file %s doesn't exist (anymore)!", UnpatchedFile.c_str()); + _error->Fatal("Unpatched file %s doesn't exist (anymore)!", UncompressedUnpatchedFile.c_str()); + State = StateErrorDiff; return; } std::string const PatchFile = GetMergeDiffsPatchFileName(UnpatchedFile, patch.file); @@ -2603,6 +2600,10 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, DestFile = GetPartialFileNameFromURI(URI); NextCompressionExtension(CurrentCompressionExtension, CompressionExtensions, false); + // store file size of the download to ensure the fetcher gives + // accurate progress reporting + FileSize = GetExpectedHashes().FileSize(); + if (CurrentCompressionExtension == "uncompressed") { Desc.URI = URI;