using namespace std;
-static void printHashSumComparision(std::string const &URI, HashStringList const &Expected, HashStringList const &Actual) /*{{{*/
+static void printHashSumComparison(std::string const &URI, HashStringList const &Expected, HashStringList const &Actual) /*{{{*/
{
if (_config->FindB("Debug::Acquire::HashSumMismatch", false) == false)
return;
return "";
}
/*}}}*/
+static std::string GetDiffIndexFileName(std::string const &Name) /*{{{*/
+{
+ return Name + ".diff/Index";
+}
+ /*}}}*/
+static std::string GetDiffIndexURI(IndexTarget const &Target) /*{{{*/
+{
+ return Target.URI + ".diff/Index";
+}
+ /*}}}*/
static bool MessageInsecureRepository(bool const isError, std::string const &msg)/*{{{*/
{
APT_CONST bool pkgAcqIndexDiffs::HashesRequired() const
{
- /* We don't always have the diff of the downloaded pdiff file.
- What we have for sure is hashes for the uncompressed file,
- but rred uncompresses them on the fly while parsing, so not handled here.
- Hashes are (also) checked while searching for (next) patch to apply. */
+ /* We can't check hashes of rred result as we don't know what the
+ hash of the file will be. We just know the hash of the patch(es),
+ the hash of the file they will apply on and the hash of the resulting
+ file. */
if (State == StateFetchDiff)
- return available_patches[0].download_hashes.empty() == false;
+ return true;
return false;
}
HashStringList pkgAcqIndexDiffs::GetExpectedHashes() const
we can check the rred result after all patches are applied as
we know the expected result rather than potentially apply more patches */
if (State == StateFetchDiff)
- return patch.download_hashes.empty() == false;
+ return true;
return State == StateApplyDiff;
}
HashStringList pkgAcqIndexMergeDiffs::GetExpectedHashes() const
for a hashsum mismatch to happen which helps nobody) */
bool pkgAcqTransactionItem::QueueURI(pkgAcquire::ItemDesc &Item)
{
+ if (TransactionManager->State != TransactionStarted)
+ {
+ if (_config->FindB("Debug::Acquire::Transaction", false))
+ std::clog << "Skip " << Target.URI << " as transaction was already dealt with!" << std::endl;
+ return false;
+ }
std::string const FinalFile = GetFinalFilename();
if (TransactionManager != NULL && TransactionManager->IMSHit == true &&
FileExists(FinalFile) == true)
// Acquire::Item::GetFinalFilename and specialisations for child classes /*{{{*/
std::string pkgAcquire::Item::GetFinalFilename() const
{
+ // Beware: Desc.URI is modified by redirections
return GetFinalFileNameFromURI(Desc.URI);
}
std::string pkgAcqDiffIndex::GetFinalFilename() const
{
- // the logic we inherent from pkgAcqBaseIndex isn't what we need here
- return pkgAcquire::Item::GetFinalFilename();
+ return GetFinalFileNameFromURI(GetDiffIndexURI(Target));
}
std::string pkgAcqIndex::GetFinalFilename() const
{
}
std::string pkgAcqDiffIndex::GetMetaKey() const
{
- return Target.MetaKey + ".diff/Index";
+ return GetDiffIndexFileName(Target.MetaKey);
}
/*}}}*/
//pkgAcqTransactionItem::TransactionState and specialisations for child classes /*{{{*/
bool const Debug = _config->FindB("Debug::Acquire::Transaction", false);
switch(state)
{
+ case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break;
case TransactionAbort:
if(Debug == true)
std::clog << " Cancel: " << DestFile << std::endl;
switch (state)
{
+ case TransactionStarted: _error->Fatal("AcqIndex %s changed to invalid transaction start state!", Target.URI.c_str()); break;
case TransactionAbort:
if (Stage == STAGE_DECOMPRESS_AND_VERIFY)
{
switch (state)
{
+ case TransactionStarted: _error->Fatal("Item %s changed to invalid transaction start state!", Target.URI.c_str()); break;
case TransactionCommit:
break;
case TransactionAbort:
}
/*}}}*/
+static void LoadLastMetaIndexParser(pkgAcqMetaClearSig * const TransactionManager, std::string const &FinalRelease, std::string const &FinalInRelease)/*{{{*/
+{
+ if (TransactionManager->IMSHit == true)
+ return;
+ if (RealFileExists(FinalInRelease) || RealFileExists(FinalRelease))
+ {
+ TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone();
+ if (TransactionManager->LastMetaIndexParser != NULL)
+ {
+ _error->PushToStack();
+ if (RealFileExists(FinalInRelease))
+ TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL);
+ else
+ TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL);
+ // its unlikely to happen, but if what we have is bad ignore it
+ if (_error->PendingError())
+ {
+ delete TransactionManager->LastMetaIndexParser;
+ TransactionManager->LastMetaIndexParser = NULL;
+ }
+ _error->RevertToStack();
+ }
+ }
+}
+ /*}}}*/
+
// AcqMetaBase - Constructor /*{{{*/
pkgAcqMetaBase::pkgAcqMetaBase(pkgAcquire * const Owner,
pkgAcqMetaClearSig * const TransactionManager,
IndexTarget const &DataTarget)
: pkgAcqTransactionItem(Owner, TransactionManager, DataTarget), d(NULL),
IndexTargets(IndexTargets),
- AuthPass(false), IMSHit(false)
+ AuthPass(false), IMSHit(false), State(TransactionStarted)
{
}
/*}}}*/
if(_config->FindB("Debug::Acquire::Transaction", false) == true)
std::clog << "AbortTransaction: " << TransactionManager << std::endl;
+ switch (TransactionManager->State)
+ {
+ case TransactionStarted: break;
+ case TransactionAbort: _error->Fatal("Transaction %s was already aborted and is aborted again", TransactionManager->Target.URI.c_str()); return;
+ case TransactionCommit: _error->Fatal("Transaction %s was already aborted and is now commited", TransactionManager->Target.URI.c_str()); return;
+ }
+ TransactionManager->State = TransactionAbort;
+
// ensure the toplevel is in error state too
for (std::vector<pkgAcqTransactionItem*>::iterator I = Transaction.begin();
I != Transaction.end(); ++I)
{
+ if ((*I)->Status != pkgAcquire::Item::StatFetching)
+ Owner->Dequeue(*I);
(*I)->TransactionState(TransactionAbort);
}
Transaction.clear();
if(_config->FindB("Debug::Acquire::Transaction", false) == true)
std::clog << "CommitTransaction: " << this << std::endl;
+ switch (TransactionManager->State)
+ {
+ case TransactionStarted: break;
+ case TransactionAbort: _error->Fatal("Transaction %s was already commited and is now aborted", TransactionManager->Target.URI.c_str()); return;
+ case TransactionCommit: _error->Fatal("Transaction %s was already commited and is again commited", TransactionManager->Target.URI.c_str()); return;
+ }
+ TransactionManager->State = TransactionCommit;
+
// move new files into place *and* remove files that are not
// part of the transaction but are still on disk
for (std::vector<pkgAcqTransactionItem*>::iterator I = Transaction.begin();
FinalInRelease = FinalFile.substr(0, FinalFile.length() - strlen("Release")) + "InRelease";
FinalRelease = FinalFile;
}
- if (RealFileExists(FinalInRelease) || RealFileExists(FinalRelease))
- {
- TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone();
- if (TransactionManager->LastMetaIndexParser != NULL)
- {
- _error->PushToStack();
- if (RealFileExists(FinalInRelease))
- TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL);
- else
- TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL);
- // its unlikely to happen, but if what we have is bad ignore it
- if (_error->PendingError())
- {
- delete TransactionManager->LastMetaIndexParser;
- TransactionManager->LastMetaIndexParser = NULL;
- }
- _error->RevertToStack();
- }
- }
+ LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
}
if (TransactionManager->MetaIndexParser->Load(DestFile, &ErrorText) == false)
if (filename.empty() == false)
{
new NoActionItem(Owner, *Target, filename);
- std::string const idxfilename = GetFinalFileNameFromURI(Target->URI + ".diff/Index");
+ std::string const idxfilename = GetFinalFileNameFromURI(GetDiffIndexURI(*Target));
if (FileExists(idxfilename))
new NoActionItem(Owner, *Target, idxfilename);
continue;
}
// check if we have patches available
- trypdiff &= TransactionManager->MetaIndexParser->Exists(Target->MetaKey + ".diff/Index");
+ trypdiff &= TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target->MetaKey));
}
else
{
return Header;
}
/*}}}*/
+void pkgAcqMetaClearSig::Finished() /*{{{*/
+{
+ if(_config->FindB("Debug::Acquire::Transaction", false) == true)
+ std::clog << "Finished: " << DestFile <<std::endl;
+ if(TransactionManager != NULL && TransactionManager->State == TransactionStarted &&
+ TransactionManager->TransactionHasError() == false)
+ TransactionManager->CommitTransaction();
+}
+ /*}}}*/
bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message, /*{{{*/
pkgAcquire::MethodConfig const * const Cnf)
{
string const FinalInRelease = GetFinalFilename();
Rename(DestFile, PartialRelease);
TransactionManager->TransactionStageCopy(this, PartialRelease, FinalRelease);
-
- if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease))
- {
- // open the last Release if we have it
- if (TransactionManager->IMSHit == false)
- {
- TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone();
- if (TransactionManager->LastMetaIndexParser != NULL)
- {
- _error->PushToStack();
- if (RealFileExists(FinalInRelease))
- TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL);
- else
- TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL);
- // its unlikely to happen, but if what we have is bad ignore it
- if (_error->PendingError())
- {
- delete TransactionManager->LastMetaIndexParser;
- TransactionManager->LastMetaIndexParser = NULL;
- }
- _error->RevertToStack();
- }
- }
- }
+ LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
// we parse the indexes here because at this point the user wanted
// a repository that may potentially harm him
}
}
/*}}}*/
-void pkgAcqMetaIndex::Finished() /*{{{*/
-{
- if(_config->FindB("Debug::Acquire::Transaction", false) == true)
- std::clog << "Finished: " << DestFile <<std::endl;
- if(TransactionManager != NULL &&
- TransactionManager->TransactionHasError() == false)
- TransactionManager->CommitTransaction();
-}
- /*}}}*/
std::string pkgAcqMetaIndex::DescURI() const /*{{{*/
{
return Target.URI;
// only allow going further if the user explicitly wants it
if (AllowInsecureRepositories(_("The repository '%s' is not signed."), MetaIndex->Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
{
- if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease))
- {
- // open the last Release if we have it
- if (TransactionManager->IMSHit == false)
- {
- TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone();
- if (TransactionManager->LastMetaIndexParser != NULL)
- {
- _error->PushToStack();
- if (RealFileExists(FinalInRelease))
- TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL);
- else
- TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL);
- // its unlikely to happen, but if what we have is bad ignore it
- if (_error->PendingError())
- {
- delete TransactionManager->LastMetaIndexParser;
- TransactionManager->LastMetaIndexParser = NULL;
- }
- _error->RevertToStack();
- }
- }
- }
+ LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
// we parse the indexes here because at this point the user wanted
// a repository that may potentially harm him
Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
Desc.Owner = this;
- Desc.Description = Target.Description + ".diff/Index";
+ Desc.Description = GetDiffIndexFileName(Target.Description);
Desc.ShortDesc = Target.ShortDesc;
- Desc.URI = Target.URI + ".diff/Index";
+ Desc.URI = GetDiffIndexURI(Target);
DestFile = GetPartialFileNameFromURI(Desc.URI);
if (Debug == true)
{
std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": Index has different hashes than parser, probably older, so fail pdiffing" << std::endl;
- printHashSumComparision(CurrentPackagesFile, ServerHashes, TargetFileHashes);
+ printHashSumComparison(CurrentPackagesFile, ServerHashes, TargetFileHashes);
}
return false;
}
return false;
}
+ for (auto const &patch: available_patches)
+ if (patch.result_hashes.usable() == false ||
+ patch.patch_hashes.usable() == false ||
+ patch.download_hashes.usable() == false)
+ {
+ if (Debug)
+ std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": provides no usable hashes for " << patch.file
+ << " so fallback to complete download" << std::endl;
+ return false;
+ }
+
// patching with too many files is rather slow compared to a fast download
unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
if (fileLimit != 0 && fileLimit < available_patches.size())
unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100);
if (sizeLimitPercent > 0 && TransactionManager->MetaIndexParser != nullptr)
{
- // compressed case
unsigned long long downloadSize = std::accumulate(available_patches.begin(),
available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) {
return T + I.download_hashes.FileSize();
return false;
}
}
- // uncompressed case
- downloadSize = std::accumulate(available_patches.begin(),
- available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) {
- return T + I.patch_hashes.FileSize();
- });
- if (downloadSize != 0)
- {
- unsigned long long const downloadSizeIdx = ServerSize;
- unsigned long long const sizeLimit = downloadSizeIdx * sizeLimitPercent;
- if ((sizeLimit/100) < downloadSize)
- {
- if (Debug)
- std::clog << "Need " << downloadSize << " uncompressed bytes (Limit is " << (sizeLimit/100) << ", "
- << "original is " << downloadSizeIdx << ") so fallback to complete download" << std::endl;
- return false;
- }
- }
}
// we have something, queue the diffs
for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
I != allPatches->end(); ++I)
if ((*I)->State == StateErrorDiff)
+ {
+ State = StateErrorDiff;
return;
+ }
// first failure means we should fallback
State = StateErrorDiff;
if (Debug)
std::clog << "Falling back to normal index file acquire" << std::endl;
RenameOnError(PDiffError);
- std::string const patchname = GetPartialFileNameFromURI(Desc.URI);
- if (RealFileExists(patchname))
- Rename(patchname, patchname + ".FAILED");
+ if (RealFileExists(DestFile))
+ Rename(DestFile, DestFile + ".FAILED");
std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI));
if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile))
Rename(UnpatchedFile, UnpatchedFile + ".FAILED");
Item::Done(Message, Hashes, Cnf);
+ if (std::any_of(allPatches->begin(), allPatches->end(),
+ [](pkgAcqIndexMergeDiffs const * const P) { return P->State == StateErrorDiff; }))
+ {
+ if(Debug)
+ std::clog << "Another patch failed already, no point in processing this one." << std::endl;
+ State = StateErrorDiff;
+ return;
+ }
+
std::string const UncompressedUnpatchedFile = GetPartialFileNameFromURI(Target.URI);
std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile);
+ if (UnpatchedFile.empty())
+ {
+ _error->Fatal("Unpatched file %s doesn't exist (anymore)!", UncompressedUnpatchedFile.c_str());
+ State = StateErrorDiff;
+ return;
+ }
std::string const PatchFile = GetMergeDiffsPatchFileName(UnpatchedFile, patch.file);
std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target);
DestFile = GetPartialFileNameFromURI(URI);
NextCompressionExtension(CurrentCompressionExtension, CompressionExtensions, false);
+ // store file size of the download to ensure the fetcher gives
+ // accurate progress reporting
+ FileSize = GetExpectedHashes().FileSize();
+
if (CurrentCompressionExtension == "uncompressed")
{
Desc.URI = URI;
projects / apt.git / blobdiff