+ // at this point the real Items are loaded in the fetcher
+ ExpectedAdditionalItems = 0;
+
+ std::set<std::string> targetsSeen;
+ bool const hasReleaseFile = TransactionManager->MetaIndexParser != NULL;
+ bool const metaBaseSupportsByHash = hasReleaseFile && TransactionManager->MetaIndexParser->GetSupportsAcquireByHash();
+ bool hasHashes = true;
+ auto IndexTargets = TransactionManager->MetaIndexParser->GetIndexTargets();
+ if (hasReleaseFile && verify == false)
+ hasHashes = std::any_of(IndexTargets.begin(), IndexTargets.end(),
+ [&](IndexTarget const &Target) { return TransactionManager->MetaIndexParser->Exists(Target.MetaKey); });
+ if (_config->FindB("Acquire::IndexTargets::Randomized", true) && likely(IndexTargets.empty() == false))
+ {
+ /* For fallback handling and to have some reasonable progress information
+ we can't randomize everything, but at least the order in the same type
+ can be as we shouldn't be telling the mirrors (and everyone else watching)
+ which is native/foreign arch, specific order of preference of translations, … */
+ auto range_start = IndexTargets.begin();
+ std::random_device rd;
+ std::default_random_engine g(rd());
+ do {
+ auto const type = range_start->Option(IndexTarget::CREATED_BY);
+ auto const range_end = std::find_if_not(range_start, IndexTargets.end(),
+ [&type](IndexTarget const &T) { return type == T.Option(IndexTarget::CREATED_BY); });
+ std::shuffle(range_start, range_end, g);
+ range_start = range_end;
+ } while (range_start != IndexTargets.end());
+ }
+ for (auto&& Target: IndexTargets)
+ {
+ // if we have seen a target which is created-by a target this one here is declared a
+ // fallback to, we skip acquiring the fallback (but we make sure we clean up)
+ if (targetsSeen.find(Target.Option(IndexTarget::FALLBACK_OF)) != targetsSeen.end())
+ {
+ targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
+ new CleanupItem(Owner, TransactionManager, Target);
+ continue;
+ }
+ // all is an implementation detail. Users shouldn't use this as arch
+ // We need this support trickery here as e.g. Debian has binary-all files already,
+ // but arch:all packages are still in the arch:any files, so we would waste precious
+ // download time, bandwidth and diskspace for nothing, BUT Debian doesn't feature all
+ // in the set of supported architectures, so we can filter based on this property rather
+ // than invent an entirely new flag we would need to carry for all of eternity.
+ if (hasReleaseFile && Target.Option(IndexTarget::ARCHITECTURE) == "all")
+ {
+ if (TransactionManager->MetaIndexParser->IsArchitectureAllSupportedFor(Target) == false)
+ {
+ new CleanupItem(Owner, TransactionManager, Target);
+ continue;
+ }
+ }
+
+ bool trypdiff = Target.OptionBool(IndexTarget::PDIFFS);
+ if (hasReleaseFile == true)
+ {
+ if (TransactionManager->MetaIndexParser->Exists(Target.MetaKey) == false)
+ {
+ // optional targets that we do not have in the Release file are skipped
+ if (hasHashes == true && Target.IsOptional)
+ {
+ new CleanupItem(Owner, TransactionManager, Target);
+ continue;
+ }
+
+ std::string const &arch = Target.Option(IndexTarget::ARCHITECTURE);
+ if (arch.empty() == false)
+ {
+ if (TransactionManager->MetaIndexParser->IsArchitectureSupported(arch) == false)
+ {
+ new CleanupItem(Owner, TransactionManager, Target);
+ _error->Notice(_("Skipping acquire of configured file '%s' as repository '%s' doesn't support architecture '%s'"),
+ Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str(), arch.c_str());
+ continue;
+ }
+ // if the architecture is officially supported but currently no packages for it available,
+ // ignore silently as this is pretty much the same as just shipping an empty file.
+ // if we don't know which architectures are supported, we do NOT ignore it to notify user about this
+ if (hasHashes == true && TransactionManager->MetaIndexParser->IsArchitectureSupported("*undefined*") == false)
+ {
+ new CleanupItem(Owner, TransactionManager, Target);
+ continue;
+ }
+ }
+
+ if (hasHashes == true)
+ {
+ Status = StatAuthError;
+ strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), Target.MetaKey.c_str());
+ return;
+ }
+ else
+ {
+ new pkgAcqIndex(Owner, TransactionManager, Target);
+ continue;
+ }
+ }
+ else if (verify)
+ {
+ auto const hashes = GetExpectedHashesFor(Target.MetaKey);
+ if (hashes.empty() == false)
+ {
+ if (hashes.usable() == false && TargetIsAllowedToBe(TransactionManager->Target, InsecureType::WEAK) == false)
+ {
+ new CleanupItem(Owner, TransactionManager, Target);
+ _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"),
+ Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str());
+ continue;
+ }
+ // empty files are skipped as acquiring the very small compressed files is a waste of time
+ else if (hashes.FileSize() == 0)
+ {
+ new CleanupItem(Owner, TransactionManager, Target);
+ targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
+ continue;
+ }
+ }
+ }
+
+ // autoselect the compression method
+ std::vector<std::string> types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' ');
+ types.erase(std::remove_if(types.begin(), types.end(), [&](std::string const &t) {
+ if (t == "uncompressed")
+ return TransactionManager->MetaIndexParser->Exists(Target.MetaKey) == false;
+ std::string const MetaKey = Target.MetaKey + "." + t;
+ return TransactionManager->MetaIndexParser->Exists(MetaKey) == false;
+ }), types.end());
+ if (types.empty() == false)
+ {
+ std::ostringstream os;
+ // add the special compressiontype byhash first if supported
+ std::string const useByHashConf = Target.Option(IndexTarget::BY_HASH);
+ bool useByHash = false;
+ if(useByHashConf == "force")
+ useByHash = true;
+ else
+ useByHash = StringToBool(useByHashConf) == true && metaBaseSupportsByHash;
+ if (useByHash == true)
+ os << "by-hash ";
+ std::copy(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " "));
+ os << *types.rbegin();
+ Target.Options["COMPRESSIONTYPES"] = os.str();
+ }
+ else
+ Target.Options["COMPRESSIONTYPES"].clear();
+
+ std::string filename = GetExistingFilename(GetFinalFileNameFromURI(Target.URI));
+ if (filename.empty() == false)
+ {
+ // if the Release file is a hit and we have an index it must be the current one
+ if (TransactionManager->IMSHit == true)
+ ;
+ else if (TransactionManager->LastMetaIndexParser != NULL)
+ {
+ // see if the file changed since the last Release file
+ // we use the uncompressed files as we might compress differently compared to the server,
+ // so the hashes might not match, even if they contain the same data.
+ HashStringList const newFile = GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, Target.MetaKey);
+ HashStringList const oldFile = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, Target.MetaKey);
+ if (newFile != oldFile)
+ filename.clear();
+ }
+ else
+ filename.clear();
+ }
+ else
+ trypdiff = false; // no file to patch
+
+ if (filename.empty() == false)
+ {
+ new NoActionItem(Owner, Target, filename);
+ std::string const idxfilename = GetFinalFileNameFromURI(GetDiffIndexURI(Target));
+ if (FileExists(idxfilename))
+ new NoActionItem(Owner, Target, idxfilename);
+ targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
+ continue;
+ }
+
+ // check if we have patches available
+ trypdiff &= TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target.MetaKey));
+ }
+ else
+ {
+ // if we have no file to patch, no point in trying
+ trypdiff &= (GetExistingFilename(GetFinalFileNameFromURI(Target.URI)).empty() == false);
+ }
+
+ // no point in patching from local sources
+ if (trypdiff)
+ {
+ std::string const proto = Target.URI.substr(0, strlen("file:/"));
+ if (proto == "file:/" || proto == "copy:/" || proto == "cdrom:")
+ trypdiff = false;
+ }
+
+ // Queue the Index file (Packages, Sources, Translation-$foo, …)
+ targetsSeen.emplace(Target.Option(IndexTarget::CREATED_BY));
+ if (trypdiff)
+ new pkgAcqDiffIndex(Owner, TransactionManager, Target);
+ else
+ new pkgAcqIndex(Owner, TransactionManager, Target);
+ }
+}
+ /*}}}*/
+bool pkgAcqMetaBase::VerifyVendor(string const &) /*{{{*/
+{
+ if (TransactionManager->MetaIndexParser->GetValidUntil() > 0)
+ {
+ time_t const invalid_since = time(NULL) - TransactionManager->MetaIndexParser->GetValidUntil();
+ if (invalid_since > 0)
+ {
+ std::string errmsg;
+ strprintf(errmsg,
+ // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
+ // the time since then the file is invalid - formatted in the same way as in
+ // the download progress display (e.g. 7d 3h 42min 1s)
+ _("Release file for %s is expired (invalid since %s). "
+ "Updates for this repository will not be applied."),
+ Target.URI.c_str(), TimeToStr(invalid_since).c_str());
+ if (ErrorText.empty())
+ ErrorText = errmsg;
+ return _error->Error("%s", errmsg.c_str());
+ }
+ }
+
+ /* Did we get a file older than what we have? This is a last minute IMS hit and doubles
+ as a prevention of downgrading us to older (still valid) files */
+ if (TransactionManager->IMSHit == false && TransactionManager->LastMetaIndexParser != NULL &&
+ TransactionManager->LastMetaIndexParser->GetDate() > TransactionManager->MetaIndexParser->GetDate())
+ {
+ TransactionManager->IMSHit = true;
+ RemoveFile("VerifyVendor", DestFile);
+ PartialFile = DestFile = GetFinalFilename();
+ // load the 'old' file in the 'new' one instead of flipping pointers as
+ // the new one isn't owned by us, while the old one is so cleanup would be confused.
+ TransactionManager->MetaIndexParser->swapLoad(TransactionManager->LastMetaIndexParser);
+ delete TransactionManager->LastMetaIndexParser;
+ TransactionManager->LastMetaIndexParser = NULL;
+ }
+
+ if (_config->FindB("Debug::pkgAcquire::Auth", false))
+ {
+ std::cerr << "Got Codename: " << TransactionManager->MetaIndexParser->GetCodename() << std::endl;
+ std::cerr << "Got Suite: " << TransactionManager->MetaIndexParser->GetSuite() << std::endl;
+ std::cerr << "Expecting Dist: " << TransactionManager->MetaIndexParser->GetExpectedDist() << std::endl;
+ }
+
+ // One day that might become fatal…
+ auto const ExpectedDist = TransactionManager->MetaIndexParser->GetExpectedDist();
+ auto const NowCodename = TransactionManager->MetaIndexParser->GetCodename();
+ if (TransactionManager->MetaIndexParser->CheckDist(ExpectedDist) == false)
+ _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
+ Desc.Description.c_str(), ExpectedDist.c_str(), NowCodename.c_str());
+ // might be okay, might be not
+ if (TransactionManager->LastMetaIndexParser != nullptr)
+ {
+ auto const LastCodename = TransactionManager->LastMetaIndexParser->GetCodename();
+ if (LastCodename.empty() == false && NowCodename.empty() == false && LastCodename != NowCodename)
+ _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
+ Desc.Description.c_str(), LastCodename.c_str(), NowCodename.c_str());
+ }
+ return true;
+}
+ /*}}}*/
+pkgAcqMetaBase::~pkgAcqMetaBase()
+{
+}
+
+pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner, /*{{{*/
+ IndexTarget const &ClearsignedTarget,
+ IndexTarget const &DetachedDataTarget, IndexTarget const &DetachedSigTarget,
+ metaIndex * const MetaIndexParser) :
+ pkgAcqMetaIndex(Owner, this, ClearsignedTarget, DetachedSigTarget),
+ d(NULL), DetachedDataTarget(DetachedDataTarget),
+ MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL)
+{
+ // index targets + (worst case:) Release/Release.gpg
+ ExpectedAdditionalItems = std::numeric_limits<decltype(ExpectedAdditionalItems)>::max();
+ TransactionManager->Add(this);
+}
+ /*}}}*/
+pkgAcqMetaClearSig::~pkgAcqMetaClearSig() /*{{{*/
+{
+ if (LastMetaIndexParser != NULL)
+ delete LastMetaIndexParser;
+}
+ /*}}}*/
+// pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
+string pkgAcqMetaClearSig::Custom600Headers() const
+{
+ string Header = pkgAcqMetaBase::Custom600Headers();
+ Header += "\nFail-Ignore: true";
+ std::string const key = TransactionManager->MetaIndexParser->GetSignedBy();
+ if (key.empty() == false)
+ Header += "\nSigned-By: " + key;
+
+ return Header;
+}
+ /*}}}*/
+void pkgAcqMetaClearSig::Finished() /*{{{*/
+{
+ if(_config->FindB("Debug::Acquire::Transaction", false) == true)
+ std::clog << "Finished: " << DestFile <<std::endl;
+ if(TransactionManager->State == TransactionStarted &&
+ TransactionManager->TransactionHasError() == false)
+ TransactionManager->CommitTransaction();
+}
+ /*}}}*/
+bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message, /*{{{*/
+ pkgAcquire::MethodConfig const * const Cnf)
+{
+ Item::VerifyDone(Message, Cnf);
+
+ if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile))
+ return RenameOnError(NotClearsigned);
+
+ return true;
+}
+ /*}}}*/
+// pkgAcqMetaClearSig::Done - We got a file /*{{{*/
+void pkgAcqMetaClearSig::Done(std::string const &Message,
+ HashStringList const &Hashes,
+ pkgAcquire::MethodConfig const * const Cnf)
+{
+ Item::Done(Message, Hashes, Cnf);
+
+ if(AuthPass == false)
+ {
+ if(CheckDownloadDone(this, Message, Hashes) == true)
+ QueueForSignatureVerify(this, DestFile, DestFile);
+ return;
+ }
+ else if(CheckAuthDone(Message) == true)
+ {
+ if (TransactionManager->IMSHit == false)
+ TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
+ else if (RealFileExists(GetFinalFilename()) == false)
+ {
+ // We got an InRelease file IMSHit, but we haven't one, which means
+ // we had a valid Release/Release.gpg combo stepping in, which we have
+ // to 'acquire' now to ensure list cleanup isn't removing them
+ new NoActionItem(Owner, DetachedDataTarget);
+ new NoActionItem(Owner, DetachedSigTarget);
+ }
+ }
+ else if (Status != StatAuthError)
+ {
+ string const FinalFile = GetFinalFileNameFromURI(DetachedDataTarget.URI);
+ string const OldFile = GetFinalFilename();
+ if (TransactionManager->IMSHit == false)
+ TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
+ else if (RealFileExists(OldFile) == false)
+ new NoActionItem(Owner, DetachedDataTarget);
+ else
+ TransactionManager->TransactionStageCopy(this, OldFile, FinalFile);
+ }
+}
+ /*}}}*/
+void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) /*{{{*/
+{
+ Item::Failed(Message, Cnf);
+
+ if (AuthPass == false)
+ {
+ if (Status == StatAuthError || Status == StatTransientNetworkError)
+ {
+ // if we expected a ClearTextSignature (InRelease) but got a network
+ // error or got a file, but it wasn't valid, we end up here (see VerifyDone).
+ // As these is usually called by web-portals we do not try Release/Release.gpg
+ // as this is gonna fail anyway and instead abort our try (LP#346386)
+ TransactionManager->AbortTransaction();
+ return;
+ }
+
+ // Queue the 'old' InRelease file for removal if we try Release.gpg
+ // as otherwise the file will stay around and gives a false-auth
+ // impression (CVE-2012-0214)
+ TransactionManager->TransactionStageRemoval(this, GetFinalFilename());
+ Status = StatDone;
+
+ new pkgAcqMetaIndex(Owner, TransactionManager, DetachedDataTarget, DetachedSigTarget);
+ }
+ else
+ {
+ if(CheckStopAuthentication(this, Message))
+ return;
+
+ if(AllowInsecureRepositories(InsecureType::UNSIGNED, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
+ {
+ Status = StatDone;
+
+ /* InRelease files become Release files, otherwise
+ * they would be considered as trusted later on */
+ string const FinalRelease = GetFinalFileNameFromURI(DetachedDataTarget.URI);
+ string const PartialRelease = GetPartialFileNameFromURI(DetachedDataTarget.URI);
+ string const FinalReleasegpg = GetFinalFileNameFromURI(DetachedSigTarget.URI);
+ string const FinalInRelease = GetFinalFilename();
+ Rename(DestFile, PartialRelease);
+ TransactionManager->TransactionStageCopy(this, PartialRelease, FinalRelease);
+ LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
+
+ // we parse the indexes here because at this point the user wanted
+ // a repository that may potentially harm him
+ if (TransactionManager->MetaIndexParser->Load(PartialRelease, &ErrorText) == false || VerifyVendor(Message) == false)
+ /* expired Release files are still a problem you need extra force for */;
+ else
+ TransactionManager->QueueIndexes(true);
+ }
+ }
+}
+ /*}}}*/
+
+pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire * const Owner, /*{{{*/
+ pkgAcqMetaClearSig * const TransactionManager,
+ IndexTarget const &DataTarget,
+ IndexTarget const &DetachedSigTarget) :
+ pkgAcqMetaBase(Owner, TransactionManager, DataTarget), d(NULL),
+ DetachedSigTarget(DetachedSigTarget)
+{
+ if(_config->FindB("Debug::Acquire::Transaction", false) == true)
+ std::clog << "New pkgAcqMetaIndex with TransactionManager "
+ << this->TransactionManager << std::endl;
+
+ DestFile = GetPartialFileNameFromURI(DataTarget.URI);
+
+ // Create the item
+ Desc.Description = DataTarget.Description;
+ Desc.Owner = this;
+ Desc.ShortDesc = DataTarget.ShortDesc;
+ Desc.URI = DataTarget.URI;
+ QueueURI(Desc);
+}
+ /*}}}*/
+void pkgAcqMetaIndex::Done(string const &Message, /*{{{*/
+ HashStringList const &Hashes,
+ pkgAcquire::MethodConfig const * const Cfg)
+{
+ Item::Done(Message,Hashes,Cfg);
+
+ if(CheckDownloadDone(this, Message, Hashes))
+ {
+ // we have a Release file, now download the Signature, all further
+ // verify/queue for additional downloads will be done in the
+ // pkgAcqMetaSig::Done() code
+ new pkgAcqMetaSig(Owner, TransactionManager, DetachedSigTarget, this);
+ }
+}
+ /*}}}*/
+// pkgAcqMetaIndex::Failed - no Release file present /*{{{*/
+void pkgAcqMetaIndex::Failed(string const &Message,
+ pkgAcquire::MethodConfig const * const Cnf)
+{
+ pkgAcquire::Item::Failed(Message, Cnf);
+ Status = StatDone;
+
+ // No Release file was present so fall
+ // back to queueing Packages files without verification
+ // only allow going further if the user explicitly wants it
+ if(AllowInsecureRepositories(InsecureType::NORELEASE, Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
+ {
+ // ensure old Release files are removed
+ TransactionManager->TransactionStageRemoval(this, GetFinalFilename());
+
+ // queue without any kind of hashsum support
+ TransactionManager->QueueIndexes(false);
+ }
+}
+ /*}}}*/
+std::string pkgAcqMetaIndex::DescURI() const /*{{{*/
+{
+ return Target.URI;
+}
+ /*}}}*/
+pkgAcqMetaIndex::~pkgAcqMetaIndex() {}
+
+// AcqMetaSig::AcqMetaSig - Constructor /*{{{*/
+pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire * const Owner,
+ pkgAcqMetaClearSig * const TransactionManager,
+ IndexTarget const &Target,
+ pkgAcqMetaIndex * const MetaIndex) :
+ pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL), MetaIndex(MetaIndex)
+{
+ DestFile = GetPartialFileNameFromURI(Target.URI);
+
+ // remove any partial downloaded sig-file in partial/.
+ // it may confuse proxies and is too small to warrant a
+ // partial download anyway
+ RemoveFile("pkgAcqMetaSig", DestFile);
+
+ // set the TransactionManager
+ if(_config->FindB("Debug::Acquire::Transaction", false) == true)
+ std::clog << "New pkgAcqMetaSig with TransactionManager "
+ << TransactionManager << std::endl;
+
+ // Create the item
+ Desc.Description = Target.Description;
+ Desc.Owner = this;
+ Desc.ShortDesc = Target.ShortDesc;
+ Desc.URI = Target.URI;
+
+ // If we got a hit for Release, we will get one for Release.gpg too (or obscure errors),
+ // so we skip the download step and go instantly to verification
+ if (TransactionManager->IMSHit == true && RealFileExists(GetFinalFilename()))
+ {
+ Complete = true;
+ Status = StatDone;
+ PartialFile = DestFile = GetFinalFilename();
+ MetaIndexFileSignature = DestFile;
+ MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
+ }
+ else
+ QueueURI(Desc);
+}
+ /*}}}*/
+pkgAcqMetaSig::~pkgAcqMetaSig() /*{{{*/
+{
+}
+ /*}}}*/
+// pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/
+std::string pkgAcqMetaSig::Custom600Headers() const
+{
+ std::string Header = pkgAcqTransactionItem::Custom600Headers();
+ std::string const key = TransactionManager->MetaIndexParser->GetSignedBy();
+ if (key.empty() == false)
+ Header += "\nSigned-By: " + key;
+ return Header;
+}
+ /*}}}*/
+// AcqMetaSig::Done - The signature was downloaded/verified /*{{{*/
+void pkgAcqMetaSig::Done(string const &Message, HashStringList const &Hashes,
+ pkgAcquire::MethodConfig const * const Cfg)
+{
+ if (MetaIndexFileSignature.empty() == false)
+ {
+ DestFile = MetaIndexFileSignature;
+ MetaIndexFileSignature.clear();
+ }
+ Item::Done(Message, Hashes, Cfg);
+
+ if(MetaIndex->AuthPass == false)
+ {
+ if(MetaIndex->CheckDownloadDone(this, Message, Hashes) == true)
+ {
+ // destfile will be modified to point to MetaIndexFile for the
+ // gpgv method, so we need to save it here
+ MetaIndexFileSignature = DestFile;
+ MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
+ }
+ return;
+ }
+ else if(MetaIndex->CheckAuthDone(Message) == true)
+ {
+ auto const Releasegpg = GetFinalFilename();
+ auto const Release = MetaIndex->GetFinalFilename();
+ // if this is an IMS-Hit on Release ensure we also have the the Release.gpg file stored
+ // (previously an unknown pubkey) – but only if the Release file exists locally (unlikely
+ // event of InRelease removed from the mirror causing fallback but still an IMS-Hit)
+ if (TransactionManager->IMSHit == false ||
+ (FileExists(Releasegpg) == false && FileExists(Release) == true))
+ {
+ TransactionManager->TransactionStageCopy(this, DestFile, Releasegpg);
+ TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, Release);
+ }
+ }
+ else if (MetaIndex->Status != StatAuthError)
+ {
+ std::string const FinalFile = MetaIndex->GetFinalFilename();
+ if (TransactionManager->IMSHit == false)
+ TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, FinalFile);
+ else
+ TransactionManager->TransactionStageCopy(MetaIndex, FinalFile, FinalFile);
+ }
+}
+ /*}}}*/
+void pkgAcqMetaSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
+{
+ Item::Failed(Message,Cnf);
+
+ // check if we need to fail at this point
+ if (MetaIndex->AuthPass == true && MetaIndex->CheckStopAuthentication(this, Message))
+ return;
+
+ // ensures that a Release.gpg file in the lists/ is removed by the transaction
+ TransactionManager->TransactionStageRemoval(this, DestFile);
+
+ // only allow going further if the user explicitly wants it
+ if (AllowInsecureRepositories(InsecureType::UNSIGNED, MetaIndex->Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true)
+ {
+ string const FinalRelease = MetaIndex->GetFinalFilename();
+ string const FinalInRelease = TransactionManager->GetFinalFilename();
+ LoadLastMetaIndexParser(TransactionManager, FinalRelease, FinalInRelease);
+
+ // we parse the indexes here because at this point the user wanted
+ // a repository that may potentially harm him
+ bool const GoodLoad = TransactionManager->MetaIndexParser->Load(MetaIndex->DestFile, &ErrorText);
+ if (MetaIndex->VerifyVendor(Message) == false)
+ /* expired Release files are still a problem you need extra force for */;
+ else
+ TransactionManager->QueueIndexes(GoodLoad);
+
+ TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, FinalRelease);
+ }
+ else if (TransactionManager->IMSHit == false)
+ Rename(MetaIndex->DestFile, MetaIndex->DestFile + ".FAILED");
+
+ // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
+ if (Cnf->LocalOnly == true ||
+ StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
+ {
+ // Ignore this
+ Status = StatDone;
+ }
+}
+ /*}}}*/
+
+
+// AcqBaseIndex - Constructor /*{{{*/
+pkgAcqBaseIndex::pkgAcqBaseIndex(pkgAcquire * const Owner,
+ pkgAcqMetaClearSig * const TransactionManager,
+ IndexTarget const &Target)
+: pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL)
+{
+}
+ /*}}}*/
+void pkgAcqBaseIndex::Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
+{
+ pkgAcquire::Item::Failed(Message, Cnf);
+ if (Status != StatAuthError)
+ return;
+
+ ErrorText.append("Release file created at: ");
+ auto const timespec = TransactionManager->MetaIndexParser->GetDate();
+ if (timespec == 0)
+ ErrorText.append("<unknown>");
+ else
+ ErrorText.append(TimeRFC1123(timespec, true));
+ ErrorText.append("\n");
+}
+ /*}}}*/
+pkgAcqBaseIndex::~pkgAcqBaseIndex() {}
+
+// AcqDiffIndex::AcqDiffIndex - Constructor /*{{{*/
+// ---------------------------------------------------------------------
+/* Get the DiffIndex file first and see if there are patches available
+ * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
+ * patches. If anything goes wrong in that process, it will fall back to
+ * the original packages file
+ */
+pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner,
+ pkgAcqMetaClearSig * const TransactionManager,
+ IndexTarget const &Target)
+ : pkgAcqIndex(Owner, TransactionManager, Target, true), d(NULL), diffs(NULL)
+{
+ // FIXME: Magic number as an upper bound on pdiffs we will reasonably acquire
+ ExpectedAdditionalItems = 40;
+ Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
+
+ CompressionExtensions.clear();
+ {
+ std::vector<std::string> types = APT::Configuration::getCompressionTypes();
+ if (types.empty() == false)
+ {
+ std::ostringstream os;
+ std::copy_if(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " "), [&](std::string const type) {
+ if (type == "uncompressed")
+ return true;
+ return TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target.MetaKey) + '.' + type);
+ });
+ os << *types.rbegin();
+ CompressionExtensions = os.str();
+ }
+ }
+ if (Target.Option(IndexTarget::COMPRESSIONTYPES).find("by-hash") != std::string::npos)
+ CompressionExtensions = "by-hash " + CompressionExtensions;
+ Init(GetDiffIndexURI(Target), GetDiffIndexFileName(Target.Description), Target.ShortDesc);
+
+ if(Debug)
+ std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
+}
+ /*}}}*/
+void pkgAcqDiffIndex::QueueOnIMSHit() const /*{{{*/
+{
+ // list cleanup needs to know that this file as well as the already
+ // present index is ours, so we create an empty diff to save it for us
+ new pkgAcqIndexDiffs(Owner, TransactionManager, Target, UsedMirror, Target.URI);
+}
+ /*}}}*/
+static bool RemoveFileForBootstrapLinking(bool const Debug, std::string const &For, std::string const &Boot)/*{{{*/
+{
+ if (FileExists(Boot) && RemoveFile("Bootstrap-linking", Boot) == false)
+ {
+ if (Debug)
+ std::clog << "Bootstrap-linking for patching " << For
+ << " by removing stale " << Boot << " failed!" << std::endl;
+ return false;
+ }
+ return true;
+}
+ /*}}}*/
+bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
+{
+ ExpectedAdditionalItems = 0;
+ // failing here is fine: our caller will take care of trying to
+ // get the complete file if patching fails
+ if(Debug)
+ std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
+ << std::endl;
+
+ FileFd Fd(IndexDiffFile, FileFd::ReadOnly, FileFd::Extension);
+ pkgTagFile TF(&Fd);
+ if (Fd.IsOpen() == false || Fd.Failed())
+ return false;
+
+ pkgTagSection Tags;
+ if(unlikely(TF.Step(Tags) == false))
+ return false;
+
+ HashStringList ServerHashes;
+ unsigned long long ServerSize = 0;
+
+ auto const &posix = std::locale::classic();
+ for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
+ {
+ std::string tagname = *type;
+ tagname.append("-Current");
+ std::string const tmp = Tags.FindS(tagname.c_str());
+ if (tmp.empty() == true)
+ continue;
+
+ string hash;
+ unsigned long long size;
+ std::stringstream ss(tmp);
+ ss.imbue(posix);
+ ss >> hash >> size;
+ if (unlikely(hash.empty() == true))
+ continue;
+ if (unlikely(ServerSize != 0 && ServerSize != size))
+ continue;
+ ServerHashes.push_back(HashString(*type, hash));
+ ServerSize = size;
+ }
+
+ if (ServerHashes.usable() == false)
+ {
+ if (Debug == true)
+ std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": Did not find a good hashsum in the index" << std::endl;
+ return false;
+ }
+
+ std::string const CurrentPackagesFile = GetFinalFileNameFromURI(Target.URI);
+ HashStringList const TargetFileHashes = GetExpectedHashesFor(Target.MetaKey);
+ if (TargetFileHashes.usable() == false || ServerHashes != TargetFileHashes)
+ {
+ if (Debug == true)
+ {
+ std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": Index has different hashes than parser, probably older, so fail pdiffing" << std::endl;
+ printHashSumComparison(CurrentPackagesFile, ServerHashes, TargetFileHashes);
+ }
+ return false;
+ }
+
+ HashStringList LocalHashes;
+ // try avoiding calculating the hash here as this is costly
+ if (TransactionManager->LastMetaIndexParser != NULL)
+ LocalHashes = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, Target.MetaKey);
+ if (LocalHashes.usable() == false)
+ {
+ FileFd fd(CurrentPackagesFile, FileFd::ReadOnly, FileFd::Auto);
+ Hashes LocalHashesCalc(ServerHashes);
+ LocalHashesCalc.AddFD(fd);
+ LocalHashes = LocalHashesCalc.GetHashStringList();
+ }
+
+ if (ServerHashes == LocalHashes)
+ {
+ // we have the same sha1 as the server so we are done here
+ if(Debug)
+ std::clog << "pkgAcqDiffIndex: Package file " << CurrentPackagesFile << " is up-to-date" << std::endl;
+ QueueOnIMSHit();
+ return true;
+ }
+
+ if(Debug)
+ std::clog << "Server-Current: " << ServerHashes.find(NULL)->toStr() << " and we start at "
+ << CurrentPackagesFile << " " << LocalHashes.FileSize() << " " << LocalHashes.find(NULL)->toStr() << std::endl;
+
+ // historically, older hashes have more info than newer ones, so start
+ // collecting with older ones first to avoid implementing complicated
+ // information merging techniques… a failure is after all always
+ // recoverable with a complete file and hashes aren't changed that often.
+ std::vector<char const *> types;
+ for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
+ types.push_back(*type);
+
+ // parse all of (provided) history
+ vector<DiffInfo> available_patches;
+ bool firstAcceptedHashes = true;
+ for (auto type = types.crbegin(); type != types.crend(); ++type)
+ {
+ if (LocalHashes.find(*type) == NULL)
+ continue;
+
+ std::string tagname = *type;
+ tagname.append("-History");
+ std::string const tmp = Tags.FindS(tagname.c_str());
+ if (tmp.empty() == true)
+ continue;
+
+ string hash, filename;
+ unsigned long long size;
+ std::stringstream ss(tmp);
+ ss.imbue(posix);
+
+ while (ss >> hash >> size >> filename)
+ {
+ if (unlikely(hash.empty() == true || filename.empty() == true))
+ continue;
+
+ // see if we have a record for this file already
+ std::vector<DiffInfo>::iterator cur = available_patches.begin();
+ for (; cur != available_patches.end(); ++cur)
+ {
+ if (cur->file != filename)
+ continue;
+ cur->result_hashes.push_back(HashString(*type, hash));
+ break;
+ }
+ if (cur != available_patches.end())
+ continue;
+ if (firstAcceptedHashes == true)
+ {
+ DiffInfo next;
+ next.file = filename;
+ next.result_hashes.push_back(HashString(*type, hash));
+ next.result_hashes.FileSize(size);
+ available_patches.push_back(next);
+ }
+ else
+ {
+ if (Debug == true)
+ std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
+ << " wasn't in the list for the first parsed hash! (history)" << std::endl;
+ break;
+ }
+ }
+ firstAcceptedHashes = false;
+ }
+
+ if (unlikely(available_patches.empty() == true))
+ {
+ if (Debug)
+ std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": "
+ << "Couldn't find any patches for the patch series." << std::endl;
+ return false;
+ }
+
+ for (auto type = types.crbegin(); type != types.crend(); ++type)
+ {
+ if (LocalHashes.find(*type) == NULL)
+ continue;
+
+ std::string tagname = *type;
+ tagname.append("-Patches");
+ std::string const tmp = Tags.FindS(tagname.c_str());
+ if (tmp.empty() == true)
+ continue;
+
+ string hash, filename;
+ unsigned long long size;
+ std::stringstream ss(tmp);
+ ss.imbue(posix);
+
+ while (ss >> hash >> size >> filename)
+ {
+ if (unlikely(hash.empty() == true || filename.empty() == true))
+ continue;
+
+ // see if we have a record for this file already
+ std::vector<DiffInfo>::iterator cur = available_patches.begin();
+ for (; cur != available_patches.end(); ++cur)
+ {
+ if (cur->file != filename)
+ continue;
+ if (cur->patch_hashes.empty())
+ cur->patch_hashes.FileSize(size);
+ cur->patch_hashes.push_back(HashString(*type, hash));
+ break;
+ }
+ if (cur != available_patches.end())
+ continue;
+ if (Debug == true)
+ std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
+ << " wasn't in the list for the first parsed hash! (patches)" << std::endl;
+ break;
+ }
+ }
+
+ for (auto type = types.crbegin(); type != types.crend(); ++type)
+ {
+ std::string tagname = *type;
+ tagname.append("-Download");
+ std::string const tmp = Tags.FindS(tagname.c_str());
+ if (tmp.empty() == true)
+ continue;
+
+ string hash, filename;
+ unsigned long long size;
+ std::stringstream ss(tmp);
+ ss.imbue(posix);
+
+ // FIXME: all of pdiff supports only .gz compressed patches
+ while (ss >> hash >> size >> filename)
+ {
+ if (unlikely(hash.empty() == true || filename.empty() == true))
+ continue;
+ if (unlikely(APT::String::Endswith(filename, ".gz") == false))
+ continue;
+ filename.erase(filename.length() - 3);
+
+ // see if we have a record for this file already
+ std::vector<DiffInfo>::iterator cur = available_patches.begin();
+ for (; cur != available_patches.end(); ++cur)
+ {
+ if (cur->file != filename)
+ continue;
+ if (cur->download_hashes.empty())
+ cur->download_hashes.FileSize(size);
+ cur->download_hashes.push_back(HashString(*type, hash));
+ break;
+ }
+ if (cur != available_patches.end())
+ continue;
+ if (Debug == true)
+ std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
+ << " wasn't in the list for the first parsed hash! (download)" << std::endl;
+ break;
+ }
+ }
+
+
+ bool foundStart = false;
+ for (std::vector<DiffInfo>::iterator cur = available_patches.begin();
+ cur != available_patches.end(); ++cur)
+ {
+ if (LocalHashes != cur->result_hashes)
+ continue;
+
+ available_patches.erase(available_patches.begin(), cur);
+ foundStart = true;
+ break;
+ }
+
+ if (foundStart == false || unlikely(available_patches.empty() == true))
+ {
+ if (Debug)
+ std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": "
+ << "Couldn't find the start of the patch series." << std::endl;
+ return false;
+ }
+
+ for (auto const &patch: available_patches)
+ if (patch.result_hashes.usable() == false ||
+ patch.patch_hashes.usable() == false ||
+ patch.download_hashes.usable() == false)
+ {
+ if (Debug)
+ std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": provides no usable hashes for " << patch.file
+ << " so fallback to complete download" << std::endl;
+ return false;
+ }
+
+ // patching with too many files is rather slow compared to a fast download
+ unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
+ if (fileLimit != 0 && fileLimit < available_patches.size())
+ {
+ if (Debug)
+ std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
+ << ") so fallback to complete download" << std::endl;
+ return false;
+ }
+
+ // calculate the size of all patches we have to get
+ unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100);
+ if (sizeLimitPercent > 0)
+ {
+ unsigned long long downloadSize = std::accumulate(available_patches.begin(),
+ available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) {
+ return T + I.download_hashes.FileSize();
+ });
+ if (downloadSize != 0)
+ {
+ unsigned long long downloadSizeIdx = 0;
+ auto const types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' ');
+ for (auto const &t : types)
+ {
+ std::string MetaKey = Target.MetaKey;
+ if (t != "uncompressed")
+ MetaKey += '.' + t;
+ HashStringList const hsl = GetExpectedHashesFor(MetaKey);
+ if (unlikely(hsl.usable() == false))
+ continue;
+ downloadSizeIdx = hsl.FileSize();
+ break;
+ }
+ unsigned long long const sizeLimit = downloadSizeIdx * sizeLimitPercent;
+ if ((sizeLimit/100) < downloadSize)
+ {
+ if (Debug)
+ std::clog << "Need " << downloadSize << " compressed bytes (Limit is " << (sizeLimit/100) << ", "
+ << "original is " << downloadSizeIdx << ") so fallback to complete download" << std::endl;
+ return false;
+ }
+ }
projects / apt.git / blobdiff