X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/245dde96193702f7f51389d3583dee547f8ba366..831ea74cddf8568b8648edddb394fdd3d1ba7b8e:/apt-pkg/acquire-item.cc?ds=sidebyside diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 3313aaabc..446551cc2 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -22,9 +22,8 @@ #include #include #include -#include #include -#include +#include #include #include #include @@ -33,6 +32,7 @@ #include #include +#include #include #include #include @@ -42,9 +42,9 @@ #include #include #include -#include #include #include +#include #include /*}}}*/ @@ -80,20 +80,19 @@ static std::string GetFinalFileNameFromURI(std::string const &uri) /*{{{*/ return _config->FindDir("Dir::State::lists") + URItoFileName(uri); } /*}}}*/ -static std::string GetCompressedFileName(std::string const &URI, std::string const &Name, std::string const &Ext) /*{{{*/ +static std::string GetKeepCompressedFileName(std::string file, IndexTarget const &Target)/*{{{*/ { - if (Ext.empty() || Ext == "uncompressed") - return Name; - - // do not reverify cdrom sources as apt-cdrom may rewrite the Packages - // file when its doing the indexcopy - if (URI.substr(0,6) == "cdrom:") - return Name; + if (Target.KeepCompressed == false) + return file; - // adjust DestFile if its compressed on disk - if (_config->FindB("Acquire::GzipIndexes",false) == true) - return Name + '.' + Ext; - return Name; + std::string const KeepCompressedAs = Target.Option(IndexTarget::KEEPCOMPRESSEDAS); + if (KeepCompressedAs.empty() == false) + { + std::string const ext = KeepCompressedAs.substr(0, KeepCompressedAs.find(' ')); + if (ext != "uncompressed") + file.append(".").append(ext); + } + return file; } /*}}}*/ static std::string GetMergeDiffsPatchFileName(std::string const &Final, std::string const &Patch)/*{{{*/ @@ -108,23 +107,65 @@ static std::string GetDiffsPatchFileName(std::string const &Final) /*{{{*/ return Final + ".ed"; } /*}}}*/ +static std::string GetExistingFilename(std::string const &File) /*{{{*/ +{ + if (RealFileExists(File)) + return File; + for (auto const &type : APT::Configuration::getCompressorExtensions()) + { + std::string const Final = File + type; + if (RealFileExists(Final)) + return Final; + } + return ""; +} + /*}}}*/ -static bool AllowInsecureRepositories(indexRecords const * const MetaIndexParser, pkgAcqMetaBase * const TransactionManager, pkgAcquire::Item * const I) /*{{{*/ +static bool MessageInsecureRepository(bool const isError, std::string const &msg)/*{{{*/ +{ + if (isError) + { + _error->Error("%s", msg.c_str()); + _error->Notice("%s", _("Updating from such a repository can't be done securely, and is therefore disabled by default.")); + } + else + { + _error->Warning("%s", msg.c_str()); + _error->Notice("%s", _("Data from such a repository can't be authenticated and is therefore potentially dangerous to use.")); + } + _error->Notice("%s", _("See apt-secure(8) manpage for repository creation and user configuration details.")); + return false; +} +static bool MessageInsecureRepository(bool const isError, char const * const msg, std::string const &repo) +{ + std::string m; + strprintf(m, msg, repo.c_str()); + return MessageInsecureRepository(isError, m); +} + /*}}}*/ +static bool AllowInsecureRepositories(char const * const msg, std::string const &repo,/*{{{*/ + metaIndex const * const MetaIndexParser, pkgAcqMetaClearSig * const TransactionManager, pkgAcquire::Item * const I) { - if(MetaIndexParser->IsAlwaysTrusted() || _config->FindB("Acquire::AllowInsecureRepositories") == true) + if(MetaIndexParser->GetTrusted() == metaIndex::TRI_YES) + return true; + + if (_config->FindB("Acquire::AllowInsecureRepositories") == true) + { + MessageInsecureRepository(false, msg, repo); return true; + } - _error->Error(_("Use --allow-insecure-repositories to force the update")); + MessageInsecureRepository(true, msg, repo); TransactionManager->AbortTransaction(); I->Status = pkgAcquire::Item::StatError; return false; } /*}}}*/ -static HashStringList GetExpectedHashesFromFor(indexRecords * const Parser, std::string const MetaKey)/*{{{*/ +static HashStringList GetExpectedHashesFromFor(metaIndex * const Parser, std::string const &MetaKey)/*{{{*/ { if (Parser == NULL) return HashStringList(); - indexRecords::checkSum * const R = Parser->Lookup(MetaKey); + metaIndex::checkSum * const R = Parser->Lookup(MetaKey); if (R == NULL) return HashStringList(); return R->Hashes; @@ -144,7 +185,8 @@ APT_CONST bool pkgAcqTransactionItem::HashesRequired() const we can at least trust them for integrity of the download itself. Only repositories without a Release file can (obviously) not have hashes – and they are very uncommon and strongly discouraged */ - return TransactionManager->MetaIndexParser != NULL; + return TransactionManager->MetaIndexParser != NULL && + TransactionManager->MetaIndexParser->GetLoadedSuccessfully() == metaIndex::TRI_YES; } HashStringList pkgAcqTransactionItem::GetExpectedHashes() const { @@ -268,7 +310,7 @@ std::string pkgAcqDiffIndex::GetFinalFilename() const std::string pkgAcqIndex::GetFinalFilename() const { std::string const FinalFile = GetFinalFileNameFromURI(Target.URI); - return GetCompressedFileName(Target.URI, FinalFile, CurrentCompressionExtension); + return GetKeepCompressedFileName(FinalFile, Target); } std::string pkgAcqMetaSig::GetFinalFilename() const { @@ -319,16 +361,66 @@ bool pkgAcqTransactionItem::TransactionState(TransactionStates const state) } break; case TransactionCommit: - if(PartialFile != "") + if(PartialFile.empty() == false) { - if(Debug == true) - std::clog << "mv " << PartialFile << " -> "<< DestFile << " # " << DescURI() << std::endl; + bool sameFile = (PartialFile == DestFile); + // we use symlinks on IMS-Hit to avoid copies + if (RealFileExists(DestFile)) + { + struct stat Buf; + if (lstat(PartialFile.c_str(), &Buf) != -1) + { + if (S_ISLNK(Buf.st_mode) && Buf.st_size > 0) + { + char partial[Buf.st_size + 1]; + ssize_t const sp = readlink(PartialFile.c_str(), partial, Buf.st_size); + if (sp == -1) + _error->Errno("pkgAcqTransactionItem::TransactionState-sp", _("Failed to readlink %s"), PartialFile.c_str()); + else + { + partial[sp] = '\0'; + sameFile = (DestFile == partial); + } + } + } + else + _error->Errno("pkgAcqTransactionItem::TransactionState-stat", _("Failed to stat %s"), PartialFile.c_str()); + } + if (sameFile == false) + { + // ensure that even without lists-cleanup all compressions are nuked + std::string FinalFile = GetFinalFileNameFromURI(Target.URI); + if (FileExists(FinalFile)) + { + if(Debug == true) + std::clog << "rm " << FinalFile << " # " << DescURI() << std::endl; + if (RemoveFile("TransactionStates-Cleanup", FinalFile) == false) + return false; + } + for (auto const &ext: APT::Configuration::getCompressorExtensions()) + { + auto const Final = FinalFile + ext; + if (FileExists(Final)) + { + if(Debug == true) + std::clog << "rm " << Final << " # " << DescURI() << std::endl; + if (RemoveFile("TransactionStates-Cleanup", Final) == false) + return false; + } + } + if(Debug == true) + std::clog << "mv " << PartialFile << " -> "<< DestFile << " # " << DescURI() << std::endl; + if (Rename(PartialFile, DestFile) == false) + return false; + } + else if(Debug == true) + std::clog << "keep " << PartialFile << " # " << DescURI() << std::endl; - Rename(PartialFile, DestFile); } else { if(Debug == true) std::clog << "rm " << DestFile << " # " << DescURI() << std::endl; - unlink(DestFile.c_str()); + if (RemoveFile("TransactionCommit", DestFile) == false) + return false; } break; } @@ -353,13 +445,13 @@ bool pkgAcqIndex::TransactionState(TransactionStates const state) { // keep the compressed file, but drop the decompressed EraseFileName.clear(); - if (PartialFile.empty() == false && flExtension(PartialFile) == "decomp") - unlink(PartialFile.c_str()); + if (PartialFile.empty() == false && flExtension(PartialFile) != CurrentCompressionExtension) + RemoveFile("TransactionAbort", PartialFile); } break; case TransactionCommit: if (EraseFileName.empty() == false) - unlink(EraseFileName.c_str()); + RemoveFile("TransactionCommit", EraseFileName); break; } return true; @@ -375,7 +467,7 @@ bool pkgAcqDiffIndex::TransactionState(TransactionStates const state) break; case TransactionAbort: std::string const Partial = GetPartialFileNameFromURI(Target.URI); - unlink(Partial.c_str()); + RemoveFile("TransactionAbort", Partial); break; } @@ -390,23 +482,29 @@ class APT_HIDDEN NoActionItem : public pkgAcquire::Item /*{{{*/ { IndexTarget const Target; public: - virtual std::string DescURI() const {return Target.URI;}; - virtual HashStringList GetExpectedHashes() const {return HashStringList();}; + virtual std::string DescURI() const APT_OVERRIDE {return Target.URI;}; + virtual HashStringList GetExpectedHashes() const APT_OVERRIDE {return HashStringList();}; - NoActionItem(pkgAcquire * const Owner, IndexTarget const Target) : + NoActionItem(pkgAcquire * const Owner, IndexTarget const &Target) : pkgAcquire::Item(Owner), Target(Target) { Status = StatDone; DestFile = GetFinalFileNameFromURI(Target.URI); } + NoActionItem(pkgAcquire * const Owner, IndexTarget const &Target, std::string const &FinalFile) : + pkgAcquire::Item(Owner), Target(Target) + { + Status = StatDone; + DestFile = FinalFile; + } }; /*}}}*/ // Acquire::Item::Item - Constructor /*{{{*/ APT_IGNORE_DEPRECATED_PUSH -pkgAcquire::Item::Item(pkgAcquire * const Owner) : +pkgAcquire::Item::Item(pkgAcquire * const owner) : FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false), Local(false), - QueueCounter(0), ExpectedAdditionalItems(0), Owner(Owner) + QueueCounter(0), ExpectedAdditionalItems(0), Owner(owner), d(NULL) { Owner->Add(this); Status = StatIdle; @@ -456,7 +554,6 @@ void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig con { if(ErrorText.empty()) ErrorText = LookupTag(Message,"Message"); - UsedMirror = LookupTag(Message,"UsedMirror"); if (QueueCounter <= 1) { /* This indicates that the file is not available right now but might @@ -514,12 +611,28 @@ void pkgAcquire::Item::Start(string const &/*Message*/, unsigned long long const FileSize = Size; } /*}}}*/ +// Acquire::Item::VerifyDone - check if Item was downloaded OK /*{{{*/ +/* Note that hash-verification is 'hardcoded' in acquire-worker and has + * already passed if this method is called. */ +bool pkgAcquire::Item::VerifyDone(std::string const &Message, + pkgAcquire::MethodConfig const * const /*Cnf*/) +{ + std::string const FileName = LookupTag(Message,"Filename"); + if (FileName.empty() == true) + { + Status = StatError; + ErrorText = "Method gave a blank filename"; + return false; + } + + return true; +} + /*}}}*/ // Acquire::Item::Done - Item downloaded OK /*{{{*/ -void pkgAcquire::Item::Done(string const &Message, HashStringList const &Hashes, +void pkgAcquire::Item::Done(string const &/*Message*/, HashStringList const &Hashes, pkgAcquire::MethodConfig const * const /*Cnf*/) { // We just downloaded something.. - UsedMirror = LookupTag(Message,"UsedMirror"); if (FileSize == 0) { unsigned long long const downloadedSize = Hashes.FileSize(); @@ -586,8 +699,8 @@ bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const Status = StatError; break; case NotClearsigned: - errtext = _("Does not start with a cleartext signature"); - Status = StatError; + strprintf(errtext, _("Clearsigned file isn't valid, got '%s' (does the network require authentication?)"), "NOSPLIT"); + Status = StatAuthError; break; case MaximumSizeExceeded: // the method is expected to report a good error for this @@ -661,8 +774,8 @@ std::string pkgAcquire::Item::HashSum() const /*{{{*/ /*}}}*/ pkgAcqTransactionItem::pkgAcqTransactionItem(pkgAcquire * const Owner, /*{{{*/ - pkgAcqMetaBase * const TransactionManager, IndexTarget const Target) : - pkgAcquire::Item(Owner), Target(Target), TransactionManager(TransactionManager) + pkgAcqMetaClearSig * const transactionManager, IndexTarget const &target) : + pkgAcquire::Item(Owner), d(NULL), Target(target), TransactionManager(transactionManager) { if (TransactionManager != this) TransactionManager->Add(this); @@ -672,7 +785,7 @@ pkgAcqTransactionItem::~pkgAcqTransactionItem() /*{{{*/ { } /*}}}*/ -HashStringList pkgAcqTransactionItem::GetExpectedHashesFor(std::string const MetaKey) const /*{{{*/ +HashStringList pkgAcqTransactionItem::GetExpectedHashesFor(std::string const &MetaKey) const /*{{{*/ { return GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, MetaKey); } @@ -680,12 +793,11 @@ HashStringList pkgAcqTransactionItem::GetExpectedHashesFor(std::string const Met // AcqMetaBase - Constructor /*{{{*/ pkgAcqMetaBase::pkgAcqMetaBase(pkgAcquire * const Owner, - pkgAcqMetaBase * const TransactionManager, - std::vector const IndexTargets, - IndexTarget const &DataTarget, - indexRecords * const MetaIndexParser) -: pkgAcqTransactionItem(Owner, TransactionManager, DataTarget), - MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL), IndexTargets(IndexTargets), + pkgAcqMetaClearSig * const TransactionManager, + std::vector const &IndexTargets, + IndexTarget const &DataTarget) +: pkgAcqTransactionItem(Owner, TransactionManager, DataTarget), d(NULL), + IndexTargets(IndexTargets), AuthPass(false), IMSHit(false) { } @@ -775,7 +887,7 @@ bool pkgAcqMetaBase::CheckStopAuthentication(pkgAcquire::Item * const I, const s _error->Warning(_("An error occurred during the signature " "verification. The repository is not updated " "and the previous index files will be used. " - "GPG error: %s: %s\n"), + "GPG error: %s: %s"), Desc.Description.c_str(), LookupTag(Message,"Message").c_str()); RunScripts("APT::Update::Auth-Failure"); @@ -785,7 +897,7 @@ bool pkgAcqMetaBase::CheckStopAuthentication(pkgAcquire::Item * const I, const s _error->Error(_("GPG error: %s: %s"), Desc.Description.c_str(), LookupTag(Message,"Message").c_str()); - I->Status = StatError; + I->Status = StatAuthError; return true; } else { _error->Warning(_("GPG error: %s: %s"), @@ -808,7 +920,6 @@ string pkgAcqMetaBase::Custom600Headers() const Header += MaximumSize; string const FinalFile = GetFinalFilename(); - struct stat Buf; if (stat(FinalFile.c_str(),&Buf) == 0) Header += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); @@ -832,14 +943,7 @@ bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const st // We have just finished downloading a Release file (it is not // verified yet) - string const FileName = LookupTag(Message,"Filename"); - if (FileName.empty() == true) - { - I->Status = StatError; - I->ErrorText = "Method gave a blank filename"; - return false; - } - + std::string const FileName = LookupTag(Message,"Filename"); if (FileName != I->DestFile && RealFileExists(I->DestFile) == false) { I->Local = true; @@ -857,7 +961,7 @@ bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const st if (RealFileExists(FinalFile) && Hashes.VerifyFile(FinalFile) == true) { IMSHit = true; - unlink(I->DestFile.c_str()); + RemoveFile("CheckDownloadDone", I->DestFile); } } @@ -901,26 +1005,28 @@ bool pkgAcqMetaBase::CheckAuthDone(string const &Message) /*{{{*/ } if (RealFileExists(FinalInRelease) || RealFileExists(FinalRelease)) { - TransactionManager->LastMetaIndexParser = new indexRecords; - _error->PushToStack(); - if (RealFileExists(FinalInRelease)) - TransactionManager->LastMetaIndexParser->Load(FinalInRelease); - else - TransactionManager->LastMetaIndexParser->Load(FinalRelease); - // its unlikely to happen, but if what we have is bad ignore it - if (_error->PendingError()) + TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone(); + if (TransactionManager->LastMetaIndexParser != NULL) { - delete TransactionManager->LastMetaIndexParser; - TransactionManager->LastMetaIndexParser = NULL; + _error->PushToStack(); + if (RealFileExists(FinalInRelease)) + TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL); + else + TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL); + // its unlikely to happen, but if what we have is bad ignore it + if (_error->PendingError()) + { + delete TransactionManager->LastMetaIndexParser; + TransactionManager->LastMetaIndexParser = NULL; + } + _error->RevertToStack(); } - _error->RevertToStack(); } } - if (TransactionManager->MetaIndexParser->Load(DestFile) == false) + if (TransactionManager->MetaIndexParser->Load(DestFile, &ErrorText) == false) { Status = StatAuthError; - ErrorText = TransactionManager->MetaIndexParser->ErrorText; return false; } @@ -945,11 +1051,29 @@ void pkgAcqMetaBase::QueueIndexes(bool const verify) /*{{{*/ // at this point the real Items are loaded in the fetcher ExpectedAdditionalItems = 0; - for (std::vector ::const_iterator Target = IndexTargets.begin(); + bool metaBaseSupportsByHash = false; + if (TransactionManager != NULL && TransactionManager->MetaIndexParser != NULL) + metaBaseSupportsByHash = TransactionManager->MetaIndexParser->GetSupportsAcquireByHash(); + + for (std::vector ::iterator Target = IndexTargets.begin(); Target != IndexTargets.end(); ++Target) { - bool trypdiff = _config->FindB("Acquire::PDiffs", true); + // all is an implementation detail. Users shouldn't use this as arch + // We need this support trickery here as e.g. Debian has binary-all files already, + // but arch:all packages are still in the arch:any files, so we would waste precious + // download time, bandwidth and diskspace for nothing, BUT Debian doesn't feature all + // in the set of supported architectures, so we can filter based on this property rather + // than invent an entirely new flag we would need to carry for all of eternity. + if (Target->Option(IndexTarget::ARCHITECTURE) == "all") + { + if (TransactionManager->MetaIndexParser->IsArchitectureSupported("all") == false) + continue; + if (TransactionManager->MetaIndexParser->IsArchitectureAllSupportedFor(*Target) == false) + continue; + } + + bool trypdiff = Target->OptionBool(IndexTarget::PDIFFS); if (verify == true) { if (TransactionManager->MetaIndexParser->Exists(Target->MetaKey) == false) @@ -958,33 +1082,103 @@ void pkgAcqMetaBase::QueueIndexes(bool const verify) /*{{{*/ if (Target->IsOptional) continue; + std::string const &arch = Target->Option(IndexTarget::ARCHITECTURE); + if (arch.empty() == false) + { + if (TransactionManager->MetaIndexParser->IsArchitectureSupported(arch) == false) + { + _error->Notice(_("Skipping acquire of configured file '%s' as repository '%s' doesn't support architecture '%s'"), + Target->MetaKey.c_str(), TransactionManager->Target.Description.c_str(), arch.c_str()); + continue; + } + // if the architecture is officially supported but currently no packages for it available, + // ignore silently as this is pretty much the same as just shipping an empty file. + // if we don't know which architectures are supported, we do NOT ignore it to notify user about this + if (TransactionManager->MetaIndexParser->IsArchitectureSupported("*undefined*") == false) + continue; + } + Status = StatAuthError; strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), Target->MetaKey.c_str()); return; } + else + { + auto const hashes = GetExpectedHashesFor(Target->MetaKey); + if (hashes.usable() == false && hashes.empty() == false) + { + _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"), + Target->MetaKey.c_str(), TransactionManager->Target.Description.c_str()); + continue; + } + } - if (RealFileExists(GetFinalFileNameFromURI(Target->URI))) + // autoselect the compression method + std::vector types = VectorizeString(Target->Option(IndexTarget::COMPRESSIONTYPES), ' '); + types.erase(std::remove_if(types.begin(), types.end(), [&](std::string const &t) { + if (t == "uncompressed") + return TransactionManager->MetaIndexParser->Exists(Target->MetaKey) == false; + std::string const MetaKey = Target->MetaKey + "." + t; + return TransactionManager->MetaIndexParser->Exists(MetaKey) == false; + }), types.end()); + if (types.empty() == false) { - if (TransactionManager->LastMetaIndexParser != NULL) + std::ostringstream os; + // add the special compressiontype byhash first if supported + std::string const useByHashConf = Target->Option(IndexTarget::BY_HASH); + bool useByHash = false; + if(useByHashConf == "force") + useByHash = true; + else + useByHash = StringToBool(useByHashConf) == true && metaBaseSupportsByHash; + if (useByHash == true) + os << "by-hash "; + std::copy(types.begin(), types.end()-1, std::ostream_iterator(os, " ")); + os << *types.rbegin(); + Target->Options["COMPRESSIONTYPES"] = os.str(); + } + else + Target->Options["COMPRESSIONTYPES"].clear(); + + std::string filename = GetExistingFilename(GetFinalFileNameFromURI(Target->URI)); + if (filename.empty() == false) + { + // if the Release file is a hit and we have an index it must be the current one + if (TransactionManager->IMSHit == true) + ; + else if (TransactionManager->LastMetaIndexParser != NULL) { + // see if the file changed since the last Release file + // we use the uncompressed files as we might compress differently compared to the server, + // so the hashes might not match, even if they contain the same data. HashStringList const newFile = GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, Target->MetaKey); HashStringList const oldFile = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, Target->MetaKey); - if (newFile == oldFile) - { - // we have the file already, no point in trying to acquire it again - new NoActionItem(Owner, *Target); - continue; - } + if (newFile != oldFile) + filename.clear(); } + else + filename.clear(); } else trypdiff = false; // no file to patch + if (filename.empty() == false) + { + new NoActionItem(Owner, *Target, filename); + std::string const idxfilename = GetFinalFileNameFromURI(Target->URI + ".diff/Index"); + if (FileExists(idxfilename)) + new NoActionItem(Owner, *Target, idxfilename); + continue; + } + // check if we have patches available trypdiff &= TransactionManager->MetaIndexParser->Exists(Target->MetaKey + ".diff/Index"); } - // if we have no file to patch, no point in trying - trypdiff &= RealFileExists(GetFinalFileNameFromURI(Target->URI)); + else + { + // if we have no file to patch, no point in trying + trypdiff &= (GetExistingFilename(GetFinalFileNameFromURI(Target->URI)).empty() == false); + } // no point in patching from local sources if (trypdiff) @@ -1039,15 +1233,15 @@ bool pkgAcqMetaBase::VerifyVendor(string const &Message) /*{{{*/ Transformed = ""; } - if (_config->FindB("Acquire::Check-Valid-Until", true) == true && - TransactionManager->MetaIndexParser->GetValidUntil() > 0) { + if (TransactionManager->MetaIndexParser->GetValidUntil() > 0) + { time_t const invalid_since = time(NULL) - TransactionManager->MetaIndexParser->GetValidUntil(); if (invalid_since > 0) { std::string errmsg; strprintf(errmsg, // TRANSLATOR: The first %s is the URL of the bad Release file, the second is - // the time since then the file is invalid - formated in the same way as in + // the time since then the file is invalid - formatted in the same way as in // the download progress display (e.g. 7d 3h 42min 1s) _("Release file for %s is expired (invalid since %s). " "Updates for this repository will not be applied."), @@ -1064,16 +1258,18 @@ bool pkgAcqMetaBase::VerifyVendor(string const &Message) /*{{{*/ TransactionManager->LastMetaIndexParser->GetDate() > TransactionManager->MetaIndexParser->GetDate()) { TransactionManager->IMSHit = true; - unlink(DestFile.c_str()); + RemoveFile("VerifyVendor", DestFile); PartialFile = DestFile = GetFinalFilename(); - delete TransactionManager->MetaIndexParser; - TransactionManager->MetaIndexParser = TransactionManager->LastMetaIndexParser; + // load the 'old' file in the 'new' one instead of flipping pointers as + // the new one isn't owned by us, while the old one is so cleanup would be confused. + TransactionManager->MetaIndexParser->swapLoad(TransactionManager->LastMetaIndexParser); + delete TransactionManager->LastMetaIndexParser; TransactionManager->LastMetaIndexParser = NULL; } if (_config->FindB("Debug::pkgAcquire::Auth", false)) { - std::cerr << "Got Codename: " << TransactionManager->MetaIndexParser->GetDist() << std::endl; + std::cerr << "Got Codename: " << TransactionManager->MetaIndexParser->GetCodename() << std::endl; std::cerr << "Expecting Dist: " << TransactionManager->MetaIndexParser->GetExpectedDist() << std::endl; std::cerr << "Transformed Dist: " << Transformed << std::endl; } @@ -1084,30 +1280,33 @@ bool pkgAcqMetaBase::VerifyVendor(string const &Message) /*{{{*/ // Status = StatAuthError; // ErrorText = "Conflicting distribution; expected " // + MetaIndexParser->GetExpectedDist() + " but got " -// + MetaIndexParser->GetDist(); +// + MetaIndexParser->GetCodename(); // return false; if (!Transformed.empty()) { _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"), Desc.Description.c_str(), Transformed.c_str(), - TransactionManager->MetaIndexParser->GetDist().c_str()); + TransactionManager->MetaIndexParser->GetCodename().c_str()); } } return true; } /*}}}*/ -pkgAcqMetaBase::~pkgAcqMetaBase() {} +pkgAcqMetaBase::~pkgAcqMetaBase() +{ +} pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner, /*{{{*/ IndexTarget const &ClearsignedTarget, IndexTarget const &DetachedDataTarget, IndexTarget const &DetachedSigTarget, - std::vector const IndexTargets, - indexRecords * const MetaIndexParser) : - pkgAcqMetaIndex(Owner, this, ClearsignedTarget, DetachedSigTarget, IndexTargets, MetaIndexParser), - ClearsignedTarget(ClearsignedTarget), - DetachedDataTarget(DetachedDataTarget), DetachedSigTarget(DetachedSigTarget) + std::vector const &IndexTargets, + metaIndex * const MetaIndexParser) : + pkgAcqMetaIndex(Owner, this, ClearsignedTarget, DetachedSigTarget, IndexTargets), + d(NULL), ClearsignedTarget(ClearsignedTarget), + DetachedDataTarget(DetachedDataTarget), + MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL) { // index targets + (worst case:) Release/Release.gpg ExpectedAdditionalItems = IndexTargets.size() + 2; @@ -1116,6 +1315,8 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner, /*{{{*/ /*}}}*/ pkgAcqMetaClearSig::~pkgAcqMetaClearSig() /*{{{*/ { + if (LastMetaIndexParser != NULL) + delete LastMetaIndexParser; } /*}}}*/ // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/ @@ -1123,9 +1324,24 @@ string pkgAcqMetaClearSig::Custom600Headers() const { string Header = pkgAcqMetaBase::Custom600Headers(); Header += "\nFail-Ignore: true"; + std::string const key = TransactionManager->MetaIndexParser->GetSignedBy(); + if (key.empty() == false) + Header += "\nSigned-By: " + key; + return Header; } /*}}}*/ +bool pkgAcqMetaClearSig::VerifyDone(std::string const &Message, /*{{{*/ + pkgAcquire::MethodConfig const * const Cnf) +{ + Item::VerifyDone(Message, Cnf); + + if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile)) + return RenameOnError(NotClearsigned); + + return true; +} + /*}}}*/ // pkgAcqMetaClearSig::Done - We got a file /*{{{*/ void pkgAcqMetaClearSig::Done(std::string const &Message, HashStringList const &Hashes, @@ -1133,17 +1349,6 @@ void pkgAcqMetaClearSig::Done(std::string const &Message, { Item::Done(Message, Hashes, Cnf); - // if we expect a ClearTextSignature (InRelease), ensure that - // this is what we get and if not fail to queue a - // Release/Release.gpg, see #346386 - if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile)) - { - pkgAcquire::Item::Failed(Message, Cnf); - RenameOnError(NotClearsigned); - TransactionManager->AbortTransaction(); - return; - } - if(AuthPass == false) { if(CheckDownloadDone(this, Message, Hashes) == true) @@ -1174,27 +1379,33 @@ void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig c if (AuthPass == false) { + if (Status == StatAuthError || Status == StatTransientNetworkError) + { + // if we expected a ClearTextSignature (InRelease) but got a network + // error or got a file, but it wasn't valid, we end up here (see VerifyDone). + // As these is usually called by web-portals we do not try Release/Release.gpg + // as this is gonna fail anyway and instead abort our try (LP#346386) + TransactionManager->AbortTransaction(); + return; + } + // Queue the 'old' InRelease file for removal if we try Release.gpg // as otherwise the file will stay around and gives a false-auth // impression (CVE-2012-0214) TransactionManager->TransactionStageRemoval(this, GetFinalFilename()); Status = StatDone; - new pkgAcqMetaIndex(Owner, TransactionManager, DetachedDataTarget, DetachedSigTarget, IndexTargets, TransactionManager->MetaIndexParser); + new pkgAcqMetaIndex(Owner, TransactionManager, DetachedDataTarget, DetachedSigTarget, IndexTargets); } else { if(CheckStopAuthentication(this, Message)) return; - _error->Warning(_("The data from '%s' is not signed. Packages " - "from that repository can not be authenticated."), - ClearsignedTarget.Description.c_str()); - // No Release file was present, or verification failed, so fall // back to queueing Packages files without verification - // only allow going further if the users explicitely wants it - if(AllowInsecureRepositories(TransactionManager->MetaIndexParser, TransactionManager, this) == true) + // only allow going further if the user explicitly wants it + if(AllowInsecureRepositories(_("The repository '%s' is not signed."), ClearsignedTarget.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true) { Status = StatDone; @@ -1212,25 +1423,28 @@ void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig c // open the last Release if we have it if (TransactionManager->IMSHit == false) { - TransactionManager->LastMetaIndexParser = new indexRecords; - _error->PushToStack(); - if (RealFileExists(FinalInRelease)) - TransactionManager->LastMetaIndexParser->Load(FinalInRelease); - else - TransactionManager->LastMetaIndexParser->Load(FinalRelease); - // its unlikely to happen, but if what we have is bad ignore it - if (_error->PendingError()) + TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone(); + if (TransactionManager->LastMetaIndexParser != NULL) { - delete TransactionManager->LastMetaIndexParser; - TransactionManager->LastMetaIndexParser = NULL; + _error->PushToStack(); + if (RealFileExists(FinalInRelease)) + TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL); + else + TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL); + // its unlikely to happen, but if what we have is bad ignore it + if (_error->PendingError()) + { + delete TransactionManager->LastMetaIndexParser; + TransactionManager->LastMetaIndexParser = NULL; + } + _error->RevertToStack(); } - _error->RevertToStack(); } } // we parse the indexes here because at this point the user wanted // a repository that may potentially harm him - if (TransactionManager->MetaIndexParser->Load(PartialRelease) == false || VerifyVendor(Message) == false) + if (TransactionManager->MetaIndexParser->Load(PartialRelease, &ErrorText) == false || VerifyVendor(Message) == false) /* expired Release files are still a problem you need extra force for */; else QueueIndexes(true); @@ -1240,12 +1454,11 @@ void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig c /*}}}*/ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire * const Owner, /*{{{*/ - pkgAcqMetaBase * const TransactionManager, + pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &DataTarget, IndexTarget const &DetachedSigTarget, - vector const IndexTargets, - indexRecords * const MetaIndexParser) : - pkgAcqMetaBase(Owner, TransactionManager, IndexTargets, DataTarget, MetaIndexParser), + vector const &IndexTargets) : + pkgAcqMetaBase(Owner, TransactionManager, IndexTargets, DataTarget), d(NULL), DetachedSigTarget(DetachedSigTarget) { if(_config->FindB("Debug::Acquire::Transaction", false) == true) @@ -1287,19 +1500,13 @@ void pkgAcqMetaIndex::Failed(string const &Message, pkgAcquire::Item::Failed(Message, Cnf); Status = StatDone; - _error->Warning(_("The repository '%s' does not have a Release file. " - "This is deprecated, please contact the owner of the " - "repository."), Target.Description.c_str()); - // No Release file was present so fall // back to queueing Packages files without verification - // only allow going further if the users explicitely wants it - if(AllowInsecureRepositories(TransactionManager->MetaIndexParser, TransactionManager, this) == true) + // only allow going further if the user explicitly wants it + if(AllowInsecureRepositories(_("The repository '%s' does not have a Release file."), Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true) { // ensure old Release files are removed TransactionManager->TransactionStageRemoval(this, GetFinalFilename()); - delete TransactionManager->MetaIndexParser; - TransactionManager->MetaIndexParser = NULL; // queue without any kind of hashsum support QueueIndexes(false); @@ -1324,17 +1531,17 @@ pkgAcqMetaIndex::~pkgAcqMetaIndex() {} // AcqMetaSig::AcqMetaSig - Constructor /*{{{*/ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire * const Owner, - pkgAcqMetaBase * const TransactionManager, - IndexTarget const Target, + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target, pkgAcqMetaIndex * const MetaIndex) : - pkgAcqTransactionItem(Owner, TransactionManager, Target), MetaIndex(MetaIndex) + pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL), MetaIndex(MetaIndex) { DestFile = GetPartialFileNameFromURI(Target.URI); // remove any partial downloaded sig-file in partial/. // it may confuse proxies and is too small to warrant a // partial download anyway - unlink(DestFile.c_str()); + RemoveFile("pkgAcqMetaSig", DestFile); // set the TransactionManager if(_config->FindB("Debug::Acquire::Transaction", false) == true) @@ -1363,6 +1570,16 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire * const Owner, /*}}}*/ pkgAcqMetaSig::~pkgAcqMetaSig() /*{{{*/ { +} + /*}}}*/ +// pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/ +std::string pkgAcqMetaSig::Custom600Headers() const +{ + std::string Header = pkgAcqTransactionItem::Custom600Headers(); + std::string const key = TransactionManager->MetaIndexParser->GetSignedBy(); + if (key.empty() == false) + Header += "\nSigned-By: " + key; + return Header; } /*}}}*/ // AcqMetaSig::Done - The signature was downloaded/verified /*{{{*/ @@ -1424,7 +1641,7 @@ void pkgAcqMetaSig::Failed(string const &Message,pkgAcquire::MethodConfig const "given to override it.")); Status = StatDone; } else { - _error->Error("%s", downgrade_msg.c_str()); + MessageInsecureRepository(true, downgrade_msg); if (TransactionManager->IMSHit == false) Rename(MetaIndex->DestFile, MetaIndex->DestFile + ".FAILED"); Item::Failed("Message: " + downgrade_msg, Cnf); @@ -1432,44 +1649,44 @@ void pkgAcqMetaSig::Failed(string const &Message,pkgAcquire::MethodConfig const return; } } - else - _error->Warning(_("The data from '%s' is not signed. Packages " - "from that repository can not be authenticated."), - MetaIndex->Target.Description.c_str()); // ensures that a Release.gpg file in the lists/ is removed by the transaction TransactionManager->TransactionStageRemoval(this, DestFile); - // only allow going further if the users explicitely wants it - if(AllowInsecureRepositories(TransactionManager->MetaIndexParser, TransactionManager, this) == true) + // only allow going further if the user explicitly wants it + if (AllowInsecureRepositories(_("The repository '%s' is not signed."), MetaIndex->Target.Description, TransactionManager->MetaIndexParser, TransactionManager, this) == true) { if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease)) { // open the last Release if we have it if (TransactionManager->IMSHit == false) { - TransactionManager->LastMetaIndexParser = new indexRecords; - _error->PushToStack(); - if (RealFileExists(FinalInRelease)) - TransactionManager->LastMetaIndexParser->Load(FinalInRelease); - else - TransactionManager->LastMetaIndexParser->Load(FinalRelease); - // its unlikely to happen, but if what we have is bad ignore it - if (_error->PendingError()) + TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone(); + if (TransactionManager->LastMetaIndexParser != NULL) { - delete TransactionManager->LastMetaIndexParser; - TransactionManager->LastMetaIndexParser = NULL; + _error->PushToStack(); + if (RealFileExists(FinalInRelease)) + TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL); + else + TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL); + // its unlikely to happen, but if what we have is bad ignore it + if (_error->PendingError()) + { + delete TransactionManager->LastMetaIndexParser; + TransactionManager->LastMetaIndexParser = NULL; + } + _error->RevertToStack(); } - _error->RevertToStack(); } } // we parse the indexes here because at this point the user wanted // a repository that may potentially harm him - if (TransactionManager->MetaIndexParser->Load(MetaIndex->DestFile) == false || MetaIndex->VerifyVendor(Message) == false) + bool const GoodLoad = TransactionManager->MetaIndexParser->Load(MetaIndex->DestFile, &ErrorText); + if (MetaIndex->VerifyVendor(Message) == false) /* expired Release files are still a problem you need extra force for */; else - MetaIndex->QueueIndexes(true); + MetaIndex->QueueIndexes(GoodLoad); TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, MetaIndex->GetFinalFilename()); } @@ -1487,9 +1704,9 @@ void pkgAcqMetaSig::Failed(string const &Message,pkgAcquire::MethodConfig const // AcqBaseIndex - Constructor /*{{{*/ pkgAcqBaseIndex::pkgAcqBaseIndex(pkgAcquire * const Owner, - pkgAcqMetaBase * const TransactionManager, - IndexTarget const Target) -: pkgAcqTransactionItem(Owner, TransactionManager, Target) + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target) +: pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL) { } /*}}}*/ @@ -1503,9 +1720,9 @@ pkgAcqBaseIndex::~pkgAcqBaseIndex() {} * the original packages file */ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner, - pkgAcqMetaBase * const TransactionManager, - IndexTarget const Target) - : pkgAcqBaseIndex(Owner, TransactionManager, Target) + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target) + : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), diffs(NULL) { Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); @@ -1527,6 +1744,9 @@ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner, /* The only header we use is the last-modified header. */ string pkgAcqDiffIndex::Custom600Headers() const { + if (TransactionManager->LastMetaIndexParser != NULL) + return "\nIndex-File: true"; + string const Final = GetFinalFilename(); if(Debug) @@ -1556,7 +1776,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ FileFd Fd(IndexDiffFile,FileFd::ReadOnly); pkgTagFile TF(&Fd); - if (_error->PendingError() == true) + if (Fd.IsOpen() == false || Fd.Failed()) return false; pkgTagSection Tags; @@ -1611,7 +1831,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ LocalHashes = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, Target.MetaKey); if (LocalHashes.usable() == false) { - FileFd fd(CurrentPackagesFile, FileFd::ReadOnly); + FileFd fd(CurrentPackagesFile, FileFd::ReadOnly, FileFd::Auto); Hashes LocalHashesCalc(ServerHashes); LocalHashesCalc.AddFD(fd); LocalHashes = LocalHashesCalc.GetHashStringList(); @@ -1630,10 +1850,18 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ std::clog << "Server-Current: " << ServerHashes.find(NULL)->toStr() << " and we start at " << CurrentPackagesFile << " " << LocalHashes.FileSize() << " " << LocalHashes.find(NULL)->toStr() << std::endl; + // historically, older hashes have more info than newer ones, so start + // collecting with older ones first to avoid implementing complicated + // information merging techniques… a failure is after all always + // recoverable with a complete file and hashes aren't changed that often. + std::vector types; + for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) + types.push_back(*type); + // parse all of (provided) history vector available_patches; bool firstAcceptedHashes = true; - for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) + for (auto type = types.crbegin(); type != types.crend(); ++type) { if (LocalHashes.find(*type) == NULL) continue; @@ -1691,7 +1919,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ return false; } - for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) + for (auto type = types.crbegin(); type != types.crend(); ++type) { if (LocalHashes.find(*type) == NULL) continue; @@ -1731,7 +1959,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ } } - for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) + for (auto type = types.crbegin(); type != types.crend(); ++type) { std::string tagname = *type; tagname.append("-Download"); @@ -1804,18 +2032,55 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ } // calculate the size of all patches we have to get - // note that all sizes are uncompressed, while we download compressed files - unsigned long long patchesSize = 0; - for (std::vector::const_iterator cur = available_patches.begin(); - cur != available_patches.end(); ++cur) - patchesSize += cur->patch_hashes.FileSize(); - unsigned long long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100); - if (sizeLimit > 0 && (sizeLimit/100) < patchesSize) - { - if (Debug) - std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100 - << ") so fallback to complete download" << std::endl; - return false; + unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100); + if (sizeLimitPercent > 0 && TransactionManager->MetaIndexParser != nullptr) + { + // compressed case + unsigned long long downloadSize = std::accumulate(available_patches.begin(), + available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) { + return T + I.download_hashes.FileSize(); + }); + if (downloadSize != 0) + { + unsigned long long downloadSizeIdx = 0; + auto const types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' '); + for (auto const &t : types) + { + std::string MetaKey = Target.MetaKey; + if (t != "uncompressed") + MetaKey += '.' + t; + HashStringList const hsl = GetExpectedHashesFor(MetaKey); + if (unlikely(hsl.usable() == false)) + continue; + downloadSizeIdx = hsl.FileSize(); + break; + } + unsigned long long const sizeLimit = downloadSizeIdx * sizeLimitPercent; + if ((sizeLimit/100) < downloadSize) + { + if (Debug) + std::clog << "Need " << downloadSize << " compressed bytes (Limit is " << (sizeLimit/100) << ", " + << "original is " << downloadSizeIdx << ") so fallback to complete download" << std::endl; + return false; + } + } + // uncompressed case + downloadSize = std::accumulate(available_patches.begin(), + available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) { + return T + I.patch_hashes.FileSize(); + }); + if (downloadSize != 0) + { + unsigned long long const downloadSizeIdx = ServerSize; + unsigned long long const sizeLimit = downloadSizeIdx * sizeLimitPercent; + if ((sizeLimit/100) < downloadSize) + { + if (Debug) + std::clog << "Need " << downloadSize << " uncompressed bytes (Limit is " << (sizeLimit/100) << ", " + << "original is " << downloadSizeIdx << ") so fallback to complete download" << std::endl; + return false; + } + } } // we have something, queue the diffs @@ -1836,11 +2101,46 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ pdiff_merge = (precedence != "merged"); } + // clean the plate + { + std::string const Final = GetExistingFilename(CurrentPackagesFile); + if (unlikely(Final.empty())) // because we wouldn't be called in such a case + return false; + std::string const PartialFile = GetPartialFileNameFromURI(Target.URI); + if (FileExists(PartialFile) && RemoveFile("Bootstrap-linking", PartialFile) == false) + { + if (Debug) + std::clog << "Bootstrap-linking for patching " << CurrentPackagesFile + << " by removing stale " << PartialFile << " failed!" << std::endl; + return false; + } + for (auto const &ext : APT::Configuration::getCompressorExtensions()) + { + std::string const Partial = PartialFile + ext; + if (FileExists(Partial) && RemoveFile("Bootstrap-linking", Partial) == false) + { + if (Debug) + std::clog << "Bootstrap-linking for patching " << CurrentPackagesFile + << " by removing stale " << Partial << " failed!" << std::endl; + return false; + } + } + std::string const Ext = Final.substr(CurrentPackagesFile.length()); + std::string const Partial = PartialFile + Ext; + if (symlink(Final.c_str(), Partial.c_str()) != 0) + { + if (Debug) + std::clog << "Bootstrap-linking for patching " << CurrentPackagesFile + << " by linking " << Final << " to " << Partial << " failed!" << std::endl; + return false; + } + } + if (pdiff_merge == false) new pkgAcqIndexDiffs(Owner, TransactionManager, Target, available_patches); else { - std::vector *diffs = new std::vector(available_patches.size()); + diffs = new std::vector(available_patches.size()); for(size_t i = 0; i < available_patches.size(); ++i) (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, TransactionManager, Target, @@ -1896,7 +2196,11 @@ void pkgAcqDiffIndex::Done(string const &Message,HashStringList const &Hashes, / return; } /*}}}*/ -pkgAcqDiffIndex::~pkgAcqDiffIndex() {} +pkgAcqDiffIndex::~pkgAcqDiffIndex() +{ + if (diffs != NULL) + delete diffs; +} // AcqIndexDiffs::AcqIndexDiffs - Constructor /*{{{*/ // --------------------------------------------------------------------- @@ -1904,13 +2208,13 @@ pkgAcqDiffIndex::~pkgAcqDiffIndex() {} * for each diff and the index */ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire * const Owner, - pkgAcqMetaBase * const TransactionManager, - IndexTarget const Target, + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target, vector const &diffs) - : pkgAcqBaseIndex(Owner, TransactionManager, Target), + : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), available_patches(diffs) { - DestFile = GetPartialFileNameFromURI(Target.URI); + DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target); Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); @@ -1921,23 +2225,11 @@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire * const Owner, if(available_patches.empty() == true) { // we are done (yeah!), check hashes against the final file - DestFile = GetFinalFileNameFromURI(Target.URI); + DestFile = GetKeepCompressedFileName(GetFinalFileNameFromURI(Target.URI), Target); Finish(true); } else { - // patching needs to be bootstrapped with the 'old' version - std::string const PartialFile = GetPartialFileNameFromURI(Target.URI); - if (RealFileExists(PartialFile) == false) - { - if (symlink(GetFinalFilename().c_str(), PartialFile.c_str()) != 0) - { - Failed("Link creation of " + PartialFile + " to " + GetFinalFilename() + " failed", NULL); - return; - } - } - - // get the next diff State = StateFetchDiff; QueueNextDiff(); } @@ -1948,14 +2240,17 @@ void pkgAcqIndexDiffs::Failed(string const &Message,pkgAcquire::MethodConfig con Item::Failed(Message,Cnf); Status = StatDone; + DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target); if(Debug) std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl - << "Falling back to normal index file acquire" << std::endl; - DestFile = GetPartialFileNameFromURI(Target.URI); + << "Falling back to normal index file acquire " << std::endl; RenameOnError(PDiffError); std::string const patchname = GetDiffsPatchFileName(DestFile); if (RealFileExists(patchname)) - rename(patchname.c_str(), std::string(patchname + ".FAILED").c_str()); + Rename(patchname, patchname + ".FAILED"); + std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI)); + if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile)) + Rename(UnpatchedFile, UnpatchedFile + ".FAILED"); new pkgAcqIndex(Owner, TransactionManager, Target); Finish(); } @@ -1972,7 +2267,8 @@ void pkgAcqIndexDiffs::Finish(bool allDone) // the file will be cleaned if(allDone) { - TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename()); + std::string const Final = GetKeepCompressedFileName(GetFinalFilename(), Target); + TransactionManager->TransactionStageCopy(this, DestFile, Final); // this is for the "real" finish Complete = true; @@ -1982,6 +2278,8 @@ void pkgAcqIndexDiffs::Finish(bool allDone) std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl; return; } + else + DestFile.clear(); if(Debug) std::clog << "Finishing: " << Desc.URI << std::endl; @@ -1994,30 +2292,28 @@ void pkgAcqIndexDiffs::Finish(bool allDone) bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ { // calc sha1 of the just patched file - std::string const FinalFile = GetPartialFileNameFromURI(Target.URI); - - if(!FileExists(FinalFile)) + std::string const PartialFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI)); + if(unlikely(PartialFile.empty())) { - Failed("Message: No FinalFile " + FinalFile + " available", NULL); + Failed("Message: The file " + GetPartialFileNameFromURI(Target.URI) + " isn't available", NULL); return false; } - FileFd fd(FinalFile, FileFd::ReadOnly); + FileFd fd(PartialFile, FileFd::ReadOnly, FileFd::Extension); Hashes LocalHashesCalc; LocalHashesCalc.AddFD(fd); HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList(); if(Debug) - std::clog << "QueueNextDiff: " << FinalFile << " (" << LocalHashes.find(NULL)->toStr() << ")" << std::endl; + std::clog << "QueueNextDiff: " << PartialFile << " (" << LocalHashes.find(NULL)->toStr() << ")" << std::endl; HashStringList const TargetFileHashes = GetExpectedHashesFor(Target.MetaKey); if (unlikely(LocalHashes.usable() == false || TargetFileHashes.usable() == false)) { - Failed("Local/Expected hashes are not usable", NULL); + Failed("Local/Expected hashes are not usable for " + PartialFile, NULL); return false; } - // final file reached before all patches are applied if(LocalHashes == TargetFileHashes) { @@ -2027,26 +2323,22 @@ bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ // remove all patches until the next matching patch is found // this requires the Index file to be ordered - for(vector::iterator I = available_patches.begin(); - available_patches.empty() == false && - I != available_patches.end() && - I->result_hashes != LocalHashes; - ++I) - { - available_patches.erase(I); - } + available_patches.erase(available_patches.begin(), + std::find_if(available_patches.begin(), available_patches.end(), [&](DiffInfo const &I) { + return I.result_hashes == LocalHashes; + })); // error checking and falling back if no patch was found if(available_patches.empty() == true) { - Failed("No patches left to reach target", NULL); + Failed("No patches left to reach target for " + PartialFile, NULL); return false; } // queue the right diff Desc.URI = Target.URI + ".diff/" + available_patches[0].file + ".gz"; Desc.Description = Description + " " + available_patches[0].file + string(".pdiff"); - DestFile = GetPartialFileNameFromURI(Target.URI + ".diff/" + available_patches[0].file); + DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI + ".diff/" + available_patches[0].file), Target); if(Debug) std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl; @@ -2059,55 +2351,51 @@ bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ void pkgAcqIndexDiffs::Done(string const &Message, HashStringList const &Hashes, /*{{{*/ pkgAcquire::MethodConfig const * const Cnf) { - if(Debug) + if (Debug) std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl; Item::Done(Message, Hashes, Cnf); - std::string const FinalFile = GetPartialFileNameFromURI(Target.URI); - std::string const PatchFile = GetDiffsPatchFileName(FinalFile); - - // success in downloading a diff, enter ApplyDiff state - if(State == StateFetchDiff) - { - Rename(DestFile, PatchFile); - - if(Debug) - std::clog << "Sending to rred method: " << FinalFile << std::endl; - - State = StateApplyDiff; - Local = true; - Desc.URI = "rred:" + FinalFile; - QueueURI(Desc); - SetActiveSubprocess("rred"); - return; - } + std::string const UncompressedUnpatchedFile = GetPartialFileNameFromURI(Target.URI); + std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile); + std::string const PatchFile = GetDiffsPatchFileName(UnpatchedFile); + std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target); - // success in download/apply a diff, queue next (if needed) - if(State == StateApplyDiff) + switch (State) { - // remove the just applied patch - available_patches.erase(available_patches.begin()); - unlink(PatchFile.c_str()); - - // move into place - if(Debug) - { - std::clog << "Moving patched file in place: " << std::endl - << DestFile << " -> " << FinalFile << std::endl; - } - Rename(DestFile,FinalFile); - chmod(FinalFile.c_str(),0644); - - // see if there is more to download - if(available_patches.empty() == false) { - new pkgAcqIndexDiffs(Owner, TransactionManager, Target, - available_patches); - return Finish(); - } else - // update - DestFile = FinalFile; - return Finish(true); + // success in downloading a diff, enter ApplyDiff state + case StateFetchDiff: + Rename(DestFile, PatchFile); + DestFile = GetKeepCompressedFileName(UncompressedUnpatchedFile + "-patched", Target); + if(Debug) + std::clog << "Sending to rred method: " << UnpatchedFile << std::endl; + State = StateApplyDiff; + Local = true; + Desc.URI = "rred:" + UnpatchedFile; + QueueURI(Desc); + SetActiveSubprocess("rred"); + return; + // success in download/apply a diff, queue next (if needed) + case StateApplyDiff: + // remove the just applied patch and base file + available_patches.erase(available_patches.begin()); + RemoveFile("pkgAcqIndexDiffs::Done", PatchFile); + RemoveFile("pkgAcqIndexDiffs::Done", UnpatchedFile); + if(Debug) + std::clog << "Moving patched file in place: " << std::endl + << DestFile << " -> " << PatchedFile << std::endl; + Rename(DestFile, PatchedFile); + + // see if there is more to download + if(available_patches.empty() == false) + { + new pkgAcqIndexDiffs(Owner, TransactionManager, Target, available_patches); + Finish(); + } else { + DestFile = PatchedFile; + Finish(true); + } + return; } } /*}}}*/ @@ -2127,11 +2415,11 @@ pkgAcqIndexDiffs::~pkgAcqIndexDiffs() {} // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire * const Owner, - pkgAcqMetaBase * const TransactionManager, - IndexTarget const Target, + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target, DiffInfo const &patch, std::vector const * const allPatches) - : pkgAcqBaseIndex(Owner, TransactionManager, Target), + : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), patch(patch), allPatches(allPatches), State(StateFetchDiff) { Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); @@ -2139,11 +2427,9 @@ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire * const Owner, Desc.Owner = this; Description = Target.Description; Desc.ShortDesc = Target.ShortDesc; - Desc.URI = Target.URI + ".diff/" + patch.file + ".gz"; - Desc.Description = Description + " " + patch.file + string(".pdiff"); - - DestFile = GetPartialFileNameFromURI(Target.URI + ".diff/" + patch.file); + Desc.Description = Description + " " + patch.file + ".pdiff"; + DestFile = GetPartialFileNameFromURI(Desc.URI); if(Debug) std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl; @@ -2170,11 +2456,14 @@ void pkgAcqIndexMergeDiffs::Failed(string const &Message,pkgAcquire::MethodConfi State = StateErrorDiff; if (Debug) std::clog << "Falling back to normal index file acquire" << std::endl; - DestFile = GetPartialFileNameFromURI(Target.URI); RenameOnError(PDiffError); - std::string const patchname = GetMergeDiffsPatchFileName(DestFile, patch.file); + std::string const patchname = GetPartialFileNameFromURI(Desc.URI); if (RealFileExists(patchname)) - rename(patchname.c_str(), std::string(patchname + ".FAILED").c_str()); + Rename(patchname, patchname + ".FAILED"); + std::string const UnpatchedFile = GetExistingFilename(GetPartialFileNameFromURI(Target.URI)); + if (UnpatchedFile.empty() == false && FileExists(UnpatchedFile)) + Rename(UnpatchedFile, UnpatchedFile + ".FAILED"); + DestFile.clear(); new pkgAcqIndex(Owner, TransactionManager, Target); } /*}}}*/ @@ -2186,67 +2475,58 @@ void pkgAcqIndexMergeDiffs::Done(string const &Message, HashStringList const &Ha Item::Done(Message, Hashes, Cnf); - string const FinalFile = GetPartialFileNameFromURI(Target.URI); - if (State == StateFetchDiff) - { - Rename(DestFile, GetMergeDiffsPatchFileName(FinalFile, patch.file)); - - // check if this is the last completed diff - State = StateDoneDiff; - for (std::vector::const_iterator I = allPatches->begin(); - I != allPatches->end(); ++I) - if ((*I)->State != StateDoneDiff) - { - if(Debug) - std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl; - return; - } - - // this is the last completed diff, so we are ready to apply now - State = StateApplyDiff; - - // patching needs to be bootstrapped with the 'old' version - if (symlink(GetFinalFilename().c_str(), FinalFile.c_str()) != 0) - { - Failed("Link creation of " + FinalFile + " to " + GetFinalFilename() + " failed", NULL); - return; - } - - if(Debug) - std::clog << "Sending to rred method: " << FinalFile << std::endl; + std::string const UncompressedUnpatchedFile = GetPartialFileNameFromURI(Target.URI); + std::string const UnpatchedFile = GetExistingFilename(UncompressedUnpatchedFile); + std::string const PatchFile = GetMergeDiffsPatchFileName(UnpatchedFile, patch.file); + std::string const PatchedFile = GetKeepCompressedFileName(UncompressedUnpatchedFile, Target); - Local = true; - Desc.URI = "rred:" + FinalFile; - QueueURI(Desc); - SetActiveSubprocess("rred"); - return; - } - // success in download/apply all diffs, clean up - else if (State == StateApplyDiff) + switch (State) { - // move the result into place - std::string const Final = GetFinalFilename(); - if(Debug) - std::clog << "Queue patched file in place: " << std::endl - << DestFile << " -> " << Final << std::endl; - - // queue for copy by the transaction manager - TransactionManager->TransactionStageCopy(this, DestFile, Final); - - // ensure the ed's are gone regardless of list-cleanup - for (std::vector::const_iterator I = allPatches->begin(); - I != allPatches->end(); ++I) - { - std::string const PartialFile = GetPartialFileNameFromURI(Target.URI); - std::string const patch = GetMergeDiffsPatchFileName(PartialFile, (*I)->patch.file); - unlink(patch.c_str()); - } - unlink(FinalFile.c_str()); + case StateFetchDiff: + Rename(DestFile, PatchFile); - // all set and done - Complete = true; - if(Debug) - std::clog << "allDone: " << DestFile << "\n" << std::endl; + // check if this is the last completed diff + State = StateDoneDiff; + for (std::vector::const_iterator I = allPatches->begin(); + I != allPatches->end(); ++I) + if ((*I)->State != StateDoneDiff) + { + if(Debug) + std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl; + return; + } + // this is the last completed diff, so we are ready to apply now + DestFile = GetKeepCompressedFileName(UncompressedUnpatchedFile + "-patched", Target); + if(Debug) + std::clog << "Sending to rred method: " << UnpatchedFile << std::endl; + State = StateApplyDiff; + Local = true; + Desc.URI = "rred:" + UnpatchedFile; + QueueURI(Desc); + SetActiveSubprocess("rred"); + return; + case StateApplyDiff: + // success in download & apply all diffs, finialize and clean up + if(Debug) + std::clog << "Queue patched file in place: " << std::endl + << DestFile << " -> " << PatchedFile << std::endl; + + // queue for copy by the transaction manager + TransactionManager->TransactionStageCopy(this, DestFile, GetKeepCompressedFileName(GetFinalFilename(), Target)); + + // ensure the ed's are gone regardless of list-cleanup + for (std::vector::const_iterator I = allPatches->begin(); + I != allPatches->end(); ++I) + RemoveFile("pkgAcqIndexMergeDiffs::Done", GetMergeDiffsPatchFileName(UnpatchedFile, (*I)->patch.file)); + RemoveFile("pkgAcqIndexMergeDiffs::Done", UnpatchedFile); + + // all set and done + Complete = true; + if(Debug) + std::clog << "allDone: " << DestFile << "\n" << std::endl; + return; + case StateDoneDiff: _error->Fatal("Done called for %s which is in an invalid Done state", PatchFile.c_str()); break; + case StateErrorDiff: _error->Fatal("Done called for %s which is in an invalid Error state", PatchFile.c_str()); break; } } /*}}}*/ @@ -2272,12 +2552,11 @@ pkgAcqIndexMergeDiffs::~pkgAcqIndexMergeDiffs() {} // AcqIndex::AcqIndex - Constructor /*{{{*/ pkgAcqIndex::pkgAcqIndex(pkgAcquire * const Owner, - pkgAcqMetaBase * const TransactionManager, - IndexTarget const Target) - : pkgAcqBaseIndex(Owner, TransactionManager, Target) + pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target) + : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), Stage(STAGE_DOWNLOAD), + CompressionExtensions(Target.Option(IndexTarget::COMPRESSIONTYPES)) { - // autoselect the compression method - AutoSelectCompression(); Init(Target.URI, Target.Description, Target.ShortDesc); if(_config->FindB("Debug::Acquire::Transaction", false) == true) @@ -2285,54 +2564,58 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire * const Owner, << TransactionManager << std::endl; } /*}}}*/ -// AcqIndex::AutoSelectCompression - Select compression /*{{{*/ -void pkgAcqIndex::AutoSelectCompression() +// AcqIndex::Init - defered Constructor /*{{{*/ +static void NextCompressionExtension(std::string &CurrentCompressionExtension, std::string &CompressionExtensions, bool const preview) { - std::vector types = APT::Configuration::getCompressionTypes(); - CompressionExtensions = ""; - if (TransactionManager->MetaIndexParser != NULL && TransactionManager->MetaIndexParser->Exists(Target.MetaKey)) + size_t const nextExt = CompressionExtensions.find(' '); + if (nextExt == std::string::npos) { - for (std::vector::const_iterator t = types.begin(); - t != types.end(); ++t) - { - std::string CompressedMetaKey = string(Target.MetaKey).append(".").append(*t); - if (*t == "uncompressed" || - TransactionManager->MetaIndexParser->Exists(CompressedMetaKey) == true) - CompressionExtensions.append(*t).append(" "); - } + CurrentCompressionExtension = CompressionExtensions; + if (preview == false) + CompressionExtensions.clear(); } else { - for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) - CompressionExtensions.append(*t).append(" "); + CurrentCompressionExtension = CompressionExtensions.substr(0, nextExt); + if (preview == false) + CompressionExtensions = CompressionExtensions.substr(nextExt+1); } - if (CompressionExtensions.empty() == false) - CompressionExtensions.erase(CompressionExtensions.end()-1); } - /*}}}*/ -// AcqIndex::Init - defered Constructor /*{{{*/ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) { Stage = STAGE_DOWNLOAD; DestFile = GetPartialFileNameFromURI(URI); + NextCompressionExtension(CurrentCompressionExtension, CompressionExtensions, false); - size_t const nextExt = CompressionExtensions.find(' '); - if (nextExt == std::string::npos) + if (CurrentCompressionExtension == "uncompressed") { - CurrentCompressionExtension = CompressionExtensions; - CompressionExtensions.clear(); + Desc.URI = URI; } - else + else if (CurrentCompressionExtension == "by-hash") { - CurrentCompressionExtension = CompressionExtensions.substr(0, nextExt); - CompressionExtensions = CompressionExtensions.substr(nextExt+1); - } + NextCompressionExtension(CurrentCompressionExtension, CompressionExtensions, true); + if(unlikely(TransactionManager->MetaIndexParser == NULL || CurrentCompressionExtension.empty())) + return; + if (CurrentCompressionExtension != "uncompressed") + { + Desc.URI = URI + '.' + CurrentCompressionExtension; + DestFile = DestFile + '.' + CurrentCompressionExtension; + } - if (CurrentCompressionExtension == "uncompressed") - { - Desc.URI = URI; + HashStringList const Hashes = GetExpectedHashes(); + HashString const * const TargetHash = Hashes.find(NULL); + if (unlikely(TargetHash == nullptr)) + return; + std::string const ByHash = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue(); + size_t const trailing_slash = Desc.URI.find_last_of("/"); + if (unlikely(trailing_slash == std::string::npos)) + return; + Desc.URI = Desc.URI.replace( + trailing_slash, + Desc.URI.substr(trailing_slash+1).size()+1, + ByHash); } else if (unlikely(CurrentCompressionExtension.empty())) return; @@ -2342,8 +2625,6 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, DestFile = DestFile + '.' + CurrentCompressionExtension; } - if(TransactionManager->MetaIndexParser != NULL) - InitByHashIfNeeded(); Desc.Description = URIDesc; Desc.Owner = this; @@ -2352,47 +2633,22 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, QueueURI(Desc); } /*}}}*/ -// AcqIndex::AdjustForByHash - modify URI for by-hash support /*{{{*/ -void pkgAcqIndex::InitByHashIfNeeded() -{ - // TODO: - // - (maybe?) add support for by-hash into the sources.list as flag - // - make apt-ftparchive generate the hashes (and expire?) - std::string HostKnob = "APT::Acquire::" + ::URI(Desc.URI).Host + "::By-Hash"; - if(_config->FindB("APT::Acquire::By-Hash", false) == true || - _config->FindB(HostKnob, false) == true || - TransactionManager->MetaIndexParser->GetSupportsAcquireByHash()) - { - HashStringList const Hashes = GetExpectedHashes(); - if(Hashes.usable()) - { - // FIXME: should we really use the best hash here? or a fixed one? - HashString const * const TargetHash = Hashes.find(""); - std::string const ByHash = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue(); - size_t const trailing_slash = Desc.URI.find_last_of("/"); - Desc.URI = Desc.URI.replace( - trailing_slash, - Desc.URI.substr(trailing_slash+1).size()+1, - ByHash); - } else { - _error->Warning( - "Fetching ByHash requested but can not find record for %s", - GetMetaKey().c_str()); - } - } -} - /*}}}*/ // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ string pkgAcqIndex::Custom600Headers() const { - string Final = GetFinalFilename(); string msg = "\nIndex-File: true"; - struct stat Buf; - if (stat(Final.c_str(),&Buf) == 0) - msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); + + if (TransactionManager->LastMetaIndexParser == NULL) + { + std::string const Final = GetFinalFilename(); + + struct stat Buf; + if (stat(Final.c_str(),&Buf) == 0) + msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); + } if(Target.IsOptional) msg += "\nFail-Ignore: true"; @@ -2422,20 +2678,6 @@ void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * TransactionManager->AbortTransaction(); } /*}}}*/ -// AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit /*{{{*/ -void pkgAcqIndex::ReverifyAfterIMS() -{ - // update destfile to *not* include the compression extension when doing - // a reverify (as its uncompressed on disk already) - DestFile = GetCompressedFileName(Target.URI, GetPartialFileNameFromURI(Target.URI), CurrentCompressionExtension); - - // copy FinalFile into partial/ so that we check the hash again - string FinalFile = GetFinalFilename(); - Stage = STAGE_DECOMPRESS_AND_VERIFY; - Desc.URI = "copy:" + FinalFile; - QueueURI(Desc); -} - /*}}}*/ // AcqIndex::Done - Finished a fetch /*{{{*/ // --------------------------------------------------------------------- /* This goes through a number of states.. On the initial fetch the @@ -2452,97 +2694,93 @@ void pkgAcqIndex::Done(string const &Message, switch(Stage) { case STAGE_DOWNLOAD: - StageDownloadDone(Message, Hashes, Cfg); + StageDownloadDone(Message); break; case STAGE_DECOMPRESS_AND_VERIFY: - StageDecompressDone(Message, Hashes, Cfg); + StageDecompressDone(); break; } } /*}}}*/ // AcqIndex::StageDownloadDone - Queue for decompress and verify /*{{{*/ -void pkgAcqIndex::StageDownloadDone(string const &Message, HashStringList const &, - pkgAcquire::MethodConfig const * const) +void pkgAcqIndex::StageDownloadDone(string const &Message) { + Local = true; Complete = true; - // Handle the unzipd case - string FileName = LookupTag(Message,"Alt-Filename"); - if (FileName.empty() == false) + std::string const AltFilename = LookupTag(Message,"Alt-Filename"); + std::string Filename = LookupTag(Message,"Filename"); + + // we need to verify the file against the current Release file again + // on if-modfied-since hit to avoid a stale attack against us + if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) { + // copy FinalFile into partial/ so that we check the hash again + string const FinalFile = GetExistingFilename(GetFinalFileNameFromURI(Target.URI)); + if (symlink(FinalFile.c_str(), DestFile.c_str()) != 0) + _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking final file %s back to %s failed", FinalFile.c_str(), DestFile.c_str()); + else + { + EraseFileName = DestFile; + Filename = DestFile; + } Stage = STAGE_DECOMPRESS_AND_VERIFY; - Local = true; - DestFile += ".decomp"; - Desc.URI = "copy:" + FileName; + Desc.URI = "store:" + Filename; QueueURI(Desc); - SetActiveSubprocess("copy"); + SetActiveSubprocess(::URI(Desc.URI).Access); return; } - - FileName = LookupTag(Message,"Filename"); - if (FileName.empty() == true) + // methods like file:// give us an alternative (uncompressed) file + else if (Target.KeepCompressed == false && AltFilename.empty() == false) { - Status = StatError; - ErrorText = "Method gave a blank filename"; + if (CurrentCompressionExtension != "uncompressed") + DestFile.erase(DestFile.length() - (CurrentCompressionExtension.length() + 1)); + Filename = AltFilename; } - // Methods like e.g. "file:" will give us a (compressed) FileName that is // not the "DestFile" we set, in this case we uncompress from the local file - if (FileName != DestFile && RealFileExists(DestFile) == false) - Local = true; - else - EraseFileName = FileName; - - // we need to verify the file against the current Release file again - // on if-modfied-since hit to avoid a stale attack against us - if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + else if (Filename != DestFile && RealFileExists(DestFile) == false) { - // The files timestamp matches, reverify by copy into partial/ - EraseFileName = ""; - ReverifyAfterIMS(); - return; + // symlinking ensures that the filename can be used for compression detection + // that is e.g. needed for by-hash which has no extension over file + if (symlink(Filename.c_str(),DestFile.c_str()) != 0) + _error->WarningE("pkgAcqIndex::StageDownloadDone", "Symlinking file %s to %s failed", Filename.c_str(), DestFile.c_str()); + else + { + EraseFileName = DestFile; + Filename = DestFile; + } } - // If we have compressed indexes enabled, queue for hash verification - if (_config->FindB("Acquire::GzipIndexes",false)) - { - DestFile = GetPartialFileNameFromURI(Target.URI + '.' + CurrentCompressionExtension); - EraseFileName = ""; - Stage = STAGE_DECOMPRESS_AND_VERIFY; - Desc.URI = "copy:" + FileName; - QueueURI(Desc); - SetActiveSubprocess("copy"); - return; - } - - // get the binary name for your used compression type - string decompProg; - if(CurrentCompressionExtension == "uncompressed") - decompProg = "copy"; + Stage = STAGE_DECOMPRESS_AND_VERIFY; + DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target); + if (Filename != DestFile && flExtension(Filename) == flExtension(DestFile)) + Desc.URI = "copy:" + Filename; else - decompProg = _config->Find(string("Acquire::CompressionTypes::").append(CurrentCompressionExtension),""); - if(decompProg.empty() == true) + Desc.URI = "store:" + Filename; + if (DestFile == Filename) { - _error->Error("Unsupported extension: %s", CurrentCompressionExtension.c_str()); - return; + if (CurrentCompressionExtension == "uncompressed") + return StageDecompressDone(); + DestFile = "/dev/null"; } + if (EraseFileName.empty()) + EraseFileName = Filename; + // queue uri for the next stage - Stage = STAGE_DECOMPRESS_AND_VERIFY; - DestFile += ".decomp"; - Desc.URI = decompProg + ":" + FileName; QueueURI(Desc); - SetActiveSubprocess(decompProg); + SetActiveSubprocess(::URI(Desc.URI).Access); } /*}}}*/ // AcqIndex::StageDecompressDone - Final verification /*{{{*/ -void pkgAcqIndex::StageDecompressDone(string const &, - HashStringList const &, - pkgAcquire::MethodConfig const * const) +void pkgAcqIndex::StageDecompressDone() { + if (DestFile == "/dev/null") + DestFile = GetKeepCompressedFileName(GetPartialFileNameFromURI(Target.URI), Target); + // Done, queue for rename on transaction finished TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename()); - return; } /*}}}*/ pkgAcqIndex::~pkgAcqIndex() {} @@ -2555,7 +2793,7 @@ pkgAcqIndex::~pkgAcqIndex() {} pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sources, pkgRecords * const Recs,pkgCache::VerIterator const &Version, string &StoreFilename) : - Item(Owner), LocalSource(false), Version(Version), Sources(Sources), Recs(Recs), + Item(Owner), d(NULL), LocalSource(false), Version(Version), Sources(Sources), Recs(Recs), StoreFilename(StoreFilename), Vf(Version.FileList()), Trusted(false) { @@ -2694,7 +2932,7 @@ bool pkgAcqArchive::QueueNext() /* Hmm, we have a file and its size does not match, this means it is an old style mismatched arch */ - unlink(FinalFile.c_str()); + RemoveFile("pkgAcqArchive::QueueNext", FinalFile); } // Check it again using the new style output filenames @@ -2713,7 +2951,7 @@ bool pkgAcqArchive::QueueNext() /* Hmm, we have a file and its size does not match, this shouldn't happen.. */ - unlink(FinalFile.c_str()); + RemoveFile("pkgAcqArchive::QueueNext", FinalFile); } DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename); @@ -2723,7 +2961,7 @@ bool pkgAcqArchive::QueueNext() { // Hmm, the partial file is too big, erase it if ((unsigned long long)Buf.st_size > Version->Size) - unlink(DestFile.c_str()); + RemoveFile("pkgAcqArchive::QueueNext", DestFile); else PartialSize = Buf.st_size; } @@ -2758,15 +2996,7 @@ void pkgAcqArchive::Done(string const &Message, HashStringList const &Hashes, Item::Done(Message, Hashes, Cfg); // Grab the output filename - string FileName = LookupTag(Message,"Filename"); - if (FileName.empty() == true) - { - Status = StatError; - ErrorText = "Method gave a blank filename"; - return; - } - - // Reference filename + std::string const FileName = LookupTag(Message,"Filename"); if (DestFile != FileName && RealFileExists(DestFile) == false) { StoreFilename = DestFile = FileName; @@ -2888,7 +3118,8 @@ void pkgAcqChangelog::Init(std::string const &DestDir, std::string const &DestFi if (DestDir.empty()) { - std::string const systemTemp = GetTempDir(); + std::string const SandboxUser = _config->Find("APT::Sandbox::User"); + std::string const systemTemp = GetTempDir(SandboxUser); char tmpname[100]; snprintf(tmpname, sizeof(tmpname), "%s/apt-changelog-XXXXXX", systemTemp.c_str()); if (NULL == mkdtemp(tmpname)) @@ -2898,6 +3129,9 @@ void pkgAcqChangelog::Init(std::string const &DestDir, std::string const &DestFi return; } DestFile = TemporaryDirectory = tmpname; + + ChangeOwnerAndPermissionOfFile("Item::QueueURI", DestFile.c_str(), + SandboxUser.c_str(), "root", 0700); } else DestFile = DestDir; @@ -2992,7 +3226,7 @@ std::string pkgAcqChangelog::URI(std::string const &Template, char const * const Component, char const * const SrcName, char const * const SrcVersion) { - if (Template.find("CHANGEPATH") == std::string::npos) + if (Template.find("@CHANGEPATH@") == std::string::npos) return ""; // the path is: COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER, e.g. main/a/apt/1.1 or contrib/liba/libapt/2.0 @@ -3004,7 +3238,7 @@ std::string pkgAcqChangelog::URI(std::string const &Template, if (Component != NULL && strlen(Component) != 0) path = std::string(Component) + "/" + path; - return SubstVar(Template, "CHANGEPATH", path); + return SubstVar(Template, "@CHANGEPATH@", path); } /*}}}*/ // AcqChangelog::Failed - Failure handler /*{{{*/ @@ -3037,7 +3271,7 @@ pkgAcqChangelog::~pkgAcqChangelog() /*{{{*/ { if (TemporaryDirectory.empty() == false) { - unlink(DestFile.c_str()); + RemoveFile("~pkgAcqChangelog", DestFile); rmdir(TemporaryDirectory.c_str()); } } @@ -3048,7 +3282,7 @@ pkgAcqFile::pkgAcqFile(pkgAcquire * const Owner,string const &URI, HashStringLis unsigned long long const Size,string const &Dsc,string const &ShortDesc, const string &DestDir, const string &DestFilename, bool const IsIndexFile) : - Item(Owner), IsIndexFile(IsIndexFile), ExpectedHashes(Hashes) + Item(Owner), d(NULL), IsIndexFile(IsIndexFile), ExpectedHashes(Hashes) { Retries = _config->FindI("Acquire::Retries",0); @@ -3074,7 +3308,7 @@ pkgAcqFile::pkgAcqFile(pkgAcquire * const Owner,string const &URI, HashStringLis { // Hmm, the partial file is too big, erase it if ((Size > 0) && (unsigned long long)Buf.st_size > Size) - unlink(DestFile.c_str()); + RemoveFile("pkgAcqFile", DestFile); else PartialSize = Buf.st_size; } @@ -3088,14 +3322,7 @@ void pkgAcqFile::Done(string const &Message,HashStringList const &CalcHashes, { Item::Done(Message,CalcHashes,Cnf); - string FileName = LookupTag(Message,"Filename"); - if (FileName.empty() == true) - { - Status = StatError; - ErrorText = "Method gave a blank filename"; - return; - } - + std::string const FileName = LookupTag(Message,"Filename"); Complete = true; // The files timestamp matches @@ -3119,7 +3346,7 @@ void pkgAcqFile::Done(string const &Message,HashStringList const &CalcHashes, if (lstat(DestFile.c_str(),&St) == 0) { if (S_ISLNK(St.st_mode) != 0) - unlink(DestFile.c_str()); + RemoveFile("pkgAcqFile::Done", DestFile); } // Symlink the file @@ -3128,7 +3355,7 @@ void pkgAcqFile::Done(string const &Message,HashStringList const &CalcHashes, _error->PushToStack(); _error->Errno("pkgAcqFile::Done", "Symlinking file %s failed", DestFile.c_str()); std::stringstream msg; - _error->DumpErrors(msg); + _error->DumpErrors(msg, GlobalError::DEBUG, false); _error->RevertToStack(); ErrorText = msg.str(); Status = StatError;