X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/1f46d14620a0b58a9239fa3faa59ba79a98f6c98..63b7249e6930c1bcb69bac32f10108119eeacc2a:/apt-pkg/acquire-item.cc diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 99013d649..e98eaa445 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -64,10 +64,12 @@ static void printHashSumComparision(std::string const &URI, HashStringList const /*}}}*/ // Acquire::Item::Item - Constructor /*{{{*/ -pkgAcquire::Item::Item(pkgAcquire *Owner, HashStringList const &ExpectedHashes) : - Owner(Owner), FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false), - Local(false), QueueCounter(0), ExpectedAdditionalItems(0), - ExpectedHashes(ExpectedHashes) +pkgAcquire::Item::Item(pkgAcquire *Owner, + HashStringList const &ExpectedHashes, + unsigned long TransactionID) + : Owner(Owner), FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false), + Local(false), QueueCounter(0), TransactionID(TransactionID), + ExpectedAdditionalItems(0), ExpectedHashes(ExpectedHashes) { Owner->Add(this); Status = StatIdle; @@ -88,7 +90,8 @@ pkgAcquire::Item::~Item() void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { Status = StatIdle; - ErrorText = LookupTag(Message,"Message"); + if(ErrorText == "") + ErrorText = LookupTag(Message,"Message"); UsedMirror = LookupTag(Message,"UsedMirror"); if (QueueCounter <= 1) { @@ -134,7 +137,7 @@ void pkgAcquire::Item::Done(string Message,unsigned long long Size,HashStringLis { // We just downloaded something.. string FileName = LookupTag(Message,"Filename"); - UsedMirror = LookupTag(Message,"UsedMirror"); + UsedMirror = LookupTag(Message,"UsedMirror"); if (Complete == false && !Local && FileName == DestFile) { if (Owner->Log != 0) @@ -238,10 +241,12 @@ void pkgAcquire::Item::ReportMirrorFailure(string FailCode) // --------------------------------------------------------------------- /* Get a sub-index file based on checksums from a 'master' file and possibly query additional files */ -pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI, - string const &URIDesc, string const &ShortDesc, - HashStringList const &ExpectedHashes) - : Item(Owner, ExpectedHashes) +pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, + unsigned long TransactionID, + string const &URI, + string const &URIDesc, string const &ShortDesc, + HashStringList const &ExpectedHashes) + : Item(Owner, ExpectedHashes, TransactionID) { /* XXX: Beware: Currently this class does nothing (of value) anymore ! */ Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false); @@ -354,10 +359,12 @@ bool pkgAcqSubIndex::ParseIndex(string const &IndexFile) /*{{{*/ * the original packages file */ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, + unsigned long TransactionID, IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) - : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser) + : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHashes, + MetaIndexParser) { Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); @@ -455,7 +462,8 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ std::clog << "Package file is up-to-date" << std::endl; // list cleanup needs to know that this file as well as the already // present index is ours, so we create an empty diff to save it for us - new pkgAcqIndexDiffs(Owner, Target, ExpectedHashes, MetaIndexParser, + new pkgAcqIndexDiffs(Owner, TransactionID, Target, + ExpectedHashes, MetaIndexParser, ServerSha1, available_patches); return true; } @@ -542,14 +550,17 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ if (pdiff_merge == false) { - new pkgAcqIndexDiffs(Owner, Target, ExpectedHashes, MetaIndexParser, + new pkgAcqIndexDiffs(Owner, TransactionID, Target, ExpectedHashes, + MetaIndexParser, ServerSha1, available_patches); } else { std::vector *diffs = new std::vector(available_patches.size()); for(size_t i = 0; i < available_patches.size(); ++i) - (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, Target, + (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, + TransactionID, + Target, ExpectedHashes, MetaIndexParser, available_patches[i], @@ -577,7 +588,7 @@ void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/ std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser); + new pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser); Complete = false; Status = StatDone; @@ -620,12 +631,13 @@ void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList * for each diff and the index */ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, + unsigned long TransactionID, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser, string ServerSha1, vector diffs) - : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser), + : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser), available_patches(diffs), ServerSha1(ServerSha1) { @@ -657,7 +669,7 @@ void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/) if(Debug) std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser); + new pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser); Finish(); } /*}}}*/ @@ -797,7 +809,7 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringLi // see if there is more to download if(available_patches.empty() == false) { - new pkgAcqIndexDiffs(Owner, Target, + new pkgAcqIndexDiffs(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser, ServerSha1, available_patches); return Finish(); @@ -808,12 +820,13 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringLi /*}}}*/ // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, + unsigned long TransactionID, struct IndexTarget const * const Target, HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser, DiffInfo const &patch, std::vector const * const allPatches) - : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser), + : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser), patch(patch), allPatches(allPatches), State(StateFetchDiff) { @@ -856,7 +869,7 @@ void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*C // first failure means we should fallback State = StateErrorDiff; std::clog << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser); + new pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser); } /*}}}*/ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ @@ -939,30 +952,39 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, HashStringList const &ExpectedHash, string comprExt) - : pkgAcqBaseIndex(Owner, NULL, ExpectedHash, NULL), RealURI(URI) + : pkgAcqBaseIndex(Owner, 0, NULL, ExpectedHash, NULL), RealURI(URI) { - if(comprExt.empty() == true) - { - // autoselect the compression method - std::vector types = APT::Configuration::getCompressionTypes(); - for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) - comprExt.append(*t).append(" "); - if (comprExt.empty() == false) - comprExt.erase(comprExt.end()-1); - } - CompressionExtension = comprExt; - - Verify = true; - + AutoSelectCompression(); Init(URI, URIDesc, ShortDesc); + + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "New pkgIndex with TransactionID " + << TransactionID << std::endl; } -pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target, + /*}}}*/ +// AcqIndex::AcqIndex - Constructor /*{{{*/ +// --------------------------------------------------------------------- +pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, + unsigned long TransactionID, + IndexTarget const *Target, HashStringList const &ExpectedHash, indexRecords *MetaIndexParser) - : pkgAcqBaseIndex(Owner, Target, ExpectedHash, MetaIndexParser), - RealURI(Target->URI) + : pkgAcqBaseIndex(Owner, TransactionID, Target, ExpectedHash, + MetaIndexParser), RealURI(Target->URI) { // autoselect the compression method + AutoSelectCompression(); + Init(Target->URI, Target->Description, Target->ShortDesc); + + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "New pkgIndex with TransactionID " + << TransactionID << std::endl; +} + /*}}}*/ +// AcqIndex::AutoSelectCompression - Select compression /*{{{*/ +// --------------------------------------------------------------------- +void pkgAcqIndex::AutoSelectCompression() +{ std::vector types = APT::Configuration::getCompressionTypes(); CompressionExtension = ""; if (ExpectedHashes.usable()) @@ -978,17 +1000,7 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target, } if (CompressionExtension.empty() == false) CompressionExtension.erase(CompressionExtension.end()-1); - - // only verify non-optional targets, see acquire-item.h for a FIXME - // to make this more flexible - if (Target->IsOptional()) - Verify = false; - else - Verify = true; - - Init(Target->URI, Target->Description, Target->ShortDesc); } - /*}}}*/ // AcqIndex::Init - defered Constructor /*{{{*/ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) { Decompression = false; @@ -998,7 +1010,6 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S DestFile += URItoFileName(URI); std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); - std::string MetaKey; if (comprExt == "uncompressed") { Desc.URI = URI; @@ -1018,6 +1029,8 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); if(Record) FileSize = Record->Size; + + InitByHashIfNeeded(MetaKey); } Desc.Description = URIDesc; @@ -1027,6 +1040,38 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S QueueURI(Desc); } /*}}}*/ +// AcqIndex::AdjustForByHash - modify URI for by-hash support /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqIndex::InitByHashIfNeeded(const std::string MetaKey) +{ + // TODO: + // - (maybe?) add support for by-hash into the sources.list as flag + // - make apt-ftparchive generate the hashes (and expire?) + std::string HostKnob = "APT::Acquire::" + ::URI(Desc.URI).Host + "::By-Hash"; + if(_config->FindB("APT::Acquire::By-Hash", false) == true || + _config->FindB(HostKnob, false) == true || + MetaIndexParser->GetSupportsAcquireByHash()) + { + indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); + if(Record) + { + // FIXME: should we really use the best hash here? or a fixed one? + const HashString *TargetHash = Record->Hashes.find(""); + std::string ByHash = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue(); + size_t trailing_slash = Desc.URI.find_last_of("/"); + Desc.URI = Desc.URI.replace( + trailing_slash, + Desc.URI.substr(trailing_slash+1).size()+1, + ByHash); + } else { + _error->Warning( + "Fetching ByHash requested but can not find record for %s", + MetaKey.c_str()); + } + } +} + /*}}}*/ // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ @@ -1049,6 +1094,9 @@ string pkgAcqIndex::Custom600Headers() const return msg; } /*}}}*/ +// pkgAcqIndex::Failed - getting the indexfile failed /*{{{*/ +// --------------------------------------------------------------------- +/* */ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ { size_t const nextExt = CompressionExtension.find(' '); @@ -1067,8 +1115,40 @@ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ } Item::Failed(Message,Cnf); + + /// cancel the entire transaction + Owner->AbortTransaction(TransactionID); } /*}}}*/ +// pkgAcqIndex::GetFinalFilename - Return the full final file path /*{{{*/ +// --------------------------------------------------------------------- +/* */ +std::string pkgAcqIndex::GetFinalFilename(std::string const &URI, + std::string const &compExt) +{ + std::string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(URI); + if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") + FinalFile += ".gz"; + return FinalFile; +} + /*}}}*/ +// AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName) +{ + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); + if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") + DestFile += ".gz"; + + // copy FinalFile into partial/ so that we check the hash again + string FinalFile = GetFinalFilename(RealURI, compExt); + Decompression = true; + Desc.URI = "copy:" + FinalFile; + QueueURI(Desc); +} + /*}}}*/ // AcqIndex::Done - Finished a fetch /*{{{*/ // --------------------------------------------------------------------- /* This goes through a number of states.. On the initial fetch the @@ -1080,6 +1160,8 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con pkgAcquire::MethodConfig *Cfg) { Item::Done(Message,Size,Hashes,Cfg); + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); + if (Decompression == true) { @@ -1087,35 +1169,36 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con { RenameOnError(HashSumMismatch); printHashSumComparision(RealURI, ExpectedHashes, Hashes); + Failed(Message, Cfg); return; } - /* Verify the index file for correctness (all indexes must - * have a Package field) (LP: #346386) (Closes: #627642) */ - if (Verify == true) + // FIXME: this can go away once we only ever download stuff that + // has a valid hash and we never do GET based probing + // + /* Always verify the index file for correctness (all indexes must + * have a Package field) (LP: #346386) (Closes: #627642) + */ + FileFd fd(DestFile, FileFd::ReadOnly); + // Only test for correctness if the file is not empty (empty is ok) + if (fd.FileSize() > 0) { - FileFd fd(DestFile, FileFd::ReadOnly); - // Only test for correctness if the file is not empty (empty is ok) - if (fd.FileSize() > 0) - { - pkgTagSection sec; - pkgTagFile tag(&fd); - - // all our current indexes have a field 'Package' in each section - if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false) - { - RenameOnError(InvalidFormat); - return; - } + pkgTagSection sec; + pkgTagFile tag(&fd); + + // all our current indexes have a field 'Package' in each section + if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false) + { + RenameOnError(InvalidFormat); + Failed(Message, Cfg); + return; } } - // Done, move it into position - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); - Rename(DestFile,FinalFile); - chmod(FinalFile.c_str(),0644); + // Done, queue for rename on transaction finished + PartialFile = DestFile; +#if 1 // FIXME: waaaay too complicated /* We restore the original name to DestFile so that the clean operation will work OK */ DestFile = _config->FindDir("Dir::State::lists") + "partial/"; @@ -1124,8 +1207,26 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con // Remove the compressed version. if (Erase == true) unlink(DestFile.c_str()); +#endif + + // Done, queue for rename on transaction finished + DestFile = GetFinalFilename(RealURI, compExt); return; + } else { + // FIXME: use the same method to find + // check the compressed hash too + if(MetaKey != "" && Hashes.size() > 0) + { + indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); + if(Record && Record->Hashes.usable() && Hashes != Record->Hashes) + { + RenameOnError(HashSumMismatch); + printHashSumComparision(RealURI, Record->Hashes, Hashes); + Failed(Message, Cfg); + return; + } + } } Erase = false; @@ -1137,7 +1238,10 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con { // The files timestamp matches if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true) + { + ReverifyAfterIMS(FileName); return; + } Decompression = true; Local = true; DestFile += ".decomp"; @@ -1154,13 +1258,10 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con ErrorText = "Method gave a blank filename"; } - std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); - // The files timestamp matches - if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) { - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") - // Update DestFile for .gz suffix so that the clean operation keeps it - DestFile += ".gz"; + if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + { + ReverifyAfterIMS(FileName); return; } @@ -1211,9 +1312,13 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashStringList(), "") { } -pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const * const Target, - HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) - : pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser) + /*}}}*/ +pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, + unsigned long TransactionID, + IndexTarget const * const Target, + HashStringList const &ExpectedHashes, + indexRecords *MetaIndexParser) + : pkgAcqIndex(Owner, TransactionID, Target, ExpectedHashes, MetaIndexParser) { // load the filesize indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey)); @@ -1248,6 +1353,7 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) return; } + // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) { @@ -1261,15 +1367,15 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Item::Failed(Message,Cnf); } /*}}}*/ -pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ +pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ + unsigned long TransactionID, string URI,string URIDesc,string ShortDesc, - string MetaIndexURI, string MetaIndexURIDesc, - string MetaIndexShortDesc, + string MetaIndexFile, const vector* IndexTargets, indexRecords* MetaIndexParser) : - Item(Owner, HashStringList()), RealURI(URI), MetaIndexURI(MetaIndexURI), - MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), - MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets) + Item(Owner, HashStringList(), TransactionID), RealURI(URI), + MetaIndexParser(MetaIndexParser), MetaIndexFile(MetaIndexFile), + IndexTargets(IndexTargets), AuthPass(false), IMSHit(false) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); @@ -1279,42 +1385,22 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ // partial download anyway unlink(DestFile.c_str()); + // set the TransactionID + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "New pkgAcqMetaSig with TransactionID " + << TransactionID << std::endl; + // Create the item Desc.Description = URIDesc; Desc.Owner = this; Desc.ShortDesc = ShortDesc; Desc.URI = URI; - - string Final = _config->FindDir("Dir::State::lists"); - Final += URItoFileName(RealURI); - if (RealFileExists(Final) == true) - { - // File was already in place. It needs to be re-downloaded/verified - // because Release might have changed, we do give it a different - // name than DestFile because otherwise the http method will - // send If-Range requests and there are too many broken servers - // out there that do not understand them - LastGoodSig = DestFile+".reverify"; - Rename(Final,LastGoodSig); - } - - // we expect the indextargets + one additional Release file - ExpectedAdditionalItems = IndexTargets->size() + 1; QueueURI(Desc); } /*}}}*/ pkgAcqMetaSig::~pkgAcqMetaSig() /*{{{*/ { - // if the file was never queued undo file-changes done in the constructor - if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false && - LastGoodSig.empty() == false) - { - string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); - if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true) - Rename(LastGoodSig, Final); - } - } /*}}}*/ // pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/ @@ -1322,8 +1408,11 @@ pkgAcqMetaSig::~pkgAcqMetaSig() /*{{{*/ /* The only header we use is the last-modified header. */ string pkgAcqMetaSig::Custom600Headers() const { + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + struct stat Buf; - if (stat(LastGoodSig.c_str(),&Buf) != 0) + if (stat(FinalFile.c_str(),&Buf) != 0) return "\nIndex-File: true"; return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); @@ -1351,21 +1440,40 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList return; } - Complete = true; + if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + IMSHit = true; - // at this point pkgAcqMetaIndex takes over - ExpectedAdditionalItems = 0; + // adjust paths if its a ims-hit + if(IMSHit) + { + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + + DestFile = PartialFile = FinalFile; + } - // put the last known good file back on i-m-s hit (it will - // be re-verified again) - // Else do nothing, we have the new file in DestFile then - if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) - Rename(LastGoodSig, DestFile); + // queue for verify + if(AuthPass == false) + { + AuthPass = true; + Desc.URI = "gpgv:" + DestFile; + DestFile = MetaIndexFile; + QueueURI(Desc); + return; + } - // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved - new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, - MetaIndexShortDesc, DestFile, IndexTargets, - MetaIndexParser); + // queue to copy the file in place if it was not a ims hit, on ims + // hit the file is already at the right place + if(IMSHit == false) + { + PartialFile = _config->FindDir("Dir::State::lists") + "partial/"; + PartialFile += URItoFileName(RealURI); + + DestFile = _config->FindDir("Dir::State::lists"); + DestFile += URItoFileName(RealURI); + } + + Complete = true; } /*}}}*/ @@ -1373,29 +1481,13 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ { string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); - // at this point pkgAcqMetaIndex takes over - ExpectedAdditionalItems = 0; - - // if we get a network error we fail gracefully - if(Status == StatTransientNetworkError) - { - Item::Failed(Message,Cnf); - // move the sigfile back on transient network failures - if(FileExists(LastGoodSig)) - Rename(LastGoodSig,Final); - - // set the status back to , Item::Failed likes to reset it - Status = pkgAcquire::Item::StatTransientNetworkError; - return; - } - - // Delete any existing sigfile when the acquire failed - unlink(Final.c_str()); - - // queue a pkgAcqMetaIndex with no sigfile - new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, - "", IndexTargets, MetaIndexParser); + // this ensures that any file in the lists/ dir is removed by the + // transaction + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI); + PartialFile = ""; + // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) { @@ -1405,33 +1497,46 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ Dequeue(); return; } - Item::Failed(Message,Cnf); } /*}}}*/ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ + unsigned long TransactionID, string URI,string URIDesc,string ShortDesc, - string SigFile, + string MetaIndexSigURI,string MetaIndexSigURIDesc, string MetaIndexSigShortDesc, const vector* IndexTargets, indexRecords* MetaIndexParser) : - Item(Owner, HashStringList()), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets), - MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false) + Item(Owner, HashStringList(), TransactionID), RealURI(URI), IndexTargets(IndexTargets), + MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false), + MetaIndexSigURI(MetaIndexSigURI), MetaIndexSigURIDesc(MetaIndexSigURIDesc), + MetaIndexSigShortDesc(MetaIndexSigShortDesc) +{ + if(TransactionID == 0) + this->TransactionID = (unsigned long)this; + + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "New pkgAcqMetaIndex with TransactionID " + << TransactionID << std::endl; + + Init(URIDesc, ShortDesc); +} + /*}}}*/ +// pkgAcqMetaIndex::Init - Delayed constructor /*{{{*/ +void pkgAcqMetaIndex::Init(std::string URIDesc, std::string ShortDesc) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(URI); + DestFile += URItoFileName(RealURI); // Create the item Desc.Description = URIDesc; Desc.Owner = this; Desc.ShortDesc = ShortDesc; - Desc.URI = URI; + Desc.URI = RealURI; // we expect more item ExpectedAdditionalItems = IndexTargets->size(); - QueueURI(Desc); } - /*}}}*/ // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ @@ -1472,25 +1577,12 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList if (SigFile == "") { - // There was no signature file, so we are finished. Download - // the indexes and do only hashsum verification if possible + // load indexes, the signature will downloaded afterwards MetaIndexParser->Load(DestFile); - QueueIndexes(false); + QueueIndexes(true); } else { - // FIXME: move this into pkgAcqMetaClearSig::Done on the next - // ABI break - - // if we expect a ClearTextSignature (InRelase), ensure that - // this is what we get and if not fail to queue a - // Release/Release.gpg, see #346386 - if (SigFile == DestFile && !StartsWithGPGClearTextSignature(DestFile)) - { - Failed(Message, Cfg); - return; - } - // There was a signature file, so pass it to gpgv for // verification if (_config->FindB("Debug::pkgAcquire::Auth", false)) @@ -1510,8 +1602,8 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList FinalFile += URItoFileName(RealURI); if (SigFile == DestFile) SigFile = FinalFile; - Rename(DestFile,FinalFile); - chmod(FinalFile.c_str(),0644); + // queue for copy in place + PartialFile = DestFile; DestFile = FinalFile; } } @@ -1546,14 +1638,24 @@ void pkgAcqMetaIndex::RetrievalDone(string Message) /*{{{*/ if (SigFile == DestFile) { SigFile = FinalFile; +#if 0 // constructor of pkgAcqMetaClearSig moved it out of the way, // now move it back in on IMS hit for the 'old' file string const OldClearSig = DestFile + ".reverify"; if (RealFileExists(OldClearSig) == true) Rename(OldClearSig, FinalFile); +#endif } DestFile = FinalFile; } + + // queue a signature + if(SigFile != DestFile) + new pkgAcqMetaSig(Owner, TransactionID, + MetaIndexSigURI, MetaIndexSigURIDesc, + MetaIndexSigShortDesc, DestFile, IndexTargets, + MetaIndexParser); + Complete = true; } /*}}}*/ @@ -1583,6 +1685,7 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ // Download further indexes with verification QueueIndexes(true); +#if 0 // is it a clearsigned MetaIndex file? if (DestFile == SigFile) return; @@ -1592,20 +1695,11 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ URItoFileName(RealURI) + ".gpg"; Rename(SigFile,VerifiedSigFile); chmod(VerifiedSigFile.c_str(),0644); +#endif } /*}}}*/ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ { -#if 0 - /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642) - * FIXME: Disabled; it breaks unsigned repositories without hashes */ - if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile)) - { - Status = StatError; - ErrorText = MetaIndexParser->ErrorText; - return; - } -#endif bool transInRelease = false; { std::vector const keys = MetaIndexParser->MetaKeys(); @@ -1620,7 +1714,6 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ // at this point the real Items are loaded in the fetcher ExpectedAdditionalItems = 0; - for (vector ::const_iterator Target = IndexTargets->begin(); Target != IndexTargets->end(); ++Target) @@ -1669,15 +1762,16 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ if ((*Target)->IsOptional() == true) { if ((*Target)->IsSubIndex() == true) - new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description, + new pkgAcqSubIndex(Owner, TransactionID, + (*Target)->URI, (*Target)->Description, (*Target)->ShortDesc, ExpectedIndexHashes); else if (transInRelease == false || Record != NULL || compressedAvailable == true) { if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true && MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true) - new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqDiffIndex(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser); else - new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqIndexTrans(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser); } continue; } @@ -1688,9 +1782,9 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ instead, but passing the required info to it is to much hassle */ if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false || MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)) - new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqDiffIndex(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser); else - new pkgAcqIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); + new pkgAcqIndex(Owner, TransactionID, *Target, ExpectedIndexHashes, MetaIndexParser); } } /*}}}*/ @@ -1774,22 +1868,19 @@ bool pkgAcqMetaIndex::VerifyVendor(string Message) /*{{{*/ // pkgAcqMetaIndex::Failed - no Release file present or no signature file present /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/) +void pkgAcqMetaIndex::Failed(string Message, + pkgAcquire::MethodConfig * /*Cnf*/) { if (AuthPass == true) { // gpgv method failed, if we have a good signature - string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI)); + string LastGoodSigFile = _config->FindDir("Dir::State::lists"); + LastGoodSigFile += URItoFileName(RealURI); if (DestFile != SigFile) LastGoodSigFile.append(".gpg"); - LastGoodSigFile.append(".reverify"); if(FileExists(LastGoodSigFile)) { - string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); - if (DestFile != SigFile) - VerifiedSigFile.append(".gpg"); - Rename(LastGoodSigFile, VerifiedSigFile); Status = StatTransientNetworkError; _error->Warning(_("An error occurred during the signature " "verification. The repository is not updated " @@ -1804,6 +1895,7 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/) _error->Error(_("GPG error: %s: %s"), Desc.Description.c_str(), LookupTag(Message,"Message").c_str()); + Status = StatError; return; } else { _error->Warning(_("GPG error: %s: %s"), @@ -1828,9 +1920,9 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/) "Release"); SigFile = FinalFile; } - Rename(DestFile,FinalFile); - chmod(FinalFile.c_str(),0644); + // Done, queue for rename on transaction finished + PartialFile = DestFile; DestFile = FinalFile; } @@ -1839,22 +1931,33 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/) QueueIndexes(false); } /*}}}*/ + +void pkgAcqMetaIndex::Finished() +{ + if(_config->FindB("Debug::Acquire::Transaction", false) == true) + std::clog << "Finished: " << DestFile <TransactionHasError(TransactionID) == false && + TransactionID > 0) + Owner->CommitTransaction(TransactionID); +} + + pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/ string const &URI, string const &URIDesc, string const &ShortDesc, string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc, string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc, const vector* IndexTargets, indexRecords* MetaIndexParser) : - pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser), - MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), - MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc) + pkgAcqMetaIndex(Owner, (unsigned long)this, URI, URIDesc, ShortDesc, MetaSigURI, MetaSigURIDesc,MetaSigShortDesc, IndexTargets, MetaIndexParser), + MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), + MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc) { SigFile = DestFile; // index targets + (worst case:) Release/Release.gpg ExpectedAdditionalItems = IndexTargets->size() + 2; - +#if 0 // keep the old InRelease around in case of transistent network errors string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); if (RealFileExists(Final) == true) @@ -1862,10 +1965,12 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/ string const LastGoodSig = DestFile + ".reverify"; Rename(Final,LastGoodSig); } +#endif } /*}}}*/ pkgAcqMetaClearSig::~pkgAcqMetaClearSig() /*{{{*/ { +#if 0 // if the file was never queued undo file-changes done in the constructor if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false) { @@ -1874,6 +1979,7 @@ pkgAcqMetaClearSig::~pkgAcqMetaClearSig() /*{{{*/ if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true) Rename(LastGoodSig, Final); } +#endif } /*}}}*/ // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/ @@ -1887,7 +1993,6 @@ string pkgAcqMetaClearSig::Custom600Headers() const struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) { - Final = DestFile + ".reverify"; if (stat(Final.c_str(),&Buf) != 0) return "\nIndex-File: true\nFail-Ignore: true\n"; } @@ -1900,18 +2005,30 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* // we failed, we will not get additional items from this method ExpectedAdditionalItems = 0; + // if we expect a ClearTextSignature (InRelase), ensure that + // this is what we get and if not fail to queue a + // Release/Release.gpg, see #346386 + if (!StartsWithGPGClearTextSignature(DestFile)) + { + //_error->Error(_("Does not start with a clear sign signature")); + pkgAcquire::Item::Failed(Message, Cnf); + return; + } + + if (AuthPass == false) { - // Remove the 'old' InRelease file if we try Release.gpg now as otherwise - // the file will stay around and gives a false-auth impression (CVE-2012-0214) + // Queue the 'old' InRelease file for removal if we try Release.gpg + // as otherwise the file will stay around and gives a false-auth + // impression (CVE-2012-0214) string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile.append(URItoFileName(RealURI)); - if (FileExists(FinalFile)) - unlink(FinalFile.c_str()); + PartialFile = ""; + DestFile = FinalFile; - new pkgAcqMetaSig(Owner, - MetaSigURI, MetaSigURIDesc, MetaSigShortDesc, + new pkgAcqMetaIndex(Owner, TransactionID, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, + MetaSigURI, MetaSigURIDesc, MetaSigShortDesc, IndexTargets, MetaIndexParser); if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)