From: Michael Vogt Date: Tue, 23 Sep 2014 12:20:27 +0000 (+0200) Subject: Merge branch 'debian/sid' into debian/experimental X-Git-Tag: 1.1.exp3~14 X-Git-Url: https://git.saurik.com/apt.git/commitdiff_plain/c511c5e8ed3f59ddee1b174b39e5cc16a2f11922?hp=-c Merge branch 'debian/sid' into debian/experimental Conflicts: apt-pkg/acquire-item.cc apt-pkg/acquire-item.h apt-pkg/cachefilter.h configure.ac debian/changelog --- c511c5e8ed3f59ddee1b174b39e5cc16a2f11922 diff --combined apt-pkg/acquire-item.cc index 3feb17ffa,36c0fa567..bbdd3897a --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@@ -50,24 -50,12 +50,24 @@@ using namespace std; +static void printHashSumComparision(std::string const &URI, HashStringList const &Expected, HashStringList const &Actual) /*{{{*/ +{ + if (_config->FindB("Debug::Acquire::HashSumMismatch", false) == false) + return; + std::cerr << std::endl << URI << ":" << std::endl << " Expected Hash: " << std::endl; + for (HashStringList::const_iterator hs = Expected.begin(); hs != Expected.end(); ++hs) + std::cerr << "\t- " << hs->toStr() << std::endl; + std::cerr << " Actual Hash: " << std::endl; + for (HashStringList::const_iterator hs = Actual.begin(); hs != Actual.end(); ++hs) + std::cerr << "\t- " << hs->toStr() << std::endl; +} + /*}}}*/ + // Acquire::Item::Item - Constructor /*{{{*/ -// --------------------------------------------------------------------- -/* */ -pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0), - PartialSize(0), Mode(0), ID(0), Complete(false), - Local(false), QueueCounter(0) +pkgAcquire::Item::Item(pkgAcquire *Owner, HashStringList const &ExpectedHashes) : + Owner(Owner), FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false), + Local(false), QueueCounter(0), ExpectedAdditionalItems(0), + ExpectedHashes(ExpectedHashes) { Owner->Add(this); Status = StatIdle; @@@ -129,7 -117,7 +129,7 @@@ void pkgAcquire::Item::Start(string /*M // Acquire::Item::Done - Item downloaded OK /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcquire::Item::Done(string Message,unsigned long long Size,string /*Hash*/, +void pkgAcquire::Item::Done(string Message,unsigned long long Size,HashStringList const &/*Hash*/, pkgAcquire::MethodConfig * /*Cnf*/) { // We just downloaded something.. @@@ -240,8 -228,8 +240,8 @@@ void pkgAcquire::Item::ReportMirrorFail possibly query additional files */ pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI, string const &URIDesc, string const &ShortDesc, - HashString const &ExpectedHash) - : Item(Owner), ExpectedHash(ExpectedHash) + HashStringList const &ExpectedHashes) + : Item(Owner, ExpectedHashes) { /* XXX: Beware: Currently this class does nothing (of value) anymore ! */ Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false); @@@ -263,7 -251,7 +263,7 @@@ // AcqSubIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqSubIndex::Custom600Headers() +string pkgAcqSubIndex::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(Desc.URI); @@@ -286,7 -274,7 +286,7 @@@ void pkgAcqSubIndex::Failed(string Mess // No good Index is provided } /*}}}*/ -void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ +void pkgAcqSubIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ pkgAcquire::MethodConfig *Cnf) { if(Debug) @@@ -308,7 -296,7 +308,7 @@@ return; } - Item::Done(Message,Size,Md5Hash,Cnf); + Item::Done(Message, Size, Hashes, Cnf); string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI); @@@ -354,19 -342,18 +354,19 @@@ bool pkgAcqSubIndex::ParseIndex(string * the original packages file */ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, - string URI,string URIDesc,string ShortDesc, - HashString ExpectedHash) - : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash), - Description(URIDesc) + IndexTarget const * const Target, + HashStringList const &ExpectedHashes, + indexRecords *MetaIndexParser) + : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser) { Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); - Desc.Description = URIDesc + "/DiffIndex"; + RealURI = Target->URI; Desc.Owner = this; - Desc.ShortDesc = ShortDesc; - Desc.URI = URI + ".diff/Index"; + Desc.Description = Target->Description + "/DiffIndex"; + Desc.ShortDesc = Target->ShortDesc; + Desc.URI = Target->URI + ".diff/Index"; DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(Desc.URI); @@@ -402,7 -389,7 +402,7 @@@ // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqDiffIndex::Custom600Headers() +string pkgAcqDiffIndex::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(Desc.URI); @@@ -455,8 -442,8 +455,8 @@@ bool pkgAcqDiffIndex::ParseDiffIndex(st std::clog << "Package file is up-to-date" << std::endl; // list cleanup needs to know that this file as well as the already // present index is ours, so we create an empty diff to save it for us - new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, - ExpectedHash, ServerSha1, available_patches); + new pkgAcqIndexDiffs(Owner, Target, ExpectedHashes, MetaIndexParser, + ServerSha1, available_patches); return true; } else @@@ -541,19 -528,14 +541,19 @@@ } if (pdiff_merge == false) - new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, - ExpectedHash, ServerSha1, available_patches); - else + { + new pkgAcqIndexDiffs(Owner, Target, ExpectedHashes, MetaIndexParser, + ServerSha1, available_patches); + } + else { std::vector *diffs = new std::vector(available_patches.size()); for(size_t i = 0; i < available_patches.size(); ++i) - (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedHash, - available_patches[i], diffs); + (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, Target, + ExpectedHashes, + MetaIndexParser, + available_patches[i], + diffs); } Complete = false; @@@ -577,20 -559,21 +577,20 @@@ void pkgAcqDiffIndex::Failed(string Mes std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc, - ExpectedHash); + new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser); Complete = false; Status = StatDone; Dequeue(); } /*}}}*/ -void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ +void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ pkgAcquire::MethodConfig *Cnf) { if(Debug) std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl; - Item::Done(Message,Size,Md5Hash,Cnf); + Item::Done(Message, Size, Hashes, Cnf); string FinalFile; FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); @@@ -620,24 -603,22 +620,24 @@@ * for each diff and the index */ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, - string URI,string URIDesc,string ShortDesc, - HashString ExpectedHash, + struct IndexTarget const * const Target, + HashStringList const &ExpectedHashes, + indexRecords *MetaIndexParser, string ServerSha1, vector diffs) - : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash), + : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser), available_patches(diffs), ServerSha1(ServerSha1) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(URI); + DestFile += URItoFileName(Target->URI); Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); - Description = URIDesc; + RealURI = Target->URI; Desc.Owner = this; - Desc.ShortDesc = ShortDesc; + Description = Target->Description; + Desc.ShortDesc = Target->ShortDesc; if(available_patches.empty() == true) { @@@ -657,7 -638,8 +657,7 @@@ void pkgAcqIndexDiffs::Failed(string Me if(Debug) std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc, - ExpectedHash); + new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser); Finish(); } /*}}}*/ @@@ -671,7 -653,7 +671,7 @@@ void pkgAcqIndexDiffs::Finish(bool allD DestFile = _config->FindDir("Dir::State::lists"); DestFile += URItoFileName(RealURI); - if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile)) + if(HashSums().usable() && !HashSums().VerifyFile(DestFile)) { RenameOnError(HashSumMismatch); Dequeue(); @@@ -749,13 -731,13 +749,13 @@@ bool pkgAcqIndexDiffs::QueueNextDiff( return true; } /*}}}*/ -void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ +void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringList const &Hashes, /*{{{*/ pkgAcquire::MethodConfig *Cnf) { if(Debug) std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl; - Item::Done(Message,Size,Md5Hash,Cnf); + Item::Done(Message, Size, Hashes, Cnf); string FinalFile; FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); @@@ -797,9 -779,8 +797,9 @@@ // see if there is more to download if(available_patches.empty() == false) { - new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, - ExpectedHash, ServerSha1, available_patches); + new pkgAcqIndexDiffs(Owner, Target, + ExpectedHashes, MetaIndexParser, + ServerSha1, available_patches); return Finish(); } else return Finish(true); @@@ -808,24 -789,22 +808,24 @@@ /*}}}*/ // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, - string const &URI, string const &URIDesc, - string const &ShortDesc, HashString const &ExpectedHash, - DiffInfo const &patch, - std::vector const * const allPatches) - : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash), - patch(patch),allPatches(allPatches), State(StateFetchDiff) + struct IndexTarget const * const Target, + HashStringList const &ExpectedHashes, + indexRecords *MetaIndexParser, + DiffInfo const &patch, + std::vector const * const allPatches) + : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser), + patch(patch), allPatches(allPatches), State(StateFetchDiff) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(URI); + DestFile += URItoFileName(Target->URI); Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); - Description = URIDesc; + RealURI = Target->URI; Desc.Owner = this; - Desc.ShortDesc = ShortDesc; + Description = Target->Description; + Desc.ShortDesc = Target->ShortDesc; Desc.URI = RealURI + ".diff/" + patch.file + ".gz"; Desc.Description = Description + " " + patch.file + string(".pdiff"); @@@ -856,16 -835,17 +856,16 @@@ void pkgAcqIndexMergeDiffs::Failed(stri // first failure means we should fallback State = StateErrorDiff; std::clog << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc, - ExpectedHash); + new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser); } /*}}}*/ -void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ +void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ pkgAcquire::MethodConfig *Cnf) { if(Debug) std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl; - Item::Done(Message,Size,Md5Hash,Cnf); + Item::Done(Message,Size,Hashes,Cnf); string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); @@@ -901,7 -881,7 +901,7 @@@ else if (State == StateApplyDiff) { // see if we really got the expected file - if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile)) + if(ExpectedHashes.usable() && !ExpectedHashes.VerifyFile(DestFile)) { RenameOnError(HashSumMismatch); return; @@@ -938,8 -918,8 +938,8 @@@ instantiated to fetch the revision file */ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, - HashString ExpectedHash, string comprExt) - : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash) + HashStringList const &ExpectedHash, string comprExt) + : pkgAcqBaseIndex(Owner, NULL, ExpectedHash, NULL), RealURI(URI) { if(comprExt.empty() == true) { @@@ -955,15 -935,13 +955,15 @@@ Init(URI, URIDesc, ShortDesc); } pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target, - HashString const &ExpectedHash, indexRecords const *MetaIndexParser) - : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash) + HashStringList const &ExpectedHash, + indexRecords *MetaIndexParser) + : pkgAcqBaseIndex(Owner, Target, ExpectedHash, MetaIndexParser), + RealURI(Target->URI) { // autoselect the compression method std::vector types = APT::Configuration::getCompressionTypes(); CompressionExtension = ""; - if (ExpectedHash.empty() == false) + if (ExpectedHashes.usable()) { for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true) @@@ -989,29 -967,10 +989,29 @@@ void pkgAcqIndex::Init(string const &UR DestFile += URItoFileName(URI); std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); + std::string MetaKey; if (comprExt == "uncompressed") + { Desc.URI = URI; + if(Target) + MetaKey = string(Target->MetaKey); + } else + { Desc.URI = URI + '.' + comprExt; + if(Target) + MetaKey = string(Target->MetaKey) + '.' + comprExt; + } + + // load the filesize + if(MetaIndexParser) + { + indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); + if(Record) + FileSize = Record->Size; + + InitByHashIfNeeded(MetaKey); + } Desc.Description = URIDesc; Desc.Owner = this; @@@ -1020,42 -979,10 +1020,42 @@@ QueueURI(Desc); } /*}}}*/ +// AcqIndex::AdjustForByHash - modify URI for by-hash support /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqIndex::InitByHashIfNeeded(const std::string MetaKey) +{ + // TODO: + // - (maybe?) add support for by-hash into the sources.list as flag + // - make apt-ftparchive generate the hashes (and expire?) + std::string HostKnob = "APT::Acquire::" + ::URI(Desc.URI).Host + "::By-Hash"; + if(_config->FindB("APT::Acquire::By-Hash", false) == true || + _config->FindB(HostKnob, false) == true || + MetaIndexParser->GetSupportsAcquireByHash()) + { + indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); + if(Record) + { + // FIXME: should we really use the best hash here? or a fixed one? + const HashString *TargetHash = Record->Hashes.find(""); + std::string ByHash = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue(); + size_t trailing_slash = Desc.URI.find_last_of("/"); + Desc.URI = Desc.URI.replace( + trailing_slash, + Desc.URI.substr(trailing_slash+1).size()+1, + ByHash); + } else { + _error->Warning( + "Fetching ByHash requested but can not find record for %s", + MetaKey.c_str()); + } + } +} + /*}}}*/ // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqIndex::Custom600Headers() +string pkgAcqIndex::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); @@@ -1094,6 -1021,31 +1094,31 @@@ void pkgAcqIndex::Failed(string Message Item::Failed(Message,Cnf); } /*}}}*/ + // pkgAcqIndex::GetFinalFilename - Return the full final file path /*{{{*/ + std::string pkgAcqIndex::GetFinalFilename(std::string const &URI, + std::string const &compExt) + { + std::string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(URI); + if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") + FinalFile += ".gz"; + return FinalFile; + } + /*}}}*/ + // AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit /*{{{*/ + void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName) + { + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); + if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") + DestFile += ".gz"; + + string FinalFile = GetFinalFilename(RealURI, compExt); + Rename(FinalFile, FileName); + Decompression = true; + Desc.URI = "copy:" + FileName; + QueueURI(Desc); + } + /*}}}*/ // AcqIndex::Done - Finished a fetch /*{{{*/ // --------------------------------------------------------------------- /* This goes through a number of states.. On the initial fetch the @@@ -1101,17 -1053,24 +1126,19 @@@ to the uncompressed version of the file. If this is so the file is copied into the partial directory. In all other cases the file is decompressed with a gzip uri. */ -void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, +void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, pkgAcquire::MethodConfig *Cfg) { - Item::Done(Message,Size,Hashes,Cfg); + Item::Done(Message,Size,Hash,Cfg); + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); if (Decompression == true) { - if (_config->FindB("Debug::pkgAcquire::Auth", false)) - { - std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash; - std::cerr << " Expected Hash: " << ExpectedHash.toStr() << std::endl; - } - - if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash) + if (ExpectedHashes.usable() && ExpectedHashes != Hashes) { + Desc.URI = RealURI; RenameOnError(HashSumMismatch); + printHashSumComparision(RealURI, ExpectedHashes, Hashes); return; } @@@ -1121,9 -1080,9 +1148,9 @@@ /* Always verify the index file for correctness (all indexes must * have a Package field) (LP: #346386) (Closes: #627642) */ - FileFd fd(DestFile, FileFd::ReadOnly); + FileFd fd(DestFile, FileFd::ReadOnlyGzip); // Only test for correctness if the file is not empty (empty is ok) - if (fd.FileSize() > 0) + if (fd.Size() > 0) { pkgTagSection sec; pkgTagFile tag(&fd); @@@ -1137,20 -1096,20 +1164,21 @@@ } // Done, move it into position - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); + string FinalFile = GetFinalFilename(RealURI, compExt); Rename(DestFile,FinalFile); chmod(FinalFile.c_str(),0644); - + /* We restore the original name to DestFile so that the clean operation will work OK */ DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(RealURI); - + if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") + DestFile += ".gz"; + // Remove the compressed version. if (Erase == true) unlink(DestFile.c_str()); + return; } @@@ -1161,9 -1120,6 +1189,6 @@@ string FileName = LookupTag(Message,"Alt-Filename"); if (FileName.empty() == false) { - // The files timestamp matches - if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true) - return; Decompression = true; Local = true; DestFile += ".decomp"; @@@ -1180,33 -1136,37 +1205,37 @@@ ErrorText = "Method gave a blank filename"; } - std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); - - // The files timestamp matches - if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) { - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") - // Update DestFile for .gz suffix so that the clean operation keeps it - DestFile += ".gz"; - return; - } - if (FileName == DestFile) Erase = true; else Local = true; - + + // do not reverify cdrom sources as apt-cdrom may rewrite the Packages + // file when its doing the indexcopy + if (RealURI.substr(0,6) == "cdrom:" && + StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + return; + + // The files timestamp matches, for non-local URLs reverify the local + // file, for local file, uncompress again to ensure the hashsum is still + // matching the Release file + if (!Local && StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + { + ReverifyAfterIMS(FileName); + return; + } string decompProg; - // If we enable compressed indexes and already have gzip, keep it - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) { - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI) + ".gz"; - Rename(DestFile,FinalFile); - chmod(FinalFile.c_str(),0644); - - // Update DestFile for .gz suffix so that the clean operation keeps it - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + // If we enable compressed indexes, queue for hash verification + if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) + { + DestFile = _config->FindDir("Dir::State::lists"); DestFile += URItoFileName(RealURI) + ".gz"; + + Decompression = true; + Desc.URI = "copy:" + FileName; + QueueURI(Desc); + return; } @@@ -1234,26 -1194,25 +1263,29 @@@ /* The Translation file is added to the queue */ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc) - : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "") + : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashStringList(), "") { } -pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target, - HashString const &ExpectedHash, indexRecords const *MetaIndexParser) - : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser) +pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const * const Target, + HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) + : pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser) { + // load the filesize + indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey)); + if(Record) + FileSize = Record->Size; } /*}}}*/ // AcqIndexTrans::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- -string pkgAcqIndexTrans::Custom600Headers() +string pkgAcqIndexTrans::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); + if (_config->FindB("Acquire::GzipIndexes",false)) + Final += ".gz"; + struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) return "\nFail-Ignore: true\nIndex-File: true"; @@@ -1293,7 -1252,7 +1325,7 @@@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquir string MetaIndexShortDesc, const vector* IndexTargets, indexRecords* MetaIndexParser) : - Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI), + Item(Owner, HashStringList()), RealURI(URI), MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets) { @@@ -1324,9 -1283,6 +1356,9 @@@ Rename(Final,LastGoodSig); } + // we expect the indextargets + one additional Release file + ExpectedAdditionalItems = IndexTargets->size() + 1; + QueueURI(Desc); } /*}}}*/ @@@ -1346,7 -1302,7 +1378,7 @@@ pkgAcqMetaSig::~pkgAcqMetaSig() /* // pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqMetaSig::Custom600Headers() +string pkgAcqMetaSig::Custom600Headers() const { struct stat Buf; if (stat(LastGoodSig.c_str(),&Buf) != 0) @@@ -1355,10 -1311,10 +1387,10 @@@ return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } -void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5, +void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cfg) { - Item::Done(Message,Size,MD5,Cfg); + Item::Done(Message, Size, Hashes, Cfg); string FileName = LookupTag(Message,"Filename"); if (FileName.empty() == true) @@@ -1379,9 -1335,6 +1411,9 @@@ Complete = true; + // at this point pkgAcqMetaIndex takes over + ExpectedAdditionalItems = 0; + // put the last known good file back on i-m-s hit (it will // be re-verified again) // Else do nothing, we have the new file in DestFile then @@@ -1399,9 -1352,6 +1431,9 @@@ void pkgAcqMetaSig::Failed(string Messa { string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); + // at this point pkgAcqMetaIndex takes over + ExpectedAdditionalItems = 0; + // if we get a network error we fail gracefully if(Status == StatTransientNetworkError) { @@@ -1438,9 -1388,9 +1470,9 @@@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ string URI,string URIDesc,string ShortDesc, string SigFile, - const vector* IndexTargets, + const vector* IndexTargets, indexRecords* MetaIndexParser) : - Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets), + Item(Owner, HashStringList()), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets), MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; @@@ -1452,16 -1402,13 +1484,16 @@@ Desc.ShortDesc = ShortDesc; Desc.URI = URI; + // we expect more item + ExpectedAdditionalItems = IndexTargets->size(); + QueueURI(Desc); } /*}}}*/ // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqMetaIndex::Custom600Headers() +string pkgAcqMetaIndex::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); @@@ -1473,10 -1420,10 +1505,10 @@@ return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } /*}}}*/ -void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash, /*{{{*/ +void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ pkgAcquire::MethodConfig *Cfg) { - Item::Done(Message,Size,Hash,Cfg); + Item::Done(Message,Size,Hashes,Cfg); // MetaIndexes are done in two passes: one to download the // metaindex with an appropriate method, and a second to verify it @@@ -1505,20 -1452,9 +1537,20 @@@ } else { + // FIXME: move this into pkgAcqMetaClearSig::Done on the next + // ABI break + + // if we expect a ClearTextSignature (InRelase), ensure that + // this is what we get and if not fail to queue a + // Release/Release.gpg, see #346386 + if (SigFile == DestFile && !StartsWithGPGClearTextSignature(DestFile)) + { + Failed(Message, Cfg); + return; + } + // There was a signature file, so pass it to gpgv for // verification - if (_config->FindB("Debug::pkgAcquire::Auth", false)) std::cerr << "Metaindex acquired, queueing gpg verification (" << SigFile << "," << DestFile << ")\n"; @@@ -1606,6 -1542,28 +1638,28 @@@ void pkgAcqMetaIndex::AuthDone(string M std::cerr << "Signature verification succeeded: " << DestFile << std::endl; + // do not trust any previously unverified content that we may have + string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI)); + if (DestFile != SigFile) + LastGoodSigFile.append(".gpg"); + LastGoodSigFile.append(".reverify"); + if(IMSHit == false && RealFileExists(LastGoodSigFile) == false) + { + for (vector ::const_iterator Target = IndexTargets->begin(); + Target != IndexTargets->end(); + ++Target) + { + // remove old indexes + std::string index = _config->FindDir("Dir::State::lists") + + URItoFileName((*Target)->URI); + unlink(index.c_str()); + // and also old gzipindexes + index += ".gz"; + unlink(index.c_str()); + } + } + + // Download further indexes with verification QueueIndexes(true); @@@ -1644,13 -1602,11 +1698,13 @@@ void pkgAcqMetaIndex::QueueIndexes(boo } } - for (vector ::const_iterator Target = IndexTargets->begin(); + // at this point the real Items are loaded in the fetcher + ExpectedAdditionalItems = 0; + for (vector ::const_iterator Target = IndexTargets->begin(); Target != IndexTargets->end(); ++Target) { - HashString ExpectedIndexHash; + HashStringList ExpectedIndexHashes; const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey); bool compressedAvailable = false; if (Record == NULL) @@@ -1674,16 -1630,14 +1728,16 @@@ } else { - ExpectedIndexHash = Record->Hash; + ExpectedIndexHashes = Record->Hashes; if (_config->FindB("Debug::pkgAcquire::Auth", false)) { - std::cerr << "Queueing: " << (*Target)->URI << std::endl; - std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl; + std::cerr << "Queueing: " << (*Target)->URI << std::endl + << "Expected Hash:" << std::endl; + for (HashStringList::const_iterator hs = ExpectedIndexHashes.begin(); hs != ExpectedIndexHashes.end(); ++hs) + std::cerr << "\t- " << hs->toStr() << std::endl; std::cerr << "For: " << Record->MetaKeyFilename << std::endl; } - if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false) + if (verify == true && ExpectedIndexHashes.empty() == true && (*Target)->IsOptional() == false) { Status = StatAuthError; strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str()); @@@ -1695,14 -1649,15 +1749,14 @@@ { if ((*Target)->IsSubIndex() == true) new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, ExpectedIndexHash); + (*Target)->ShortDesc, ExpectedIndexHashes); else if (transInRelease == false || Record != NULL || compressedAvailable == true) { if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true && MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true) - new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, ExpectedIndexHash); + new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); else - new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser); + new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); } continue; } @@@ -1713,9 -1668,10 +1767,9 @@@ instead, but passing the required info to it is to much hassle */ if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false || MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)) - new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, ExpectedIndexHash); + new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); else - new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser); + new pkgAcqIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); } } /*}}}*/ @@@ -1868,7 -1824,7 +1922,7 @@@ pkgAcqMetaClearSig::pkgAcqMetaClearSig( string const &URI, string const &URIDesc, string const &ShortDesc, string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc, string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc, - const vector* IndexTargets, + const vector* IndexTargets, indexRecords* MetaIndexParser) : pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser), MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), @@@ -1876,10 -1832,6 +1930,10 @@@ { SigFile = DestFile; + // index targets + (worst case:) Release/Release.gpg + ExpectedAdditionalItems = IndexTargets->size() + 2; + + // keep the old InRelease around in case of transistent network errors string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); if (RealFileExists(Final) == true) @@@ -1904,7 -1856,7 +1958,7 @@@ pkgAcqMetaClearSig::~pkgAcqMetaClearSig // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- // FIXME: this can go away once the InRelease file is used widely -string pkgAcqMetaClearSig::Custom600Headers() +string pkgAcqMetaClearSig::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); @@@ -1922,9 -1874,6 +1976,9 @@@ /*}}}*/ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ { + // we failed, we will not get additional items from this method + ExpectedAdditionalItems = 0; + if (AuthPass == false) { // Remove the 'old' InRelease file if we try Release.gpg now as otherwise @@@ -1953,7 -1902,7 +2007,7 @@@ pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources, pkgRecords *Recs,pkgCache::VerIterator const &Version, string &StoreFilename) : - Item(Owner), Version(Version), Sources(Sources), Recs(Recs), + Item(Owner, HashStringList()), Version(Version), Sources(Sources), Recs(Recs), StoreFilename(StoreFilename), Vf(Version.FileList()), Trusted(false) { @@@ -2038,6 -1987,7 +2092,6 @@@ checking later. */ bool pkgAcqArchive::QueueNext() { - string const ForceHash = _config->Find("Acquire::ForceHash"); for (; Vf.end() == false; ++Vf) { // Ignore not source sources @@@ -2058,10 -2008,31 +2112,10 @@@ pkgRecords::Parser &Parse = Recs->Lookup(Vf); if (_error->PendingError() == true) return false; - + string PkgFile = Parse.FileName(); - if (ForceHash.empty() == false) - { - if(stringcasecmp(ForceHash, "sha512") == 0) - ExpectedHash = HashString("SHA512", Parse.SHA512Hash()); - else if(stringcasecmp(ForceHash, "sha256") == 0) - ExpectedHash = HashString("SHA256", Parse.SHA256Hash()); - else if (stringcasecmp(ForceHash, "sha1") == 0) - ExpectedHash = HashString("SHA1", Parse.SHA1Hash()); - else - ExpectedHash = HashString("MD5Sum", Parse.MD5Hash()); - } - else - { - string Hash; - if ((Hash = Parse.SHA512Hash()).empty() == false) - ExpectedHash = HashString("SHA512", Hash); - else if ((Hash = Parse.SHA256Hash()).empty() == false) - ExpectedHash = HashString("SHA256", Hash); - else if ((Hash = Parse.SHA1Hash()).empty() == false) - ExpectedHash = HashString("SHA1", Hash); - else - ExpectedHash = HashString("MD5Sum", Parse.MD5Hash()); - } + ExpectedHashes = Parse.Hashes(); + if (PkgFile.empty() == true) return _error->Error(_("The package index files are corrupted. No Filename: " "field for package %s."), @@@ -2148,10 -2119,10 +2202,10 @@@ // AcqArchive::Done - Finished fetching /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash, +void pkgAcqArchive::Done(string Message,unsigned long long Size, HashStringList const &CalcHashes, pkgAcquire::MethodConfig *Cfg) { - Item::Done(Message,Size,CalcHash,Cfg); + Item::Done(Message, Size, CalcHashes, Cfg); // Check the size if (Size != Version->Size) @@@ -2159,12 -2130,11 +2213,12 @@@ RenameOnError(SizeMismatch); return; } - - // Check the hash - if(ExpectedHash.toStr() != CalcHash) + + // FIXME: could this empty() check impose *any* sort of security issue? + if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes) { RenameOnError(HashSumMismatch); + printHashSumComparision(DestFile, ExpectedHashes, CalcHashes); return; } @@@ -2236,7 -2206,7 +2290,7 @@@ void pkgAcqArchive::Failed(string Messa /*}}}*/ // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/ // --------------------------------------------------------------------- -APT_PURE bool pkgAcqArchive::IsTrusted() +APT_PURE bool pkgAcqArchive::IsTrusted() const { return Trusted; } @@@ -2255,11 -2225,11 +2309,11 @@@ void pkgAcqArchive::Finished( // AcqFile::pkgAcqFile - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The file is added to the queue */ -pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash, +pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI, HashStringList const &Hashes, unsigned long long Size,string Dsc,string ShortDesc, const string &DestDir, const string &DestFilename, bool IsIndexFile) : - Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile) + Item(Owner, Hashes), IsIndexFile(IsIndexFile) { Retries = _config->FindI("Acquire::Retries",0); @@@ -2296,16 -2266,15 +2350,16 @@@ // AcqFile::Done - Item downloaded OK /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash, +void pkgAcqFile::Done(string Message,unsigned long long Size,HashStringList const &CalcHashes, pkgAcquire::MethodConfig *Cnf) { - Item::Done(Message,Size,CalcHash,Cnf); + Item::Done(Message,Size,CalcHashes,Cnf); // Check the hash - if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash) + if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes) { RenameOnError(HashSumMismatch); + printHashSumComparision(DestFile, ExpectedHashes, CalcHashes); return; } @@@ -2376,7 -2345,7 +2430,7 @@@ void pkgAcqFile::Failed(string Message, // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqFile::Custom600Headers() +string pkgAcqFile::Custom600Headers() const { if (IsIndexFile) return "\nIndex-File: true"; diff --combined apt-pkg/acquire-item.h index 0500a3627,384c5ee2b..c027a2764 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@@ -46,7 -46,6 +46,7 @@@ class indexRecords; class pkgRecords; class pkgSourceList; +class IndexTarget; /** \brief Represents the process by which a pkgAcquire object should {{{ * retrieve a file or a collection of files. @@@ -167,16 -166,6 +167,16 @@@ class pkgAcquire::Item : public WeakPoi * \sa pkgAcquire */ unsigned int QueueCounter; + + /** \brief The number of additional fetch items that are expected + * once this item is done. + * + * Some items like pkgAcqMeta{Index,Sig} will queue additional + * items. This variable can be set by the methods if it knows + * in advance how many items to expect to get a more accurate + * progress. + */ + unsigned int ExpectedAdditionalItems; /** \brief The name of the file into which the retrieved object * will be written. @@@ -212,12 -201,12 +212,12 @@@ * \param Message Data from the acquire method. Use LookupTag() * to parse it. * \param Size The size of the object that was fetched. - * \param Hash The HashSum of the object that was fetched. + * \param Hashes The HashSums of the object that was fetched. * \param Cnf The method via which the object was fetched. * * \sa pkgAcqMethod */ - virtual void Done(std::string Message,unsigned long long Size,std::string Hash, + virtual void Done(std::string Message, unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); /** \brief Invoked when the worker starts to fetch this object. @@@ -239,35 -228,34 +239,35 @@@ * line, so they should (if nonempty) have a leading newline and * no trailing newline. */ - virtual std::string Custom600Headers() {return std::string();}; + virtual std::string Custom600Headers() const {return std::string();}; /** \brief A "descriptive" URI-like string. * * \return a URI that should be used to describe what is being fetched. */ - virtual std::string DescURI() = 0; + virtual std::string DescURI() const = 0; /** \brief Short item description. * * \return a brief description of the object being fetched. */ - virtual std::string ShortDesc() {return DescURI();} + virtual std::string ShortDesc() const {return DescURI();} /** \brief Invoked by the worker when the download is completely done. */ virtual void Finished() {}; - /** \brief HashSum + /** \brief HashSums * - * \return the HashSum of this object, if applicable; otherwise, an - * empty string. + * \return the HashSums of this object, if applicable; otherwise, an + * empty list. */ - virtual std::string HashSum() {return std::string();}; + HashStringList HashSums() const {return ExpectedHashes;}; + std::string HashSum() const {HashStringList const hashes = HashSums(); HashString const * const hs = hashes.find(NULL); return hs != NULL ? hs->toStr() : ""; }; /** \return the acquire process with which this item is associated. */ - pkgAcquire *GetOwner() {return Owner;}; + pkgAcquire *GetOwner() const {return Owner;}; /** \return \b true if this object is being fetched from a trusted source. */ - virtual bool IsTrusted() {return false;}; + virtual bool IsTrusted() const {return false;}; // report mirror problems /** \brief Report mirror problem @@@ -286,11 -274,12 +286,11 @@@ * process, but does not place it into any fetch queues (you must * manually invoke QueueURI() to do so). * - * Initializes all fields of the item other than Owner to 0, - * false, or the empty string. - * * \param Owner The new owner of this item. + * \param ExpectedHashes of the file represented by this item */ - Item(pkgAcquire *Owner); + Item(pkgAcquire *Owner, + HashStringList const &ExpectedHashes=HashStringList()); /** \brief Remove this item from its owner's queue by invoking * pkgAcquire::Remove. @@@ -310,12 -299,6 +310,12 @@@ * \param state respresenting the error we encountered */ bool RenameOnError(RenameOnErrorState const state); + + /** \brief The HashSums of the item is supposed to have than done */ + HashStringList ExpectedHashes; + + /** \brief The item that is currently being downloaded. */ + pkgAcquire::ItemDesc Desc; }; /*}}}*/ /** \brief Information about an index patch (aka diff). */ /*{{{*/ @@@ -342,13 -325,20 +342,13 @@@ class pkgAcqSubIndex : public pkgAcquir /** \brief If \b true, debugging information will be written to std::clog. */ bool Debug; - /** \brief The item that is currently being downloaded. */ - pkgAcquire::ItemDesc Desc; - - /** \brief The Hash that this file should have after download - */ - HashString ExpectedHash; - public: // Specialized action members virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size,std::string Md5Hash, + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); - virtual std::string DescURI() {return Desc.URI;}; - virtual std::string Custom600Headers(); + virtual std::string DescURI() const {return Desc.URI;}; + virtual std::string Custom600Headers() const; virtual bool ParseIndex(std::string const &IndexFile); /** \brief Create a new pkgAcqSubIndex. @@@ -361,31 -351,10 +361,31 @@@ * * \param ShortDesc A short description of the list file to download. * - * \param ExpectedHash The list file's MD5 signature. + * \param ExpectedHashes The list file's hashsums which are expected. */ pkgAcqSubIndex(pkgAcquire *Owner, std::string const &URI,std::string const &URIDesc, - std::string const &ShortDesc, HashString const &ExpectedHash); + std::string const &ShortDesc, HashStringList const &ExpectedHashes); +}; + /*}}}*/ + +/** \brief Common base class for all classes that deal with fetching {{{ + indexes + */ +class pkgAcqBaseIndex : public pkgAcquire::Item +{ + protected: + /** \brief Pointer to the IndexTarget data + */ + const struct IndexTarget * Target; + indexRecords *MetaIndexParser; + + pkgAcqBaseIndex(pkgAcquire *Owner, + struct IndexTarget const * const Target, + HashStringList const &ExpectedHashes, + indexRecords *MetaIndexParser) + : Item(Owner, ExpectedHashes), Target(Target), + MetaIndexParser(MetaIndexParser) {}; + }; /*}}}*/ /** \brief An item that is responsible for fetching an index file of {{{ @@@ -397,17 -366,25 +397,17 @@@ * * \sa pkgAcqIndexDiffs, pkgAcqIndex */ -class pkgAcqDiffIndex : public pkgAcquire::Item +class pkgAcqDiffIndex : public pkgAcqBaseIndex { protected: /** \brief If \b true, debugging information will be written to std::clog. */ bool Debug; - /** \brief The item that is currently being downloaded. */ - pkgAcquire::ItemDesc Desc; - /** \brief The URI of the index file to recreate at our end (either * by downloading it or by applying partial patches). */ std::string RealURI; - /** \brief The Hash that the real index file should have after - * all patches have been applied. - */ - HashString ExpectedHash; - /** \brief The index file which will be patched to generate the new * file. */ @@@ -421,10 -398,10 +421,10 @@@ public: // Specialized action members virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size,std::string Md5Hash, + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); - virtual std::string DescURI() {return RealURI + "Index";}; - virtual std::string Custom600Headers(); + virtual std::string DescURI() const {return RealURI + "Index";}; + virtual std::string Custom600Headers() const; /** \brief Parse the Index file for a set of Packages diffs. * @@@ -449,12 -426,10 +449,12 @@@ * * \param ShortDesc A short description of the list file to download. * - * \param ExpectedHash The list file's MD5 signature. + * \param ExpectedHashes The list file's hashsums which are expected. */ - pkgAcqDiffIndex(pkgAcquire *Owner,std::string URI,std::string URIDesc, - std::string ShortDesc, HashString ExpectedHash); + pkgAcqDiffIndex(pkgAcquire *Owner, + struct IndexTarget const * const Target, + HashStringList const &ExpectedHashes, + indexRecords *MetaIndexParser); }; /*}}}*/ /** \brief An item that is responsible for fetching client-merge patches {{{ @@@ -468,7 -443,7 +468,7 @@@ * * \sa pkgAcqDiffIndex, pkgAcqIndex */ -class pkgAcqIndexMergeDiffs : public pkgAcquire::Item +class pkgAcqIndexMergeDiffs : public pkgAcqBaseIndex { protected: @@@ -477,11 -452,21 +477,11 @@@ */ bool Debug; - /** \brief description of the item that is currently being - * downloaded. - */ - pkgAcquire::ItemDesc Desc; - /** \brief URI of the package index file that is being * reconstructed. */ std::string RealURI; - /** \brief HashSum of the package index file that is being - * reconstructed. - */ - HashString ExpectedHash; - /** \brief description of the file being downloaded. */ std::string Description; @@@ -514,9 -499,10 +514,9 @@@ * outright; its arguments are ignored. */ virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - - virtual void Done(std::string Message,unsigned long long Size,std::string Md5Hash, - pkgAcquire::MethodConfig *Cnf); - virtual std::string DescURI() {return RealURI + "Index";}; + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, + pkgAcquire::MethodConfig *Cnf); + virtual std::string DescURI() const {return RealURI + "Index";}; /** \brief Create an index merge-diff item. * @@@ -529,7 -515,7 +529,7 @@@ * * \param ShortDesc A brief description of this item. * - * \param ExpectedHash The expected md5sum of the completely + * \param ExpectedHashes The expected md5sum of the completely * reconstructed package index file; the index file will be tested * against this value when it is entirely reconstructed. * @@@ -539,12 -525,9 +539,12 @@@ * \param allPatches contains all related items so that each item can * check if it was the last one to complete the download step */ - pkgAcqIndexMergeDiffs(pkgAcquire *Owner,std::string const &URI,std::string const &URIDesc, - std::string const &ShortDesc, HashString const &ExpectedHash, - DiffInfo const &patch, std::vector const * const allPatches); + pkgAcqIndexMergeDiffs(pkgAcquire *Owner, + struct IndexTarget const * const Target, + HashStringList const &ExpectedHash, + indexRecords *MetaIndexParser, + DiffInfo const &patch, + std::vector const * const allPatches); }; /*}}}*/ /** \brief An item that is responsible for fetching server-merge patches {{{ @@@ -558,7 -541,7 +558,7 @@@ * * \sa pkgAcqDiffIndex, pkgAcqIndex */ -class pkgAcqIndexDiffs : public pkgAcquire::Item +class pkgAcqIndexDiffs : public pkgAcqBaseIndex { private: @@@ -576,8 -559,8 +576,8 @@@ /** \brief Handle tasks that must be performed after the item * finishes downloading. * - * Dequeues the item and checks the resulting file's md5sum - * against ExpectedHash after the last patch was applied. + * Dequeues the item and checks the resulting file's hashsums + * against ExpectedHashes after the last patch was applied. * There is no need to check the md5/sha1 after a "normal" * patch because QueueNextDiff() will check the sha1 later. * @@@ -593,11 -576,21 +593,11 @@@ */ bool Debug; - /** \brief A description of the item that is currently being - * downloaded. - */ - pkgAcquire::ItemDesc Desc; - /** \brief The URI of the package index file that is being * reconstructed. */ std::string RealURI; - /** \brief The HashSum of the package index file that is being - * reconstructed. - */ - HashString ExpectedHash; - /** A description of the file being downloaded. */ std::string Description; @@@ -639,9 -632,9 +639,9 @@@ */ virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size,std::string Md5Hash, + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); - virtual std::string DescURI() {return RealURI + "Index";}; + virtual std::string DescURI() const {return RealURI + "Index";}; /** \brief Create an index diff item. * @@@ -657,7 -650,7 +657,7 @@@ * * \param ShortDesc A brief description of this item. * - * \param ExpectedHash The expected md5sum of the completely + * \param ExpectedHashes The expected md5sum of the completely * reconstructed package index file; the index file will be tested * against this value when it is entirely reconstructed. * @@@ -667,10 -660,8 +667,10 @@@ * should be ordered so that each diff appears before any diff * that depends on it. */ - pkgAcqIndexDiffs(pkgAcquire *Owner,std::string URI,std::string URIDesc, - std::string ShortDesc, HashString ExpectedHash, + pkgAcqIndexDiffs(pkgAcquire *Owner, + struct IndexTarget const * const Target, + HashStringList const &ExpectedHash, + indexRecords *MetaIndexParser, std::string ServerSha1, std::vector diffs=std::vector()); }; @@@ -682,7 -673,7 +682,7 @@@ * * \todo Why does pkgAcqIndex have protected members? */ -class pkgAcqIndex : public pkgAcquire::Item +class pkgAcqIndex : public pkgAcqBaseIndex { protected: @@@ -697,27 -688,41 +697,36 @@@ // Unused, used to be used to verify that "Packages: " header was there bool __DELME_ON_NEXT_ABI_BREAK_Verify; - /** \brief The download request that is currently being - * processed. - */ - pkgAcquire::ItemDesc Desc; - /** \brief The object that is actually being fetched (minus any * compression-related extensions). */ std::string RealURI; - /** \brief The expected hashsum of the decompressed index file. */ - HashString ExpectedHash; - /** \brief The compression-related file extensions that are being * added to the downloaded file one by one if first fails (e.g., "gz bz2"). */ std::string CompressionExtension; ++ + /** \brief Do the changes needed to fetch via AptByHash (if needed) */ + void InitByHashIfNeeded(const std::string MetaKey); + + /** \brief Get the full pathname of the final file for the given URI + */ + std::string GetFinalFilename(std::string const &URI, + std::string const &compExt); + + /** \brief Schedule file for verification after a IMS hit */ + void ReverifyAfterIMS(std::string const &FileName); + public: // Specialized action members virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size,std::string Md5Hash, + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); - virtual std::string Custom600Headers(); - virtual std::string DescURI() {return Desc.URI;}; - virtual std::string HashSum() {return ExpectedHash.toStr(); }; + virtual std::string Custom600Headers() const; + virtual std::string DescURI() const {return Desc.URI;}; /** \brief Create a pkgAcqIndex. * @@@ -730,7 -735,7 +739,7 @@@ * * \param ShortDesc A brief description of this index file. * - * \param ExpectedHash The expected hashsum of this index file. + * \param ExpectedHashes The expected hashsum of this index file. * * \param compressExt The compression-related extension with which * this index file should be downloaded, or "" to autodetect @@@ -739,14 -744,11 +748,14 @@@ * fallback is ".gz" or none. */ pkgAcqIndex(pkgAcquire *Owner,std::string URI,std::string URIDesc, - std::string ShortDesc, HashString ExpectedHash, + std::string ShortDesc, HashStringList const &ExpectedHashes, std::string compressExt=""); - pkgAcqIndex(pkgAcquire *Owner, struct IndexTarget const * const Target, - HashString const &ExpectedHash, indexRecords const *MetaIndexParser); - void Init(std::string const &URI, std::string const &URIDesc, std::string const &ShortDesc); + pkgAcqIndex(pkgAcquire *Owner, + IndexTarget const * const Target, + HashStringList const &ExpectedHash, + indexRecords *MetaIndexParser); + void Init(std::string const &URI, std::string const &URIDesc, + std::string const &ShortDesc); }; /*}}}*/ /** \brief An acquire item that is responsible for fetching a {{{ @@@ -761,7 -763,7 +770,7 @@@ class pkgAcqIndexTrans : public pkgAcqI public: virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual std::string Custom600Headers(); + virtual std::string Custom600Headers() const; /** \brief Create a pkgAcqIndexTrans. * @@@ -776,8 -778,8 +785,8 @@@ */ pkgAcqIndexTrans(pkgAcquire *Owner,std::string URI,std::string URIDesc, std::string ShortDesc); - pkgAcqIndexTrans(pkgAcquire *Owner, struct IndexTarget const * const Target, - HashString const &ExpectedHash, indexRecords const *MetaIndexParser); + pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const * const Target, + HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser); }; /*}}}*/ /** \brief Information about an index file. */ /*{{{*/ @@@ -845,6 -847,9 +854,6 @@@ class pkgAcqMetaSig : public pkgAcquire /** \brief The last good signature file */ std::string LastGoodSig; - /** \brief The fetch request that is currently being processed. */ - pkgAcquire::ItemDesc Desc; - /** \brief The URI of the signature file. Unlike Desc.URI, this is * never modified; it is used to determine the file that is being * downloaded. @@@ -872,21 -877,21 +881,21 @@@ * * \todo Why a list of pointers instead of a list of structs? */ - const std::vector* IndexTargets; + const std::vector* IndexTargets; public: // Specialized action members virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size,std::string Md5Hash, + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); - virtual std::string Custom600Headers(); - virtual std::string DescURI() {return RealURI; }; + virtual std::string Custom600Headers() const; + virtual std::string DescURI() const {return RealURI; }; /** \brief Create a new pkgAcqMetaSig. */ pkgAcqMetaSig(pkgAcquire *Owner,std::string URI,std::string URIDesc, std::string ShortDesc, std::string MetaIndexURI, std::string MetaIndexURIDesc, std::string MetaIndexShortDesc, - const std::vector* IndexTargets, + const std::vector* IndexTargets, indexRecords* MetaIndexParser); virtual ~pkgAcqMetaSig(); }; @@@ -904,6 -909,9 +913,6 @@@ class pkgAcqMetaIndex : public pkgAcquire::Item { protected: - /** \brief The fetch command that is currently being processed. */ - pkgAcquire::ItemDesc Desc; - /** \brief The URI that is actually being downloaded; never * modified by pkgAcqMetaIndex. */ @@@ -917,7 -925,7 +926,7 @@@ std::string SigFile; /** \brief The index files to download. */ - const std::vector* IndexTargets; + const std::vector* IndexTargets; /** \brief The parser for the meta-index file. */ indexRecords* MetaIndexParser; @@@ -971,16 -979,16 +980,16 @@@ // Specialized action members virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size, std::string Hash, + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); - virtual std::string Custom600Headers(); - virtual std::string DescURI() {return RealURI; }; + virtual std::string Custom600Headers() const; + virtual std::string DescURI() const {return RealURI; }; /** \brief Create a new pkgAcqMetaIndex. */ pkgAcqMetaIndex(pkgAcquire *Owner, std::string URI,std::string URIDesc, std::string ShortDesc, std::string SigFile, - const std::vector* IndexTargets, + const std::vector* IndexTargets, indexRecords* MetaIndexParser); }; /*}}}*/ @@@ -1007,14 -1015,14 +1016,14 @@@ class pkgAcqMetaClearSig : public pkgAc public: void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual std::string Custom600Headers(); + virtual std::string Custom600Headers() const; /** \brief Create a new pkgAcqMetaClearSig. */ pkgAcqMetaClearSig(pkgAcquire *Owner, std::string const &URI, std::string const &URIDesc, std::string const &ShortDesc, std::string const &MetaIndexURI, std::string const &MetaIndexURIDesc, std::string const &MetaIndexShortDesc, std::string const &MetaSigURI, std::string const &MetaSigURIDesc, std::string const &MetaSigShortDesc, - const std::vector* IndexTargets, + const std::vector* IndexTargets, indexRecords* MetaIndexParser); virtual ~pkgAcqMetaClearSig(); }; @@@ -1030,6 -1038,9 +1039,6 @@@ class pkgAcqArchive : public pkgAcquire /** \brief The package version being fetched. */ pkgCache::VerIterator Version; - /** \brief The fetch command that is currently being processed. */ - pkgAcquire::ItemDesc Desc; - /** \brief The list of sources from which to pick archives to * download this package from. */ @@@ -1040,6 -1051,9 +1049,6 @@@ */ pkgRecords *Recs; - /** \brief The hashsum of this package. */ - HashString ExpectedHash; - /** \brief A location in which the actual filename of the package * should be stored. */ @@@ -1066,12 -1080,13 +1075,12 @@@ public: virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size,std::string Hash, + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); - virtual std::string DescURI() {return Desc.URI;}; - virtual std::string ShortDesc() {return Desc.ShortDesc;}; + virtual std::string DescURI() const {return Desc.URI;}; + virtual std::string ShortDesc() const {return Desc.ShortDesc;}; virtual void Finished(); - virtual std::string HashSum() {return ExpectedHash.toStr(); }; - virtual bool IsTrusted(); + virtual bool IsTrusted() const; /** \brief Create a new pkgAcqArchive. * @@@ -1104,6 -1119,12 +1113,6 @@@ */ class pkgAcqFile : public pkgAcquire::Item { - /** \brief The currently active download process. */ - pkgAcquire::ItemDesc Desc; - - /** \brief The hashsum of the file to download, if it is known. */ - HashString ExpectedHash; - /** \brief How many times to retry the download, set from * Acquire::Retries. */ @@@ -1116,10 -1137,11 +1125,10 @@@ // Specialized action members virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size,std::string CalcHash, + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &CalcHashes, pkgAcquire::MethodConfig *Cnf); - virtual std::string DescURI() {return Desc.URI;}; - virtual std::string HashSum() {return ExpectedHash.toStr(); }; - virtual std::string Custom600Headers(); + virtual std::string DescURI() const {return Desc.URI;}; + virtual std::string Custom600Headers() const; /** \brief Create a new pkgAcqFile object. * @@@ -1128,8 -1150,8 +1137,8 @@@ * * \param URI The URI to download. * - * \param Hash The hashsum of the file to download, if it is known; - * otherwise "". + * \param Hashes The hashsums of the file to download, if they are known; + * otherwise empty list. * * \param Size The size of the file to download, if it is known; * otherwise 0. @@@ -1152,7 -1174,7 +1161,7 @@@ * is the absolute name to which the file should be downloaded. */ - pkgAcqFile(pkgAcquire *Owner, std::string URI, std::string Hash, unsigned long long Size, + pkgAcqFile(pkgAcquire *Owner, std::string URI, HashStringList const &Hashes, unsigned long long Size, std::string Desc, std::string ShortDesc, const std::string &DestDir="", const std::string &DestFilename="", bool IsIndexFile=false); diff --combined apt-pkg/contrib/fileutl.h index e04f75e2a,667057067..a8e255b86 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@@ -85,7 -85,9 +85,9 @@@ class FileF bool Skip(unsigned long long To); bool Truncate(unsigned long long To); unsigned long long Tell(); + // the size of the file content (compressed files will be uncompressed first) unsigned long long Size(); + // the size of the file itself unsigned long long FileSize(); time_t ModificationTime(); @@@ -168,8 -170,6 +170,8 @@@ time_t GetModificationTime(std::string bool Rename(std::string From, std::string To); std::string GetTempDir(); +FileFd* GetTempFile(std::string const &Prefix = "", + bool ImmediateUnlink = true); /** \brief Ensure the existence of the given Path * @@@ -193,13 -193,6 +195,13 @@@ pid_t ExecFork(std::set keep_fds) void MergeKeepFdsFromConfiguration(std::set &keep_fds); bool ExecWait(pid_t Pid,const char *Name,bool Reap = false); + +// check if the given file starts with a PGP cleartext signature +bool StartsWithGPGClearTextSignature(std::string const &FileName); + +// process releated +bool DropPrivs(); + // File string manipulators std::string flNotDir(std::string File); std::string flNotFile(std::string File); @@@ -207,23 -200,7 +209,23 @@@ std::string flNoLink(std::string File) std::string flExtension(std::string File); std::string flCombine(std::string Dir,std::string File); +/** \brief Takes a file path and returns the absolute path + */ +std::string flAbsPath(std::string File); + // simple c++ glob std::vector Glob(std::string const &pattern, int flags=0); +/** \brief Popen() implementation that execv() instead of using a shell + * + * \param Args the execv style command to run + * \param FileFd is a referenz to the FileFd to use for input or output + * \param Child a reference to the integer that stores the child pid + * Note that you must call ExecWait() or similar to cleanup + * \param Mode is either FileFd::ReadOnly or FileFd::WriteOnly + * \return true on success, false on failure with _error set + */ +bool Popen(const char* Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode); + + #endif diff --combined apt-pkg/contrib/strutl.cc index 9238966cd,87f57a30e..aad358a55 --- a/apt-pkg/contrib/strutl.cc +++ b/apt-pkg/contrib/strutl.cc @@@ -45,14 -45,26 +45,26 @@@ using namespace std // --------------------------------------------------------------------- namespace APT { namespace String { - std::string Strip(const std::string &s) + std::string Strip(const std::string &str) { - size_t start = s.find_first_not_of(" \t\n"); - // only whitespace - if (start == string::npos) + // ensure we have at least one character + if (str.empty() == true) + return str; + + char const * const s = str.c_str(); + size_t start = 0; + for (; isspace(s[start]) != 0; ++start) + ; // find the first not-space + + // string contains only whitespaces + if (s[start] == '\0') return ""; - size_t end = s.find_last_not_of(" \t\n"); - return s.substr(start, end-start+1); + + size_t end = str.length() - 1; + for (; isspace(s[end]) != 0; --end) + ; // find the last not-space + + return str.substr(start, end - start + 1); } bool Endswith(const std::string &s, const std::string &end) @@@ -62,13 -74,6 +74,13 @@@ return (s.substr(s.size() - end.size(), s.size()) == end); } +bool Startswith(const std::string &s, const std::string &start) +{ + if (start.size() > s.size()) + return false; + return (s.substr(0, start.size()) == start); +} + } } /*}}}*/ @@@ -1056,7 -1061,7 +1068,7 @@@ bool StrToNum(const char *Str,unsigned // --------------------------------------------------------------------- /* This is used in decoding the 256bit encoded fixed length fields in tar files */ -bool Base256ToNum(const char *Str,unsigned long &Res,unsigned int Len) +bool Base256ToNum(const char *Str,unsigned long long &Res,unsigned int Len) { if ((Str[0] & 0x80) == 0) return false; @@@ -1069,23 -1074,6 +1081,23 @@@ } } /*}}}*/ +// Base256ToNum - Convert a fixed length binary to a number /*{{{*/ +// --------------------------------------------------------------------- +/* This is used in decoding the 256bit encoded fixed length fields in + tar files */ +bool Base256ToNum(const char *Str,unsigned long &Res,unsigned int Len) +{ + unsigned long long Num; + bool rc; + + rc = Base256ToNum(Str, Num, Len); + Res = Num; + if (Res != Num) + return false; + + return rc; +} + /*}}}*/ // HexDigit - Convert a hex character into an integer /*{{{*/ // --------------------------------------------------------------------- /* Helper for Hex2Num */ diff --combined apt-pkg/deb/dpkgpm.cc index 8295a7c1e,04a13a86c..a5c05d5ea --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@@ -59,8 -59,8 +59,8 @@@ class pkgDPkgPMPrivat { public: pkgDPkgPMPrivate() : stdin_is_dev_null(false), dpkgbuf_pos(0), - term_out(NULL), history_out(NULL), - progress(NULL), master(-1), slave(-1) + term_out(NULL), history_out(NULL), + progress(NULL), master(-1), slave(NULL) { dpkgbuf[0] = '\0'; } @@@ -77,9 -77,9 +77,9 @@@ APT::Progress::PackageManager *progress; // pty stuff - struct termios tt; + struct termios tt; int master; - int slave; + char * slave; // signals sigset_t sigmask; @@@ -510,14 -510,14 +510,14 @@@ bool pkgDPkgPM::RunScriptsWithPkgs(cons return result; } /*}}}*/ - // DPkgPM::DoStdin - Read stdin and pass to slave pty /*{{{*/ + // DPkgPM::DoStdin - Read stdin and pass to master pty /*{{{*/ // --------------------------------------------------------------------- /* */ void pkgDPkgPM::DoStdin(int master) { unsigned char input_buf[256] = {0,}; - ssize_t len = read(0, input_buf, sizeof(input_buf)); + ssize_t len = read(STDIN_FILENO, input_buf, sizeof(input_buf)); if (len) FileFd::Write(master, input_buf, len); else @@@ -564,8 -564,8 +564,8 @@@ void pkgDPkgPM::ProcessDpkgStatusLine(c 'status: : ' 'status: :: ' - 'processing: {install,configure,remove,purge,disappear,trigproc}: pkg' - 'processing: {install,configure,remove,purge,disappear,trigproc}: trigger' + 'processing: {install,upgrade,configure,remove,purge,disappear,trigproc}: pkg' + 'processing: {install,upgrade,configure,remove,purge,disappear,trigproc}: trigger' */ // we need to split on ": " (note the appended space) as the ':' is @@@ -589,12 -589,15 +589,15 @@@ std::string action; // "processing" has the form "processing: action: pkg or trigger" - // with action = ["install", "configure", "remove", "purge", "disappear", - // "trigproc"] + // with action = ["install", "upgrade", "configure", "remove", "purge", + // "disappear", "trigproc"] if (prefix == "processing") { pkgname = APT::String::Strip(list[2]); action = APT::String::Strip(list[1]); + // we don't care for the difference (as dpkg doesn't really either) + if (action == "upgrade") + action = "install"; } // "status" has the form: "status: pkg: state" // with state in ["half-installed", "unpacked", "half-configured", @@@ -638,27 -641,26 +641,26 @@@ // at this point we know that we should have a valid pkgname, so build all // the info from it - // dpkg does not send always send "pkgname:arch" so we add it here - // if needed + // dpkg does not always send "pkgname:arch" so we add it here if needed if (pkgname.find(":") == std::string::npos) { - // find the package in the group that is in a touched by dpkg - // if there are multiple dpkg will send us a full pkgname:arch + // find the package in the group that is touched by dpkg + // if there are multiple pkgs dpkg would send us a full pkgname:arch pkgCache::GrpIterator Grp = Cache.FindGrp(pkgname); - if (Grp.end() == false) + if (Grp.end() == false) { - pkgCache::PkgIterator P = Grp.PackageList(); - for (; P.end() != true; P = Grp.NextPkg(P)) - { - if(Cache[P].Mode != pkgDepCache::ModeKeep) - { - pkgname = P.FullName(); - break; - } - } + pkgCache::PkgIterator P = Grp.PackageList(); + for (; P.end() != true; P = Grp.NextPkg(P)) + { + if(Cache[P].Keep() == false || Cache[P].ReInstall() == true) + { + pkgname = P.FullName(); + break; + } + } } } - + const char* const pkg = pkgname.c_str(); std::string short_pkgname = StringSplit(pkgname, ":")[0]; std::string arch = ""; @@@ -697,28 -699,29 +699,29 @@@ if (prefix == "status") { vector const &states = PackageOps[pkg]; - const char *next_action = NULL; if(PackageOpsDone[pkg] < states.size()) - next_action = states[PackageOpsDone[pkg]].state; - // check if the package moved to the next dpkg state - if(next_action && (action == next_action)) { - // only read the translation if there is actually a next - // action - const char *translation = _(states[PackageOpsDone[pkg]].str); - std::string msg; - - // we moved from one dpkg state to a new one, report that - PackageOpsDone[pkg]++; - PackagesDone++; - - strprintf(msg, translation, i18n_pkgname.c_str()); - d->progress->StatusChanged(pkgname, PackagesDone, PackagesTotal, msg); - + char const * const next_action = states[PackageOpsDone[pkg]].state; + if (next_action && Debug == true) + std::clog << "(parsed from dpkg) pkg: " << short_pkgname + << " action: " << action << " (expected: '" << next_action << "' " + << PackageOpsDone[pkg] << " of " << states.size() << ")" << endl; + + // check if the package moved to the next dpkg state + if(next_action && (action == next_action)) + { + // only read the translation if there is actually a next action + char const * const translation = _(states[PackageOpsDone[pkg]].str); + + // we moved from one dpkg state to a new one, report that + ++PackageOpsDone[pkg]; + ++PackagesDone; + + std::string msg; + strprintf(msg, translation, i18n_pkgname.c_str()); + d->progress->StatusChanged(pkgname, PackagesDone, PackagesTotal, msg); + } } - if (Debug == true) - std::clog << "(parsed from dpkg) pkg: " << short_pkgname - << " action: " << action << endl; } } /*}}}*/ @@@ -1031,6 -1034,7 +1034,6 @@@ void pkgDPkgPM::BuildPackagesProgressMa } } /*}}}*/ -#if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR < 13) bool pkgDPkgPM::Go(int StatusFd) { APT::Progress::PackageManager *progress = NULL; @@@ -1039,67 -1043,135 +1042,134 @@@ else progress = new APT::Progress::PackageManagerProgressFd(StatusFd); - return GoNoABIBreak(progress); + return Go(progress); } -#endif void pkgDPkgPM::StartPtyMagic() { if (_config->FindB("Dpkg::Use-Pty", true) == false) { - d->master = d->slave = -1; + d->master = -1; + if (d->slave != NULL) + free(d->slave); + d->slave = NULL; return; } - // setup the pty and stuff - struct winsize win; - + _error->PushToStack(); // if tcgetattr for both stdin/stdout returns 0 (no error) // we do the pty magic - _error->PushToStack(); - if (tcgetattr(STDIN_FILENO, &d->tt) == 0 && - tcgetattr(STDOUT_FILENO, &d->tt) == 0) + if (tcgetattr(STDOUT_FILENO, &d->tt) == 0 && + tcgetattr(STDIN_FILENO, &d->tt) == 0) { - if (ioctl(STDOUT_FILENO, TIOCGWINSZ, (char *)&win) < 0) - { - _error->Errno("ioctl", _("ioctl(TIOCGWINSZ) failed")); - } else if (openpty(&d->master, &d->slave, NULL, &d->tt, &win) < 0) - { - _error->Errno("openpty", _("Can not write log (%s)"), _("Is /dev/pts mounted?")); - d->master = d->slave = -1; - } else { - struct termios rtt; - rtt = d->tt; - cfmakeraw(&rtt); - rtt.c_lflag &= ~ECHO; - rtt.c_lflag |= ISIG; + d->master = posix_openpt(O_RDWR | O_NOCTTY); + if (d->master == -1) + _error->Errno("posix_openpt", _("Can not write log (%s)"), _("Is /dev/pts mounted?")); + else if (unlockpt(d->master) == -1) + { + _error->Errno("unlockpt", "Unlocking the slave of master fd %d failed!", d->master); + close(d->master); + d->master = -1; + } + else + { + char const * const slave_name = ptsname(d->master); + if (slave_name == NULL) + { + _error->Errno("unlockpt", "Getting name for slave of master fd %d failed!", d->master); + close(d->master); + d->master = -1; + } + else + { + d->slave = strdup(slave_name); + if (d->slave == NULL) + { + _error->Errno("strdup", "Copying name %s for slave of master fd %d failed!", slave_name, d->master); + close(d->master); + d->master = -1; + } + struct winsize win; + if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &win) < 0) + _error->Errno("ioctl", "Getting TIOCGWINSZ from stdout failed!"); + if (ioctl(d->master, TIOCSWINSZ, &win) < 0) + _error->Errno("ioctl", "Setting TIOCSWINSZ for master fd %d failed!", d->master); + if (tcsetattr(d->master, TCSANOW, &d->tt) == -1) + _error->Errno("tcsetattr", "Setting in Start via TCSANOW for master fd %d failed!", d->master); + + struct termios raw_tt; + raw_tt = d->tt; + cfmakeraw(&raw_tt); + raw_tt.c_lflag &= ~ECHO; + raw_tt.c_lflag |= ISIG; // block SIGTTOU during tcsetattr to prevent a hang if // the process is a member of the background process group // http://www.opengroup.org/onlinepubs/000095399/functions/tcsetattr.html sigemptyset(&d->sigmask); sigaddset(&d->sigmask, SIGTTOU); sigprocmask(SIG_BLOCK,&d->sigmask, &d->original_sigmask); - tcsetattr(0, TCSAFLUSH, &rtt); - sigprocmask(SIG_SETMASK, &d->original_sigmask, 0); - } + if (tcsetattr(STDIN_FILENO, TCSAFLUSH, &raw_tt) == -1) + _error->Errno("tcsetattr", "Setting in Start via TCSAFLUSH for stdout failed!"); + sigprocmask(SIG_SETMASK, &d->original_sigmask, NULL); + } } - // complain only if stdout is either a terminal (but still failed) or is an invalid + } + else + { + // complain only if stdout is either a terminal (but still failed) or is an invalid // descriptor otherwise we would complain about redirection to e.g. /dev/null as well. - else if (isatty(STDOUT_FILENO) == 1 || errno == EBADF) - _error->Errno("tcgetattr", _("Can not write log (%s)"), _("Is stdout a terminal?")); + if (isatty(STDOUT_FILENO) == 1 || errno == EBADF) + _error->Errno("tcgetattr", _("Can not write log (%s)"), _("Is stdout a terminal?")); + } - if (_error->PendingError() == true) - _error->DumpErrors(std::cerr); - _error->RevertToStack(); + if (_error->PendingError() == true) + { + if (d->master != -1) + { + close(d->master); + d->master = -1; + } + _error->DumpErrors(std::cerr); + } + _error->RevertToStack(); } + void pkgDPkgPM::SetupSlavePtyMagic() + { + if(d->master == -1) + return; + + if (close(d->master) == -1) + _error->FatalE("close", "Closing master %d in child failed!", d->master); + if (setsid() == -1) + _error->FatalE("setsid", "Starting a new session for child failed!"); + int const slaveFd = open(d->slave, O_RDWR); + if (slaveFd == -1) + _error->FatalE("open", _("Can not write log (%s)"), _("Is /dev/pts mounted?")); + + if (ioctl(slaveFd, TIOCSCTTY, 0) < 0) + _error->FatalE("ioctl", "Setting TIOCSCTTY for slave fd %d failed!", slaveFd); + else + { + for (unsigned short i = 0; i < 3; ++i) + if (dup2(slaveFd, i) == -1) + _error->FatalE("dup2", "Dupping %d to %d in child failed!", slaveFd, i); + + if (tcsetattr(0, TCSANOW, &d->tt) < 0) + _error->FatalE("tcsetattr", "Setting in Setup via TCSANOW for slave fd %d failed!", slaveFd); + } + } void pkgDPkgPM::StopPtyMagic() { - if(d->slave > 0) - close(d->slave); + if (d->slave != NULL) + free(d->slave); + d->slave = NULL; if(d->master >= 0) { - tcsetattr(0, TCSAFLUSH, &d->tt); + if (tcsetattr(0, TCSAFLUSH, &d->tt) == -1) + _error->FatalE("tcsetattr", "Setting in Stop via TCSAFLUSH for stdin failed!"); close(d->master); + d->master = -1; } } @@@ -1112,7 -1184,11 +1182,7 @@@ * through to human readable (and i10n-able) * names and calculates a percentage for each step. */ -#if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR >= 13) bool pkgDPkgPM::Go(APT::Progress::PackageManager *progress) -#else -bool pkgDPkgPM::GoNoABIBreak(APT::Progress::PackageManager *progress) -#endif { pkgPackageManager::SigINTStop = false; d->progress = progress; @@@ -1407,22 -1483,8 +1477,8 @@@ pid_t Child = ExecFork(KeepFDs); if (Child == 0) { - // This is the child - if(d->slave >= 0 && d->master >= 0) - { - setsid(); - int res = ioctl(d->slave, TIOCSCTTY, 0); - if (res < 0) { - std::cerr << "ioctl(TIOCSCTTY) failed for fd: " - << d->slave << std::endl; - } else { - close(d->master); - dup2(d->slave, 0); - dup2(d->slave, 1); - dup2(d->slave, 2); - close(d->slave); - } - } + // This is the child + SetupSlavePtyMagic(); close(fd[0]); // close the read end of the pipe dpkgChrootDirectory(); diff --combined apt-pkg/deb/dpkgpm.h index e967bfe99,2c1805015..6bd6ce0ee --- a/apt-pkg/deb/dpkgpm.h +++ b/apt-pkg/deb/dpkgpm.h @@@ -110,6 -110,7 +110,7 @@@ class pkgDPkgPM : public pkgPackageMana // helper void BuildPackagesProgressMap(); void StartPtyMagic(); + void SetupSlavePtyMagic(); void StopPtyMagic(); // input processing @@@ -117,14 -118,27 +118,14 @@@ void DoTerminalPty(int master); void DoDpkgStatusFd(int statusfd); void ProcessDpkgStatusLine(char *line); -#if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR < 13) - void DoDpkgStatusFd(int statusfd, int /*unused*/) { - DoDpkgStatusFd(statusfd); - } - void ProcessDpkgStatusLine(int /*unused*/, char *line) { - ProcessDpkgStatusLine(line); - } -#endif - // The Actuall installation implementation virtual bool Install(PkgIterator Pkg,std::string File); virtual bool Configure(PkgIterator Pkg); virtual bool Remove(PkgIterator Pkg,bool Purge = false); -#if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR >= 13) virtual bool Go(APT::Progress::PackageManager *progress); -#else virtual bool Go(int StatusFd=-1); - bool GoNoABIBreak(APT::Progress::PackageManager *progress); -#endif virtual void Reset(); diff --combined apt-pkg/tagfile.cc index 7085e7d69,bf865bdc4..26c895417 --- a/apt-pkg/tagfile.cc +++ b/apt-pkg/tagfile.cc @@@ -47,43 -47,16 +47,43 @@@ public unsigned long long Size; }; +static unsigned long AlphaHash(const char *Text, size_t Length) /*{{{*/ +{ + /* This very simple hash function for the last 8 letters gives + very good performance on the debian package files */ + if (Length > 8) + { + Text += (Length - 8); + Length = 8; + } + unsigned long Res = 0; + for (size_t i = 0; i < Length; ++i) + Res = ((unsigned long)(Text[i]) & 0xDF) ^ (Res << 1); + return Res & 0xFF; +} + /*}}}*/ + // TagFile::pkgTagFile - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ pkgTagFile::pkgTagFile(FileFd *pFd,unsigned long long Size) + : d(NULL) +{ + Init(pFd, Size); +} + +void pkgTagFile::Init(FileFd *pFd,unsigned long long Size) { /* The size is increased by 4 because if we start with the Size of the filename we need to try to read 1 char more to see an EOF faster, 1 char the end-pointer can be on and maybe 2 newlines need to be added to the end of the file -> 4 extra chars */ Size += 4; + if(d != NULL) + { + free(d->Buffer); + delete d; + } d = new pkgTagFilePrivate(pFd, Size); if (d->Fd.IsOpen() == false) @@@ -155,23 -128,18 +155,23 @@@ bool pkgTagFile::Resize(unsigned long l */ bool pkgTagFile::Step(pkgTagSection &Tag) { - while (Tag.Scan(d->Start,d->End - d->Start) == false) + if(Tag.Scan(d->Start,d->End - d->Start) == false) { - if (Fill() == false) - return false; - - if(Tag.Scan(d->Start,d->End - d->Start)) - break; + do + { + if (Fill() == false) + return false; + + if(Tag.Scan(d->Start,d->End - d->Start, false)) + break; + + if (Resize() == false) + return _error->Error(_("Unable to parse package file %s (1)"), + d->Fd.Name().c_str()); - if (Resize() == false) - return _error->Error(_("Unable to parse package file %s (1)"), - d->Fd.Name().c_str()); + } while (Tag.Scan(d->Start,d->End - d->Start, false) == false); } + d->Start += Tag.size(); d->iOffset += Tag.size(); @@@ -265,7 -233,7 +265,7 @@@ bool pkgTagFile::Jump(pkgTagSection &Ta if (Fill() == false) return false; - if (Tag.Scan(d->Start, d->End - d->Start) == false) + if (Tag.Scan(d->Start, d->End - d->Start, false) == false) return _error->Error(_("Unable to parse package file %s (2)"),d->Fd.Name().c_str()); return true; @@@ -275,46 -243,27 +275,46 @@@ // --------------------------------------------------------------------- /* */ pkgTagSection::pkgTagSection() - : Section(0), TagCount(0), d(NULL), Stop(0) + : Section(0), d(NULL), Stop(0) { - memset(&Indexes, 0, sizeof(Indexes)); - memset(&AlphaIndexes, 0, sizeof(AlphaIndexes)); + memset(&LookupTable, 0, sizeof(LookupTable)); } /*}}}*/ // TagSection::Scan - Scan for the end of the header information /*{{{*/ -// --------------------------------------------------------------------- -/* This looks for the first double new line in the data stream. - It also indexes the tags in the section. */ -bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength) +bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength, bool const Restart) { + Section = Start; const char *End = Start + MaxLength; - Stop = Section = Start; - memset(AlphaIndexes,0,sizeof(AlphaIndexes)); + + if (Restart == false && Tags.empty() == false) + { + Stop = Section + Tags.back().StartTag; + if (End <= Stop) + return false; + Stop = (const char *)memchr(Stop,'\n',End - Stop); + if (Stop == NULL) + return false; + ++Stop; + } + else + { + Stop = Section; + if (Tags.empty() == false) + { + memset(&LookupTable, 0, sizeof(LookupTable)); + Tags.clear(); + } + Tags.reserve(0x100); + } + size_t TagCount = Tags.size(); if (Stop == 0) return false; - TagCount = 0; - while (TagCount+1 < sizeof(Indexes)/sizeof(Indexes[0]) && Stop < End) + TagData lastTagData(0); + lastTagData.EndTag = 0; + unsigned long lastTagHash = 0; + while (Stop < End) { TrimRecord(true,End); @@@ -326,39 -275,12 +326,39 @@@ // Start a new index and add it to the hash if (isspace(Stop[0]) == 0) { - Indexes[TagCount++] = Stop - Section; - AlphaIndexes[AlphaHash(Stop,End)] = TagCount; + // store the last found tag + if (lastTagData.EndTag != 0) + { + if (LookupTable[lastTagHash] != 0) + lastTagData.NextInBucket = LookupTable[lastTagHash]; + LookupTable[lastTagHash] = TagCount; + Tags.push_back(lastTagData); + } + + ++TagCount; + lastTagData = TagData(Stop - Section); + // find the colon separating tag and value + char const * Colon = (char const *) memchr(Stop, ':', End - Stop); + if (Colon == NULL) + return false; + // find the end of the tag (which might or might not be the colon) + char const * EndTag = Colon; + --EndTag; + for (; EndTag > Stop && isspace(*EndTag) != 0; --EndTag) + ; + ++EndTag; + lastTagData.EndTag = EndTag - Section; + lastTagHash = AlphaHash(Stop, EndTag - Stop); + // find the beginning of the value + Stop = Colon + 1; + for (; isspace(*Stop) != 0; ++Stop); + if (Stop >= End) + return false; + lastTagData.StartValue = Stop - Section; } Stop = (const char *)memchr(Stop,'\n',End - Stop); - + if (Stop == 0) return false; @@@ -369,16 -291,7 +369,16 @@@ // Double newline marks the end of the record if (Stop+1 < End && Stop[1] == '\n') { - Indexes[TagCount] = Stop - Section; + if (lastTagData.EndTag != 0) + { + if (LookupTable[lastTagHash] != 0) + lastTagData.NextInBucket = LookupTable[lastTagHash]; + LookupTable[lastTagHash] = TagCount; + Tags.push_back(lastTagData); + } + + TagData const td(Stop - Section); + Tags.push_back(td); TrimRecord(false,End); return true; } @@@ -407,8 -320,8 +407,8 @@@ void pkgTagSection::Trim( for (; Stop > Section + 2 && (Stop[-2] == '\n' || Stop[-2] == '\r'); Stop--); } /*}}}*/ -// TagSection::Exists - return True if a tag exists /*{{{*/ -bool pkgTagSection::Exists(const char* const Tag) +// TagSection::Exists - return True if a tag exists /*{{{*/ +bool pkgTagSection::Exists(const char* const Tag) const { unsigned int tmp; return Find(Tag, tmp); @@@ -419,43 -332,73 +419,43 @@@ /* This searches the section for a tag that matches the given string. */ bool pkgTagSection::Find(const char *Tag,unsigned int &Pos) const { - unsigned int Length = strlen(Tag); - unsigned int I = AlphaIndexes[AlphaHash(Tag)]; - if (I == 0) + size_t const Length = strlen(Tag); + unsigned int Bucket = LookupTable[AlphaHash(Tag, Length)]; + if (Bucket == 0) return false; - I--; - - for (unsigned int Counter = 0; Counter != TagCount; Counter++, - I = (I+1)%TagCount) + + for (; Bucket != 0; Bucket = Tags[Bucket - 1].NextInBucket) { - const char *St; - St = Section + Indexes[I]; - if (strncasecmp(Tag,St,Length) != 0) + if ((Tags[Bucket - 1].EndTag - Tags[Bucket - 1].StartTag) != Length) continue; - // Make sure the colon is in the right place - const char *C = St + Length; - for (; isspace(*C) != 0; C++); - if (*C != ':') + char const * const St = Section + Tags[Bucket - 1].StartTag; + if (strncasecmp(Tag,St,Length) != 0) continue; - Pos = I; + + Pos = Bucket - 1; return true; } Pos = 0; return false; } - /*}}}*/ -// TagSection::Find - Locate a tag /*{{{*/ -// --------------------------------------------------------------------- -/* This searches the section for a tag that matches the given string. */ bool pkgTagSection::Find(const char *Tag,const char *&Start, const char *&End) const { - unsigned int Length = strlen(Tag); - unsigned int I = AlphaIndexes[AlphaHash(Tag)]; - if (I == 0) + unsigned int Pos; + if (Find(Tag, Pos) == false) return false; - I--; - - for (unsigned int Counter = 0; Counter != TagCount; Counter++, - I = (I+1)%TagCount) - { - const char *St; - St = Section + Indexes[I]; - if (strncasecmp(Tag,St,Length) != 0) - continue; - - // Make sure the colon is in the right place - const char *C = St + Length; - for (; isspace(*C) != 0; C++); - if (*C != ':') - continue; - // Strip off the gunk from the start end - Start = C; - End = Section + Indexes[I+1]; - if (Start >= End) - return _error->Error("Internal parsing error"); - - for (; (isspace(*Start) != 0 || *Start == ':') && Start < End; Start++); - for (; isspace(End[-1]) != 0 && End > Start; End--); - - return true; - } - - Start = End = 0; - return false; + Start = Section + Tags[Pos].StartValue; + // Strip off the gunk from the end + End = Section + Tags[Pos + 1].StartTag; + if (unlikely(Start > End)) + return _error->Error("Internal parsing error"); + + for (; isspace(End[-1]) != 0 && End > Start; --End); + + return true; } /*}}}*/ // TagSection::FindS - Find a string /*{{{*/ @@@ -518,17 -461,6 +518,17 @@@ unsigned long long pkgTagSection::FindU return Result; } /*}}}*/ +// TagSection::FindB - Find boolean value /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgTagSection::FindB(const char *Tag, bool const &Default) const +{ + const char *Start, *Stop; + if (Find(Tag, Start, Stop) == false) + return Default; + return StringToBool(string(Start, Stop)); +} + /*}}}*/ // TagSection::FindFlag - Locate a yes/no type flag /*{{{*/ // --------------------------------------------------------------------- /* The bits marked in Flag are masked on/off in Flags */ @@@ -561,13 -493,6 +561,13 @@@ bool pkgTagSection::FindFlag(unsigned l return true; } /*}}}*/ +APT_PURE unsigned int pkgTagSection::Count() const { /*{{{*/ + if (Tags.empty() == true) + return 0; + // the last element is just marking the end and isn't a real one + return Tags.size() - 1; +} + /*}}}*/ // TFRewrite - Rewrite a control record /*{{{*/ // --------------------------------------------------------------------- /* This writes the control record to stdout rewriting it as necessary. The @@@ -601,7 -526,7 +601,7 @@@ static const char *iTFRewritePackageOrd "Conffiles", "Filename", "Size", - "MD5Sum", + "MD5sum", "SHA1", "SHA256", "SHA512", diff --combined apt-private/private-list.cc index e85aaf64c,536348640..aa3a2c24b --- a/apt-private/private-list.cc +++ b/apt-private/private-list.cc @@@ -37,20 -37,28 +37,20 @@@ struct PackageSortAlphabetic /*{{{ return (l_name < r_name); } }; - /*}}}*/ -class PackageNameMatcher : public Matcher /*{{{*/ + +class PackageNameMatcher : public Matcher { -#ifdef PACKAGE_MATCHER_ABI_COMPAT -#define PackageMatcher PackageNameMatchesFnmatch -#endif public: PackageNameMatcher(const char **patterns) { for(int i=0; patterns[i] != NULL; ++i) { std::string pattern = patterns[i]; -#ifdef PACKAGE_MATCHER_ABI_COMPAT - APT::CacheFilter::PackageNameMatchesFnmatch *cachefilter = NULL; - cachefilter = new APT::CacheFilter::PackageNameMatchesFnmatch(pattern); -#else APT::CacheFilter::PackageMatcher *cachefilter = NULL; if(_config->FindB("APT::Cmd::Use-Regexp", false) == true) cachefilter = new APT::CacheFilter::PackageNameMatchesRegEx(pattern); else cachefilter = new APT::CacheFilter::PackageNameMatchesFnmatch(pattern); -#endif filters.push_back(cachefilter); } } @@@ -77,15 -85,14 +77,14 @@@ private }; /*}}}*/ static void ListAllVersions(pkgCacheFile &CacheFile, pkgRecords &records,/*{{{*/ - pkgCache::PkgIterator P, - std::ostream &outs, - bool include_summary=true) + pkgCache::PkgIterator const &P, std::ostream &outs, + std::string const &format) { for (pkgCache::VerIterator Ver = P.VersionList(); Ver.end() == false; ++Ver) { - ListSingleVersion(CacheFile, records, Ver, outs, include_summary); - outs << "\n"; + ListSingleVersion(CacheFile, records, Ver, outs, format); + outs << std::endl; } } /*}}}*/ @@@ -109,10 -116,9 +108,9 @@@ bool DoList(CommandLine &Cmd patterns = Cmd.FileList + 1; } - std::map output_map; - std::map::const_iterator K; - - bool includeSummary = _config->FindB("APT::Cmd::List-Include-Summary"); + std::string format = "${color:highlight}${Package}${color:neutral}/${Origin} ${Version} ${Architecture}${ }${apt:Status}"; + if (_config->FindB("APT::Cmd::List-Include-Summary", false) == true) + format += "\n ${Description}\n"; PackageNameMatcher matcher(patterns); LocalitySortedVersionSet bag; @@@ -121,37 -127,34 +119,34 @@@ Cache->Head().PackageCount, Cache->Head().PackageCount, _("Listing")); - GetLocalitySortedVersionSet(CacheFile, bag, matcher, progress); - bool ShowAllVersions = _config->FindB("APT::Cmd::All-Versions", false); + GetLocalitySortedVersionSet(CacheFile, &bag, matcher, &progress); + bool const ShowAllVersions = _config->FindB("APT::Cmd::All-Versions", false); + std::map output_map; for (LocalitySortedVersionSet::iterator V = bag.begin(); V != bag.end(); ++V) { std::stringstream outs; if(ShowAllVersions == true) - { - ListAllVersions(CacheFile, records, V.ParentPkg(), outs, includeSummary); - output_map.insert(std::make_pair( - V.ParentPkg().Name(), outs.str())); - } else { - ListSingleVersion(CacheFile, records, V, outs, includeSummary); - output_map.insert(std::make_pair( - V.ParentPkg().Name(), outs.str())); - } + ListAllVersions(CacheFile, records, V.ParentPkg(), outs, format); + else + ListSingleVersion(CacheFile, records, V, outs, format); + output_map.insert(std::make_pair( + V.ParentPkg().Name(), outs.str())); } // FIXME: SORT! and make sorting flexible (alphabetic, by pkg status) // output the sorted map + std::map::const_iterator K; for (K = output_map.begin(); K != output_map.end(); ++K) std::cout << (*K).second << std::endl; - // be nice and tell the user if there is more to see if (bag.size() == 1 && ShowAllVersions == false) { // start with -1 as we already displayed one version int versions = -1; pkgCache::VerIterator Ver = *bag.begin(); - for ( ; Ver.end() == false; Ver++) - versions++; + for ( ; Ver.end() == false; ++Ver) + ++versions; if (versions > 0) _error->Notice(P_("There is %i additional version. Please use the '-a' switch to see it", "There are %i additional versions. Please use the '-a' switch to see them.", versions), versions); } diff --combined configure.ac index 356c40614,2e591cdf2..89950fccd --- a/configure.ac +++ b/configure.ac @@@ -18,7 -18,7 +18,7 @@@ AC_CONFIG_AUX_DIR(buildlib AC_CONFIG_HEADER(include/config.h:buildlib/config.h.in include/apti18n.h:buildlib/apti18n.h.in) PACKAGE="apt" - PACKAGE_VERSION="1.1~exp2" -PACKAGE_VERSION="1.0.9.1" ++PACKAGE_VERSION="1.1~exp3" PACKAGE_MAIL="APT Development Team " AC_DEFINE_UNQUOTED(PACKAGE,"$PACKAGE") AC_DEFINE_UNQUOTED(PACKAGE_VERSION,"$PACKAGE_VERSION") diff --combined debian/changelog index 5b885ad6c,36353bd95..32447d5e1 --- a/debian/changelog +++ b/debian/changelog @@@ -1,73 -1,81 +1,151 @@@ +apt (1.1~exp2) experimental; urgency=medium + + [ Guillem Jover ] + * Add new Base256ToNum long long overload function + * Fix ar and tar code to be LFS-safe (Closes: #742882) + + [ Michael Vogt ] + * increase libapt-inst to version 1.6 + * Only allow "apt-get build-dep path" when path starts with ./ or / + * Allow passing a full path to apt-get install /foo/bar.deb (CLoses: #752327) + * merge changes from the 1.0.6 upload + + -- Michael Vogt Thu, 10 Jul 2014 13:18:08 +0200 + +apt (1.1~exp1) experimental; urgency=low + + [ David Kalnischkies ] + * [API Break] change "std::string pkgAcquire::Item::DescURI()" to + "std::string pkgAcquire::Item::DescURI() const" + * [ABI-Break] increase hashtable size for packages/groups by factor 5 + * [ABI-Break] cleanup datatypes mix used in binary cache + * [internal API-Break] remove the Section member from package struct + * use 'best' hash for source authentication (LP: 1098738) + * use HashStringList in the acquire system + * deal with hashes in ftparchive more dynamic as well + * reenable pipelining via hashsum reordering support + * parse and retrieve multiple Descriptions in one record + * improve pkgTagSection scanning and parsing + * invalid cache if architecture set doesn't match (Closes: 745036) + + [ Michael Vogt ] + * add support for "apt-get build-dep foo.dsc" + * add support for "apt-get build-dep unpacked-source-dir" + * add support for "apt-get install foo_1.0_all.deb" + * make "apt-get update" progress much more accurate by loading the + sizes of the targets into the fetcher early + * Implement simple by-hash for apt update to improve reliability of + the update. Apt will try to fetch the Packages file via + /by-hash/$hash_type/$hash_value if the repo supports that. + - add APT::Acquire::$(host)::By-Hash=1 knob + - add Acquire-By-Hash=1 to Release file + * add Debug::Acquire::Progress debug option + * [ABI-Break] lp:~mvo/apt/source-hashes: + - use sha{512,256,1} for deb-src when available LP: #1098738 + * [ABI-Break] stop exporting the accidently exported parsenetrc() symbol + * [ABI-Break] remove the PACKAGE_MATCHER_ABI_COMPAT defines + * [ABI BREAK] apt-pkg/pkgcache.h: + - adjust pkgCache::State::VerPriority enum, to match reality + * test/integration/test-debsrc-hashes: + - add integration test, thanks to Daniel Hartwig + * [ABI-Break] remove the PACKAGE_MATCHER_ABI_COMPAT defines + * [ABI-Break] Pass struct IndexTarget/indexRecords to + pkgAcqIndex{,Merge}Diffs + * [internal API-Break] rename pkgCache::Package::NextPackage to + pkgCache::Package::Next + * Calculate Percent as part of pkgAcquireStatus to provide a weighted + percent for both items and bytes + * apt-pkg/contrib/macros.h: bump library version to 4.13 + * apt-private/acqprogress.cc: do not show file size on IMSHit, it wasn't + fetched + * Fix warnings from clang -Wall/clang -fsanitize=address + * add DropPrivs() and drop privileges to nobody when running the + the buildin apt and dump solvers + * lp:~mvo/apt/webserver-simulate-broken-with-fix346386: + - fix invalid InRelease file download checking and add regression + test to server broken files to the buildin test webserver + - add regression test for LP: #34638 + + -- Michael Vogt Thu, 19 Jun 2014 12:01:48 +0200 + + apt (1.0.9.1) unstable; urgency=high + + [ Michael Vogt ] + * Allow override of Proxy-Auto-Detect by the users configuration + (Closes: 759264) + * fix ci autopkgtest + * fix regression from 1.0.9 when file:/// source are used and + those are on a different partition than the apt state directory + and add regression test + + [ Trần Ngọc Quân ] + * l10n: vi.po (636t): Update program translation + + [ Chris Leick ] + * Updated German documentation translation + + [ Mert Dirik ] + * Turkish program translation update (Closes: 761394) + + -- Michael Vogt Tue, 16 Sep 2014 20:52:25 +0200 + + apt (1.0.9) unstable; urgency=high + + * SECURITY UPDATE: + - incorrect invalidating of unauthenticated data (CVE-2014-0488) + - incorect verification of 304 reply (CVE-2014-0487) + - incorrect verification of Acquire::Gzip indexes (CVE-2014-0489) + + -- Michael Vogt Mon, 15 Sep 2014 08:34:46 +0200 + + apt (1.0.8) unstable; urgency=medium + + [ Holger Wansing ] + * German program translation update (Closes: 758837) + + [ Américo Monteiro ] + * Portuguese manpages translation update (Closes: 759608) + + [ Warren He ] + * initialize iPolicyBrokenCount in DepCache::Update (Closes: 758397) + + [ Andreas Oberritter ] + * Avoid yielding blank lines with APT::Cmd::use-format=true + + [ Michael Vogt ] + * Make Proxy-Auto-Detect check for each host (Closes: #759264) + * Add testcase for apt list --all-versions + * * apt-pkg/deb/dpkgpm.cc: + - update string matching for dpkg I/O errors. (LP: #1363257) + - properly parse the dpkg status line so that package name + is properly set and an apport report is created. Thanks + to Anders Kaseorg for the patch (LP: #1353171) + * Use heap to allocate PatternMatch to avoid potential stack overflow + (Closes: 759612) + * Run autopkgtest tests with "env -i" to avoid pollution from the host env + (Closes: #759655) + * test/integration/test-ubuntu-bug-346386-apt-get-update-paywall: + - use downloadfile() to fix test failure + * Fix incorrect upgradable listing in "apt list" + (thanks to Michael Musenbrock) (Closes: #753297) + * apt-pkg/cachefile.cc: + - ensure we have a Policy in CacheFile.BuildDepCache() + * methods/http.cc: + - Improve Debug::Acquire::http debug output + + [ Dimitri John Ledkov ] + * apt-ftparchive: make Packages & Sources generation optional, + during Generate call + + [ David Kalnischkies ] + * support regular expressions in 'apt search' + * implement --full in apt search + * fix progress report for upgrade and reinstall + * rework PTY magic to fix stair-stepping on kfreebsd (Closes: 759684) + * don't call pager in non-terminals for changelog (Closes: 755040) + + -- Michael Vogt Tue, 09 Sep 2014 20:09:11 +0200 + apt (1.0.7) unstable; urgency=medium [ Michael Vogt ] diff --combined debian/gbp.conf index 4991e409e,ec6d9894e..135522d40 --- a/debian/gbp.conf +++ b/debian/gbp.conf @@@ -1,5 -1,7 +1,7 @@@ [DEFAULT] prebuild = ./prepare-release pre-export postbuild = ./prepare-release post-build -debian-branch = debian/sid +debian-branch = debian/experimental debian-tag = %(version)s + export-dir = ../build-area + sign-tags = True diff --combined methods/server.cc index c91d3b218,92d94e638..ff67eb09b --- a/methods/server.cc +++ b/methods/server.cc @@@ -44,7 -44,8 +44,8 @@@ time_t ServerMethod::FailTime = 0 // --------------------------------------------------------------------- /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header parse error occurred */ - ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File) + ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File, + const std::string &Uri) { State = Header; @@@ -66,7 -67,7 +67,7 @@@ continue; if (Owner->Debug == true) - clog << Data; + clog << "Answer for: " << Uri << endl << Data; for (string::const_iterator I = Data.begin(); I < Data.end(); ++I) { @@@ -392,16 -393,9 +393,16 @@@ bool ServerMethod::Fetch(FetchItem * for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++) { - // If pipelining is disabled, we only queue 1 request - if (Server->Pipeline == false && Depth >= 0) - break; + if (Depth >= 0) + { + // If pipelining is disabled, we only queue 1 request + if (Server->Pipeline == false) + break; + // if we have no hashes, do at most one such request + // as we can't fixup pipeling misbehaviors otherwise + else if (I->ExpectedHashes.usable() == false) + break; + } // Make sure we stick with the same server if (Server->Comp(I->Uri) == false) @@@ -485,7 -479,7 +486,7 @@@ int ServerMethod::Loop( Fetch(0); // Fetch the next URL header data from the server. - switch (Server->RunHeaders(File)) + switch (Server->RunHeaders(File, Queue->Uri)) { case ServerState::RUN_HEADERS_OK: break; @@@ -553,38 -547,7 +554,38 @@@ // Send status to APT if (Result == true) { - Res.TakeHashes(*Server->GetHashes()); + Hashes * const resultHashes = Server->GetHashes(); + HashStringList const hashList = resultHashes->GetHashStringList(); + if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList) + { + // we did not get the expected hash… mhhh: + // could it be that server/proxy messed up pipelining? + FetchItem * BeforeI = Queue; + for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next) + { + if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList) + { + // yes, he did! Disable pipelining and rewrite queue + if (Server->Pipeline == true) + { + // FIXME: fake a warning message as we have no proper way of communicating here + std::string out; + strprintf(out, _("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::PipelineDepth"); + std::cerr << "W: " << out << std::endl; + Server->Pipeline = false; + // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well + } + Rename(Res.Filename, I->DestFile); + Res.Filename = I->DestFile; + BeforeI->Next = I->Next; + I->Next = Queue; + Queue = I; + break; + } + BeforeI = I; + } + } + Res.TakeHashes(*resultHashes); URIDone(Res); } else diff --combined methods/server.h index 5299b3954,f5e68d902..aa692ea93 --- a/methods/server.h +++ b/methods/server.h @@@ -68,7 -68,7 +68,7 @@@ struct ServerStat RUN_HEADERS_PARSE_ERROR }; /** \brief Get the headers before the data */ - RunHeadersResult RunHeaders(FileFd * const File); + RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri); bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; @@@ -140,7 -140,7 +140,7 @@@ class ServerMethod : public pkgAcqMetho virtual ServerState * CreateServerState(URI uri) = 0; virtual void RotateDNS() = 0; - ServerMethod(const char *Ver,unsigned long Flags = 0) : pkgAcqMethod(Ver, Flags), Server(NULL), File(NULL), PipelineDepth(0), AllowRedirect(false), Debug(false) {}; + ServerMethod(const char *Ver,unsigned long Flags = 0) : pkgAcqMethod(Ver, Flags), Server(NULL), File(NULL), PipelineDepth(10), AllowRedirect(false), Debug(false) {}; virtual ~ServerMethod() {}; }; diff --combined test/integration/framework index 8f1e7cca5,ff010a5c4..7cf4d8a6d --- a/test/integration/framework +++ b/test/integration/framework @@@ -4,7 -4,7 +4,7 @@@ EXIT_CODE= # we all like colorful messages if [ "$MSGCOLOR" != 'NO' ]; then - if ! expr match "$(readlink -f /proc/$$/fd/1)" '/dev/pts/[0-9]\+' > /dev/null; then + if [ ! -t 1 ]; then # but check that we output to a terminal export MSGCOLOR='NO' fi fi @@@ -102,10 -102,10 +102,10 @@@ runapt() local CMD="$1" shift case $CMD in - sh|aptitude|*/*) ;; + sh|aptitude|*/*|command) ;; *) CMD="${BUILDDIRECTORY}/$CMD";; esac - MALLOC_PERTURB_=21 MALLOC_CHECK_=2 APT_CONFIG="$(getaptconfig)" LD_LIBRARY_PATH=${BUILDDIRECTORY} $CMD "$@" + MALLOC_PERTURB_=21 MALLOC_CHECK_=2 APT_CONFIG="$(getaptconfig)" LD_LIBRARY_PATH=${LIBRARYPATH} $CMD "$@" } aptconfig() { runapt apt-config "$@"; } aptcache() { runapt apt-cache "$@"; } @@@ -129,9 -129,11 +129,9 @@@ dpkgcheckbuilddeps() command dpkg-checkbuilddeps --admindir=${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg "$@" } gdb() { - echo "gdb: run »$*«" - CMD="$1" + local CMD="$1" shift - - APT_CONFIG=aptconfig.conf LD_LIBRARY_PATH=${LIBRARYPATH} command gdb ${BUILDDIRECTORY}/$CMD --args ${BUILDDIRECTORY}/$CMD "$@" + runapt command gdb --quiet -ex run "${BUILDDIRECTORY}/$CMD" --args "${BUILDDIRECTORY}/$CMD" "$@" } gpg() { # see apt-key for the whole trickery. Setup is done in setupenvironment @@@ -233,7 -235,6 +233,7 @@@ setupenvironment() fi echo "DPKG::options:: \"--log=${TMPWORKINGDIRECTORY}/rootdir/var/log/dpkg.log\";" >> aptconfig.conf echo 'quiet::NoUpdate "true";' >> aptconfig.conf + echo 'quiet::NoStatistic "true";' >> aptconfig.conf echo "Acquire::https::CaInfo \"${TESTDIR}/apt.pem\";" > rootdir/etc/apt/apt.conf.d/99https echo "Apt::Cmd::Disable-Script-Warning \"1\";" > rootdir/etc/apt/apt.conf.d/apt-binary configcompression '.' 'gz' #'bz2' 'lzma' 'xz' @@@ -252,8 -253,7 +252,8 @@@ gpg --quiet --check-trustdb --secret-keyring $SECRETKEYRING --keyring $SECRETKEYRING >/dev/null 2>&1 # cleanup the environment a bit - export PATH="${PATH}:/usr/local/sbin:/usr/sbin:/sbin" + # prefer our apt binaries over the system apt binaries + export PATH="${BUILDDIRECTORY}:${PATH}:/usr/local/sbin:/usr/sbin:/sbin" export LC_ALL=C.UTF-8 unset LANGUAGE APT_CONFIG unset GREP_OPTIONS DEB_BUILD_PROFILES @@@ -1019,7 -1019,13 +1019,13 @@@ testfileequal() testempty() { msgtest "Test for no output of" "$*" - test -z "$($* 2>&1)" && msgpass || msgfail + local COMPAREFILE="${TMPWORKINGDIRECTORY}/rootdir/tmp/testempty.comparefile" + if $* >$COMPAREFILE 2>&1 && test ! -s $COMPAREFILE; then + msgpass + else + cat $COMPAREFILE + msgfail + fi } testequal() { diff --combined test/integration/test-apt-progress-fd index 7ddf355f3,d72e7e72d..c147c4517 --- a/test/integration/test-apt-progress-fd +++ b/test/integration/test-apt-progress-fd @@@ -16,7 -16,7 +16,7 @@@ setupaptarchiv exec 3> apt-progress.log testsuccess aptget install testing=0.1 -y -o APT::Status-Fd=3 testequal "dlstatus:1:0:Retrieving file 1 of 1 -dlstatus:1:0:Retrieving file 1 of 1 +dlstatus:1:20:Retrieving file 1 of 1 pmstatus:dpkg-exec:0:Running dpkg pmstatus:testing:0:Installing testing (amd64) pmstatus:testing:20:Preparing testing (amd64) @@@ -31,8 -31,24 +31,24 @@@ pmstatus:testing:100:Installed testing exec 3> apt-progress.log testsuccess aptget install testing=0.8.15 -y -o APT::Status-Fd=3 testequal "dlstatus:1:0:Retrieving file 1 of 1 -dlstatus:1:0:Retrieving file 1 of 1 +dlstatus:1:20:Retrieving file 1 of 1 pmstatus:dpkg-exec:0:Running dpkg + pmstatus:testing:0:Installing testing (amd64) + pmstatus:testing:20:Preparing testing (amd64) + pmstatus:testing:40:Unpacking testing (amd64) + pmstatus:testing:60:Preparing to configure testing (amd64) + pmstatus:dpkg-exec:60:Running dpkg + pmstatus:testing:60:Configuring testing (amd64) + pmstatus:testing:80:Configuring testing (amd64) + pmstatus:testing:100:Installed testing (amd64)" cat apt-progress.log + + # reinstall + exec 3> apt-progress.log + testsuccess aptget install testing=0.8.15 --reinstall -y -o APT::Status-Fd=3 + testequal "dlstatus:1:0:Retrieving file 1 of 1 + dlstatus:1:0:Retrieving file 1 of 1 + pmstatus:dpkg-exec:0:Running dpkg + pmstatus:testing:0:Installing testing (amd64) pmstatus:testing:20:Preparing testing (amd64) pmstatus:testing:40:Unpacking testing (amd64) pmstatus:testing:60:Preparing to configure testing (amd64) @@@ -56,7 -72,7 +72,7 @@@ testsuccess aptget install testing2:i38 # and compare testequal "dlstatus:1:0:Retrieving file 1 of 1 -dlstatus:1:0:Retrieving file 1 of 1 +dlstatus:1:20:Retrieving file 1 of 1 pmstatus:dpkg-exec:0:Running dpkg pmstatus:testing2:0:Installing testing2 (i386) pmstatus:testing2:20:Preparing testing2 (i386) diff --combined test/libapt/strutil_test.cc index 1c2f0abac,194c9c074..8dd9114ec --- a/test/libapt/strutil_test.cc +++ b/test/libapt/strutil_test.cc @@@ -19,6 -19,21 +19,21 @@@ TEST(StrUtilTest,DeEscapeString EXPECT_EQ("foo\\ x", DeEscapeString("foo\\\\ x")); EXPECT_EQ("\\foo\\", DeEscapeString("\\\\foo\\\\")); } + TEST(StrUtilTest,StringStrip) + { + EXPECT_EQ("", APT::String::Strip("")); + EXPECT_EQ("foobar", APT::String::Strip("foobar")); + EXPECT_EQ("foo bar", APT::String::Strip("foo bar")); + + EXPECT_EQ("", APT::String::Strip(" ")); + EXPECT_EQ("", APT::String::Strip(" \r\n \t ")); + + EXPECT_EQ("foo bar", APT::String::Strip("foo bar ")); + EXPECT_EQ("foo bar", APT::String::Strip("foo bar \r\n \t ")); + EXPECT_EQ("foo bar", APT::String::Strip("\r\n \t foo bar")); + EXPECT_EQ("bar foo", APT::String::Strip("\r\n \t bar foo \r\n \t ")); + EXPECT_EQ("bar \t\r\n foo", APT::String::Strip("\r\n \t bar \t\r\n foo \r\n \t ")); + } TEST(StrUtilTest,StringSplitBasic) { std::vector result = StringSplit("", ""); @@@ -70,15 -85,6 +85,15 @@@ TEST(StrUtilTest,EndsWith EXPECT_FALSE(Endswith("abcd", "x")); EXPECT_FALSE(Endswith("abcd", "abcndefg")); } +TEST(StrUtilTest,StartWith) +{ + using APT::String::Startswith; + EXPECT_TRUE(Startswith("abcd", "a")); + EXPECT_TRUE(Startswith("abcd", "ab")); + EXPECT_TRUE(Startswith("abcd", "abcd")); + EXPECT_FALSE(Startswith("abcd", "x")); + EXPECT_FALSE(Startswith("abcd", "abcndefg")); +} TEST(StrUtilTest,SubstVar) { EXPECT_EQ("", SubstVar("", "fails", "passes"));