From: Michael Vogt Date: Mon, 29 Sep 2014 06:33:33 +0000 (+0200) Subject: Merge branch 'debian/sid' into debian/experimental X-Git-Tag: 1.1.exp4~23 X-Git-Url: https://git.saurik.com/apt.git/commitdiff_plain/48f3f1b24b48066cfc23342a5c6470ddf5e0fa6a?ds=inline;hp=-c Merge branch 'debian/sid' into debian/experimental Conflicts: apt-pkg/acquire-item.cc --- 48f3f1b24b48066cfc23342a5c6470ddf5e0fa6a diff --combined apt-pkg/acquire-item.cc index d1c1f8c5a,2401364a9..f46c8a6e4 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@@ -50,35 -50,16 +50,35 @@@ using namespace std; +static void printHashSumComparision(std::string const &URI, HashStringList const &Expected, HashStringList const &Actual) /*{{{*/ +{ + if (_config->FindB("Debug::Acquire::HashSumMismatch", false) == false) + return; + std::cerr << std::endl << URI << ":" << std::endl << " Expected Hash: " << std::endl; + for (HashStringList::const_iterator hs = Expected.begin(); hs != Expected.end(); ++hs) + std::cerr << "\t- " << hs->toStr() << std::endl; + std::cerr << " Actual Hash: " << std::endl; + for (HashStringList::const_iterator hs = Actual.begin(); hs != Actual.end(); ++hs) + std::cerr << "\t- " << hs->toStr() << std::endl; +} + /*}}}*/ + // Acquire::Item::Item - Constructor /*{{{*/ -// --------------------------------------------------------------------- -/* */ -pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0), - PartialSize(0), Mode(0), ID(0), Complete(false), - Local(false), QueueCounter(0) +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif +pkgAcquire::Item::Item(pkgAcquire *Owner, HashStringList const &ExpectedHashes) : + Owner(Owner), FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false), + Local(false), QueueCounter(0), ExpectedAdditionalItems(0), + ExpectedHashes(ExpectedHashes) { Owner->Add(this); Status = StatIdle; } +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif /*}}}*/ // Acquire::Item::~Item - Destructor /*{{{*/ // --------------------------------------------------------------------- @@@ -136,7 -117,7 +136,7 @@@ void pkgAcquire::Item::Start(string /*M // Acquire::Item::Done - Item downloaded OK /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcquire::Item::Done(string Message,unsigned long long Size,string /*Hash*/, +void pkgAcquire::Item::Done(string Message,unsigned long long Size,HashStringList const &/*Hash*/, pkgAcquire::MethodConfig * /*Cnf*/) { // We just downloaded something.. @@@ -247,8 -228,8 +247,8 @@@ void pkgAcquire::Item::ReportMirrorFail possibly query additional files */ pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI, string const &URIDesc, string const &ShortDesc, - HashString const &ExpectedHash) - : Item(Owner), ExpectedHash(ExpectedHash) + HashStringList const &ExpectedHashes) + : Item(Owner, ExpectedHashes) { /* XXX: Beware: Currently this class does nothing (of value) anymore ! */ Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false); @@@ -270,7 -251,7 +270,7 @@@ // AcqSubIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqSubIndex::Custom600Headers() +string pkgAcqSubIndex::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(Desc.URI); @@@ -293,7 -274,7 +293,7 @@@ void pkgAcqSubIndex::Failed(string Mess // No good Index is provided } /*}}}*/ -void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ +void pkgAcqSubIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ pkgAcquire::MethodConfig *Cnf) { if(Debug) @@@ -315,7 -296,7 +315,7 @@@ return; } - Item::Done(Message,Size,Md5Hash,Cnf); + Item::Done(Message, Size, Hashes, Cnf); string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI); @@@ -361,19 -342,18 +361,19 @@@ bool pkgAcqSubIndex::ParseIndex(string * the original packages file */ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, - string URI,string URIDesc,string ShortDesc, - HashString ExpectedHash) - : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash), - Description(URIDesc) + IndexTarget const * const Target, + HashStringList const &ExpectedHashes, + indexRecords *MetaIndexParser) + : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser) { Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); - Desc.Description = URIDesc + "/DiffIndex"; + RealURI = Target->URI; Desc.Owner = this; - Desc.ShortDesc = ShortDesc; - Desc.URI = URI + ".diff/Index"; + Desc.Description = Target->Description + "/DiffIndex"; + Desc.ShortDesc = Target->ShortDesc; + Desc.URI = Target->URI + ".diff/Index"; DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(Desc.URI); @@@ -409,7 -389,7 +409,7 @@@ // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqDiffIndex::Custom600Headers() +string pkgAcqDiffIndex::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(Desc.URI); @@@ -462,8 -442,8 +462,8 @@@ bool pkgAcqDiffIndex::ParseDiffIndex(st std::clog << "Package file is up-to-date" << std::endl; // list cleanup needs to know that this file as well as the already // present index is ours, so we create an empty diff to save it for us - new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, - ExpectedHash, ServerSha1, available_patches); + new pkgAcqIndexDiffs(Owner, Target, ExpectedHashes, MetaIndexParser, + ServerSha1, available_patches); return true; } else @@@ -548,19 -528,14 +548,19 @@@ } if (pdiff_merge == false) - new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, - ExpectedHash, ServerSha1, available_patches); - else + { + new pkgAcqIndexDiffs(Owner, Target, ExpectedHashes, MetaIndexParser, + ServerSha1, available_patches); + } + else { std::vector *diffs = new std::vector(available_patches.size()); for(size_t i = 0; i < available_patches.size(); ++i) - (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedHash, - available_patches[i], diffs); + (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, Target, + ExpectedHashes, + MetaIndexParser, + available_patches[i], + diffs); } Complete = false; @@@ -584,20 -559,21 +584,20 @@@ void pkgAcqDiffIndex::Failed(string Mes std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc, - ExpectedHash); + new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser); Complete = false; Status = StatDone; Dequeue(); } /*}}}*/ -void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ +void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ pkgAcquire::MethodConfig *Cnf) { if(Debug) std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl; - Item::Done(Message,Size,Md5Hash,Cnf); + Item::Done(Message, Size, Hashes, Cnf); string FinalFile; FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); @@@ -627,24 -603,22 +627,24 @@@ * for each diff and the index */ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, - string URI,string URIDesc,string ShortDesc, - HashString ExpectedHash, + struct IndexTarget const * const Target, + HashStringList const &ExpectedHashes, + indexRecords *MetaIndexParser, string ServerSha1, vector diffs) - : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash), + : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser), available_patches(diffs), ServerSha1(ServerSha1) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(URI); + DestFile += URItoFileName(Target->URI); Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); - Description = URIDesc; + RealURI = Target->URI; Desc.Owner = this; - Desc.ShortDesc = ShortDesc; + Description = Target->Description; + Desc.ShortDesc = Target->ShortDesc; if(available_patches.empty() == true) { @@@ -664,7 -638,8 +664,7 @@@ void pkgAcqIndexDiffs::Failed(string Me if(Debug) std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc, - ExpectedHash); + new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser); Finish(); } /*}}}*/ @@@ -678,7 -653,7 +678,7 @@@ void pkgAcqIndexDiffs::Finish(bool allD DestFile = _config->FindDir("Dir::State::lists"); DestFile += URItoFileName(RealURI); - if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile)) + if(HashSums().usable() && !HashSums().VerifyFile(DestFile)) { RenameOnError(HashSumMismatch); Dequeue(); @@@ -756,13 -731,13 +756,13 @@@ bool pkgAcqIndexDiffs::QueueNextDiff( return true; } /*}}}*/ -void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ +void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringList const &Hashes, /*{{{*/ pkgAcquire::MethodConfig *Cnf) { if(Debug) std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl; - Item::Done(Message,Size,Md5Hash,Cnf); + Item::Done(Message, Size, Hashes, Cnf); string FinalFile; FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); @@@ -781,15 -756,7 +781,15 @@@ Local = true; Desc.URI = "rred:" + FinalFile; QueueURI(Desc); + ActiveSubprocess = "rred"; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif Mode = "rred"; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif return; } @@@ -812,9 -779,8 +812,9 @@@ // see if there is more to download if(available_patches.empty() == false) { - new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, - ExpectedHash, ServerSha1, available_patches); + new pkgAcqIndexDiffs(Owner, Target, + ExpectedHashes, MetaIndexParser, + ServerSha1, available_patches); return Finish(); } else return Finish(true); @@@ -823,24 -789,22 +823,24 @@@ /*}}}*/ // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, - string const &URI, string const &URIDesc, - string const &ShortDesc, HashString const &ExpectedHash, - DiffInfo const &patch, - std::vector const * const allPatches) - : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash), - patch(patch),allPatches(allPatches), State(StateFetchDiff) + struct IndexTarget const * const Target, + HashStringList const &ExpectedHashes, + indexRecords *MetaIndexParser, + DiffInfo const &patch, + std::vector const * const allPatches) + : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser), + patch(patch), allPatches(allPatches), State(StateFetchDiff) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(URI); + DestFile += URItoFileName(Target->URI); Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); - Description = URIDesc; + RealURI = Target->URI; Desc.Owner = this; - Desc.ShortDesc = ShortDesc; + Description = Target->Description; + Desc.ShortDesc = Target->ShortDesc; Desc.URI = RealURI + ".diff/" + patch.file + ".gz"; Desc.Description = Description + " " + patch.file + string(".pdiff"); @@@ -871,16 -835,17 +871,16 @@@ void pkgAcqIndexMergeDiffs::Failed(stri // first failure means we should fallback State = StateErrorDiff; std::clog << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc, - ExpectedHash); + new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser); } /*}}}*/ -void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ +void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ pkgAcquire::MethodConfig *Cnf) { if(Debug) std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl; - Item::Done(Message,Size,Md5Hash,Cnf); + Item::Done(Message,Size,Hashes,Cnf); string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); @@@ -909,22 -874,14 +909,22 @@@ Local = true; Desc.URI = "rred:" + FinalFile; QueueURI(Desc); + ActiveSubprocess = "rred"; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif Mode = "rred"; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif return; } // success in download/apply all diffs, clean up else if (State == StateApplyDiff) { // see if we really got the expected file - if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile)) + if(ExpectedHashes.usable() && !ExpectedHashes.VerifyFile(DestFile)) { RenameOnError(HashSumMismatch); return; @@@ -961,8 -918,8 +961,8 @@@ instantiated to fetch the revision file */ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, - HashString ExpectedHash, string comprExt) - : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash) + HashStringList const &ExpectedHash, string comprExt) + : pkgAcqBaseIndex(Owner, NULL, ExpectedHash, NULL), RealURI(URI) { if(comprExt.empty() == true) { @@@ -978,15 -935,13 +978,15 @@@ Init(URI, URIDesc, ShortDesc); } pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target, - HashString const &ExpectedHash, indexRecords const *MetaIndexParser) - : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash) + HashStringList const &ExpectedHash, + indexRecords *MetaIndexParser) + : pkgAcqBaseIndex(Owner, Target, ExpectedHash, MetaIndexParser), + RealURI(Target->URI) { // autoselect the compression method std::vector types = APT::Configuration::getCompressionTypes(); CompressionExtension = ""; - if (ExpectedHash.empty() == false) + if (ExpectedHashes.usable()) { for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true) @@@ -1012,28 -967,11 +1012,29 @@@ void pkgAcqIndex::Init(string const &UR DestFile += URItoFileName(URI); std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); + std::string MetaKey; if (comprExt == "uncompressed") + { Desc.URI = URI; - else { + if(Target) + MetaKey = string(Target->MetaKey); + } + else + { Desc.URI = URI + '.' + comprExt; + DestFile = DestFile + '.' + comprExt; + if(Target) + MetaKey = string(Target->MetaKey) + '.' + comprExt; + } + + // load the filesize + if(MetaIndexParser) + { + indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); + if(Record) + FileSize = Record->Size; + + InitByHashIfNeeded(MetaKey); } Desc.Description = URIDesc; @@@ -1043,50 -981,22 +1044,51 @@@ QueueURI(Desc); } /*}}}*/ +// AcqIndex::AdjustForByHash - modify URI for by-hash support /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqIndex::InitByHashIfNeeded(const std::string MetaKey) +{ + // TODO: + // - (maybe?) add support for by-hash into the sources.list as flag + // - make apt-ftparchive generate the hashes (and expire?) + std::string HostKnob = "APT::Acquire::" + ::URI(Desc.URI).Host + "::By-Hash"; + if(_config->FindB("APT::Acquire::By-Hash", false) == true || + _config->FindB(HostKnob, false) == true || + MetaIndexParser->GetSupportsAcquireByHash()) + { + indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); + if(Record) + { + // FIXME: should we really use the best hash here? or a fixed one? + const HashString *TargetHash = Record->Hashes.find(""); + std::string ByHash = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue(); + size_t trailing_slash = Desc.URI.find_last_of("/"); + Desc.URI = Desc.URI.replace( + trailing_slash, + Desc.URI.substr(trailing_slash+1).size()+1, + ByHash); + } else { + _error->Warning( + "Fetching ByHash requested but can not find record for %s", + MetaKey.c_str()); + } + } +} + /*}}}*/ // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqIndex::Custom600Headers() +string pkgAcqIndex::Custom600Headers() const { + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); if (_config->FindB("Acquire::GzipIndexes",false)) - Final += ".gz"; + Final += compExt; string msg = "\nIndex-File: true"; - // FIXME: this really should use "IndexTarget::IsOptional()" but that - // seems to be difficult without breaking ABI - if (ShortDesc().find("Translation") != 0) - msg += "\nFail-Ignore: true"; + struct stat Buf; if (stat(Final.c_str(),&Buf) == 0) msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); @@@ -1120,8 -1030,8 +1122,8 @@@ std::string pkgAcqIndex::GetFinalFilena { std::string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(URI); - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") - FinalFile += ".gz"; + if (_config->FindB("Acquire::GzipIndexes",false) == true) + FinalFile += '.' + compExt; return FinalFile; } /*}}}*/ @@@ -1129,8 -1039,8 +1131,8 @@@ void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName) { std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") - DestFile += ".gz"; + if (_config->FindB("Acquire::GzipIndexes",false) == true) + DestFile += compExt; string FinalFile = GetFinalFilename(RealURI, compExt); Rename(FinalFile, FileName); @@@ -1146,19 -1056,24 +1148,19 @@@ to the uncompressed version of the file. If this is so the file is copied into the partial directory. In all other cases the file is decompressed with a gzip uri. */ -void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, +void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, pkgAcquire::MethodConfig *Cfg) { - Item::Done(Message,Size,Hash,Cfg); + Item::Done(Message,Size,Hashes,Cfg); std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); if (Decompression == true) { - if (_config->FindB("Debug::pkgAcquire::Auth", false)) - { - std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash; - std::cerr << " Expected Hash: " << ExpectedHash.toStr() << std::endl; - } - - if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash) + if (ExpectedHashes.usable() && ExpectedHashes != Hashes) { Desc.URI = RealURI; RenameOnError(HashSumMismatch); + printHashSumComparision(RealURI, ExpectedHashes, Hashes); return; } @@@ -1168,7 -1083,7 +1170,7 @@@ /* Always verify the index file for correctness (all indexes must * have a Package field) (LP: #346386) (Closes: #627642) */ - FileFd fd(DestFile, FileFd::ReadOnlyGzip); + FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Extension); // Only test for correctness if the file is not empty (empty is ok) if (fd.Size() > 0) { @@@ -1187,18 -1102,17 +1189,18 @@@ string FinalFile = GetFinalFilename(RealURI, compExt); Rename(DestFile,FinalFile); chmod(FinalFile.c_str(),0644); - + /* We restore the original name to DestFile so that the clean operation will work OK */ DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(RealURI); - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") - DestFile += ".gz"; + if (_config->FindB("Acquire::GzipIndexes",false)) + DestFile += '.' + compExt; // Remove the compressed version. if (Erase == true) unlink(DestFile.c_str()); + return; } @@@ -1214,15 -1128,7 +1216,15 @@@ DestFile += ".decomp"; Desc.URI = "copy:" + FileName; QueueURI(Desc); + ActiveSubprocess = "copy"; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif Mode = "copy"; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif return; } @@@ -1249,16 -1155,23 +1251,23 @@@ // matching the Release file if (!Local && StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) { + // set destfile to the final destfile + if(_config->FindB("Acquire::GzipIndexes",false) == false) + { + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI); + } + ReverifyAfterIMS(FileName); return; } string decompProg; // If we enable compressed indexes, queue for hash verification - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) + if (_config->FindB("Acquire::GzipIndexes",false)) { DestFile = _config->FindDir("Dir::State::lists"); - DestFile += URItoFileName(RealURI) + ".gz"; + DestFile += URItoFileName(RealURI) + '.' + compExt; Decompression = true; Desc.URI = "copy:" + FileName; @@@ -1282,15 -1195,8 +1291,15 @@@ Desc.URI = decompProg + ":" + FileName; QueueURI(Desc); - // FIXME: this points to a c++ string that goes out of scope - Mode = decompProg.c_str(); + ActiveSubprocess = decompProg; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + Mode = ActiveSubprocess.c_str(); +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif } /*}}}*/ // AcqIndexTrans::pkgAcqIndexTrans - Constructor /*{{{*/ @@@ -1298,28 -1204,24 +1307,28 @@@ /* The Translation file is added to the queue */ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc) - : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "") + : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashStringList(), "") { } -pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target, - HashString const &ExpectedHash, indexRecords const *MetaIndexParser) - : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser) +pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const * const Target, + HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) + : pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser) { + // load the filesize + indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey)); + if(Record) + FileSize = Record->Size; } /*}}}*/ // AcqIndexTrans::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- -string pkgAcqIndexTrans::Custom600Headers() +string pkgAcqIndexTrans::Custom600Headers() const { + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); - if (_config->FindB("Acquire::GzipIndexes",false)) - Final += ".gz"; + Final += compExt; struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) @@@ -1360,7 -1262,7 +1369,7 @@@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquir string MetaIndexShortDesc, const vector* IndexTargets, indexRecords* MetaIndexParser) : - Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI), + Item(Owner, HashStringList()), RealURI(URI), MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets) { @@@ -1391,9 -1293,6 +1400,9 @@@ Rename(Final,LastGoodSig); } + // we expect the indextargets + one additional Release file + ExpectedAdditionalItems = IndexTargets->size() + 1; + QueueURI(Desc); } /*}}}*/ @@@ -1413,7 -1312,7 +1422,7 @@@ pkgAcqMetaSig::~pkgAcqMetaSig() /* // pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqMetaSig::Custom600Headers() +string pkgAcqMetaSig::Custom600Headers() const { struct stat Buf; if (stat(LastGoodSig.c_str(),&Buf) != 0) @@@ -1422,10 -1321,10 +1431,10 @@@ return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } -void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5, +void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cfg) { - Item::Done(Message,Size,MD5,Cfg); + Item::Done(Message, Size, Hashes, Cfg); string FileName = LookupTag(Message,"Filename"); if (FileName.empty() == true) @@@ -1446,9 -1345,6 +1455,9 @@@ Complete = true; + // at this point pkgAcqMetaIndex takes over + ExpectedAdditionalItems = 0; + // put the last known good file back on i-m-s hit (it will // be re-verified again) // Else do nothing, we have the new file in DestFile then @@@ -1466,9 -1362,6 +1475,9 @@@ void pkgAcqMetaSig::Failed(string Messa { string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); + // at this point pkgAcqMetaIndex takes over + ExpectedAdditionalItems = 0; + // if we get a network error we fail gracefully if(Status == StatTransientNetworkError) { @@@ -1505,9 -1398,9 +1514,9 @@@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ string URI,string URIDesc,string ShortDesc, string SigFile, - const vector* IndexTargets, + const vector* IndexTargets, indexRecords* MetaIndexParser) : - Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets), + Item(Owner, HashStringList()), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets), MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; @@@ -1519,16 -1412,13 +1528,16 @@@ Desc.ShortDesc = ShortDesc; Desc.URI = URI; + // we expect more item + ExpectedAdditionalItems = IndexTargets->size(); + QueueURI(Desc); } /*}}}*/ // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqMetaIndex::Custom600Headers() +string pkgAcqMetaIndex::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); @@@ -1540,10 -1430,10 +1549,10 @@@ return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } /*}}}*/ -void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash, /*{{{*/ +void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ pkgAcquire::MethodConfig *Cfg) { - Item::Done(Message,Size,Hash,Cfg); + Item::Done(Message,Size,Hashes,Cfg); // MetaIndexes are done in two passes: one to download the // metaindex with an appropriate method, and a second to verify it @@@ -1572,35 -1462,16 +1581,35 @@@ } else { + // FIXME: move this into pkgAcqMetaClearSig::Done on the next + // ABI break + + // if we expect a ClearTextSignature (InRelase), ensure that + // this is what we get and if not fail to queue a + // Release/Release.gpg, see #346386 + if (SigFile == DestFile && !StartsWithGPGClearTextSignature(DestFile)) + { + Failed(Message, Cfg); + return; + } + // There was a signature file, so pass it to gpgv for // verification - if (_config->FindB("Debug::pkgAcquire::Auth", false)) std::cerr << "Metaindex acquired, queueing gpg verification (" << SigFile << "," << DestFile << ")\n"; AuthPass = true; Desc.URI = "gpgv:" + SigFile; QueueURI(Desc); - Mode = "gpgv"; + ActiveSubprocess = "gpgv"; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + Mode = "gpgv"; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif return; } } @@@ -1697,8 -1568,12 +1706,12 @@@ void pkgAcqMetaIndex::AuthDone(string M URItoFileName((*Target)->URI); unlink(index.c_str()); // and also old gzipindexes - index += ".gz"; - unlink(index.c_str()); + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + index += '.' + (*t); + unlink(index.c_str()); + } } } @@@ -1741,13 -1616,11 +1754,13 @@@ void pkgAcqMetaIndex::QueueIndexes(boo } } - for (vector ::const_iterator Target = IndexTargets->begin(); + // at this point the real Items are loaded in the fetcher + ExpectedAdditionalItems = 0; + for (vector ::const_iterator Target = IndexTargets->begin(); Target != IndexTargets->end(); ++Target) { - HashString ExpectedIndexHash; + HashStringList ExpectedIndexHashes; const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey); bool compressedAvailable = false; if (Record == NULL) @@@ -1771,16 -1644,14 +1784,16 @@@ } else { - ExpectedIndexHash = Record->Hash; + ExpectedIndexHashes = Record->Hashes; if (_config->FindB("Debug::pkgAcquire::Auth", false)) { - std::cerr << "Queueing: " << (*Target)->URI << std::endl; - std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl; + std::cerr << "Queueing: " << (*Target)->URI << std::endl + << "Expected Hash:" << std::endl; + for (HashStringList::const_iterator hs = ExpectedIndexHashes.begin(); hs != ExpectedIndexHashes.end(); ++hs) + std::cerr << "\t- " << hs->toStr() << std::endl; std::cerr << "For: " << Record->MetaKeyFilename << std::endl; } - if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false) + if (verify == true && ExpectedIndexHashes.empty() == true && (*Target)->IsOptional() == false) { Status = StatAuthError; strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str()); @@@ -1792,14 -1663,15 +1805,14 @@@ { if ((*Target)->IsSubIndex() == true) new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, ExpectedIndexHash); + (*Target)->ShortDesc, ExpectedIndexHashes); else if (transInRelease == false || Record != NULL || compressedAvailable == true) { if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true && MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true) - new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, ExpectedIndexHash); + new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); else - new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser); + new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); } continue; } @@@ -1810,9 -1682,10 +1823,9 @@@ instead, but passing the required info to it is to much hassle */ if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false || MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)) - new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, ExpectedIndexHash); + new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); else - new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser); + new pkgAcqIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); } } /*}}}*/ @@@ -1965,7 -1838,7 +1978,7 @@@ pkgAcqMetaClearSig::pkgAcqMetaClearSig( string const &URI, string const &URIDesc, string const &ShortDesc, string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc, string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc, - const vector* IndexTargets, + const vector* IndexTargets, indexRecords* MetaIndexParser) : pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser), MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), @@@ -1973,10 -1846,6 +1986,10 @@@ { SigFile = DestFile; + // index targets + (worst case:) Release/Release.gpg + ExpectedAdditionalItems = IndexTargets->size() + 2; + + // keep the old InRelease around in case of transistent network errors string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); if (RealFileExists(Final) == true) @@@ -2001,7 -1870,7 +2014,7 @@@ pkgAcqMetaClearSig::~pkgAcqMetaClearSig // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- // FIXME: this can go away once the InRelease file is used widely -string pkgAcqMetaClearSig::Custom600Headers() +string pkgAcqMetaClearSig::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); @@@ -2019,9 -1888,6 +2032,9 @@@ /*}}}*/ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ { + // we failed, we will not get additional items from this method + ExpectedAdditionalItems = 0; + if (AuthPass == false) { // Remove the 'old' InRelease file if we try Release.gpg now as otherwise @@@ -2050,7 -1916,7 +2063,7 @@@ pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources, pkgRecords *Recs,pkgCache::VerIterator const &Version, string &StoreFilename) : - Item(Owner), Version(Version), Sources(Sources), Recs(Recs), + Item(Owner, HashStringList()), Version(Version), Sources(Sources), Recs(Recs), StoreFilename(StoreFilename), Vf(Version.FileList()), Trusted(false) { @@@ -2135,6 -2001,7 +2148,6 @@@ checking later. */ bool pkgAcqArchive::QueueNext() { - string const ForceHash = _config->Find("Acquire::ForceHash"); for (; Vf.end() == false; ++Vf) { // Ignore not source sources @@@ -2155,10 -2022,31 +2168,10 @@@ pkgRecords::Parser &Parse = Recs->Lookup(Vf); if (_error->PendingError() == true) return false; - + string PkgFile = Parse.FileName(); - if (ForceHash.empty() == false) - { - if(stringcasecmp(ForceHash, "sha512") == 0) - ExpectedHash = HashString("SHA512", Parse.SHA512Hash()); - else if(stringcasecmp(ForceHash, "sha256") == 0) - ExpectedHash = HashString("SHA256", Parse.SHA256Hash()); - else if (stringcasecmp(ForceHash, "sha1") == 0) - ExpectedHash = HashString("SHA1", Parse.SHA1Hash()); - else - ExpectedHash = HashString("MD5Sum", Parse.MD5Hash()); - } - else - { - string Hash; - if ((Hash = Parse.SHA512Hash()).empty() == false) - ExpectedHash = HashString("SHA512", Hash); - else if ((Hash = Parse.SHA256Hash()).empty() == false) - ExpectedHash = HashString("SHA256", Hash); - else if ((Hash = Parse.SHA1Hash()).empty() == false) - ExpectedHash = HashString("SHA1", Hash); - else - ExpectedHash = HashString("MD5Sum", Parse.MD5Hash()); - } + ExpectedHashes = Parse.Hashes(); + if (PkgFile.empty() == true) return _error->Error(_("The package index files are corrupted. No Filename: " "field for package %s."), @@@ -2245,10 -2133,10 +2258,10 @@@ // AcqArchive::Done - Finished fetching /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash, +void pkgAcqArchive::Done(string Message,unsigned long long Size, HashStringList const &CalcHashes, pkgAcquire::MethodConfig *Cfg) { - Item::Done(Message,Size,CalcHash,Cfg); + Item::Done(Message, Size, CalcHashes, Cfg); // Check the size if (Size != Version->Size) @@@ -2256,12 -2144,11 +2269,12 @@@ RenameOnError(SizeMismatch); return; } - - // Check the hash - if(ExpectedHash.toStr() != CalcHash) + + // FIXME: could this empty() check impose *any* sort of security issue? + if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes) { RenameOnError(HashSumMismatch); + printHashSumComparision(DestFile, ExpectedHashes, CalcHashes); return; } @@@ -2333,7 -2220,7 +2346,7 @@@ void pkgAcqArchive::Failed(string Messa /*}}}*/ // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/ // --------------------------------------------------------------------- -APT_PURE bool pkgAcqArchive::IsTrusted() +APT_PURE bool pkgAcqArchive::IsTrusted() const { return Trusted; } @@@ -2352,11 -2239,11 +2365,11 @@@ void pkgAcqArchive::Finished( // AcqFile::pkgAcqFile - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The file is added to the queue */ -pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash, +pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI, HashStringList const &Hashes, unsigned long long Size,string Dsc,string ShortDesc, const string &DestDir, const string &DestFilename, bool IsIndexFile) : - Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile) + Item(Owner, Hashes), IsIndexFile(IsIndexFile) { Retries = _config->FindI("Acquire::Retries",0); @@@ -2393,16 -2280,15 +2406,16 @@@ // AcqFile::Done - Item downloaded OK /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash, +void pkgAcqFile::Done(string Message,unsigned long long Size,HashStringList const &CalcHashes, pkgAcquire::MethodConfig *Cnf) { - Item::Done(Message,Size,CalcHash,Cnf); + Item::Done(Message,Size,CalcHashes,Cnf); // Check the hash - if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash) + if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes) { RenameOnError(HashSumMismatch); + printHashSumComparision(DestFile, ExpectedHashes, CalcHashes); return; } @@@ -2473,7 -2359,7 +2486,7 @@@ void pkgAcqFile::Failed(string Message, // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqFile::Custom600Headers() +string pkgAcqFile::Custom600Headers() const { if (IsIndexFile) return "\nIndex-File: true"; diff --combined apt-pkg/deb/debindexfile.cc index c35a89ade,5b4289e92..f90731dd2 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@@ -30,7 -30,6 +30,7 @@@ #include #include #include +#include #include #include @@@ -81,14 -80,18 +81,18 @@@ pkgSrcRecords::Parser *debSourcesIndex: { string SourcesURI = _config->FindDir("Dir::State::lists") + URItoFileName(IndexURI("Sources")); - string SourcesURIgzip = SourcesURI + ".gz"; - if (!FileExists(SourcesURI) && !FileExists(SourcesURIgzip)) - return NULL; - else if (!FileExists(SourcesURI) && FileExists(SourcesURIgzip)) - SourcesURI = SourcesURIgzip; - - return new debSrcRecordParser(SourcesURI,this); + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + string p; + p = SourcesURI + '.' + *t; + if (FileExists(p)) + return new debSrcRecordParser(p, this); + } + if (FileExists(SourcesURI)) + return new debSrcRecordParser(SourcesURI, this); + return NULL; } /*}}}*/ // SourcesIndex::Describe - Give a descriptive path to the index /*{{{*/ @@@ -130,11 -133,15 +134,15 @@@ string debSourcesIndex::Info(const cha inline string debSourcesIndex::IndexFile(const char *Type) const { string s = URItoFileName(IndexURI(Type)); - string sgzip = s + ".gz"; - if (!FileExists(s) && FileExists(sgzip)) - return sgzip; - else - return s; + + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + string p = s + '.' + *t; + if (FileExists(p)) + return p; + } + return s; } string debSourcesIndex::IndexURI(const char *Type) const @@@ -260,11 -267,15 +268,15 @@@ string debPackagesIndex::Info(const cha inline string debPackagesIndex::IndexFile(const char *Type) const { string s =_config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type)); - string sgzip = s + ".gz"; - if (!FileExists(s) && FileExists(sgzip)) - return sgzip; - else - return s; + + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + string p = s + '.' + *t; + if (FileExists(p)) + return p; + } + return s; } string debPackagesIndex::IndexURI(const char *Type) const { @@@ -412,11 -423,15 +424,15 @@@ debTranslationsIndex::debTranslationsIn inline string debTranslationsIndex::IndexFile(const char *Type) const { string s =_config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type)); - string sgzip = s + ".gz"; - if (!FileExists(s) && FileExists(sgzip)) - return sgzip; - else - return s; + + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + string p = s + '.' + *t; + if (FileExists(p)) + return p; + } + return s; } string debTranslationsIndex::IndexURI(const char *Type) const { @@@ -619,7 -634,7 +635,7 @@@ bool debStatusIndex::Merge(pkgCacheGene pkgCache::PkgFileIterator CFile = Gen.GetCurFile(); CFile->Size = Pkg.FileSize(); CFile->mtime = Pkg.ModificationTime(); - map_ptrloc const storage = Gen.WriteUniqString("now"); + map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::MIXED, "now"); CFile->Archive = storage; if (Gen.MergeList(Parser) == false) @@@ -668,143 -683,6 +684,143 @@@ APT_CONST bool debStatusIndex::Exists( } /*}}}*/ +// debDebPkgFile - Single .deb file /*{{{*/ +// --------------------------------------------------------------------- +debDebPkgFileIndex::debDebPkgFileIndex(std::string DebFile) + : pkgIndexFile(true), DebFile(DebFile) +{ + DebFileFullPath = flAbsPath(DebFile); +} + +std::string debDebPkgFileIndex::ArchiveURI(std::string /*File*/) const +{ + return "file:" + DebFileFullPath; +} + +bool debDebPkgFileIndex::Exists() const +{ + return FileExists(DebFile); +} +bool debDebPkgFileIndex::Merge(pkgCacheGenerator& Gen, OpProgress* Prog) const +{ + if(Prog) + Prog->SubProgress(0, "Reading deb file"); + + // get the control data out of the deb file vid dpkg -I + // ... can I haz libdpkg? + Configuration::Item const *Opts = _config->Tree("DPkg::Options"); + std::string dpkg = _config->Find("Dir::Bin::dpkg","dpkg"); + std::vector Args; + Args.push_back(dpkg.c_str()); + if (Opts != 0) + { + Opts = Opts->Child; + for (; Opts != 0; Opts = Opts->Next) + { + if (Opts->Value.empty() == true) + continue; + Args.push_back(Opts->Value.c_str()); + } + } + Args.push_back("-I"); + Args.push_back(DebFile.c_str()); + Args.push_back("control"); + Args.push_back(NULL); + FileFd PipeFd; + pid_t Child; + if(Popen((const char**)&Args[0], PipeFd, Child, FileFd::ReadOnly) == false) + return _error->Error("Popen failed"); + // FIXME: static buffer + char buf[8*1024]; + unsigned long long n = 0; + if(PipeFd.Read(buf, sizeof(buf)-1, &n) == false) + return _error->Errno("read", "Failed to read dpkg pipe"); + ExecWait(Child, "Popen"); + + // now write the control data to a tempfile + SPtr DebControl = GetTempFile("deb-file-" + flNotDir(DebFile)); + if(DebControl == NULL) + return false; + DebControl->Write(buf, n); + // append size of the file + FileFd Fd(DebFile, FileFd::ReadOnly); + string Size; + strprintf(Size, "Size: %llu\n", Fd.Size()); + DebControl->Write(Size.c_str(), Size.size()); + // and rewind for the listparser + DebControl->Seek(0); + + // and give it to the list parser + debDebFileParser Parser(DebControl, DebFile); + if(Gen.SelectFile(DebFile, "local", *this) == false) + return _error->Error("Problem with SelectFile %s", DebFile.c_str()); + + pkgCache::PkgFileIterator File = Gen.GetCurFile(); + File->Size = DebControl->Size(); + File->mtime = DebControl->ModificationTime(); + + if (Gen.MergeList(Parser) == false) + return _error->Error("Problem with MergeLister for %s", DebFile.c_str()); + + return true; +} +pkgCache::PkgFileIterator debDebPkgFileIndex::FindInCache(pkgCache &Cache) const +{ + pkgCache::PkgFileIterator File = Cache.FileBegin(); + for (; File.end() == false; ++File) + { + if (File.FileName() == NULL || DebFile != File.FileName()) + continue; + + return File; + } + + return File; +} +unsigned long debDebPkgFileIndex::Size() const +{ + struct stat buf; + if(stat(DebFile.c_str(), &buf) != 0) + return 0; + return buf.st_size; +} + /*}}}*/ + +// debDscFileIndex stuff +debDscFileIndex::debDscFileIndex(std::string &DscFile) + : pkgIndexFile(true), DscFile(DscFile) +{ +} + +bool debDscFileIndex::Exists() const +{ + return FileExists(DscFile); +} + +unsigned long debDscFileIndex::Size() const +{ + struct stat buf; + if(stat(DscFile.c_str(), &buf) == 0) + return buf.st_size; + return 0; +} + +// DscFileIndex::CreateSrcParser - Get a parser for the .dsc file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgSrcRecords::Parser *debDscFileIndex::CreateSrcParser() const +{ + if (!FileExists(DscFile)) + return NULL; + + return new debDscRecordParser(DscFile,this); +} + /*}}}*/ + + + + +// --------------------------------------------------------------------- // Index File types for Debian /*{{{*/ class debIFTypeSrc : public pkgIndexFile::Type { @@@ -837,42 -715,10 +853,42 @@@ class debIFTypeStatus : public pkgIndex }; debIFTypeStatus() {Label = "Debian dpkg status file";}; }; +class debIFTypeDebPkgFile : public pkgIndexFile::Type +{ + public: + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const + { + return new debDebFileRecordParser(File.FileName(),*File.Cache()); + }; + debIFTypeDebPkgFile() {Label = "deb Package file";}; +}; +class debIFTypeDscFile : public pkgIndexFile::Type +{ + public: + virtual pkgSrcRecords::Parser *CreateSrcPkgParser(std::string DscFile) const + { + return new debDscRecordParser(DscFile, NULL); + }; + debIFTypeDscFile() {Label = "dsc File Source Index";}; +}; +class debIFTypeDebianSourceDir : public pkgIndexFile::Type +{ + public: + virtual pkgSrcRecords::Parser *CreateSrcPkgParser(std::string SourceDir) const + { + return new debDscRecordParser(SourceDir + string("/debian/control"), NULL); + }; + debIFTypeDebianSourceDir() {Label = "debian/control File Source Index";}; +}; + static debIFTypeSrc _apt_Src; static debIFTypePkg _apt_Pkg; static debIFTypeTrans _apt_Trans; static debIFTypeStatus _apt_Status; +static debIFTypeDebPkgFile _apt_DebPkgFile; +// file based pseudo indexes +static debIFTypeDscFile _apt_DscFile; +static debIFTypeDebianSourceDir _apt_DebianSourceDir; const pkgIndexFile::Type *debSourcesIndex::GetType() const { @@@ -890,16 -736,5 +906,16 @@@ const pkgIndexFile::Type *debStatusInde { return &_apt_Status; } - +const pkgIndexFile::Type *debDebPkgFileIndex::GetType() const +{ + return &_apt_DebPkgFile; +} +const pkgIndexFile::Type *debDscFileIndex::GetType() const +{ + return &_apt_DscFile; +} +const pkgIndexFile::Type *debDebianSourceDirIndex::GetType() const +{ + return &_apt_DebianSourceDir; +} /*}}}*/ diff --combined cmdline/apt-get.cc index c8c3ca56b,2e283da5a..0cea05cb3 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@@ -195,7 -195,7 +195,7 @@@ static std::string GetReleaseForSourceR // FindSrc - Find a source record /*{{{*/ // --------------------------------------------------------------------- /* */ -static pkgSrcRecords::Parser *FindSrc(const char *Name,pkgRecords &Recs, +static pkgSrcRecords::Parser *FindSrc(const char *Name, pkgSrcRecords &SrcRecs,string &Src, CacheFile &CacheFile) { @@@ -303,10 -303,16 +303,10 @@@ (VF.File().Archive() != 0 && VF.File().Archive() == RelTag) || (VF.File().Codename() != 0 && VF.File().Codename() == RelTag)) { - pkgRecords::Parser &Parse = Recs.Lookup(VF); - Src = Parse.SourcePkg(); - // no SourcePkg name, so it is the "binary" name - if (Src.empty() == true) - Src = TmpSrc; + Src = Ver.SourcePkgName(); // the Version we have is possibly fuzzy or includes binUploads, - // so we use the Version of the SourcePkg (empty if same as package) - VerTag = Parse.SourceVer(); - if (VerTag.empty() == true) - VerTag = Ver.VerStr(); + // so we use the Version of the SourcePkg + VerTag = Ver.SourceVerStr(); break; } } @@@ -337,10 -343,10 +337,10 @@@ pkgCache::VerIterator Ver = Cache->GetCandidateVer(Pkg); if (Ver.end() == false) { - pkgRecords::Parser &Parse = Recs.Lookup(Ver.FileList()); - Src = Parse.SourcePkg(); - if (VerTag.empty() == true) - VerTag = Parse.SourceVer(); + if (strcmp(Ver.SourcePkgName(),Ver.ParentPkg().Name()) != 0) + Src = Ver.SourcePkgName(); + if (VerTag.empty() == true && strcmp(Ver.SourceVerStr(),Ver.VerStr()) != 0) + VerTag = Ver.SourceVerStr(); } } } @@@ -534,7 -540,7 +534,7 @@@ static bool DoDSelectUpgrade(CommandLin } // Now upgrade everything - if (pkgAllUpgrade(Cache) == false) + if (APT::Upgrade::Upgrade(Cache, APT::Upgrade::FORBID_REMOVE_PACKAGES | APT::Upgrade::FORBID_INSTALL_NEW_PACKAGES) == false) { ShowBroken(c1out,Cache,false); return _error->Error(_("Internal error, problem resolver broke stuff")); @@@ -549,44 -555,30 +549,44 @@@ static bool DoClean(CommandLine &) { std::string const archivedir = _config->FindDir("Dir::Cache::archives"); - std::string const pkgcache = _config->FindFile("Dir::cache::pkgcache"); - std::string const srcpkgcache = _config->FindFile("Dir::cache::srcpkgcache"); + std::string const listsdir = _config->FindDir("Dir::state::lists"); if (_config->FindB("APT::Get::Simulate") == true) { + std::string const pkgcache = _config->FindFile("Dir::cache::pkgcache"); + std::string const srcpkgcache = _config->FindFile("Dir::cache::srcpkgcache"); cout << "Del " << archivedir << "* " << archivedir << "partial/*"<< endl + << "Del " << listsdir << "partial/*" << endl << "Del " << pkgcache << " " << srcpkgcache << endl; return true; } - + + bool const NoLocking = _config->FindB("Debug::NoLocking",false); // Lock the archive directory FileFd Lock; - if (_config->FindB("Debug::NoLocking",false) == false) + if (NoLocking == false) { int lock_fd = GetLock(archivedir + "lock"); if (lock_fd < 0) - return _error->Error(_("Unable to lock the download directory")); + return _error->Error(_("Unable to lock directory %s"), archivedir.c_str()); Lock.Fd(lock_fd); } - + pkgAcquire Fetcher; Fetcher.Clean(archivedir); Fetcher.Clean(archivedir + "partial/"); + if (NoLocking == false) + { + Lock.Close(); + int lock_fd = GetLock(listsdir + "lock"); + if (lock_fd < 0) + return _error->Error(_("Unable to lock directory %s"), listsdir.c_str()); + Lock.Fd(lock_fd); + } + + Fetcher.Clean(listsdir + "partial/"); + pkgCacheFile::RemoveCaches(); return true; @@@ -640,14 -632,14 +640,14 @@@ static bool DoDownload(CommandLine &Cmd APT::CacheSetHelper helper(c0out); APT::VersionSet verset = APT::VersionSet::FromCommandLine(Cache, - CmdL.FileList + 1, APT::VersionSet::CANDIDATE, helper); + CmdL.FileList + 1, APT::CacheSetHelper::CANDIDATE, helper); if (verset.empty() == true) return false; AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet", 0)); pkgAcquire Fetcher; - if (Fetcher.Setup(&Stat) == false) + if (Fetcher.Setup(&Stat, "", false) == false) return false; pkgRecords Recs(Cache); @@@ -673,7 -665,7 +673,7 @@@ { pkgAcquire::UriIterator I = Fetcher.UriBegin(); for (; I != Fetcher.UriEnd(); ++I) - cout << '\'' << I->URI << "' " << flNotDir(I->Owner->DestFile) << ' ' << + cout << '\'' << I->URI << "' " << flNotDir(I->Owner->DestFile) << ' ' << I->Owner->FileSize << ' ' << I->Owner->HashSum() << endl; return true; } @@@ -739,6 -731,7 +739,6 @@@ static bool DoSource(CommandLine &CmdL pkgSourceList *List = Cache.GetSourceList(); // Create the text record parsers - pkgRecords Recs(Cache); pkgSrcRecords SrcRecs(*List); if (_error->PendingError() == true) return false; @@@ -767,7 -760,7 +767,7 @@@ for (const char **I = CmdL.FileList + 1; *I != 0; I++, J++) { string Src; - pkgSrcRecords::Parser *Last = FindSrc(*I,Recs,SrcRecs,Src,Cache); + pkgSrcRecords::Parser *Last = FindSrc(*I,SrcRecs,Src,Cache); if (Last == 0) { return _error->Error(_("Unable to find a source package for %s"),Src.c_str()); @@@ -839,25 -832,23 +839,25 @@@ queued.insert(Last->Index().ArchiveURI(I->Path)); // check if we have a file with that md5 sum already localy - if(!I->MD5Hash.empty() && FileExists(flNotDir(I->Path))) - { - FileFd Fd(flNotDir(I->Path), FileFd::ReadOnly); - MD5Summation sum; - sum.AddFD(Fd.Fd(), Fd.Size()); - Fd.Close(); - if((string)sum.Result() == I->MD5Hash) + std::string localFile = flNotDir(I->Path); + if (FileExists(localFile) == true) + if(I->Hashes.VerifyFile(localFile) == true) { ioprintf(c1out,_("Skipping already downloaded file '%s'\n"), - flNotDir(I->Path).c_str()); + localFile.c_str()); continue; } + + // see if we have a hash (Acquire::ForceHash is the only way to have none) + if (I->Hashes.usable() == false && _config->FindB("APT::Get::AllowUnauthenticated",false) == false) + { + ioprintf(c1out, "Skipping download of file '%s' as requested hashsum is not available for authentication\n", + localFile.c_str()); + continue; } new pkgAcqFile(&Fetcher,Last->Index().ArchiveURI(I->Path), - I->MD5Hash,I->Size, - Last->Index().SourceInfo(*Last,*I),Src); + I->Hashes, I->Size, Last->Index().SourceInfo(*Last,*I), Src); } } @@@ -1044,6 -1035,7 +1044,6 @@@ static bool DoBuildDep(CommandLine &Cmd pkgSourceList *List = Cache.GetSourceList(); // Create the text record parsers - pkgRecords Recs(Cache); pkgSrcRecords SrcRecs(*List); if (_error->PendingError() == true) return false; @@@ -1070,35 -1062,7 +1070,35 @@@ for (const char **I = CmdL.FileList + 1; *I != 0; I++, J++) { string Src; - pkgSrcRecords::Parser *Last = FindSrc(*I,Recs,SrcRecs,Src,Cache); + pkgSrcRecords::Parser *Last = 0; + + // an unpacked debian source tree + using APT::String::Startswith; + if ((Startswith(*I, "./") || Startswith(*I, "/")) && + DirectoryExists(*I)) + { + ioprintf(c1out, _("Note, using directory '%s' to get the build dependencies\n"), *I); + // FIXME: how can we make this more elegant? + std::string TypeName = "debian/control File Source Index"; + pkgIndexFile::Type *Type = pkgIndexFile::Type::GetType(TypeName.c_str()); + if(Type != NULL) + Last = Type->CreateSrcPkgParser(*I); + } + // if its a local file (e.g. .dsc) use this + else if (FileExists(*I)) + { + ioprintf(c1out, _("Note, using file '%s' to get the build dependencies\n"), *I); + + // see if we can get a parser for this pkgIndexFile type + string TypeName = flExtension(*I) + " File Source Index"; + pkgIndexFile::Type *Type = pkgIndexFile::Type::GetType(TypeName.c_str()); + if(Type != NULL) + Last = Type->CreateSrcPkgParser(*I); + } else { + // normal case, search the cache for the source file + Last = FindSrc(*I,SrcRecs,Src,Cache); + } + if (Last == 0) return _error->Error(_("Unable to find a source package for %s"),Src.c_str()); @@@ -1116,7 -1080,7 +1116,7 @@@ } else if (Last->BuildDepends(BuildDeps, _config->FindB("APT::Get::Arch-Only", false), StripMultiArch) == false) return _error->Error(_("Unable to get build-dependency information for %s"),Src.c_str()); - + // Also ensure that build-essential packages are present Configuration::Item const *Opts = _config->Tree("APT::Build-Essential"); if (Opts) @@@ -1447,15 -1411,21 +1447,15 @@@ * pool/ next to the deb itself) * Example return: "pool/main/a/apt/apt_0.8.8ubuntu3" */ -static string GetChangelogPath(CacheFile &Cache, - pkgCache::PkgIterator Pkg, +static string GetChangelogPath(CacheFile &Cache, pkgCache::VerIterator Ver) { - string path; - pkgRecords Recs(Cache); pkgRecords::Parser &rec=Recs.Lookup(Ver.FileList()); - string srcpkg = rec.SourcePkg().empty() ? Pkg.Name() : rec.SourcePkg(); - string ver = Ver.VerStr(); - // if there is a source version it always wins - if (rec.SourceVer() != "") - ver = rec.SourceVer(); - path = flNotFile(rec.FileName()); - path += srcpkg + "_" + StripEpoch(ver); + string path = flNotFile(rec.FileName()); + path.append(Ver.SourcePkgName()); + path.append("_"); + path.append(StripEpoch(Ver.SourceVerStr())); return path; } /*}}}*/ @@@ -1469,6 -1439,7 +1469,6 @@@ * http://packages.medibuntu.org/pool/non-free/m/mplayer/mplayer_1.0~rc4~try1.dsfg1-1ubuntu1+medibuntu1.changelog */ static bool GuessThirdPartyChangelogUri(CacheFile &Cache, - pkgCache::PkgIterator Pkg, pkgCache::VerIterator Ver, string &out_uri) { @@@ -1483,7 -1454,7 +1483,7 @@@ return false; // get archive uri for the binary deb - string path_without_dot_changelog = GetChangelogPath(Cache, Pkg, Ver); + string path_without_dot_changelog = GetChangelogPath(Cache, Ver); out_uri = index->ArchiveURI(path_without_dot_changelog + ".changelog"); // now strip away the filename and add srcpkg_srcver.changelog @@@ -1501,20 -1472,25 +1501,20 @@@ static bool DownloadChangelog(CacheFil * GuessThirdPartyChangelogUri for details how) */ { - string path; - string descr; - string server; - string changelog_uri; - - // data structures we need - pkgCache::PkgIterator Pkg = Ver.ParentPkg(); - // make the server root configurable - server = _config->Find("Apt::Changelogs::Server", + string const server = _config->Find("Apt::Changelogs::Server", "http://packages.debian.org/changelogs"); - path = GetChangelogPath(CacheFile, Pkg, Ver); + string const path = GetChangelogPath(CacheFile, Ver); + string changelog_uri; strprintf(changelog_uri, "%s/%s/changelog", server.c_str(), path.c_str()); if (_config->FindB("APT::Get::Print-URIs", false) == true) { std::cout << '\'' << changelog_uri << '\'' << std::endl; return true; } + pkgCache::PkgIterator const Pkg = Ver.ParentPkg(); + string descr; strprintf(descr, _("Changelog for %s (%s)"), Pkg.Name(), changelog_uri.c_str()); // queue it new pkgAcqFile(&Fetcher, changelog_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile); @@@ -1525,7 -1501,7 +1525,7 @@@ if (!FileExists(targetfile)) { string third_party_uri; - if (GuessThirdPartyChangelogUri(CacheFile, Pkg, Ver, third_party_uri)) + if (GuessThirdPartyChangelogUri(CacheFile, Ver, third_party_uri)) { strprintf(descr, _("Changelog for %s (%s)"), Pkg.Name(), third_party_uri.c_str()); new pkgAcqFile(&Fetcher, third_party_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile); @@@ -1550,7 -1526,7 +1550,7 @@@ static bool DoChangelog(CommandLine &Cm APT::CacheSetHelper helper(c0out); APT::VersionList verset = APT::VersionList::FromCommandLine(Cache, - CmdL.FileList + 1, APT::VersionList::CANDIDATE, helper); + CmdL.FileList + 1, APT::CacheSetHelper::CANDIDATE, helper); if (verset.empty() == true) return false; pkgAcquire Fetcher; @@@ -1565,8 -1541,7 +1565,8 @@@ } AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0)); - Fetcher.Setup(&Stat); + if (Fetcher.Setup(&Stat, "",false) == false) + return false; bool const downOnly = _config->FindB("APT::Get::Download-Only", false); diff --combined methods/copy.cc index b65bc4dd5,40f8f85ec..f542a27c0 --- a/methods/copy.cc +++ b/methods/copy.cc @@@ -37,15 -37,12 +37,12 @@@ class CopyMethod : public pkgAcqMetho void CopyMethod::CalculateHashes(FetchResult &Res) { - // For gzip indexes we need to look inside the gzip for the hash - // We can not use the extension here as its not used in partial - // on a IMS hit - FileFd::OpenMode OpenMode = FileFd::ReadOnly; + Hashes Hash; + FileFd::CompressMode CompressMode = FileFd::None; if (_config->FindB("Acquire::GzipIndexes", false) == true) - OpenMode = FileFd::ReadOnlyGzip; + CompressMode = FileFd::Extension; - Hashes Hash; - FileFd Fd(Res.Filename, OpenMode); + FileFd Fd(Res.Filename, FileFd::ReadOnly, CompressMode); Hash.AddFD(Fd); Res.TakeHashes(Hash); } @@@ -119,6 -116,5 +116,6 @@@ int main( setlocale(LC_ALL, ""); CopyMethod Mth; + return Mth.Run(); } diff --combined test/integration/framework index fcdca34ce,7923e23d9..e83606fae --- a/test/integration/framework +++ b/test/integration/framework @@@ -102,10 -102,10 +102,10 @@@ runapt() local CMD="$1" shift case $CMD in - sh|aptitude|*/*) ;; + sh|aptitude|*/*|command) ;; *) CMD="${BUILDDIRECTORY}/$CMD";; esac - MALLOC_PERTURB_=21 MALLOC_CHECK_=2 APT_CONFIG="$(getaptconfig)" LD_LIBRARY_PATH=${BUILDDIRECTORY} $CMD "$@" + MALLOC_PERTURB_=21 MALLOC_CHECK_=2 APT_CONFIG="$(getaptconfig)" LD_LIBRARY_PATH=${LIBRARYPATH} $CMD "$@" } aptconfig() { runapt apt-config "$@"; } aptcache() { runapt apt-cache "$@"; } @@@ -129,9 -129,18 +129,9 @@@ dpkgcheckbuilddeps() command dpkg-checkbuilddeps --admindir=${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg "$@" } gdb() { - echo "gdb: run »$*«" - CMD="$1" + local CMD="$1" shift - - APT_CONFIG=aptconfig.conf LD_LIBRARY_PATH=${LIBRARYPATH} command gdb ${BUILDDIRECTORY}/$CMD --args ${BUILDDIRECTORY}/$CMD "$@" -} -gpg() { - # see apt-key for the whole trickery. Setup is done in setupenvironment - command gpg --ignore-time-conflict --no-options --no-default-keyring \ - --homedir "${TMPWORKINGDIRECTORY}/gnupghome" \ - --no-auto-check-trustdb --trust-model always \ - "$@" + runapt command gdb --quiet -ex run "${BUILDDIRECTORY}/$CMD" --args "${BUILDDIRECTORY}/$CMD" "$@" } exitwithstatus() { @@@ -216,7 -225,6 +216,7 @@@ setupenvironment() echo "Debug::NoLocking \"true\";" >> aptconfig.conf echo "APT::Get::Show-User-Simulation-Note \"false\";" >> aptconfig.conf echo "Dir::Bin::Methods \"${METHODSDIR}\";" >> aptconfig.conf + echo "Dir::Bin::apt-key \"${BUILDDIRECTORY}/apt-key\";" >> aptconfig.conf echo "Dir::Bin::dpkg \"fakeroot\";" >> aptconfig.conf echo "DPKG::options:: \"dpkg\";" >> aptconfig.conf echo "DPKG::options:: \"--root=${TMPWORKINGDIRECTORY}/rootdir\";" >> aptconfig.conf @@@ -227,14 -235,25 +227,14 @@@ fi echo "DPKG::options:: \"--log=${TMPWORKINGDIRECTORY}/rootdir/var/log/dpkg.log\";" >> aptconfig.conf echo 'quiet::NoUpdate "true";' >> aptconfig.conf + echo 'quiet::NoStatistic "true";' >> aptconfig.conf echo "Acquire::https::CaInfo \"${TESTDIR}/apt.pem\";" > rootdir/etc/apt/apt.conf.d/99https echo "Apt::Cmd::Disable-Script-Warning \"1\";" > rootdir/etc/apt/apt.conf.d/apt-binary configcompression '.' 'gz' #'bz2' 'lzma' 'xz' - # gpg needs a trustdb to function, but it can't be invalid (not even empty) - # see also apt-key where this trickery comes from: - local TRUSTDBDIR="${TMPWORKINGDIRECTORY}/gnupghome" - mkdir "$TRUSTDBDIR" - chmod 700 "$TRUSTDBDIR" - # We also don't use a secret keyring, of course, but gpg panics and - # implodes if there isn't one available - and writeable for imports - local SECRETKEYRING="${TRUSTDBDIR}/secring.gpg" - touch $SECRETKEYRING - # now create the trustdb with an (empty) dummy keyring - # newer gpg versions are fine without it, but play it safe for now - gpg --quiet --check-trustdb --secret-keyring $SECRETKEYRING --keyring $SECRETKEYRING >/dev/null 2>&1 - # cleanup the environment a bit - export PATH="${PATH}:/usr/local/sbin:/usr/sbin:/sbin" + # prefer our apt binaries over the system apt binaries + export PATH="${BUILDDIRECTORY}:${PATH}:/usr/local/sbin:/usr/sbin:/sbin" export LC_ALL=C.UTF-8 unset LANGUAGE APT_CONFIG unset GREP_OPTIONS DEB_BUILD_PROFILES @@@ -320,6 -339,36 +320,36 @@@ configcompression() done > ${TMPWORKINGDIRECTORY}/rootdir/etc/testcase-compressor.conf } + forcecompressor() { + COMPRESSOR="$1" + COMPRESSOR_CMD="$1" + case $COMPRESSOR in + gzip) COMPRESS='gz';; + bzip2) COMPRESS='bz2';; + lzma) COMPRESS='lzma';; + xz) COMPRESS='xz';; + *) msgdie "Compressor $COMPRESSOR is unknown to framework, so can't be forced by forcecompressor!";; + esac + local CONFFILE="${TMPWORKINGDIRECTORY}/rootdir/etc/apt/apt.conf.d/00force-compressor" + echo "Acquire::CompressionTypes::Order { \"${COMPRESS}\"; }; + Dir::Bin::uncompressed \"/does/not/exist\"; + Dir::Bin::gzip \"/does/not/exist\"; + Dir::Bin::bzip2 \"/does/not/exist\"; + Dir::Bin::lzma \"/does/not/exist\"; + Dir::Bin::xz \"/does/not/exist\";" > "$CONFFILE" + if [ -e "/bin/${COMPRESSOR}" ]; then + echo "Dir::Bin::${COMPRESSOR} \"/bin/${COMPRESSOR}\";" >> "$CONFFILE" + elif [ -e "/usr/bin/${COMPRESSOR}" ]; then + echo "Dir::Bin::${COMPRESSOR} \"/usr/bin/${COMPRESSOR}\";" >> "$CONFFILE" + elif [ "${COMPRESSOR}" = 'lzma' ]; then + echo 'Dir::Bin::xz "/usr/bin/xz";' >> "$CONFFILE" + COMPRESSOR_CMD='xz --format=lzma' + else + msgtest 'Test for availability of compressor' "${COMPRESSOR}" + msgfail + fi + } + setupsimplenativepackage() { local NAME="$1" local ARCH="$2" @@@ -429,8 -478,8 +459,8 @@@ Package: $NAME" >> ${BUILDDIR}/debian/c | while read SRC; do echo "pool/${SRC}" >> ${BUILDDIR}/../${RELEASE}.${DISTSECTION}.srclist # if expr match "${SRC}" '.*\.dsc' >/dev/null 2>&1; then -# gpg --yes --secret-keyring ./keys/joesixpack.sec \ -# --keyring ./keys/joesixpack.pub --default-key 'Joe Sixpack' \ +# aptkey --keyring ./keys/joesixpack.pub --secret-keyring ./keys/joesixpack.sec --quiet --readonly \ +# adv --yes --default-key 'Joe Sixpack' \ # --clearsign -o "${BUILDDIR}/../${SRC}.sign" "${BUILDDIR}/../$SRC" # mv "${BUILDDIR}/../${SRC}.sign" "${BUILDDIR}/../$SRC" # fi @@@ -815,9 -864,8 +845,9 @@@ setupaptarchive() signreleasefiles() { local SIGNER="${1:-Joe Sixpack}" - local GPG="gpg --batch --yes" - msgninfo "\tSign archive with $SIGNER key… " + local KEY="keys/$(echo "$SIGNER" | tr 'A-Z' 'a-z' | sed 's# ##g')" + local GPG="aptkey --quiet --keyring ${KEY}.pub --secret-keyring ${KEY}.sec --readonly adv --batch --yes" + msgninfo "\tSign archive with $SIGNER key $KEY… " local REXKEY='keys/rexexpired' local SECEXPIREBAK="${REXKEY}.sec.bak" local PUBEXPIREBAK="${REXKEY}.pub.bak" @@@ -833,14 -881,17 +863,14 @@@ cp $SECUNEXPIRED ${REXKEY}.sec cp $PUBUNEXPIRED ${REXKEY}.pub else - printf "expire\n1w\nsave\n" | $GPG --keyring ${REXKEY}.pub --secret-keyring ${REXKEY}.sec --command-fd 0 --edit-key "${SIGNER}" >/dev/null 2>&1 || true + if ! printf "expire\n1w\nsave\n" | $GPG --default-key "$SIGNER" --command-fd 0 --edit-key "${SIGNER}" >setexpire.gpg 2>&1; then + cat setexpire.gpg + exit 1 + fi cp ${REXKEY}.sec $SECUNEXPIRED cp ${REXKEY}.pub $PUBUNEXPIRED fi fi - for KEY in $(find keys/ -name '*.sec'); do - GPG="$GPG --secret-keyring $KEY" - done - for KEY in $(find keys/ -name '*.pub'); do - GPG="$GPG --keyring $KEY" - done for RELEASE in $(find aptarchive/ -name Release); do $GPG --default-key "$SIGNER" --armor --detach-sign --sign --output ${RELEASE}.gpg ${RELEASE} local INRELEASE="$(echo "${RELEASE}" | sed 's#/Release$#/InRelease#')" diff --combined test/integration/test-bug-595691-empty-and-broken-archive-files index 8da0a52d2,aea340203..683c174bd --- a/test/integration/test-bug-595691-empty-and-broken-archive-files +++ b/test/integration/test-bug-595691-empty-and-broken-archive-files @@@ -13,7 -13,7 +13,7 @@@ setupflataptarchiv testaptgetupdate() { rm -rf rootdir/var/lib/apt aptget update 2>> testaptgetupdate.diff >> testaptgetupdate.diff || true - sed -i -e '/^Fetched / d' -e '/Ign / d' -e '/Release/ d' -e 's#Get:[0-9]\+ #Get: #' -e 's#\[[0-9]* [kMGTPY]*B\]#\[\]#' testaptgetupdate.diff + sed -i -e '/Ign / d' -e '/Release/ d' -e 's#Get:[0-9]\+ #Get: #' -e 's#\[[0-9]* [kMGTPY]*B\]#\[\]#' testaptgetupdate.diff GIVEN="$1" shift msgtest "Test for correctness of" "apt-get update with $*" @@@ -48,37 -48,8 +48,8 @@@ createemptyfile() rm -f aptarchive/Packages } - setupcompressor() { - COMPRESSOR="$1" - COMPRESSOR_CMD="$1" - case $COMPRESSOR in - gzip) COMPRESS="gz";; - bzip2) COMPRESS="bz2";; - lzma) COMPRESS="lzma";; - xz) COMPRESS="xz";; - esac - echo "Acquire::CompressionTypes::Order { \"${COMPRESS}\"; }; - Dir::Bin::uncompressed \"/does/not/exist\"; - Dir::Bin::gzip \"/does/not/exist\"; - Dir::Bin::bzip2 \"/does/not/exist\"; - Dir::Bin::lzma \"/does/not/exist\"; - Dir::Bin::xz \"/does/not/exist\";" > rootdir/etc/apt/apt.conf.d/00compressor - if [ -e "/bin/${COMPRESSOR}" ]; then - echo "Dir::Bin::${COMPRESSOR} \"/bin/${COMPRESSOR}\";" >> rootdir/etc/apt/apt.conf.d/00compressor - elif [ -e "/usr/bin/${COMPRESSOR}" ]; then - echo "Dir::Bin::${COMPRESSOR} \"/usr/bin/${COMPRESSOR}\";" >> rootdir/etc/apt/apt.conf.d/00compressor - elif [ "${COMPRESSOR}" = 'lzma' ]; then - echo "Dir::Bin::xz \"/usr/bin/xz\";" >> rootdir/etc/apt/apt.conf.d/00compressor - COMPRESSOR_CMD='xz --format=lzma' - else - msgtest "Test for availability of compressor" "${COMPRESSOR}" - msgfail - #exit 1 - fi - } - testoverfile() { - setupcompressor "$1" + forcecompressor "$1" createemptyfile 'en' testaptgetupdate 'Reading package lists...' "empty file en.$COMPRESS over file" @@@ -100,7 -71,7 +71,7 @@@ E: Some index files failed to download } testoverhttp() { - setupcompressor "$1" + forcecompressor "$1" createemptyfile 'en' testaptgetupdate "Get: http://localhost:8080 Packages [] @@@ -121,7 -92,7 +92,7 @@@ Reading package lists..." "empty archiv testaptgetupdate "Get: http://localhost:8080 Packages Err http://localhost:8080 Packages Empty files can't be valid archives - W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages) Empty files can't be valid archives + W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages.${COMPRESS}) Empty files can't be valid archives E: Some index files failed to download. They have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over http" }