X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/4a858c3921aabe5253204063bedb2307a60f30e0..304731b8f23e03d15d1c2d6f66e1bf5f26ac0ca2:/apt-pkg/acquire-item.cc diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index aee187227..12000a8c1 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -13,25 +13,30 @@ ##################################################################### */ /*}}}*/ // Include Files /*{{{*/ +#include + #include #include +#include #include -#include #include #include #include #include #include #include +#include +#include -#include - #include #include #include #include #include #include +#include + +#include /*}}}*/ using namespace std; @@ -80,7 +85,7 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Status = StatError; Dequeue(); } - + // report mirror failure back to LP if we actually use a mirror string FailReason = LookupTag(Message, "FailReason"); if(FailReason.size() != 0) @@ -93,7 +98,7 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) // --------------------------------------------------------------------- /* Stash status and the file size. Note that setting Complete means sub-phases of the acquire process such as decompresion are operating */ -void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size) +void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size) { Status = StatFetching; if (FileSize == 0 && Complete == false) @@ -103,7 +108,7 @@ void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size) // Acquire::Item::Done - Item downloaded OK /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash, +void pkgAcquire::Item::Done(string Message,unsigned long long Size,string Hash, pkgAcquire::MethodConfig *Cnf) { // We just downloaded something.. @@ -138,7 +143,8 @@ void pkgAcquire::Item::Rename(string From,string To) } } /*}}}*/ - +// Acquire::Item::ReportMirrorFailure /*{{{*/ +// --------------------------------------------------------------------- void pkgAcquire::Item::ReportMirrorFailure(string FailCode) { // we only act if a mirror was used at all @@ -180,10 +186,120 @@ void pkgAcquire::Item::ReportMirrorFailure(string FailCode) _config->Find("Methods::Mirror::ProblemReporting").c_str()); } } + /*}}}*/ +// AcqSubIndex::AcqSubIndex - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* Get a sub-index file based on checksums from a 'master' file and + possibly query additional files */ +pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI, + string const &URIDesc, string const &ShortDesc, + HashString const &ExpectedHash) + : Item(Owner), ExpectedHash(ExpectedHash) +{ + /* XXX: Beware: Currently this class does nothing (of value) anymore ! */ + Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false); + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(URI); + Desc.URI = URI; + Desc.Description = URIDesc; + Desc.Owner = this; + Desc.ShortDesc = ShortDesc; + + QueueURI(Desc); -// AcqDiffIndex::AcqDiffIndex - Constructor + if(Debug) + std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl; +} + /*}}}*/ +// AcqSubIndex::Custom600Headers - Insert custom request headers /*{{{*/ +// --------------------------------------------------------------------- +/* The only header we use is the last-modified header. */ +string pkgAcqSubIndex::Custom600Headers() +{ + string Final = _config->FindDir("Dir::State::lists"); + Final += URItoFileName(Desc.URI); + + struct stat Buf; + if (stat(Final.c_str(),&Buf) != 0) + return "\nIndex-File: true\nFail-Ignore: true\n"; + return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); +} + /*}}}*/ +void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ +{ + if(Debug) + std::clog << "pkgAcqSubIndex failed: " << Desc.URI << std::endl; + + Complete = false; + Status = StatDone; + Dequeue(); + + // No good Index is provided +} + /*}}}*/ +void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ + pkgAcquire::MethodConfig *Cnf) +{ + if(Debug) + std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl; + + string FileName = LookupTag(Message,"Filename"); + if (FileName.empty() == true) + { + Status = StatError; + ErrorText = "Method gave a blank filename"; + return; + } + + if (FileName != DestFile) + { + Local = true; + Desc.URI = "copy:" + FileName; + QueueURI(Desc); + return; + } + + Item::Done(Message,Size,Md5Hash,Cnf); + + string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI); + + /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */ + indexRecords SubIndexParser; + if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) { + Status = StatError; + ErrorText = SubIndexParser.ErrorText; + return; + } + + // sucess in downloading the index + // rename the index + if(Debug) + std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl; + Rename(DestFile,FinalFile); + chmod(FinalFile.c_str(),0644); + DestFile = FinalFile; + + if(ParseIndex(DestFile) == false) + return Failed("", NULL); + + Complete = true; + Status = StatDone; + Dequeue(); + return; +} + /*}}}*/ +bool pkgAcqSubIndex::ParseIndex(string const &IndexFile) /*{{{*/ +{ + indexRecords SubIndexParser; + if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false) + return false; + // so something with the downloaded index + return true; +} + /*}}}*/ +// AcqDiffIndex::AcqDiffIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* Get the DiffIndex file first and see if there are patches availabe * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the @@ -234,7 +350,7 @@ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, QueueURI(Desc); } - + /*}}}*/ // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ @@ -252,9 +368,8 @@ string pkgAcqDiffIndex::Custom600Headers() return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } - - -bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) + /*}}}*/ +bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ { if(Debug) std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile @@ -271,19 +386,19 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) if(TF.Step(Tags) == true) { - string local_sha1; bool found = false; DiffInfo d; string size; - string tmp = Tags.FindS("SHA1-Current"); + string const tmp = Tags.FindS("SHA1-Current"); std::stringstream ss(tmp); - ss >> ServerSha1; + ss >> ServerSha1 >> size; + unsigned long const ServerSize = atol(size.c_str()); FileFd fd(CurrentPackagesFile, FileFd::ReadOnly); SHA1Summation SHA1; - SHA1.AddFD(fd.Fd(), fd.Size()); - local_sha1 = string(SHA1.Result()); + SHA1.AddFD(fd); + string const local_sha1 = SHA1.Result(); if(local_sha1 == ServerSha1) { @@ -297,23 +412,59 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) else { if(Debug) - std::clog << "SHA1-Current: " << ServerSha1 << std::endl; + std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl; // check the historie and see what patches we need - string history = Tags.FindS("SHA1-History"); + string const history = Tags.FindS("SHA1-History"); std::stringstream hist(history); - while(hist >> d.sha1 >> size >> d.file) + while(hist >> d.sha1 >> size >> d.file) { - d.size = atoi(size.c_str()); // read until the first match is found + // from that point on, we probably need all diffs if(d.sha1 == local_sha1) found=true; - // from that point on, we probably need all diffs - if(found) + else if (found == false) + continue; + + if(Debug) + std::clog << "Need to get diff: " << d.file << std::endl; + available_patches.push_back(d); + } + + if (available_patches.empty() == false) + { + // patching with too many files is rather slow compared to a fast download + unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0); + if (fileLimit != 0 && fileLimit < available_patches.size()) + { + if (Debug) + std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit + << ") so fallback to complete download" << std::endl; + return false; + } + + // see if the patches are too big + found = false; // it was true and it will be true again at the end + d = *available_patches.begin(); + string const firstPatch = d.file; + unsigned long patchesSize = 0; + std::stringstream patches(Tags.FindS("SHA1-Patches")); + while(patches >> d.sha1 >> size >> d.file) + { + if (firstPatch == d.file) + found = true; + else if (found == false) + continue; + + patchesSize += atol(size.c_str()); + } + unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100); + if (sizeLimit > 0 && (sizeLimit/100) < patchesSize) { - if(Debug) - std::clog << "Need to get diff: " << d.file << std::endl; - available_patches.push_back(d); + if (Debug) + std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100 + << ") so fallback to complete download" << std::endl; + return false; } } } @@ -322,11 +473,11 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) if(found) { // queue the diffs - string::size_type last_space = Description.rfind(" "); + string::size_type const last_space = Description.rfind(" "); if(last_space != string::npos) Description.erase(last_space, Description.size()-last_space); new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, - ExpectedHash, available_patches); + ExpectedHash, ServerSha1, available_patches); Complete = false; Status = StatDone; Dequeue(); @@ -341,8 +492,8 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) std::clog << "Can't find a patch in the index file" << std::endl; return false; } - -void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) + /*}}}*/ +void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ { if(Debug) std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl @@ -355,8 +506,8 @@ void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Status = StatDone; Dequeue(); } - -void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash, + /*}}}*/ +void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ pkgAcquire::MethodConfig *Cnf) { if(Debug) @@ -385,10 +536,8 @@ void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash, Dequeue(); return; } - - - -// AcqIndexDiffs::AcqIndexDiffs - Constructor + /*}}}*/ +// AcqIndexDiffs::AcqIndexDiffs - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The package diff is added to the queue. one object is constructed * for each diff and the index @@ -396,9 +545,10 @@ void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash, pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, HashString ExpectedHash, + string ServerSha1, vector diffs) : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash), - available_patches(diffs) + available_patches(diffs), ServerSha1(ServerSha1) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; @@ -410,7 +560,7 @@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, Desc.Owner = this; Desc.ShortDesc = ShortDesc; - if(available_patches.size() == 0) + if(available_patches.empty() == true) { // we are done (yeah!) Finish(true); @@ -422,9 +572,8 @@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, QueueNextDiff(); } } - - -void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf) + /*}}}*/ +void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ { if(Debug) std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl @@ -433,9 +582,8 @@ void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf) ExpectedHash); Finish(); } - - -// helper that cleans the item out of the fetcher queue + /*}}}*/ +// Finish - helper that cleans the item out of the fetcher queue /*{{{*/ void pkgAcqIndexDiffs::Finish(bool allDone) { // we restore the original name, this is required, otherwise @@ -470,10 +618,8 @@ void pkgAcqIndexDiffs::Finish(bool allDone) Dequeue(); return; } - - - -bool pkgAcqIndexDiffs::QueueNextDiff() + /*}}}*/ +bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ { // calc sha1 of the just patched file @@ -482,26 +628,33 @@ bool pkgAcqIndexDiffs::QueueNextDiff() FileFd fd(FinalFile, FileFd::ReadOnly); SHA1Summation SHA1; - SHA1.AddFD(fd.Fd(), fd.Size()); + SHA1.AddFD(fd); string local_sha1 = string(SHA1.Result()); if(Debug) std::clog << "QueueNextDiff: " << FinalFile << " (" << local_sha1 << ")"<::iterator I=available_patches.begin(); - available_patches.size() > 0 && + available_patches.empty() == false && I != available_patches.end() && - (*I).sha1 != local_sha1; - I++) + I->sha1 != local_sha1; + ++I) { available_patches.erase(I); } // error checking and falling back if no patch was found - if(available_patches.size() == 0) - { + if(available_patches.empty() == true) + { Failed("", NULL); return false; } @@ -519,10 +672,8 @@ bool pkgAcqIndexDiffs::QueueNextDiff() return true; } - - - -void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash, + /*}}}*/ +void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ pkgAcquire::MethodConfig *Cnf) { if(Debug) @@ -534,24 +685,7 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash, FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); // sucess in downloading a diff, enter ApplyDiff state - if(State == StateFetchDiff) - { - - if(Debug) - std::clog << "Sending to gzip method: " << FinalFile << std::endl; - - string FileName = LookupTag(Message,"Filename"); - State = StateUnzipDiff; - Local = true; - Desc.URI = "gzip:" + FileName; - DestFile += ".decomp"; - QueueURI(Desc); - Mode = "gzip"; - return; - } - - // sucess in downloading a diff, enter ApplyDiff state - if(State == StateUnzipDiff) + if(State == StateFetchDiff) { // rred excepts the patch as $FinalFile.ed @@ -585,16 +719,15 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash, chmod(FinalFile.c_str(),0644); // see if there is more to download - if(available_patches.size() > 0) { + if(available_patches.empty() == false) { new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, - ExpectedHash, available_patches); + ExpectedHash, ServerSha1, available_patches); return Finish(); } else return Finish(true); } } - - + /*}}}*/ // AcqIndex::AcqIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The package file is added to the queue and a second class is @@ -604,28 +737,68 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, HashString ExpectedHash, string comprExt) : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash) { + if(comprExt.empty() == true) + { + // autoselect the compression method + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + comprExt.append(*t).append(" "); + if (comprExt.empty() == false) + comprExt.erase(comprExt.end()-1); + } + CompressionExtension = comprExt; + + Init(URI, URIDesc, ShortDesc); +} +pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target, + HashString const &ExpectedHash, indexRecords const *MetaIndexParser) + : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash) +{ + // autoselect the compression method + std::vector types = APT::Configuration::getCompressionTypes(); + CompressionExtension = ""; + if (ExpectedHash.empty() == false) + { + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true) + CompressionExtension.append(*t).append(" "); + } + else + { + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + CompressionExtension.append(*t).append(" "); + } + if (CompressionExtension.empty() == false) + CompressionExtension.erase(CompressionExtension.end()-1); + + // only verify non-optional targets, see acquire-item.h for a FIXME + // to make this more flexible + if (Target->IsOptional()) + Verify = false; + else + Verify = true; + + Init(Target->URI, Target->Description, Target->ShortDesc); +} + /*}}}*/ +// AcqIndex::Init - defered Constructor /*{{{*/ +void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) { Decompression = false; Erase = false; DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); - if(comprExt.empty()) - { - // autoselect the compression method - if(FileExists("/bin/bzip2")) - CompressionExtension = ".bz2"; - else - CompressionExtension = ".gz"; - } else { - CompressionExtension = comprExt; - } - Desc.URI = URI + CompressionExtension; + std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); + if (comprExt == "uncompressed") + Desc.URI = URI; + else + Desc.URI = URI + '.' + comprExt; Desc.Description = URIDesc; Desc.Owner = this; Desc.ShortDesc = ShortDesc; - + QueueURI(Desc); } /*}}}*/ @@ -636,40 +809,41 @@ string pkgAcqIndex::Custom600Headers() { string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); + if (_config->FindB("Acquire::GzipIndexes",false)) + Final += ".gz"; + string msg = "\nIndex-File: true"; + // FIXME: this really should use "IndexTarget::IsOptional()" but that + // seems to be difficult without breaking ABI + if (ShortDesc().find("Translation") != 0) + msg += "\nFail-Ignore: true"; struct stat Buf; - if (stat(Final.c_str(),&Buf) != 0) - return "\nIndex-File: true"; - return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); + if (stat(Final.c_str(),&Buf) == 0) + msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); + + return msg; } /*}}}*/ - -void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) +void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ { - // no .bz2 found, retry with .gz - if(Desc.URI.substr(Desc.URI.size()-3) == "bz2") { - Desc.URI = Desc.URI.substr(0,Desc.URI.size()-3) + "gz"; - - // retry with a gzip one - new pkgAcqIndex(Owner, RealURI, Desc.Description,Desc.ShortDesc, - ExpectedHash, string(".gz")); - Status = StatDone; - Complete = false; - Dequeue(); + size_t const nextExt = CompressionExtension.find(' '); + if (nextExt != std::string::npos) + { + CompressionExtension = CompressionExtension.substr(nextExt+1); + Init(RealURI, Desc.Description, Desc.ShortDesc); return; - } - + } + // on decompression failure, remove bad versions in partial/ - if(Decompression && Erase) { + if (Decompression && Erase) { string s = _config->FindDir("Dir::State::lists") + "partial/"; - s += URItoFileName(RealURI); + s.append(URItoFileName(RealURI)); unlink(s.c_str()); } Item::Failed(Message,Cnf); } - - + /*}}}*/ // AcqIndex::Done - Finished a fetch /*{{{*/ // --------------------------------------------------------------------- /* This goes through a number of states.. On the initial fetch the @@ -677,7 +851,7 @@ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) to the uncompressed version of the file. If this is so the file is copied into the partial directory. In all other cases the file is decompressed with a gzip uri. */ -void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash, +void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, pkgAcquire::MethodConfig *Cfg) { Item::Done(Message,Size,Hash,Cfg); @@ -698,6 +872,31 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash, ReportMirrorFailure("HashChecksumFailure"); return; } + + /* Verify the index file for correctness (all indexes must + * have a Package field) (LP: #346386) (Closes: #627642) */ + if (Verify == true) + { + FileFd fd(DestFile, FileFd::ReadOnly); + pkgTagSection sec; + pkgTagFile tag(&fd); + + // Only test for correctness if the file is not empty (empty is ok) + if (fd.Size() > 0) { + if (_error->PendingError() || !tag.Step(sec)) { + Status = StatError; + _error->DumpErrors(); + Rename(DestFile,DestFile + ".FAILED"); + return; + } else if (!sec.Exists("Package")) { + Status = StatError; + ErrorText = ("Encountered a section with no Package: header"); + Rename(DestFile,DestFile + ".FAILED"); + return; + } + } + } + // Done, move it into position string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); @@ -740,22 +939,42 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash, Status = StatError; ErrorText = "Method gave a blank filename"; } - + + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); + // The files timestamp matches - if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) { + if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") + // Update DestFile for .gz suffix so that the clean operation keeps it + DestFile += ".gz"; return; + } if (FileName == DestFile) Erase = true; else Local = true; - string compExt = Desc.URI.substr(Desc.URI.size()-3); - const char *decompProg; - if(compExt == "bz2") - decompProg = "bzip2"; - else if(compExt == ".gz") - decompProg = "gzip"; + string decompProg; + + // If we enable compressed indexes and already have gzip, keep it + if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) { + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI) + ".gz"; + Rename(DestFile,FinalFile); + chmod(FinalFile.c_str(),0644); + + // Update DestFile for .gz suffix so that the clean operation keeps it + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI) + ".gz"; + return; + } + + // get the binary name for your used compression type + decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),""); + if(decompProg.empty() == false); + else if(compExt == "uncompressed") + decompProg = "copy"; else { _error->Error("Unsupported extension: %s", compExt.c_str()); return; @@ -763,11 +982,11 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash, Decompression = true; DestFile += ".decomp"; - Desc.URI = string(decompProg) + ":" + FileName; + Desc.URI = decompProg + ":" + FileName; QueueURI(Desc); - Mode = decompProg; + Mode = decompProg.c_str(); } - + /*}}}*/ // AcqIndexTrans::pkgAcqIndexTrans - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The Translation file is added to the queue */ @@ -776,13 +995,39 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "") { } +pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target, + HashString const &ExpectedHash, indexRecords const *MetaIndexParser) + : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser) +{ +} + /*}}}*/ +// AcqIndexTrans::Custom600Headers - Insert custom request headers /*{{{*/ +// --------------------------------------------------------------------- +string pkgAcqIndexTrans::Custom600Headers() +{ + string Final = _config->FindDir("Dir::State::lists"); + Final += URItoFileName(RealURI); + struct stat Buf; + if (stat(Final.c_str(),&Buf) != 0) + return "\nFail-Ignore: true\nIndex-File: true"; + return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); +} /*}}}*/ // AcqIndexTrans::Failed - Silence failure messages for missing files /*{{{*/ // --------------------------------------------------------------------- /* */ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { + size_t const nextExt = CompressionExtension.find(' '); + if (nextExt != std::string::npos) + { + CompressionExtension = CompressionExtension.substr(nextExt+1); + Init(RealURI, Desc.Description, Desc.ShortDesc); + Status = StatIdle; + return; + } + if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) { @@ -792,12 +1037,11 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Dequeue(); return; } - + Item::Failed(Message,Cnf); } /*}}}*/ - -pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, +pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ string URI,string URIDesc,string ShortDesc, string MetaIndexURI, string MetaIndexURIDesc, string MetaIndexShortDesc, @@ -823,8 +1067,7 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); - struct stat Buf; - if (stat(Final.c_str(),&Buf) == 0) + if (RealFileExists(Final) == true) { // File was already in place. It needs to be re-downloaded/verified // because Release might have changed, we do give it a differnt @@ -836,6 +1079,19 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, } QueueURI(Desc); +} + /*}}}*/ +pkgAcqMetaSig::~pkgAcqMetaSig() /*{{{*/ +{ + // if the file was never queued undo file-changes done in the constructor + if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false && + LastGoodSig.empty() == false) + { + string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); + if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true) + Rename(LastGoodSig, Final); + } + } /*}}}*/ // pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/ @@ -850,7 +1106,7 @@ string pkgAcqMetaSig::Custom600Headers() return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } -void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5, +void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5, pkgAcquire::MethodConfig *Cfg) { Item::Done(Message,Size,MD5,Cfg); @@ -887,7 +1143,7 @@ void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5, } /*}}}*/ -void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) +void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ { string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); @@ -923,8 +1179,8 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Item::Failed(Message,Cnf); } - -pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, + /*}}}*/ +pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ string URI,string URIDesc,string ShortDesc, string SigFile, const vector* IndexTargets, @@ -943,7 +1199,6 @@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, QueueURI(Desc); } - /*}}}*/ // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- @@ -959,8 +1214,8 @@ string pkgAcqMetaIndex::Custom600Headers() return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } - -void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash, + /*}}}*/ +void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash, /*{{{*/ pkgAcquire::MethodConfig *Cfg) { Item::Done(Message,Size,Hash,Cfg); @@ -975,12 +1230,6 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash, // all cool, move Release file into place Complete = true; - - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); - Rename(DestFile,FinalFile); - chmod(FinalFile.c_str(),0644); - DestFile = FinalFile; } else { @@ -992,14 +1241,26 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash, if (SigFile == "") { // There was no signature file, so we are finished. Download - // the indexes without verification. + // the indexes and do only hashsum verification if possible + MetaIndexParser->Load(DestFile); QueueIndexes(false); } else { + // FIXME: move this into pkgAcqMetaClearSig::Done on the next + // ABI break + + // if we expect a ClearTextSignature (InRelase), ensure that + // this is what we get and if not fail to queue a + // Release/Release.gpg, see #346386 + if (SigFile == DestFile && !StartsWithGPGClearTextSignature(DestFile)) + { + Failed(Message, Cfg); + return; + } + // There was a signature file, so pass it to gpgv for // verification - if (_config->FindB("Debug::pkgAcquire::Auth", false)) std::cerr << "Metaindex acquired, queueing gpg verification (" << SigFile << "," << DestFile << ")\n"; @@ -1007,11 +1268,23 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash, Desc.URI = "gpgv:" + SigFile; QueueURI(Desc); Mode = "gpgv"; + return; } } -} -void pkgAcqMetaIndex::RetrievalDone(string Message) + if (Complete == true) + { + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + if (SigFile == DestFile) + SigFile = FinalFile; + Rename(DestFile,FinalFile); + chmod(FinalFile.c_str(),0644); + DestFile = FinalFile; + } +} + /*}}}*/ +void pkgAcqMetaIndex::RetrievalDone(string Message) /*{{{*/ { // We have just finished downloading a Release file (it is not // verified yet) @@ -1038,12 +1311,21 @@ void pkgAcqMetaIndex::RetrievalDone(string Message) { string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); + if (SigFile == DestFile) + { + SigFile = FinalFile; + // constructor of pkgAcqMetaClearSig moved it out of the way, + // now move it back in on IMS hit for the 'old' file + string const OldClearSig = DestFile + ".reverify"; + if (RealFileExists(OldClearSig) == true) + Rename(OldClearSig, FinalFile); + } DestFile = FinalFile; } Complete = true; } - -void pkgAcqMetaIndex::AuthDone(string Message) + /*}}}*/ +void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ { // At this point, the gpgv method has succeeded, so there is a // valid signature from a key in the trusted keyring. We @@ -1069,81 +1351,116 @@ void pkgAcqMetaIndex::AuthDone(string Message) // Download further indexes with verification QueueIndexes(true); + // is it a clearsigned MetaIndex file? + if (DestFile == SigFile) + return; + // Done, move signature file into position string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI) + ".gpg"; Rename(SigFile,VerifiedSigFile); chmod(VerifiedSigFile.c_str(),0644); } - -void pkgAcqMetaIndex::QueueIndexes(bool verify) + /*}}}*/ +void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ { +#if 0 + /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642) + * FIXME: Disabled; it breaks unsigned repositories without hashes */ + if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile)) + { + Status = StatError; + ErrorText = MetaIndexParser->ErrorText; + return; + } +#endif + bool transInRelease = false; + { + std::vector const keys = MetaIndexParser->MetaKeys(); + for (std::vector::const_iterator k = keys.begin(); k != keys.end(); ++k) + // FIXME: Feels wrong to check for hardcoded string here, but what should we do else… + if (k->find("Translation-") != std::string::npos) + { + transInRelease = true; + break; + } + } + for (vector ::const_iterator Target = IndexTargets->begin(); Target != IndexTargets->end(); - Target++) + ++Target) { HashString ExpectedIndexHash; - if (verify) + const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey); + bool compressedAvailable = false; + if (Record == NULL) { - const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey); - if (!Record) - { - Status = StatAuthError; - ErrorText = "Unable to find expected entry " - + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)"; - return; - } - ExpectedIndexHash = Record->Hash; - if (_config->FindB("Debug::pkgAcquire::Auth", false)) - { - std::cerr << "Queueing: " << (*Target)->URI << std::endl; - std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl; - } - if (ExpectedIndexHash.empty()) - { - Status = StatAuthError; - ErrorText = "Unable to find hash sum for " - + (*Target)->MetaKey + " in Meta-index file"; - return; - } + if ((*Target)->IsOptional() == true) + { + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + if (MetaIndexParser->Exists(string((*Target)->MetaKey).append(".").append(*t)) == true) + { + compressedAvailable = true; + break; + } + } + else if (verify == true) + { + Status = StatAuthError; + strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str()); + return; + } } - - // Queue Packages file (either diff or full packages files, depending - // on the users option) - if(_config->FindB("Acquire::PDiffs",false) == true) + else + { + ExpectedIndexHash = Record->Hash; + if (_config->FindB("Debug::pkgAcquire::Auth", false)) + { + std::cerr << "Queueing: " << (*Target)->URI << std::endl; + std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl; + std::cerr << "For: " << Record->MetaKeyFilename << std::endl; + } + if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false) + { + Status = StatAuthError; + strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str()); + return; + } + } + + if ((*Target)->IsOptional() == true) + { + if ((*Target)->IsSubIndex() == true) + new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description, + (*Target)->ShortDesc, ExpectedIndexHash); + else if (transInRelease == false || Record != NULL || compressedAvailable == true) + { + if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true && + MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true) + new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description, + (*Target)->ShortDesc, ExpectedIndexHash); + else + new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser); + } + continue; + } + + /* Queue Packages file (either diff or full packages files, depending + on the users option) - we also check if the PDiff Index file is listed + in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this + instead, but passing the required info to it is to much hassle */ + if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false || + MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true)) new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description, (*Target)->ShortDesc, ExpectedIndexHash); - else - new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, ExpectedIndexHash); + else + new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser); } } - -bool pkgAcqMetaIndex::VerifyVendor(string Message) + /*}}}*/ +bool pkgAcqMetaIndex::VerifyVendor(string Message) /*{{{*/ { -// // Maybe this should be made available from above so we don't have -// // to read and parse it every time? -// pkgVendorList List; -// List.ReadMainList(); - -// const Vendor* Vndr = NULL; -// for (std::vector::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++) -// { -// string::size_type pos = (*I).find("VALIDSIG "); -// if (_config->FindB("Debug::Vendor", false)) -// std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos -// << std::endl; -// if (pos != std::string::npos) -// { -// string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG")); -// if (_config->FindB("Debug::Vendor", false)) -// std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." << -// std::endl; -// Vndr = List.FindVendor(Fingerprint) != ""; -// if (Vndr != NULL); -// break; -// } -// } string::size_type pos; // check for missing sigs (that where not fatal because otherwise we had @@ -1179,6 +1496,19 @@ bool pkgAcqMetaIndex::VerifyVendor(string Message) Transformed = ""; } + if (_config->FindB("Acquire::Check-Valid-Until", true) == true && + MetaIndexParser->GetValidUntil() > 0) { + time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil(); + if (invalid_since > 0) + // TRANSLATOR: The first %s is the URL of the bad Release file, the second is + // the time since then the file is invalid - formated in the same way as in + // the download progress display (e.g. 7d 3h 42min 1s) + return _error->Error( + _("Release file for %s is expired (invalid since %s). " + "Updates for this repository will not be applied."), + RealURI.c_str(), TimeToStr(invalid_since).c_str()); + } + if (_config->FindB("Debug::pkgAcquire::Auth", false)) { std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl; @@ -1196,7 +1526,7 @@ bool pkgAcqMetaIndex::VerifyVendor(string Message) // return false; if (!Transformed.empty()) { - _error->Warning("Conflicting distribution: %s (expected %s but got %s)", + _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"), Desc.Description.c_str(), Transformed.c_str(), MetaIndexParser->GetDist().c_str()); @@ -1205,9 +1535,8 @@ bool pkgAcqMetaIndex::VerifyVendor(string Message) return true; } - /*}}}*/ -// pkgAcqMetaIndex::Failed - no Release file present or no signature -// file present /*{{{*/ + /*}}}*/ +// pkgAcqMetaIndex::Failed - no Release file present or no signature file present /*{{{*/ // --------------------------------------------------------------------- /* */ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) @@ -1215,22 +1544,32 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) if (AuthPass == true) { // gpgv method failed, if we have a good signature - string LastGoodSigFile = _config->FindDir("Dir::State::lists") + - "partial/" + URItoFileName(RealURI) + ".gpg.reverify"; + string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI)); + if (DestFile != SigFile) + LastGoodSigFile.append(".gpg"); + LastGoodSigFile.append(".reverify"); + if(FileExists(LastGoodSigFile)) { - string VerifiedSigFile = _config->FindDir("Dir::State::lists") + - URItoFileName(RealURI) + ".gpg"; - Rename(LastGoodSigFile,VerifiedSigFile); + string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); + if (DestFile != SigFile) + VerifiedSigFile.append(".gpg"); + Rename(LastGoodSigFile, VerifiedSigFile); Status = StatTransientNetworkError; - _error->Warning(_("A error occurred during the signature " + _error->Warning(_("An error occurred during the signature " "verification. The repository is not updated " - "and the previous index files will be used." + "and the previous index files will be used. " "GPG error: %s: %s\n"), Desc.Description.c_str(), LookupTag(Message,"Message").c_str()); RunScripts("APT::Update::Auth-Failure"); return; + } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) { + /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */ + _error->Error(_("GPG error: %s: %s"), + Desc.Description.c_str(), + LookupTag(Message,"Message").c_str()); + return; } else { _error->Warning(_("GPG error: %s: %s"), Desc.Description.c_str(), @@ -1240,13 +1579,106 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) ReportMirrorFailure("GPGFailure"); } + /* Always move the meta index, even if gpgv failed. This ensures + * that PackageFile objects are correctly filled in */ + if (FileExists(DestFile)) { + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + /* InRelease files become Release files, otherwise + * they would be considered as trusted later on */ + if (SigFile == DestFile) { + RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9, + "Release"); + FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9, + "Release"); + SigFile = FinalFile; + } + Rename(DestFile,FinalFile); + chmod(FinalFile.c_str(),0644); + + DestFile = FinalFile; + } + // No Release file was present, or verification failed, so fall // back to queueing Packages files without verification QueueIndexes(false); } + /*}}}*/ +pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/ + string const &URI, string const &URIDesc, string const &ShortDesc, + string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc, + string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc, + const vector* IndexTargets, + indexRecords* MetaIndexParser) : + pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser), + MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), + MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc) +{ + SigFile = DestFile; + // keep the old InRelease around in case of transistent network errors + string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); + if (RealFileExists(Final) == true) + { + string const LastGoodSig = DestFile + ".reverify"; + Rename(Final,LastGoodSig); + } +} /*}}}*/ +pkgAcqMetaClearSig::~pkgAcqMetaClearSig() /*{{{*/ +{ + // if the file was never queued undo file-changes done in the constructor + if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false) + { + string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); + string const LastGoodSig = DestFile + ".reverify"; + if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true) + Rename(LastGoodSig, Final); + } +} + /*}}}*/ +// pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/ +// --------------------------------------------------------------------- +// FIXME: this can go away once the InRelease file is used widely +string pkgAcqMetaClearSig::Custom600Headers() +{ + string Final = _config->FindDir("Dir::State::lists"); + Final += URItoFileName(RealURI); + + struct stat Buf; + if (stat(Final.c_str(),&Buf) != 0) + { + Final = DestFile + ".reverify"; + if (stat(Final.c_str(),&Buf) != 0) + return "\nIndex-File: true\nFail-Ignore: true\n"; + } + + return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); +} + /*}}}*/ +void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ +{ + if (AuthPass == false) + { + // Remove the 'old' InRelease file if we try Release.gpg now as otherwise + // the file will stay around and gives a false-auth impression (CVE-2012-0214) + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile.append(URItoFileName(RealURI)); + if (FileExists(FinalFile)) + unlink(FinalFile.c_str()); + new pkgAcqMetaSig(Owner, + MetaSigURI, MetaSigURIDesc, MetaSigShortDesc, + MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, + IndexTargets, MetaIndexParser); + if (Cnf->LocalOnly == true || + StringToBool(LookupTag(Message, "Transient-Failure"), false) == false) + Dequeue(); + } + else + pkgAcqMetaIndex::Failed(Message, Cnf); +} + /*}}}*/ // AcqArchive::AcqArchive - Constructor /*{{{*/ // --------------------------------------------------------------------- /* This just sets up the initial fetch environment and queues the first @@ -1273,7 +1705,7 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources, assumption here that all the available sources for this version share the same extension.. */ // Skip not source sources, they do not have file fields. - for (; Vf.end() == false; Vf++) + for (; Vf.end() == false; ++Vf) { if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0) continue; @@ -1297,7 +1729,7 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources, // check if we have one trusted source for the package. if so, switch // to "TrustedOnly" mode - for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++) + for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i) { pkgIndexFile *Index; if (Sources->FindIndex(i.File(),Index) == false) @@ -1321,7 +1753,7 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources, // Select a source if (QueueNext() == false && _error->PendingError() == false) - _error->Error(_("I wasn't able to locate file for the %s package. " + _error->Error(_("I wasn't able to locate a file for the %s package. " "This might mean you need to manually fix this package."), Version.ParentPkg().Name()); } @@ -1332,8 +1764,9 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources, the archive is already available in the cache and stashs the MD5 for checking later. */ bool pkgAcqArchive::QueueNext() -{ - for (; Vf.end() == false; Vf++) +{ + string const ForceHash = _config->Find("Acquire::ForceHash"); + for (; Vf.end() == false; ++Vf) { // Ignore not source sources if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0) @@ -1355,12 +1788,29 @@ bool pkgAcqArchive::QueueNext() return false; string PkgFile = Parse.FileName(); - if(Parse.SHA256Hash() != "") - ExpectedHash = HashString("SHA256", Parse.SHA256Hash()); - else if (Parse.SHA1Hash() != "") - ExpectedHash = HashString("SHA1", Parse.SHA1Hash()); - else - ExpectedHash = HashString("MD5Sum", Parse.MD5Hash()); + if (ForceHash.empty() == false) + { + if(stringcasecmp(ForceHash, "sha512") == 0) + ExpectedHash = HashString("SHA512", Parse.SHA512Hash()); + else if(stringcasecmp(ForceHash, "sha256") == 0) + ExpectedHash = HashString("SHA256", Parse.SHA256Hash()); + else if (stringcasecmp(ForceHash, "sha1") == 0) + ExpectedHash = HashString("SHA1", Parse.SHA1Hash()); + else + ExpectedHash = HashString("MD5Sum", Parse.MD5Hash()); + } + else + { + string Hash; + if ((Hash = Parse.SHA512Hash()).empty() == false) + ExpectedHash = HashString("SHA512", Hash); + else if ((Hash = Parse.SHA256Hash()).empty() == false) + ExpectedHash = HashString("SHA256", Hash); + else if ((Hash = Parse.SHA1Hash()).empty() == false) + ExpectedHash = HashString("SHA1", Hash); + else + ExpectedHash = HashString("MD5Sum", Parse.MD5Hash()); + } if (PkgFile.empty() == true) return _error->Error(_("The package index files are corrupted. No Filename: " "field for package %s."), @@ -1378,7 +1828,7 @@ bool pkgAcqArchive::QueueNext() if (stat(FinalFile.c_str(),&Buf) == 0) { // Make sure the size matches - if ((unsigned)Buf.st_size == Version->Size) + if ((unsigned long long)Buf.st_size == Version->Size) { Complete = true; Local = true; @@ -1397,7 +1847,7 @@ bool pkgAcqArchive::QueueNext() if (stat(FinalFile.c_str(),&Buf) == 0) { // Make sure the size matches - if ((unsigned)Buf.st_size == Version->Size) + if ((unsigned long long)Buf.st_size == Version->Size) { Complete = true; Local = true; @@ -1417,12 +1867,23 @@ bool pkgAcqArchive::QueueNext() if (stat(DestFile.c_str(),&Buf) == 0) { // Hmm, the partial file is too big, erase it - if ((unsigned)Buf.st_size > Version->Size) + if ((unsigned long long)Buf.st_size > Version->Size) unlink(DestFile.c_str()); else PartialSize = Buf.st_size; } - + + // Disables download of archives - useful if no real installation follows, + // e.g. if we are just interested in proposed installation order + if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true) + { + Complete = true; + Local = true; + Status = StatDone; + StoreFilename = DestFile = FinalFile; + return true; + } + // Create the item Local = false; Desc.URI = Index->ArchiveURI(PkgFile); @@ -1431,7 +1892,7 @@ bool pkgAcqArchive::QueueNext() Desc.ShortDesc = Version.ParentPkg().Name(); QueueURI(Desc); - Vf++; + ++Vf; return true; } return false; @@ -1440,7 +1901,7 @@ bool pkgAcqArchive::QueueNext() // AcqArchive::Done - Finished fetching /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash, +void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash, pkgAcquire::MethodConfig *Cfg) { Item::Done(Message,Size,CalcHash,Cfg); @@ -1505,7 +1966,7 @@ void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf) StringToBool(LookupTag(Message,"Transient-Failure"),false) == true) { // Vf = Version.FileList(); - while (Vf.end() == false) Vf++; + while (Vf.end() == false) ++Vf; StoreFilename = string(); Item::Failed(Message,Cnf); return; @@ -1529,14 +1990,13 @@ void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf) } } /*}}}*/ -// AcqArchive::IsTrusted - Determine whether this archive comes from a -// trusted source /*{{{*/ +// AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/ // --------------------------------------------------------------------- bool pkgAcqArchive::IsTrusted() { return Trusted; } - + /*}}}*/ // AcqArchive::Finished - Fetching has finished, tidy up /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -1548,14 +2008,14 @@ void pkgAcqArchive::Finished() StoreFilename = string(); } /*}}}*/ - // AcqFile::pkgAcqFile - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The file is added to the queue */ pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash, - unsigned long Size,string Dsc,string ShortDesc, - const string &DestDir, const string &DestFilename) : - Item(Owner), ExpectedHash(Hash) + unsigned long long Size,string Dsc,string ShortDesc, + const string &DestDir, const string &DestFilename, + bool IsIndexFile) : + Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile) { Retries = _config->FindI("Acquire::Retries",0); @@ -1580,7 +2040,7 @@ pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash, if (stat(DestFile.c_str(),&Buf) == 0) { // Hmm, the partial file is too big, erase it - if ((unsigned)Buf.st_size > Size) + if ((unsigned long long)Buf.st_size > Size) unlink(DestFile.c_str()); else PartialSize = Buf.st_size; @@ -1592,7 +2052,7 @@ pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash, // AcqFile::Done - Item downloaded OK /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash, +void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash, pkgAcquire::MethodConfig *Cnf) { Item::Done(Message,Size,CalcHash,Cnf); @@ -1601,7 +2061,7 @@ void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash, if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash) { Status = StatError; - ErrorText = "Hash Sum mismatch"; + ErrorText = _("Hash Sum mismatch"); Rename(DestFile,DestFile + ".FAILED"); return; } @@ -1670,3 +2130,13 @@ void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Item::Failed(Message,Cnf); } /*}}}*/ +// AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ +// --------------------------------------------------------------------- +/* The only header we use is the last-modified header. */ +string pkgAcqFile::Custom600Headers() +{ + if (IsIndexFile) + return "\nIndex-File: true"; + return ""; +} + /*}}}*/