From: Michael Vogt Date: Mon, 17 Oct 2005 14:56:57 +0000 (+0000) Subject: * merged pdiff support X-Git-Tag: 0.7.21~284^2~51 X-Git-Url: https://git.saurik.com/apt.git/commitdiff_plain/5e9d46df79517125d0fd72e7671f5e0995746677?hp=-c * merged pdiff support Patches applied: * michael.vogt@canonical.com--2004/apt--bwlimit--0--base-0 tag of apt@packages.debian.org/apt--main--0--patch-47 * michael.vogt@canonical.com--2004/apt--bwlimit--0--patch-1 * fist attempt for bwlimit * michael.vogt@canonical.com--2004/apt--bwlimit--0--patch-2 * add a nanosleep, making it less cpu intensive * michael.vogt@canonical.com--2004/apt--bwlimit--0--patch-3 * use SingleInstance when DlLimit is activated * michael.vogt@canonical.com--2004/apt--bwlimit--0--patch-4 * cleanups, no code changes * michael.vogt@ubuntu.com--2005/apt--bwlimit--0--base-0 tag of apt@packages.debian.org/apt--main--0--patch-76 * michael.vogt@ubuntu.com--2005/apt--bwlimit--0--patch-1 * merged the apt--main with the 2004 bwlimit tree * michael.vogt@ubuntu.com--2005/apt--local-install--0--base-0 tag of michael.vogt@ubuntu.com--2005/apt--bwlimit--0--patch-1 * michael.vogt@ubuntu.com--2005/apt--pdiff--0--base-0 tag of michael.vogt@ubuntu.com--2005/apt--local-install--0--base-0 * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-1 * first rewrite of the apt pdiff support patch finished * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-2 * use "red" now, print meaningfull pdiff names, use IMS for the IndexDiff * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-3 * merge with matt, search for the right patch next instead of applying them in order * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-4 * fix a bug QueueNextDiff * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-5 * another bug in QueueNextDiff fixed * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-6 * merged with apt--main and apt--fixes * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-7 * added methods/rred.cc (thanks to robertle) * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-8 * use the new cool rred method for the patchting * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-9 * merged with mainline * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-10 * rred.cc: use the d'tor of FileFd to close the file-fd; acquuire-item.cc: make the QueueNextDiff() code more robust * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-11 * bugfix/updates in the rred.cc code (big thanks for helping Robert!) * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-12 * method/rred.cc cleanup (thanks to robertle) * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-13 * debug support (Debug::pkgAcquire::RRed) for rred method added * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-14 * split the diff acquire code in a bit that gets the index file and one that gets the patches. that make the code cleaner and the list-cleaner happy * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-15 * fixed apt-pkg/acquire-item.h (looks like the last commit didn't include it?) * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-16 * fix a missing StatDone if nothing needs to be fetched * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-17 * be less verbose and call the Url-Remap: APT::URL-Remap * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-18 * dont bump the soname, it doesn't break the abi! * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-19 * merged with michael.vogt@ubuntu.com--2005/apt--mvo--0 to be in sync with debian * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-20 * don't forget the final md5sum checking! * michael.vogt@ubuntu.com--2005/apt--pdiff--0--patch-21 * removed the URL-Reamp hack (we have pdiff support in the main archive now), removed bw-limit support (NO idea how that got into this archive in the first place) --- 5e9d46df79517125d0fd72e7671f5e0995746677 diff --combined apt-pkg/acquire-item.cc index e0e0611f0,8a9a4c5bb..75a7faa2a --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@@ -24,6 -24,8 +24,8 @@@ #include #include #include + #include + #include #include @@@ -31,6 -33,7 +33,7 @@@ #include #include #include + #include #include /*}}}*/ @@@ -131,14 -134,410 +134,410 @@@ void pkgAcquire::Item::Rename(string Fr } /*}}}*/ + + // AcqDiffIndex::AcqDiffIndex - Constructor + // --------------------------------------------------------------------- + /* Get the DiffIndex file first and see if there are patches availabe + * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the + * patches. If anything goes wrong in that process, it will fall back to + * the original packages file + */ + pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, + string URI,string URIDesc,string ShortDesc, + string ExpectedMD5) + : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), Description(URIDesc) + { + + Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + + Desc.Description = URIDesc + "/DiffIndex"; + Desc.Owner = this; + Desc.ShortDesc = ShortDesc; + Desc.URI = URI + ".diff/Index"; + + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(URI) + string(".DiffIndex"); + + if(Debug) + std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl; + + // look for the current package file + CurrentPackagesFile = _config->FindDir("Dir::State::lists"); + CurrentPackagesFile += URItoFileName(RealURI); + + if(!FileExists(CurrentPackagesFile) || + !_config->FindB("Acquire::Diffs",true)) { + // we don't have a pkg file or we don't want to queue + if(Debug) + std::clog << "No index file or canceld by user" << std::endl; + Failed("", NULL); + return; + } + + if(Debug) { + std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): " + << CurrentPackagesFile << std::endl; + } + + QueueURI(Desc); + + } + + // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ + // --------------------------------------------------------------------- + /* The only header we use is the last-modified header. */ + string pkgAcqDiffIndex::Custom600Headers() + { + string Final = _config->FindDir("Dir::State::lists"); + Final += URItoFileName(RealURI) + string(".IndexDiff"); + + if(Debug) + std::clog << "Custom600Header-IMS: " << Final << std::endl; + + struct stat Buf; + if (stat(Final.c_str(),&Buf) != 0) + return "\nIndex-File: true"; + + return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); + } + + + bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) + { + if(Debug) + std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile + << std::endl; + + pkgTagSection Tags; + string ServerSha1; + vector available_patches; + + FileFd Fd(IndexDiffFile,FileFd::ReadOnly); + pkgTagFile TF(&Fd); + if (_error->PendingError() == true) + return false; + + if(TF.Step(Tags) == true) + { + string local_sha1; + bool found = false; + DiffInfo d; + string size; + + string tmp = Tags.FindS("SHA1-Current"); + std::stringstream ss(tmp); + ss >> ServerSha1; + + FileFd fd(CurrentPackagesFile, FileFd::ReadOnly); + SHA1Summation SHA1; + SHA1.AddFD(fd.Fd(), fd.Size()); + local_sha1 = string(SHA1.Result()); + + if(local_sha1 == ServerSha1) { + if(Debug) + std::clog << "Package file is up-to-date" << std::endl; + // set found to true, this will queue a pkgAcqIndexDiffs with + // a empty availabe_patches + found = true; + } else { + if(Debug) + std::clog << "SHA1-Current: " << ServerSha1 << std::endl; + + // check the historie and see what patches we need + string history = Tags.FindS("SHA1-History"); + std::stringstream hist(history); + while(hist >> d.sha1 >> size >> d.file) { + d.size = atoi(size.c_str()); + // read until the first match is found + if(d.sha1 == local_sha1) + found=true; + // from that point on, we probably need all diffs + if(found) { + if(Debug) + std::clog << "Need to get diff: " << d.file << std::endl; + available_patches.push_back(d); + } + } + } + + // no information how to get the patches, bail out + if(!found) { + if(Debug) + std::clog << "Can't find a patch in the index file" << std::endl; + // Failed will queue a big package file + Failed("", NULL); + } else { + // queue the diffs + new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, + ExpectedMD5, available_patches); + Complete = false; + Status = StatDone; + Dequeue(); + return true; + } + } + + return false; + } + + void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) + { + if(Debug) + std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl + << "Falling back to normal index file aquire" << std::endl; + + new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc, + ExpectedMD5); + + Complete = false; + Status = StatDone; + Dequeue(); + } + + void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash, + pkgAcquire::MethodConfig *Cnf) + { + if(Debug) + std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl; + + Item::Done(Message,Size,Md5Hash,Cnf); + + string FinalFile; + FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); + + // sucess in downloading the index + // rename the index + FinalFile += string(".IndexDiff"); + if(Debug) + std::clog << "Renaming: " << DestFile << " -> " << FinalFile + << std::endl; + Rename(DestFile,FinalFile); + chmod(FinalFile.c_str(),0644); + DestFile = FinalFile; + + if(!ParseDiffIndex(DestFile)) + return Failed("", NULL); + + Complete = true; + Status = StatDone; + Dequeue(); + return; + } + + + + // AcqIndexDiffs::AcqIndexDiffs - Constructor + // --------------------------------------------------------------------- + /* The package diff is added to the queue. one object is constructed + * for each diff and the index + */ + pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, + string URI,string URIDesc,string ShortDesc, + string ExpectedMD5, vector diffs) + : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), + available_patches(diffs) + { + + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(URI); + + Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); + + Desc.Description = URIDesc; + Desc.Owner = this; + Desc.ShortDesc = ShortDesc; + + if(available_patches.size() == 0) { + // we are done (yeah!) + Finish(true); + } else { + // get the next diff + State = StateFetchDiff; + QueueNextDiff(); + } + } + + + void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf) + { + if(Debug) + std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl + << "Falling back to normal index file aquire" << std::endl; + new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc, + ExpectedMD5); + Finish(); + } + + + // helper that cleans the item out of the fetcher queue + void pkgAcqIndexDiffs::Finish(bool allDone) + { + // we restore the original name, this is required, otherwise + // the file will be cleaned + if(allDone) { + DestFile = _config->FindDir("Dir::State::lists"); + DestFile += URItoFileName(RealURI); + + // do the final md5sum checking + MD5Summation sum; + FileFd Fd(DestFile, FileFd::ReadOnly); + sum.AddFD(Fd.Fd(), Fd.Size()); + Fd.Close(); + string MD5 = (string)sum.Result(); + + if (!ExpectedMD5.empty() && MD5 != ExpectedMD5) + { + Status = StatAuthError; + ErrorText = _("MD5Sum mismatch"); + Rename(DestFile,DestFile + ".FAILED"); + Dequeue(); + return; + } + + // this is for the "real" finish + Complete = true; + Status = StatDone; + Dequeue(); + if(Debug) + std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl; + return; + } + + if(Debug) + std::clog << "Finishing: " << Desc.URI << std::endl; + Complete = false; + Status = StatDone; + Dequeue(); + return; + } + + + + bool pkgAcqIndexDiffs::QueueNextDiff() + { + + // calc sha1 of the just patched file + string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(RealURI); + + FileFd fd(FinalFile, FileFd::ReadOnly); + SHA1Summation SHA1; + SHA1.AddFD(fd.Fd(), fd.Size()); + string local_sha1 = string(SHA1.Result()); + if(Debug) + std::clog << "QueueNextDiff: " + << FinalFile << " (" << local_sha1 << ")"<::iterator I=available_patches.begin(); + available_patches.size() > 0 && I != available_patches.end() + && (*I).sha1 != local_sha1; + I++) { + available_patches.erase(I); + } + + // error checking and falling back if no patch was found + if(available_patches.size() == 0) { + Failed("", NULL); + return false; + } + + // queue the right diff + Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz"; + Desc.Description = available_patches[0].file + string(".pdiff"); + + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file); + + if(Debug) + std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl; + + QueueURI(Desc); + + return true; + } + + + + void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash, + pkgAcquire::MethodConfig *Cnf) + { + if(Debug) + std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl; + + Item::Done(Message,Size,Md5Hash,Cnf); + + string FinalFile; + FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); + + // sucess in downloading a diff, enter ApplyDiff state + if(State == StateFetchDiff) + { + + if(Debug) + std::clog << "Sending to gzip method: " << FinalFile << std::endl; + + string FileName = LookupTag(Message,"Filename"); + State = StateUnzipDiff; + Desc.URI = "gzip:" + FileName; + DestFile += ".decomp"; + QueueURI(Desc); + Mode = "gzip"; + return; + } + + // sucess in downloading a diff, enter ApplyDiff state + if(State == StateUnzipDiff) + { + + // rred excepts the patch as $FinalFile.ed + Rename(DestFile,FinalFile+".ed"); + + if(Debug) + std::clog << "Sending to rred method: " << FinalFile << std::endl; + + State = StateApplyDiff; + Desc.URI = "rred:" + FinalFile; + QueueURI(Desc); + Mode = "rred"; + return; + } + + + // success in download/apply a diff, queue next (if needed) + if(State == StateApplyDiff) + { + // remove the just applied patch + available_patches.erase(available_patches.begin()); + + // move into place + if(Debug) + { + std::clog << "Moving patched file in place: " << std::endl + << DestFile << " -> " << FinalFile << std::endl; + } + Rename(DestFile,FinalFile); + + // see if there is more to download + if(available_patches.size() > 0) { + new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, + ExpectedMD5, available_patches); + return Finish(); + } else + return Finish(true); + } + } + + // AcqIndex::AcqIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The package file is added to the queue and a second class is instantiated to fetch the revision file */ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, - string ExpectedMD5, string comprExt) : - Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5) + string ExpectedMD5, string comprExt) + : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5) { Decompression = false; Erase = false; @@@ -308,35 -707,6 +707,35 @@@ void pkgAcqIndex::Done(string Message,u Mode = decompProg; } +// AcqIndexTrans::pkgAcqIndexTrans - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* The Translation file is added to the queue */ +pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, + string URI,string URIDesc,string ShortDesc) : + pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, "", "") +{ +} + + /*}}}*/ +// AcqIndexTrans::Failed - Silence failure messages for missing files /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) +{ + if (Cnf->LocalOnly == true || + StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) + { + // Ignore this + Status = StatDone; + Complete = false; + Dequeue(); + return; + } + + Item::Failed(Message,Cnf); +} + /*}}}*/ + pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, string MetaIndexURI, string MetaIndexURIDesc, @@@ -636,8 -1006,8 +1035,8 @@@ void pkgAcqMetaIndex::QueueIndexes(boo } // Queue Packages file - new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, ExpectedIndexMD5); + new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description, + (*Target)->ShortDesc, ExpectedIndexMD5); } } diff --combined apt-pkg/acquire-item.h index ae2fba4d5,3a6e93b1a..34fcc2a76 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@@ -9,8 -9,8 +9,8 @@@ the Owner Acquire class. Derived classes will then call QueueURI to register all the URI's they wish to fetch at the initial moment. - Two item classes are provided to provide functionality for downloading - of Index files and downloading of Packages. + Three item classes are provided to provide functionality for + downloading of Index, Translation and Packages files. A Archive class is provided for downloading .deb files. It does Md5 checking and source location as well as a retry algorithm. @@@ -82,6 -82,70 +82,70 @@@ class pkgAcquire::Ite virtual ~Item(); }; + // item for index diffs + + struct DiffInfo { + string file; + string sha1; + unsigned long size; + }; + + class pkgAcqDiffIndex : public pkgAcquire::Item + { + protected: + bool Debug; + pkgAcquire::ItemDesc Desc; + string RealURI; + string ExpectedMD5; + string CurrentPackagesFile; + string Description; + + public: + // Specialized action members + virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); + virtual void Done(string Message,unsigned long Size,string Md5Hash, + pkgAcquire::MethodConfig *Cnf); + virtual string DescURI() {return RealURI + "Index";}; + virtual string Custom600Headers(); + + // helpers + bool ParseDiffIndex(string IndexDiffFile); + + pkgAcqDiffIndex(pkgAcquire *Owner,string URI,string URIDesc, + string ShortDesct, string ExpectedMD5); + }; + + class pkgAcqIndexDiffs : public pkgAcquire::Item + { + protected: + bool Debug; + pkgAcquire::ItemDesc Desc; + string RealURI; + string ExpectedMD5; + + // this is the SHA-1 sum we expect after the patching + string Description; + vector available_patches; + enum {StateFetchIndex,StateFetchDiff,StateUnzipDiff,StateApplyDiff} State; + + public: + + // Specialized action members + virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); + virtual void Done(string Message,unsigned long Size,string Md5Hash, + pkgAcquire::MethodConfig *Cnf); + virtual string DescURI() {return RealURI + "Index";}; + + // various helpers + bool QueueNextDiff(); + bool ApplyDiff(string PatchFile); + void Finish(bool allDone=false); + + pkgAcqIndexDiffs(pkgAcquire *Owner,string URI,string URIDesc, + string ShortDesct, string ExpectedMD5, + vector diffs=vector()); + }; + // Item class for index files class pkgAcqIndex : public pkgAcquire::Item { @@@ -107,16 -171,6 +171,16 @@@ string ShortDesct, string ExpectedMD5, string compressExt=""); }; +// Item class for translated package index files +class pkgAcqIndexTrans : public pkgAcqIndex +{ + public: + + virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); + pkgAcqIndexTrans(pkgAcquire *Owner,string URI,string URIDesc, + string ShortDesct); +}; + struct IndexTarget { string URI;