From 50bd6fd3794dd1f61185302129dc6cd218d20b98 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 15 Jan 2014 17:23:05 +0100 Subject: [PATCH 1/1] integrate Anthonys rred with POC for client-side merge Providing the benefits of both without the downsides :) (ABI breaks or external dependencies) For this Anthonys rred is equipped with: - magic-filename-pickup of patches rather than explicit messages - use of FileFd instead of FILE* to get on-the-fly uncompress of the gzip compressed pdiff patches The acquire code in turn stops checking for apt-file's helper as our own rred is now clever enough for our needs. --- apt-pkg/acquire-item.cc | 11 +--- methods/rred.cc | 94 +++++++++---------------------- test/integration/test-pdiff-usage | 23 +++++--- 3 files changed, 45 insertions(+), 83 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 7f6443555..1185908f3 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -513,14 +513,9 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true); if (pdiff_merge == true) { - // this perl script is provided by apt-file - pdiff_merge = FileExists(_config->FindFile("Dir::Bin::rred", "/usr/bin/diffindex-rred")); - if (pdiff_merge == true) - { - // reprepro adds this flag if it has merged patches on the server - std::string const precedence = Tags.FindS("X-Patch-Precedence"); - pdiff_merge = (precedence != "merged"); - } + // reprepro adds this flag if it has merged patches on the server + std::string const precedence = Tags.FindS("X-Patch-Precedence"); + pdiff_merge = (precedence != "merged"); } if (pdiff_merge == false) diff --git a/methods/rred.cc b/methods/rred.cc index ed3fcc82e..313166160 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -407,13 +407,13 @@ class Patch { public: - void read_diff(FILE *f) + void read_diff(FileFd &f) { char buffer[BLOCK_SIZE]; bool cmdwanted = true; Change ch(0); - while(fgets(buffer, sizeof(buffer), f)) + while(f.ReadLine(buffer, sizeof(buffer))) { if (cmdwanted) { char *m, *c; @@ -534,66 +534,11 @@ class Patch { } }; -bool LookupPatches(const std::string &Message, std::vector &lines) -{ - const char *Tag = "Patches"; - const size_t Length = strlen(Tag); - - std::string::const_iterator I, J; - - std::clog << "Looking for \"Patches:\" section in message:\n\n" << Message << "\n\n"; - std::clog.flush(); - - for (I = Message.begin(); I + Length < Message.end(); ++I) - { - if (I[Length] == ':' && stringcasecmp(I, I+Length, Tag) == 0) - { - // found the tag, now read the patches - for(;;) { - for (; I < Message.end() && *I != '\n'; ++I); - if (I < Message.end()) I++; - if (I == Message.end() || *I != ' ') - break; - while (I < Message.end() && isspace(*I)) I++; - for (J = I; J < Message.end() && *J != '\n'; ++J) - ; - do - J--; - while (I < J && isspace(*J)); - if (I < J) - lines.push_back(std::string(I,++J)); - else - break; - I = J; - } - std::clog << "Found " << lines.size() << " patches!\n"; - std::clog.flush(); - return true; - } - } - std::clog << "Found no patches! :(\n"; - std::clog.flush(); - return false; -} - - class RredMethod : public pkgAcqMethod { private: bool Debug; - std::vector patchpaths; protected: - virtual bool HandleMessage(int Number, std::string Message) { - if (Number == 600) - { - patchpaths.clear(); - LookupPatches(Message, patchpaths); - std::clog << "Ended up with " << patchpaths.size() << " patches!\n"; - std::clog.flush(); - } - return pkgAcqMethod::HandleMessage(Number, Message); - } - virtual bool Fetch(FetchItem *Itm) { Debug = _config->FindB("Debug::pkgAcquire::RRed", false); URI Get = Itm->Uri; @@ -601,17 +546,29 @@ class RredMethod : public pkgAcqMethod { FetchResult Res; Res.Filename = Itm->DestFile; - if (Itm->Uri.empty()) { + if (Itm->Uri.empty()) + { Path = Itm->DestFile; Itm->DestFile.append(".result"); } else URIStart(Res); + std::vector patchpaths; Patch patch; - if (patchpaths.empty()) - { + if (FileExists(Path + ".ed") == true) patchpaths.push_back(Path + ".ed"); + else + { + _error->PushToStack(); + std::vector patches = GetListOfFilesInDir(flNotFile(Path), "gz", true, false); + _error->RevertToStack(); + + std::string const baseName = Path + ".ed."; + for (std::vector::const_iterator p = patches.begin(); + p != patches.end(); ++p) + if (p->compare(0, baseName.length(), baseName) == 0) + patchpaths.push_back(*p); } std::string patch_name; @@ -624,13 +581,15 @@ class RredMethod : public pkgAcqMethod { std::clog << "Patching " << Path << " with " << patch_name << std::endl; - FILE *p = fopen(patch_name.c_str(), "r"); - if (p == NULL) { - std::clog << "Could not open patch file " << patch_name << std::endl; + FileFd p; + // all patches are compressed, even if the name doesn't reflect it + if (p.Open(patch_name, FileFd::ReadOnly, FileFd::Gzip) == false) { + std::cerr << "Could not open patch file " << patch_name << std::endl; + _error->DumpErrors(std::cerr); abort(); } patch.read_diff(p); - fclose(p); + p.Close(); } if (Debug == true) @@ -697,10 +656,9 @@ int main(int argc, char **argv) } for (; i < argc; i++) { - FILE *p; - p = fopen(argv[i], "r"); - if (!p) { - perror(argv[i]); + FileFd p; + if (p.Open(argv[i], FileFd::ReadOnly) == false) { + _error->DumpErrors(std::cerr); exit(1); } patch.read_diff(p); diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index 5a06e0ccb..ad31511b9 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -20,10 +20,19 @@ chmod +x extrred echo 'Dir::Bin::rred "./extrred";' > rootdir/etc/apt/apt.conf.d/99rred wasmergeused() { - testsuccess aptget update "$@" + msgtest 'Test for successful execution of' "$*" + local OUTPUT=$(mktemp) + addtrap "rm $OUTPUT;" + if aptget update "$@" >${OUTPUT} 2>&1; then + msgpass + else + echo + cat $OUTPUT + msgfail + fi + msgtest 'Check if the right pdiff merger was used' - if [ -e ./merge-was-used ]; then - rm -f ./merge-was-used + if grep -q '^pkgAcqIndexMergeDiffs::Done(): rred' $OUTPUT; then if echo "$*" | grep -q -- '-o Acquire::PDiffs::Merge=1'; then msgpass else @@ -50,7 +59,7 @@ testrun() { testequal "$(cat ${PKGFILE}) " aptcache show apt oldstuff - # apply with one patch + msgmsg 'Testcase: apply with one patch' cp ${PKGFILE}-new aptarchive/Packages compressfile 'aptarchive/Packages' mkdir -p aptarchive/Packages.diff @@ -73,13 +82,13 @@ SHA1-Patches: testequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff - # index is already up-to-date + msgmsg 'Testcase: index is already up-to-date' find rootdir/var/lib/apt/lists -name '*.IndexDiff' -type f -delete testsuccess aptget update "$@" testequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff - # apply with two patches + msgmsg 'Testcase: apply with two patches' cp ${PKGFILE}-new aptarchive/Packages echo ' Package: futurestuff @@ -120,7 +129,7 @@ SHA1-Patches: testequal "$(cat Packages-future) " aptcache show apt newstuff futurestuff - # patch applying fails, but successful fallback + msgmsg 'Testcase: patch applying fails, but successful fallback' rm -rf rootdir/var/lib/apt/lists cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists cp ${PKGFILE}-new aptarchive/Packages -- 2.45.2