support hashes for compressed pdiff files

At the moment we only have hashes for the uncompressed pdiff files, but
via the new '$HASH-Download' field in the .diff/Index hashes can be
provided for the .gz compressed pdiff file, which apt will pick up now
and use to verify the download. Now, we "just" need a buy in from the
creators of repositories…

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 7b69ee9937ffe72e4f7652d5abb1a7b3135f4717..a3f47242ffe0447653bfcd4ffc3540856192bebf 100644 (file)
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -152,14 +152,18 @@ HashStringList pkgAcqMetaBase::GetExpectedHashes() const
 
 APT_CONST bool pkgAcqIndexDiffs::HashesRequired() const
 {
-   /* FIXME: We have only hashes for uncompressed pdiffs.
-      rred uncompresses them on the fly while parsing.
-      In StateFetchDiff state we also uncompress on the fly for hash check.
-      Hashes are checked while searching for (next) patch to apply. */
+   /* We don't always have the diff of the downloaded pdiff file.
+      What we have for sure is hashes for the uncompressed file,
+      but rred uncompresses them on the fly while parsing, so not handled here.
+      Hashes are (also) checked while searching for (next) patch to apply. */
+   if (State == StateFetchDiff)
+      return available_patches[0].download_hashes.empty() == false;
    return false;
 }
 HashStringList pkgAcqIndexDiffs::GetExpectedHashes() const
 {
+   if (State == StateFetchDiff)
+      return available_patches[0].download_hashes;
    return HashStringList();
 }
 
@@ -168,11 +172,15 @@ APT_CONST bool pkgAcqIndexMergeDiffs::HashesRequired() const
    /* @see #pkgAcqIndexDiffs::HashesRequired, with the difference that
       we can check the rred result after all patches are applied as
       we know the expected result rather than potentially apply more patches */
+   if (State == StateFetchDiff)
+      return patch.download_hashes.empty() == false;
    return State == StateApplyDiff;
 }
 HashStringList pkgAcqIndexMergeDiffs::GetExpectedHashes() const
 {
-   if (State == StateApplyDiff)
+   if (State == StateFetchDiff)
+      return patch.download_hashes;
+   else if (State == StateApplyDiff)
       return GetExpectedHashesFor(Target->MetaKey);
    return HashStringList();
 }
@@ -1618,7 +1626,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
         std::vector<DiffInfo>::iterator cur = available_patches.begin();
         for (; cur != available_patches.end(); ++cur)
         {
-           if (cur->file != filename || unlikely(cur->result_size != size))
+           if (cur->file != filename)
               continue;
            cur->result_hashes.push_back(HashString(*type, hash));
            break;
@@ -1630,8 +1638,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
            DiffInfo next;
            next.file = filename;
            next.result_hashes.push_back(HashString(*type, hash));
-           next.result_size = size;
-           next.patch_size = 0;
+           next.result_hashes.FileSize(size);
            available_patches.push_back(next);
         }
         else
@@ -1679,10 +1686,9 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)        /*{{{*/
         {
            if (cur->file != filename)
               continue;
-           if (unlikely(cur->patch_size != 0 && cur->patch_size != size))
-              continue;
+           if (cur->patch_hashes.empty())
+              cur->patch_hashes.FileSize(size);
            cur->patch_hashes.push_back(HashString(*type, hash));
-           cur->patch_size = size;
            break;
         }
         if (cur != available_patches.end())
@@ -1694,6 +1700,48 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)        /*{{{*/
       }
    }
 
+   for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
+   {
+      std::string tagname = *type;
+      tagname.append("-Download");
+      std::string const tmp = Tags.FindS(tagname.c_str());
+      if (tmp.empty() == true)
+        continue;
+
+      string hash, filename;
+      unsigned long long size;
+      std::stringstream ss(tmp);
+
+      // FIXME: all of pdiff supports only .gz compressed patches
+      while (ss >> hash >> size >> filename)
+      {
+        if (unlikely(hash.empty() == true || filename.empty() == true))
+           continue;
+        if (unlikely(APT::String::Endswith(filename, ".gz") == false))
+           continue;
+        filename.erase(filename.length() - 3);
+
+        // see if we have a record for this file already
+        std::vector<DiffInfo>::iterator cur = available_patches.begin();
+        for (; cur != available_patches.end(); ++cur)
+        {
+           if (cur->file != filename)
+              continue;
+           if (cur->download_hashes.empty())
+              cur->download_hashes.FileSize(size);
+           cur->download_hashes.push_back(HashString(*type, hash));
+           break;
+        }
+        if (cur != available_patches.end())
+              continue;
+        if (Debug == true)
+           std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
+              << " wasn't in the list for the first parsed hash! (download)" << std::endl;
+        break;
+      }
+   }
+
+
    bool foundStart = false;
    for (std::vector<DiffInfo>::iterator cur = available_patches.begin();
         cur != available_patches.end(); ++cur)
@@ -1729,7 +1777,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/
    unsigned long long patchesSize = 0;
    for (std::vector<DiffInfo>::const_iterator cur = available_patches.begin();
         cur != available_patches.end(); ++cur)
-      patchesSize += cur->patch_size;
+      patchesSize += cur->patch_hashes.FileSize();
    unsigned long long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
    if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
    {

diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index f24af1aeca81520f116e1959a1ae8e19524b6a73..910e4131bd21ee069625460c8db3f4556baec9c8 100644 (file)
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -705,17 +705,14 @@ struct APT_HIDDEN DiffInfo {                                              /*{{{*/
    /** The filename of the diff. */
    std::string file;
 
-   /** The hashes of the diff */
+   /** The hashes of the file after the diff is applied */
    HashStringList result_hashes;
 
-   /** The hashes of the file after the diff is applied */
+   /** The hashes of the diff */
    HashStringList patch_hashes;
 
-   /** The size of the file after the diff is applied */
-   unsigned long long result_size;
-
-   /** The size of the diff itself */
-   unsigned long long patch_size;
+   /** The hashes of the compressed diff */
+   HashStringList download_hashes;
 };
                                                                        /*}}}*/
 /** \brief An item that is responsible for fetching client-merge patches {{{

diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc
index 11a7e479b9ac7ddce55de262f805ee9e018556b6..46cf0ba083af22cd62f182a6688df7b5f4669137 100644 (file)
--- a/apt-pkg/contrib/hashes.cc
+++ b/apt-pkg/contrib/hashes.cc
@@ -188,6 +188,13 @@ unsigned long long HashStringList::FileSize() const                        /*{{{*/
    return strtoull(hv.c_str(), NULL, 10);
 }
                                                                        /*}}}*/
+bool HashStringList::FileSize(unsigned long long const Size)           /*{{{*/
+{
+   std::string size;
+   strprintf(size, "%llu", Size);
+   return push_back(HashString("Checksum-FileSize", size));
+}
+                                                                       /*}}}*/
 bool HashStringList::supported(char const * const type)                        /*{{{*/
 {
    for (char const * const * t = HashString::SupportedHashes(); *t != NULL; ++t)
@@ -361,9 +368,7 @@ APT_IGNORE_DEPRECATED_PUSH
    if ((d->CalcHashes & SHA512SUM) == SHA512SUM)
       hashes.push_back(HashString("SHA512", SHA512.Result().Value()));
 APT_IGNORE_DEPRECATED_POP
-   std::string SizeStr;
-   strprintf(SizeStr, "%llu", d->FileSize);
-   hashes.push_back(HashString("Checksum-FileSize", SizeStr));
+   hashes.FileSize(d->FileSize);
    return hashes;
 }
 APT_IGNORE_DEPRECATED_PUSH

diff --git a/apt-pkg/contrib/hashes.h b/apt-pkg/contrib/hashes.h
index 176ce4faa8593a3394c578f91ed2f18dea102e4a..e8d84da9e766c69ccd878fe8b4acc444fb85cbbd 100644 (file)
--- a/apt-pkg/contrib/hashes.h
+++ b/apt-pkg/contrib/hashes.h
@@ -96,6 +96,13 @@ class HashStringList
     */
    unsigned long long FileSize() const;
 
+   /** sets the filesize hash
+    *
+    * @param Size of the file
+    * @return @see #push_back
+    */
+   bool FileSize(unsigned long long const Size);
+
    /** check if the given hash type is supported
     *
     * @param type to check

diff --git a/apt-pkg/indexrecords.cc b/apt-pkg/indexrecords.cc
index c26868cac3d22a73ca036fbbb393597344580565..7e6da9558e017af0ea799b32bc3aa958891d5d03 100644 (file)
--- a/apt-pkg/indexrecords.cc
+++ b/apt-pkg/indexrecords.cc
@@ -121,9 +121,7 @@ bool indexRecords::Load(const string Filename)                              /*{{{*/
             indexRecords::checkSum *Sum = new indexRecords::checkSum;
             Sum->MetaKeyFilename = Name;
             Sum->Size = Size;
-           std::string SizeStr;
-           strprintf(SizeStr, "%llu", Size);
-           Sum->Hashes.push_back(HashString("Checksum-FileSize", SizeStr));
+           Sum->Hashes.FileSize(Size);
             APT_IGNORE_DEPRECATED(Sum->Hash = HashString(HashString::SupportedHashes()[i],Hash);)
             Entries[Name] = Sum;
          }

diff --git a/methods/rred.cc b/methods/rred.cc
index 81ecf855318610a33a42089034901d8f7fe2b320..12cf2b4a5dc4a0d9b3f3d0e83e473581f1d5551b 100644 (file)
--- a/methods/rred.cc
+++ b/methods/rred.cc
@@ -627,7 +627,7 @@ class RredMethod : public pkgAcqMethod {
            p.Close();
            HashStringList const hsl = patch_hash.GetHashStringList();
            if (hsl != I->ExpectedHashes)
-              return _error->Error("Patch %s doesn't have the expected hashsum", patch_name.c_str());
+              return _error->Error("Hash Sum mismatch for uncompressed patch %s", patch_name.c_str());
         }
 
         if (Debug == true)

diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
index 7a9f6496b5c8beda81363b02acce08661a03ae84..3295d549752e69dfa33cfb89c0bbc23ed581b93b 100755 (executable)
--- a/test/integration/test-pdiff-usage
+++ b/test/integration/test-pdiff-usage
@@ -42,6 +42,8 @@ wasmergeused() {
 
 testrun() {
        msgmsg "Testcase: setup the base with: $*"
+       local DOWNLOADHASH=true
+       if [ "$1" = 'nohash' ]; then DOWNLOADHASH=false; shift; fi
        find aptarchive -name 'Packages*' -type f -delete
        cp ${PKGFILE} aptarchive/Packages
        compressfile 'aptarchive/Packages'
@@ -76,6 +78,15 @@ SHA256-History:
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
  $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX
+       if $DOWNLOADHASH; then
+               echo "SHA1-Download:
+ 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
+ $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
+SHA256-Download:
+ d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
+ $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX
+       fi
+
        generatereleasefiles '+1hour'
        signreleasefiles
        find aptarchive -name 'Packages*' -type f -delete
@@ -131,6 +142,17 @@ SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
  $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)
  $(sha256sum ${PATCHFILE2} | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}) $(basename ${PATCHFILE2})" > $PATCHINDEX
+       if $DOWNLOADHASH; then
+               echo "SHA1-Download:
+ 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
+ $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
+ $(sha1sum ${PATCHFILE2}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}.gz) $(basename ${PATCHFILE2}.gz)
+SHA256-Download:
+ d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
+ $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
+ $(sha256sum ${PATCHFILE2}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}.gz) $(basename ${PATCHFILE2}.gz)" >> $PATCHINDEX
+       fi
+
        generatereleasefiles '+2hour'
        signreleasefiles
        cp -a aptarchive/Packages Packages-future
@@ -150,6 +172,7 @@ SHA256-Patches:
        mkdir -p aptarchive/Packages.diff
        PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)"
        diff -e ${PKGFILE} ${PKGFILE}-new > ${PATCHFILE} || true
+       cat $PATCHFILE | gzip > ${PATCHFILE}.gz
        PATCHINDEX='aptarchive/Packages.diff/Index'
        echo "SHA1-Current: $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new)
 SHA1-History:
@@ -165,14 +188,22 @@ SHA256-History:
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
  $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX
+       if $DOWNLOADHASH; then
+               echo "SHA1-Download:
+ 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
+ $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
+SHA256-Download:
+ d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
+ $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX
+       fi
        # needs to look like a valid command, otherwise the parser will fail before hashes are checked
-       echo '1d' >> $PATCHFILE
+       echo '1d' > $PATCHFILE
        cat $PATCHFILE | gzip > ${PATCHFILE}.gz
        generatereleasefiles '+1hour'
        signreleasefiles
        testsuccess aptget update "$@"
        cp -f rootdir/tmp/testsuccess.output rootdir/tmp/aptgetupdate.output
-       testsuccess grep 'have the expected hashsum' rootdir/tmp/aptgetupdate.output
+       testsuccess grep 'Hash Sum mismatch' rootdir/tmp/aptgetupdate.output
        testnopackage oldstuff
        testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt newstuff
@@ -201,6 +232,14 @@ SHA256-History:
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
  $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE)000 $(basename $PATCHFILE)" > $PATCHINDEX
+       if $DOWNLOADHASH; then
+               echo "SHA1-Download:
+ 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
+ $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz)000 $(basename ${PATCHFILE}.gz)
+SHA256-Download:
+ d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
+ $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz)000 $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX
+       fi
        generatereleasefiles '+1hour'
        signreleasefiles
        #find aptarchive -name 'Packages*' -type f -delete
@@ -215,6 +254,9 @@ echo 'Debug::pkgAcquire::Diffs "true";
 Debug::Acquire::Transaction "true";
 Debug::pkgAcquire::rred "true";' > rootdir/etc/apt/apt.conf.d/rreddebug.conf
 
+testrun nohash -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1
+testrun nohash -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1
+
 testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1
 testrun -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1
 testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=0