use filesize of compressed pdiffs for the limit if possible

With the addition of the $HASH-Download field in the .diff/Index we got
the size of the compressed patches for 'free', so if that information is
available we can use it for a more fitting calculation of the size
requirements of the patches vs. the complete file.

Note that this predicts a too small size in the transition case in which
the information isn't available for all patches, but figuring this out
would be a lot of code for practically nothing as only one update can
ever be in such a transition phase.

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 7c7a204c46a6ef8a0f80aee9fe268d1e1c1367e5..b73c4985f0d4dd3ad446a3e9e4fe8840a5ec29a8 100644 (file)
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -2032,18 +2032,55 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)       /*{{{*/
    }
 
    // calculate the size of all patches we have to get
-   // note that all sizes are uncompressed, while we download compressed files
-   unsigned long long patchesSize = 0;
-   for (std::vector<DiffInfo>::const_iterator cur = available_patches.begin();
-        cur != available_patches.end(); ++cur)
-      patchesSize += cur->patch_hashes.FileSize();
-   unsigned long long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
-   if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
-   {
-      if (Debug)
-        std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
-           << ") so fallback to complete download" << std::endl;
-      return false;
+   unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100);
+   if (sizeLimitPercent > 0 && TransactionManager->MetaIndexParser != nullptr)
+   {
+      // compressed case
+      unsigned long long downloadSize = std::accumulate(available_patches.begin(),
+           available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) {
+           return T + I.download_hashes.FileSize();
+           });
+      if (downloadSize != 0)
+      {
+        unsigned long long downloadSizeIdx = 0;
+        auto const types = VectorizeString(Target.Option(IndexTarget::COMPRESSIONTYPES), ' ');
+        for (auto const &t : types)
+        {
+           std::string MetaKey = Target.MetaKey;
+           if (t != "uncompressed")
+              MetaKey += '.' + t;
+           HashStringList const hsl = GetExpectedHashesFor(MetaKey);
+           if (unlikely(hsl.usable() == false))
+              continue;
+           downloadSizeIdx = hsl.FileSize();
+           break;
+        }
+        unsigned long long const sizeLimit = downloadSizeIdx * sizeLimitPercent;
+        if ((sizeLimit/100) < downloadSize)
+        {
+           if (Debug)
+              std::clog << "Need " << downloadSize << " compressed bytes (Limit is " << (sizeLimit/100) << ", "
+                 << "original is " << downloadSizeIdx << ") so fallback to complete download" << std::endl;
+           return false;
+        }
+      }
+      // uncompressed case
+      downloadSize = std::accumulate(available_patches.begin(),
+           available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) {
+           return T + I.patch_hashes.FileSize();
+           });
+      if (downloadSize != 0)
+      {
+        unsigned long long const downloadSizeIdx = ServerSize;
+        unsigned long long const sizeLimit = downloadSizeIdx * sizeLimitPercent;
+        if ((sizeLimit/100) < downloadSize)
+        {
+           if (Debug)
+              std::clog << "Need " << downloadSize << " uncompressed bytes (Limit is " << (sizeLimit/100) << ", "
+                 << "original is " << downloadSizeIdx << ") so fallback to complete download" << std::endl;
+           return false;
+        }
+      }
    }
 
    // we have something, queue the diffs

diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
index 32240da4e9314f42948a9f1e2fb57894c1b1f736..bf7779e4031b58c2a0affd34cce536baea0b30c0 100755 (executable)
--- a/test/integration/test-pdiff-usage
+++ b/test/integration/test-pdiff-usage
@@ -215,20 +215,24 @@ SHA256-Download:
        diff -e "${PKGFILE}" "${PKGFILE}-new" > "${PATCHFILE}" || true
        cat "$PATCHFILE" | gzip > "${PATCHFILE}.gz"
        PATCHINDEX='aptarchive/Packages.diff/Index'
+       BIGSIZE="$(stat -c%s "$PATCHFILE")"
+       if ! $DOWNLOADHASH; then
+               BIGSIZE="${BIGSIZE}000"
+       fi
        echo "SHA1-Current: $(sha1sum "${PKGFILE}-new" | cut -d' ' -f 1) $(stat -c%s "${PKGFILE}-new")
 SHA1-History:
  9f4148e06d7faa37062994ff10d0c842d7017513 33053002 2010-08-18-2013.28
  $(sha1sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
 SHA1-Patches:
  7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-2013.28
- $(sha1sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE")000 $(basename "$PATCHFILE")
+ $(sha1sum "$PATCHFILE" | cut -d' ' -f 1) $BIGSIZE $(basename "$PATCHFILE")
 SHA256-Current: $(sha256sum "${PKGFILE}-new" | cut -d' ' -f 1) $(stat -c%s "${PKGFILE}-new")
 SHA256-History:
  01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b 33053002 2010-08-18-2013.28
  $(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
- $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE")000 $(basename "$PATCHFILE")" > "$PATCHINDEX"
+ $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $BIGSIZE $(basename "$PATCHFILE")" > "$PATCHINDEX"
        if $DOWNLOADHASH; then
                echo "SHA1-Download:
  2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz