silently skip acquire of empty index files

There is just no point in taking the time to acquire empty files –
especially as it will be tiny non-empty compressed files usually.

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 0569c6ddabdfa0a0266a33d57daef62366695371..bc5e91fed32ed2e72fb18f4bdf7a2f3ad490c43e 100644 (file)
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -1151,11 +1151,17 @@ void pkgAcqMetaBase::QueueIndexes(bool const verify)                    /*{{{*/
         else
         {
            auto const hashes = GetExpectedHashesFor(Target->MetaKey);
-           if (hashes.usable() == false && hashes.empty() == false)
+           if (hashes.empty() == false)
            {
-              _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"),
+              if (hashes.usable() == false)
+              {
+                 _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' provides only weak security information for it"),
                        Target->MetaKey.c_str(), TransactionManager->Target.Description.c_str());
-              continue;
+                 continue;
+              }
+              // empty files are skipped as acquiring the very small compressed files is a waste of time
+              else if (hashes.FileSize() == 0)
+                 continue;
            }
         }
 

diff --git a/test/integration/framework b/test/integration/framework
index fc59c64503e650f5c4234c4ce02f4698c429c373..213169a98375e4373654aa31b282c0d6d4602231 100644 (file)
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -566,6 +566,11 @@ forcecompressor() {
        local CONFFILE="${TMPWORKINGDIRECTORY}/rootdir/etc/apt/apt.conf.d/00force-compressor"
        echo "Acquire::CompressionTypes::Order { \"${COMPRESS}\"; };
 Dir::Bin::uncompressed \"/does/not/exist\";" > "$CONFFILE"
+       for COMP in $(aptconfig dump 'APT::Compressor' --format '%f%n' | cut -d':' -f 5 | uniq); do
+               if [ -z "$COMP" -o "$COMP" = '.' -o "$COMP" = "$COMPRESSOR" ]; then continue; fi
+               echo "Dir::Bin::${COMP} \"/does/not/exist\";" >> "$CONFFILE"
+               echo "APT::Compressor::${COMP}::Name \"${COMP}-disabled\";" >> "$CONFFILE"
+       done
 }
 
 setupsimplenativepackage() {
@@ -1843,6 +1848,7 @@ listcurrentlistsdirectory() {
        } | sort
 }
 forallsupportedcompressors() {
+       rm -f "${TMPWORKINGDIRECTORY}/rootdir/etc/apt/apt.conf.d/00force-compressor"
        for COMP in $(aptconfig dump 'APT::Compressor' --format '%f%n' | cut -d':' -f 5 | uniq); do
                if [ -z "$COMP" -o "$COMP" = '.' ]; then continue; fi
                "$@" "$COMP"

diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files
index 522b3adcb3bbc61c6317f8f97cd002bd3382253c..afeeadd4f7ee9d712ef233abdd75077972795889 100755 (executable)
--- a/test/integration/test-apt-acquire-additional-files
+++ b/test/integration/test-apt-acquire-additional-files
@@ -15,6 +15,7 @@ configcompression '.' 'gz'
 LOWCOSTEXT='lz4'
 
 buildsimplenativepackage 'foo' 'amd64' '1' 'unstable'
+buildsimplenativepackage 'bar' 'all' '1' 'unstable'
 
 setupaptarchive --no-update
 changetowebserver
@@ -110,7 +111,7 @@ testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_mai
 testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT" find rootdir/var/lib/apt/lists -name '*Contents-all*'
 testequal "$(readfile Contents-amd64.$LOWCOSTEXT Contents-all.$LOWCOSTEXT)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
 testequal "$(apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT)" apthelper cat-file 'aptarchive/dists/unstable/main/Contents-amd64.gz'
-testempty apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT
+testequal "$(apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT)" apthelper cat-file 'aptarchive/dists/unstable/main/Contents-all.gz'
 
 rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT
 rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT

diff --git a/test/integration/test-apt-get-build-dep-file b/test/integration/test-apt-get-build-dep-file
index 2cd4f57ec32d3d22a32ec00c7bf2429f7ed052dc..233aed93460e9335d7963750b317a0dbb3a403ff 100755 (executable)
--- a/test/integration/test-apt-get-build-dep-file
+++ b/test/integration/test-apt-get-build-dep-file
@@ -10,6 +10,7 @@ configarchitecture 'i386'
 insertpackage 'stable' 'debhelper' 'i386' '7'
 insertpackage 'stable' 'build-essential' 'i386' '1'
 insertpackage 'stable' 'build-depends' 'i386' '1'
+insertsource 'stable' 'unrelated' 'any' '1'
 insertinstalledpackage 'build-conflict' 'i386' '1'
 
 setupaptarchive

diff --git a/test/integration/test-apt-update-empty-files b/test/integration/test-apt-update-empty-files
index e4f6d62bd98bc5da85b71f09c2a4664504be9f1e..42329bb4bbe5a188d87047130c0200000c6e5685 100755 (executable)
--- a/test/integration/test-apt-update-empty-files
+++ b/test/integration/test-apt-update-empty-files
@@ -17,9 +17,13 @@ msgmsg 'Test with file'
 rm -rf rootdir/var/lib/apt/lists
 testsuccess apt update -o Debug::pkgAcquire::Worker=1
 cp rootdir/tmp/testsuccess.output apt.output
-testsuccess grep '%0aAlt-Filename:%20/.*/Sources%0a' apt.output
+testfailure grep '%0a\(Alt\)\?Filename:%20/.*/Sources\(\.gz\)\?%0a' apt.output
+testempty find rootdir/var/lib/apt/lists -name '*_Sources'
 
 msgmsg 'Test with http'
 changetowebserver
 rm -rf rootdir/var/lib/apt/lists
 testsuccess apt update -o Debug::pkgAcquire::Worker=1
+cp rootdir/tmp/testsuccess.output apt.output
+testfailure grep 'http:600.*Sources' apt.output
+testempty find rootdir/var/lib/apt/lists -name '*_Sources'

diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files
index 8d06f09c01d86c7cd2eccd17c1108bebf2f5dfaa..d71bd2ce5a875840973fb4d6d9e98662c14ebad0 100755 (executable)
--- a/test/integration/test-bug-595691-empty-and-broken-archive-files
+++ b/test/integration/test-bug-595691-empty-and-broken-archive-files
@@ -10,35 +10,28 @@ buildaptarchive
 touch aptarchive/Packages
 setupflataptarchive
 
-testaptgetupdate() {
+failureupdate() {
        rm -rf rootdir/var/lib/apt
-       aptget update >testaptgetupdate.diff 2>&1 || true
-       sed -i -e '/Ign /,+1d' -e '/Release/ d' -e 's#\[[0-9]* [kMGTPY]*B\]#\[\]#' testaptgetupdate.diff
-       GIVEN="$1"
-       shift
-       msgtest "Test for correctness of" "apt-get update with $*"
-       if [ -z "$GIVEN" ]; then
-               echo -n "" | checkdiff - testaptgetupdate.diff && msgpass || msgfail
-       else
-               echo "$GIVEN" | checkdiff - testaptgetupdate.diff && msgpass || msgfail
-       fi
-       rm testaptgetupdate.diff
+       testfailure aptget update --allow-insecure-repositories
+       testsuccess grep "^E: Failed to fetch store:${1}  Empty files can't be valid archives$" rootdir/tmp/testfailure.output
+}
+successupdate() {
+       rm -rf rootdir/var/lib/apt
+       testwarning aptget update --allow-insecure-repositories
 }
 
 createemptyarchive() {
        find aptarchive/ \( -name "Packages*" -o -name "en*" \) -type f -delete
        touch aptarchive/Packages
        echo -n "" | $COMPRESSOR_CMD > aptarchive/${1}.$COMPRESS
-       generatereleasefiles
-       signreleasefiles
+       find aptarchive -name '*Release*' -delete
        rm -f aptarchive/Packages
 }
 
 createemptyfile() {
        find aptarchive/ \( -name "Packages*" -o -name "en*" \) -type f -delete
        touch aptarchive/Packages aptarchive/${1}.$COMPRESS
-       generatereleasefiles
-       signreleasefiles
+       find aptarchive -name '*Release*' -delete
        rm -f aptarchive/Packages
 }
 
@@ -46,34 +39,21 @@ testoverfile() {
        local APTARCHIVE="$(readlink -f ./aptarchive)"
        forcecompressor "$1"
 
+       msgmsg 'archive over file' "Packages.$COMPRESS"
        createemptyarchive 'Packages'
-       testaptgetupdate "Get:2 file:$APTARCHIVE  Packages []
-Reading package lists..." "empty archive Packages.$COMPRESS over file"
-
+       successupdate
        createemptyfile 'Packages'
-       testaptgetupdate "Get:2 file:$APTARCHIVE  Packages
-Err:2 file:$APTARCHIVE  Packages
-  Empty files can't be valid archives
-Reading package lists...
-E: Failed to fetch store:$(readlink -f rootdir/var/lib/apt/lists/partial/$(echo "$APTARCHIVE" | sed -e 's#/#_#g')_Packages.${COMPRESS})  Empty files can't be valid archives
-E: Some index files failed to download. They have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over file"
+       failureupdate "$(readlink -f rootdir/var/lib/apt/lists/partial/$(echo "$APTARCHIVE" | sed -e 's#/#_#g')_Packages.${COMPRESS})"
 }
 
 testoverhttp() {
        forcecompressor "$1"
 
+       msgmsg 'archive over http' "Packages.$COMPRESS"
        createemptyarchive 'Packages'
-       testaptgetupdate "Get:2 http://localhost:${APTHTTPPORT}  Packages []
-Reading package lists..." "empty archive Packages.$COMPRESS over http"
-
+       successupdate
        createemptyfile 'Packages'
-       #FIXME: we should response with a good error message instead
-       testaptgetupdate "Get:2 http://localhost:${APTHTTPPORT}  Packages
-Err:2 http://localhost:${APTHTTPPORT}  Packages
-  Empty files can't be valid archives
-Reading package lists...
-E: Failed to fetch store:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:${APTHTTPPORT}_Packages.${COMPRESS})  Empty files can't be valid archives
-E: Some index files failed to download. They have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over http"
+       failureupdate "$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:${APTHTTPPORT}_Packages.${COMPRESS})"
 }
 
 forallsupportedcompressors testoverfile

diff --git a/test/integration/test-compressed-indexes b/test/integration/test-compressed-indexes
index 573a86511486340426eaa5f0e45a48c4e3056d43..7ddf2e2e2904972183befaa9c369f13df615695a 100755 (executable)
--- a/test/integration/test-compressed-indexes
+++ b/test/integration/test-compressed-indexes
@@ -33,11 +33,9 @@ testrun() {
        msgtest 'Check if all index files are' "${1:-uncompressed}"
        if [ "$1" = 'compressed' ]; then
                ! test -e rootdir/var/lib/apt/lists/*i386_Packages || F=1
-               ! test -e rootdir/var/lib/apt/lists/*all_Packages || F=1
                ! test -e rootdir/var/lib/apt/lists/*_Sources || F=1
                ! test -e rootdir/var/lib/apt/lists/*_Translation-en || F=1
                test -e rootdir/var/lib/apt/lists/*i386_Packages.$LOWCOSTEXT || F=1
-               test -e rootdir/var/lib/apt/lists/*all_Packages.$LOWCOSTEXT || F=1
                test -e rootdir/var/lib/apt/lists/*_Sources.$LOWCOSTEXT || F=1
                test -e rootdir/var/lib/apt/lists/*_Translation-en.$LOWCOSTEXT || F=1
                # there is no point in trying pdiff if we have compressed indexes
@@ -47,11 +45,9 @@ testrun() {
                # clear the faked pdiff indexes so the glob below works
                rm -f rootdir/var/lib/apt/lists/*diff_Index
                test -e rootdir/var/lib/apt/lists/*i386_Packages || F=1
-               test -e rootdir/var/lib/apt/lists/*all_Packages || F=1
                test -e rootdir/var/lib/apt/lists/*_Sources || F=1
                test -e rootdir/var/lib/apt/lists/*_Translation-en || F=1
                ! test -e rootdir/var/lib/apt/lists/*i386_Packages.* || F=1
-               ! test -e rootdir/var/lib/apt/lists/*all_Packages.* || F=1
                ! test -e rootdir/var/lib/apt/lists/*_Sources.* || F=1
                ! test -e rootdir/var/lib/apt/lists/*_Translation-en.* || F=1
        fi
@@ -112,6 +108,7 @@ testovermethod() {
                        INDCOMP='uncompressed'
                else
                        INDCOMP='compressed'
+                       echo 'APT::Compressor::lz4::Name "lz4";' >> rootdir/etc/apt/apt.conf.d/02compressindex
                fi
 
                msgmsg "${1}: ${COMPRESSOR}: Test with $INDCOMP indexes gzip=$INDEX"

diff --git a/test/integration/test-handle-redirect-as-used-mirror-change b/test/integration/test-handle-redirect-as-used-mirror-change
index 41dd0772e68935323f8ddf832a10b97ac5bc689d..3a8b1fecd905da1ce9de9616a29ffe3d9514d052 100755 (executable)
--- a/test/integration/test-handle-redirect-as-used-mirror-change
+++ b/test/integration/test-handle-redirect-as-used-mirror-change
@@ -16,9 +16,8 @@ rewritesourceslist "http://localhost:${APTHTTPPORT}/redirectme"
 
 testsuccessequal "Get:1 http://0.0.0.0:${APTHTTPPORT} unstable InRelease [$(stat -c %s aptarchive/dists/unstable/InRelease) B]
 Get:2 http://0.0.0.0:${APTHTTPPORT} unstable/main Sources [$(stat -c %s aptarchive/dists/unstable/main/source/Sources.gz) B]
-Get:3 http://0.0.0.0:${APTHTTPPORT} unstable/main amd64 Packages [$(stat -c %s aptarchive/dists/unstable/main/binary-amd64/Packages.gz) B]
-Get:4 http://0.0.0.0:${APTHTTPPORT} unstable/main all Packages [$(stat -c %s aptarchive/dists/unstable/main/binary-all/Packages.gz) B]
-Get:5 http://0.0.0.0:${APTHTTPPORT} unstable/main Translation-en [$(stat -c %s aptarchive/dists/unstable/main/i18n/Translation-en.gz) B]
+Get:3 http://0.0.0.0:${APTHTTPPORT} unstable/main all Packages [$(stat -c %s aptarchive/dists/unstable/main/binary-all/Packages.gz) B]
+Get:4 http://0.0.0.0:${APTHTTPPORT} unstable/main Translation-en [$(stat -c %s aptarchive/dists/unstable/main/i18n/Translation-en.gz) B]
 Reading package lists..." aptget update
 
 testsuccessequal "Hit:1 http://0.0.0.0:${APTHTTPPORT} unstable InRelease