keep compressed indexes in a low-cost format

[apt.git] / test / integration / test-apt-acquire-additional-files

diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files
index ee7908a2d972102fd66c20e2dfebd565c1b60c88..bf7850a7ad25815aa61d2bdf7c0bfc85d6ec7dcb 100755 (executable)
--- a/test/integration/test-apt-acquire-additional-files
+++ b/test/integration/test-apt-acquire-additional-files
@@ -1,34 +1,38 @@
 #!/bin/sh
 set -e
 
 #!/bin/sh
 set -e
 

-TESTDIR=$(readlink -f $(dirname $0))
-. $TESTDIR/framework
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"

 
 setupenvironment
 configarchitecture 'amd64'
 
 
 setupenvironment
 configarchitecture 'amd64'
 

-# note that in --print-uri we talk about .bz2 because that is the default.
+# note that in --print-uri we talk about .xz because that is the default.

 # This doesn't mean it is actually attempt to download it.
 configcompression '.' 'gz'
 # This doesn't mean it is actually attempt to download it.
 configcompression '.' 'gz'

+LOWCOSTEXT='lz4'

 
 buildsimplenativepackage 'foo' 'amd64' '1' 'unstable'
 
 setupaptarchive --no-update
 changetowebserver
 
 
 buildsimplenativepackage 'foo' 'amd64' '1' 'unstable'
 
 setupaptarchive --no-update
 changetowebserver
 

-testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 
-'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 
-'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 
-'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 " aptget update --print-uris
-
-testsuccessequal "Get:1 http://localhost:8080 unstable InRelease [$(stat -c%s aptarchive/dists/unstable/InRelease) B]
-Get:2 http://localhost:8080 unstable/main Sources [$(stat -c%s aptarchive/dists/unstable/main/source/Sources.gz) B]
-Get:3 http://localhost:8080 unstable/main amd64 Packages [$(stat -c%s aptarchive/dists/unstable/main/binary-amd64/Packages.gz) B]
-Get:4 http://localhost:8080 unstable/main Translation-en [$(stat -c%s aptarchive/dists/unstable/main/i18n/Translation-en.gz) B]
+testequal "'http://localhost:${APTHTTPPORT}/dists/unstable/InRelease' localhost:${APTHTTPPORT}_dists_unstable_InRelease 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/source/Sources.xz' localhost:${APTHTTPPORT}_dists_unstable_main_source_Sources 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-amd64/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-amd64_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-all/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-all_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/i18n/Translation-en.xz' localhost:${APTHTTPPORT}_dists_unstable_main_i18n_Translation-en 0 " aptget update --print-uris
+
+testsuccessequal "Get:1 http://localhost:${APTHTTPPORT} unstable InRelease [$(stat -c%s aptarchive/dists/unstable/InRelease) B]
+Get:2 http://localhost:${APTHTTPPORT} unstable/main Sources [$(stat -c%s aptarchive/dists/unstable/main/source/Sources.gz) B]
+Get:3 http://localhost:${APTHTTPPORT} unstable/main amd64 Packages [$(stat -c%s aptarchive/dists/unstable/main/binary-amd64/Packages.gz) B]
+Get:4 http://localhost:${APTHTTPPORT} unstable/main all Packages [$(stat -c%s aptarchive/dists/unstable/main/binary-all/Packages.gz) B]
+Get:5 http://localhost:${APTHTTPPORT} unstable/main Translation-en [$(stat -c%s aptarchive/dists/unstable/main/i18n/Translation-en.gz) B]

 Reading package lists..." aptget update
 
 testempty find rootdir/var/lib/apt/lists -name '*Contents*'
 
 Reading package lists..." aptget update
 
 testempty find rootdir/var/lib/apt/lists -name '*Contents*'
 

+msgmsg "Normal Contents file"

 cat > rootdir/etc/apt/apt.conf.d/content-target.conf <<EOF
 Acquire::IndexTargets::deb::Contents {
        MetaKey "\$(COMPONENT)/Contents-\$(ARCHITECTURE)";
 cat > rootdir/etc/apt/apt.conf.d/content-target.conf <<EOF
 Acquire::IndexTargets::deb::Contents {
        MetaKey "\$(COMPONENT)/Contents-\$(ARCHITECTURE)";


@@ -37,42 +41,78 @@ Acquire::IndexTargets::deb::Contents {
 };
 EOF
 
 };
 EOF
 

-testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64)" aptget indextargets --no-release-info --format '$(FILENAME)' 'Created-By: Contents'
+readfile() {
+       while [ -n "$1" ]; do
+               readlink -f "./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_${1}"
+               shift
+       done
+}
+
+testequal "$(readfile Contents-amd64 Contents-all)" aptget indextargets --no-release-info --format '$(FILENAME)' 'Created-By: Contents'

 testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
 # lets fake the existence of a compressed Contents file
 testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
 # lets fake the existence of a compressed Contents file

-touch ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz
-testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
+touch "./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz"
+testequal "$(readfile Contents-amd64.gz)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
+touch "./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz"
+testequal "$(readfile Contents-amd64.gz Contents-all.gz)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
+
+testequal "'http://localhost:${APTHTTPPORT}/dists/unstable/InRelease' localhost:${APTHTTPPORT}_dists_unstable_InRelease 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/source/Sources.xz' localhost:${APTHTTPPORT}_dists_unstable_main_source_Sources 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-amd64/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-amd64_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-all/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-all_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/i18n/Translation-en.xz' localhost:${APTHTTPPORT}_dists_unstable_main_i18n_Translation-en 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/Contents-amd64.xz' localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/Contents-all.xz' localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all 0 " aptget update --print-uris
+
+# apt believes the Contents files we faked are good
+testsuccessequal "Hit:1 http://localhost:${APTHTTPPORT} unstable InRelease
+Reading package lists..." aptget update
+testequal "$(readfile Contents-amd64.gz Contents-all.gz)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
+
+find rootdir/var/lib/apt/lists -name '*Contents*' -delete

 
 

-testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 
-'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 
-'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 
-'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 
-'http://localhost:8080/dists/unstable/main/Contents-amd64.bz2' localhost:8080_dists_unstable_main_Contents-amd64 0 " aptget update --print-uris
+testequal "'http://localhost:${APTHTTPPORT}/dists/unstable/InRelease' localhost:${APTHTTPPORT}_dists_unstable_InRelease 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/source/Sources.xz' localhost:${APTHTTPPORT}_dists_unstable_main_source_Sources 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-amd64/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-amd64_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-all/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-all_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/i18n/Translation-en.xz' localhost:${APTHTTPPORT}_dists_unstable_main_i18n_Translation-en 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/Contents-amd64.xz' localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/Contents-all.xz' localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all 0 " aptget update --print-uris

 
 

-testsuccessequal "Hit:1 http://localhost:8080 unstable InRelease
-Get:2 http://localhost:8080 unstable/main amd64 Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
+testsuccessequal "Hit:1 http://localhost:${APTHTTPPORT} unstable InRelease
+Get:2 http://localhost:${APTHTTPPORT} unstable/main amd64 Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
+Get:3 http://localhost:${APTHTTPPORT} unstable/main all Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-all.gz) B]

 Reading package lists..." aptget update
 
 Reading package lists..." aptget update
 

-testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64' find rootdir/var/lib/apt/lists -name '*Contents*'
-testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
-testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64' 'aptarchive/dists/unstable/main/Contents-amd64'
+testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64" find rootdir/var/lib/apt/lists -name '*Contents-amd64*'
+testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all" find rootdir/var/lib/apt/lists -name '*Contents-all*'
+testequal "$(readfile Contents-amd64 Contents-all)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
+testsuccess cmp "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64" 'aptarchive/dists/unstable/main/Contents-amd64'
+testsuccess cmp "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all" 'aptarchive/dists/unstable/main/Contents-all'

 
 

-rm ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64
+rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64 \
+       ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all

 testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
 
 # if we asked for keeping it compressed, keep it
 testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
 
 # if we asked for keeping it compressed, keep it

+msgmsg "Normal Contents file with KeepCompressed"

 echo 'Acquire::IndexTargets::deb::Contents::KeepCompressed "true";' >> rootdir/etc/apt/apt.conf.d/content-target.conf
 echo 'Acquire::IndexTargets::deb::Contents::KeepCompressed "true";' >> rootdir/etc/apt/apt.conf.d/content-target.conf

-testsuccessequal "Hit:1 http://localhost:8080 unstable InRelease
-Get:2 http://localhost:8080 unstable/main amd64 Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
+testsuccessequal "Hit:1 http://localhost:${APTHTTPPORT} unstable InRelease
+Get:2 http://localhost:${APTHTTPPORT} unstable/main amd64 Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
+Get:3 http://localhost:${APTHTTPPORT} unstable/main all Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-all.gz) B]

 Reading package lists..." aptget update
 
 Reading package lists..." aptget update
 

-testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' find rootdir/var/lib/apt/lists -name '*Contents*'
-testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
-testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' 'aptarchive/dists/unstable/main/Contents-amd64.gz'
+testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT" find rootdir/var/lib/apt/lists -name '*Contents-amd64*'
+testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT" find rootdir/var/lib/apt/lists -name '*Contents-all*'
+testequal "$(readfile Contents-amd64.$LOWCOSTEXT Contents-all.$LOWCOSTEXT)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
+testequal "$(apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT)" apthelper cat-file 'aptarchive/dists/unstable/main/Contents-amd64.gz'
+testempty apthelper cat-file rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT

 
 

-rm ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz
+rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.$LOWCOSTEXT
+rm ./rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.$LOWCOSTEXT

 testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
 
 testempty aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
 

+msgmsg "Compressed Contents file"

 # and no automatic uncompress based on the name please,
 # only if we downloaded a compressed file, but target was uncompressed
 cat > rootdir/etc/apt/apt.conf.d/content-target.conf <<EOF
 # and no automatic uncompress based on the name please,
 # only if we downloaded a compressed file, but target was uncompressed
 cat > rootdir/etc/apt/apt.conf.d/content-target.conf <<EOF


@@ -84,28 +124,35 @@ Acquire::IndexTargets::deb::Contents {
 EOF
 
 # the last line is utter bogus of course, but how should apt know…
 EOF
 
 # the last line is utter bogus of course, but how should apt know…

-testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 
-'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 
-'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 
-'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 
-'http://localhost:8080/dists/unstable/main/Contents-amd64.gz.bz2' localhost:8080_dists_unstable_main_Contents-amd64.gz 0 " aptget update --print-uris
-
-testsuccessequal "Hit:1 http://localhost:8080 unstable InRelease
-Get:2 http://localhost:8080 unstable/main amd64 Contents.gz [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
+testequal "'http://localhost:${APTHTTPPORT}/dists/unstable/InRelease' localhost:${APTHTTPPORT}_dists_unstable_InRelease 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/source/Sources.xz' localhost:${APTHTTPPORT}_dists_unstable_main_source_Sources 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-amd64/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-amd64_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-all/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-all_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/i18n/Translation-en.xz' localhost:${APTHTTPPORT}_dists_unstable_main_i18n_Translation-en 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/Contents-amd64.gz.xz' localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/Contents-all.gz.xz' localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz 0 " aptget update --print-uris
+
+testsuccessequal "Hit:1 http://localhost:${APTHTTPPORT} unstable InRelease
+Get:2 http://localhost:${APTHTTPPORT} unstable/main amd64 Contents.gz [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
+Get:3 http://localhost:${APTHTTPPORT} unstable/main all Contents.gz [$(stat -c%s aptarchive/dists/unstable/main/Contents-all.gz) B]

 Reading package lists..." aptget update
 
 Reading package lists..." aptget update
 

-testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' find rootdir/var/lib/apt/lists -name '*Contents*'
-testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
-testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' 'aptarchive/dists/unstable/main/Contents-amd64.gz'
+testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz" find rootdir/var/lib/apt/lists -name '*Contents-amd64*'
+testequal "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz" find rootdir/var/lib/apt/lists -name '*Contents-all*'
+testequal "$(readfile Contents-amd64.gz Contents-all.gz)" aptget indextargets --format '$(FILENAME)' 'Created-By: Contents'
+testsuccess cmp "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-amd64.gz" 'aptarchive/dists/unstable/main/Contents-amd64.gz'
+testsuccess cmp "rootdir/var/lib/apt/lists/localhost:${APTHTTPPORT}_dists_unstable_main_Contents-all.gz" 'aptarchive/dists/unstable/main/Contents-all.gz'

 
 rm -f rootdir/etc/apt/apt.conf.d/content-target.conf
 
 rm -f rootdir/etc/apt/apt.conf.d/content-target.conf

+msgmsg "No Contents file"

 
 

-testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 
-'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 
-'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 
-'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 " aptget update --print-uris
+testequal "'http://localhost:${APTHTTPPORT}/dists/unstable/InRelease' localhost:${APTHTTPPORT}_dists_unstable_InRelease 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/source/Sources.xz' localhost:${APTHTTPPORT}_dists_unstable_main_source_Sources 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-amd64/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-amd64_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/binary-all/Packages.xz' localhost:${APTHTTPPORT}_dists_unstable_main_binary-all_Packages 0 
+'http://localhost:${APTHTTPPORT}/dists/unstable/main/i18n/Translation-en.xz' localhost:${APTHTTPPORT}_dists_unstable_main_i18n_Translation-en 0 " aptget update --print-uris

 
 

-testsuccessequal "Hit:1 http://localhost:8080 unstable InRelease
+testsuccessequal "Hit:1 http://localhost:${APTHTTPPORT} unstable InRelease

 Reading package lists..." aptget update
 
 testempty find rootdir/var/lib/apt/lists -name '*Contents*'
 Reading package lists..." aptget update
 
 testempty find rootdir/var/lib/apt/lists -name '*Contents*'