condense parallel requests with the same hashes to one

It shouldn't be too common, but sometimes people have multiple mirrors
in the sources or otherwise repositories with the same content. Now that
we gracefully can handle multiple requests to the same URI, we can also
fold multiple requests with the same expected hashes into one. Note that
this isn't trying to find oppertunities for merging, but just merges if
it happens to encounter the oppertunity for it.

This is most obvious in the new testcase actually as it needs to delay
the action to give the acquire system enough time to figure out that
they can be merged.

diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index 14c8863dcdf7fcf28f8661a34bb6c379ebfb64c8..34afab181c906ebf393b030afa567c08097ab1f9 100644 (file)
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -680,9 +680,12 @@ bool pkgAcquire::Queue::Enqueue(ItemDesc &Item)
 {
    QItem **I = &Items;
    // move to the end of the queue and check for duplicates here
 {
    QItem **I = &Items;
    // move to the end of the queue and check for duplicates here

+   HashStringList const hsl = Item.Owner->GetExpectedHashes();

    for (; *I != 0; I = &(*I)->Next)
    for (; *I != 0; I = &(*I)->Next)

-      if (Item.URI == (*I)->URI) 
+      if (Item.URI == (*I)->URI || hsl == (*I)->Owner->GetExpectedHashes())

       {
       {

+        if (_config->FindB("Debug::pkgAcquire::Worker",false) == true)
+           std::cerr << " @ Queue: Action combined for " << Item.URI << " and " << (*I)->URI << std::endl;

         (*I)->Owners.push_back(Item.Owner);
         Item.Owner->Status = (*I)->Owner->Status;
         return false;
         (*I)->Owners.push_back(Item.Owner);
         Item.Owner->Status = (*I)->Owner->Status;
         return false;

diff --git a/apt-private/acqprogress.cc b/apt-private/acqprogress.cc
index dc92e3b2a139985bd1e2932e33973a1b2de26f5c..f6c3d1204ac8feab6441b0634ab0e7bbdbc084ff 100644 (file)
--- a/apt-private/acqprogress.cc
+++ b/apt-private/acqprogress.cc
@@ -116,14 +116,10 @@ void AcqTextStatus::Fail(pkgAcquire::ItemDesc &Itm)
    if (Quiet > 1)
       return;
 
    if (Quiet > 1)
       return;
 

-   // Ignore certain kinds of transient failures (bad code)
-   if (Itm.Owner->Status == pkgAcquire::Item::StatIdle)
-      return;
-

    AssignItemID(Itm);
    clearLastLine();
 
    AssignItemID(Itm);
    clearLastLine();
 

-   if (Itm.Owner->Status == pkgAcquire::Item::StatDone)
+   if (Itm.Owner->Status == pkgAcquire::Item::StatDone || Itm.Owner->Status == pkgAcquire::Item::StatIdle)

    {
       // TRANSLATOR: Very short word to be displayed for files in 'apt-get update'
       // which failed to download, but the error is ignored (compare "Err:")
    {
       // TRANSLATOR: Very short word to be displayed for files in 'apt-get update'
       // which failed to download, but the error is ignored (compare "Err:")

diff --git a/test/integration/framework b/test/integration/framework
index 1b99929e2f19d78b5cd95843b6272d0e5191acd6..d8f7567d9680bd619397a5fc88c5c47d97cd053f 100644 (file)
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -245,7 +245,7 @@ setupenvironment() {
        echo "Dir \"${TMPWORKINGDIRECTORY}/rootdir\";" > aptconfig.conf
        echo "Dir::state::status \"${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg/status\";" >> aptconfig.conf
        echo "APT::Get::Show-User-Simulation-Note \"false\";" >> aptconfig.conf
        echo "Dir \"${TMPWORKINGDIRECTORY}/rootdir\";" > aptconfig.conf
        echo "Dir::state::status \"${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg/status\";" >> aptconfig.conf
        echo "APT::Get::Show-User-Simulation-Note \"false\";" >> aptconfig.conf

-       echo "Dir::Bin::Methods \"${METHODSDIR}\";" >> aptconfig.conf
+       echo "Dir::Bin::Methods \"${TMPWORKINGDIRECTORY}/rootdir/usr/lib/apt/methods\";" >> aptconfig.conf

        # store apt-key were we can access it, even if we run it as a different user
        # destroys coverage reporting though, so just do it for root for now
        if [ "$(id -u)" = '0' ]; then
        # store apt-key were we can access it, even if we run it as a different user
        # destroys coverage reporting though, so just do it for root for now
        if [ "$(id -u)" = '0' ]; then


@@ -786,6 +786,8 @@ insertsource() {
        local SPATH="aptarchive/dists/${RELEASE}/main/source"
        mkdir -p $SPATH
        local FILE="${SPATH}/Sources"
        local SPATH="aptarchive/dists/${RELEASE}/main/source"
        mkdir -p $SPATH
        local FILE="${SPATH}/Sources"

+       local DSCFILE="${NAME}_${VERSION}.dsc"
+       local TARFILE="${NAME}_${VERSION}.tar.gz"

        echo "Package: $NAME
 Binary: $NAME
 Version: $VERSION
        echo "Package: $NAME
 Binary: $NAME
 Version: $VERSION


@@ -793,8 +795,8 @@ Maintainer: Joe Sixpack <joe@example.org>
 Architecture: $ARCH" >> $FILE
        test -z "$DEPENDENCIES" || echo "$DEPENDENCIES" >> $FILE
        echo "Files:
 Architecture: $ARCH" >> $FILE
        test -z "$DEPENDENCIES" || echo "$DEPENDENCIES" >> $FILE
        echo "Files:

- d41d8cd98f00b204e9800998ecf8427e 0 ${NAME}_${VERSION}.dsc
- d41d8cd98f00b204e9800998ecf8427e 0 ${NAME}_${VERSION}.tar.gz
+ $(echo -n "$DSCFILE" | md5sum | cut -d' ' -f 1) $(echo -n "$DSCFILE" | wc -c) $DSCFILE
+ $(echo -n "$TARFILE" | md5sum | cut -d' ' -f 1) $(echo -n "$TARFILE" | wc -c) $TARFILE

 " >> $FILE
 }
 
 " >> $FILE
 }
 

diff --git a/test/integration/skip-aptwebserver b/test/integration/skip-aptwebserver
deleted file mode 100755 (executable)
index 0622941..0000000
--- a/test/integration/skip-aptwebserver
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/sh
-set -e
-
-TESTDIR=$(readlink -f $(dirname $0))
-. $TESTDIR/framework
-
-setupenvironment
-configarchitecture 'amd64'
-
-buildsimplenativepackage 'apt' 'all' '1.0' 'stable'
-
-setupaptarchive
-changetowebserver
-
-rm -rf rootdir/var/lib/apt/lists
-aptget update -qq
-testequal 'Hit http://localhost stable InRelease
-Hit http://localhost stable/main Sources
-Hit http://localhost stable/main amd64 Packages
-Hit http://localhost stable/main Translation-en
-Reading package lists...' aptget update
-
-mv rootdir/var/lib/apt/lists/localhost* rootdir/var/lib/apt/lists/partial
-aptget update
-

diff --git a/test/integration/test-acquire-same-file-multiple-times b/test/integration/test-acquire-same-file-multiple-times
index d329a39cb343f0339c874c1fead736a04f2326b0..5267655215c92dd7f0b3f77976a4b2bed3296a12 100755 (executable)
--- a/test/integration/test-acquire-same-file-multiple-times
+++ b/test/integration/test-acquire-same-file-multiple-times
@@ -45,7 +45,7 @@ changetowebserver -o aptwebserver::redirect::replace::/foo2=/foo
 
 httpdown() {
        msgtest 'Downloading the same URI to different files' 'twice over http'
 
 httpdown() {
        msgtest 'Downloading the same URI to different files' 'twice over http'

-       testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1
+       testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1

        cp rootdir/tmp/testsuccess.output download.log
        testsuccess cmp $TESTDIR/framework ./downloaded/foo1
        testsuccess cmp ./downloaded/foo1 ./downloaded/foo2
        cp rootdir/tmp/testsuccess.output download.log
        testsuccess cmp $TESTDIR/framework ./downloaded/foo1
        testsuccess cmp ./downloaded/foo1 ./downloaded/foo2


@@ -57,7 +57,7 @@ testrun 'httpdown'
 
 httpredirectdown() {
        msgtest 'Redirect leads' 'first URI to the second URI'
 
 httpredirectdown() {
        msgtest 'Redirect leads' 'first URI to the second URI'

-       testsuccess --nomsg apthelper download-file http://localhost:8080/foo2 ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1
+       testsuccess --nomsg apthelper download-file http://localhost:8080/foo2 ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1

        cp rootdir/tmp/testsuccess.output download.log
        testsuccess cmp $TESTDIR/framework ./downloaded/foo1
        testsuccess cmp ./downloaded/foo1 ./downloaded/foo2
        cp rootdir/tmp/testsuccess.output download.log
        testsuccess cmp $TESTDIR/framework ./downloaded/foo1
        testsuccess cmp ./downloaded/foo1 ./downloaded/foo2

diff --git a/test/integration/test-acquire-same-repository-multiple-times b/test/integration/test-acquire-same-repository-multiple-times
new file mode 100755 (executable)
index 0000000..bfeaf88
--- /dev/null
+++ b/test/integration/test-acquire-same-repository-multiple-times
@@ -0,0 +1,81 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+configarchitecture 'amd64'
+
+TESTFILE="$TESTDIR/framework"
+cp $TESTFILE aptarchive/foo
+APTARCHIVE="$(readlink -f ./aptarchive)"
+
+getcodenamefromsuite() { echo "jessie"; }
+buildsimplenativepackage 'foo' 'all' '1.0' 'stable'
+setupaptarchive --no-update
+ln -s "${APTARCHIVE}/dists/stable" "${APTARCHIVE}/dists/jessie"
+for FILE in rootdir/etc/apt/sources.list.d/*-stable-* ; do
+       sed 's#stable#jessie#g' $FILE > $(echo "$FILE" | sed 's#stable#jessie#g')
+done
+
+# install a slowed down file: otherwise its to fast to reproduce combining
+NEWMETHODS="$(readlink -f rootdir)/usr/lib/apt/methods"
+OLDMETHODS="$(readlink -f rootdir/usr/lib/apt/methods)"
+rm $NEWMETHODS
+mkdir $NEWMETHODS
+for METH in $(find $OLDMETHODS ! -type d); do
+       ln -s $OLDMETHODS/$(basename $METH) $NEWMETHODS
+done
+rm $NEWMETHODS/file
+cat >$NEWMETHODS/file <<EOF
+#!/bin/sh
+while read line; do
+       echo "\$line"
+       if [ -z "\$line" ]; then
+               sleep 0.2
+       fi
+done | $OLDMETHODS/file
+EOF
+chmod +x $NEWMETHODS/file
+
+tworepos() {
+       msgtest "Downloading the same repository twice over $1" "$3"
+       testsuccess --nomsg aptget update -o Debug::pkgAcquire::Worker=1
+       cp rootdir/tmp/testsuccess.output download.log
+       #cat download.log
+       aptget files --format '$(FILENAME)' --no-release-info | sort > file.lst
+       testequal "$(find $(readlink -f ./rootdir/var/lib/apt/lists) -name '*_dists_*' \( ! -name '*InRelease' \) -type f | sort)" cat file.lst
+       testsuccess aptcache policy
+       testequal "foo:
+  Installed: (none)
+  Candidate: 1.0
+  Version table:
+     1.0 0
+        500 $1:$2 jessie/main amd64 Packages
+        500 $1:$2 stable/main amd64 Packages" aptcache policy foo
+       testfailure aptcache show foo/unstable
+       testsuccess aptcache show foo/stable
+       testsuccess aptcache show foo/jessie
+}
+
+tworepos 'file' "$APTARCHIVE" 'no partial'
+testequal '12' grep -c '200%20URI%20Start' ./download.log
+testequal '12' grep -c '201%20URI%20Done' ./download.log
+testequal '6' grep -c '^ @ Queue: Action combined' ./download.log
+tworepos 'file' "$APTARCHIVE" 'hit'
+testequal '6' grep -c '200%20URI%20Start' ./download.log
+testequal '6' grep -c '201%20URI%20Done' ./download.log
+testequal '0' grep -c '^ @ Queue: Action combined' ./download.log
+rm -rf rootdir/var/lib/apt/lists
+
+changetowebserver
+
+tworepos 'http' '//localhost:8080' 'no partial'
+testequal '10' grep -c '200%20URI%20Start' ./download.log
+testequal '10' grep -c '201%20URI%20Done' ./download.log
+testequal '6' grep -c '^ @ Queue: Action combined' ./download.log
+tworepos 'http' '//localhost:8080' 'hit'
+testequal '2' grep -c '200%20URI%20Start' ./download.log
+testequal '4' grep -c '201%20URI%20Done' ./download.log
+testequal '0' grep -c '^ @ Queue: Action combined' ./download.log
+rm -rf rootdir/var/lib/apt/lists

diff --git a/test/integration/test-apt-get-source b/test/integration/test-apt-get-source
index 9db24370f6e35983b0e37dde0457593df05da1b6..22f01b99752c2871e4753504d97613e20a955277 100755 (executable)
--- a/test/integration/test-apt-get-source
+++ b/test/integration/test-apt-get-source
@@ -34,43 +34,40 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 # normal operation gets highest version number
 HEADER="Reading package lists...
 Building dependency tree..."
 # normal operation gets highest version number
 HEADER="Reading package lists...
 Building dependency tree..."

+DOWNLOAD1="Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 11 MD5Sum:b998e085e36cf162e6a33c2801318fef
+'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 14 MD5Sum:d46b9a02af8487cbeb49165540c88184"
+DOWNLOAD2="Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 11 MD5Sum:c0de572c6f8aa576c8ff78c81132ed55
+'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 14 MD5Sum:e10bb487c375b2b938d27bd31c2d1f5f"

 testsuccessequal "$HEADER
 testsuccessequal "$HEADER

-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo
+$DOWNLOAD2" aptget source -q --print-uris foo

 
 # select by release: suite
 testsuccessequal "$HEADER
 Selected version '1.0' (stable) for foo
 
 # select by release: suite
 testsuccessequal "$HEADER
 Selected version '1.0' (stable) for foo

-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/stable
+$DOWNLOAD1" aptget source -q --print-uris foo/stable

 testsuccessequal "$HEADER
 Selected version '2.0' (unstable) for foo
 testsuccessequal "$HEADER
 Selected version '2.0' (unstable) for foo

-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/unstable
+$DOWNLOAD2" aptget source -q --print-uris foo/unstable

 
 # select by release: codename
 testsuccessequal "$HEADER
 Selected version '2.0' (sid) for foo
 
 # select by release: codename
 testsuccessequal "$HEADER
 Selected version '2.0' (sid) for foo

-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/sid
+$DOWNLOAD2" aptget source -q --print-uris foo/sid

 
 # select by version
 testsuccessequal "$HEADER
 
 # select by version
 testsuccessequal "$HEADER

-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo=1.0
+$DOWNLOAD1" aptget source -q --print-uris foo=1.0

 
 # select by release with no binary package (Bug#731102) but ensure to get
 # highest version
 
 # select by release with no binary package (Bug#731102) but ensure to get
 # highest version

+DOWNLOAD01="Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 11 MD5Sum:0811a4d85238056c613ea897f49f01af
+'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 14 MD5Sum:fa1ecb7a1a53e8e6f6551ca7db888a61"

 testsuccessequal "$HEADER
 Selected version '0.1' (wheezy) for foo
 testsuccessequal "$HEADER
 Selected version '0.1' (wheezy) for foo

-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/wheezy
+$DOWNLOAD01" aptget source -q --print-uris foo/wheezy

 
 # unavailable one
 testfailureequal "$HEADER
 
 # unavailable one
 testfailureequal "$HEADER


@@ -78,11 +75,12 @@ E: Can not find version '9.9-not-there' of package 'foo'
 E: Unable to find a source package for foo" aptget source -q --print-uris foo=9.9-not-there
 
 # version and release
 E: Unable to find a source package for foo" aptget source -q --print-uris foo=9.9-not-there
 
 # version and release

+DOWNLOAD001="Need to get 0 B/29 B of source archives.
+'file://${APTARCHIVE}/foo_0.0.1.dsc' foo_0.0.1.dsc 13 MD5Sum:6c819ebf0a21b1a480e1dbf6b8edfebd
+'file://${APTARCHIVE}/foo_0.0.1.tar.gz' foo_0.0.1.tar.gz 16 MD5Sum:a3c7e1ac2159fc0faf522e110d6906fd"

 testsuccessequal "$HEADER
 testsuccessequal "$HEADER

-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_0.0.1.dsc' foo_0.0.1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_0.0.1.tar.gz' foo_0.0.1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris -t unstable foo=0.0.1
+$DOWNLOAD001" aptget source -q --print-uris -t unstable foo=0.0.1

 
 testsuccessequal "$HEADER
 
 testsuccessequal "$HEADER

-Need to get 0 B of source archives.
+Need to get 0 B/25 B of source archives.

 Fetch source foo" aptget source -q -s foo
 Fetch source foo" aptget source -q -s foo

diff --git a/test/integration/test-apt-get-source-arch b/test/integration/test-apt-get-source-arch
index c757982094676d348de6bc941e88b042e6d4b693..f54bb6012ef32736007513210a93336a15f3b7d2 100755 (executable)
--- a/test/integration/test-apt-get-source-arch
+++ b/test/integration/test-apt-get-source-arch
@@ -28,31 +28,30 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 
 HEADER="Reading package lists...
 Building dependency tree..."
 
 HEADER="Reading package lists...
 Building dependency tree..."

+DOWNLOAD10="Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 11 MD5Sum:b998e085e36cf162e6a33c2801318fef
+'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 14 MD5Sum:d46b9a02af8487cbeb49165540c88184"

 
 # pick :amd64
 testsuccessequal "$HEADER
 
 # pick :amd64
 testsuccessequal "$HEADER

-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:amd64
+$DOWNLOAD10" aptget source -q --print-uris foo:amd64

 
 # pick :i386
 testsuccessequal "$HEADER
 
 # pick :i386
 testsuccessequal "$HEADER

-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:i386
+Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 11 MD5Sum:c0de572c6f8aa576c8ff78c81132ed55
+'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 14 MD5Sum:e10bb487c375b2b938d27bd31c2d1f5f" aptget source -q --print-uris foo:i386

 
 # pick :i386 by release
 testsuccessequal "$HEADER
 Selected version '0.1' (oldstable) for foo
 
 # pick :i386 by release
 testsuccessequal "$HEADER
 Selected version '0.1' (oldstable) for foo

-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:i386/oldstable
+Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 11 MD5Sum:0811a4d85238056c613ea897f49f01af
+'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 14 MD5Sum:fa1ecb7a1a53e8e6f6551ca7db888a61" aptget source -q --print-uris foo:i386/oldstable

 
 # pick :i386 by version
 testsuccessequal "$HEADER
 
 # pick :i386 by version
 testsuccessequal "$HEADER

-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:i386=1.0
+$DOWNLOAD10" aptget source -q --print-uris foo:i386=1.0

 
 # error on unknown arch
 testfailureequal "$HEADER
 
 # error on unknown arch
 testfailureequal "$HEADER

diff --git a/test/integration/test-apt-get-source-multisources b/test/integration/test-apt-get-source-multisources
index 03d0400a0e3bd400fc8172f893e055023e2b86aa..887a3068507881de3f9bf284579f4100634645e0 100755 (executable)
--- a/test/integration/test-apt-get-source-multisources
+++ b/test/integration/test-apt-get-source-multisources
@@ -20,11 +20,11 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 HEADER="Reading package lists...
 Building dependency tree..."
 testsuccessequal "$HEADER
 HEADER="Reading package lists...
 Building dependency tree..."
 testsuccessequal "$HEADER

-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -qdy --print-uris --dsc-only adduser=3.113 python-fll=0.9.11
+Need to get 0 B/43 B of source archives.
+'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 22 MD5Sum:255405ab5af211238ef53b7a1dd8ca4b
+'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 21 MD5Sum:740a9dbf02a295932f15b1415d0dc0df" aptget source -qdy --print-uris --dsc-only adduser=3.113 python-fll=0.9.11

 
 testsuccessequal "$HEADER
 
 testsuccessequal "$HEADER

-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e"  aptget source -qdy --print-uris --dsc-only python-fll=0.9.11 adduser=3.113
+Need to get 0 B/43 B of source archives.
+'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 21 MD5Sum:740a9dbf02a295932f15b1415d0dc0df
+'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 22 MD5Sum:255405ab5af211238ef53b7a1dd8ca4b" aptget source -qdy --print-uris --dsc-only python-fll=0.9.11 adduser=3.113

diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning
index 5b81d56d29e34683a3c04557f21c46c0d79bdae6..fcabf244ab1a85a472438b34750fcc48d4bfac22 100755 (executable)
--- a/test/integration/test-apt-get-update-unauth-warning
+++ b/test/integration/test-apt-get-update-unauth-warning
@@ -20,8 +20,10 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 rm -f $APTARCHIVE/dists/unstable/*Release*
 
 # update without authenticated files leads to warning
 rm -f $APTARCHIVE/dists/unstable/*Release*
 
 # update without authenticated files leads to warning

-testfailureequal "Ign:1 file:$APTARCHIVE unstable InRelease
+testfailureequal "Get:1 file:$APTARCHIVE unstable InRelease
+Ign:1 file:$APTARCHIVE unstable InRelease

   File not found
   File not found

+Get:2 file:$APTARCHIVE unstable Release

 Err:2 file:$APTARCHIVE unstable Release
   File not found
 W: The repository 'file:$APTARCHIVE unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository.
 Err:2 file:$APTARCHIVE unstable Release
   File not found
 W: The repository 'file:$APTARCHIVE unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository.


@@ -36,10 +38,41 @@ filesize() {
        stat -c%s "$(aptget files --no-release-info --format '$(URI)' "Created-By: $1" | cut -d'/' -f 3- ).gz"
 }
 # allow override
        stat -c%s "$(aptget files --no-release-info --format '$(URI)' "Created-By: $1" | cut -d'/' -f 3- ).gz"
 }
 # allow override

-testwarningequal "Ign:1 file:$APTARCHIVE unstable InRelease
+#aptget update --allow-insecure-repositories -o Debug::pkgAcquire::worker=1
+#exit
+testwarningequal "Get:1 file:$APTARCHIVE unstable InRelease
+Ign:1 file:$APTARCHIVE unstable InRelease

   File not found
   File not found

+Get:2 file:$APTARCHIVE unstable Release

 Ign:2 file:$APTARCHIVE unstable Release
   File not found
 Ign:2 file:$APTARCHIVE unstable Release
   File not found

+Get:3 file:$APTARCHIVE unstable/main Sources
+Ign:3 file:$APTARCHIVE unstable/main Sources
+  File not found
+Get:4 file:$APTARCHIVE unstable/main i386 Packages
+Ign:4 file:$APTARCHIVE unstable/main i386 Packages
+  File not found
+Get:5 file:$APTARCHIVE unstable/main Translation-en
+Ign:5 file:$APTARCHIVE unstable/main Translation-en
+  File not found
+Get:3 file:$APTARCHIVE unstable/main Sources
+Ign:3 file:$APTARCHIVE unstable/main Sources
+  File not found
+Get:4 file:$APTARCHIVE unstable/main i386 Packages
+Ign:4 file:$APTARCHIVE unstable/main i386 Packages
+  File not found
+Get:5 file:$APTARCHIVE unstable/main Translation-en
+Ign:5 file:$APTARCHIVE unstable/main Translation-en
+  File not found
+Get:3 file:$APTARCHIVE unstable/main Sources
+Ign:3 file:$APTARCHIVE unstable/main Sources
+  File not found
+Get:4 file:$APTARCHIVE unstable/main i386 Packages
+Ign:4 file:$APTARCHIVE unstable/main i386 Packages
+  File not found
+Get:5 file:$APTARCHIVE unstable/main Translation-en
+Ign:5 file:$APTARCHIVE unstable/main Translation-en
+  File not found

 Get:3 file:$APTARCHIVE unstable/main Sources [$(filesize 'Sources') B]
 Get:4 file:$APTARCHIVE unstable/main i386 Packages [$(filesize 'Packages') B]
 Get:5 file:$APTARCHIVE unstable/main Translation-en [$(filesize 'Translations') B]
 Get:3 file:$APTARCHIVE unstable/main Sources [$(filesize 'Sources') B]
 Get:4 file:$APTARCHIVE unstable/main i386 Packages [$(filesize 'Packages') B]
 Get:5 file:$APTARCHIVE unstable/main Translation-en [$(filesize 'Translations') B]

diff --git a/test/integration/test-apt-progress-fd b/test/integration/test-apt-progress-fd
index 99b4ea050f45d0debf785070c875fbbbbb8fa877..e30d503cbcee6713742df8fa99375155717646d2 100755 (executable)
--- a/test/integration/test-apt-progress-fd
+++ b/test/integration/test-apt-progress-fd
@@ -16,7 +16,6 @@ setupaptarchive
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.1 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.1 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1

-dlstatus:1:0:Retrieving file 1 of 1

 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)


@@ -33,7 +32,6 @@ pmstatus:dpkg-exec:83.3333:Running dpkg'
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.8.15 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.8.15 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1

-dlstatus:1:0:Retrieving file 1 of 1

 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)


@@ -50,7 +48,6 @@ pmstatus:dpkg-exec:83.3333:Running dpkg'
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.8.15 --reinstall -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.8.15 --reinstall -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1

-dlstatus:1:0:Retrieving file 1 of 1

 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)


@@ -77,7 +74,6 @@ pmstatus:dpkg-exec:75:Running dpkg'
 exec 3> apt-progress.log
 testsuccess aptget install testing2:i386 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
 exec 3> apt-progress.log
 testsuccess aptget install testing2:i386 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1

-dlstatus:1:0:Retrieving file 1 of 1

 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing2:0:Installing testing2 (i386)
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing2:0:Installing testing2 (i386)

diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index 2b662171c722eccaa540c2f55b4d7142927e6005..33b4ed1b91908c67d3a9414791d9c3299a4bbf53 100755 (executable)
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -134,6 +134,24 @@ EXPECT="Ign:1 http://localhost:8080 unstable InRelease
   404  Not Found
 Ign:2 http://localhost:8080 unstable Release
   404  Not Found
   404  Not Found
 Ign:2 http://localhost:8080 unstable Release
   404  Not Found

+Ign:3 http://localhost:8080 unstable/main Sources
+  404  Not Found
+Ign:4 http://localhost:8080 unstable/main amd64 Packages
+  404  Not Found
+Ign:5 http://localhost:8080 unstable/main Translation-en
+  404  Not Found
+Ign:3 http://localhost:8080 unstable/main Sources
+  404  Not Found
+Ign:4 http://localhost:8080 unstable/main amd64 Packages
+  404  Not Found
+Ign:5 http://localhost:8080 unstable/main Translation-en
+  404  Not Found
+Ign:3 http://localhost:8080 unstable/main Sources
+  404  Not Found
+Ign:4 http://localhost:8080 unstable/main amd64 Packages
+  404  Not Found
+Ign:5 http://localhost:8080 unstable/main Translation-en
+  404  Not Found

 Hit:3 http://localhost:8080 unstable/main Sources
 Hit:4 http://localhost:8080 unstable/main amd64 Packages
 Hit:5 http://localhost:8080 unstable/main Translation-en
 Hit:3 http://localhost:8080 unstable/main Sources
 Hit:4 http://localhost:8080 unstable/main amd64 Packages
 Hit:5 http://localhost:8080 unstable/main Translation-en

diff --git a/test/integration/test-bug-722207-print-uris-even-if-very-quiet b/test/integration/test-bug-722207-print-uris-even-if-very-quiet
index e51d72ccd6e2f00fb62906b764a9ba0c1e3a1adc..1fa94de7db005ba1dfa7126c83abe811b4190b04 100755 (executable)
--- a/test/integration/test-bug-722207-print-uris-even-if-very-quiet
+++ b/test/integration/test-bug-722207-print-uris-even-if-very-quiet
@@ -21,11 +21,11 @@ testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.d
 testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget dist-upgrade -qq --print-uris
 testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget install apt -qq --print-uris
 testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget download apt -qq --print-uris
 testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget dist-upgrade -qq --print-uris
 testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget install apt -qq --print-uris
 testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget download apt -qq --print-uris

-testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source apt -qq --print-uris
+testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 9 MD5Sum:16ff470aaedad0f06fb951ed89ffdd3a
+'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 12 MD5Sum:ab2b546f59ff9e8f5cc7a2d987ff3373" aptget source apt -qq --print-uris

 testsuccessequal "'http://metadata.ftp-master.debian.org/changelogs/main/a/apt/apt_2_changelog' apt.changelog" aptget changelog apt -qq --print-uris
 
 testsuccessequal "'http://metadata.ftp-master.debian.org/changelogs/main/a/apt/apt_2_changelog' apt.changelog" aptget changelog apt -qq --print-uris
 

-testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/apt2_1.dsc' apt2_1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/apt2_1.tar.gz' apt2_1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source apt apt2 -qq --print-uris
+testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 9 MD5Sum:16ff470aaedad0f06fb951ed89ffdd3a
+'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 12 MD5Sum:ab2b546f59ff9e8f5cc7a2d987ff3373
+'file://${APTARCHIVE}/apt2_1.dsc' apt2_1.dsc 10 MD5Sum:4c572ce45f1e2bedbb30da7f5e1c241c
+'file://${APTARCHIVE}/apt2_1.tar.gz' apt2_1.tar.gz 13 MD5Sum:2a96fec139f8722d93312a1ff8281232" aptget source apt apt2 -qq --print-uris

diff --git a/test/integration/test-bug-738785-switch-protocol b/test/integration/test-bug-738785-switch-protocol
index 5390802000016f35fbc16cd9004e5101ed47eba0..e86f28824c29d0f24432a4bd696eb9c8bd868067 100755 (executable)
--- a/test/integration/test-bug-738785-switch-protocol
+++ b/test/integration/test-bug-738785-switch-protocol
@@ -39,18 +39,15 @@ cd - >/dev/null
 testsuccess aptget install apt -y
 testdpkginstalled 'apt'
 
 testsuccess aptget install apt -y
 testdpkginstalled 'apt'
 

-# create a copy of all methods, expect https
-eval `aptconfig shell METHODS Dir::Bin::Methods/d`
-COPYMETHODS='usr/lib/apt/methods'
-mv rootdir/${COPYMETHODS} rootdir/${COPYMETHODS}.bak
-mkdir -p rootdir/$COPYMETHODS
-cd rootdir/$COPYMETHODS
-find $METHODS \! -type d | while read meth; do
-       ln -s $meth
+# install a slowed down file: otherwise its to fast to reproduce combining
+NEWMETHODS="$(readlink -f rootdir)/usr/lib/apt/methods"
+OLDMETHODS="$(readlink -f rootdir/usr/lib/apt/methods)"
+rm $NEWMETHODS
+mkdir $NEWMETHODS
+for METH in $(find $OLDMETHODS ! -type d); do
+       ln -s $OLDMETHODS/$(basename $METH) $NEWMETHODS

 done
 done

-rm https
-cd - >/dev/null
-echo "Dir::Bin::Methods \"${COPYMETHODS}\";" >> aptconfig.conf
+rm $NEWMETHODS/https

 
 cd downloaded
 testfailureequal "E: The method driver $(readlink -f './../')/rootdir/usr/lib/apt/methods/https could not be found.
 
 cd downloaded
 testfailureequal "E: The method driver $(readlink -f './../')/rootdir/usr/lib/apt/methods/https could not be found.


@@ -59,8 +56,7 @@ testfailure test -e apt_1.0_all.deb
 cd - >/dev/null
 
 # revert to all methods
 cd - >/dev/null
 
 # revert to all methods

-rm -rf rootdir/$COPYMETHODS
-mv rootdir/${COPYMETHODS}.bak rootdir/${COPYMETHODS}
+ln -s $OLDMETHODS/https $NEWMETHODS

 
 # check that downgrades from https to http are not allowed
 webserverconfig 'aptwebserver::support::http' 'true'
 
 # check that downgrades from https to http are not allowed
 webserverconfig 'aptwebserver::support::http' 'true'