retry without partial data after a 416 response

If we get a 416 from the server it means the Range we asked for is above
the real filesize of the file on the server. Mostly this happens if the
server isn't supporting If-Range, but regardless of how we end up with
the partial data, the data is invalid so we discard it and retry with a
fresh plate and hope for the best.

Old behavior was to consider 416 an error and retry with a different
compression until we ran out of compression and requested the
uncompressed file (which doesn't exist on most mirrors) with an accept
line which server answered with "406 Not Acceptable".

Closes: 710924

diff --git a/methods/http.cc b/methods/http.cc
index 278ddb290519fba10b3dc44b03b80e39d7fa81d6..2a8b7f0fccdfc883c9ef853ef825b205d604316d 100644 (file)
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -604,6 +604,8 @@ bool ServerState::HeaderLine(string Line)
       Size = strtoull(Val.c_str(), NULL, 10);
       if (Size >= std::numeric_limits<unsigned long long>::max())
         return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
       Size = strtoull(Val.c_str(), NULL, 10);
       if (Size >= std::numeric_limits<unsigned long long>::max())
         return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));

+      else if (Size == 0)
+        HaveContent = false;

       return true;
    }
 
       return true;
    }
 


@@ -616,8 +618,14 @@ bool ServerState::HeaderLine(string Line)
    if (stringcasecmp(Tag,"Content-Range:") == 0)
    {
       HaveContent = true;
    if (stringcasecmp(Tag,"Content-Range:") == 0)
    {
       HaveContent = true;

-      
-      if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
+
+      // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
+      if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
+      {
+        StartPos = 1; // ignore Content-Length, it would override Size
+        HaveContent = false;
+      }
+      else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)

         return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
       if ((unsigned long long)StartPos > Size)
         return _error->Error(_("This HTTP server has broken range support"));
         return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
       if ((unsigned long long)StartPos > Size)
         return _error->Error(_("This HTTP server has broken range support"));


@@ -995,6 +1003,13 @@ HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
       }
       /* else pass through for error message */
    }
       }
       /* else pass through for error message */
    }

+   // retry after an invalid range response without partial data
+   else if (Srv->Result == 416 && FileExists(Queue->DestFile) == true &&
+        unlink(Queue->DestFile.c_str()) == 0)
+   {
+      NextURI = Queue->Uri;
+      return TRY_AGAIN_OR_REDIRECT;
+   }

  
    /* We have a reply we dont handle. This should indicate a perm server
       failure */
  
    /* We have a reply we dont handle. This should indicate a perm server
       failure */

diff --git a/test/integration/test-releasefile-verification b/test/integration/test-releasefile-verification
index a9f4b977523d0ba3be51e5b56899e3ccdd547ece..9d34a521a45fe972e1e415147988c08e2e80102f 100755 (executable)
--- a/test/integration/test-releasefile-verification
+++ b/test/integration/test-releasefile-verification
@@ -11,18 +11,24 @@ buildaptarchive
 setupflataptarchive
 changetowebserver
 
 setupflataptarchive
 changetowebserver
 

+downloadfile "http://localhost:8080/_config/set/aptwebserver::support::range/false" '/dev/null' >/dev/null
+

 prepare() {
        local DATE="${2:-now}"
 prepare() {
        local DATE="${2:-now}"

-       if [ "$DATE" = 'now' -a "$1" = "${PKGFILE}-new" ]; then
-               DATE='now + 6 days'
+       if [ "$DATE" = 'now' ]; then
+               if [ "$1" = "${PKGFILE}-new" ]; then
+                       DATE='now - 1 day'
+               else
+                       DATE='now - 7 day'
+               fi

        fi
        for release in $(find rootdir/var/lib/apt/lists 2> /dev/null); do
        fi
        for release in $(find rootdir/var/lib/apt/lists 2> /dev/null); do

-               touch -d 'now - 6 hours' $release
+               touch -d 'now - 1 year' $release

        done
        aptget clean
        cp $1 aptarchive/Packages
        find aptarchive -name 'Release' -delete
        done
        aptget clean
        cp $1 aptarchive/Packages
        find aptarchive -name 'Release' -delete

-       compressfile 'aptarchive/Packages'
+       compressfile 'aptarchive/Packages' "$DATE"

        generatereleasefiles "$DATE"
 }
 
        generatereleasefiles "$DATE"
 }
 


@@ -85,13 +91,34 @@ touch aptarchive/apt.deb
 
 PKGFILE="${TESTDIR}/$(echo "$(basename $0)" | sed 's#^test-#Packages-#')"
 
 
 PKGFILE="${TESTDIR}/$(echo "$(basename $0)" | sed 's#^test-#Packages-#')"
 

+updatesuccess() {
+       local LOG='update.log'
+       if aptget update >$LOG 2>&1 || grep -q -E '^(W|E): ' $LOG; then
+               msgpass
+       else
+               cat $LOG
+               msgfail
+       fi
+}
+
+updatefailure() {
+       local LOG='update.log'
+       aptget update >$LOG 2>&1 || true
+       if grep -q -E "$1" $LOG; then
+               msgpass
+       else
+               cat $LOG
+               msgfail
+       fi
+}
+

 runtest() {
        prepare ${PKGFILE}
        rm -rf rootdir/var/lib/apt/lists
        signreleasefiles 'Joe Sixpack'
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Cold archive signed by' 'Joe Sixpack'
 runtest() {
        prepare ${PKGFILE}
        rm -rf rootdir/var/lib/apt/lists
        signreleasefiles 'Joe Sixpack'
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Cold archive signed by' 'Joe Sixpack'

-       aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+       updatesuccess

        testequal "$(cat ${PKGFILE})
 " aptcache show apt
        installaptold
        testequal "$(cat ${PKGFILE})
 " aptcache show apt
        installaptold


@@ -100,7 +127,7 @@ runtest() {
        signreleasefiles 'Joe Sixpack'
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Good warm archive signed by' 'Joe Sixpack'
        signreleasefiles 'Joe Sixpack'
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Good warm archive signed by' 'Joe Sixpack'

-       aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+       updatesuccess

        testequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
        installaptnew
        testequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
        installaptnew


@@ -111,7 +138,7 @@ runtest() {
        signreleasefiles 'Rex Expired'
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Cold archive signed by' 'Rex Expired'
        signreleasefiles 'Rex Expired'
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Cold archive signed by' 'Rex Expired'

-       aptget update 2>&1 | grep -E '^W: .* KEYEXPIRED' > /dev/null && msgpass || msgfail
+       updatefailure '^W: .* KEYEXPIRED'

        testequal "$(cat ${PKGFILE})
 " aptcache show apt
        failaptold
        testequal "$(cat ${PKGFILE})
 " aptcache show apt
        failaptold


@@ -122,7 +149,7 @@ runtest() {
        signreleasefiles 'Marvin Paranoid'
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Cold archive signed by' 'Marvin Paranoid'
        signreleasefiles 'Marvin Paranoid'
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Cold archive signed by' 'Marvin Paranoid'

-       aptget update 2>&1 | grep -E '^W: .* NO_PUBKEY' > /dev/null && msgpass || msgfail
+       updatefailure '^W: .* NO_PUBKEY'

        testequal "$(cat ${PKGFILE})
 " aptcache show apt
        failaptold
        testequal "$(cat ${PKGFILE})
 " aptcache show apt
        failaptold


@@ -136,7 +163,7 @@ runtest() {
        signreleasefiles 'Joe Sixpack'
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Bad warm archive signed by' 'Joe Sixpack'
        signreleasefiles 'Joe Sixpack'
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Bad warm archive signed by' 'Joe Sixpack'

-       aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+       updatesuccess

        testequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
        installaptnew
        testequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
        installaptnew


@@ -147,7 +174,7 @@ runtest() {
        signreleasefiles 'Joe Sixpack'
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Cold archive signed by' 'Joe Sixpack'
        signreleasefiles 'Joe Sixpack'
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Cold archive signed by' 'Joe Sixpack'

-       aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+       updatesuccess

        testequal "$(cat ${PKGFILE})
 " aptcache show apt
        installaptold
        testequal "$(cat ${PKGFILE})
 " aptcache show apt
        installaptold


@@ -156,7 +183,7 @@ runtest() {
        signreleasefiles 'Marvin Paranoid'
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Good warm archive signed by' 'Marvin Paranoid'
        signreleasefiles 'Marvin Paranoid'
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Good warm archive signed by' 'Marvin Paranoid'

-       aptget update 2>&1 | grep -E '^W: .* NO_PUBKEY' > /dev/null && msgpass || msgfail
+       updatefailure '^W: .* NO_PUBKEY'

        testequal "$(cat ${PKGFILE})
 " aptcache show apt
        installaptold
        testequal "$(cat ${PKGFILE})
 " aptcache show apt
        installaptold


@@ -166,7 +193,7 @@ runtest() {
        signreleasefiles 'Rex Expired'
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Good warm archive signed by' 'Rex Expired'
        signreleasefiles 'Rex Expired'
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Good warm archive signed by' 'Rex Expired'

-       aptget update 2>&1 | grep -E '^W: .* KEYEXPIRED' > /dev/null && msgpass || msgfail
+       updatefailure '^W: .* KEYEXPIRED'

        testequal "$(cat ${PKGFILE})
 " aptcache show apt
        installaptold
        testequal "$(cat ${PKGFILE})
 " aptcache show apt
        installaptold


@@ -176,7 +203,7 @@ runtest() {
        signreleasefiles
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Good warm archive signed by' 'Joe Sixpack'
        signreleasefiles
        find aptarchive/ -name "$DELETEFILE" -delete
        msgtest 'Good warm archive signed by' 'Joe Sixpack'

-       aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+       updatesuccess

        testequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
        installaptnew
        testequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
        installaptnew


@@ -187,7 +214,7 @@ runtest2() {
        rm -rf rootdir/var/lib/apt/lists
        signreleasefiles 'Joe Sixpack'
        msgtest 'Cold archive signed by' 'Joe Sixpack'
        rm -rf rootdir/var/lib/apt/lists
        signreleasefiles 'Joe Sixpack'
        msgtest 'Cold archive signed by' 'Joe Sixpack'

-       aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+       updatesuccess

 
        # New .deb but now an unsigned archive. For example MITM to circumvent
        # package verification.
 
        # New .deb but now an unsigned archive. For example MITM to circumvent
        # package verification.


@@ -195,7 +222,7 @@ runtest2() {
        find aptarchive/ -name InRelease -delete
        find aptarchive/ -name Release.gpg -delete
        msgtest 'Warm archive signed by' 'nobody'
        find aptarchive/ -name InRelease -delete
        find aptarchive/ -name Release.gpg -delete
        msgtest 'Warm archive signed by' 'nobody'

-       aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+       updatesuccess

        testequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
        failaptnew
        testequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
        failaptnew


@@ -203,7 +230,7 @@ runtest2() {
        # Unsigned archive from the beginning must also be detected.
        rm -rf rootdir/var/lib/apt/lists
        msgtest 'Cold archive signed by' 'nobody'
        # Unsigned archive from the beginning must also be detected.
        rm -rf rootdir/var/lib/apt/lists
        msgtest 'Cold archive signed by' 'nobody'

-       aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+       updatesuccess

        testequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
        failaptnew
        testequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
        failaptnew