dispose http(s) 416 error page as non-content

Real webservers (like apache) actually send an error page with a 416
response, but our client didn't expect it leaving the page on the socket
to be parsed as response for the next request (http) or as file content
(https), which isn't what we want at all… Symptom is a "Bad header line"
as html usually doesn't parse that well to an http-header.

This manifests itself e.g. if we have a complete file (or larger) in
partial/ which isn't discarded by If-Range as the server doesn't support
it (or it is just newer, think: mirror rotation).
It is a sort-of regression of 78c72d0ce22e00b194251445aae306df357d5c1a,
which removed the filesize - 1 trick, but this had its own problems…

To properly test this our webserver gains the ability to reply with
transfer-encoding: chunked as most real webservers will use it to send
the dynamically generated error pages.

(The tests and their binary helpers had to be slightly modified to
apply, but the patch to fix the issue itself is unchanged.)

Closes: 768797

diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc
index dd43ea1bce28f87750ac7f71a0034c173e4f9189..63f70983cc9846dc9d8428fa0961973964d5afbf 100644 (file)
--- a/cmdline/apt-helper.cc
+++ b/cmdline/apt-helper.cc
@@ -48,23 +48,34 @@ static bool DoDownloadFile(CommandLine &CmdL)
    if (CmdL.FileSize() <= 2)
       return _error->Error(_("Must specify at least one pair url/filename"));
 
    if (CmdL.FileSize() <= 2)
       return _error->Error(_("Must specify at least one pair url/filename"));
 

-

    pkgAcquire Fetcher;
    AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0));
    Fetcher.Setup(&Stat);
    pkgAcquire Fetcher;
    AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0));
    Fetcher.Setup(&Stat);

-   std::string download_uri = CmdL.FileList[1];
-   std::string targetfile = CmdL.FileList[2];
-   std::string hash;
-   if (CmdL.FileSize() > 3)
-      hash = CmdL.FileList[3];
-   // we use download_uri as descr and targetfile as short-descr
-   new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile, 
-                  "dest-dir-ignored", targetfile);
-   Fetcher.Run();
+
+   size_t fileind = 0;
+   std::vector<std::string> targetfiles;
+   while (fileind + 2 <= CmdL.FileSize())
+   {
+      std::string download_uri = CmdL.FileList[fileind + 1];
+      std::string targetfile = CmdL.FileList[fileind + 2];
+      std::string hash;
+      if (CmdL.FileSize() > fileind + 3)
+        hash = CmdL.FileList[fileind + 3];
+      // we use download_uri as descr and targetfile as short-descr
+      new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile,
+           "dest-dir-ignored", targetfile);
+      targetfiles.push_back(targetfile);
+      fileind += 3;
+   }
+

    bool Failed = false;
    bool Failed = false;

-   if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true ||
-        FileExists(targetfile) == false)
+   if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true)

       return _error->Error(_("Download Failed"));
       return _error->Error(_("Download Failed"));

+   if (targetfiles.empty() == false)
+      for (std::vector<std::string>::const_iterator f = targetfiles.begin(); f != targetfiles.end(); ++f)
+        if (FileExists(*f) == false)
+           return _error->Error(_("Download Failed"));
+

    return true;
 }
 
    return true;
 }
 

diff --git a/methods/http.cc b/methods/http.cc
index f2a4a4db631add976bcbf36d329a57d45d3304a5..1b996db9816bab01f688a645a0f855b90a0c138f 100644 (file)
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -440,6 +440,8 @@ bool HttpServerState::RunData(FileFd * const File)
          loss of the connection means we are done */
       if (Encoding == Closes)
         In.Limit(-1);
          loss of the connection means we are done */
       if (Encoding == Closes)
         In.Limit(-1);

+      else if (JunkSize != 0)
+        In.Limit(JunkSize);

       else
         In.Limit(Size - StartPos);
       
       else
         In.Limit(Size - StartPos);
       

diff --git a/methods/https.cc b/methods/https.cc
index 0499af0c5126bc629bc0ac8b96c8a45ca98bf814..65a744e2a926fb07e90e49793b55d4e4103bb11a 100644 (file)
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -59,6 +59,9 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
       {
          me->Server->Result = 200;
         me->Server->StartPos = me->Server->Size;
       {
          me->Server->Result = 200;
         me->Server->StartPos = me->Server->Size;

+        // the actual size is not important for https as curl will deal with it
+        // by itself and e.g. doesn't bother us with transport-encoding…
+        me->Server->JunkSize = std::numeric_limits<unsigned long long>::max();

       }
       else
         me->Server->StartPos = 0;
       }
       else
         me->Server->StartPos = 0;


@@ -76,13 +79,18 @@ size_t
 HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
 {
    HttpsMethod *me = (HttpsMethod *)userp;
 HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
 {
    HttpsMethod *me = (HttpsMethod *)userp;

+   size_t buffer_size = size * nmemb;
+   // we don't need to count the junk here, just drop anything we get as
+   // we don't always know how long it would be, e.g. in chunked encoding.
+   if (me->Server->JunkSize != 0)
+      return buffer_size;

 
    if (me->Res.Size == 0)
       me->URIStart(me->Res);
 
    if (me->Res.Size == 0)
       me->URIStart(me->Res);

-   if(me->File->Write(buffer, size*nmemb) != true)
+   if(me->File->Write(buffer, buffer_size) != true)

       return false;
 
       return false;
 

-   return size*nmemb;
+   return buffer_size;

 }
 
 int
 }
 
 int

diff --git a/methods/server.cc b/methods/server.cc
index 92d94e63830118c2b360317be4ca90acbdc428fe..cb0341d5fb7b9447ba06ea85cd68ec7bb19b2e6d 100644 (file)
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -55,6 +55,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
    Minor = 0; 
    Result = 0; 
    Size = 0; 
    Minor = 0; 
    Result = 0; 
    Size = 0; 

+   JunkSize = 0;

    StartPos = 0;
    Encoding = Closes;
    HaveContent = false;
    StartPos = 0;
    Encoding = Closes;
    HaveContent = false;


@@ -163,14 +164,14 @@ bool ServerState::HeaderLine(string Line)
         Encoding = Stream;
       HaveContent = true;
 
         Encoding = Stream;
       HaveContent = true;
 

-      // The length is already set from the Content-Range header
-      if (StartPos != 0)
-        return true;
+      unsigned long long * SizePtr = &Size;
+      if (Result == 416)
+        SizePtr = &JunkSize;

 
 

-      Size = strtoull(Val.c_str(), NULL, 10);
-      if (Size >= std::numeric_limits<unsigned long long>::max())
+      *SizePtr = strtoull(Val.c_str(), NULL, 10);
+      if (*SizePtr >= std::numeric_limits<unsigned long long>::max())

         return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
         return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));

-      else if (Size == 0)
+      else if (*SizePtr == 0)

         HaveContent = false;
       return true;
    }
         HaveContent = false;
       return true;
    }


@@ -187,10 +188,7 @@ bool ServerState::HeaderLine(string Line)
 
       // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
       if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
 
       // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
       if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)

-      {
-        StartPos = 1; // ignore Content-Length, it would override Size
-        HaveContent = false;
-      }
+        ; // we got the expected filesize which is all we wanted

       else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
         return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
       if ((unsigned long long)StartPos > Size)
       else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
         return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
       if ((unsigned long long)StartPos > Size)


@@ -308,9 +306,15 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
         if ((unsigned long long)SBuf.st_size == Server->Size)
         {
            // the file is completely downloaded, but was not moved
         if ((unsigned long long)SBuf.st_size == Server->Size)
         {
            // the file is completely downloaded, but was not moved

+           if (Server->HaveContent == true)
+           {
+              // Send to error page to dev/null
+              FileFd DevNull("/dev/null",FileFd::WriteExists);
+              Server->RunData(&DevNull);
+           }
+           Server->HaveContent = false;

            Server->StartPos = Server->Size;
            Server->Result = 200;
            Server->StartPos = Server->Size;
            Server->Result = 200;

-           Server->HaveContent = false;

         }
         else if (unlink(Queue->DestFile.c_str()) == 0)
         {
         }
         else if (unlink(Queue->DestFile.c_str()) == 0)
         {

diff --git a/methods/server.h b/methods/server.h
index f5e68d9027b6373d10da5d507365c37c817eea79..1b81e3549b5d05370700f1985aa532d2e3fff2aa 100644 (file)
--- a/methods/server.h
+++ b/methods/server.h
@@ -34,7 +34,8 @@ struct ServerState
    char Code[360];
 
    // These are some statistics from the last parsed header lines
    char Code[360];
 
    // These are some statistics from the last parsed header lines

-   unsigned long long Size;
+   unsigned long long Size; // size of the usable content (aka: the file)
+   unsigned long long JunkSize; // size of junk content (aka: server error pages)

    unsigned long long StartPos;
    time_t Date;
    bool HaveContent;
    unsigned long long StartPos;
    time_t Date;
    bool HaveContent;


@@ -71,7 +72,7 @@ struct ServerState
    RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri);
 
    bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
    RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri);
 
    bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};

-   virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0;
+   virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0;

                 StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false;
                 State = Header; Persistent = false; Pipeline = true;};
    virtual bool WriteResponse(std::string const &Data) = 0;
                 StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false;
                 State = Header; Persistent = false; Pipeline = true;};
    virtual bool WriteResponse(std::string const &Data) = 0;

diff --git a/test/integration/framework b/test/integration/framework
index df1942ff9cdf0543a538681330e480fc88331954..ac482a7a0bed9c43217a22a82a4a54a7c7039ee7 100644 (file)
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1064,8 +1064,8 @@ acquire::cdrom::autodetect 0;" > rootdir/etc/apt/apt.conf.d/00cdrom
 }
 
 downloadfile() {
 }
 
 downloadfile() {

-       local PROTO="$(echo "$1" | cut -d':' -f 1 )"
-       apthelper -o Debug::Acquire::${PROTO}=1 \
+       local PROTO="${1%%:*}"
+       apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \

                download-file "$1" "$2" 2>&1 || true
        # only if the file exists the download was successful
        if [ -e "$2" ]; then
                download-file "$1" "$2" 2>&1 || true
        # only if the file exists the download was successful
        if [ -e "$2" ]; then


@@ -1221,7 +1221,7 @@ testsuccess() {
                msgtest 'Test for successful execution of' "$*"
        fi
        local OUTPUT="${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output"
                msgtest 'Test for successful execution of' "$*"
        fi
        local OUTPUT="${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output"

-       if $@ >${OUTPUT} 2>&1; then
+       if "$@" >${OUTPUT} 2>&1; then

                msgpass
        else
                echo >&2
                msgpass
        else
                echo >&2

diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper
index c749224ca2afb294985436bb2d4d5aee2bfc2cb2..31e471677c3d6992e2b5bcd995787b06460ef1df 100755 (executable)
--- a/test/integration/test-apt-helper
+++ b/test/integration/test-apt-helper
@@ -10,34 +10,36 @@ configarchitecture "i386"
 changetohttpswebserver
 
 test_apt_helper_download() {
 changetohttpswebserver
 
 test_apt_helper_download() {

-    echo "foo" > aptarchive/foo
+    echo 'foo' > aptarchive/foo
+    echo 'bar' > aptarchive/foo2

 
     msgtest 'apt-file download-file md5sum'
 
     msgtest 'apt-file download-file md5sum'

-    apthelper -qq download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00

     testfileequal foo2 'foo'
 
     msgtest 'apt-file download-file sha1'
     testfileequal foo2 'foo'
 
     msgtest 'apt-file download-file sha1'

-    apthelper -qq download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15

     testfileequal foo1 'foo'
 
     msgtest 'apt-file download-file sha256'
     testfileequal foo1 'foo'
 
     msgtest 'apt-file download-file sha256'

-    apthelper -qq download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c

     testfileequal foo3 'foo'
 
     msgtest 'apt-file download-file no-hash'
     testfileequal foo3 'foo'
 
     msgtest 'apt-file download-file no-hash'

-    apthelper -qq download-file http://localhost:8080/foo foo4 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo4

     testfileequal foo4 'foo'
     
     msgtest 'apt-file download-file wrong hash'
     testfileequal foo4 'foo'
     
     msgtest 'apt-file download-file wrong hash'

-    if ! apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc 2>&1 2> download.stderr; then
-        msgpass
-    else
-        msgfail
-    fi
-    testfileequal download.stderr 'E: Failed to fetch http://localhost:8080/foo  Hash Sum mismatch
+    testfailure --nomsg apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc
+    testfileequal rootdir/tmp/testfailure.output 'E: Failed to fetch http://localhost:8080/foo  Hash Sum mismatch

 
 E: Download Failed'
     testfileequal foo5.FAILED 'foo'
 
 E: Download Failed'
     testfileequal foo5.FAILED 'foo'

+
+    msgtest 'apt-file download-file md5sum sha1'
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo6 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 http://localhost:8080/foo2 foo7 SHA1:e242ed3bffccdf271b7fbaf34ed72d089537b42f
+    testfileequal foo6 'foo'
+    testfileequal foo7 'bar'

 }
 
 test_apt_helper_detect_proxy() {
 }
 
 test_apt_helper_detect_proxy() {

diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support
index 5ab326defce3efaa056e8c0fe136245c6b54bb59..160d451b6c85dc9b2a4906d5c3654009d18ecd89 100755 (executable)
--- a/test/integration/test-partial-file-support
+++ b/test/integration/test-partial-file-support
@@ -24,13 +24,25 @@ testdownloadfile() {
        else
                msgpass
        fi
        else
                msgpass
        fi

-       cat "$DOWNLOADLOG" | while read field hash; do
+       sed -e '/^ <- / s#%20# #g' -e '/^ <- / s#%0a#\n#g' "$DOWNLOADLOG" | grep '^.*-Hash: ' > receivedhashes.log
+       testsuccess test -s receivedhashes.log
+       local HASHES_OK=0
+       local HASHES_BAD=0
+       while read field hash; do

                local EXPECTED
                case "$field" in
                'MD5Sum-Hash:') EXPECTED="$(md5sum "$TESTFILE" | cut -d' ' -f 1)";;
                'SHA1-Hash:') EXPECTED="$(sha1sum "$TESTFILE" | cut -d' ' -f 1)";;
                'SHA256-Hash:') EXPECTED="$(sha256sum "$TESTFILE" | cut -d' ' -f 1)";;
                'SHA512-Hash:') EXPECTED="$(sha512sum "$TESTFILE" | cut -d' ' -f 1)";;
                local EXPECTED
                case "$field" in
                'MD5Sum-Hash:') EXPECTED="$(md5sum "$TESTFILE" | cut -d' ' -f 1)";;
                'SHA1-Hash:') EXPECTED="$(sha1sum "$TESTFILE" | cut -d' ' -f 1)";;
                'SHA256-Hash:') EXPECTED="$(sha256sum "$TESTFILE" | cut -d' ' -f 1)";;
                'SHA512-Hash:') EXPECTED="$(sha512sum "$TESTFILE" | cut -d' ' -f 1)";;

+               'Checksum-FileSize-Hash:')
+                       #filesize is too weak to check for !=
+                       if [ "$4" = '=' ]; then
+                               EXPECTED="$(stat -c '%s' "$TESTFILE")"
+                       else
+                               continue
+                       fi
+                       ;;

                *) continue;;
                esac
                if [ "$4" = '=' ]; then
                *) continue;;
                esac
                if [ "$4" = '=' ]; then


@@ -40,15 +52,41 @@ testdownloadfile() {
                fi
                if [ "$EXPECTED" "$4" "$hash" ]; then
                        msgpass
                fi
                if [ "$EXPECTED" "$4" "$hash" ]; then
                        msgpass

+                       HASHES_OK=$((HASHES_OK+1));

                else
                else

-                       cat >&2 "$DOWNLOADLOG"

                        msgfail "expected: $EXPECTED ; got: $hash"
                        msgfail "expected: $EXPECTED ; got: $hash"

+                       HASHES_BAD=$((HASHES_BAD+1));

                fi
                fi

-       done
+       done < receivedhashes.log
+       msgtest 'At least one good hash and no bad ones'
+       if [ $HASHES_OK -eq 0 ] || [ $HASHES_BAD -ne 0 ]; then
+               cat >&2 "$DOWNLOADLOG"
+               msgfail
+       else
+               msgpass
+       fi

 }
 
 TESTFILE='aptarchive/testfile'
 cp -a ${TESTDIR}/framework $TESTFILE
 }
 
 TESTFILE='aptarchive/testfile'
 cp -a ${TESTDIR}/framework $TESTFILE

+cp -a ${TESTDIR}/framework "${TESTFILE}2"
+
+followuprequest() {
+       local DOWN='./testfile'
+
+       copysource $TESTFILE 1M $DOWN
+       testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '='
+       testwebserverlaststatuscode '416' "$DOWNLOADLOG"
+
+       copysource $TESTFILE 1M $DOWN
+       copysource "${TESTFILE}2" 20 "${DOWN}2"
+       msgtest 'Testing download of files with' 'completely downloaded file + partial file'
+       testsuccess --nomsg apthelper -o Debug::Acquire::${1%%:*}=1 -o Debug::pkgAcquire::Worker=1 \
+               download-file "$1/testfile" "$DOWN" '' "$1/testfile2" "${DOWN}2"
+       testwebserverlaststatuscode '206' 'rootdir/tmp/testsuccess.output'
+       testsuccess diff -u "$TESTFILE" "${DOWN}"
+       testsuccess diff -u "${DOWN}" "${DOWN}2"
+}

 
 testrun() {
        webserverconfig 'aptwebserver::support::range' 'true'
 
 testrun() {
        webserverconfig 'aptwebserver::support::range' 'true'


@@ -65,9 +103,11 @@ testrun() {
        testdownloadfile 'invalid partial data' "${1}/testfile" './testfile' '!='
        testwebserverlaststatuscode '206' "$DOWNLOADLOG"
 
        testdownloadfile 'invalid partial data' "${1}/testfile" './testfile' '!='
        testwebserverlaststatuscode '206' "$DOWNLOADLOG"
 

-       copysource $TESTFILE 1M ./testfile
-       testdownloadfile 'completely downloaded file' "${1}/testfile" './testfile' '='
-       testwebserverlaststatuscode '416' "$DOWNLOADLOG"
+       webserverconfig 'aptwebserver::closeOnError' 'false'
+       followuprequest "$1"
+       webserverconfig 'aptwebserver::closeOnError' 'true'
+       followuprequest "$1"
+       webserverconfig 'aptwebserver::closeOnError' 'false'

 
        copysource /dev/zero 1M ./testfile
        testdownloadfile 'too-big partial file' "${1}/testfile" './testfile' '='
 
        copysource /dev/zero 1M ./testfile
        testdownloadfile 'too-big partial file' "${1}/testfile" './testfile' '='


@@ -85,8 +125,18 @@ testrun() {
        testwebserverlaststatuscode '200' "$DOWNLOADLOG"
 }
 
        testwebserverlaststatuscode '200' "$DOWNLOADLOG"
 }
 

+msgmsg 'http: Test with Content-Length'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false'
+testrun 'http://localhost:8080'
+msgmsg 'http: Test with Transfer-Encoding: chunked'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true'

 testrun 'http://localhost:8080'
 
 changetohttpswebserver
 
 testrun 'http://localhost:8080'
 
 changetohttpswebserver
 

+msgmsg 'https: Test with Content-Length'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false'
+testrun 'https://localhost:4433'
+msgmsg 'https: Test with Transfer-Encoding: chunked'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true'

 testrun 'https://localhost:4433'
 testrun 'https://localhost:4433'

diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 34476e1af44452753338e0320e6f07ea46b7b7f2..cd52da692a8339cffdbab68ff61b039ea72275a8 100644 (file)
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -19,6 +19,8 @@
 #include <sys/stat.h>
 #include <time.h>
 #include <unistd.h>
 #include <sys/stat.h>
 #include <time.h>
 #include <unistd.h>

+
+#include <algorithm>

 #include <iostream>
 #include <sstream>
 #include <list>
 #include <iostream>
 #include <sstream>
 #include <list>


@@ -79,12 +81,21 @@ static char const * httpcodeToStr(int const httpcode)               /*{{{*/
    return NULL;
 }
                                                                        /*}}}*/
    return NULL;
 }
                                                                        /*}}}*/

+static bool chunkedTransferEncoding(std::list<std::string> const &headers) {
+   if (std::find(headers.begin(), headers.end(), "Transfer-Encoding: chunked") != headers.end())
+      return true;
+   if (_config->FindB("aptwebserver::chunked-transfer-encoding", false) == true)
+      return true;
+   return false;
+}

 static void addFileHeaders(std::list<std::string> &headers, FileFd &data)/*{{{*/
 {
 static void addFileHeaders(std::list<std::string> &headers, FileFd &data)/*{{{*/
 {

-   std::ostringstream contentlength;
-   contentlength << "Content-Length: " << data.FileSize();
-   headers.push_back(contentlength.str());
-
+   if (chunkedTransferEncoding(headers) == false)
+   {
+      std::ostringstream contentlength;
+      contentlength << "Content-Length: " << data.FileSize();
+      headers.push_back(contentlength.str());
+   }

    std::string lastmodified("Last-Modified: ");
    lastmodified.append(TimeRFC1123(data.ModificationTime()));
    headers.push_back(lastmodified);
    std::string lastmodified("Last-Modified: ");
    lastmodified.append(TimeRFC1123(data.ModificationTime()));
    headers.push_back(lastmodified);


@@ -92,9 +103,12 @@ static void addFileHeaders(std::list<std::string> &headers, FileFd &data)/*{{{*/
                                                                        /*}}}*/
 static void addDataHeaders(std::list<std::string> &headers, std::string &data)/*{{{*/
 {
                                                                        /*}}}*/
 static void addDataHeaders(std::list<std::string> &headers, std::string &data)/*{{{*/
 {

-   std::ostringstream contentlength;
-   contentlength << "Content-Length: " << data.size();
-   headers.push_back(contentlength.str());
+   if (chunkedTransferEncoding(headers) == false)
+   {
+      std::ostringstream contentlength;
+      contentlength << "Content-Length: " << data.size();
+      headers.push_back(contentlength.str());
+   }

 }
                                                                        /*}}}*/
 static bool sendHead(int const client, int const httpcode, std::list<std::string> &headers)/*{{{*/
 }
                                                                        /*}}}*/
 static bool sendHead(int const client, int const httpcode, std::list<std::string> &headers)/*{{{*/


@@ -114,6 +128,9 @@ static bool sendHead(int const client, int const httpcode, std::list<std::string
    date.append(TimeRFC1123(time(NULL)));
    headers.push_back(date);
 
    date.append(TimeRFC1123(time(NULL)));
    headers.push_back(date);
 

+   if (chunkedTransferEncoding(headers) == true)
+      headers.push_back("Transfer-Encoding: chunked");
+

    std::clog << ">>> RESPONSE to " << client << " >>>" << std::endl;
    bool Success = true;
    for (std::list<std::string>::const_iterator h = headers.begin();
    std::clog << ">>> RESPONSE to " << client << " >>>" << std::endl;
    bool Success = true;
    for (std::list<std::string>::const_iterator h = headers.begin();


@@ -130,25 +147,55 @@ static bool sendHead(int const client, int const httpcode, std::list<std::string
    return Success;
 }
                                                                        /*}}}*/
    return Success;
 }
                                                                        /*}}}*/

-static bool sendFile(int const client, FileFd &data)                   /*{{{*/
+static bool sendFile(int const client, std::list<std::string> const &headers, FileFd &data)/*{{{*/

 {
    bool Success = true;
 {
    bool Success = true;

+   bool const chunked = chunkedTransferEncoding(headers);

    char buffer[500];
    unsigned long long actual = 0;
    while ((Success &= data.Read(buffer, sizeof(buffer), &actual)) == true)
    {
       if (actual == 0)
         break;
    char buffer[500];
    unsigned long long actual = 0;
    while ((Success &= data.Read(buffer, sizeof(buffer), &actual)) == true)
    {
       if (actual == 0)
         break;

-      Success &= FileFd::Write(client, buffer, actual);
+
+      if (chunked == true)
+      {
+        std::string size;
+        strprintf(size, "%llX\r\n", actual);
+        Success &= FileFd::Write(client, size.c_str(), size.size());
+        Success &= FileFd::Write(client, buffer, actual);
+        Success &= FileFd::Write(client, "\r\n", strlen("\r\n"));
+      }
+      else
+        Success &= FileFd::Write(client, buffer, actual);
+   }
+   if (chunked == true)
+   {
+      char const * const finish = "0\r\n\r\n";
+      Success &= FileFd::Write(client, finish, strlen(finish));

    }
    if (Success == false)
    }
    if (Success == false)

-      std::cerr << "SENDFILE: READ/WRITE ERROR to " << client << std::endl;
+      std::cerr << "SENDFILE:" << (chunked ? " CHUNKED" : "") << " READ/WRITE ERROR to " << client << std::endl;

    return Success;
 }
                                                                        /*}}}*/
    return Success;
 }
                                                                        /*}}}*/

-static bool sendData(int const client, std::string const &data)                /*{{{*/
+static bool sendData(int const client, std::list<std::string> const &headers, std::string const &data)/*{{{*/

 {
 {

-   if (FileFd::Write(client, data.c_str(), data.size()) == false)
+   if (chunkedTransferEncoding(headers) == true)
+   {
+      unsigned long long const ullsize = data.length();
+      std::string size;
+      strprintf(size, "%llX\r\n", ullsize);
+      char const * const finish = "\r\n0\r\n\r\n";
+      if (FileFd::Write(client, size.c_str(), size.length()) == false ||
+           FileFd::Write(client, data.c_str(), ullsize) == false ||
+           FileFd::Write(client, finish, strlen(finish)) == false)
+      {
+        std::cerr << "SENDDATA: CHUNK WRITE ERROR to " << client << std::endl;
+        return false;
+      }
+   }
+   else if (FileFd::Write(client, data.c_str(), data.size()) == false)

    {
       std::cerr << "SENDDATA: WRITE ERROR to " << client << std::endl;
       return false;
    {
       std::cerr << "SENDDATA: WRITE ERROR to " << client << std::endl;
       return false;


@@ -157,34 +204,38 @@ static bool sendData(int const client, std::string const &data)           /*{{{*/
 }
                                                                        /*}}}*/
 static void sendError(int const client, int const httpcode, std::string const &request,/*{{{*/
 }
                                                                        /*}}}*/
 static void sendError(int const client, int const httpcode, std::string const &request,/*{{{*/

-              bool content, std::string const &error = "")
+              bool const content, std::string const &error, std::list<std::string> &headers)

 {
 {

-   std::list<std::string> headers;

    std::string response("<html><head><title>");
    response.append(httpcodeToStr(httpcode)).append("</title></head>");
    response.append("<body><h1>").append(httpcodeToStr(httpcode)).append("</h1>");
    if (httpcode != 200)
    std::string response("<html><head><title>");
    response.append(httpcodeToStr(httpcode)).append("</title></head>");
    response.append("<body><h1>").append(httpcodeToStr(httpcode)).append("</h1>");
    if (httpcode != 200)

-   {
-      if (error.empty() == false)
-        response.append("<p><em>Error</em>: ").append(error).append("</p>");
-      response.append("This error is a result of the request: <pre>");
-   }
+      response.append("<p><em>Error</em>: ");
+   else
+      response.append("<p><em>Success</em>: ");
+   if (error.empty() == false)
+      response.append(error);
+   else
+      response.append(httpcodeToStr(httpcode));
+   if (httpcode != 200)
+      response.append("</p>This error is a result of the request: <pre>");

    else
    else

-   {
-      if (error.empty() == false)
-        response.append("<p><em>Success</em>: ").append(error).append("</p>");

       response.append("The successfully executed operation was requested by: <pre>");
       response.append("The successfully executed operation was requested by: <pre>");

-   }

    response.append(request).append("</pre></body></html>");
    response.append(request).append("</pre></body></html>");

+   if (httpcode != 200)
+   {
+      if (_config->FindB("aptwebserver::closeOnError", false) == true)
+        headers.push_back("Connection: close");
+   }

    addDataHeaders(headers, response);
    sendHead(client, httpcode, headers);
    if (content == true)
    addDataHeaders(headers, response);
    sendHead(client, httpcode, headers);
    if (content == true)

-      sendData(client, response);
+      sendData(client, headers, response);

 }
 static void sendSuccess(int const client, std::string const &request,
 }
 static void sendSuccess(int const client, std::string const &request,

-              bool content, std::string const &error = "")
+              bool const content, std::string const &error, std::list<std::string> &headers)

 {
 {

-   sendError(client, 200, request, content, error);
+   sendError(client, 200, request, content, error, headers);

 }
                                                                        /*}}}*/
 static void sendRedirect(int const client, int const httpcode, std::string const &uri,/*{{{*/
 }
                                                                        /*}}}*/
 static void sendRedirect(int const client, int const httpcode, std::string const &uri,/*{{{*/


@@ -221,7 +272,7 @@ static void sendRedirect(int const client, int const httpcode, std::string const
    headers.push_back(location);
    sendHead(client, httpcode, headers);
    if (content == true)
    headers.push_back(location);
    sendHead(client, httpcode, headers);
    if (content == true)

-      sendData(client, response);
+      sendData(client, headers, response);

 }
                                                                        /*}}}*/
 static int filter_hidden_files(const struct dirent *a)                 /*{{{*/
 }
                                                                        /*}}}*/
 static int filter_hidden_files(const struct dirent *a)                 /*{{{*/


@@ -263,16 +314,15 @@ static int grouped_alpha_case_sort(const struct dirent **a, const struct dirent
 }
                                                                        /*}}}*/
 static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*/
 }
                                                                        /*}}}*/
 static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*/

-                         std::string const &request, bool content)
+                         std::string const &request, bool content, std::list<std::string> &headers)

 {
 {

-   std::list<std::string> headers;

    std::ostringstream listing;
 
    struct dirent **namelist;
    int const counter = scandir(dir.c_str(), &namelist, filter_hidden_files, grouped_alpha_case_sort);
    if (counter == -1)
    {
    std::ostringstream listing;
 
    struct dirent **namelist;
    int const counter = scandir(dir.c_str(), &namelist, filter_hidden_files, grouped_alpha_case_sort);
    if (counter == -1)
    {

-      sendError(client, 500, request, content);
+      sendError(client, 500, request, content, "scandir failed", headers);

       return;
    }
 
       return;
    }
 


@@ -311,18 +361,18 @@ static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*
    addDataHeaders(headers, response);
    sendHead(client, 200, headers);
    if (content == true)
    addDataHeaders(headers, response);
    sendHead(client, 200, headers);
    if (content == true)

-      sendData(client, response);
+      sendData(client, headers, response);

 }
                                                                        /*}}}*/
 static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
                    std::string &filename, std::string &params, bool &sendContent,
 }
                                                                        /*}}}*/
 static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
                    std::string &filename, std::string &params, bool &sendContent,

-                   bool &closeConnection)
+                   bool &closeConnection, std::list<std::string> &headers)

 {
    if (strncmp(request.c_str(), "HEAD ", 5) == 0)
       sendContent = false;
    if (strncmp(request.c_str(), "GET ", 4) != 0)
    {
 {
    if (strncmp(request.c_str(), "HEAD ", 5) == 0)
       sendContent = false;
    if (strncmp(request.c_str(), "GET ", 4) != 0)
    {

-      sendError(client, 501, request, true);
+      sendError(client, 501, request, true, "", headers);

       return false;
    }
 
       return false;
    }
 


@@ -333,7 +383,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (lineend == std::string::npos || filestart == std::string::npos ||
         fileend == std::string::npos || filestart == fileend)
    {
    if (lineend == std::string::npos || filestart == std::string::npos ||
         fileend == std::string::npos || filestart == fileend)
    {

-      sendError(client, 500, request, sendContent, "Filename can't be extracted");
+      sendError(client, 500, request, sendContent, "Filename can't be extracted", headers);

       return false;
    }
 
       return false;
    }
 


@@ -345,14 +395,14 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       closeConnection = strcasecmp(LookupTag(request, "Connection", "Keep-Alive").c_str(), "close") == 0;
    else
    {
       closeConnection = strcasecmp(LookupTag(request, "Connection", "Keep-Alive").c_str(), "close") == 0;
    else
    {

-      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request");
+      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request", headers);

       return false;
    }
 
    filename = request.substr(filestart, fileend - filestart);
    if (filename.find(' ') != std::string::npos)
    {
       return false;
    }
 
    filename = request.substr(filestart, fileend - filestart);
    if (filename.find(' ') != std::string::npos)
    {

-      sendError(client, 500, request, sendContent, "Filename contains an unencoded space");
+      sendError(client, 500, request, sendContent, "Filename contains an unencoded space", headers);

       return false;
    }
 
       return false;
    }
 


@@ -360,7 +410,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (host.empty() == true)
    {
       // RFC 2616 §14.23 requires Host
    if (host.empty() == true)
    {
       // RFC 2616 §14.23 requires Host

-      sendError(client, 400, request, sendContent, "Host header is required");
+      sendError(client, 400, request, sendContent, "Host header is required", headers);

       return false;
    }
    host = "http://" + host;
       return false;
    }
    host = "http://" + host;


@@ -371,7 +421,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    {
       if (absolute.find("uri") == std::string::npos)
       {
    {
       if (absolute.find("uri") == std::string::npos)
       {

-        sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that");
+        sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that", headers);

         return false;
       }
       // strip the host from the request to make it an absolute path
         return false;
       }
       // strip the host from the request to make it an absolute path


@@ -379,7 +429,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    }
    else if (absolute.find("path") == std::string::npos)
    {
    }
    else if (absolute.find("path") == std::string::npos)
    {

-      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that");
+      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that", headers);

       return false;
    }
 
       return false;
    }
 


@@ -398,7 +448,8 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
        filename.find_first_of("\r\n\t\f\v") != std::string::npos ||
        filename.find("/../") != std::string::npos)
    {
        filename.find_first_of("\r\n\t\f\v") != std::string::npos ||
        filename.find("/../") != std::string::npos)
    {

-      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)");
+      std::list<std::string> headers;
+      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)", headers);

       return false;
    }
 
       return false;
    }
 


@@ -434,46 +485,45 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    return true;
 }
                                                                        /*}}}*/
    return true;
 }
                                                                        /*}}}*/

-static bool handleOnTheFlyReconfiguration(int const client, std::string const &request, std::vector<std::string> const &parts)/*{{{*/
+static bool handleOnTheFlyReconfiguration(int const client, std::string const &request,/*{{{*/
+      std::vector<std::string> parts, std::list<std::string> &headers)

 {
    size_t const pcount = parts.size();
    if (pcount == 4 && parts[1] == "set")
    {
       _config->Set(parts[2], parts[3]);
 {
    size_t const pcount = parts.size();
    if (pcount == 4 && parts[1] == "set")
    {
       _config->Set(parts[2], parts[3]);

-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!", headers);

       return true;
    }
    else if (pcount == 4 && parts[1] == "find")
    {
       return true;
    }
    else if (pcount == 4 && parts[1] == "find")
    {

-      std::list<std::string> headers;

       std::string response = _config->Find(parts[2], parts[3]);
       addDataHeaders(headers, response);
       sendHead(client, 200, headers);
       std::string response = _config->Find(parts[2], parts[3]);
       addDataHeaders(headers, response);
       sendHead(client, 200, headers);

-      sendData(client, response);
+      sendData(client, headers, response);

       return true;
    }
    else if (pcount == 3 && parts[1] == "find")
    {
       return true;
    }
    else if (pcount == 3 && parts[1] == "find")
    {

-      std::list<std::string> headers;

       if (_config->Exists(parts[2]) == true)
       {
         std::string response = _config->Find(parts[2]);
         addDataHeaders(headers, response);
         sendHead(client, 200, headers);
       if (_config->Exists(parts[2]) == true)
       {
         std::string response = _config->Find(parts[2]);
         addDataHeaders(headers, response);
         sendHead(client, 200, headers);

-        sendData(client, response);
+        sendData(client, headers, response);

         return true;
       }
         return true;
       }

-      sendError(client, 404, request, "Requested Configuration option doesn't exist.");
+      sendError(client, 404, request, true, "Requested Configuration option doesn't exist", headers);

       return false;
    }
    else if (pcount == 3 && parts[1] == "clear")
    {
       _config->Clear(parts[2]);
       return false;
    }
    else if (pcount == 3 && parts[1] == "clear")
    {
       _config->Clear(parts[2]);

-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.", headers);

       return true;
    }
 
       return true;
    }
 

-   sendError(client, 400, request, true, "Unknown on-the-fly configuration request");
+   sendError(client, 400, request, true, "Unknown on-the-fly configuration request", headers);

    return false;
 }
                                                                        /*}}}*/
    return false;
 }
                                                                        /*}}}*/


@@ -482,18 +532,22 @@ static void * handleClient(void * voidclient)                             /*{{{*/
    int client = *((int*)(voidclient));
    std::clog << "ACCEPT client " << client << std::endl;
    std::vector<std::string> messages;
    int client = *((int*)(voidclient));
    std::clog << "ACCEPT client " << client << std::endl;
    std::vector<std::string> messages;

-   while (ReadMessages(client, messages))
+   bool closeConnection = false;
+   std::list<std::string> headers;
+   while (closeConnection == false && ReadMessages(client, messages))

    {
    {

-      bool closeConnection = false;
+      // if we announced a closing, do the close
+      if (std::find(headers.begin(), headers.end(), std::string("Connection: close")) != headers.end())
+        break;
+      headers.clear();

       for (std::vector<std::string>::const_iterator m = messages.begin();
            m != messages.end() && closeConnection == false; ++m) {
         std::clog << ">>> REQUEST from " << client << " >>>" << std::endl << *m
            << std::endl << "<<<<<<<<<<<<<<<<" << std::endl;
       for (std::vector<std::string>::const_iterator m = messages.begin();
            m != messages.end() && closeConnection == false; ++m) {
         std::clog << ">>> REQUEST from " << client << " >>>" << std::endl << *m
            << std::endl << "<<<<<<<<<<<<<<<<" << std::endl;

-        std::list<std::string> headers;

         std::string filename;
         std::string params;
         bool sendContent = true;
         std::string filename;
         std::string params;
         bool sendContent = true;

-        if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection) == false)
+        if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection, headers) == false)

            continue;
 
         // special webserver command request
            continue;
 
         // special webserver command request


@@ -502,7 +556,7 @@ static void * handleClient(void * voidclient)                               /*{{{*/
            std::vector<std::string> parts = VectorizeString(filename, '/');
            if (parts[0] == "_config")
            {
            std::vector<std::string> parts = VectorizeString(filename, '/');
            if (parts[0] == "_config")
            {

-              handleOnTheFlyReconfiguration(client, *m, parts);
+              handleOnTheFlyReconfiguration(client, *m, parts, headers);

               continue;
            }
         }
               continue;
            }
         }


@@ -534,7 +588,7 @@ static void * handleClient(void * voidclient)                               /*{{{*/
               {
                  char error[300];
                  regerror(res, pattern, error, sizeof(error));
               {
                  char error[300];
                  regerror(res, pattern, error, sizeof(error));

-                 sendError(client, 500, *m, sendContent, error);
+                 sendError(client, 500, *m, sendContent, error, headers);

                  continue;
               }
               if (regexec(pattern, filename.c_str(), 0, 0, 0) == 0)
                  continue;
               }
               if (regexec(pattern, filename.c_str(), 0, 0, 0) == 0)


@@ -553,7 +607,7 @@ static void * handleClient(void * voidclient)                               /*{{{*/
         if (_config->FindB("aptwebserver::support::http", true) == false &&
               LookupTag(*m, "Host").find(":4433") == std::string::npos)
         {
         if (_config->FindB("aptwebserver::support::http", true) == false &&
               LookupTag(*m, "Host").find(":4433") == std::string::npos)
         {

-           sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS");
+           sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS", headers);

            continue;
         }
         else if (RealFileExists(filename) == true)
            continue;
         }
         else if (RealFileExists(filename) == true)


@@ -609,17 +663,16 @@ static void * handleClient(void * voidclient)                             /*{{{*/
                        headers.push_back(contentrange.str());
                        sendHead(client, 206, headers);
                        if (sendContent == true)
                        headers.push_back(contentrange.str());
                        sendHead(client, 206, headers);
                        if (sendContent == true)

-                          sendFile(client, data);
+                          sendFile(client, headers, data);

                        continue;
                     }
                     else
                     {
                        continue;
                     }
                     else
                     {

-                       headers.push_back("Content-Length: 0");

                        std::ostringstream contentrange;
                        contentrange << "Content-Range: bytes */" << filesize;
                        headers.push_back(contentrange.str());
                        std::ostringstream contentrange;
                        contentrange << "Content-Range: bytes */" << filesize;
                        headers.push_back(contentrange.str());

-                       sendHead(client, 416, headers);
-                       continue;
+                       sendError(client, 416, *m, sendContent, "", headers);
+                       break;

                     }
                  }
               }
                     }
                  }
               }


@@ -628,22 +681,20 @@ static void * handleClient(void * voidclient)                             /*{{{*/
            addFileHeaders(headers, data);
            sendHead(client, 200, headers);
            if (sendContent == true)
            addFileHeaders(headers, data);
            sendHead(client, 200, headers);
            if (sendContent == true)

-              sendFile(client, data);
+              sendFile(client, headers, data);

         }
         else if (DirectoryExists(filename) == true)
         {
            if (filename[filename.length()-1] == '/')
         }
         else if (DirectoryExists(filename) == true)
         {
            if (filename[filename.length()-1] == '/')

-              sendDirectoryListing(client, filename, *m, sendContent);
+              sendDirectoryListing(client, filename, *m, sendContent, headers);

            else
               sendRedirect(client, 301, filename.append("/"), *m, sendContent);
         }
         else
            else
               sendRedirect(client, 301, filename.append("/"), *m, sendContent);
         }
         else

-           sendError(client, 404, *m, sendContent);
+           sendError(client, 404, *m, sendContent, "", headers);

       }
       _error->DumpErrors(std::cerr);
       messages.clear();
       }
       _error->DumpErrors(std::cerr);
       messages.clear();

-      if (closeConnection == true)
-        break;

    }
    close(client);
    std::clog << "CLOSE client " << client << std::endl;
    }
    close(client);
    std::clog << "CLOSE client " << client << std::endl;