From e1ae0531bfad0fce8590c26d1e38825df22d812a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 24 Nov 2016 12:14:39 +0100 Subject: [PATCH] optional write aptwebserver log to client specific files The test test-handle-redirect-as-used-mirror-change serves multiple clients at the same time, so the order of the output is undefined and once in a while the two clients will intermix their lines causing the grep we perform on it later to fail making our tests fail. Solved by introducing client-specific logfiles which we all grep and sort the result to have the results more stable. Git-Dch: Ignore --- test/integration/framework | 2 +- test/integration/test-apt-download-progress | 10 +- .../test-apt-update-filesize-mismatch | 2 +- .../test-apt-update-hashsum-mismatch | 2 +- .../test-bug-602412-dequote-redirect | 5 +- ...test-handle-redirect-as-used-mirror-change | 19 ++- test/interactive-helper/aptwebserver.cc | 155 +++++++++--------- test/interactive-helper/teestream.h | 62 +++++++ test/libapt/teestream_test.cc | 39 +++++ 9 files changed, 200 insertions(+), 96 deletions(-) create mode 100644 test/interactive-helper/teestream.h create mode 100644 test/libapt/teestream_test.cc diff --git a/test/integration/framework b/test/integration/framework index 9a114ae69..05c8fcd2a 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1305,7 +1305,7 @@ changetowebserver() { if test -x "${APTTESTHELPERSBINDIR}/aptwebserver"; then cd aptarchive local LOG="webserver.log" - if ! aptwebserver --port 0 -o aptwebserver::fork=1 -o aptwebserver::portfile='aptwebserver.port' "$@" >$LOG 2>&1 ; then + if ! aptwebserver --port 0 -o aptwebserver::fork=1 -o aptwebserver::portfile='aptwebserver.port' -o aptwebserver::logfiles="$(readlink -f .)/$LOG" "$@" >$LOG 2>&1 ; then cat "$LOG" false fi diff --git a/test/integration/test-apt-download-progress b/test/integration/test-apt-download-progress index 7ac044a57..13a18f7c3 100755 --- a/test/integration/test-apt-download-progress +++ b/test/integration/test-apt-download-progress @@ -20,14 +20,14 @@ assertprogress() { # actually report progress - but not too big to ensure its not delaying the # test too much TESTFILE=testfile.big -testsuccess dd if=/dev/zero of=./aptarchive/$TESTFILE bs=16000k count=1 +testsuccess dd if=/dev/zero of=./aptarchive/$TESTFILE bs=1600k count=1 OPT='-o APT::Status-Fd=3 -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1 -o Debug::Acquire::https=1' msgtest 'download progress works via' 'http' -for i in 1 2 3 4 5 6 7 8 9 10; do +for i in 2 5 7 10 12 15; do exec 3> apt-progress-http.log - testsuccess --nomsg apthelper download-file "http://localhost:${APTHTTPPORT}/$TESTFILE" ./downloaded/http-$TESTFILE $OPT -o Acquire::http::Dl-Limit=$((16000/i)) + testsuccess --nomsg apthelper download-file "http://localhost:${APTHTTPPORT}/$TESTFILE" ./downloaded/http-$TESTFILE $OPT -o Acquire::http::Dl-Limit=$((1600/i)) if [ "$(wc -l apt-progress-http.log | awk '{print $1}')" -ge 3 ]; then break fi @@ -35,9 +35,9 @@ done assertprogress apt-progress-http.log msgtest 'download progress works via' 'https' -for i in 1 2 3 4 5 6 7 8 9 10; do +for i in 2 5 7 10 12 15; do exec 3> apt-progress-https.log - testsuccess --nomsg apthelper download-file "https://localhost:${APTHTTPSPORT}/$TESTFILE" ./downloaded/https-$TESTFILE $OPT -o Acquire::https::Dl-Limit=$((16000/i)) + testsuccess --nomsg apthelper download-file "https://localhost:${APTHTTPSPORT}/$TESTFILE" ./downloaded/https-$TESTFILE $OPT -o Acquire::https::Dl-Limit=$((1600/i)) if [ "$(wc -l apt-progress-https.log | awk '{print $1}')" -ge 3 ]; then break fi diff --git a/test/integration/test-apt-update-filesize-mismatch b/test/integration/test-apt-update-filesize-mismatch index 9f95906b5..9467e77b6 100755 --- a/test/integration/test-apt-update-filesize-mismatch +++ b/test/integration/test-apt-update-filesize-mismatch @@ -24,7 +24,7 @@ testsuccess aptget update testsuccess aptcache show foo testsuccess aptget install foo -s -for get in $(sed -n 's#^GET /\([^ ]\+\.gz\) HTTP.\+$#\1#p' aptarchive/webserver.log); do +for get in $(sed -n 's#^GET /\([^ ]\+\.gz\) HTTP.\+$#\1#p' aptarchive/webserver.log.client*.log); do for ext in '' '.gz'; do COMPRESSFILE="$get" get="${get}${ext}" diff --git a/test/integration/test-apt-update-hashsum-mismatch b/test/integration/test-apt-update-hashsum-mismatch index f3f6a4236..4d4c33286 100755 --- a/test/integration/test-apt-update-hashsum-mismatch +++ b/test/integration/test-apt-update-hashsum-mismatch @@ -27,7 +27,7 @@ testsuccess aptget update testsuccess aptcache show foo testsuccess aptget install foo -s -for get in $(sed -n 's#^GET /\([^ ]\+\.gz\) HTTP.\+$#\1#p' aptarchive/webserver.log); do +for get in $(sed -n 's#^GET /\([^ ]\+\.gz\) HTTP.\+$#\1#p' aptarchive/webserver.log.client*.log); do msgmsg 'Test hashsum mismatch with file' "$get" rm -rf rootdir/var/lib/apt/lists webserverconfig 'aptwebserver::overwrite' '' diff --git a/test/integration/test-bug-602412-dequote-redirect b/test/integration/test-bug-602412-dequote-redirect index b4da876cb..9c6aa3945 100755 --- a/test/integration/test-bug-602412-dequote-redirect +++ b/test/integration/test-bug-602412-dequote-redirect @@ -32,8 +32,9 @@ Reading package lists..." aptget update for CODE in 301 302 307; do webserverconfig 'aptwebserver::redirect::httpcode' "$CODE" + rm -f aptarchive/webserver.log.client*.log testrun "$CODE" "http://localhost:${APTHTTPPORT}" - testsuccess grep "^HTTP/1.1 $CODE " aptarchive/webserver.log + testsuccess grep "^HTTP/1.1 $CODE " aptarchive/webserver.log.client*.log rm -rf rootdir/var/lib/apt/lists rootdir/var/cache/apt/archives done @@ -41,7 +42,9 @@ changetohttpswebserver for CODE in 301 302 307; do webserverconfig 'aptwebserver::redirect::httpcode' "$CODE" + rm -f aptarchive/webserver.log.client*.log testrun "$CODE" "https://localhost:${APTHTTPSPORT}" + testsuccess grep "^HTTP/1.1 $CODE " aptarchive/webserver.log.client*.log rm -rf rootdir/var/lib/apt/lists rootdir/var/cache/apt/archives done diff --git a/test/integration/test-handle-redirect-as-used-mirror-change b/test/integration/test-handle-redirect-as-used-mirror-change index 2655f713c..254bdd54a 100755 --- a/test/integration/test-handle-redirect-as-used-mirror-change +++ b/test/integration/test-handle-redirect-as-used-mirror-change @@ -21,23 +21,30 @@ Get:3 http://0.0.0.0:${APTHTTPPORT} unstable/main all Packages [$(stat -c %s apt Get:4 http://0.0.0.0:${APTHTTPPORT} unstable/main Translation-en [$(stat -c %s aptarchive/dists/unstable/main/i18n/Translation-en.gz) B] Reading package lists..." aptget update +grepwebserverlogs() { + testsuccess grep -h "$1" aptarchive/webserver.log.client*.log + shift + sort rootdir/tmp/testsuccess.output > aptwebserver.log + testfileequal 'aptwebserver.log' "$@" +} + # ensure we asked the redirector only once -testsuccessequal "Location: http://0.0.0.0:${APTHTTPPORT}/dists/unstable/InRelease" grep '^Location:' aptarchive/webserver.log +grepwebserverlogs '^Location:' "Location: http://0.0.0.0:${APTHTTPPORT}/dists/unstable/InRelease" testsuccessequal "Hit:1 http://0.0.0.0:${APTHTTPPORT} unstable InRelease Reading package lists..." aptget update -testsuccessequal "Location: http://0.0.0.0:${APTHTTPPORT}/dists/unstable/InRelease -Location: http://0.0.0.0:${APTHTTPPORT}/dists/unstable/InRelease" grep '^Location:' aptarchive/webserver.log +grepwebserverlogs '^Location:' "Location: http://0.0.0.0:${APTHTTPPORT}/dists/unstable/InRelease +Location: http://0.0.0.0:${APTHTTPPORT}/dists/unstable/InRelease" rm -rf rootdir/var/lib/apt/lists testsuccess apt update -o Debug::Acquire::http=1 -o Acquire::SameMirrorForAllIndexes=0 -testsuccessequal "Location: http://0.0.0.0:${APTHTTPPORT}/dists/unstable/InRelease +grepwebserverlogs '^Location:' "Location: http://0.0.0.0:${APTHTTPPORT}/dists/unstable/InRelease Location: http://0.0.0.0:${APTHTTPPORT}/dists/unstable/InRelease Location: http://0.0.0.0:${APTHTTPPORT}/dists/unstable/InRelease -Location: http://0.0.0.0:${APTHTTPPORT}/dists/unstable/main/source/Sources.gz Location: http://0.0.0.0:${APTHTTPPORT}/dists/unstable/main/binary-all/Packages.gz -Location: http://0.0.0.0:${APTHTTPPORT}/dists/unstable/main/i18n/Translation-en.gz" grep '^Location:' aptarchive/webserver.log +Location: http://0.0.0.0:${APTHTTPPORT}/dists/unstable/main/i18n/Translation-en.gz +Location: http://0.0.0.0:${APTHTTPPORT}/dists/unstable/main/source/Sources.gz" cd downloaded testsuccess apthelper download-file "http://localhost:${APTHTTPPORT}/redirectme/dists/unstable/InRelease" inrelease diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 950a17bc1..4b98cbd7c 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -6,6 +6,8 @@ #include #include +#include "teestream.h" + #include #include #include @@ -22,9 +24,11 @@ #include #include +#include #include #include #include +#include #include static std::string httpcodeToStr(int const httpcode) /*{{{*/ @@ -114,7 +118,7 @@ static void addDataHeaders(std::list &headers, std::string &data)/* } } /*}}}*/ -static bool sendHead(int const client, int const httpcode, std::list &headers)/*{{{*/ +static bool sendHead(std::ostream &log, int const client, int const httpcode, std::list &headers)/*{{{*/ { std::string response("HTTP/1.1 "); response.append(httpcodeToStr(httpcode)); @@ -137,7 +141,7 @@ static bool sendHead(int const client, int const httpcode, std::list>> RESPONSE to " << client << " >>>" << std::endl; + log << ">>> RESPONSE to " << client << " >>>" << std::endl; bool Success = true; for (std::list::const_iterator h = headers.begin(); Success == true && h != headers.end(); ++h) @@ -145,11 +149,11 @@ static bool sendHead(int const client, int const httpcode, std::listc_str(), h->size()); if (Success == true) Success &= FileFd::Write(client, "\r\n", 2); - std::clog << *h << std::endl; + log << *h << std::endl; } if (Success == true) Success &= FileFd::Write(client, "\r\n", 2); - std::clog << "<<<<<<<<<<<<<<<<" << std::endl; + log << "<<<<<<<<<<<<<<<<" << std::endl; return Success; } /*}}}*/ @@ -209,7 +213,7 @@ static bool sendData(int const client, std::list const &headers, st return true; } /*}}}*/ -static void sendError(int const client, int const httpcode, std::string const &request,/*{{{*/ +static void sendError(std::ostream &log, int const client, int const httpcode, std::string const &request,/*{{{*/ bool const content, std::string const &error, std::list &headers) { std::string response(""); @@ -234,18 +238,18 @@ static void sendError(int const client, int const httpcode, std::string const &r headers.push_back("Connection: close"); } addDataHeaders(headers, response); - sendHead(client, httpcode, headers); + sendHead(log, client, httpcode, headers); if (content == true) sendData(client, headers, response); } -static void sendSuccess(int const client, std::string const &request, +static void sendSuccess(std::ostream &log, int const client, std::string const &request, bool const content, std::string const &error, std::list<std::string> &headers) { - sendError(client, 200, request, content, error, headers); + sendError(log, client, 200, request, content, error, headers); } /*}}}*/ -static void sendRedirect(int const client, int const httpcode, std::string const &uri,/*{{{*/ - std::string const &request, bool content) +static void sendRedirect(std::ostream &log, int const client, int const httpcode,/*{{{*/ + std::string const &uri, std::string const &request, bool content) { std::list<std::string> headers; std::string response("<!doctype html><html><head><title>"); @@ -279,7 +283,7 @@ static void sendRedirect(int const client, int const httpcode, std::string const else location.append(uri); headers.push_back(location); - sendHead(client, httpcode, headers); + sendHead(log, client, httpcode, headers); if (content == true) sendData(client, headers, response); } @@ -322,19 +326,18 @@ static int grouped_alpha_case_sort(const struct dirent **a, const struct dirent return strcasecmp((*a)->d_name, (*b)->d_name); } /*}}}*/ -static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*/ +static void sendDirectoryListing(std::ostream &log, int const client, std::string const &dir,/*{{{*/ std::string const &request, bool content, std::list<std::string> &headers) { - std::ostringstream listing; - struct dirent **namelist; int const counter = scandir(dir.c_str(), &namelist, filter_hidden_files, grouped_alpha_case_sort); if (counter == -1) { - sendError(client, 500, request, content, "scandir failed", headers); + sendError(log, client, 500, request, content, "scandir failed", headers); return; } + std::ostringstream listing; listing << "<!doctype html><html><head><title>Index of " << dir << "" << "