X-Git-Url: https://git.saurik.com/redis.git/blobdiff_plain/b04ce2a35ce084a043ef8749ca4fa0e62b92bd03..bfc197c3b604baf0dba739ea174d5054284133f0:/tests/test_helper.tcl?ds=sidebyside diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index ee7fa3e1..853193cc 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -9,16 +9,70 @@ source tests/support/tmpfile.tcl source tests/support/test.tcl source tests/support/util.tcl +set ::all_tests { + unit/printver + unit/auth + unit/protocol + unit/basic + unit/type/list + unit/type/list-2 + unit/type/list-3 + unit/type/set + unit/type/zset + unit/type/hash + unit/sort + unit/expire + unit/other + unit/cas + unit/quit + unit/aofrw + integration/replication + integration/replication-2 + integration/replication-3 + integration/replication-4 + integration/aof + integration/rdb + integration/convert-zipmap-hash-on-load + unit/pubsub + unit/slowlog + unit/scripting + unit/maxmemory + unit/introspection + unit/limits + unit/obuf-limits + unit/dump + unit/bitops +} +# Index to the next test to run in the ::all_tests list. +set ::next_test 0 + set ::host 127.0.0.1 -set ::port 16379 +set ::port 21111 set ::traceleaks 0 set ::valgrind 0 +set ::verbose 0 +set ::quiet 0 set ::denytags {} set ::allowtags {} set ::external 0; # If "1" this means, we are running against external instance +set ::file ""; # If set, runs only the tests in this comma separated list +set ::curfile ""; # Hold the filename of the current suite +set ::accurate 0; # If true runs fuzz tests with more iterations +set ::force_failure 0 + +# Set to 1 when we are running in client mode. The Redis test uses a +# server-client model to run tests simultaneously. The server instance +# runs the specified number of client instances that will actually run tests. +# The server is responsible of showing the result to the user, and exit with +# the appropriate exit code depending on the test outcome. +set ::client 0 +set ::numclients 16 proc execute_tests name { - source "tests/$name.tcl" + set path "tests/$name.tcl" + set ::curfile $path + source $path + send_data_packet $::test_server_fd done "$name" } # Setup a list to hold a stack of server configs. When calls to start_server @@ -49,6 +103,28 @@ proc r {args} { [srv $level "client"] {*}$args } +proc reconnect {args} { + set level [lindex $args 0] + if {[string length $level] == 0 || ![string is integer $level]} { + set level 0 + } + + set srv [lindex $::servers end+$level] + set host [dict get $srv "host"] + set port [dict get $srv "port"] + set config [dict get $srv "config"] + set client [redis $host $port] + dict set srv "client" $client + + # select the right db when we don't have to authenticate + if {![dict exists $config "requirepass"]} { + $client select 9 + } + + # re-set $srv in the servers list + lset ::servers end+$level $srv +} + proc redis_deferring_client {args} { set level 0 if {[llength $args] > 0 && [string is integer [lindex $args 0]]} { @@ -76,49 +152,207 @@ proc s {args} { } proc cleanup {} { + if {!$::quiet} {puts -nonewline "Cleanup: may take some time... "} + flush stdout catch {exec rm -rf {*}[glob tests/tmp/redis.conf.*]} catch {exec rm -rf {*}[glob tests/tmp/server.*]} + if {!$::quiet} {puts "OK"} } -proc main {} { - cleanup - execute_tests "unit/auth" - execute_tests "unit/protocol" - execute_tests "unit/basic" - execute_tests "unit/type/list" - execute_tests "unit/type/set" - execute_tests "unit/type/zset" - execute_tests "unit/type/hash" - execute_tests "unit/sort" - execute_tests "unit/expire" - execute_tests "unit/other" - execute_tests "unit/cas" - execute_tests "integration/replication" - execute_tests "integration/aof" -# execute_tests "integration/redis-cli" - execute_tests "unit/pubsub" - - # run tests with VM enabled - set ::global_overrides {vm-enabled yes} - execute_tests "unit/protocol" - execute_tests "unit/basic" - execute_tests "unit/type/list" - execute_tests "unit/type/set" - execute_tests "unit/type/zset" - execute_tests "unit/type/hash" - execute_tests "unit/sort" - execute_tests "unit/expire" - execute_tests "unit/other" - execute_tests "unit/cas" +proc find_available_port start { + for {set j $start} {$j < $start+1024} {incr j} { + if {[catch { + set fd [socket 127.0.0.1 $j] + }]} { + return $j + } else { + close $fd + } + } + if {$j == $start+1024} { + error "Can't find a non busy port in the $start-[expr {$start+1023}] range." + } +} +proc test_server_main {} { cleanup - puts "\n[expr $::passed+$::failed] tests, $::passed passed, $::failed failed" - if {$::failed > 0} { - puts "\n*** WARNING!!! $::failed FAILED TESTS ***\n" + # Open a listening socket, trying different ports in order to find a + # non busy one. + set port [find_available_port 11111] + if {!$::quiet} { + puts "Starting test server at port $port" + } + socket -server accept_test_clients $port + + # Start the client instances + set ::clients_pids {} + set start_port [expr {$::port+100}] + for {set j 0} {$j < $::numclients} {incr j} { + set start_port [find_available_port $start_port] + set p [exec tclsh8.5 [info script] {*}$::argv \ + --client $port --port $start_port &] + lappend ::clients_pids $p + incr start_port 10 + } + + # Setup global state for the test server + set ::idle_clients {} + set ::active_clients {} + array set ::clients_start_time {} + set ::clients_time_history {} + set ::failed_tests {} + + # Enter the event loop to handle clients I/O + after 100 test_server_cron + vwait forever +} + +# This function gets called 10 times per second, for now does nothing but +# may be used in the future in order to detect test clients taking too much +# time to execute the task. +proc test_server_cron {} { +} + +proc accept_test_clients {fd addr port} { + fileevent $fd readable [list read_from_test_client $fd] +} + +# This is the readable handler of our test server. Clients send us messages +# in the form of a status code such and additional data. Supported +# status types are: +# +# ready: the client is ready to execute the command. Only sent at client +# startup. The server will queue the client FD in the list of idle +# clients. +# testing: just used to signal that a given test started. +# ok: a test was executed with success. +# err: a test was executed with an error. +# exception: there was a runtime exception while executing the test. +# done: all the specified test file was processed, this test client is +# ready to accept a new task. +proc read_from_test_client fd { + set bytes [gets $fd] + set payload [read $fd $bytes] + foreach {status data} $payload break + if {$status eq {ready}} { + if {!$::quiet} { + puts "\[$status\]: $data" + } + signal_idle_client $fd + } elseif {$status eq {done}} { + set elapsed [expr {[clock seconds]-$::clients_start_time($fd)}] + set all_tests_count [llength $::all_tests] + set running_tests_count [expr {[llength $::active_clients]-1}] + set completed_tests_count [expr {$::next_test-$running_tests_count}] + puts "\[$completed_tests_count/$all_tests_count [colorstr yellow $status]\]: $data ($elapsed seconds)" + lappend ::clients_time_history $elapsed $data + signal_idle_client $fd + } elseif {$status eq {ok}} { + if {!$::quiet} { + puts "\[[colorstr green $status]\]: $data" + } + } elseif {$status eq {err}} { + set err "\[[colorstr red $status]\]: $data" + puts $err + lappend ::failed_tests $err + } elseif {$status eq {exception}} { + puts "\[[colorstr red $status]\]: $data" + foreach p $::clients_pids { + catch {exec kill -9 $p} + } + exit 1 + } elseif {$status eq {testing}} { + # No op + } else { + if {!$::quiet} { + puts "\[$status\]: $data" + } + } +} + +# A new client is idle. Remove it from the list of active clients and +# if there are still test units to run, launch them. +proc signal_idle_client fd { + # Remove this fd from the list of active clients. + set ::active_clients \ + [lsearch -all -inline -not -exact $::active_clients $fd] + # New unit to process? + if {$::next_test != [llength $::all_tests]} { + if {!$::quiet} { + puts [colorstr bold-white "Testing [lindex $::all_tests $::next_test]"] + } + set ::clients_start_time($fd) [clock seconds] + send_data_packet $fd run [lindex $::all_tests $::next_test] + lappend ::active_clients $fd + incr ::next_test + } else { + lappend ::idle_clients $fd + if {[llength $::active_clients] == 0} { + the_end + } + } +} + +# The the_end funciton gets called when all the test units were already +# executed, so the test finished. +proc the_end {} { + # TODO: print the status, exit with the rigth exit code. + puts "\n The End\n" + puts "Execution time of different units:" + foreach {time name} $::clients_time_history { + puts " $time seconds - $name" + } + if {[llength $::failed_tests]} { + puts "\n[colorstr bold-red {!!! WARNING}] The following tests failed:\n" + foreach failed $::failed_tests { + puts "*** $failed" + } + cleanup exit 1 + } else { + puts "\n[colorstr bold-white {\o/}] [colorstr bold-green {All tests passed without errors!}]\n" + cleanup + exit 0 + } +} + +# The client is not even driven (the test server is instead) as we just need +# to read the command, execute, reply... all this in a loop. +proc test_client_main server_port { + set ::test_server_fd [socket localhost $server_port] + send_data_packet $::test_server_fd ready [pid] + while 1 { + set bytes [gets $::test_server_fd] + set payload [read $::test_server_fd $bytes] + foreach {cmd data} $payload break + if {$cmd eq {run}} { + execute_tests $data + } else { + error "Unknown test client command: $cmd" + } } } +proc send_data_packet {fd status data} { + set payload [list $status $data] + puts $fd [string length $payload] + puts -nonewline $fd $payload + flush $fd +} + +proc print_help_screen {} { + puts [join { + "--valgrind Run the test over valgrind." + "--accurate Run slow randomized tests for more iterations." + "--quiet Don't show individual tests." + "--single Just execute the specified unit (see next option)." + "--list-tests List all the available test units." + "--clients Number of test clients (16)." + "--force-failure Force the execution of a test that always fails." + "--help Print this help screen." + } "\n"] +} + # parse arguments for {set j 0} {$j < [llength $argv]} {incr j} { set opt [lindex $argv $j] @@ -132,6 +366,10 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } } incr j + } elseif {$opt eq {--valgrind}} { + set ::valgrind 1 + } elseif {$opt eq {--quiet}} { + set ::quiet 1 } elseif {$opt eq {--host}} { set ::external 1 set ::host $arg @@ -139,18 +377,67 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } elseif {$opt eq {--port}} { set ::port $arg incr j + } elseif {$opt eq {--accurate}} { + set ::accurate 1 + } elseif {$opt eq {--force-failure}} { + set ::force_failure 1 + } elseif {$opt eq {--single}} { + set ::all_tests $arg + incr j + } elseif {$opt eq {--list-tests}} { + foreach t $::all_tests { + puts $t + } + exit 0 + } elseif {$opt eq {--client}} { + set ::client 1 + set ::test_server_port $arg + incr j + } elseif {$opt eq {--clients}} { + set ::numclients $arg + incr j + } elseif {$opt eq {--help}} { + print_help_screen + exit 0 } else { puts "Wrong argument: $opt" exit 1 } } -if {[catch { main } err]} { - if {[string length $err] > 0} { - # only display error when not generated by the test suite - if {$err ne "exception"} { - puts $err +# With the parallel test running multiple Redis instances at the same time +# we need a fast enough computer, otherwise a lot of tests may generate +# false positives. +# If the computer is too slow we revert the sequetial test without any +# parallelism, that is, clients == 1. +proc is_a_slow_computer {} { + set start [clock milliseconds] + for {set j 0} {$j < 1000000} {incr j} {} + set elapsed [expr [clock milliseconds]-$start] + expr {$elapsed > 200} +} + +if {$::client} { + if {[catch { test_client_main $::test_server_port } err]} { + set estr "Executing test client: $err.\n$::errorInfo" + if {[catch {send_data_packet $::test_server_fd exception $estr}]} { + puts $estr } exit 1 } +} else { + if {[is_a_slow_computer]} { + puts "** SLOW COMPUTER ** Using a single client to avoid false positives." + set ::numclients 1 + } + + if {[catch { test_server_main } err]} { + if {[string length $err] > 0} { + # only display error when not generated by the test suite + if {$err ne "exception"} { + puts $::errorInfo + } + exit 1 + } + } }