If the computer running the Redis test is slow, we revert to --clients 1 to avoid...

[redis.git] / tests / test_helper.tcl

diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 9f76e0e9426f45eea15b62e4921b21e8e17c2481..7874256da64763037d5c2f1d54f17b1f88b11381 100644 (file)
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -9,22 +9,69 @@ source tests/support/tmpfile.tcl
 source tests/support/test.tcl
 source tests/support/util.tcl
 
 source tests/support/test.tcl
 source tests/support/util.tcl
 

+set ::all_tests {
+    unit/printver
+    unit/auth
+    unit/protocol
+    unit/basic
+    unit/type/list
+    unit/type/list-2
+    unit/type/list-3
+    unit/type/set
+    unit/type/zset
+    unit/type/hash
+    unit/sort
+    unit/expire
+    unit/other
+    unit/cas
+    unit/quit
+    unit/aofrw
+    integration/replication
+    integration/replication-2
+    integration/replication-3
+    integration/replication-4
+    integration/aof
+    integration/rdb
+    integration/convert-zipmap-hash-on-load
+    unit/pubsub
+    unit/slowlog
+    unit/scripting
+    unit/maxmemory
+    unit/introspection
+    unit/limits
+    unit/obuf-limits
+    unit/dump
+}
+# Index to the next test to run in the ::all_tests list.
+set ::next_test 0
+

 set ::host 127.0.0.1
 set ::host 127.0.0.1

-set ::port 16379
+set ::port 21111

 set ::traceleaks 0
 set ::valgrind 0
 set ::traceleaks 0
 set ::valgrind 0

-set ::verbose 1
+set ::verbose 0
+set ::quiet 0

 set ::denytags {}
 set ::allowtags {}
 set ::external 0; # If "1" this means, we are running against external instance
 set ::file ""; # If set, runs only the tests in this comma separated list
 set ::curfile ""; # Hold the filename of the current suite
 set ::denytags {}
 set ::allowtags {}
 set ::external 0; # If "1" this means, we are running against external instance
 set ::file ""; # If set, runs only the tests in this comma separated list
 set ::curfile ""; # Hold the filename of the current suite

-set ::diskstore 0; # Don't touch this by hand. The test itself will toggle it.
+set ::accurate 0; # If true runs fuzz tests with more iterations
+set ::force_failure 0
+
+# Set to 1 when we are running in client mode. The Redis test uses a
+# server-client model to run tests simultaneously. The server instance
+# runs the specified number of client instances that will actually run tests.
+# The server is responsible of showing the result to the user, and exit with
+# the appropriate exit code depending on the test outcome.
+set ::client 0
+set ::numclients 16

 
 proc execute_tests name {
     set path "tests/$name.tcl"
     set ::curfile $path
     source $path
 
 proc execute_tests name {
     set path "tests/$name.tcl"
     set ::curfile $path
     source $path

+    send_data_packet $::test_server_fd done "$name"

 }
 
 # Setup a list to hold a stack of server configs. When calls to start_server
 }
 
 # Setup a list to hold a stack of server configs. When calls to start_server


@@ -74,7 +121,7 @@ proc reconnect {args} {
     }
 
     # re-set $srv in the servers list
     }
 
     # re-set $srv in the servers list

-    set ::servers [lreplace $::servers end+$level 1 $srv]
+    lset ::servers end+$level $srv

 }
 
 proc redis_deferring_client {args} {
 }
 
 proc redis_deferring_client {args} {


@@ -104,84 +151,207 @@ proc s {args} {
 }
 
 proc cleanup {} {
 }
 
 proc cleanup {} {

-    puts "Cleanup: warning may take some time..."
+    if {!$::quiet} {puts -nonewline "Cleanup: may take some time... "}
+    flush stdout

     catch {exec rm -rf {*}[glob tests/tmp/redis.conf.*]}
     catch {exec rm -rf {*}[glob tests/tmp/server.*]}
     catch {exec rm -rf {*}[glob tests/tmp/redis.conf.*]}
     catch {exec rm -rf {*}[glob tests/tmp/server.*]}

+    if {!$::quiet} {puts "OK"}
+}
+
+proc find_available_port start {
+    for {set j $start} {$j < $start+1024} {incr j} {
+        if {[catch {
+            set fd [socket 127.0.0.1 $j]
+        }]} {
+            return $j
+        } else {
+            close $fd
+        }
+    }
+    if {$j == $start+1024} {
+        error "Can't find a non busy port in the $start-[expr {$start+1023}] range."
+    }

 }
 
 }
 

-proc execute_everything {} {
-    if 0 {
-    execute_tests "unit/auth"
-    execute_tests "unit/protocol"
-    execute_tests "unit/basic"
-    execute_tests "unit/type/list"
-    execute_tests "unit/type/set"
-    execute_tests "unit/type/zset"
-    execute_tests "unit/type/hash"
-    execute_tests "unit/sort"
-    execute_tests "unit/expire"
-    execute_tests "unit/other"
-    execute_tests "unit/cas"
-    execute_tests "unit/quit"
-    execute_tests "integration/replication"
-    execute_tests "integration/aof"
-#    execute_tests "integration/redis-cli"
-    execute_tests "unit/pubsub"
-    }
-
-    # run tests with diskstore enabled
-    set ::diskstore 1
-    lappend ::denytags nodiskstore
-    set ::global_overrides {diskstore-enabled yes}
-#    execute_tests "unit/protocol"
-#    execute_tests "unit/basic"
-#    execute_tests "unit/type/list"
-#    execute_tests "unit/type/set"
-#    execute_tests "unit/type/zset"
-#    execute_tests "unit/type/hash"
-#    execute_tests "unit/sort"
-#    execute_tests "unit/expire"
-    execute_tests "unit/other"
-    execute_tests "unit/cas"
-}
-
-proc main {} {
+proc test_server_main {} {

     cleanup
     cleanup

+    # Open a listening socket, trying different ports in order to find a
+    # non busy one.
+    set port [find_available_port 11111]
+    if {!$::quiet} {
+        puts "Starting test server at port $port"
+    }
+    socket -server accept_test_clients $port
+
+    # Start the client instances
+    set ::clients_pids {}
+    set start_port [expr {$::port+100}]
+    for {set j 0} {$j < $::numclients} {incr j} {
+        set start_port [find_available_port $start_port]
+        set p [exec tclsh8.5 [info script] {*}$::argv \
+            --client $port --port $start_port &]
+        lappend ::clients_pids $p
+        incr start_port 10
+    }
+
+    # Setup global state for the test server
+    set ::idle_clients {}
+    set ::active_clients {}
+    array set ::clients_start_time {}
+    set ::clients_time_history {}
+    set ::failed_tests {}
+
+    # Enter the event loop to handle clients I/O
+    after 100 test_server_cron
+    vwait forever
+}

 
 

-    if {[string length $::file] > 0} {
-        foreach {file} [split $::file ,] {
-            execute_tests $file
+# This function gets called 10 times per second, for now does nothing but
+# may be used in the future in order to detect test clients taking too much
+# time to execute the task.
+proc test_server_cron {} {
+}
+
+proc accept_test_clients {fd addr port} {
+    fileevent $fd readable [list read_from_test_client $fd]
+}
+
+# This is the readable handler of our test server. Clients send us messages
+# in the form of a status code such and additional data. Supported
+# status types are:
+#
+# ready: the client is ready to execute the command. Only sent at client
+#        startup. The server will queue the client FD in the list of idle
+#        clients.
+# testing: just used to signal that a given test started.
+# ok: a test was executed with success.
+# err: a test was executed with an error.
+# exception: there was a runtime exception while executing the test.
+# done: all the specified test file was processed, this test client is
+#       ready to accept a new task.
+proc read_from_test_client fd {
+    set bytes [gets $fd]
+    set payload [read $fd $bytes]
+    foreach {status data} $payload break
+    if {$status eq {ready}} {
+        if {!$::quiet} {
+            puts "\[$status\]: $data"

         }
         }

+        signal_idle_client $fd
+    } elseif {$status eq {done}} {
+        set elapsed [expr {[clock seconds]-$::clients_start_time($fd)}]
+        set all_tests_count [llength $::all_tests]
+        set running_tests_count [expr {[llength $::active_clients]-1}]
+        set completed_tests_count [expr {$::next_test-$running_tests_count}]
+        puts "\[$completed_tests_count/$all_tests_count [colorstr yellow $status]\]: $data ($elapsed seconds)"
+        lappend ::clients_time_history $elapsed $data
+        signal_idle_client $fd
+    } elseif {$status eq {ok}} {
+        if {!$::quiet} {
+            puts "\[[colorstr green $status]\]: $data"
+        }
+    } elseif {$status eq {err}} {
+        set err "\[[colorstr red $status]\]: $data"
+        puts $err
+        lappend ::failed_tests $err
+    } elseif {$status eq {exception}} {
+        puts "\[[colorstr red $status]\]: $data"
+        foreach p $::clients_pids {
+            catch {exec kill -9 $p}
+        }
+        exit 1
+    } elseif {$status eq {testing}} {
+        # No op

     } else {
     } else {

-        execute_everything
+        if {!$::quiet} {
+            puts "\[$status\]: $data"
+        }

     }
     }

+}

 
 

-    cleanup
-    puts "\n[expr $::num_tests] tests, $::num_passed passed, $::num_failed failed\n"
-    if {$::num_failed > 0} {
-        set curheader ""
-        puts "Failures:"
-        foreach {test} $::tests_failed {
-            set header [lindex $test 0]
-            append header " ("
-            append header [join [lindex $test 1] ","]
-            append header ")"
-
-            if {$curheader ne $header} {
-                set curheader $header
-                puts "\n$curheader:"
-            }
-
-            set name [lindex $test 2]
-            set msg [lindex $test 3]
-            puts "- $name: $msg"
+# A new client is idle. Remove it from the list of active clients and
+# if there are still test units to run, launch them.
+proc signal_idle_client fd {
+    # Remove this fd from the list of active clients.
+    set ::active_clients \
+        [lsearch -all -inline -not -exact $::active_clients $fd]
+    # New unit to process?
+    if {$::next_test != [llength $::all_tests]} {
+        if {!$::quiet} {
+            puts [colorstr bold-white "Testing [lindex $::all_tests $::next_test]"]

         }
         }

+        set ::clients_start_time($fd) [clock seconds]
+        send_data_packet $fd run [lindex $::all_tests $::next_test]
+        lappend ::active_clients $fd
+        incr ::next_test
+    } else {
+        lappend ::idle_clients $fd
+        if {[llength $::active_clients] == 0} {
+            the_end
+        }
+    }
+}

 
 

-        puts ""
+# The the_end funciton gets called when all the test units were already
+# executed, so the test finished.
+proc the_end {} {
+    # TODO: print the status, exit with the rigth exit code.
+    puts "\n                   The End\n"
+    puts "Execution time of different units:"
+    foreach {time name} $::clients_time_history {
+        puts "  $time seconds - $name"
+    }
+    if {[llength $::failed_tests]} {
+        puts "\n[colorstr bold-red {!!! WARNING}] The following tests failed:\n"
+        foreach failed $::failed_tests {
+            puts "*** $failed"
+        }
+        cleanup

         exit 1
         exit 1

+    } else {
+        puts "\n[colorstr bold-white {\o/}] [colorstr bold-green {All tests passed without errors!}]\n"
+        cleanup
+        exit 0

     }
 }
 
     }
 }
 

+# The client is not even driven (the test server is instead) as we just need
+# to read the command, execute, reply... all this in a loop.
+proc test_client_main server_port {
+    set ::test_server_fd [socket localhost $server_port]
+    send_data_packet $::test_server_fd ready [pid]
+    while 1 {
+        set bytes [gets $::test_server_fd]
+        set payload [read $::test_server_fd $bytes]
+        foreach {cmd data} $payload break
+        if {$cmd eq {run}} {
+            execute_tests $data
+        } else {
+            error "Unknown test client command: $cmd"
+        }
+    }
+}
+
+proc send_data_packet {fd status data} {
+    set payload [list $status $data]
+    puts $fd [string length $payload]
+    puts -nonewline $fd $payload
+    flush $fd
+}
+
+proc print_help_screen {} {
+    puts [join {
+        "--valgrind         Run the test over valgrind."
+        "--accurate         Run slow randomized tests for more iterations."
+        "--quiet            Don't show individual tests."
+        "--single <unit>    Just execute the specified unit (see next option)."
+        "--list-tests       List all the available test units."
+        "--clients <num>    Number of test clients (16)."
+        "--force-failure    Force the execution of a test that always fails."
+        "--help             Print this help screen."
+    } "\n"]
+}
+

 # parse arguments
 for {set j 0} {$j < [llength $argv]} {incr j} {
     set opt [lindex $argv $j]
 # parse arguments
 for {set j 0} {$j < [llength $argv]} {incr j} {
     set opt [lindex $argv $j]


@@ -197,9 +367,8 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
         incr j
     } elseif {$opt eq {--valgrind}} {
         set ::valgrind 1
         incr j
     } elseif {$opt eq {--valgrind}} {
         set ::valgrind 1

-    } elseif {$opt eq {--file}} {
-        set ::file $arg
-        incr j
+    } elseif {$opt eq {--quiet}} {
+        set ::quiet 1

     } elseif {$opt eq {--host}} {
         set ::external 1
         set ::host $arg
     } elseif {$opt eq {--host}} {
         set ::external 1
         set ::host $arg


@@ -207,20 +376,67 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
     } elseif {$opt eq {--port}} {
         set ::port $arg
         incr j
     } elseif {$opt eq {--port}} {
         set ::port $arg
         incr j

-    } elseif {$opt eq {--verbose}} {
-        set ::verbose 1
+    } elseif {$opt eq {--accurate}} {
+        set ::accurate 1
+    } elseif {$opt eq {--force-failure}} {
+        set ::force_failure 1
+    } elseif {$opt eq {--single}} {
+        set ::all_tests $arg
+        incr j
+    } elseif {$opt eq {--list-tests}} {
+        foreach t $::all_tests {
+            puts $t
+        }
+        exit 0
+    } elseif {$opt eq {--client}} {
+        set ::client 1
+        set ::test_server_port $arg
+        incr j
+    } elseif {$opt eq {--clients}} {
+        set ::numclients $arg
+        incr j
+    } elseif {$opt eq {--help}} {
+        print_help_screen
+        exit 0

     } else {
         puts "Wrong argument: $opt"
         exit 1
     }
 }
 
     } else {
         puts "Wrong argument: $opt"
         exit 1
     }
 }
 

-if {[catch { main } err]} {
-    if {[string length $err] > 0} {
-        # only display error when not generated by the test suite
-        if {$err ne "exception"} {
-            puts $::errorInfo
+# With the parallel test running multiple Redis instances at the same time
+# we need a fast enough computer, otherwise a lot of tests may generate
+# false positives.
+# If the computer is too slow we revert the sequetial test without any
+# parallelism, that is, clients == 1.
+proc is_a_slow_computer {} {
+    set start [clock milliseconds]
+    for {set j 0} {$j < 1000000} {incr j} {}
+    set elapsed [expr [clock milliseconds]-$start]
+    expr {$elapsed > 200}
+}
+
+if {$::client} {
+    if {[catch { test_client_main $::test_server_port } err]} {
+        set estr "Executing test client: $err.\n$::errorInfo"
+        if {[catch {send_data_packet $::test_server_fd exception $estr}]} {
+            puts $estr

         }
         exit 1
     }
         }
         exit 1
     }

+} else {
+    if {[is_a_slow_computer]} {
+        puts "** SLOW COMPUTER ** Using a single client to avoid false positives."
+        set ::numclients 1
+    }
+
+    if {[catch { test_server_main } err]} {
+        if {[string length $err] > 0} {
+            # only display error when not generated by the test suite
+            if {$err ne "exception"} {
+                puts $::errorInfo
+            }
+            exit 1
+        }
+    }

 }
 }