Merge pull request #461 from schlenk/unstable

[redis.git] / tests / test_helper.tcl

diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl
index 8629fa57ca9c14ffa4940b35ea904c0eeb389df3..598a392916538a1900cb711dd3ec31668444bb64 100644 (file)
--- a/tests/test_helper.tcl
+++ b/tests/test_helper.tcl
@@ -25,26 +25,39 @@ set ::all_tests {
     unit/other
     unit/cas
     unit/quit
     unit/other
     unit/cas
     unit/quit

+    unit/aofrw

     integration/replication
     integration/replication-2
     integration/replication-3
     integration/replication
     integration/replication-2
     integration/replication-3

+    integration/replication-4

     integration/aof
     integration/aof

+    integration/rdb
+    integration/convert-zipmap-hash-on-load

     unit/pubsub
     unit/slowlog
     unit/pubsub
     unit/slowlog

+    unit/scripting
+    unit/maxmemory
+    unit/introspection
+    unit/limits
+    unit/obuf-limits
+    unit/dump

 }
 # Index to the next test to run in the ::all_tests list.
 set ::next_test 0
 
 set ::host 127.0.0.1
 }
 # Index to the next test to run in the ::all_tests list.
 set ::next_test 0
 
 set ::host 127.0.0.1

-set ::port 16379
+set ::port 21111

 set ::traceleaks 0
 set ::valgrind 0
 set ::verbose 0
 set ::traceleaks 0
 set ::valgrind 0
 set ::verbose 0

+set ::quiet 0

 set ::denytags {}
 set ::allowtags {}
 set ::external 0; # If "1" this means, we are running against external instance
 set ::file ""; # If set, runs only the tests in this comma separated list
 set ::curfile ""; # Hold the filename of the current suite
 set ::denytags {}
 set ::allowtags {}
 set ::external 0; # If "1" this means, we are running against external instance
 set ::file ""; # If set, runs only the tests in this comma separated list
 set ::curfile ""; # Hold the filename of the current suite

+set ::accurate 0; # If true runs fuzz tests with more iterations
+set ::force_failure 0

 
 # Set to 1 when we are running in client mode. The Redis test uses a
 # server-client model to run tests simultaneously. The server instance
 
 # Set to 1 when we are running in client mode. The Redis test uses a
 # server-client model to run tests simultaneously. The server instance


@@ -108,7 +121,7 @@ proc reconnect {args} {
     }
 
     # re-set $srv in the servers list
     }
 
     # re-set $srv in the servers list

-    set ::servers [lreplace $::servers end+$level 1 $srv]
+    lset ::servers end+$level $srv

 }
 
 proc redis_deferring_client {args} {
 }
 
 proc redis_deferring_client {args} {


@@ -138,43 +151,47 @@ proc s {args} {
 }
 
 proc cleanup {} {
 }
 
 proc cleanup {} {

-    puts -nonewline "Cleanup: warning may take some time... "
+    if {!$::quiet} {puts -nonewline "Cleanup: may take some time... "}

     flush stdout
     catch {exec rm -rf {*}[glob tests/tmp/redis.conf.*]}
     catch {exec rm -rf {*}[glob tests/tmp/server.*]}
     flush stdout
     catch {exec rm -rf {*}[glob tests/tmp/redis.conf.*]}
     catch {exec rm -rf {*}[glob tests/tmp/server.*]}

-    puts "OK"
+    if {!$::quiet} {puts "OK"}
+}
+
+proc find_available_port start {
+    for {set j $start} {$j < $start+1024} {incr j} {
+        if {[catch {
+            set fd [socket 127.0.0.1 $j]
+        }]} {
+            return $j
+        } else {
+            close $fd
+        }
+    }
+    if {$j == $start+1024} {
+        error "Can't find a non busy port in the $start-[expr {$start+1023}] range."
+    }

 }
 
 proc test_server_main {} {
     cleanup
     # Open a listening socket, trying different ports in order to find a
     # non busy one.
 }
 
 proc test_server_main {} {
     cleanup
     # Open a listening socket, trying different ports in order to find a
     # non busy one.

-    set port 11111
-    while 1 {
+    set port [find_available_port 11111]
+    if {!$::quiet} {

         puts "Starting test server at port $port"
         puts "Starting test server at port $port"

-        if {[catch {socket -server accept_test_clients $port} e]} {
-            if {[string match {*address already in use*} $e]} {
-                if {$port == 20000} {
-                    puts "Can't find an available TCP port for test server."
-                    exit 1
-                } else {
-                    incr port
-                }
-            } else {
-                puts "Fatal error starting test server: $e"
-                exit 1
-            }
-        } else {
-            break
-        }

     }
     }

+    socket -server accept_test_clients $port

 
     # Start the client instances
     set ::clients_pids {}
 
     # Start the client instances
     set ::clients_pids {}

+    set start_port [expr {$::port+100}]

     for {set j 0} {$j < $::numclients} {incr j} {
     for {set j 0} {$j < $::numclients} {incr j} {

+        set start_port [find_available_port $start_port]

         set p [exec tclsh8.5 [info script] {*}$::argv \
         set p [exec tclsh8.5 [info script] {*}$::argv \

-            --client $port --port [expr {$::port+($j*10)}] &]
+            --client $port --port $start_port &]

         lappend ::clients_pids $p
         lappend ::clients_pids $p

+        incr start_port 10

     }
 
     # Setup global state for the test server
     }
 
     # Setup global state for the test server


@@ -182,6 +199,7 @@ proc test_server_main {} {
     set ::active_clients {}
     array set ::clients_start_time {}
     set ::clients_time_history {}
     set ::active_clients {}
     array set ::clients_start_time {}
     set ::clients_time_history {}

+    set ::failed_tests {}

 
     # Enter the event loop to handle clients I/O
     after 100 test_server_cron
 
     # Enter the event loop to handle clients I/O
     after 100 test_server_cron


@@ -216,18 +234,26 @@ proc read_from_test_client fd {
     set payload [read $fd $bytes]
     foreach {status data} $payload break
     if {$status eq {ready}} {
     set payload [read $fd $bytes]
     foreach {status data} $payload break
     if {$status eq {ready}} {

-        puts "\[$status\]: $data"
+        if {!$::quiet} {
+            puts "\[$status\]: $data"
+        }

         signal_idle_client $fd
     } elseif {$status eq {done}} {
         set elapsed [expr {[clock seconds]-$::clients_start_time($fd)}]
         signal_idle_client $fd
     } elseif {$status eq {done}} {
         set elapsed [expr {[clock seconds]-$::clients_start_time($fd)}]

-        puts "\[[colorstr yellow $status]\]: $data ($elapsed seconds)"
-        puts "+++ [expr {[llength $::active_clients]-1}] units still in execution."
+        set all_tests_count [llength $::all_tests]
+        set running_tests_count [expr {[llength $::active_clients]-1}]
+        set completed_tests_count [expr {$::next_test-$running_tests_count}]
+        puts "\[$completed_tests_count/$all_tests_count [colorstr yellow $status]\]: $data ($elapsed seconds)"

         lappend ::clients_time_history $elapsed $data
         signal_idle_client $fd
     } elseif {$status eq {ok}} {
         lappend ::clients_time_history $elapsed $data
         signal_idle_client $fd
     } elseif {$status eq {ok}} {

-        puts "\[[colorstr green $status]\]: $data"
+        if {!$::quiet} {
+            puts "\[[colorstr green $status]\]: $data"
+        }

     } elseif {$status eq {err}} {
     } elseif {$status eq {err}} {

-        puts "\[[colorstr red $status]\]: $data"
+        set err "\[[colorstr red $status]\]: $data"
+        puts $err
+        lappend ::failed_tests $err

     } elseif {$status eq {exception}} {
         puts "\[[colorstr red $status]\]: $data"
         foreach p $::clients_pids {
     } elseif {$status eq {exception}} {
         puts "\[[colorstr red $status]\]: $data"
         foreach p $::clients_pids {


@@ -237,7 +263,9 @@ proc read_from_test_client fd {
     } elseif {$status eq {testing}} {
         # No op
     } else {
     } elseif {$status eq {testing}} {
         # No op
     } else {

-        puts "\[$status\]: $data"
+        if {!$::quiet} {
+            puts "\[$status\]: $data"
+        }

     }
 }
 
     }
 }
 


@@ -249,7 +277,9 @@ proc signal_idle_client fd {
         [lsearch -all -inline -not -exact $::active_clients $fd]
     # New unit to process?
     if {$::next_test != [llength $::all_tests]} {
         [lsearch -all -inline -not -exact $::active_clients $fd]
     # New unit to process?
     if {$::next_test != [llength $::all_tests]} {

-        puts [colorstr bold-white "Testing [lindex $::all_tests $::next_test]"]
+        if {!$::quiet} {
+            puts [colorstr bold-white "Testing [lindex $::all_tests $::next_test]"]
+        }

         set ::clients_start_time($fd) [clock seconds]
         send_data_packet $fd run [lindex $::all_tests $::next_test]
         lappend ::active_clients $fd
         set ::clients_start_time($fd) [clock seconds]
         send_data_packet $fd run [lindex $::all_tests $::next_test]
         lappend ::active_clients $fd


@@ -266,12 +296,23 @@ proc signal_idle_client fd {
 # executed, so the test finished.
 proc the_end {} {
     # TODO: print the status, exit with the rigth exit code.
 # executed, so the test finished.
 proc the_end {} {
     # TODO: print the status, exit with the rigth exit code.

-    puts "The End\n"
+    puts "\n                   The End\n"

     puts "Execution time of different units:"
     foreach {time name} $::clients_time_history {
         puts "  $time seconds - $name"
     }
     puts "Execution time of different units:"
     foreach {time name} $::clients_time_history {
         puts "  $time seconds - $name"
     }

-    exit 1
+    if {[llength $::failed_tests]} {
+        puts "\n[colorstr bold-red {!!! WARNING}] The following tests failed:\n"
+        foreach failed $::failed_tests {
+            puts "*** $failed"
+        }
+        cleanup
+        exit 1
+    } else {
+        puts "\n[colorstr bold-white {\o/}] [colorstr bold-green {All tests passed without errors!}]\n"
+        cleanup
+        exit 0
+    }

 }
 
 # The client is not even driven (the test server is instead) as we just need
 }
 
 # The client is not even driven (the test server is instead) as we just need


@@ -298,6 +339,19 @@ proc send_data_packet {fd status data} {
     flush $fd
 }
 
     flush $fd
 }
 

+proc print_help_screen {} {
+    puts [join {
+        "--valgrind         Run the test over valgrind."
+        "--accurate         Run slow randomized tests for more iterations."
+        "--quiet            Don't show individual tests."
+        "--single <unit>    Just execute the specified unit (see next option)."
+        "--list-tests       List all the available test units."
+        "--clients <num>    Number of test clients (16)."
+        "--force-failure    Force the execution of a test that always fails."
+        "--help             Print this help screen."
+    } "\n"]
+}
+

 # parse arguments
 for {set j 0} {$j < [llength $argv]} {incr j} {
     set opt [lindex $argv $j]
 # parse arguments
 for {set j 0} {$j < [llength $argv]} {incr j} {
     set opt [lindex $argv $j]


@@ -313,9 +367,8 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
         incr j
     } elseif {$opt eq {--valgrind}} {
         set ::valgrind 1
         incr j
     } elseif {$opt eq {--valgrind}} {
         set ::valgrind 1

-    } elseif {$opt eq {--file}} {
-        set ::file $arg
-        incr j
+    } elseif {$opt eq {--quiet}} {
+        set ::quiet 1

     } elseif {$opt eq {--host}} {
         set ::external 1
         set ::host $arg
     } elseif {$opt eq {--host}} {
         set ::external 1
         set ::host $arg


@@ -323,12 +376,28 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
     } elseif {$opt eq {--port}} {
         set ::port $arg
         incr j
     } elseif {$opt eq {--port}} {
         set ::port $arg
         incr j

-    } elseif {$opt eq {--verbose}} {
-        set ::verbose 1
+    } elseif {$opt eq {--accurate}} {
+        set ::accurate 1
+    } elseif {$opt eq {--force-failure}} {
+        set ::force_failure 1
+    } elseif {$opt eq {--single}} {
+        set ::all_tests $arg
+        incr j
+    } elseif {$opt eq {--list-tests}} {
+        foreach t $::all_tests {
+            puts $t
+        }
+        exit 0

     } elseif {$opt eq {--client}} {
         set ::client 1
         set ::test_server_port $arg
         incr j
     } elseif {$opt eq {--client}} {
         set ::client 1
         set ::test_server_port $arg
         incr j

+    } elseif {$opt eq {--clients}} {
+        set ::numclients $arg
+        incr j
+    } elseif {$opt eq {--help}} {
+        print_help_screen
+        exit 0

     } else {
         puts "Wrong argument: $opt"
         exit 1
     } else {
         puts "Wrong argument: $opt"
         exit 1