tests/mozilla/runtests.pl

   1 #!/tools/ns/bin/perl5
   2 #
   3 # simple script that executes JavaScript tests.  you have to build the
   4 # stand-alone, js shell executable (which is not the same as the dll that gets
   5 # built for mozilla).  see the readme at
   6 # http://lxr.mozilla.org/mozilla/source/js/src/README.html for instructions on
   7 # how to build the jsshell.
   8 #
   9 # this is just a quick-n-dirty script.  for full reporting, you need to run
  10 # the test driver, which requires java and is currently not available on
  11 # mozilla.org.
  12 #
  13 # this test looks for an executable JavaScript shell in
  14 # %MOZ_SRC/mozilla/js/src/[platform]-[platform-version]-OPT.OBJ/js,
  15 # which is the default build location when you build using the instructions
  16 # at http://lxr.mozilla.org/mozilla/source/js/src/README.html
  17 #
  18 #
  19 # christine@netscape.com
  20 #
  21
  22 &parse_args;
  23 &setup_env;
  24 &main_test_loop;
  25 &cleanup_env;
  26
  27 #
  28 # given a main directory, assume that there is a file called 'shell.js'
  29 # in it.  then, open all the subdirectories, and look for js files.
  30 # for each test.js that is found, execute the shell, and pass shell.js
  31 # and the test.js as file arguments.  redirect all process output to a
  32 # file.
  33 #
  34 sub main_test_loop {
  35     foreach $suite ( &get_subdirs( $test_dir )) {
  36         foreach $subdir (&get_subdirs( $suite, $test_dir )) {
  37             @jsfiles = &get_js_files($subdir);
  38             execute_js_tests(@jsfiles);
  39         }
  40     }
  41 }
  42
  43 #
  44 # given a directory, return an array of all subdirectories
  45 #
  46 sub get_subdirs{
  47     local ($dir, $path)  = @_;
  48     local @subdirs;
  49
  50     local $dir_path = $path . $dir;
  51     chdir $dir_path;
  52
  53     opendir ( DIR, ${dir_path} );
  54     local @testdir_contents = readdir( DIR );
  55     closedir( DIR );
  56
  57     foreach (@testdir_contents) {
  58         if ( (-d $_) && ($_ !~ 'CVS') && ( $_ ne '.') && ($_ ne '..')) {
  59             @subdirs[$#subdirs+1] = $_;
  60         }
  61     }
  62     chdir $path;
  63     return @subdirs;
  64 }
  65
  66 #
  67 # given a directory, return an array of all the js files that are in it.
  68 #
  69 sub get_js_files {
  70     ( $test_subdir ) = @_;
  71     local @js_file_array;
  72
  73     $current_test_dir = $test_dir  ."/". $suite . "/" .$test_subdir;
  74     chdir $current_test_dir;
  75
  76     opendir ( TEST_SUBDIR, ${current_test_dir} );
  77     @subdir_files = readdir( TEST_SUBDIR );
  78     closedir( TOP_LEVEL_BUILD_DIR );
  79
  80     foreach ( @subdir_files ) {
  81         if ( $_ =~ /\.js$/ ) {
  82             $js_file_array[$#js_file_array+1] = $_;
  83         }
  84     }
  85
  86     return @js_file_array;
  87 }
  88
  89 #
  90 # given an array of test.js files, execute the shell command and pass
  91 # the shell.js and test.js files as file arguments.  redirect process
  92 # output to a file.  if $js_verbose is set (not recommended), write all
  93 # testcase output to the output file.  if $js_quiet is set, only write
  94 # failed test case information to the output file.  the default setting
  95 # is to write a line for each test file, and whether each file passed
  96 # or failed.
  97 #
  98 sub execute_js_tests {
  99     (@js_file_array) = @_;
 100
 101     $js_printed_suitename = 0;
 102     if ( !$js_quiet ) {
 103         &js_print_suitename;
 104     }
 105
 106     foreach $js_test (@js_file_array) {
 107         $js_printed_filename = 0;
 108         $js_test_bugnumber = 0;
 109         $runtime_error = "";
 110
 111         local $passed = -1;
 112
 113         # create the test command
 114         $test_command =
 115             $shell_command .
 116             " -f $test_dir/$suite/shell.js " .
 117             " -f $test_dir/$suite/$subdir/$js_test";
 118
 119         if ( !$js_quiet ) {
 120             &js_print_filename;
 121         } else {
 122             print '.';
 123         }
 124
 125         $test_path = $test_dir ."/" . $suite ."/". $test_subdir ."/". $js_test;
 126
 127
 128         if ( !-e $test_path ) {
 129             &js_print( " FAILED! file not found\n",
 130                 "<font color=#990000>", "</font><br>\n");
 131         } else {
 132             open( RUNNING_TEST,  "$test_command" . ' 2>&1 |');
 133
 134
 135                         # this is where we want the tests to provide a lot more information
 136                         # that this script must parse so that we can
 137
 138                 while( <RUNNING_TEST> ){
 139                 if ( $js_verbose && !$js_quiet ) {
 140                     &js_print ($_ ."\n", "", "<br>\n");
 141                 }
 142                 if ( $_ =~ /BUGNUMBER/ ) {
 143                     $js_test_bugnumber = $_;
 144                 }
 145                                 if ( $_ =~ /PASSED/ && $passed == -1 ) {
 146                                         $passed = 1;
 147                                 }
 148                 if ( $_ =~ /FAILED/ && $_ =~ /expected/) {
 149                     &js_print_suitename;
 150                     &js_print_filename;
 151                     &js_print_bugnumber;
 152
 153                     local @msg = split ( "FAILED", $_ );
 154                     &js_print ( $passed ? "\n" : "" );
 155                     &js_print( "    " . $msg[0], "&nbsp;&nbsp;<tt>" );
 156                     &js_print( "FAILED", "<font color=#990000>", "</font>");
 157                     &js_print( $msg[1], "", "</tt><br>\n" );
 158                     $passed = 0;
 159                 }
 160                 if ( $_ =~ /$js_test/ ) {
 161                     $runtime_error .= $_;
 162                 }
 163             }
 164             close( RUNNING_TEST );
 165
 166             #
 167             # figure out whether the test passed or failed.  print out an
 168             # appropriate level of output based on the value of $js_quiet
 169             #
 170             if ( $js_test =~ /-n\.js$/ ) {
 171                 if ( $runtime_error ) {
 172                     if ( !$js_quiet ) {
 173                         &js_print( " PASSED!\n ",
 174                             "<font color=#009900>&nbsp;&nbsp",
 175                             "</font><br>" );
 176                         if ( $js_errors ) {
 177                             &js_print( $runtime_error, "<pre>", "</pre>");
 178                         }
 179                     }
 180                 } else {
 181                     &js_print_suitename;
 182                     &js_print_filename;
 183                     &js_print_bugnumber;
 184                     &js_print( " FAILED! ", "&nbsp;&nbsp;<font color=#990000>",
 185                         "</font>");
 186                     &js_print( " Should have resulted in an error\n",
 187                         "","<br>" );
 188                 }
 189             } else {
 190                 if ( $passed == 1 && !$js_quiet) {
 191                     &js_print( " PASSED!\n " , "&nbsp;&nbsp;<font color=#009900>",
 192                         "</font><br>" );
 193                 } else {
 194                                         if ($passed == -1) {
 195                                                 &js_print_suitename;
 196                                                 &js_print_filename;
 197                                                 &js_print_bugnumber;
 198                                                 &js_print( " FAILED!\n " , "&nbsp;&nbsp;<font color=#990000>",
 199                                                 "</font><br>" );
 200                                                 &js_print( " Missing 'PASSED' in output\n", "","<br>" );
 201                                                 &js_print( $log, "output:<br><pre>", "</pre>" );
 202                      }
 203                                 }
 204
 205             }
 206         }
 207     }
 208 }
 209
 210 #
 211 # figure out what os we're on, the default name of the object directory
 212 #
 213 sub setup_env {
 214     # MOZ_SRC must be set, so we can figure out where the
 215     # JavaScript executable is
 216     $moz_src = $ENV{"MOZ_SRC"}
 217         || die( "You need to set your MOZ_SRC environment variable.\n" );
 218     $src_dir = $moz_src . '/mozilla/js/src/';
 219
 220     # JS_TEST_DIR must be set so we can figure out where the tests are.
 221     $test_dir = $ENV{"JS_TEST_DIR"};
 222
 223     # if it's not set, look for it relative to $moz_src
 224     if ( !$test_dir ) {
 225         $test_dir = $moz_src . '/mozilla/js/tests/';
 226     }
 227
 228     # make sure that the test dir exists
 229     if ( ! -e $test_dir ) {
 230         die "The JavaScript Test Library could not be found at $test_dir.\n" .
 231             "Check the tests out from /mozilla/js/tests or\n" .
 232             "Set the value of your JS_TEST_DIR environment variable\n " .
 233             "to the location of the test library.\n";
 234     }
 235
 236     # make sure that the test dir ends with a trailing slash
 237     $test_dir .= '/';
 238
 239     chdir $src_dir;
 240
 241     # figure out which platform we're on, and figure out where the object
 242     # directory is
 243
 244     $machine_os = `uname -s`;
 245
 246     if ( $machine_os =~ /WIN/ ) {
 247         $machine_os = 'WIN';
 248         $object_dir = ($js_debug) ? 'Debug' : 'Release';
 249         $js_exe = 'jsshell.exe';
 250     } else {
 251         chop $machine_os;
 252         $js_exe = 'js';
 253
 254         # figure out what the object directory is.  on all platforms,
 255         # it's the directory that ends in OBJ.  if $js_debug is set,
 256         # look the directory that ends with or DBG.OBJ; otherwise
 257         # look for the directory that ends with OPT.OBJ
 258
 259         opendir ( SRC_DIR_FILES, $src_dir );
 260         @src_dir_files = readdir( SRC_DIR_FILES );
 261         closedir ( SRC_DIR_FILES );
 262
 263         $object_pattern = $js_debug ? 'DBG.OBJ' : 'OPT.OBJ';
 264
 265         foreach (@src_dir_files) {
 266             if ( $_ =~ /$object_pattern/ && $_ =~ $machine_os) {
 267                 $object_dir = $_;
 268             }
 269         }
 270     }
 271     if ( ! $object_dir ) {
 272         die( "Couldn't find an object directory in $src_dir.\n" );
 273     }
 274
 275     # figure out what the name of the javascript executable should be, and
 276     # make sure it's there.  if it's not there, give a helpful message so
 277     # the user can figure out what they need to do next.
 278
 279
 280     if ( ! $js_exe_full_path ) {
 281         $shell_command = $src_dir . $object_dir .'/'. $js_exe;
 282     } else {
 283         $shell_command = $js_exe_full_path;
 284     }
 285
 286     if ( !-e $shell_command ) {
 287         die ("Could not find JavaScript shell executable $shell_command.\n" .
 288             "Check the value of your MOZ_SRC environment variable.\n" .
 289             "Currently, MOZ_SRC is set to $ENV{\"MOZ_SRC\"}\n".
 290             "See the readme at http://lxr.mozilla.org/mozilla/src/js/src/ " .
 291             "for instructions on building the JavaScript shell.\n" );
 292     }
 293
 294     # set the output file name.  let's base its name on the date and platform,
 295     # and give it a sequence number.
 296
 297     if ( $get_output ) {
 298         $js_output = &get_output;
 299     }
 300     if ($js_output) {
 301         print( "Writing results to $js_output\n" );
 302         chdir $test_dir;
 303         open( JS_OUTPUT, "> ${js_output}" ) ||
 304             die "Can't open log file $js_output\n";
 305         close JS_OUTPUT;
 306     }
 307
 308     # get the start time
 309     $start_time = time;
 310
 311     # print out some nice stuff
 312     $start_date = &get_date;
 313     &js_print( "JavaScript tests started: " . $start_date, "<p><tt>", "</tt></p>" );
 314
 315     &js_print ("Executing all the tests under $test_dir\n against " .
 316         "$shell_command\n", "<p><tt>", "</tt></p>" );
 317 }
 318
 319 #
 320 # parse arguments.  see usage for what arguments are expected.
 321 #
 322 sub parse_args {
 323     $i = 0;
 324     while( $i < @ARGV ){
 325         if ( $ARGV[$i] eq '--threaded' ) {
 326             $js_threaded = 1;
 327         } elsif ( $ARGV[$i] eq '--d' ) {
 328             $js_debug = 1;
 329         } elsif ( $ARGV[$i] eq '--14' ) {
 330             $js_version = '14';
 331         } elsif ( $ARGV[$i] eq '--v' ) {
 332             $js_verbose = 1;
 333         } elsif ( $ARGV[$i] eq '-f' ) {
 334             $js_output = $ARGV[++$i];
 335         } elsif ( $ARGV[$i] eq '--o' ) {
 336             $get_output = 1;
 337         } elsif ($ARGV[$i] eq '--e' ) {
 338             $js_errors = 1;
 339         } elsif ($ARGV[$i] eq '--q' ) {
 340             $js_quiet = 1;
 341         } elsif ($ARGV[$i] eq '--h' ) {
 342             die &usage;
 343         } elsif ( $ARGV[$i] eq '-E' ) {
 344             $js_exe_full_path = $ARGV[$i+1];
 345             $i++;
 346         } else {
 347             die &usage;
 348         }
 349         $i++;
 350     }
 351
 352     #
 353     # if no output options are provided, show some output and write to file
 354     #
 355     if ( !$js_verbose && !$js_output && !$get_output ) {
 356         $get_output = 1;
 357     }
 358 }
 359
 360 #
 361 # print the arguments that this script expects
 362 #
 363 sub usage {
 364     die ("usage: $0\n" .
 365         "--q       Quiet mode -- only show information for tests that failed\n".
 366         "--e       Show runtime error messages for negative tests\n" .
 367         "--v       Verbose output -- show all test cases (not recommended)\n" .
 368         "--o       Send output to file whose generated name is based on date\n".
 369         "--d       Look for a debug JavaScript executable (default is optimized)\n" .
 370         "-f <file> Redirect output to file named <file>\n"
 371         );
 372 }
 373
 374 #
 375 # if $js_output is set, print to file as well as stdout
 376 #
 377 sub js_print {
 378     ($string, $start_tag, $end_tag) = @_;
 379
 380     if ($js_output) {
 381         open( JS_OUTPUT, ">> ${js_output}" ) ||
 382             die "Can't open log file $js_output\n";
 383
 384         print JS_OUTPUT "$start_tag $string $end_tag";
 385         close JS_OUTPUT;
 386     }
 387     print $string;
 388 }
 389
 390 #
 391 # close open files
 392 #
 393 sub cleanup_env {
 394     # print out some nice stuff
 395     $end_date = &get_date;
 396     &js_print( "\nTests complete at $end_date", "<hr><tt>", "</tt>" );
 397
 398     # print out how long it took to complete
 399     $end_time = time;
 400
 401     $test_seconds = ( $end_time - $start_time );
 402
 403     &js_print( "Start Date: $start_date\n", "<tt><br>" );
 404     &js_print( "End Date:   $end_date\n", "<br>" );
 405     &js_print( "Test Time:  $test_seconds seconds\n", "<br>" );
 406
 407     if ($js_output ) {
 408         if ( !$js_verbose) {
 409             &js_print( "Results were written to " . $js_output ."\n",
 410                 "<br>", "</tt>" );
 411         }
 412         close JS_OUTPUT;
 413     }
 414 }
 415
 416
 417 #
 418 # get the current date and time
 419 #
 420 sub get_date {
 421     &get_localtime;
 422     $now = $year ."/". $mon ."/". $mday ." ". $hour .":".
 423         $min .":". $sec ."\n";
 424     return $now;
 425
 426 }
 427 sub get_localtime {
 428     ( $sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst ) =
 429         localtime;
 430     $mon++;
 431     $mon = &zero_pad($mon);
 432     $year= ($year < 2000) ? "19" . $year : $year;
 433     $mday= &zero_pad($mday);
 434     $sec = &zero_pad($sec);
 435     $min = &zero_pad($min);
 436     $hour = &zero_pad($hour);
 437 }
 438 sub zero_pad {
 439     local ($string) = @_;
 440     $string = ($string < 10) ? "0" . $string : $string;
 441     return $string;
 442 }
 443
 444 #
 445 # generate an output file name based on the date
 446 #
 447 sub get_output {
 448     &get_localtime;
 449
 450     chdir $test_dir;
 451
 452     $js_output = $test_dir ."/". $year .'-'. $mon .'-'. $mday ."\.1.html";
 453
 454     $output_file_found = 0;
 455
 456     while ( !$output_file_found ) {
 457         if ( -e $js_output ) {
 458         # get the last sequence number - everything after the dot
 459             @seq_no = split( /\./, $js_output, 2 );
 460             $js_output = $seq_no[0] .".". (++$seq_no[1]) . "\.html";
 461         } else {
 462             $output_file_found = 1;
 463         }
 464     }
 465     return $js_output;
 466 }
 467
 468 sub js_print_suitename {
 469     if ( !$js_printed_suitename ) {
 470         &js_print( "$suite\\$subdir\n", "<hr><font size+=1><b>",
 471             "</font></b><br>" );
 472     }
 473     $js_printed_suitename = 1;
 474 }
 475
 476 sub js_print_filename {
 477     if ( !$js_printed_filename ) {
 478         &js_print( "$js_test\n", "<b>", "</b><br>" );
 479         $js_printed_filename = 1;
 480     }
 481 }
 482
 483 sub js_print_bugnumber {
 484     if ( !$js_printed_bugnumber ) {
 485         if ( $js_bugnumber =~ /^http/ ) {
 486             &js_print( "$js_bugnumber", "<a href=$js_bugnumber>", "</a>" );
 487         } else {
 488             &js_print( "$js_bugnumber",
 489                 "<a href=http://scopus.mcom.com/bugsplat/show_bug.cgi?id=" .
 490                     $js_bugnumber .">",
 491                 "</a>" );
 492         }
 493         $js_printed_bugnumber = 1;
 494     }
 495 }