test/test.pl

   1 #!/usr/bin/perl
   2
   3 # test.pl
   4 # Run unit tests.
   5
   6 use strict;
   7 use File::Basename;
   8
   9 chdir dirname $0;
  10 chomp (my $DIR = `pwd`);
  11
  12 my $TESTLIBNAME = "libobjc.A.dylib";
  13 my $TESTLIBPATH = "/usr/lib/$TESTLIBNAME";
  14
  15 my $BUILDDIR = "/tmp/test-$TESTLIBNAME-build";
  16
  17 # xterm colors
  18 my $red = "\e[41;37m";
  19 my $yellow = "\e[43;37m";
  20 my $def = "\e[0m";
  21
  22 # clean, help
  23 if (scalar(@ARGV) == 1) {
  24     my $arg = $ARGV[0];
  25     if ($arg eq "clean") {
  26         my $cmd = "rm -rf $BUILDDIR *~";
  27         print "$cmd\n";
  28         `$cmd`;
  29         exit 0;
  30     }
  31     elsif ($arg eq "-h" || $arg eq "-H" || $arg eq "-help" || $arg eq "help") {
  32         print(<<END);
  33 usage: $0 [options] [testname ...]
  34        $0 clean
  35        $0 help
  36
  37 testname:
  38     `testname` runs a specific test. If no testnames are given, runs all tests.
  39
  40 options:
  41     ARCH=<arch>
  42     SDK=<sdk name>
  43     ROOT=/path/to/project.roots/
  44
  45     CC=<compiler name>
  46
  47     LANGUAGE=c,c++,objective-c,objective-c++,swift
  48     MEM=mrc,arc,gc
  49     STDLIB=libc++,libstdc++
  50     GUARDMALLOC=0|1
  51
  52     BUILD=0|1
  53     RUN=0|1
  54     VERBOSE=0|1
  55
  56 examples:
  57
  58     test installed library, x86_64, no gc
  59     $0
  60
  61     test buildit-built root, i386 and x86_64, MRC and ARC and GC, clang compiler
  62     $0 ARCH=i386,x86_64 ROOT=/tmp/libclosure.roots MEM=mrc,arc,gc CC=clang
  63
  64     test buildit-built root with iOS simulator
  65     $0 ARCH=i386 ROOT=/tmp/libclosure.roots SDK=iphonesimulator
  66
  67     test buildit-built root on attached iOS device
  68     $0 ARCH=armv7 ROOT=/tmp/libclosure.roots SDK=iphoneos
  69 END
  70         exit 0;
  71     }
  72 }
  73
  74 #########################################################################
  75 ## Tests
  76
  77 my %ALL_TESTS;
  78
  79 #########################################################################
  80 ## Variables for use in complex build and run rules
  81
  82 # variable         # example value
  83
  84 # things you can multiplex on the command line
  85 # ARCH=i386,x86_64,armv6,armv7
  86 # SDK=macosx,iphoneos,iphonesimulator
  87 # LANGUAGE=c,c++,objective-c,objective-c++,swift
  88 # CC=clang,gcc-4.2,llvm-gcc-4.2
  89 # MEM=mrc,arc,gc
  90 # STDLIB=libc++,libstdc++
  91 # GUARDMALLOC=0,1
  92
  93 # things you can set once on the command line
  94 # ROOT=/path/to/project.roots
  95 # BUILD=0|1
  96 # RUN=0|1
  97 # VERBOSE=0|1
  98
  99
 100
 101 my $BUILD;
 102 my $RUN;
 103 my $VERBOSE;
 104
 105 my $crashcatch = <<'END';
 106 // interpose-able code to catch crashes, print, and exit cleanly
 107 #include <signal.h>
 108 #include <string.h>
 109 #include <unistd.h>
 110
 111 // from dyld-interposing.h
 112 #define DYLD_INTERPOSE(_replacement,_replacee) __attribute__((used)) static struct{ const void* replacement; const void* replacee; } _interpose_##_replacee __attribute__ ((section ("__DATA,__interpose"))) = { (const void*)(unsigned long)&_replacement, (const void*)(unsigned long)&_replacee };
 113
 114 static void catchcrash(int sig)
 115 {
 116     const char *msg;
 117     switch (sig) {
 118     case SIGILL:  msg = "CRASHED: SIGILL\\n";  break;
 119     case SIGBUS:  msg = "CRASHED: SIGBUS\\n";  break;
 120     case SIGSYS:  msg = "CRASHED: SIGSYS\\n";  break;
 121     case SIGSEGV: msg = "CRASHED: SIGSEGV\\n"; break;
 122     case SIGTRAP: msg = "CRASHED: SIGTRAP\\n"; break;
 123     case SIGABRT: msg = "CRASHED: SIGABRT\\n"; break;
 124     default: msg = "SIG\?\?\?\?\\n"; break;
 125     }
 126     write(STDERR_FILENO, msg, strlen(msg));
 127     _exit(0);
 128 }
 129
 130 static void setupcrash(void) __attribute__((constructor));
 131 static void setupcrash(void)
 132 {
 133     signal(SIGILL, &catchcrash);
 134     signal(SIGBUS, &catchcrash);
 135     signal(SIGSYS, &catchcrash);
 136     signal(SIGSEGV, &catchcrash);
 137     signal(SIGTRAP, &catchcrash);
 138     signal(SIGABRT, &catchcrash);
 139 }
 140
 141
 142 static int hacked = 0;
 143 ssize_t hacked_write(int fildes, const void *buf, size_t nbyte)
 144 {
 145     if (!hacked) {
 146         setupcrash();
 147         hacked = 1;
 148     }
 149     return write(fildes, buf, nbyte);
 150 }
 151
 152 DYLD_INTERPOSE(hacked_write, write);
 153
 154 END
 155
 156
 157 #########################################################################
 158 ## Harness
 159
 160
 161 # map language to buildable extensions for that language
 162 my %extensions_for_language = (
 163     "c"     => ["c"],
 164     "objective-c" => ["c", "m"],
 165     "c++" => ["c", "cc", "cp", "cpp", "cxx", "c++"],
 166     "objective-c++" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"],
 167     "swift" => ["swift"],
 168
 169     "any" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm", "swift"],
 170     );
 171
 172 # map extension to languages
 173 my %languages_for_extension = (
 174     "c" => ["c", "objective-c", "c++", "objective-c++"],
 175     "m" => ["objective-c", "objective-c++"],
 176     "mm" => ["objective-c++"],
 177     "cc" => ["c++", "objective-c++"],
 178     "cp" => ["c++", "objective-c++"],
 179     "cpp" => ["c++", "objective-c++"],
 180     "cxx" => ["c++", "objective-c++"],
 181     "c++" => ["c++", "objective-c++"],
 182     "swift" => ["swift"],
 183     );
 184
 185 # Run some newline-separated commands like `make` would, stopping if any fail
 186 # run("cmd1 \n cmd2 \n cmd3")
 187 sub make {
 188     my $output = "";
 189     my @cmds = split("\n", $_[0]);
 190     die if scalar(@cmds) == 0;
 191     $? = 0;
 192     foreach my $cmd (@cmds) {
 193         chomp $cmd;
 194         next if $cmd =~ /^\s*$/;
 195         $cmd .= " 2>&1";
 196         print "$cmd\n" if $VERBOSE;
 197         $output .= `$cmd`;
 198         last if $?;
 199     }
 200     print "$output\n" if $VERBOSE;
 201     return $output;
 202 }
 203
 204 sub chdir_verbose {
 205     my $dir = shift;
 206     chdir $dir || die;
 207     print "cd $dir\n" if $VERBOSE;
 208 }
 209
 210
 211 # Return test names from the command line.
 212 # Returns all tests if no tests were named.
 213 sub gettests {
 214     my @tests;
 215
 216     foreach my $arg (@ARGV) {
 217         push @tests, $arg  if ($arg !~ /=/  &&  $arg !~ /^-/);
 218     }
 219
 220     opendir(my $dir, $DIR) || die;
 221     while (my $file = readdir($dir)) {
 222         my ($name, $ext) = ($file =~ /^([^.]+)\.([^.]+)$/);
 223         next if ! $languages_for_extension{$ext};
 224
 225         open(my $in, "< $file") || die "$file";
 226         my $contents = join "", <$in>;
 227         if (defined $ALL_TESTS{$name}) {
 228             print "${yellow}SKIP: multiple tests named '$name'; skipping file '$file'.${def}\n";
 229         } else {
 230             $ALL_TESTS{$name} = $ext  if ($contents =~ m#^[/*\s]*TEST_#m);
 231         }
 232         close($in);
 233     }
 234     closedir($dir);
 235
 236     if (scalar(@tests) == 0) {
 237         @tests = keys %ALL_TESTS;
 238     }
 239
 240     @tests = sort @tests;
 241
 242     return @tests;
 243 }
 244
 245
 246 # Turn a C compiler name into a C++ compiler name.
 247 sub cplusplus {
 248     my ($c) = @_;
 249     if ($c =~ /cc/) {
 250         $c =~ s/cc/\+\+/;
 251         return $c;
 252     }
 253     return $c . "++";                         # e.g. clang => clang++
 254 }
 255
 256 # Turn a C compiler name into a Swift compiler name
 257 sub swift {
 258     my ($c) = @_;
 259     $c =~ s#[^/]*$#swift#;
 260     return $c;
 261 }
 262
 263 # Returns an array of all sdks from `xcodebuild -showsdks`
 264 my @sdks_memo;
 265 sub getsdks {
 266     if (!@sdks_memo) {
 267         @sdks_memo = (`xcodebuild -showsdks` =~ /-sdk (.+)$/mg);
 268     }
 269     return @sdks_memo;
 270 }
 271
 272 # Returns whether the given sdk supports -lauto
 273 sub supportslibauto {
 274     my ($sdk) = @_;
 275     return 1 if $sdk =~ /^macosx/;
 276     return 0 if $sdk =~ /^iphone/;
 277     die;
 278 }
 279
 280 # print text with a colored prefix on each line
 281 sub colorprint {
 282     my $color = shift;
 283     while (my @lines = split("\n", shift)) {
 284         for my $line (@lines) {
 285             chomp $line;
 286             print "$color $def$line\n";
 287         }
 288     }
 289 }
 290
 291 sub rewind {
 292     seek($_[0], 0, 0);
 293 }
 294
 295 # parse name=value,value pairs
 296 sub readconditions {
 297     my ($conditionstring) = @_;
 298
 299     my %results;
 300     my @conditions = ($conditionstring =~ /\w+=(?:[^\s,]+,?)+/g);
 301     for my $condition (@conditions) {
 302         my ($name, $values) = ($condition =~ /(\w+)=(.+)/);
 303         $results{$name} = [split ',', $values];
 304     }
 305
 306     return %results;
 307 }
 308
 309 # Get the name of the system SDK from sw_vers
 310 sub systemsdkname {
 311     my @lines = `/usr/bin/sw_vers`;
 312     my $name;
 313     my $vers;
 314     for my $line (@lines) {
 315         ($name) = ($line =~ /^ProductName:\s+(.*)/)  if !$name;
 316         ($vers) = ($line =~ /^ProductVersion:\s+(.*)/)  if !$vers;
 317     }
 318
 319     $name =~ s/ //g;
 320     $name = lc($name);
 321     my $internal = "";
 322     if (-d "/usr/local/include/objc") {
 323         if ($name eq "macosx") {
 324             $internal = "internal";
 325         } else {
 326             $internal = ".internal";
 327         }
 328     }
 329     return $name . $vers . $internal;
 330 }
 331
 332 sub check_output {
 333     my %C = %{shift()};
 334     my $name = shift;
 335     my @output = @_;
 336
 337     my %T = %{$C{"TEST_$name"}};
 338
 339     # Quietly strip MallocScribble before saving the "original" output
 340     # because it is distracting.
 341     filter_malloc(\@output);
 342
 343     my @original_output = @output;
 344
 345     # Run result-checking passes, reducing @output each time
 346     my $xit = 1;
 347     my $bad = "";
 348     my $warn = "";
 349     my $runerror = $T{TEST_RUN_OUTPUT};
 350     filter_hax(\@output);
 351     filter_verbose(\@output);
 352     $warn = filter_warn(\@output);
 353     $bad |= filter_guardmalloc(\@output) if ($C{GUARDMALLOC});
 354     $bad |= filter_valgrind(\@output) if ($C{VALGRIND});
 355     $bad = filter_expected(\@output, \%C, $name) if ($bad eq "");
 356     $bad = filter_bad(\@output)  if ($bad eq "");
 357
 358     # OK line should be the only one left
 359     $bad = "(output not 'OK: $name')" if ($bad eq ""  &&  (scalar(@output) != 1  ||  $output[0] !~ /^OK: $name/));
 360
 361     if ($bad ne "") {
 362         print "${red}FAIL: /// test '$name' \\\\\\$def\n";
 363         colorprint($red, @original_output);
 364         print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
 365         print "${red}FAIL: $name: $bad$def\n";
 366         $xit = 0;
 367     }
 368     elsif ($warn ne "") {
 369         print "${yellow}PASS: /// test '$name' \\\\\\$def\n";
 370         colorprint($yellow, @original_output);
 371         print "${yellow}PASS: \\\\\\ test '$name' ///$def\n";
 372         print "PASS: $name (with warnings)\n";
 373     }
 374     else {
 375         print "PASS: $name\n";
 376     }
 377     return $xit;
 378 }
 379
 380 sub filter_expected
 381 {
 382     my $outputref = shift;
 383     my %C = %{shift()};
 384     my $name = shift;
 385
 386     my %T = %{$C{"TEST_$name"}};
 387     my $runerror = $T{TEST_RUN_OUTPUT}  ||  return "";
 388
 389     my $bad = "";
 390
 391     my $output = join("\n", @$outputref) . "\n";
 392     if ($output !~ /$runerror/) {
 393         $bad = "(run output does not match TEST_RUN_OUTPUT)";
 394         @$outputref = ("FAIL: $name");
 395     } else {
 396         @$outputref = ("OK: $name");  # pacify later filter
 397     }
 398
 399     return $bad;
 400 }
 401
 402 sub filter_bad
 403 {
 404     my $outputref = shift;
 405     my $bad = "";
 406
 407     my @new_output;
 408     for my $line (@$outputref) {
 409         if ($line =~ /^BAD: (.*)/) {
 410             $bad = "(failed)";
 411         } else {
 412             push @new_output, $line;
 413         }
 414     }
 415
 416     @$outputref = @new_output;
 417     return $bad;
 418 }
 419
 420 sub filter_warn
 421 {
 422     my $outputref = shift;
 423     my $warn = "";
 424
 425     my @new_output;
 426     for my $line (@$outputref) {
 427         if ($line !~ /^WARN: (.*)/) {
 428             push @new_output, $line;
 429         } else {
 430             $warn = "(warned)";
 431         }
 432     }
 433
 434     @$outputref = @new_output;
 435     return $warn;
 436 }
 437
 438 sub filter_verbose
 439 {
 440     my $outputref = shift;
 441
 442     my @new_output;
 443     for my $line (@$outputref) {
 444         if ($line !~ /^VERBOSE: (.*)/) {
 445             push @new_output, $line;
 446         }
 447     }
 448
 449     @$outputref = @new_output;
 450 }
 451
 452 sub filter_hax
 453 {
 454     my $outputref = shift;
 455
 456     my @new_output;
 457     for my $line (@$outputref) {
 458         if ($line !~ /Class OS_tcp_/) {
 459             push @new_output, $line;
 460         }
 461     }
 462
 463     @$outputref = @new_output;
 464 }
 465
 466 sub filter_valgrind
 467 {
 468     my $outputref = shift;
 469     my $errors = 0;
 470     my $leaks = 0;
 471
 472     my @new_output;
 473     for my $line (@$outputref) {
 474         if ($line =~ /^Approx: do_origins_Dirty\([RW]\): missed \d bytes$/) {
 475             # --track-origins warning (harmless)
 476             next;
 477         }
 478         if ($line =~ /^UNKNOWN __disable_threadsignal is unsupported. This warning will not be repeated.$/) {
 479             # signals unsupported (harmless)
 480             next;
 481         }
 482         if ($line =~ /^UNKNOWN __pthread_sigmask is unsupported. This warning will not be repeated.$/) {
 483             # signals unsupported (harmless)
 484             next;
 485         }
 486         if ($line !~ /^^\.*==\d+==/) {
 487             # not valgrind output
 488             push @new_output, $line;
 489             next;
 490         }
 491
 492         my ($errcount) = ($line =~ /==\d+== ERROR SUMMARY: (\d+) errors/);
 493         if (defined $errcount  &&  $errcount > 0) {
 494             $errors = 1;
 495         }
 496
 497         (my $leakcount) = ($line =~ /==\d+==\s+(?:definitely|possibly) lost:\s+([0-9,]+)/);
 498         if (defined $leakcount  &&  $leakcount > 0) {
 499             $leaks = 1;
 500         }
 501     }
 502
 503     @$outputref = @new_output;
 504
 505     my $bad = "";
 506     $bad .= "(valgrind errors)" if ($errors);
 507     $bad .= "(valgrind leaks)" if ($leaks);
 508     return $bad;
 509 }
 510
 511
 512
 513 sub filter_malloc
 514 {
 515     my $outputref = shift;
 516     my $errors = 0;
 517
 518     my @new_output;
 519     my $count = 0;
 520     for my $line (@$outputref) {
 521         # Ignore MallocScribble prologue.
 522         # Ignore MallocStackLogging prologue.
 523         if ($line =~ /malloc: enabling scribbling to detect mods to free/  ||
 524             $line =~ /Deleted objects will be dirtied by the collector/  ||
 525             $line =~ /malloc: stack logs being written into/  ||
 526             $line =~ /malloc: recording malloc and VM allocation stacks/)
 527         {
 528             next;
 529         }
 530
 531         # not malloc output
 532         push @new_output, $line;
 533
 534     }
 535
 536     @$outputref = @new_output;
 537 }
 538
 539 sub filter_guardmalloc
 540 {
 541     my $outputref = shift;
 542     my $errors = 0;
 543
 544     my @new_output;
 545     my $count = 0;
 546     for my $line (@$outputref) {
 547         if ($line !~ /^GuardMalloc\[[^\]]+\]: /) {
 548             # not guardmalloc output
 549             push @new_output, $line;
 550             next;
 551         }
 552
 553         # Ignore 4 lines of guardmalloc prologue.
 554         # Anything further is a guardmalloc error.
 555         if (++$count > 4) {
 556             $errors = 1;
 557         }
 558     }
 559
 560     @$outputref = @new_output;
 561
 562     my $bad = "";
 563     $bad .= "(guardmalloc errors)" if ($errors);
 564     return $bad;
 565 }
 566
 567 # TEST_SOMETHING
 568 # text
 569 # text
 570 # END
 571 sub extract_multiline {
 572     my ($flag, $contents, $name) = @_;
 573     if ($contents =~ /$flag\n/) {
 574         my ($output) = ($contents =~ /$flag\n(.*?\n)END[ *\/]*\n/s);
 575         die "$name used $flag without END\n"  if !defined($output);
 576         return $output;
 577     }
 578     return undef;
 579 }
 580
 581
 582 # TEST_SOMETHING
 583 # text
 584 # OR
 585 # text
 586 # END
 587 sub extract_multiple_multiline {
 588     my ($flag, $contents, $name) = @_;
 589     if ($contents =~ /$flag\n/) {
 590         my ($output) = ($contents =~ /$flag\n(.*?\n)END[ *\/]*\n/s);
 591         die "$name used $flag without END\n"  if !defined($output);
 592
 593         $output =~ s/\nOR\n/\n|/sg;
 594         $output = "^(" . $output . ")\$";
 595         return $output;
 596     }
 597     return undef;
 598 }
 599
 600
 601 sub gather_simple {
 602     my $CREF = shift;
 603     my %C = %{$CREF};
 604     my $name = shift;
 605     chdir_verbose $DIR;
 606
 607     my $ext = $ALL_TESTS{$name};
 608     my $file = "$name.$ext";
 609     return 0 if !$file;
 610
 611     # search file for 'TEST_CONFIG' or '#include "test.h"'
 612     # also collect other values:
 613     # TEST_CONFIG test conditions
 614     # TEST_ENV environment prefix
 615     # TEST_CFLAGS compile flags
 616     # TEST_BUILD build instructions
 617     # TEST_BUILD_OUTPUT expected build stdout/stderr
 618     # TEST_RUN_OUTPUT expected run stdout/stderr
 619     open(my $in, "< $file") || die;
 620     my $contents = join "", <$in>;
 621
 622     my $test_h = ($contents =~ /^\s*#\s*(include|import)\s*"test\.h"/m);
 623     my $disabled = ($contents =~ /\bTEST_DISABLED\b/m);
 624     my $crashes = ($contents =~ /\bTEST_CRASHES\b/m);
 625     my ($conditionstring) = ($contents =~ /\bTEST_CONFIG\b(.*)$/m);
 626     my ($envstring) = ($contents =~ /\bTEST_ENV\b(.*)$/m);
 627     my ($cflags) = ($contents =~ /\bTEST_CFLAGS\b(.*)$/m);
 628     my ($buildcmd) = extract_multiline("TEST_BUILD", $contents, $name);
 629     my ($builderror) = extract_multiple_multiline("TEST_BUILD_OUTPUT", $contents, $name);
 630     my ($runerror) = extract_multiple_multiline("TEST_RUN_OUTPUT", $contents, $name);
 631
 632     return 0 if !$test_h && !$disabled && !$crashes && !defined($conditionstring) && !defined($envstring) && !defined($cflags) && !defined($buildcmd) && !defined($builderror) && !defined($runerror);
 633
 634     if ($disabled) {
 635         print "${yellow}SKIP: $name    (disabled by TEST_DISABLED)$def\n";
 636         return 0;
 637     }
 638
 639     # check test conditions
 640
 641     my $run = 1;
 642     my %conditions = readconditions($conditionstring);
 643     if (! $conditions{LANGUAGE}) {
 644         # implicit language restriction from file extension
 645         $conditions{LANGUAGE} = $languages_for_extension{$ext};
 646     }
 647     for my $condkey (keys %conditions) {
 648         my @condvalues = @{$conditions{$condkey}};
 649
 650         # special case: RUN=0 does not affect build
 651         if ($condkey eq "RUN"  &&  @condvalues == 1  &&  $condvalues[0] == 0) {
 652             $run = 0;
 653             next;
 654         }
 655
 656         my $testvalue = $C{$condkey};
 657         next if !defined($testvalue);
 658         # testvalue is the configuration being run now
 659         # condvalues are the allowed values for this test
 660
 661         my $ok = 0;
 662         for my $condvalue (@condvalues) {
 663
 664             # special case: objc and objc++
 665             if ($condkey eq "LANGUAGE") {
 666                 $condvalue = "objective-c" if $condvalue eq "objc";
 667                 $condvalue = "objective-c++" if $condvalue eq "objc++";
 668             }
 669
 670             $ok = 1  if ($testvalue eq $condvalue);
 671
 672             # special case: SDK allows prefixes
 673             if ($condkey eq "SDK") {
 674                 $ok = 1  if ($testvalue =~ /^$condvalue/);
 675             }
 676
 677             # special case: CC and CXX allow substring matches
 678             if ($condkey eq "CC"  ||  $condkey eq "CXX") {
 679                 $ok = 1  if ($testvalue =~ /$condvalue/);
 680             }
 681
 682             last if $ok;
 683         }
 684
 685         if (!$ok) {
 686             my $plural = (@condvalues > 1) ? "one of: " : "";
 687             print "SKIP: $name    ($condkey=$testvalue, but test requires $plural", join(' ', @condvalues), ")\n";
 688             return 0;
 689         }
 690     }
 691
 692     # save some results for build and run phases
 693     $$CREF{"TEST_$name"} = {
 694         TEST_BUILD => $buildcmd,
 695         TEST_BUILD_OUTPUT => $builderror,
 696         TEST_CRASHES => $crashes,
 697         TEST_RUN_OUTPUT => $runerror,
 698         TEST_CFLAGS => $cflags,
 699         TEST_ENV => $envstring,
 700         TEST_RUN => $run,
 701     };
 702
 703     return 1;
 704 }
 705
 706 # Builds a simple test
 707 sub build_simple {
 708     my %C = %{shift()};
 709     my $name = shift;
 710     my %T = %{$C{"TEST_$name"}};
 711     chdir_verbose "$C{DIR}/$name.build";
 712
 713     my $ext = $ALL_TESTS{$name};
 714     my $file = "$DIR/$name.$ext";
 715
 716     if ($T{TEST_CRASHES}) {
 717         `echo '$crashcatch' > crashcatch.c`;
 718         make("$C{COMPILE_C} -dynamiclib -o libcrashcatch.dylib -x c crashcatch.c");
 719         die "$?" if $?;
 720     }
 721
 722     my $cmd = $T{TEST_BUILD} ? eval "return \"$T{TEST_BUILD}\"" : "$C{COMPILE}   $T{TEST_CFLAGS} $file -o $name.out";
 723
 724     my $output = make($cmd);
 725
 726     # rdar://10163155
 727     $output =~ s/ld: warning: could not create compact unwind for [^\n]+: does not use standard frame\n//g;
 728
 729     my $ok;
 730     if (my $builderror = $T{TEST_BUILD_OUTPUT}) {
 731         # check for expected output and ignore $?
 732         if ($output =~ /$builderror/) {
 733             $ok = 1;
 734         } else {
 735             print "${red}FAIL: /// test '$name' \\\\\\$def\n";
 736             colorprint $red, $output;
 737             print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
 738             print "${red}FAIL: $name (build output does not match TEST_BUILD_OUTPUT)$def\n";
 739             $ok = 0;
 740         }
 741     } elsif ($?) {
 742         print "${red}FAIL: /// test '$name' \\\\\\$def\n";
 743         colorprint $red, $output;
 744         print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
 745         print "${red}FAIL: $name (build failed)$def\n";
 746         $ok = 0;
 747     } elsif ($output ne "") {
 748         print "${red}FAIL: /// test '$name' \\\\\\$def\n";
 749         colorprint $red, $output;
 750         print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
 751         print "${red}FAIL: $name (unexpected build output)$def\n";
 752         $ok = 0;
 753     } else {
 754         $ok = 1;
 755     }
 756
 757
 758     if ($ok) {
 759         foreach my $file (glob("*.out *.dylib *.bundle")) {
 760             make("dsymutil $file");
 761         }
 762     }
 763
 764     return $ok;
 765 }
 766
 767 # Run a simple test (testname.out, with error checking of stdout and stderr)
 768 sub run_simple {
 769     my %C = %{shift()};
 770     my $name = shift;
 771     my %T = %{$C{"TEST_$name"}};
 772
 773     if (! $T{TEST_RUN}) {
 774         print "PASS: $name (build only)\n";
 775         return 1;
 776     }
 777     else {
 778         chdir_verbose "$C{DIR}/$name.build";
 779     }
 780
 781     my $env = "$C{ENV} $T{TEST_ENV}";
 782     if ($T{TEST_CRASHES}) {
 783         $env .= " DYLD_INSERT_LIBRARIES=libcrashcatch.dylib";
 784     }
 785
 786     my $output;
 787
 788     if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) {
 789         # run on iOS device
 790
 791         my $remotedir = "/var/root/test/" . basename($C{DIR}) . "/$name.build";
 792         my $remotedyld = "";
 793         $remotedyld .= " DYLD_LIBRARY_PATH=$remotedir";
 794         $remotedyld .= ":/var/root/test/"  if ($C{TESTLIB} ne $TESTLIBPATH);
 795
 796         # elide host-specific paths
 797         $env =~ s/DYLD_LIBRARY_PATH=\S+//;
 798         $env =~ s/DYLD_ROOT_PATH=\S+//;
 799
 800         my $cmd = "ssh iphone 'cd $remotedir && env $env $remotedyld ./$name.out'";
 801         $output = make("$cmd");
 802     }
 803     else {
 804         # run locally
 805
 806         my $cmd = "env $env ./$name.out";
 807         $output = make("sh -c '$cmd 2>&1' 2>&1");
 808         # need extra sh level to capture "sh: Illegal instruction" after crash
 809         # fixme fail if $? except tests that expect to crash
 810     }
 811
 812     return check_output(\%C, $name, split("\n", $output));
 813 }
 814
 815
 816 my %compiler_memo;
 817 sub find_compiler {
 818     my ($cc, $sdk, $sdk_path) = @_;
 819
 820     # memoize
 821     my $key = $cc . ':' . $sdk;
 822     my $result = $compiler_memo{$key};
 823     return $result if defined $result;
 824
 825     $result  = `xcrun -sdk $sdk -find $cc 2>/dev/null`;
 826
 827     chomp $result;
 828     $compiler_memo{$key} = $result;
 829     return $result;
 830 }
 831
 832 sub make_one_config {
 833     my $configref = shift;
 834     my $root = shift;
 835     my %C = %{$configref};
 836
 837     # Aliases
 838     $C{LANGUAGE} = "objective-c"  if $C{LANGUAGE} eq "objc";
 839     $C{LANGUAGE} = "objective-c++"  if $C{LANGUAGE} eq "objc++";
 840
 841     # Look up SDK
 842     # Try exact match first.
 843     # Then try lexically-last prefix match (so "macosx" => "macosx10.7internal").
 844     my @sdks = getsdks();
 845     if ($VERBOSE) {
 846         print "Installed SDKs: @sdks\n";
 847     }
 848     my $exactsdk = undef;
 849     my $prefixsdk = undef;
 850     foreach my $sdk (@sdks) {
 851         my $SDK = $C{SDK};
 852         $exactsdk = $sdk  if ($sdk eq $SDK);
 853         # check for digits to prevent e.g. "iphone" => "iphonesimulator4.2"
 854         $prefixsdk = $sdk  if ($sdk =~ /^$SDK[0-9]/  &&  $sdk gt $prefixsdk);
 855     }
 856     if ($exactsdk) {
 857         $C{SDK} = $exactsdk;
 858     } elsif ($prefixsdk) {
 859         $C{SDK} = $prefixsdk;
 860     } else {
 861         die "unknown SDK '$C{SDK}'\nInstalled SDKs: @sdks\n";
 862     }
 863
 864     # set the config name now, after massaging the language and sdk,
 865     # but before adding other settings
 866     my $configname = config_name(%C);
 867     die if ($configname =~ /'/);
 868     die if ($configname =~ / /);
 869     ($C{NAME} = $configname) =~ s/~/ /g;
 870     (my $configdir = $configname) =~ s#/##g;
 871     $C{DIR} = "$BUILDDIR/$configdir";
 872
 873     ($C{SDK_PATH}) = (`xcodebuild -version -sdk $C{SDK} Path` =~ /^\s*(.+?)\s*$/);
 874
 875     # Look up test library (possible in root or SDK_PATH)
 876
 877     my $rootarg = $root;
 878     my $symroot;
 879     my @sympaths = ( (glob "$root/*~sym")[0],
 880                      (glob "$root/BuildRecords/*_install/Symbols")[0],
 881                      "$root/Symbols" );
 882     my @dstpaths = ( (glob "$root/*~dst")[0],
 883                      (glob "$root/BuildRecords/*_install/Root")[0],
 884                      "$root/Root" );
 885     for(my $i = 0; $i < scalar(@sympaths); $i++) {
 886         if (-e $sympaths[$i]  &&  -e $dstpaths[$i]) {
 887             $symroot = $sympaths[$i];
 888             $root = $dstpaths[$i];
 889             last;
 890         }
 891     }
 892
 893     if ($root ne ""  &&  -e "$root$C{SDK_PATH}$TESTLIBPATH") {
 894         $C{TESTLIB} = "$root$C{SDK_PATH}$TESTLIBPATH";
 895     } elsif (-e "$root$TESTLIBPATH") {
 896         $C{TESTLIB} = "$root$TESTLIBPATH";
 897     } elsif (-e "$root/$TESTLIBNAME") {
 898         $C{TESTLIB} = "$root/$TESTLIBNAME";
 899     } else {
 900         die "No $TESTLIBNAME in root '$rootarg' for sdk '$C{SDK_PATH}'\n"
 901             # . join("\n", @dstpaths) . "\n"
 902             ;
 903     }
 904
 905     if (-e "$symroot/$TESTLIBNAME.dSYM") {
 906         $C{TESTDSYM} = "$symroot/$TESTLIBNAME.dSYM";
 907     }
 908
 909     if ($VERBOSE) {
 910         my @uuids = `/usr/bin/dwarfdump -u '$C{TESTLIB}'`;
 911         while (my $uuid = shift @uuids) {
 912             print "note: $uuid";
 913         }
 914     }
 915
 916     # Look up compilers
 917     my $cc = $C{CC};
 918     my $cxx = cplusplus($C{CC});
 919     my $swift = swift($C{CC});
 920     if (! $BUILD) {
 921         $C{CC} = $cc;
 922         $C{CXX} = $cxx;
 923         $C{SWIFT} = $swift
 924     } else {
 925         $C{CC} = find_compiler($cc, $C{SDK}, $C{SDK_PATH});
 926         $C{CXX} = find_compiler($cxx, $C{SDK}, $C{SDK_PATH});
 927         $C{SWIFT} = find_compiler($swift, $C{SDK}, $C{SDK_PATH});
 928
 929         die "No compiler '$cc' ('$C{CC}') in SDK '$C{SDK}'\n" if !-e $C{CC};
 930         die "No compiler '$cxx' ('$C{CXX}') in SDK '$C{SDK}'\n" if !-e $C{CXX};
 931         die "No compiler '$swift' ('$C{SWIFT}') in SDK '$C{SDK}'\n" if !-e $C{SWIFT};
 932     }
 933
 934     # Populate cflags
 935
 936     # save-temps so dsymutil works so debug info works
 937     my $cflags = "-I$DIR -W -Wall -Wno-deprecated-declarations -Wshorten-64-to-32 -g -save-temps -Os -arch $C{ARCH} ";
 938     my $objcflags = "";
 939     my $swiftflags = "-g ";
 940
 941     $cflags .= " -isysroot '$C{SDK_PATH}'";
 942     $cflags .= " '-Wl,-syslibroot,$C{SDK_PATH}'";
 943     $swiftflags .= " -sdk '$C{SDK_PATH}'";
 944
 945     my $target = "";
 946     if ($C{SDK} =~ /^iphoneos[0-9]/  &&  $cflags !~ /-mios-version-min/) {
 947         my ($vers) = ($C{SDK} =~ /^iphoneos([0-9]+\.[0-9]+)/);
 948         $cflags .= " -mios-version-min=$vers";
 949         $target = "$C{ARCH}-apple-ios$vers";
 950     }
 951     elsif ($C{SDK} =~ /^iphonesimulator[0-9]/  &&  $cflags !~ /-mios-simulator-version-min/) {
 952         my ($vers) = ($C{SDK} =~ /^iphonesimulator([0-9]+\.[0-9]+)/);
 953         $cflags .= " -mios-simulator-version-min=$vers";
 954         $target = "$C{ARCH}-apple-ios$vers";
 955     }
 956     else {
 957         my ($vers) = ($C{SDK} =~ /^macosx([0-9]+\.[0-9]+)/);
 958         $vers = ""  if !defined($vers);
 959         $target = "$C{ARCH}-apple-macosx$vers";
 960     }
 961     $swiftflags .= " -target $target";
 962
 963     if ($C{SDK} =~ /^iphonesimulator/  &&  $C{ARCH} eq "i386") {
 964         $objcflags .= " -fobjc-abi-version=2 -fobjc-legacy-dispatch";
 965     }
 966
 967     if ($root ne "") {
 968         my $library_path = dirname($C{TESTLIB});
 969         $cflags .= " -L$library_path";
 970         $cflags .= " -I '$root/usr/include'";
 971         $cflags .= " -I '$root/usr/local/include'";
 972
 973         if ($C{SDK_PATH} ne "/") {
 974             $cflags .= " -I '$root$C{SDK_PATH}/usr/include'";
 975             $cflags .= " -I '$root$C{SDK_PATH}/usr/local/include'";
 976         }
 977     }
 978
 979     if ($C{CC} =~ /clang/) {
 980         $cflags .= " -Qunused-arguments -fno-caret-diagnostics";
 981         $cflags .= " -stdlib=$C{STDLIB}"; # fixme -fno-objc-link-runtime"
 982         $cflags .= " -Wl,-segalign,0x4000 ";
 983     }
 984
 985
 986     # Populate objcflags
 987
 988     $objcflags .= " -lobjc";
 989     if ($C{MEM} eq "gc") {
 990         $objcflags .= " -fobjc-gc";
 991     }
 992     elsif ($C{MEM} eq "arc") {
 993         $objcflags .= " -fobjc-arc";
 994     }
 995     elsif ($C{MEM} eq "mrc") {
 996         # nothing
 997     }
 998     else {
 999         die "unrecognized MEM '$C{MEM}'\n";
1000     }
1001
1002     if (supportslibauto($C{SDK})) {
1003         # do this even for non-GC tests
1004         $objcflags .= " -lauto";
1005     }
1006
1007     # Populate ENV_PREFIX
1008     $C{ENV} = "LANG=C MallocScribble=1";
1009     $C{ENV} .= " VERBOSE=1"  if $VERBOSE;
1010     if ($root ne "") {
1011         my $library_path = dirname($C{TESTLIB});
1012         die "no spaces allowed in root" if $library_path =~ /\s+/;
1013         $C{ENV} .= " DYLD_LIBRARY_PATH=$library_path"  if ($library_path ne "/usr/lib");
1014     }
1015     if ($C{SDK_PATH} ne "/") {
1016         die "no spaces allowed in sdk" if $C{SDK_PATH} =~ /\s+/;
1017         $C{ENV} .= " DYLD_ROOT_PATH=$C{SDK_PATH}";
1018     }
1019     if ($C{GUARDMALLOC}) {
1020         $ENV{GUARDMALLOC} = "1";  # checked by tests and errcheck.pl
1021         $C{ENV} .= " DYLD_INSERT_LIBRARIES=/usr/lib/libgmalloc.dylib";
1022     }
1023     if ($C{SDK} =~ /^iphonesimulator[0-9]/) {
1024         my ($vers) = ($C{SDK} =~ /^iphonesimulator([0-9]+\.[0-9+])/);
1025         $C{ENV} .=
1026             " CFFIXED_USER_HOME=$ENV{HOME}/Library/Application\\ Support/iPhone\\ Simulator/$vers" .
1027             " IPHONE_SIMULATOR_ROOT=$C{SDK_PATH}" .
1028             " IPHONE_SHARED_RESOURCES_DIRECTORY=$ENV{HOME}/Library/Application\\ Support/iPhone\\ Simulator/$vers";
1029     }
1030
1031     # Populate compiler commands
1032     $C{COMPILE_C}   = "env LANG=C '$C{CC}'  $cflags -x c -std=gnu99";
1033     $C{COMPILE_CXX} = "env LANG=C '$C{CXX}' $cflags -x c++";
1034     $C{COMPILE_M}   = "env LANG=C '$C{CC}'  $cflags $objcflags -x objective-c -std=gnu99";
1035     $C{COMPILE_MM}  = "env LANG=C '$C{CXX}' $cflags $objcflags -x objective-c++";
1036     $C{COMPILE_SWIFT} = "env LANG=C '$C{SWIFT}' $swiftflags";
1037
1038     $C{COMPILE} = $C{COMPILE_C}      if $C{LANGUAGE} eq "c";
1039     $C{COMPILE} = $C{COMPILE_CXX}    if $C{LANGUAGE} eq "c++";
1040     $C{COMPILE} = $C{COMPILE_M}      if $C{LANGUAGE} eq "objective-c";
1041     $C{COMPILE} = $C{COMPILE_MM}     if $C{LANGUAGE} eq "objective-c++";
1042     $C{COMPILE} = $C{COMPILE_SWIFT}  if $C{LANGUAGE} eq "swift";
1043     die "unknown language '$C{LANGUAGE}'\n" if !defined $C{COMPILE};
1044
1045     ($C{COMPILE_NOMEM} = $C{COMPILE}) =~ s/ -fobjc-(?:gc|arc)\S*//g;
1046     ($C{COMPILE_NOLINK} = $C{COMPILE}) =~ s/ '?-(?:Wl,|l)\S*//g;
1047     ($C{COMPILE_NOLINK_NOMEM} = $C{COMPILE_NOMEM}) =~ s/ '?-(?:Wl,|l)\S*//g;
1048
1049
1050     # Reject some self-inconsistent configurations
1051     if ($C{MEM} !~ /^(mrc|arc|gc)$/) {
1052         die "unknown MEM=$C{MEM} (expected one of mrc arc gc)\n";
1053     }
1054
1055     if ($C{MEM} eq "gc"  &&  $C{SDK} =~ /^iphone/) {
1056         print "note: skipping configuration $C{NAME}\n";
1057         print "note:   because SDK=$C{SDK} does not support MEM=$C{MEM}\n";
1058         return 0;
1059     }
1060     if ($C{MEM} eq "arc"  &&  $C{SDK} !~ /^iphone/  &&  $C{ARCH} eq "i386") {
1061         print "note: skipping configuration $C{NAME}\n";
1062         print "note:   because 32-bit Mac does not support MEM=$C{MEM}\n";
1063         return 0;
1064     }
1065     if ($C{MEM} eq "arc"  &&  $C{CC} !~ /clang/) {
1066         print "note: skipping configuration $C{NAME}\n";
1067         print "note:   because CC=$C{CC} does not support MEM=$C{MEM}\n";
1068         return 0;
1069     }
1070
1071     if ($C{STDLIB} ne "libstdc++"  &&  $C{CC} !~ /clang/) {
1072         print "note: skipping configuration $C{NAME}\n";
1073         print "note:   because CC=$C{CC} does not support STDLIB=$C{STDLIB}\n";
1074         return 0;
1075     }
1076
1077     # fixme
1078     if ($C{LANGUAGE} eq "swift"  &&  $C{ARCH} =~ /^arm/) {
1079         print "note: skipping configuration $C{NAME}\n";
1080         print "note:   because ARCH=$C{ARCH} does not support LANGAUGE=SWIFT\n";
1081         return 0;
1082     }
1083
1084     %$configref = %C;
1085 }
1086
1087 sub make_configs {
1088     my ($root, %args) = @_;
1089
1090     my @results = ({});  # start with one empty config
1091
1092     for my $key (keys %args) {
1093         my @newresults;
1094         my @values = @{$args{$key}};
1095         for my $configref (@results) {
1096             my %config = %{$configref};
1097             for my $value (@values) {
1098                 my %newconfig = %config;
1099                 $newconfig{$key} = $value;
1100                 push @newresults, \%newconfig;
1101             }
1102         }
1103         @results = @newresults;
1104     }
1105
1106     my @newresults;
1107     for my $configref(@results) {
1108         if (make_one_config($configref, $root)) {
1109             push @newresults, $configref;
1110         }
1111     }
1112
1113     return @newresults;
1114 }
1115
1116 sub config_name {
1117     my %config = @_;
1118     my $name = "";
1119     for my $key (sort keys %config) {
1120         $name .= '~'  if $name ne "";
1121         $name .= "$key=$config{$key}";
1122     }
1123     return $name;
1124 }
1125
1126 sub run_one_config {
1127     my %C = %{shift()};
1128     my @tests = @_;
1129
1130     # Build and run
1131     my $testcount = 0;
1132     my $failcount = 0;
1133
1134     my @gathertests;
1135     foreach my $test (@tests) {
1136         if ($VERBOSE) {
1137             print "\nGATHER $test\n";
1138         }
1139
1140         if ($ALL_TESTS{$test}) {
1141             gather_simple(\%C, $test) || next;  # not pass, not fail
1142             push @gathertests, $test;
1143         } else {
1144             die "No test named '$test'\n";
1145         }
1146     }
1147
1148     my @builttests;
1149     if (!$BUILD) {
1150         @builttests = @gathertests;
1151         $testcount = scalar(@gathertests);
1152     } else {
1153         my $configdir = $C{DIR};
1154         print $configdir, "\n"  if $VERBOSE;
1155         mkdir $configdir  || die;
1156
1157         foreach my $test (@gathertests) {
1158             if ($VERBOSE) {
1159                 print "\nBUILD $test\n";
1160             }
1161             mkdir "$configdir/$test.build"  || die;
1162
1163             if ($ALL_TESTS{$test}) {
1164                 $testcount++;
1165                 if (!build_simple(\%C, $test)) {
1166                     $failcount++;
1167                 } else {
1168                     push @builttests, $test;
1169                 }
1170             } else {
1171                 die "No test named '$test'\n";
1172             }
1173         }
1174     }
1175
1176     if (!$RUN  ||  !scalar(@builttests)) {
1177         # nothing to do
1178     }
1179     else {
1180         if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) {
1181             # upload all tests to iOS device
1182             make("RSYNC_PASSWORD=alpine rsync -av $C{DIR} rsync://root\@localhost:10873/root/var/root/test/");
1183             die "Couldn't rsync tests to device\n" if ($?);
1184
1185             # upload library to iOS device
1186             if ($C{TESTLIB} ne $TESTLIBPATH) {
1187                 make("RSYNC_PASSWORD=alpine rsync -av $C{TESTLIB} rsync://root\@localhost:10873/root/var/root/test/");
1188                 die "Couldn't rsync $C{TESTLIB} to device\n" if ($?);
1189                 make("RSYNC_PASSWORD=alpine rsync -av $C{TESTDSYM} rsync://root\@localhost:10873/root/var/root/test/");
1190             }
1191         }
1192
1193         foreach my $test (@builttests) {
1194             print "\nRUN $test\n"  if ($VERBOSE);
1195
1196             if ($ALL_TESTS{$test})
1197             {
1198                 if (!run_simple(\%C, $test)) {
1199                     $failcount++;
1200                 }
1201             } else {
1202                 die "No test named '$test'\n";
1203             }
1204         }
1205     }
1206
1207     return ($testcount, $failcount);
1208 }
1209
1210
1211
1212 # Return value if set by "$argname=value" on the command line
1213 # Return $default if not set.
1214 sub getargs {
1215     my ($argname, $default) = @_;
1216
1217     foreach my $arg (@ARGV) {
1218         my ($value) = ($arg =~ /^$argname=(.+)$/);
1219         return [split ',', $value] if defined $value;
1220     }
1221
1222     return [split ',', $default];
1223 }
1224
1225 # Return 1 or 0 if set by "$argname=1" or "$argname=0" on the
1226 # command line. Return $default if not set.
1227 sub getbools {
1228     my ($argname, $default) = @_;
1229
1230     my @values = @{getargs($argname, $default)};
1231     return [( map { ($_ eq "0") ? 0 : 1 } @values )];
1232 }
1233
1234 sub getarg {
1235     my ($argname, $default) = @_;
1236     my @values = @{getargs($argname, $default)};
1237     die "Only one value allowed for $argname\n"  if @values > 1;
1238     return $values[0];
1239 }
1240
1241 sub getbool {
1242     my ($argname, $default) = @_;
1243     my @values = @{getbools($argname, $default)};
1244     die "Only one value allowed for $argname\n"  if @values > 1;
1245     return $values[0];
1246 }
1247
1248
1249 # main
1250 my %args;
1251
1252
1253 my $default_arch = (`/usr/sbin/sysctl hw.optional.x86_64` eq "hw.optional.x86_64: 1\n") ? "x86_64" : "i386";
1254 $args{ARCH} = getargs("ARCH", 0);
1255 $args{ARCH} = getargs("ARCHS", $default_arch)  if !@{$args{ARCH}}[0];
1256
1257 $args{SDK} = getargs("SDK", "macosx");
1258
1259 $args{MEM} = getargs("MEM", "mrc");
1260 $args{LANGUAGE} = [ map { lc($_) } @{getargs("LANGUAGE", "objective-c,swift")} ];
1261 $args{STDLIB} = getargs("STDLIB", "libstdc++");
1262
1263 $args{CC} = getargs("CC", "clang");
1264
1265 $args{GUARDMALLOC} = getbools("GUARDMALLOC", 0);
1266
1267 $BUILD = getbool("BUILD", 1);
1268 $RUN = getbool("RUN", 1);
1269 $VERBOSE = getbool("VERBOSE", 0);
1270
1271 my $root = getarg("ROOT", "");
1272 $root =~ s#/*$##;
1273
1274 my @tests = gettests();
1275
1276 print "note: -----\n";
1277 print "note: testing root '$root'\n";
1278
1279 my @configs = make_configs($root, %args);
1280
1281 print "note: -----\n";
1282 print "note: testing ", scalar(@configs), " configurations:\n";
1283 for my $configref (@configs) {
1284     my $configname = $$configref{NAME};
1285     print "note: configuration $configname\n";
1286 }
1287
1288 if ($BUILD) {
1289     `rm -rf '$BUILDDIR'`;
1290     mkdir "$BUILDDIR" || die;
1291 }
1292
1293 my $failed = 0;
1294
1295 my $testconfigs = @configs;
1296 my $failconfigs = 0;
1297 my $testcount = 0;
1298 my $failcount = 0;
1299 for my $configref (@configs) {
1300     my $configname = $$configref{NAME};
1301     print "note: -----\n";
1302     print "note: \nnote: $configname\nnote: \n";
1303
1304     (my $t, my $f) = eval { run_one_config($configref, @tests); };
1305     if ($@) {
1306         chomp $@;
1307         print "${red}FAIL: $configname${def}\n";
1308         print "${red}FAIL: $@${def}\n";
1309         $failconfigs++;
1310     } else {
1311         my $color = ($f ? $red : "");
1312         print "note:\n";
1313         print "${color}note: $configname$def\n";
1314         print "${color}note: $t tests, $f failures$def\n";
1315         $testcount += $t;
1316         $failcount += $f;
1317         $failconfigs++ if ($f);
1318     }
1319 }
1320
1321 print "note: -----\n";
1322 my $color = ($failconfigs ? $red : "");
1323 print "${color}note: $testconfigs configurations, $failconfigs with failures$def\n";
1324 print "${color}note: $testcount tests, $failcount failures$def\n";
1325
1326 $failed = ($failconfigs ? 1 : 0);
1327
1328 exit ($failed ? 1 : 0);