test/test.pl

   1 #!/usr/bin/perl
   2
   3 # test.pl
   4 # Run unit tests.
   5
   6 use strict;
   7 use File::Basename;
   8
   9 chdir dirname $0;
  10 chomp (my $DIR = `pwd`);
  11
  12 my $TESTLIBNAME = "libobjc.A.dylib";
  13 my $TESTLIBPATH = "/usr/lib/$TESTLIBNAME";
  14
  15 my $BUILDDIR = "/tmp/test-$TESTLIBNAME-build";
  16
  17 # xterm colors
  18 my $red = "\e[41;37m";
  19 my $yellow = "\e[43;37m";
  20 my $def = "\e[0m";
  21
  22 # clean, help
  23 if (scalar(@ARGV) == 1) {
  24     my $arg = $ARGV[0];
  25     if ($arg eq "clean") {
  26         my $cmd = "rm -rf $BUILDDIR *~";
  27         print "$cmd\n";
  28         `$cmd`;
  29         exit 0;
  30     }
  31     elsif ($arg eq "-h" || $arg eq "-H" || $arg eq "-help" || $arg eq "help") {
  32         print(<<END);
  33 usage: $0 [options] [testname ...]
  34        $0 clean
  35        $0 help
  36
  37 testname:
  38     `testname` runs a specific test. If no testnames are given, runs all tests.
  39
  40 options:
  41     ARCH=<arch>
  42     SDK=<sdk name>
  43     ROOT=/path/to/project.roots/
  44
  45     CC=<compiler name>
  46
  47     MEM=mrc,arc,gc
  48     STDLIB=libc++,libstdc++
  49     GUARDMALLOC=0|1
  50
  51     BUILD=0|1
  52     RUN=0|1
  53     VERBOSE=0|1
  54
  55 examples:
  56
  57     test installed library, x86_64, no gc
  58     $0
  59
  60     test buildit-built root, i386 and x86_64, MRC and ARC and GC, clang compiler
  61     $0 ARCH=i386,x86_64 ROOT=/tmp/libclosure.roots MEM=mrc,arc,gc CC=clang
  62
  63     test buildit-built root with iOS simulator
  64     $0 ARCH=i386 ROOT=/tmp/libclosure.roots SDK=iphonesimulator
  65
  66     test buildit-built root on attached iOS device
  67     $0 ARCH=armv7 ROOT=/tmp/libclosure.roots SDK=iphoneos
  68 END
  69         exit 0;
  70     }
  71 }
  72
  73 #########################################################################
  74 ## Tests
  75
  76 my %ALL_TESTS;
  77
  78 #########################################################################
  79 ## Variables for use in complex build and run rules
  80
  81 # variable         # example value
  82
  83 # things you can multiplex on the command line
  84 # ARCH=i386,x86_64,armv6,armv7
  85 # SDK=system,macosx,iphoneos,iphonesimulator
  86 # LANGUAGE=c,c++,objective-c,objective-c++
  87 # CC=clang,gcc-4.2,llvm-gcc-4.2
  88 # MEM=mrc,arc,gc
  89 # STDLIB=libc++,libstdc++
  90 # GUARDMALLOC=0,1
  91
  92 # things you can set once on the command line
  93 # ROOT=/path/to/project.roots
  94 # BUILD=0|1
  95 # RUN=0|1
  96 # VERBOSE=0|1
  97
  98
  99
 100 my $BUILD;
 101 my $RUN;
 102 my $VERBOSE;
 103
 104 my $crashcatch = <<'END';
 105 // interpose-able code to catch crashes, print, and exit cleanly
 106 #include <signal.h>
 107 #include <string.h>
 108 #include <unistd.h>
 109
 110 // from dyld-interposing.h
 111 #define DYLD_INTERPOSE(_replacement,_replacee) __attribute__((used)) static struct{ const void* replacement; const void* replacee; } _interpose_##_replacee __attribute__ ((section ("__DATA,__interpose"))) = { (const void*)(unsigned long)&_replacement, (const void*)(unsigned long)&_replacee };
 112
 113 static void catchcrash(int sig)
 114 {
 115     const char *msg;
 116     switch (sig) {
 117     case SIGILL:  msg = "CRASHED: SIGILL\\n";  break;
 118     case SIGBUS:  msg = "CRASHED: SIGBUS\\n";  break;
 119     case SIGSYS:  msg = "CRASHED: SIGSYS\\n";  break;
 120     case SIGSEGV: msg = "CRASHED: SIGSEGV\\n"; break;
 121     case SIGTRAP: msg = "CRASHED: SIGTRAP\\n"; break;
 122     case SIGABRT: msg = "CRASHED: SIGABRT\\n"; break;
 123     default: msg = "SIG\?\?\?\?\\n"; break;
 124     }
 125     write(STDERR_FILENO, msg, strlen(msg));
 126     _exit(0);
 127 }
 128
 129 static void setupcrash(void) __attribute__((constructor));
 130 static void setupcrash(void)
 131 {
 132     signal(SIGILL, &catchcrash);
 133     signal(SIGBUS, &catchcrash);
 134     signal(SIGSYS, &catchcrash);
 135     signal(SIGSEGV, &catchcrash);
 136     signal(SIGTRAP, &catchcrash);
 137     signal(SIGABRT, &catchcrash);
 138 }
 139
 140
 141 static int hacked = 0;
 142 ssize_t hacked_write(int fildes, const void *buf, size_t nbyte)
 143 {
 144     if (!hacked) {
 145         setupcrash();
 146         hacked = 1;
 147     }
 148     return write(fildes, buf, nbyte);
 149 }
 150
 151 DYLD_INTERPOSE(hacked_write, write);
 152
 153 END
 154
 155
 156 #########################################################################
 157 ## Harness
 158
 159
 160 # map language to buildable extensions for that language
 161 my %extensions_for_language = (
 162     "c"     => ["c"],
 163     "objective-c" => ["c", "m"],
 164     "c++" => ["c", "cc", "cp", "cpp", "cxx", "c++"],
 165     "objective-c++" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"],
 166
 167     "any" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"],
 168     );
 169
 170 # map extension to languages
 171 my %languages_for_extension = (
 172     "c" => ["c", "objective-c", "c++", "objective-c++"],
 173     "m" => ["objective-c", "objective-c++"],
 174     "mm" => ["objective-c++"],
 175     "cc" => ["c++", "objective-c++"],
 176     "cp" => ["c++", "objective-c++"],
 177     "cpp" => ["c++", "objective-c++"],
 178     "cxx" => ["c++", "objective-c++"],
 179     "c++" => ["c++", "objective-c++"],
 180     );
 181
 182 # Run some newline-separated commands like `make` would, stopping if any fail
 183 # run("cmd1 \n cmd2 \n cmd3")
 184 sub make {
 185     my $output = "";
 186     my @cmds = split("\n", $_[0]);
 187     die if scalar(@cmds) == 0;
 188     $? = 0;
 189     foreach my $cmd (@cmds) {
 190         chomp $cmd;
 191         next if $cmd =~ /^\s*$/;
 192         $cmd .= " 2>&1";
 193         print "$cmd\n" if $VERBOSE;
 194         $output .= `$cmd`;
 195         last if $?;
 196     }
 197     print "$output\n" if $VERBOSE;
 198     return $output;
 199 }
 200
 201 sub chdir_verbose {
 202     my $dir = shift;
 203     chdir $dir || die;
 204     print "cd $dir\n" if $VERBOSE;
 205 }
 206
 207
 208 # Return test names from the command line.
 209 # Returns all tests if no tests were named.
 210 sub gettests {
 211     my @tests;
 212
 213     foreach my $arg (@ARGV) {
 214         push @tests, $arg  if ($arg !~ /=/  &&  $arg !~ /^-/);
 215     }
 216
 217     opendir(my $dir, $DIR) || die;
 218     while (my $file = readdir($dir)) {
 219         my ($name, $ext) = ($file =~ /^([^.]+)\.([^.]+)$/);
 220         next if ! $languages_for_extension{$ext};
 221
 222         open(my $in, "< $file") || die "$file";
 223         my $contents = join "", <$in>;
 224         die if defined $ALL_TESTS{$name};
 225         $ALL_TESTS{$name} = $ext  if ($contents =~ m#^[/*\s]*TEST_#m);
 226         close($in);
 227     }
 228     closedir($dir);
 229
 230     if (scalar(@tests) == 0) {
 231         @tests = keys %ALL_TESTS;
 232     }
 233
 234     @tests = sort @tests;
 235
 236     return @tests;
 237 }
 238
 239
 240 # Turn a C compiler name into a C++ compiler name.
 241 sub cplusplus {
 242     my ($c) = @_;
 243     if ($c =~ /cc/) {
 244         $c =~ s/cc/\+\+/;
 245         return $c;
 246     }
 247     return $c . "++";                         # e.g. clang => clang++
 248 }
 249
 250 # Returns an array of all sdks from `xcodebuild -showsdks`
 251 my @sdks_memo;
 252 sub getsdks {
 253     if (!@sdks_memo) {
 254         @sdks_memo = ("system", `xcodebuild -showsdks` =~ /-sdk (.+)$/mg);
 255     }
 256     return @sdks_memo;
 257 }
 258
 259 # Returns whether the given sdk supports -lauto
 260 sub supportslibauto {
 261     my ($sdk) = @_;
 262     return 1 if $sdk eq "system";
 263     return 1 if $sdk =~ /^macosx/;
 264     return 0 if $sdk =~ /^iphone/;
 265     die;
 266 }
 267
 268 # print text with a colored prefix on each line
 269 sub colorprint {
 270     my $color = shift;
 271     while (my @lines = split("\n", shift)) {
 272         for my $line (@lines) {
 273             chomp $line;
 274             print "$color $def$line\n";
 275         }
 276     }
 277 }
 278
 279 sub rewind {
 280     seek($_[0], 0, 0);
 281 }
 282
 283 # parse name=value,value pairs
 284 sub readconditions {
 285     my ($conditionstring) = @_;
 286
 287     my %results;
 288     my @conditions = ($conditionstring =~ /\w+=(?:[^\s,]+,?)+/g);
 289     for my $condition (@conditions) {
 290         my ($name, $values) = ($condition =~ /(\w+)=(.+)/);
 291         $results{$name} = [split ',', $values];
 292     }
 293
 294     return %results;
 295 }
 296
 297 # Get the name of the system SDK from sw_vers
 298 sub systemsdkname {
 299     my @lines = `/usr/bin/sw_vers`;
 300     my $name;
 301     my $vers;
 302     for my $line (@lines) {
 303         ($name) = ($line =~ /^ProductName:\s+(.*)/)  if !$name;
 304         ($vers) = ($line =~ /^ProductVersion:\s+(.*)/)  if !$vers;
 305     }
 306
 307     $name =~ s/ //g;
 308     $name = lc($name);
 309     my $internal = "";
 310     if (-d "/usr/local/include/objc") {
 311         if ($name eq "macosx") {
 312             $internal = "internal";
 313         } else {
 314             $internal = ".internal";
 315         }
 316     }
 317     return $name . $vers . $internal;
 318 }
 319
 320 sub check_output {
 321     my %C = %{shift()};
 322     my $name = shift;
 323     my @output = @_;
 324
 325     my %T = %{$C{"TEST_$name"}};
 326     my @original_output = @output;
 327
 328     # Run result-checking passes, reducing @output each time
 329     my $xit = 1;
 330     my $bad = "";
 331     my $warn = "";
 332     my $runerror = $T{TEST_RUN_OUTPUT};
 333     filter_verbose(\@output);
 334     $warn = filter_warn(\@output);
 335     $bad |= filter_guardmalloc(\@output) if ($C{GUARDMALLOC});
 336     $bad |= filter_valgrind(\@output) if ($C{VALGRIND});
 337     $bad = filter_expected(\@output, \%C, $name) if ($bad eq "");
 338     $bad = filter_bad(\@output)  if ($bad eq "");
 339
 340     # OK line should be the only one left
 341     $bad = "(output not 'OK: $name')" if ($bad eq ""  &&  (scalar(@output) != 1  ||  $output[0] !~ /^OK: $name/));
 342
 343     if ($bad ne "") {
 344         my $red = "\e[41;37m";
 345         my $def = "\e[0m";
 346         print "${red}FAIL: /// test '$name' \\\\\\$def\n";
 347         colorprint($red, @original_output);
 348         print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
 349         print "${red}FAIL: $name: $bad$def\n";
 350         $xit = 0;
 351     }
 352     elsif ($warn ne "") {
 353         my $yellow = "\e[43;37m";
 354         my $def = "\e[0m";
 355         print "${yellow}PASS: /// test '$name' \\\\\\$def\n";
 356         colorprint($yellow, @original_output);
 357         print "${yellow}PASS: \\\\\\ test '$name' ///$def\n";
 358         print "PASS: $name (with warnings)\n";
 359     }
 360     else {
 361         print "PASS: $name\n";
 362     }
 363     return $xit;
 364 }
 365
 366 sub filter_expected
 367 {
 368     my $outputref = shift;
 369     my %C = %{shift()};
 370     my $name = shift;
 371
 372     my %T = %{$C{"TEST_$name"}};
 373     my $runerror = $T{TEST_RUN_OUTPUT}  ||  return "";
 374
 375     my $bad = "";
 376
 377     my $output = join("\n", @$outputref) . "\n";
 378     if ($output !~ /$runerror/) {
 379         $bad = "(run output does not match TEST_RUN_OUTPUT)";
 380         @$outputref = ("FAIL: $name");
 381     } else {
 382         @$outputref = ("OK: $name");  # pacify later filter
 383     }
 384
 385     return $bad;
 386 }
 387
 388 sub filter_bad
 389 {
 390     my $outputref = shift;
 391     my $bad = "";
 392
 393     my @new_output;
 394     for my $line (@$outputref) {
 395         if ($line =~ /^BAD: (.*)/) {
 396             $bad = "(failed)";
 397         } else {
 398             push @new_output, $line;
 399         }
 400     }
 401
 402     @$outputref = @new_output;
 403     return $bad;
 404 }
 405
 406 sub filter_warn
 407 {
 408     my $outputref = shift;
 409     my $warn = "";
 410
 411     my @new_output;
 412     for my $line (@$outputref) {
 413         if ($line !~ /^WARN: (.*)/) {
 414             push @new_output, $line;
 415         } else {
 416             $warn = "(warned)";
 417         }
 418     }
 419
 420     @$outputref = @new_output;
 421     return $warn;
 422 }
 423
 424 sub filter_verbose
 425 {
 426     my $outputref = shift;
 427
 428     my @new_output;
 429     for my $line (@$outputref) {
 430         if ($line !~ /^VERBOSE: (.*)/) {
 431             push @new_output, $line;
 432         }
 433     }
 434
 435     @$outputref = @new_output;
 436 }
 437
 438 sub filter_valgrind
 439 {
 440     my $outputref = shift;
 441     my $errors = 0;
 442     my $leaks = 0;
 443
 444     my @new_output;
 445     for my $line (@$outputref) {
 446         if ($line =~ /^Approx: do_origins_Dirty\([RW]\): missed \d bytes$/) {
 447             # --track-origins warning (harmless)
 448             next;
 449         }
 450         if ($line =~ /^UNKNOWN __disable_threadsignal is unsupported. This warning will not be repeated.$/) {
 451             # signals unsupported (harmless)
 452             next;
 453         }
 454         if ($line =~ /^UNKNOWN __pthread_sigmask is unsupported. This warning will not be repeated.$/) {
 455             # signals unsupported (harmless)
 456             next;
 457         }
 458         if ($line !~ /^^\.*==\d+==/) {
 459             # not valgrind output
 460             push @new_output, $line;
 461             next;
 462         }
 463
 464         my ($errcount) = ($line =~ /==\d+== ERROR SUMMARY: (\d+) errors/);
 465         if (defined $errcount  &&  $errcount > 0) {
 466             $errors = 1;
 467         }
 468
 469         (my $leakcount) = ($line =~ /==\d+==\s+(?:definitely|possibly) lost:\s+([0-9,]+)/);
 470         if (defined $leakcount  &&  $leakcount > 0) {
 471             $leaks = 1;
 472         }
 473     }
 474
 475     @$outputref = @new_output;
 476
 477     my $bad = "";
 478     $bad .= "(valgrind errors)" if ($errors);
 479     $bad .= "(valgrind leaks)" if ($leaks);
 480     return $bad;
 481 }
 482
 483 sub filter_guardmalloc
 484 {
 485     my $outputref = shift;
 486     my $errors = 0;
 487
 488     my @new_output;
 489     my $count = 0;
 490     for my $line (@$outputref) {
 491         if ($line !~ /^GuardMalloc\[[^\]]+\]: /) {
 492             # not guardmalloc output
 493             push @new_output, $line;
 494             next;
 495         }
 496
 497         # Ignore 4 lines of guardmalloc prologue.
 498         # Anything further is a guardmalloc error.
 499         if (++$count > 4) {
 500             $errors = 1;
 501         }
 502     }
 503
 504     @$outputref = @new_output;
 505
 506     my $bad = "";
 507     $bad .= "(guardmalloc errors)" if ($errors);
 508     return $bad;
 509 }
 510
 511 sub gather_simple {
 512     my $CREF = shift;
 513     my %C = %{$CREF};
 514     my $name = shift;
 515     chdir_verbose $DIR;
 516
 517     my $ext = $ALL_TESTS{$name};
 518     my $file = "$name.$ext";
 519     return 0 if !$file;
 520
 521     # search file for 'TEST_CONFIG' or '#include "test.h"'
 522     # also collect other values:
 523     # TEST_CONFIG test conditions
 524     # TEST_ENV environment prefix
 525     # TEST_CFLAGS compile flags
 526     # TEST_BUILD build instructions
 527     # TEST_BUILD_OUTPUT expected build stdout/stderr
 528     # TEST_RUN_OUTPUT expected run stdout/stderr
 529     open(my $in, "< $file") || die;
 530     my $contents = join "", <$in>;
 531
 532     my $test_h = ($contents =~ /^\s*#\s*(include|import)\s*"test\.h"/m);
 533     my $disabled = ($contents =~ /\bTEST_DISABLED\b/m);
 534     my $crashes = ($contents =~ /\bTEST_CRASHES\b/m);
 535     my ($conditionstring) = ($contents =~ /\bTEST_CONFIG\b(.*)$/m);
 536     my ($envstring) = ($contents =~ /\bTEST_ENV\b(.*)$/m);
 537     my ($cflags) = ($contents =~ /\bTEST_CFLAGS\b(.*)$/m);
 538     my ($buildcmd) = ($contents =~ /TEST_BUILD\n(.*?\n)END[ *\/]*\n/s);
 539     my ($builderror) = ($contents =~ /TEST_BUILD_OUTPUT\n(.*?\n)END[ *\/]*\n/s);
 540     my ($runerror) = ($contents =~ /TEST_RUN_OUTPUT\n(.*?\n)END[ *\/]*\n/s);
 541
 542     return 0 if !$test_h && !$disabled && !$crashes && !defined($conditionstring) && !defined($envstring) && !defined($cflags) && !defined($buildcmd) && !defined($builderror) && !defined($runerror);
 543
 544     if ($disabled) {
 545         print "${yellow}SKIP: $name    (disabled by TEST_DISABLED)$def\n";
 546         return 0;
 547     }
 548
 549     # check test conditions
 550
 551     my $run = 1;
 552     my %conditions = readconditions($conditionstring);
 553     if (! $conditions{LANGUAGE}) {
 554         # implicit language restriction from file extension
 555         $conditions{LANGUAGE} = $languages_for_extension{$ext};
 556     }
 557     for my $condkey (keys %conditions) {
 558         my @condvalues = @{$conditions{$condkey}};
 559
 560         # special case: RUN=0 does not affect build
 561         if ($condkey eq "RUN"  &&  @condvalues == 1  &&  $condvalues[0] == 0) {
 562             $run = 0;
 563             next;
 564         }
 565
 566         my $testvalue = $C{$condkey};
 567         next if !defined($testvalue);
 568         # testvalue is the configuration being run now
 569         # condvalues are the allowed values for this test
 570
 571         # special case: look up the name of SDK "system"
 572         if ($condkey eq "SDK"  &&  $testvalue eq "system") {
 573             $testvalue = systemsdkname();
 574         }
 575
 576         my $ok = 0;
 577         for my $condvalue (@condvalues) {
 578
 579             # special case: objc and objc++
 580             if ($condkey eq "LANGUAGE") {
 581                 $condvalue = "objective-c" if $condvalue eq "objc";
 582                 $condvalue = "objective-c++" if $condvalue eq "objc++";
 583             }
 584
 585             $ok = 1  if ($testvalue eq $condvalue);
 586
 587             # special case: SDK allows prefixes, and "system" is "macosx"
 588             if ($condkey eq "SDK") {
 589                 $ok = 1  if ($testvalue =~ /^$condvalue/);
 590                 $ok = 1  if ($testvalue eq "system"  &&  "macosx" =~ /^$condvalue/);
 591             }
 592
 593             # special case: CC and CXX allow substring matches
 594             if ($condkey eq "CC"  ||  $condkey eq "CXX") {
 595                 $ok = 1  if ($testvalue =~ /$condvalue/);
 596             }
 597
 598             last if $ok;
 599         }
 600
 601         if (!$ok) {
 602             my $plural = (@condvalues > 1) ? "one of: " : "";
 603             print "SKIP: $name    ($condkey=$testvalue, but test requires $plural", join(' ', @condvalues), ")\n";
 604             return 0;
 605         }
 606     }
 607
 608     # builderror is multiple REs separated by OR
 609     if (defined $builderror) {
 610         $builderror =~ s/\nOR\n/\n|/sg;
 611         $builderror = "^(" . $builderror . ")\$";
 612     }
 613     # runerror is multiple REs separated by OR
 614     if (defined $runerror) {
 615         $runerror =~ s/\nOR\n/\n|/sg;
 616         $runerror = "^(" . $runerror . ")\$";
 617     }
 618
 619     # save some results for build and run phases
 620     $$CREF{"TEST_$name"} = {
 621         TEST_BUILD => $buildcmd,
 622         TEST_BUILD_OUTPUT => $builderror,
 623         TEST_CRASHES => $crashes,
 624         TEST_RUN_OUTPUT => $runerror,
 625         TEST_CFLAGS => $cflags,
 626         TEST_ENV => $envstring,
 627         TEST_RUN => $run,
 628     };
 629
 630     return 1;
 631 }
 632
 633 # Builds a simple test
 634 sub build_simple {
 635     my %C = %{shift()};
 636     my $name = shift;
 637     my %T = %{$C{"TEST_$name"}};
 638     chdir_verbose "$C{DIR}/$name.build";
 639
 640     my $ext = $ALL_TESTS{$name};
 641     my $file = "$DIR/$name.$ext";
 642
 643     if ($T{TEST_CRASHES}) {
 644         `echo '$crashcatch' > crashcatch.c`;
 645         make("$C{COMPILE_C} -dynamiclib -o libcrashcatch.dylib -x c crashcatch.c");
 646         die "$?" if $?;
 647     }
 648
 649     my $cmd = $T{TEST_BUILD} ? eval "return \"$T{TEST_BUILD}\"" : "$C{COMPILE}   $T{TEST_CFLAGS} $file -o $name.out";
 650
 651     my $output = make($cmd);
 652
 653     my $ok;
 654     if (my $builderror = $T{TEST_BUILD_OUTPUT}) {
 655         # check for expected output and ignore $?
 656         if ($output =~ /$builderror/) {
 657             $ok = 1;
 658         } else {
 659             print "${red}FAIL: /// test '$name' \\\\\\$def\n";
 660             colorprint $red, $output;
 661             print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
 662             print "${red}FAIL: $name (build output does not match TEST_BUILD_OUTPUT)$def\n";
 663             $ok = 0;
 664         }
 665     } elsif ($?) {
 666         print "${red}FAIL: /// test '$name' \\\\\\$def\n";
 667         colorprint $red, $output;
 668         print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
 669         print "${red}FAIL: $name (build failed)$def\n";
 670         $ok = 0;
 671     } elsif ($output ne "") {
 672         print "${red}FAIL: /// test '$name' \\\\\\$def\n";
 673         colorprint $red, $output;
 674         print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
 675         print "${red}FAIL: $name (unexpected build output)$def\n";
 676         $ok = 0;
 677     } else {
 678         $ok = 1;
 679     }
 680
 681
 682     if ($ok) {
 683         foreach my $file (glob("*.out *.dylib *.bundle")) {
 684             make("dsymutil $file");
 685         }
 686     }
 687
 688     return $ok;
 689 }
 690
 691 # Run a simple test (testname.out, with error checking of stdout and stderr)
 692 sub run_simple {
 693     my %C = %{shift()};
 694     my $name = shift;
 695     my %T = %{$C{"TEST_$name"}};
 696
 697     if (! $T{TEST_RUN}) {
 698         print "PASS: $name (build only)\n";
 699         return 1;
 700     }
 701     else {
 702         chdir_verbose "$C{DIR}/$name.build";
 703     }
 704
 705     my $env = "$C{ENV} $T{TEST_ENV}";
 706     if ($T{TEST_CRASHES}) {
 707         $env .= " DYLD_INSERT_LIBRARIES=libcrashcatch.dylib";
 708     }
 709
 710     my $output;
 711
 712     if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) {
 713         # run on iOS device
 714
 715         my $remotedir = "/var/root/test/" . basename($C{DIR}) . "/$name.build";
 716         my $remotedyld = " DYLD_LIBRARY_PATH=$remotedir";
 717         $remotedyld .= ":/var/root/test/"  if ($C{TESTLIB} ne $TESTLIBPATH);
 718
 719         # elide host-specific paths
 720         $env =~ s/DYLD_LIBRARY_PATH=\S+//;
 721         $env =~ s/DYLD_ROOT_PATH=\S+//;
 722
 723         my $cmd = "ssh iphone 'cd $remotedir && env $env $remotedyld ./$name.out'";
 724         $output = make("$cmd");
 725     }
 726     else {
 727         # run locally
 728
 729         my $cmd = "env $env ./$name.out";
 730         $output = make("sh -c '$cmd 2>&1' 2>&1");
 731         # need extra sh level to capture "sh: Illegal instruction" after crash
 732         # fixme fail if $? except tests that expect to crash
 733     }
 734
 735     return check_output(\%C, $name, split("\n", $output));
 736 }
 737
 738
 739 my %compiler_memo;
 740 sub find_compiler {
 741     my ($cc, $sdk, $sdk_path) = @_;
 742
 743     # memoize
 744     my $key = $cc . ':' . $sdk;
 745     my $result = $compiler_memo{$key};
 746     return $result if defined $result;
 747
 748     if (-e $cc) {
 749         $result = $cc;
 750     } elsif (-e "$sdk_path/$cc") {
 751         $result = "$sdk_path/$cc";
 752     } elsif ($sdk eq "system"  &&  -e "/usr/bin/$cc") {
 753         $result = "/usr/bin/$cc";
 754     } elsif ($sdk eq "system") {
 755         $result  = `xcrun -find $cc 2>/dev/null`;
 756     } else {
 757         $result  = `xcrun -sdk $sdk -find $cc 2>/dev/null`;
 758     }
 759
 760     chomp $result;
 761     $compiler_memo{$key} = $result;
 762     return $result;
 763 }
 764
 765 sub make_one_config {
 766     my $configref = shift;
 767     my $root = shift;
 768     my %C = %{$configref};
 769
 770     # Aliases
 771     $C{LANGUAGE} = "objective-c"  if $C{LANGUAGE} eq "objc";
 772     $C{LANGUAGE} = "objective-c++"  if $C{LANGUAGE} eq "objc++";
 773
 774     # Look up SDK
 775     # Try exact match first.
 776     # Then try lexically-last prefix match (so "macosx" => "macosx10.7internal").
 777     my @sdks = getsdks();
 778     if ($VERBOSE) {
 779         print "Installed SDKs: @sdks\n";
 780     }
 781     my $exactsdk = undef;
 782     my $prefixsdk = undef;
 783     foreach my $sdk (@sdks) {
 784         my $SDK = $C{SDK};
 785         $exactsdk = $sdk  if ($sdk eq $SDK);
 786         # check for digits to prevent e.g. "iphone" => "iphonesimulator4.2"
 787         $prefixsdk = $sdk  if ($sdk =~ /^$SDK[0-9]/  &&  $sdk gt $prefixsdk);
 788     }
 789     if ($exactsdk) {
 790         $C{SDK} = $exactsdk;
 791     } elsif ($prefixsdk) {
 792         $C{SDK} = $prefixsdk;
 793     } else {
 794         die "unknown SDK '$C{SDK}'\nInstalled SDKs: @sdks\n";
 795     }
 796
 797     # set the config name now, after massaging the language and sdk,
 798     # but before adding other settings
 799     my $configname = config_name(%C);
 800     die if ($configname =~ /'/);
 801     die if ($configname =~ / /);
 802     ($C{NAME} = $configname) =~ s/~/ /g;
 803     (my $configdir = $configname) =~ s#/##g;
 804     $C{DIR} = "$BUILDDIR/$configdir";
 805
 806     $C{SDK_PATH} = "/";
 807     if ($C{SDK} ne "system") {
 808         ($C{SDK_PATH}) = (`xcodebuild -version -sdk $C{SDK} Path` =~ /^\s*(.+?)\s*$/);
 809     }
 810
 811     # Look up test library (possible in root or SDK_PATH)
 812
 813     if (-e (glob "$root/*~dst")[0]) {
 814         $root = (glob "$root/*~dst")[0];
 815     }
 816
 817     if ($root ne ""  &&  -e "$root$C{SDK_PATH}$TESTLIBPATH") {
 818         $C{TESTLIB} = "$root$C{SDK_PATH}$TESTLIBPATH";
 819     } elsif (-e "$root$TESTLIBPATH") {
 820         $C{TESTLIB} = "$root$TESTLIBPATH";
 821     } elsif (-e "$root/$TESTLIBNAME") {
 822         $C{TESTLIB} = "$root/$TESTLIBNAME";
 823     } else {
 824         die "No $TESTLIBNAME in root '$root' for sdk '$C{SDK_PATH}'\n";
 825     }
 826
 827     if ($VERBOSE) {
 828         my @uuids = `/usr/bin/dwarfdump -u '$C{TESTLIB}'`;
 829         while (my $uuid = shift @uuids) {
 830             print "note: $uuid";
 831         }
 832     }
 833
 834     # Look up compilers
 835     my $cc = $C{CC};
 836     my $cxx = cplusplus($C{CC});
 837     if (! $BUILD) {
 838         $C{CC} = $cc;
 839         $C{CXX} = $cxx;
 840     } else {
 841         $C{CC} = find_compiler($cc, $C{SDK}, $C{SDK_PATH});
 842         $C{CXX} = find_compiler($cxx, $C{SDK}, $C{SDK_PATH});
 843
 844         die "No compiler '$cc' ('$C{CC}') in SDK '$C{SDK}'\n" if !-e $C{CC};
 845         die "No compiler '$cxx' ('$C{CXX}') in SDK '$C{SDK}'\n" if !-e $C{CXX};
 846     }
 847
 848     # Populate cflags
 849
 850     # save-temps so dsymutil works so debug info works
 851     my $cflags = "-I$DIR -W -Wall -Wno-deprecated-declarations -Wshorten-64-to-32 -g -save-temps -Os -arch $C{ARCH} ";
 852     my $objcflags = "";
 853
 854     if ($C{SDK} ne "system") {
 855         $cflags .= " -isysroot '$C{SDK_PATH}'";
 856         $cflags .= " '-Wl,-syslibroot,$C{SDK_PATH}'";
 857     }
 858
 859     if ($C{SDK} =~ /^iphoneos[0-9]/  &&  $cflags !~ /-miphoneos-version-min/) {
 860         my ($vers) = ($C{SDK} =~ /^iphoneos([0-9]+\.[0-9+])/);
 861         $cflags .= " -miphoneos-version-min=$vers";
 862     }
 863     if ($C{SDK} =~ /^iphonesimulator[0-9]/  &&  $cflags !~ /-D__IPHONE_OS_VERSION_MIN_REQUIRED/) {
 864         my ($vers) = ($C{SDK} =~ /^iphonesimulator([0-9]+\.[0-9+])/);
 865         $vers = int($vers * 10000);  # 4.2 => 42000
 866         $cflags .= " -D__IPHONE_OS_VERSION_MIN_REQUIRED=$vers";
 867     }
 868     if ($C{SDK} =~ /^iphonesimulator/) {
 869         $objcflags .= " -fobjc-abi-version=2 -fobjc-legacy-dispatch";
 870     }
 871
 872     if ($root ne "") {
 873         my $library_path = dirname($C{TESTLIB});
 874         $cflags .= " -L$library_path";
 875         $cflags .= " -isystem '$root/usr/include'";
 876         $cflags .= " -isystem '$root/usr/local/include'";
 877
 878         if ($C{SDK_PATH} ne "/") {
 879             $cflags .= " -isystem '$root$C{SDK_PATH}/usr/include'";
 880             $cflags .= " -isystem '$root$C{SDK_PATH}/usr/local/include'";
 881         }
 882     }
 883
 884     if ($C{CC} =~ /clang/) {
 885         $cflags .= " -Qunused-arguments -fno-caret-diagnostics";
 886         $cflags .= " -stdlib=$C{STDLIB} -fno-objc-link-runtime";
 887     }
 888
 889
 890     # Populate objcflags
 891
 892     $objcflags .= " -lobjc";
 893     if ($C{MEM} eq "gc") {
 894         $objcflags .= " -fobjc-gc";
 895     }
 896     elsif ($C{MEM} eq "arc") {
 897         $objcflags .= " -fobjc-arc";
 898     }
 899     elsif ($C{MEM} eq "mrc") {
 900         # nothing
 901     }
 902     else {
 903         die "unrecognized MEM '$C{MEM}'\n";
 904     }
 905
 906     if (supportslibauto($C{SDK})) {
 907         # do this even for non-GC tests
 908         $objcflags .= " -lauto";
 909     }
 910
 911     # Populate ENV_PREFIX
 912     $C{ENV} = "LANG=C";
 913     $C{ENV} .= " VERBOSE=1"  if $VERBOSE;
 914     if ($root ne "") {
 915         my $library_path = dirname($C{TESTLIB});
 916         die "no spaces allowed in root" if $library_path =~ /\s+/;
 917         $C{ENV} .= " DYLD_LIBRARY_PATH=$library_path"  if ($library_path ne "/usr/lib");
 918     }
 919     if ($C{SDK_PATH} ne "/") {
 920         die "no spaces allowed in sdk" if $C{SDK_PATH} =~ /\s+/;
 921         $C{ENV} .= " DYLD_ROOT_PATH=$C{SDK_PATH}";
 922     }
 923     if ($C{GUARDMALLOC}) {
 924         $ENV{GUARDMALLOC} = "1";  # checked by tests and errcheck.pl
 925         $C{ENV} .= " DYLD_INSERT_LIBRARIES=/usr/lib/libgmalloc.dylib";
 926     }
 927     if ($C{SDK} =~ /^iphonesimulator[0-9]/) {
 928         my ($vers) = ($C{SDK} =~ /^iphonesimulator([0-9]+\.[0-9+])/);
 929         $C{ENV} .=
 930             " CFFIXED_USER_HOME=$ENV{HOME}/Library/Application\\ Support/iPhone\\ Simulator/$vers" .
 931             " IPHONE_SIMULATOR_ROOT=$C{SDK_PATH}" .
 932             " IPHONE_SHARED_RESOURCES_DIRECTORY=$ENV{HOME}/Library/Application\\ Support/iPhone\\ Simulator/$vers";
 933     }
 934
 935     # Populate compiler commands
 936     $C{COMPILE_C}   = "env LANG=C '$C{CC}'  $cflags -x c -std=gnu99";
 937     $C{COMPILE_CXX} = "env LANG=C '$C{CXX}' $cflags -x c++";
 938     $C{COMPILE_M}   = "env LANG=C '$C{CC}'  $cflags $objcflags -x objective-c -std=gnu99";
 939     $C{COMPILE_MM}  = "env LANG=C '$C{CXX}' $cflags $objcflags -x objective-c++";
 940
 941     $C{COMPILE} = $C{COMPILE_C}    if $C{LANGUAGE} eq "c";
 942     $C{COMPILE} = $C{COMPILE_CXX}  if $C{LANGUAGE} eq "c++";
 943     $C{COMPILE} = $C{COMPILE_M}    if $C{LANGUAGE} eq "objective-c";
 944     $C{COMPILE} = $C{COMPILE_MM}   if $C{LANGUAGE} eq "objective-c++";
 945     die "unknown language '$C{LANGUAGE}'\n" if !defined $C{COMPILE};
 946
 947     ($C{COMPILE_NOMEM} = $C{COMPILE}) =~ s/ -fobjc-(?:gc|arc)\S*//g;
 948     ($C{COMPILE_NOLINK} = $C{COMPILE}) =~ s/ '?-(?:Wl,|l)\S*//g;
 949     ($C{COMPILE_NOLINK_NOMEM} = $C{COMPILE_NOMEM}) =~ s/ '?-(?:Wl,|l)\S*//g;
 950
 951
 952     # Reject some self-inconsistent configurations
 953     if ($C{MEM} !~ /^(mrc|arc|gc)$/) {
 954         die "unknown MEM=$C{MEM} (expected one of mrc arc gc)\n";
 955     }
 956
 957     if ($C{MEM} eq "gc"  &&  $C{SDK} =~ /^iphone/) {
 958         print "note: skipping configuration $C{NAME}\n";
 959         print "note:   because SDK=$C{SDK} does not support MEM=$C{MEM}\n";
 960         return 0;
 961     }
 962     if ($C{MEM} eq "arc"  &&  $C{SDK} !~ /^iphone/  &&  $C{ARCH} eq "i386") {
 963         print "note: skipping configuration $C{NAME}\n";
 964         print "note:   because 32-bit Mac does not support MEM=$C{MEM}\n";
 965         return 0;
 966     }
 967     if ($C{MEM} eq "arc"  &&  $C{CC} !~ /clang/) {
 968         print "note: skipping configuration $C{NAME}\n";
 969         print "note:   because CC=$C{CC} does not support MEM=$C{MEM}\n";
 970         return 0;
 971     }
 972
 973     if ($C{STDLIB} ne "libstdc++"  &&  $C{CC} !~ /clang/) {
 974         print "note: skipping configuration $C{NAME}\n";
 975         print "note:   because CC=$C{CC} does not support STDLIB=$C{STDLIB}\n";
 976         return 0;
 977     }
 978
 979     %$configref = %C;
 980 }
 981
 982 sub make_configs {
 983     my ($root, %args) = @_;
 984
 985     my @results = ({});  # start with one empty config
 986
 987     for my $key (keys %args) {
 988         my @newresults;
 989         my @values = @{$args{$key}};
 990         for my $configref (@results) {
 991             my %config = %{$configref};
 992             for my $value (@values) {
 993                 my %newconfig = %config;
 994                 $newconfig{$key} = $value;
 995                 push @newresults, \%newconfig;
 996             }
 997         }
 998         @results = @newresults;
 999     }
1000
1001     my @newresults;
1002     for my $configref(@results) {
1003         if (make_one_config($configref, $root)) {
1004             push @newresults, $configref;
1005         }
1006     }
1007
1008     return @newresults;
1009 }
1010
1011 sub config_name {
1012     my %config = @_;
1013     my $name = "";
1014     for my $key (sort keys %config) {
1015         $name .= '~'  if $name ne "";
1016         $name .= "$key=$config{$key}";
1017     }
1018     return $name;
1019 }
1020
1021 sub run_one_config {
1022     my %C = %{shift()};
1023     my @tests = @_;
1024
1025     # Build and run
1026     my $testcount = 0;
1027     my $failcount = 0;
1028
1029     my @gathertests;
1030     foreach my $test (@tests) {
1031         if ($VERBOSE) {
1032             print "\nGATHER $test\n";
1033         }
1034
1035         if ($ALL_TESTS{$test}) {
1036             gather_simple(\%C, $test) || next;  # not pass, not fail
1037             push @gathertests, $test;
1038         } else {
1039             die "No test named '$test'\n";
1040         }
1041     }
1042
1043     my @builttests;
1044     if (!$BUILD) {
1045         @builttests = @gathertests;
1046         $testcount = scalar(@gathertests);
1047     } else {
1048         my $configdir = $C{DIR};
1049         print $configdir, "\n"  if $VERBOSE;
1050         mkdir $configdir  || die;
1051
1052         foreach my $test (@gathertests) {
1053             if ($VERBOSE) {
1054                 print "\nBUILD $test\n";
1055             }
1056             mkdir "$configdir/$test.build"  || die;
1057
1058             if ($ALL_TESTS{$test}) {
1059                 $testcount++;
1060                 if (!build_simple(\%C, $test)) {
1061                     $failcount++;
1062                 } else {
1063                     push @builttests, $test;
1064                 }
1065             } else {
1066                 die "No test named '$test'\n";
1067             }
1068         }
1069     }
1070
1071     if (!$RUN  ||  !scalar(@builttests)) {
1072         # nothing to do
1073     }
1074     else {
1075         if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) {
1076             # upload all tests to iOS device
1077             make("RSYNC_PASSWORD=alpine rsync -av $C{DIR} rsync://root\@localhost:10873/root/var/root/test/");
1078             die "Couldn't rsync tests to device\n" if ($?);
1079
1080             # upload library to iOS device
1081             if ($C{TESTLIB} ne $TESTLIBPATH) {
1082                 # hack - send thin library because device may use lib=armv7
1083                 # even though app=armv6, and we want to set the lib's arch
1084                 make("lipo -output /tmp/$TESTLIBNAME -thin $C{ARCH} $C{TESTLIB}  ||  cp $C{TESTLIB} /tmp/$TESTLIBNAME");
1085                 die "Couldn't thin $C{TESTLIB} to $C{ARCH}\n" if ($?);
1086                 make("RSYNC_PASSWORD=alpine rsync -av /tmp/$TESTLIBNAME rsync://root\@localhost:10873/root/var/root/test/");
1087                 die "Couldn't rsync $C{TESTLIB} to device\n" if ($?);
1088             }
1089         }
1090
1091         foreach my $test (@builttests) {
1092             print "\nRUN $test\n"  if ($VERBOSE);
1093
1094             if ($ALL_TESTS{$test})
1095             {
1096                 if (!run_simple(\%C, $test)) {
1097                     $failcount++;
1098                 }
1099             } else {
1100                 die "No test named '$test'\n";
1101             }
1102         }
1103     }
1104
1105     return ($testcount, $failcount);
1106 }
1107
1108
1109
1110 # Return value if set by "$argname=value" on the command line
1111 # Return $default if not set.
1112 sub getargs {
1113     my ($argname, $default) = @_;
1114
1115     foreach my $arg (@ARGV) {
1116         my ($value) = ($arg =~ /^$argname=(.+)$/);
1117         return [split ',', $value] if defined $value;
1118     }
1119
1120     return [$default];
1121 }
1122
1123 # Return 1 or 0 if set by "$argname=1" or "$argname=0" on the
1124 # command line. Return $default if not set.
1125 sub getbools {
1126     my ($argname, $default) = @_;
1127
1128     my @values = @{getargs($argname, $default)};
1129     return [( map { ($_ eq "0") ? 0 : 1 } @values )];
1130 }
1131
1132 sub getarg {
1133     my ($argname, $default) = @_;
1134     my @values = @{getargs($argname, $default)};
1135     die "Only one value allowed for $argname\n"  if @values > 1;
1136     return $values[0];
1137 }
1138
1139 sub getbool {
1140     my ($argname, $default) = @_;
1141     my @values = @{getbools($argname, $default)};
1142     die "Only one value allowed for $argname\n"  if @values > 1;
1143     return $values[0];
1144 }
1145
1146
1147 # main
1148 my %args;
1149
1150
1151 my $default_arch = (`/usr/sbin/sysctl hw.optional.x86_64` eq "hw.optional.x86_64: 1\n") ? "x86_64" : "i386";
1152 $args{ARCH} = getargs("ARCH", 0);
1153 $args{ARCH} = getargs("ARCHS", $default_arch)  if !@{$args{ARCH}}[0];
1154
1155 $args{SDK} = getargs("SDK", "system");
1156
1157 $args{MEM} = getargs("MEM", "mrc");
1158 $args{LANGUAGE} = [ map { lc($_) } @{getargs("LANGUAGE", "objective-c")} ];
1159 $args{STDLIB} = getargs("STDLIB", "libstdc++");
1160
1161 $args{CC} = getargs("CC", "clang");
1162
1163 $args{GUARDMALLOC} = getbools("GUARDMALLOC", 0);
1164
1165 $BUILD = getbool("BUILD", 1);
1166 $RUN = getbool("RUN", 1);
1167 $VERBOSE = getbool("VERBOSE", 0);
1168
1169 my $root = getarg("ROOT", "");
1170 $root =~ s#/*$##;
1171
1172 my @tests = gettests();
1173
1174 print "note: -----\n";
1175 print "note: testing root '$root'\n";
1176
1177 my @configs = make_configs($root, %args);
1178
1179 print "note: -----\n";
1180 print "note: testing ", scalar(@configs), " configurations:\n";
1181 for my $configref (@configs) {
1182     my $configname = $$configref{NAME};
1183     print "note: configuration $configname\n";
1184 }
1185
1186 if ($BUILD) {
1187     `rm -rf '$BUILDDIR'`;
1188     mkdir "$BUILDDIR" || die;
1189 }
1190
1191 my $failed = 0;
1192
1193 my $testconfigs = @configs;
1194 my $failconfigs = 0;
1195 my $testcount = 0;
1196 my $failcount = 0;
1197 for my $configref (@configs) {
1198     my $configname = $$configref{NAME};
1199     print "note: -----\n";
1200     print "note: \nnote: $configname\nnote: \n";
1201
1202     (my $t, my $f) = eval { run_one_config($configref, @tests); };
1203     if ($@) {
1204         chomp $@;
1205         print "${red}FAIL: $configname${def}\n";
1206         print "${red}FAIL: $@${def}\n";
1207         $failconfigs++;
1208     } else {
1209         my $color = ($f ? $red : "");
1210         print "note:\n";
1211         print "${color}note: $configname$def\n";
1212         print "${color}note: $t tests, $f failures$def\n";
1213         $testcount += $t;
1214         $failcount += $f;
1215         $failconfigs++ if ($f);
1216     }
1217 }
1218
1219 print "note: -----\n";
1220 my $color = ($failconfigs ? $red : "");
1221 print "${color}note: $testconfigs configurations, $failconfigs with failures$def\n";
1222 print "${color}note: $testcount tests, $failcount failures$def\n";
1223
1224 $failed = ($failconfigs ? 1 : 0);
1225
1226 exit ($failed ? 1 : 0);