test/test.pl

   1 #!/usr/bin/perl
   2
   3 # test.pl
   4 # Run unit tests.
   5
   6 use strict;
   7 use File::Basename;
   8
   9 chdir dirname $0;
  10 chomp (my $DIR = `pwd`);
  11
  12 my $TESTLIBNAME = "libobjc.A.dylib";
  13 my $TESTLIBPATH = "/usr/lib/$TESTLIBNAME";
  14
  15 my $BUILDDIR = "/tmp/test-$TESTLIBNAME-build";
  16
  17 # xterm colors
  18 my $red = "\e[41;37m";
  19 my $yellow = "\e[43;37m";
  20 my $def = "\e[0m";
  21
  22 # clean, help
  23 if (scalar(@ARGV) == 1) {
  24     my $arg = $ARGV[0];
  25     if ($arg eq "clean") {
  26         my $cmd = "rm -rf $BUILDDIR *~";
  27         print "$cmd\n";
  28         `$cmd`;
  29         exit 0;
  30     }
  31     elsif ($arg eq "-h" || $arg eq "-H" || $arg eq "-help" || $arg eq "help") {
  32         print(<<END);
  33 usage: $0 [options] [testname ...]
  34        $0 clean
  35        $0 help
  36
  37 testname:
  38     `testname` runs a specific test. If no testnames are given, runs all tests.
  39
  40 options:
  41     ARCH=<arch>
  42     GC=0|1
  43     SDK=<sdk name>
  44     ROOT=/path/to/project.roots/
  45
  46     CC=<compiler name>
  47
  48     GUARDMALLOC=0|1
  49
  50     BUILD=0|1
  51     RUN=0|1
  52     VERBOSE=0|1
  53
  54 examples:
  55
  56     test installed library, x86_64, no gc
  57     $0
  58
  59     test buildit-built root, i386 and x86_64, gc and no gc, clang compiler
  60     $0 ARCH=i386,x86_64 ROOT=/tmp/libclosure.roots GC=1,0 CC=clang
  61
  62     test buildit-built root with iOS simulator
  63     $0 ARCH=i386 ROOT=/tmp/libclosure.roots SDK=iphonesimulator
  64
  65     test buildit-built root on attached iOS device
  66     $0 ARCH=armv7 ROOT=/tmp/libclosure.roots SDK=iphoneos
  67 END
  68         exit 0;
  69     }
  70 }
  71
  72 #########################################################################
  73 ## Tests
  74
  75 my %ALL_TESTS;
  76
  77 #########################################################################
  78 ## Variables for use in complex build and run rules
  79
  80 # variable         # example value
  81
  82 # things you can multiplex on the command line
  83 # ARCH=i386,x86_64,armv6,armv7
  84 # SDK=system,macosx,iphoneos,iphonesimulator
  85 # LANGUAGE=c,c++,objective-c,objective-c++
  86 # CC=clang,gcc-4.2,llvm-gcc-4.2
  87 # GC=0,1
  88 # GUARDMALLOC=0,1
  89
  90 # things you can set once on the command line
  91 # ROOT=/path/to/project.roots
  92 # BUILD=0|1
  93 # RUN=0|1
  94 # VERBOSE=0|1
  95
  96
  97
  98 my $BUILD;
  99 my $RUN;
 100 my $VERBOSE;
 101
 102 my $crashcatch = <<'END';
 103 // interpose-able code to catch crashes, print, and exit cleanly
 104 #include <signal.h>
 105 #include <string.h>
 106 #include <unistd.h>
 107 #include <mach-o/dyld-interposing.h>
 108
 109 static void catchcrash(int sig)
 110 {
 111     const char *msg;
 112     switch (sig) {
 113     case SIGILL:  msg = "CRASHED: SIGILL\\n";  break;
 114     case SIGBUS:  msg = "CRASHED: SIGBUS\\n";  break;
 115     case SIGSYS:  msg = "CRASHED: SIGSYS\\n";  break;
 116     case SIGSEGV: msg = "CRASHED: SIGSEGV\\n"; break;
 117     case SIGTRAP: msg = "CRASHED: SIGTRAP\\n"; break;
 118     case SIGABRT: msg = "CRASHED: SIGABRT\\n"; break;
 119     default: msg = "SIG\?\?\?\?\\n"; break;
 120     }
 121     write(STDERR_FILENO, msg, strlen(msg));
 122     _exit(0);
 123 }
 124
 125 static void setupcrash(void) __attribute__((constructor));
 126 static void setupcrash(void)
 127 {
 128     signal(SIGILL, &catchcrash);
 129     signal(SIGBUS, &catchcrash);
 130     signal(SIGSYS, &catchcrash);
 131     signal(SIGSEGV, &catchcrash);
 132     signal(SIGTRAP, &catchcrash);
 133     signal(SIGABRT, &catchcrash);
 134 }
 135
 136
 137 static int hacked = 0;
 138 ssize_t hacked_write(int fildes, const void *buf, size_t nbyte)
 139 {
 140     if (!hacked) {
 141         setupcrash();
 142         hacked = 1;
 143     }
 144     return write(fildes, buf, nbyte);
 145 }
 146
 147 DYLD_INTERPOSE(hacked_write, write);
 148
 149 END
 150
 151
 152 #########################################################################
 153 ## Harness
 154
 155
 156 # map language to buildable extensions for that language
 157 my %extensions_for_language = (
 158     "c"     => ["c"],
 159     "objective-c" => ["c", "m"],
 160     "c++" => ["c", "cc", "cp", "cpp", "cxx", "c++"],
 161     "objective-c++" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"],
 162
 163     "any" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"],
 164     );
 165
 166 # map extension to languages
 167 my %languages_for_extension = (
 168     "c" => ["c", "objective-c", "c++", "objective-c++"],
 169     "m" => ["objective-c", "objective-c++"],
 170     "mm" => ["objective-c++"],
 171     "cc" => ["c++", "objective-c++"],
 172     "cp" => ["c++", "objective-c++"],
 173     "cpp" => ["c++", "objective-c++"],
 174     "cxx" => ["c++", "objective-c++"],
 175     "c++" => ["c++", "objective-c++"],
 176     );
 177
 178 # Run some newline-separated commands like `make` would, stopping if any fail
 179 # run("cmd1 \n cmd2 \n cmd3")
 180 sub make {
 181     my $output = "";
 182     my @cmds = split("\n", $_[0]);
 183     die if scalar(@cmds) == 0;
 184     $? = 0;
 185     foreach my $cmd (@cmds) {
 186         chomp $cmd;
 187         next if $cmd =~ /^\s*$/;
 188         $cmd .= " 2>&1";
 189         print "$cmd\n" if $VERBOSE;
 190         $output .= `$cmd`;
 191         last if $?;
 192     }
 193     print "$output\n" if $VERBOSE;
 194     return $output;
 195 }
 196
 197 sub chdir_verbose {
 198     my $dir = shift;
 199     chdir $dir || die;
 200     print "cd $dir\n" if $VERBOSE;
 201 }
 202
 203
 204 # Return test names from the command line.
 205 # Returns all tests if no tests were named.
 206 sub gettests {
 207     my @tests;
 208
 209     foreach my $arg (@ARGV) {
 210         push @tests, $arg  if ($arg !~ /=/  &&  $arg !~ /^-/);
 211     }
 212
 213     opendir(my $dir, $DIR) || die;
 214     while (my $file = readdir($dir)) {
 215         my ($name, $ext) = ($file =~ /^([^.]+)\.([^.]+)$/);
 216         next if ! $languages_for_extension{$ext};
 217
 218         open(my $in, "< $file") || die "$file";
 219         my $contents = join "", <$in>;
 220         die if defined $ALL_TESTS{$name};
 221         $ALL_TESTS{$name} = $ext  if ($contents =~ m#^[/*\s]*TEST_#m);
 222         close($in);
 223     }
 224     closedir($dir);
 225
 226     if (scalar(@tests) == 0) {
 227         @tests = keys %ALL_TESTS;
 228     }
 229
 230     @tests = sort @tests;
 231
 232     return @tests;
 233 }
 234
 235
 236 # Turn a C compiler name into a C++ compiler name.
 237 sub cplusplus {
 238     my ($c) = @_;
 239     if ($c =~ /cc/) {
 240         $c =~ s/cc/\+\+/;
 241         return $c;
 242     }
 243     return $c . "++";                         # e.g. clang => clang++
 244 }
 245
 246 # Returns an array of all sdks from `xcodebuild -showsdks`
 247 sub getsdks {
 248     return ("system", `xcodebuild -showsdks` =~ /-sdk (.+)$/mg);
 249 }
 250
 251 # Returns whether the given sdk supports GC
 252 sub supportsgc {
 253     my ($sdk) = @_;
 254     return 1 if $sdk eq "system";
 255     return 1 if $sdk =~ /^macosx/;
 256     return 0 if $sdk =~ /^iphone/;
 257     die;
 258 }
 259
 260 # print text with a colored prefix on each line
 261 sub colorprint {
 262     my $color = shift;
 263     while (my @lines = split("\n", shift)) {
 264         for my $line (@lines) {
 265             chomp $line;
 266             print "$color $def$line\n";
 267         }
 268     }
 269 }
 270
 271 sub rewind {
 272     seek($_[0], 0, 0);
 273 }
 274
 275 # parse name=value,value pairs
 276 sub readconditions {
 277     my ($conditionstring) = @_;
 278
 279     my %results;
 280     my @conditions = ($conditionstring =~ /\w+=(?:[^\s,]+,?)+/g);
 281     for my $condition (@conditions) {
 282         my ($name, $values) = ($condition =~ /(\w+)=(.+)/);
 283         $results{$name} = [split ',', $values];
 284     }
 285
 286     return %results;
 287 }
 288
 289 # Get the name of the system SDK from sw_vers
 290 sub systemsdkname {
 291     my @lines = `/usr/bin/sw_vers`;
 292     my $name;
 293     my $vers;
 294     for my $line (@lines) {
 295         ($name) = ($line =~ /^ProductName:\s+(.*)/)  if !$name;
 296         ($vers) = ($line =~ /^ProductVersion:\s+(.*)/)  if !$vers;
 297     }
 298
 299     $name =~ s/ //g;
 300     $name = lc($name);
 301     my $internal = "";
 302     if (-d "/usr/local/include/objc") {
 303         if ($name eq "macosx") {
 304             $internal = "internal";
 305         } else {
 306             $internal = ".internal";
 307         }
 308     }
 309     return $name . $vers . $internal;
 310 }
 311
 312 sub check_output {
 313     my %C = %{shift()};
 314     my $name = shift;
 315     my @output = @_;
 316
 317     my %T = %{$C{"TEST_$name"}};
 318     my @original_output = @output;
 319
 320     # Run result-checking passes, reducing @output each time
 321     my $xit = 1;
 322     my $bad = "";
 323     my $warn = "";
 324     my $runerror = $T{TEST_RUN_OUTPUT};
 325     filter_verbose(\@output);
 326     $warn = filter_warn(\@output);
 327     $bad |= filter_guardmalloc(\@output) if ($C{GUARDMALLOC});
 328     $bad |= filter_valgrind(\@output) if ($C{VALGRIND});
 329     $bad = filter_expected(\@output, \%C, $name) if ($bad eq "");
 330     $bad = filter_bad(\@output)  if ($bad eq "");
 331
 332     # OK line should be the only one left
 333     $bad = "(output not 'OK: $name')" if ($bad eq ""  &&  (scalar(@output) != 1  ||  $output[0] !~ /^OK: $name/));
 334
 335     if ($bad ne "") {
 336         my $red = "\e[41;37m";
 337         my $def = "\e[0m";
 338         print "${red}FAIL: /// test '$name' \\\\\\$def\n";
 339         colorprint($red, @original_output);
 340         print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
 341         print "${red}FAIL: $name: $bad$def\n";
 342         $xit = 0;
 343     }
 344     elsif ($warn ne "") {
 345         my $yellow = "\e[43;37m";
 346         my $def = "\e[0m";
 347         print "${yellow}PASS: /// test '$name' \\\\\\$def\n";
 348         colorprint($yellow, @original_output);
 349         print "${yellow}PASS: \\\\\\ test '$name' ///$def\n";
 350         print "PASS: $name (with warnings)\n";
 351     }
 352     else {
 353         print "PASS: $name\n";
 354     }
 355     return $xit;
 356 }
 357
 358 sub filter_expected
 359 {
 360     my $outputref = shift;
 361     my %C = %{shift()};
 362     my $name = shift;
 363
 364     my %T = %{$C{"TEST_$name"}};
 365     my $check = $T{TEST_RUN_OUTPUT}  ||  return "";
 366
 367     my $bad = "";
 368
 369     my $output = join("\n", @$outputref) . "\n";
 370     if ($output !~ /$check/s) {
 371         $bad = "(run output does not match TEST_RUN_OUTPUT)";
 372         @$outputref = ("FAIL: $name");
 373     } else {
 374         @$outputref = ("OK: $name");  # pacify later filter
 375     }
 376
 377     return $bad;
 378 }
 379
 380 sub filter_bad
 381 {
 382     my $outputref = shift;
 383     my $bad = "";
 384
 385     my @new_output;
 386     for my $line (@$outputref) {
 387         if ($line =~ /^BAD: (.*)/) {
 388             $bad = "(failed)";
 389         } else {
 390             push @new_output, $line;
 391         }
 392     }
 393
 394     @$outputref = @new_output;
 395     return $bad;
 396 }
 397
 398 sub filter_warn
 399 {
 400     my $outputref = shift;
 401     my $warn = "";
 402
 403     my @new_output;
 404     for my $line (@$outputref) {
 405         if ($line !~ /^WARN: (.*)/) {
 406             push @new_output, $line;
 407         } else {
 408             $warn = "(warned)";
 409         }
 410     }
 411
 412     @$outputref = @new_output;
 413     return $warn;
 414 }
 415
 416 sub filter_verbose
 417 {
 418     my $outputref = shift;
 419
 420     my @new_output;
 421     for my $line (@$outputref) {
 422         if ($line !~ /^VERBOSE: (.*)/) {
 423             push @new_output, $line;
 424         }
 425     }
 426
 427     @$outputref = @new_output;
 428 }
 429
 430 sub filter_valgrind
 431 {
 432     my $outputref = shift;
 433     my $errors = 0;
 434     my $leaks = 0;
 435
 436     my @new_output;
 437     for my $line (@$outputref) {
 438         if ($line =~ /^Approx: do_origins_Dirty\([RW]\): missed \d bytes$/) {
 439             # --track-origins warning (harmless)
 440             next;
 441         }
 442         if ($line =~ /^UNKNOWN __disable_threadsignal is unsupported. This warning will not be repeated.$/) {
 443             # signals unsupported (harmless)
 444             next;
 445         }
 446         if ($line =~ /^UNKNOWN __pthread_sigmask is unsupported. This warning will not be repeated.$/) {
 447             # signals unsupported (harmless)
 448             next;
 449         }
 450         if ($line !~ /^^\.*==\d+==/) {
 451             # not valgrind output
 452             push @new_output, $line;
 453             next;
 454         }
 455
 456         my ($errcount) = ($line =~ /==\d+== ERROR SUMMARY: (\d+) errors/);
 457         if (defined $errcount  &&  $errcount > 0) {
 458             $errors = 1;
 459         }
 460
 461         (my $leakcount) = ($line =~ /==\d+==\s+(?:definitely|possibly) lost:\s+([0-9,]+)/);
 462         if (defined $leakcount  &&  $leakcount > 0) {
 463             $leaks = 1;
 464         }
 465     }
 466
 467     @$outputref = @new_output;
 468
 469     my $bad = "";
 470     $bad .= "(valgrind errors)" if ($errors);
 471     $bad .= "(valgrind leaks)" if ($leaks);
 472     return $bad;
 473 }
 474
 475 sub filter_guardmalloc
 476 {
 477     my $outputref = shift;
 478     my $errors = 0;
 479
 480     my @new_output;
 481     for my $line (@$outputref) {
 482         if ($line =~ /^GuardMalloc: /) {
 483             # guardmalloc prologue
 484             next;
 485         }
 486         if ($line !~ /^GuardMalloc\[[^\]]+\]: /) {
 487             # not guardmalloc output
 488             push @new_output, $line;
 489             next;
 490         }
 491
 492         $errors = 1;
 493     }
 494
 495     @$outputref = @new_output;
 496
 497     my $bad = "";
 498     $bad .= "(guardmalloc errors)" if ($errors);
 499     return $bad;
 500 }
 501
 502 sub gather_simple {
 503     my $CREF = shift;
 504     my %C = %{$CREF};
 505     my $name = shift;
 506     chdir_verbose $DIR;
 507
 508     my $ext = $ALL_TESTS{$name};
 509     my $file = "$name.$ext";
 510     return 0 if !$file;
 511
 512     # search file for 'TEST_CONFIG' or '#include "test.h"'
 513     # also collect other values:
 514     # TEST_CONFIG test conditions
 515     # TEST_ENV environment prefix
 516     # TEST_CFLAGS compile flags
 517     # TEST_BUILD build instructions
 518     # TEST_BUILD_OUTPUT expected build stdout/stderr
 519     # TEST_RUN_OUTPUT expected run stdout/stderr
 520     open(my $in, "< $file") || die;
 521     my $contents = join "", <$in>;
 522
 523     my $test_h = ($contents =~ /^\s*#\s*(include|import)\s*"test\.h"/m);
 524     my $disabled = ($contents =~ /\bTEST_DISABLED\b/m);
 525     my $crashes = ($contents =~ /\bTEST_CRASHES\b/m);
 526     my ($conditionstring) = ($contents =~ /\bTEST_CONFIG\b(.*)$/m);
 527     my ($envstring) = ($contents =~ /\bTEST_ENV\b(.*)$/m);
 528     my ($cflags) = ($contents =~ /\bTEST_CFLAGS\b(.*)$/m);
 529     my ($buildcmd) = ($contents =~ /TEST_BUILD\n(.*?\n)END[ *\/]*\n/s);
 530     my ($builderror) = ($contents =~ /TEST_BUILD_OUTPUT\n(.*?\n)END[ *\/]*\n/s);
 531     my ($runerror) = ($contents =~ /TEST_RUN_OUTPUT\n(.*?\n)END[ *\/]*\n/s);
 532
 533     return 0 if !$test_h && !$disabled && !$crashes && !defined($conditionstring) && !defined($envstring) && !defined($cflags) && !defined($buildcmd) && !defined($builderror) && !defined($runerror);
 534
 535     if ($disabled) {
 536         print "${yellow}SKIP: $name (disabled by TEST_DISABLED)$def\n";
 537         return 0;
 538     }
 539
 540     # check test conditions
 541
 542     my $run = 1;
 543     my %conditions = readconditions($conditionstring);
 544     if (! $conditions{LANGUAGE}) {
 545         # implicit language restriction from file extension
 546         $conditions{LANGUAGE} = $languages_for_extension{$ext};
 547     }
 548     for my $condkey (keys %conditions) {
 549         my @condvalues = @{$conditions{$condkey}};
 550
 551         # special case: RUN=0 does not affect build
 552         if ($condkey eq "RUN"  &&  @condvalues == 1  &&  $condvalues[0] == 0) {
 553             $run = 0;
 554             next;
 555         }
 556
 557         my $testvalue = $C{$condkey};
 558         next if !defined($testvalue);
 559         # testvalue is the configuration being run now
 560         # condvalues are the allowed values for this test
 561
 562         # special case: look up the name of SDK "system"
 563         if ($condkey eq "SDK"  &&  $testvalue eq "system") {
 564             $testvalue = systemsdkname();
 565         }
 566
 567         my $ok = 0;
 568         for my $condvalue (@condvalues) {
 569
 570             # special case: objc and objc++
 571             if ($condkey eq "LANGUAGE") {
 572                 $condvalue = "objective-c" if $condvalue eq "objc";
 573                 $condvalue = "objective-c++" if $condvalue eq "objc++";
 574             }
 575
 576             $ok = 1  if ($testvalue eq $condvalue);
 577
 578             # special case: SDK allows prefixes, and "system" is "macosx"
 579             if ($condkey eq "SDK") {
 580                 $ok = 1  if ($testvalue =~ /^$condvalue/);
 581                 $ok = 1  if ($testvalue eq "system"  &&  "macosx" =~ /^$condvalue/);
 582             }
 583
 584             # special case: CC and CXX allow substring matches
 585             if ($condkey eq "CC"  ||  $condkey eq "CXX") {
 586                 $ok = 1  if ($testvalue =~ /$condvalue/);
 587             }
 588
 589             last if $ok;
 590         }
 591
 592         if (!$ok) {
 593             my $plural = (@condvalues > 1) ? "one of: " : "";
 594             print "SKIP: $name ($condkey=$testvalue, but test requires $plural", join(' ', @condvalues), ")\n";
 595             return 0;
 596         }
 597     }
 598
 599     # builderror is multiple REs separated by OR
 600     if (defined $builderror) {
 601         $builderror =~ s/\nOR\n/\n|/sg;
 602         $builderror = "^(" . $builderror . ")\$";
 603     }
 604     # runerror is multiple REs separated by OR
 605     if (defined $runerror) {
 606         $runerror =~ s/\nOR\n/\n|/sg;
 607         $runerror = "^(" . $runerror . ")\$";
 608     }
 609
 610     # save some results for build and run phases
 611     $$CREF{"TEST_$name"} = {
 612         TEST_BUILD => $buildcmd,
 613         TEST_BUILD_OUTPUT => $builderror,
 614         TEST_CRASHES => $crashes,
 615         TEST_RUN_OUTPUT => $runerror,
 616         TEST_CFLAGS => $cflags,
 617         TEST_ENV => $envstring,
 618         TEST_RUN => $run,
 619     };
 620
 621     return 1;
 622 }
 623
 624 # Builds a simple test
 625 sub build_simple {
 626     my %C = %{shift()};
 627     my $name = shift;
 628     my %T = %{$C{"TEST_$name"}};
 629     chdir_verbose "$C{DIR}/$name.build";
 630
 631     my $ext = $ALL_TESTS{$name};
 632     my $file = "$DIR/$name.$ext";
 633
 634     if ($T{TEST_CRASHES}) {
 635         `echo '$crashcatch' > crashcatch.c`;
 636         make("$C{COMPILE_C} -dynamiclib -o libcrashcatch.dylib -x c crashcatch.c");
 637         die "$?" if $?;
 638     }
 639
 640     my $cmd = $T{TEST_BUILD} ? eval "return \"$T{TEST_BUILD}\"" : "$C{COMPILE}   $T{TEST_CFLAGS} $file -o $name.out";
 641
 642     my $output = make($cmd);
 643
 644     my $ok;
 645     if (my $builderror = $T{TEST_BUILD_OUTPUT}) {
 646         # check for expected output and ignore $?
 647         if ($output =~ /$builderror/s) {
 648             $ok = 1;
 649         } else {
 650             print "${red}FAIL: /// test '$name' \\\\\\$def\n";
 651             colorprint $red, $output;
 652             print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
 653             print "${red}FAIL: $name (build output does not match TEST_BUILD_OUTPUT)$def\n";
 654             $ok = 0;
 655         }
 656     } elsif ($?) {
 657         print "${red}FAIL: /// test '$name' \\\\\\$def\n";
 658         colorprint $red, $output;
 659         print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
 660         print "${red}FAIL: $name (build failed)$def\n";
 661         $ok = 0;
 662     } elsif ($output ne "") {
 663         print "${red}FAIL: /// test '$name' \\\\\\$def\n";
 664         colorprint $red, $output;
 665         print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
 666         print "${red}FAIL: $name (unexpected build output)$def\n";
 667         $ok = 0;
 668     } else {
 669         $ok = 1;
 670     }
 671
 672
 673     if ($ok) {
 674         foreach my $file (glob("*.out *.dylib *.bundle")) {
 675             make("dsymutil $file");
 676         }
 677     }
 678
 679     return $ok;
 680 }
 681
 682 # Run a simple test (testname.out, with error checking of stdout and stderr)
 683 sub run_simple {
 684     my %C = %{shift()};
 685     my $name = shift;
 686     my %T = %{$C{"TEST_$name"}};
 687
 688     if (! $T{TEST_RUN}) {
 689         print "PASS: $name (build only)\n";
 690         return 1;
 691     }
 692     else {
 693         chdir_verbose "$C{DIR}/$name.build";
 694     }
 695
 696     my $env = "$C{ENV} $T{TEST_ENV}";
 697     if ($T{TEST_CRASHES}) {
 698         $env .= " DYLD_INSERT_LIBRARIES=libcrashcatch.dylib";
 699     }
 700
 701     my $output;
 702
 703     if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) {
 704         # run on iOS device
 705
 706         my $remotedir = "/var/root/test/" . basename($C{DIR}) . "/$name.build";
 707         my $remotedyld = " DYLD_LIBRARY_PATH=$remotedir";
 708         $remotedyld .= ":/var/root/test/"  if ($C{TESTLIB} ne $TESTLIBPATH);
 709
 710         # elide host-specific paths
 711         $env =~ s/DYLD_LIBRARY_PATH=\S+//;
 712         $env =~ s/DYLD_ROOT_PATH=\S+//;
 713
 714         my $cmd = "ssh iphone 'cd $remotedir && $remotedyld $env ./$name.out'";
 715         $output = make("$cmd");
 716     }
 717     else {
 718         # run locally
 719
 720         my $cmd = "$env ./$name.out";
 721         $output = make("sh -c '$cmd 2>&1' 2>&1");
 722         # need extra sh level to capture "sh: Illegal instruction" after crash
 723         # fixme fail if $? except tests that expect to crash
 724     }
 725
 726     return check_output(\%C, $name, split("\n", $output));
 727 }
 728
 729
 730 sub make_one_config {
 731     my $configref = shift;
 732     my $root = shift;
 733     my %C = %{$configref};
 734
 735     $C{LANGUAGE} = "objective-c"  if $C{LANGUAGE} eq "objc";
 736     $C{LANGUAGE} = "objective-c++"  if $C{LANGUAGE} eq "objc++";
 737
 738     # Look up SDK
 739     # Try exact match first.
 740     # Then try lexically-last prefix match (so "macosx" => "macosx10.7internal").
 741     my @sdks = getsdks();
 742     if ($VERBOSE) {
 743         print "Installed SDKs: @sdks\n";
 744     }
 745     my $exactsdk = undef;
 746     my $prefixsdk = undef;
 747     foreach my $sdk (@sdks) {
 748         my $SDK = $C{SDK};
 749         $exactsdk = $sdk  if ($sdk eq $SDK);
 750         # check for digits to prevent e.g. "iphone" => "iphonesimulator4.2"
 751         $prefixsdk = $sdk  if ($sdk =~ /^$SDK[0-9]/  &&  $sdk gt $prefixsdk);
 752     }
 753     if ($exactsdk) {
 754         $C{SDK} = $exactsdk;
 755     } elsif ($prefixsdk) {
 756         $C{SDK} = $prefixsdk;
 757     } else {
 758         die "unknown SDK '$C{SDK}'\nInstalled SDKs: @sdks\n";
 759     }
 760
 761     # set the config name now, after massaging the language and sdk,
 762     # but before adding other settings
 763     my $configname = config_name(%C);
 764     die if ($configname =~ /'/);
 765     die if ($configname =~ /\//);
 766     die if ($configname =~ / /);
 767     $C{DIR} = "$BUILDDIR/$configname";
 768     ($C{NAME} = $configname) =~ s/~/ /g;
 769
 770     $C{SDK_PATH} = "/";
 771     if ($C{SDK} ne "system") {
 772         ($C{SDK_PATH}) = (`xcodebuild -version -sdk $C{SDK} Path` =~ /^\s*(.+?)\s*$/);
 773     }
 774
 775     # Look up test library (possible in root or SDK_PATH)
 776
 777     if (-e (glob "$root/*~dst")[0]) {
 778         $root = (glob "$root/*~dst")[0];
 779     }
 780
 781     if ($root ne ""  &&  -e "$root$C{SDK_PATH}$TESTLIBPATH") {
 782         $C{TESTLIB} = "$root$C{SDK_PATH}$TESTLIBPATH";
 783     } elsif (-e "$root$TESTLIBPATH") {
 784         $C{TESTLIB} = "$root$TESTLIBPATH";
 785     } elsif (-e "$root/$TESTLIBNAME") {
 786         $C{TESTLIB} = "$root/$TESTLIBNAME";
 787     } else {
 788         die "No $TESTLIBNAME in root '$root' and sdk '$C{SDK_PATH}'\n";
 789     }
 790
 791     # Look up compilers
 792     $C{CXX} = cplusplus($C{CC});
 793     if ($BUILD) {
 794         my $oldcc = $C{CC};
 795         my $oldcxx = $C{CXX};
 796
 797         if (-e $C{CC}) {
 798             # use it
 799         } elsif (-e "$C{SDK_PATH}/$C{CC}") {
 800             $C{CC} = "$C{SDK_PATH}/$C{CC}";
 801         } elsif ($C{SDK} eq "system"  &&  -e "/usr/bin/$C{CC}") {
 802             $C{CC} = "/usr/bin/$C{CC}";
 803         } elsif ($C{SDK} eq "system") {
 804             $C{CC}  = `xcrun -find $C{CC} 2>/dev/null`;
 805             chomp $C{CC};
 806         } else {
 807             $C{CC}  = `xcrun -sdk $C{SDK} -find $C{CC} 2>/dev/null`;
 808             chomp $C{CC};
 809         }
 810
 811         if (-e $C{CXX}) {
 812             # use it
 813         } elsif (-e "$C{SDK_PATH}/$C{CXX}") {
 814             $C{CXX} = "$C{SDK_PATH}/$C{CXX}";
 815         } elsif ($C{SDK} eq "system"  &&  -e "/usr/bin/$C{CXX}") {
 816             $C{CXX} = "/usr/bin/$C{CXX}";
 817         } elsif ($C{SDK} eq "system") {
 818             $C{CXX}  = `xcrun -find $C{CXX} 2>/dev/null`;
 819             chomp $C{CXX};
 820         } else {
 821             $C{CXX}  = `xcrun -sdk $C{SDK} -find $C{CXX} 2>/dev/null`;
 822             chomp $C{CXX};
 823         }
 824
 825         die "No compiler '$oldcc' in SDK '$C{SDK}'\n" if ! -e $C{CC};
 826         die "No compiler '$oldcxx' '$C{CXX}' in SDK '$C{SDK}'\n" if ! -e $C{CXX};
 827     }
 828
 829
 830     # Populate cflags
 831
 832     # save-temps so dsymutil works so debug info works
 833     my $cflags = "-I$DIR -W -Wall -Wno-deprecated-declarations -Wshorten-64-to-32 -g -save-temps -Os -arch $C{ARCH} ";
 834     my $objcflags = "";
 835
 836     if ($C{SDK} ne "system") {
 837         $cflags .= " -isysroot '$C{SDK_PATH}'";
 838         $cflags .= " '-Wl,-syslibroot,$C{SDK_PATH}'";
 839     }
 840
 841     if ($C{SDK} =~ /^iphoneos[0-9]/  &&  $cflags !~ /-miphoneos-version-min/) {
 842         my ($vers) = ($C{SDK} =~ /^iphoneos([0-9]+\.[0-9+])/);
 843         $cflags .= " -miphoneos-version-min=$vers";
 844     }
 845     if ($C{SDK} =~ /^iphonesimulator[0-9]/  &&  $cflags !~ /-D__IPHONE_OS_VERSION_MIN_REQUIRED/) {
 846         my ($vers) = ($C{SDK} =~ /^iphonesimulator([0-9]+\.[0-9+])/);
 847         $vers = int($vers * 10000);  # 4.2 => 42000
 848         $cflags .= " -D__IPHONE_OS_VERSION_MIN_REQUIRED=$vers";
 849     }
 850     if ($C{SDK} =~ /^iphonesimulator/) {
 851         $objcflags .= " -fobjc-abi-version=2 -fobjc-legacy-dispatch";
 852     }
 853
 854     if ($root ne "") {
 855         my $library_path = dirname($C{TESTLIB});
 856         $cflags .= " -L$library_path";
 857         $cflags .= " -isystem '$root/usr/include'";
 858         $cflags .= " -isystem '$root/usr/local/include'";
 859
 860         if ($C{SDK_PATH} ne "/") {
 861             $cflags .= " -isystem '$root$C{SDK_PATH}/usr/include'";
 862             $cflags .= " -isystem '$root$C{SDK_PATH}/usr/local/include'";
 863         }
 864     }
 865
 866     if ($C{CC} =~ /clang/) {
 867         $cflags .= " -Qunused-arguments -fno-caret-diagnostics";
 868     }
 869
 870     # Populate objcflags
 871
 872     $objcflags .= " -lobjc";
 873     if ($C{GC}) {
 874         $objcflags .= " -fobjc-gc";
 875     }
 876     if (supportsgc($C{SDK})) {
 877         $objcflags .= " -lauto";
 878     }
 879
 880     # Populate ENV_PREFIX
 881     $C{ENV} = "LANG=C";
 882     $C{ENV} .= " VERBOSE=1"  if $VERBOSE;
 883     if ($root ne "") {
 884         my $library_path = dirname($C{TESTLIB});
 885         die "no spaces allowed in root" if $library_path =~ /\s+/;
 886         $C{ENV} .= " DYLD_LIBRARY_PATH=$library_path"  if ($library_path ne "/usr/lib");
 887     }
 888     if ($C{SDK_PATH} ne "/") {
 889         die "no spaces allowed in sdk" if $C{SDK_PATH} =~ /\s+/;
 890         $C{ENV} .= " DYLD_ROOT_PATH=$C{SDK_PATH}";
 891     }
 892     if ($C{GUARDMALLOC}) {
 893         $ENV{GUARDMALLOC} = "1";  # checked by tests and errcheck.pl
 894         $C{ENV} .= " DYLD_INSERT_LIBRARIES=/usr/lib/libgmalloc.dylib";
 895     }
 896     if ($C{SDK} =~ /^iphonesimulator[0-9]/) {
 897         my ($vers) = ($C{SDK} =~ /^iphonesimulator([0-9]+\.[0-9+])/);
 898         $C{ENV} .=
 899             " CFFIXED_USER_HOME=$ENV{HOME}/Library/Application\\ Support/iPhone\\ Simulator/$vers" .
 900             " IPHONE_SIMULATOR_ROOT=$C{SDK_PATH}" .
 901             " IPHONE_SHARED_RESOURCES_DIRECTORY=$ENV{HOME}/Library/Application\\ Support/iPhone\\ Simulator/$vers";
 902     }
 903
 904     # Populate compiler commands
 905     $C{COMPILE_C}   = "LANG=C '$C{CC}'  $cflags -x c -std=gnu99";
 906     $C{COMPILE_CXX} = "LANG=C '$C{CXX}' $cflags -x c++";
 907     $C{COMPILE_M}   = "LANG=C '$C{CC}'  $cflags $objcflags -x objective-c -std=gnu99";
 908     $C{COMPILE_MM}  = "LANG=C '$C{CXX}' $cflags $objcflags -x objective-c++";
 909
 910     $C{COMPILE} = $C{COMPILE_C}    if $C{LANGUAGE} eq "c";
 911     $C{COMPILE} = $C{COMPILE_CXX}  if $C{LANGUAGE} eq "c++";
 912     $C{COMPILE} = $C{COMPILE_M}    if $C{LANGUAGE} eq "objective-c";
 913     $C{COMPILE} = $C{COMPILE_MM}   if $C{LANGUAGE} eq "objective-c++";
 914     die "unknown language '$C{LANGUAGE}'\n" if !defined $C{COMPILE};
 915
 916     ($C{COMPILE_NOGC} = $C{COMPILE}) =~ s/-fobjc-gc\S*//;
 917
 918     %$configref = %C;
 919 }
 920
 921 sub make_configs {
 922     my ($root, %args) = @_;
 923
 924     my @results = ({});  # start with one empty config
 925
 926     for my $key (keys %args) {
 927         my @newresults;
 928         my @values = @{$args{$key}};
 929         for my $configref (@results) {
 930             my %config = %{$configref};
 931             for my $value (@values) {
 932                 my %newconfig = %config;
 933                 $newconfig{$key} = $value;
 934                 push @newresults, \%newconfig;
 935             }
 936         }
 937         @results = @newresults;
 938     }
 939
 940     for my $configref(@results) {
 941         make_one_config($configref, $root);
 942     }
 943
 944     return @results;
 945 }
 946
 947 sub config_name {
 948     my %config = @_;
 949     my $name = "";
 950     for my $key (sort keys %config) {
 951         $name .= '~'  if $name ne "";
 952         $name .= "$key=$config{$key}";
 953     }
 954     return $name;
 955 }
 956
 957 sub run_one_config {
 958     my %C = %{shift()};
 959     my @tests = @_;
 960
 961     # Build and run
 962     my $testcount = 0;
 963     my $failcount = 0;
 964
 965     my @gathertests;
 966     foreach my $test (@tests) {
 967         if ($VERBOSE) {
 968             print "\nGATHER $test\n";
 969         }
 970
 971         if ($ALL_TESTS{$test}) {
 972             gather_simple(\%C, $test) || next;  # not pass, not fail
 973             push @gathertests, $test;
 974         } else {
 975             die "No test named '$test'\n";
 976         }
 977     }
 978
 979     my @builttests;
 980     if (!$BUILD) {
 981         @builttests = @gathertests;
 982         $testcount = scalar(@gathertests);
 983     } else {
 984         my $configdir = $C{DIR};
 985         print $configdir, "\n"  if $VERBOSE;
 986         mkdir $configdir  || die;
 987
 988         foreach my $test (@gathertests) {
 989             if ($VERBOSE) {
 990                 print "\nBUILD $test\n";
 991             }
 992             mkdir "$configdir/$test.build"  || die;
 993
 994             if ($ALL_TESTS{$test}) {
 995                 $testcount++;
 996                 if (!build_simple(\%C, $test)) {
 997                     $failcount++;
 998                 } else {
 999                     push @builttests, $test;
1000                 }
1001             } else {
1002                 die "No test named '$test'\n";
1003             }
1004         }
1005     }
1006
1007     if (!$RUN  ||  !scalar(@builttests)) {
1008         # nothing to do
1009     }
1010     else {
1011         if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) {
1012             # upload all tests to iOS device
1013             make("RSYNC_PASSWORD=alpine rsync -av $C{DIR} rsync://root\@localhost:10873/root/var/root/test/");
1014             die "Couldn't rsync tests to device\n" if ($?);
1015
1016             # upload library to iOS device
1017             if ($C{TESTLIB} ne $TESTLIBPATH) {
1018                 # hack - send thin library because device may use lib=armv7
1019                 # even though app=armv6, and we want to set the lib's arch
1020                 make("lipo -output /tmp/$TESTLIBNAME -thin $C{ARCH} $C{TESTLIB}  ||  cp $C{TESTLIB} /tmp/$TESTLIBNAME");
1021                 die "Couldn't thin $C{TESTLIB} to $C{ARCH}\n" if ($?);
1022                 make("RSYNC_PASSWORD=alpine rsync -av /tmp/$TESTLIBNAME rsync://root\@localhost:10873/root/var/root/test/");
1023                 die "Couldn't rsync $C{TESTLIB} to device\n" if ($?);
1024             }
1025         }
1026
1027         foreach my $test (@builttests) {
1028             print "\nRUN $test\n"  if ($VERBOSE);
1029
1030             if ($ALL_TESTS{$test})
1031             {
1032                 if (!run_simple(\%C, $test)) {
1033                     $failcount++;
1034                 }
1035             } else {
1036                 die "No test named '$test'\n";
1037             }
1038         }
1039     }
1040
1041     return ($testcount, $failcount);
1042 }
1043
1044
1045
1046 # Return value if set by "$argname=value" on the command line
1047 # Return $default if not set.
1048 sub getargs {
1049     my ($argname, $default) = @_;
1050
1051     foreach my $arg (@ARGV) {
1052         my ($value) = ($arg =~ /^$argname=(.+)$/);
1053         return [split ',', $value] if defined $value;
1054     }
1055
1056     return [$default];
1057 }
1058
1059 # Return 1 or 0 if set by "$argname=1" or "$argname=0" on the
1060 # command line. Return $default if not set.
1061 sub getbools {
1062     my ($argname, $default) = @_;
1063
1064     my @values = @{getargs($argname, $default)};
1065     return [( map { ($_ eq "0") ? 0 : 1 } @values )];
1066 }
1067
1068 sub getarg {
1069     my ($argname, $default) = @_;
1070     my @values = @{getargs($argname, $default)};
1071     die "Only one value allowed for $argname\n"  if @values > 1;
1072     return $values[0];
1073 }
1074
1075 sub getbool {
1076     my ($argname, $default) = @_;
1077     my @values = @{getbools($argname, $default)};
1078     die "Only one value allowed for $argname\n"  if @values > 1;
1079     return $values[0];
1080 }
1081
1082
1083 # main
1084 my %args;
1085
1086
1087 my $default_arch = (`/usr/sbin/sysctl hw.optional.x86_64` eq "hw.optional.x86_64: 1\n") ? "x86_64" : "i386";
1088 $args{ARCH} = getargs("ARCH", 0);
1089 $args{ARCH} = getargs("ARCHS", $default_arch)  if !@{$args{ARCH}}[0];
1090
1091 $args{SDK} = getargs("SDK", "system");
1092
1093 $args{GC} = getbools("GC", 0);
1094 $args{LANGUAGE} = [ map { lc($_) } @{getargs("LANGUAGE", "objective-c")} ];
1095
1096 $args{CC} = getargs("CC", "llvm-gcc-4.2");
1097
1098 $args{GUARDMALLOC} = getbools("GUARDMALLOC", 0);
1099
1100 $BUILD = getbool("BUILD", 1);
1101 $RUN = getbool("RUN", 1);
1102 $VERBOSE = getbool("VERBOSE", 0);
1103
1104 my $root = getarg("ROOT", "");
1105 $root =~ s#/*$##;
1106
1107 my @tests = gettests();
1108
1109 print "note: -----\n";
1110 print "note: testing root '$root'\n";
1111
1112 my @configs = make_configs($root, %args);
1113
1114 print "note: -----\n";
1115 print "note: testing ", scalar(@configs), " configurations:\n";
1116 for my $configref (@configs) {
1117     my $configname = $$configref{NAME};
1118     print "note: configuration $configname\n";
1119 }
1120
1121 if ($BUILD) {
1122     `rm -rf '$BUILDDIR'`;
1123     mkdir "$BUILDDIR" || die;
1124 }
1125
1126 my $failed = 0;
1127
1128 my $testconfigs = @configs;
1129 my $failconfigs = 0;
1130 my $testcount = 0;
1131 my $failcount = 0;
1132 for my $configref (@configs) {
1133     my $configname = $$configref{NAME};
1134     print "note: -----\n";
1135     print "note: \nnote: $configname\nnote: \n";
1136
1137     (my $t, my $f) = eval { run_one_config($configref, @tests); };
1138     if ($@) {
1139         chomp $@;
1140         print "${red}FAIL: $configname${def}\n";
1141         print "${red}FAIL: $@${def}\n";
1142         $failconfigs++;
1143     } else {
1144         my $color = ($f ? $red : "");
1145         print "note:\n";
1146         print "${color}note: $configname$def\n";
1147         print "${color}note: $t tests, $f failures$def\n";
1148         $testcount += $t;
1149         $failcount += $f;
1150         $failconfigs++ if ($f);
1151     }
1152 }
1153
1154 print "note: -----\n";
1155 my $color = ($failconfigs ? $red : "");
1156 print "${color}note: $testconfigs configurations, $failconfigs with failures$def\n";
1157 print "${color}note: $testcount tests, $failcount failures$def\n";
1158
1159 $failed = ($failconfigs ? 1 : 0);
1160
1161 exit ($failed ? 1 : 0);