test/test.pl

   1 #!/usr/bin/perl
   2
   3 # test.pl
   4 # Run unit tests.
   5
   6 use strict;
   7 use File::Basename;
   8
   9 chdir dirname $0;
  10 chomp (my $DIR = `pwd`);
  11
  12 my $TESTLIBNAME = "libobjc.A.dylib";
  13 my $TESTLIBPATH = "/usr/lib/$TESTLIBNAME";
  14
  15 my $BUILDDIR = "/tmp/test-$TESTLIBNAME-build";
  16
  17 # xterm colors
  18 my $red = "\e[41;37m";
  19 my $yellow = "\e[43;30m";
  20 my $nocolor = "\e[0m";
  21
  22 # clean, help
  23 if (scalar(@ARGV) == 1) {
  24     my $arg = $ARGV[0];
  25     if ($arg eq "clean") {
  26         my $cmd = "rm -rf $BUILDDIR *~";
  27         print "$cmd\n";
  28         `$cmd`;
  29         exit 0;
  30     }
  31     elsif ($arg eq "-h" || $arg eq "-H" || $arg eq "-help" || $arg eq "help") {
  32         print(<<END);
  33 usage: $0 [options] [testname ...]
  34        $0 clean
  35        $0 help
  36
  37 testname:
  38     `testname` runs a specific test. If no testnames are given, runs all tests.
  39
  40 options:
  41     ARCH=<arch>
  42     OS=<sdk name>[sdk version][-<deployment target>[-<run target>]]
  43     ROOT=/path/to/project.roots/
  44
  45     CC=<compiler name>
  46
  47     LANGUAGE=c,c++,objective-c,objective-c++,swift
  48     MEM=mrc,arc,gc
  49     STDLIB=libc++,libstdc++
  50     GUARDMALLOC=0|1|before|after
  51
  52     BUILD=0|1
  53     RUN=0|1
  54     VERBOSE=0|1|2
  55
  56 examples:
  57
  58     test installed library, x86_64, no gc
  59     $0
  60
  61     test buildit-built root, i386 and x86_64, MRC and ARC and GC, clang compiler
  62     $0 ARCH=i386,x86_64 ROOT=/tmp/libclosure.roots MEM=mrc,arc,gc CC=clang
  63
  64     test buildit-built root with iOS simulator, deploy to iOS 7, run on iOS 8
  65     $0 ARCH=i386 ROOT=/tmp/libclosure.roots OS=iphonesimulator-7.0-8.0
  66
  67     test buildit-built root on attached iOS device
  68     $0 ARCH=armv7 ROOT=/tmp/libclosure.roots OS=iphoneos
  69 END
  70         exit 0;
  71     }
  72 }
  73
  74 #########################################################################
  75 ## Tests
  76
  77 my %ALL_TESTS;
  78
  79 #########################################################################
  80 ## Variables for use in complex build and run rules
  81
  82 # variable         # example value
  83
  84 # things you can multiplex on the command line
  85 # ARCH=i386,x86_64,armv6,armv7
  86 # OS=macosx,iphoneos,iphonesimulator (plus sdk/deployment/run versions)
  87 # LANGUAGE=c,c++,objective-c,objective-c++,swift
  88 # CC=clang,gcc-4.2,llvm-gcc-4.2
  89 # MEM=mrc,arc,gc
  90 # STDLIB=libc++,libstdc++
  91 # GUARDMALLOC=0,1,before,after
  92
  93 # things you can set once on the command line
  94 # ROOT=/path/to/project.roots
  95 # BUILD=0|1
  96 # RUN=0|1
  97 # VERBOSE=0|1|2
  98
  99
 100
 101 my $BUILD;
 102 my $RUN;
 103 my $VERBOSE;
 104
 105 my $crashcatch = <<'END';
 106 // interpose-able code to catch crashes, print, and exit cleanly
 107 #include <signal.h>
 108 #include <string.h>
 109 #include <unistd.h>
 110
 111 // from dyld-interposing.h
 112 #define DYLD_INTERPOSE(_replacement,_replacee) __attribute__((used)) static struct{ const void* replacement; const void* replacee; } _interpose_##_replacee __attribute__ ((section ("__DATA,__interpose"))) = { (const void*)(unsigned long)&_replacement, (const void*)(unsigned long)&_replacee };
 113
 114 static void catchcrash(int sig)
 115 {
 116     const char *msg;
 117     switch (sig) {
 118     case SIGILL:  msg = "CRASHED: SIGILL\\n";  break;
 119     case SIGBUS:  msg = "CRASHED: SIGBUS\\n";  break;
 120     case SIGSYS:  msg = "CRASHED: SIGSYS\\n";  break;
 121     case SIGSEGV: msg = "CRASHED: SIGSEGV\\n"; break;
 122     case SIGTRAP: msg = "CRASHED: SIGTRAP\\n"; break;
 123     case SIGABRT: msg = "CRASHED: SIGABRT\\n"; break;
 124     default: msg = "SIG\?\?\?\?\\n"; break;
 125     }
 126     write(STDERR_FILENO, msg, strlen(msg));
 127     _exit(0);
 128 }
 129
 130 static void setupcrash(void) __attribute__((constructor));
 131 static void setupcrash(void)
 132 {
 133     signal(SIGILL, &catchcrash);
 134     signal(SIGBUS, &catchcrash);
 135     signal(SIGSYS, &catchcrash);
 136     signal(SIGSEGV, &catchcrash);
 137     signal(SIGTRAP, &catchcrash);
 138     signal(SIGABRT, &catchcrash);
 139 }
 140
 141
 142 static int hacked = 0;
 143 ssize_t hacked_write(int fildes, const void *buf, size_t nbyte)
 144 {
 145     if (!hacked) {
 146         setupcrash();
 147         hacked = 1;
 148     }
 149     return write(fildes, buf, nbyte);
 150 }
 151
 152 DYLD_INTERPOSE(hacked_write, write);
 153
 154 END
 155
 156
 157 #########################################################################
 158 ## Harness
 159
 160
 161 # map language to buildable extensions for that language
 162 my %extensions_for_language = (
 163     "c"     => ["c"],
 164     "objective-c" => ["c", "m"],
 165     "c++" => ["c", "cc", "cp", "cpp", "cxx", "c++"],
 166     "objective-c++" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"],
 167     "swift" => ["swift"],
 168
 169     "any" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm", "swift"],
 170     );
 171
 172 # map extension to languages
 173 my %languages_for_extension = (
 174     "c" => ["c", "objective-c", "c++", "objective-c++"],
 175     "m" => ["objective-c", "objective-c++"],
 176     "mm" => ["objective-c++"],
 177     "cc" => ["c++", "objective-c++"],
 178     "cp" => ["c++", "objective-c++"],
 179     "cpp" => ["c++", "objective-c++"],
 180     "cxx" => ["c++", "objective-c++"],
 181     "c++" => ["c++", "objective-c++"],
 182     "swift" => ["swift"],
 183     );
 184
 185 # Run some newline-separated commands like `make` would, stopping if any fail
 186 # run("cmd1 \n cmd2 \n cmd3")
 187 sub make {
 188     my $output = "";
 189     my @cmds = split("\n", $_[0]);
 190     die if scalar(@cmds) == 0;
 191     $? = 0;
 192     foreach my $cmd (@cmds) {
 193         chomp $cmd;
 194         next if $cmd =~ /^\s*$/;
 195         $cmd .= " 2>&1";
 196         print "$cmd\n" if $VERBOSE;
 197         $output .= `$cmd`;
 198         last if $?;
 199     }
 200     print "$output\n" if $VERBOSE;
 201     return $output;
 202 }
 203
 204 sub chdir_verbose {
 205     my $dir = shift;
 206     print "cd $dir\n" if $VERBOSE;
 207     chdir $dir || die;
 208 }
 209
 210
 211 # Return test names from the command line.
 212 # Returns all tests if no tests were named.
 213 sub gettests {
 214     my @tests;
 215
 216     foreach my $arg (@ARGV) {
 217         push @tests, $arg  if ($arg !~ /=/  &&  $arg !~ /^-/);
 218     }
 219
 220     opendir(my $dir, $DIR) || die;
 221     while (my $file = readdir($dir)) {
 222         my ($name, $ext) = ($file =~ /^([^.]+)\.([^.]+)$/);
 223         next if ! $languages_for_extension{$ext};
 224
 225         open(my $in, "< $file") || die "$file";
 226         my $contents = join "", <$in>;
 227         if (defined $ALL_TESTS{$name}) {
 228             print "${yellow}SKIP: multiple tests named '$name'; skipping file '$file'.${nocolor}\n";
 229         } else {
 230             $ALL_TESTS{$name} = $ext  if ($contents =~ m#^[/*\s]*TEST_#m);
 231         }
 232         close($in);
 233     }
 234     closedir($dir);
 235
 236     if (scalar(@tests) == 0) {
 237         @tests = keys %ALL_TESTS;
 238     }
 239
 240     @tests = sort @tests;
 241
 242     return @tests;
 243 }
 244
 245
 246 # Turn a C compiler name into a C++ compiler name.
 247 sub cplusplus {
 248     my ($c) = @_;
 249     if ($c =~ /cc/) {
 250         $c =~ s/cc/\+\+/;
 251         return $c;
 252     }
 253     return $c . "++";                         # e.g. clang => clang++
 254 }
 255
 256 # Turn a C compiler name into a Swift compiler name
 257 sub swift {
 258     my ($c) = @_;
 259     $c =~ s#[^/]*$#swift#;
 260     return $c;
 261 }
 262
 263 # Returns an array of all sdks from `xcodebuild -showsdks`
 264 my @sdks_memo;
 265 sub getsdks {
 266     if (!@sdks_memo) {
 267         @sdks_memo = (`xcodebuild -showsdks` =~ /-sdk (.+)$/mg);
 268     }
 269     return @sdks_memo;
 270 }
 271
 272 my %sdk_path_memo = {};
 273 sub getsdkpath {
 274     my ($sdk) = @_;
 275     if (!defined $sdk_path_memo{$sdk}) {
 276         ($sdk_path_memo{$sdk}) = (`xcodebuild -version -sdk '$sdk' Path` =~ /^\s*(.+?)\s*$/);
 277     }
 278     return $sdk_path_memo{$sdk};
 279 }
 280
 281 # Extract a version number from a string.
 282 # Ignore trailing "internal".
 283 sub versionsuffix {
 284     my ($str) = @_;
 285     my ($vers) = ($str =~ /([0-9]+\.[0-9]+)(?:\.?internal)?$/);
 286     return $vers;
 287 }
 288 sub majorversionsuffix {
 289     my ($str) = @_;
 290     my ($vers) = ($str =~ /([0-9]+)\.[0-9]+(?:\.?internal)?$/);
 291     return $vers;
 292 }
 293 sub minorversionsuffix {
 294     my ($str) = @_;
 295     my ($vers) = ($str =~ /[0-9]+\.([0-9]+)(?:\.?internal)?$/);
 296     return $vers;
 297 }
 298
 299 # Compares two SDK names and returns the newer one.
 300 # Assumes the two SDKs are the same OS.
 301 sub newersdk {
 302     my ($lhs, $rhs) = @_;
 303
 304     # Major version wins.
 305     my $lhsMajor = majorversionsuffix($lhs);
 306     my $rhsMajor = majorversionsuffix($rhs);
 307     if ($lhsMajor > $rhsMajor) { return $lhs; }
 308     if ($lhsMajor < $rhsMajor) { return $rhs; }
 309
 310     # Minor version wins.
 311     my $lhsMinor = minorversionsuffix($lhs);
 312     my $rhsMinor = minorversionsuffix($rhs);
 313     if ($lhsMinor > $rhsMinor) { return $lhs; }
 314     if ($lhsMinor < $rhsMinor) { return $rhs; }
 315
 316     # Lexically-last wins (i.e. internal is better than not internal)
 317     if ($lhs gt $rhs) { return $lhs; }
 318     return $rhs;
 319 }
 320
 321 # Returns whether the given sdk supports -lauto
 322 sub supportslibauto {
 323     my ($sdk) = @_;
 324     return 1 if $sdk =~ /^macosx/;
 325     return 0;
 326 }
 327
 328 # print text with a colored prefix on each line
 329 sub colorprint {
 330     my $color = shift;
 331     while (my @lines = split("\n", shift)) {
 332         for my $line (@lines) {
 333             chomp $line;
 334             print "$color $nocolor$line\n";
 335         }
 336     }
 337 }
 338
 339 sub rewind {
 340     seek($_[0], 0, 0);
 341 }
 342
 343 # parse name=value,value pairs
 344 sub readconditions {
 345     my ($conditionstring) = @_;
 346
 347     my %results;
 348     my @conditions = ($conditionstring =~ /\w+=(?:[^\s,]+,?)+/g);
 349     for my $condition (@conditions) {
 350         my ($name, $values) = ($condition =~ /(\w+)=(.+)/);
 351         $results{$name} = [split ',', $values];
 352     }
 353
 354     return %results;
 355 }
 356
 357 sub check_output {
 358     my %C = %{shift()};
 359     my $name = shift;
 360     my @output = @_;
 361
 362     my %T = %{$C{"TEST_$name"}};
 363
 364     # Quietly strip MallocScribble before saving the "original" output
 365     # because it is distracting.
 366     filter_malloc(\@output);
 367
 368     my @original_output = @output;
 369
 370     # Run result-checking passes, reducing @output each time
 371     my $xit = 1;
 372     my $bad = "";
 373     my $warn = "";
 374     my $runerror = $T{TEST_RUN_OUTPUT};
 375     filter_hax(\@output);
 376     filter_verbose(\@output);
 377     filter_simulator(\@output);
 378     $warn = filter_warn(\@output);
 379     $bad |= filter_guardmalloc(\@output) if ($C{GUARDMALLOC});
 380     $bad |= filter_valgrind(\@output) if ($C{VALGRIND});
 381     $bad = filter_expected(\@output, \%C, $name) if ($bad eq "");
 382     $bad = filter_bad(\@output)  if ($bad eq "");
 383
 384     # OK line should be the only one left
 385     $bad = "(output not 'OK: $name')" if ($bad eq ""  &&  (scalar(@output) != 1  ||  $output[0] !~ /^OK: $name/));
 386
 387     if ($bad ne "") {
 388         print "${red}FAIL: /// test '$name' \\\\\\$nocolor\n";
 389         colorprint($red, @original_output);
 390         print "${red}FAIL: \\\\\\ test '$name' ///$nocolor\n";
 391         print "${red}FAIL: $name: $bad$nocolor\n";
 392         $xit = 0;
 393     }
 394     elsif ($warn ne "") {
 395         print "${yellow}PASS: /// test '$name' \\\\\\$nocolor\n";
 396         colorprint($yellow, @original_output);
 397         print "${yellow}PASS: \\\\\\ test '$name' ///$nocolor\n";
 398         print "PASS: $name (with warnings)\n";
 399     }
 400     else {
 401         print "PASS: $name\n";
 402     }
 403     return $xit;
 404 }
 405
 406 sub filter_expected
 407 {
 408     my $outputref = shift;
 409     my %C = %{shift()};
 410     my $name = shift;
 411
 412     my %T = %{$C{"TEST_$name"}};
 413     my $runerror = $T{TEST_RUN_OUTPUT}  ||  return "";
 414
 415     my $bad = "";
 416
 417     my $output = join("\n", @$outputref) . "\n";
 418     if ($output !~ /$runerror/) {
 419         $bad = "(run output does not match TEST_RUN_OUTPUT)";
 420         @$outputref = ("FAIL: $name");
 421     } else {
 422         @$outputref = ("OK: $name");  # pacify later filter
 423     }
 424
 425     return $bad;
 426 }
 427
 428 sub filter_bad
 429 {
 430     my $outputref = shift;
 431     my $bad = "";
 432
 433     my @new_output;
 434     for my $line (@$outputref) {
 435         if ($line =~ /^BAD: (.*)/) {
 436             $bad = "(failed)";
 437         } else {
 438             push @new_output, $line;
 439         }
 440     }
 441
 442     @$outputref = @new_output;
 443     return $bad;
 444 }
 445
 446 sub filter_warn
 447 {
 448     my $outputref = shift;
 449     my $warn = "";
 450
 451     my @new_output;
 452     for my $line (@$outputref) {
 453         if ($line !~ /^WARN: (.*)/) {
 454             push @new_output, $line;
 455         } else {
 456             $warn = "(warned)";
 457         }
 458     }
 459
 460     @$outputref = @new_output;
 461     return $warn;
 462 }
 463
 464 sub filter_verbose
 465 {
 466     my $outputref = shift;
 467
 468     my @new_output;
 469     for my $line (@$outputref) {
 470         if ($line !~ /^VERBOSE: (.*)/) {
 471             push @new_output, $line;
 472         }
 473     }
 474
 475     @$outputref = @new_output;
 476 }
 477
 478 sub filter_simulator
 479 {
 480     my $outputref = shift;
 481
 482     my @new_output;
 483     for my $line (@$outputref) {
 484         if ($line !~ /No simulator devices appear to be running/) {
 485             push @new_output, $line;
 486         }
 487     }
 488
 489     @$outputref = @new_output;
 490 }
 491
 492 sub filter_simulator
 493 {
 494     my $outputref = shift;
 495
 496     my @new_output;
 497     for my $line (@$outputref) {
 498         if ($line !~ /No simulator devices appear to be running/) {
 499             push @new_output, $line;
 500         }
 501     }
 502
 503     @$outputref = @new_output;
 504 }
 505
 506 sub filter_hax
 507 {
 508     my $outputref = shift;
 509
 510     my @new_output;
 511     for my $line (@$outputref) {
 512         if ($line !~ /Class OS_tcp_/) {
 513             push @new_output, $line;
 514         }
 515     }
 516
 517     @$outputref = @new_output;
 518 }
 519
 520 sub filter_valgrind
 521 {
 522     my $outputref = shift;
 523     my $errors = 0;
 524     my $leaks = 0;
 525
 526     my @new_output;
 527     for my $line (@$outputref) {
 528         if ($line =~ /^Approx: do_origins_Dirty\([RW]\): missed \d bytes$/) {
 529             # --track-origins warning (harmless)
 530             next;
 531         }
 532         if ($line =~ /^UNKNOWN __disable_threadsignal is unsupported. This warning will not be repeated.$/) {
 533             # signals unsupported (harmless)
 534             next;
 535         }
 536         if ($line =~ /^UNKNOWN __pthread_sigmask is unsupported. This warning will not be repeated.$/) {
 537             # signals unsupported (harmless)
 538             next;
 539         }
 540         if ($line !~ /^^\.*==\d+==/) {
 541             # not valgrind output
 542             push @new_output, $line;
 543             next;
 544         }
 545
 546         my ($errcount) = ($line =~ /==\d+== ERROR SUMMARY: (\d+) errors/);
 547         if (defined $errcount  &&  $errcount > 0) {
 548             $errors = 1;
 549         }
 550
 551         (my $leakcount) = ($line =~ /==\d+==\s+(?:definitely|possibly) lost:\s+([0-9,]+)/);
 552         if (defined $leakcount  &&  $leakcount > 0) {
 553             $leaks = 1;
 554         }
 555     }
 556
 557     @$outputref = @new_output;
 558
 559     my $bad = "";
 560     $bad .= "(valgrind errors)" if ($errors);
 561     $bad .= "(valgrind leaks)" if ($leaks);
 562     return $bad;
 563 }
 564
 565
 566
 567 sub filter_malloc
 568 {
 569     my $outputref = shift;
 570     my $errors = 0;
 571
 572     my @new_output;
 573     my $count = 0;
 574     for my $line (@$outputref) {
 575         # Ignore MallocScribble prologue.
 576         # Ignore MallocStackLogging prologue.
 577         if ($line =~ /malloc: enabling scribbling to detect mods to free/  ||
 578             $line =~ /Deleted objects will be dirtied by the collector/  ||
 579             $line =~ /malloc: stack logs being written into/  ||
 580             $line =~ /malloc: stack logs deleted from/  ||
 581             $line =~ /malloc: process \d+ no longer exists/  ||
 582             $line =~ /malloc: recording malloc and VM allocation stacks/)
 583         {
 584             next;
 585         }
 586
 587         # not malloc output
 588         push @new_output, $line;
 589
 590     }
 591
 592     @$outputref = @new_output;
 593 }
 594
 595 sub filter_guardmalloc
 596 {
 597     my $outputref = shift;
 598     my $errors = 0;
 599
 600     my @new_output;
 601     my $count = 0;
 602     for my $line (@$outputref) {
 603         if ($line !~ /^GuardMalloc\[[^\]]+\]: /) {
 604             # not guardmalloc output
 605             push @new_output, $line;
 606             next;
 607         }
 608
 609         # Ignore 4 lines of guardmalloc prologue.
 610         # Anything further is a guardmalloc error.
 611         if (++$count > 4) {
 612             $errors = 1;
 613         }
 614     }
 615
 616     @$outputref = @new_output;
 617
 618     my $bad = "";
 619     $bad .= "(guardmalloc errors)" if ($errors);
 620     return $bad;
 621 }
 622
 623 # TEST_SOMETHING
 624 # text
 625 # text
 626 # END
 627 sub extract_multiline {
 628     my ($flag, $contents, $name) = @_;
 629     if ($contents =~ /$flag\n/) {
 630         my ($output) = ($contents =~ /$flag\n(.*?\n)END[ *\/]*\n/s);
 631         die "$name used $flag without END\n"  if !defined($output);
 632         return $output;
 633     }
 634     return undef;
 635 }
 636
 637
 638 # TEST_SOMETHING
 639 # text
 640 # OR
 641 # text
 642 # END
 643 sub extract_multiple_multiline {
 644     my ($flag, $contents, $name) = @_;
 645     if ($contents =~ /$flag\n/) {
 646         my ($output) = ($contents =~ /$flag\n(.*?\n)END[ *\/]*\n/s);
 647         die "$name used $flag without END\n"  if !defined($output);
 648
 649         $output =~ s/\nOR\n/\n|/sg;
 650         $output = "^(" . $output . ")\$";
 651         return $output;
 652     }
 653     return undef;
 654 }
 655
 656
 657 sub gather_simple {
 658     my $CREF = shift;
 659     my %C = %{$CREF};
 660     my $name = shift;
 661     chdir_verbose $DIR;
 662
 663     my $ext = $ALL_TESTS{$name};
 664     my $file = "$name.$ext";
 665     return 0 if !$file;
 666
 667     # search file for 'TEST_CONFIG' or '#include "test.h"'
 668     # also collect other values:
 669     # TEST_DISABLED disable test with an optional message
 670     # TEST_CRASHES test is expected to crash
 671     # TEST_CONFIG test conditions
 672     # TEST_ENV environment prefix
 673     # TEST_CFLAGS compile flags
 674     # TEST_BUILD build instructions
 675     # TEST_BUILD_OUTPUT expected build stdout/stderr
 676     # TEST_RUN_OUTPUT expected run stdout/stderr
 677     open(my $in, "< $file") || die;
 678     my $contents = join "", <$in>;
 679
 680     my $test_h = ($contents =~ /^\s*#\s*(include|import)\s*"test\.h"/m);
 681     my ($disabled) = ($contents =~ /\b(TEST_DISABLED\b.*)$/m);
 682     my $crashes = ($contents =~ /\bTEST_CRASHES\b/m);
 683     my ($conditionstring) = ($contents =~ /\bTEST_CONFIG\b(.*)$/m);
 684     my ($envstring) = ($contents =~ /\bTEST_ENV\b(.*)$/m);
 685     my ($cflags) = ($contents =~ /\bTEST_CFLAGS\b(.*)$/m);
 686     my ($buildcmd) = extract_multiline("TEST_BUILD", $contents, $name);
 687     my ($builderror) = extract_multiple_multiline("TEST_BUILD_OUTPUT", $contents, $name);
 688     my ($runerror) = extract_multiple_multiline("TEST_RUN_OUTPUT", $contents, $name);
 689
 690     return 0 if !$test_h && !$disabled && !$crashes && !defined($conditionstring) && !defined($envstring) && !defined($cflags) && !defined($buildcmd) && !defined($builderror) && !defined($runerror);
 691
 692     if ($disabled) {
 693         print "${yellow}SKIP: $name    (disabled by $disabled)$nocolor\n";
 694         return 0;
 695     }
 696
 697     # check test conditions
 698
 699     my $run = 1;
 700     my %conditions = readconditions($conditionstring);
 701     if (! $conditions{LANGUAGE}) {
 702         # implicit language restriction from file extension
 703         $conditions{LANGUAGE} = $languages_for_extension{$ext};
 704     }
 705     for my $condkey (keys %conditions) {
 706         my @condvalues = @{$conditions{$condkey}};
 707
 708         # special case: RUN=0 does not affect build
 709         if ($condkey eq "RUN"  &&  @condvalues == 1  &&  $condvalues[0] == 0) {
 710             $run = 0;
 711             next;
 712         }
 713
 714         my $testvalue = $C{$condkey};
 715         next if !defined($testvalue);
 716         # testvalue is the configuration being run now
 717         # condvalues are the allowed values for this test
 718
 719         my $ok = 0;
 720         for my $condvalue (@condvalues) {
 721
 722             # special case: objc and objc++
 723             if ($condkey eq "LANGUAGE") {
 724                 $condvalue = "objective-c" if $condvalue eq "objc";
 725                 $condvalue = "objective-c++" if $condvalue eq "objc++";
 726             }
 727
 728             $ok = 1  if ($testvalue eq $condvalue);
 729
 730             # special case: CC and CXX allow substring matches
 731             if ($condkey eq "CC"  ||  $condkey eq "CXX") {
 732                 $ok = 1  if ($testvalue =~ /$condvalue/);
 733             }
 734
 735             last if $ok;
 736         }
 737
 738         if (!$ok) {
 739             my $plural = (@condvalues > 1) ? "one of: " : "";
 740             print "SKIP: $name    ($condkey=$testvalue, but test requires $plural", join(' ', @condvalues), ")\n";
 741             return 0;
 742         }
 743     }
 744
 745     # save some results for build and run phases
 746     $$CREF{"TEST_$name"} = {
 747         TEST_BUILD => $buildcmd,
 748         TEST_BUILD_OUTPUT => $builderror,
 749         TEST_CRASHES => $crashes,
 750         TEST_RUN_OUTPUT => $runerror,
 751         TEST_CFLAGS => $cflags,
 752         TEST_ENV => $envstring,
 753         TEST_RUN => $run,
 754     };
 755
 756     return 1;
 757 }
 758
 759 # Builds a simple test
 760 sub build_simple {
 761     my %C = %{shift()};
 762     my $name = shift;
 763     my %T = %{$C{"TEST_$name"}};
 764     chdir_verbose "$C{DIR}/$name.build";
 765
 766     my $ext = $ALL_TESTS{$name};
 767     my $file = "$DIR/$name.$ext";
 768
 769     if ($T{TEST_CRASHES}) {
 770         `echo '$crashcatch' > crashcatch.c`;
 771         make("$C{COMPILE_C} -dynamiclib -o libcrashcatch.dylib -x c crashcatch.c");
 772         die "$?" if $?;
 773     }
 774
 775     my $cmd = $T{TEST_BUILD} ? eval "return \"$T{TEST_BUILD}\"" : "$C{COMPILE}   $T{TEST_CFLAGS} $file -o $name.out";
 776
 777     my $output = make($cmd);
 778
 779     # rdar://10163155
 780     $output =~ s/ld: warning: could not create compact unwind for [^\n]+: does not use standard frame\n//g;
 781
 782     my $ok;
 783     if (my $builderror = $T{TEST_BUILD_OUTPUT}) {
 784         # check for expected output and ignore $?
 785         if ($output =~ /$builderror/) {
 786             $ok = 1;
 787         } else {
 788             print "${red}FAIL: /// test '$name' \\\\\\$nocolor\n";
 789             colorprint $red, $output;
 790             print "${red}FAIL: \\\\\\ test '$name' ///$nocolor\n";
 791             print "${red}FAIL: $name (build output does not match TEST_BUILD_OUTPUT)$nocolor\n";
 792             $ok = 0;
 793         }
 794     } elsif ($?) {
 795         print "${red}FAIL: /// test '$name' \\\\\\$nocolor\n";
 796         colorprint $red, $output;
 797         print "${red}FAIL: \\\\\\ test '$name' ///$nocolor\n";
 798         print "${red}FAIL: $name (build failed)$nocolor\n";
 799         $ok = 0;
 800     } elsif ($output ne "") {
 801         print "${red}FAIL: /// test '$name' \\\\\\$nocolor\n";
 802         colorprint $red, $output;
 803         print "${red}FAIL: \\\\\\ test '$name' ///$nocolor\n";
 804         print "${red}FAIL: $name (unexpected build output)$nocolor\n";
 805         $ok = 0;
 806     } else {
 807         $ok = 1;
 808     }
 809
 810
 811     if ($ok) {
 812         foreach my $file (glob("*.out *.dylib *.bundle")) {
 813             make("dsymutil $file");
 814         }
 815     }
 816
 817     return $ok;
 818 }
 819
 820 # Run a simple test (testname.out, with error checking of stdout and stderr)
 821 sub run_simple {
 822     my %C = %{shift()};
 823     my $name = shift;
 824     my %T = %{$C{"TEST_$name"}};
 825
 826     if (! $T{TEST_RUN}) {
 827         print "PASS: $name (build only)\n";
 828         return 1;
 829     }
 830
 831     my $testdir = "$C{DIR}/$name.build";
 832     chdir_verbose $testdir;
 833
 834     my $env = "$C{ENV} $T{TEST_ENV}";
 835
 836     my $output;
 837
 838     if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) {
 839         # run on iOS or watchos device
 840
 841         my $remotedir = "/var/root/objctest/" . basename($C{DIR}) . "/$name.build";
 842
 843         # Add test dir and libobjc's dir to DYLD_LIBRARY_PATH.
 844         # Insert libcrashcatch.dylib if necessary.
 845         $env .= " DYLD_LIBRARY_PATH=$remotedir";
 846         $env .= ":/var/root/objctest/"  if ($C{TESTLIB} ne $TESTLIBPATH);
 847         if ($T{TEST_CRASHES}) {
 848             $env .= " DYLD_INSERT_LIBRARIES=$remotedir/libcrashcatch.dylib";
 849         }
 850
 851         my $cmd = "ssh iphone 'cd $remotedir && env $env ./$name.out'";
 852         $output = make("$cmd");
 853     }
 854     elsif ($C{OS} =~ /simulator/) {
 855         # run locally in an iOS simulator
 856         # fixme appletvsimulator and watchsimulator
 857         # fixme SDK
 858         my $sim = "xcrun -sdk iphonesimulator simctl spawn 'iPhone 6'";
 859
 860         # Add test dir and libobjc's dir to DYLD_LIBRARY_PATH.
 861         # Insert libcrashcatch.dylib if necessary.
 862         $env .= " DYLD_LIBRARY_PATH=$testdir";
 863         $env .= ":" . dirname($C{TESTLIB})  if ($C{TESTLIB} ne $TESTLIBPATH);
 864         if ($T{TEST_CRASHES}) {
 865             $env .= " DYLD_INSERT_LIBRARIES=$testdir/libcrashcatch.dylib";
 866         }
 867
 868         my $simenv = "";
 869         foreach my $keyvalue (split(' ', $env)) {
 870             $simenv .= "SIMCTL_CHILD_$keyvalue ";
 871         }
 872         # Use the full path here so hack_cwd in test.h works.
 873         $output = make("env $simenv $sim $testdir/$name.out");
 874     }
 875     else {
 876         # run locally
 877
 878         # Add test dir and libobjc's dir to DYLD_LIBRARY_PATH.
 879         # Insert libcrashcatch.dylib if necessary.
 880         $env .= " DYLD_LIBRARY_PATH=$testdir";
 881         $env .= ":" . dirname($C{TESTLIB})  if ($C{TESTLIB} ne $TESTLIBPATH);
 882         if ($T{TEST_CRASHES}) {
 883             $env .= " DYLD_INSERT_LIBRARIES=$testdir/libcrashcatch.dylib";
 884         }
 885
 886         $output = make("sh -c '$env ./$name.out'");
 887     }
 888
 889     return check_output(\%C, $name, split("\n", $output));
 890 }
 891
 892
 893 my %compiler_memo;
 894 sub find_compiler {
 895     my ($cc, $toolchain, $sdk_path) = @_;
 896
 897     # memoize
 898     my $key = $cc . ':' . $toolchain;
 899     my $result = $compiler_memo{$key};
 900     return $result if defined $result;
 901
 902     $result  = make("xcrun -toolchain $toolchain -find $cc 2>/dev/null");
 903
 904     chomp $result;
 905     $compiler_memo{$key} = $result;
 906     return $result;
 907 }
 908
 909 sub make_one_config {
 910     my $configref = shift;
 911     my $root = shift;
 912     my %C = %{$configref};
 913
 914     # Aliases
 915     $C{LANGUAGE} = "objective-c"  if $C{LANGUAGE} eq "objc";
 916     $C{LANGUAGE} = "objective-c++"  if $C{LANGUAGE} eq "objc++";
 917
 918     # Interpret OS version string from command line.
 919     my ($sdk_arg, $deployment_arg, $run_arg, undef) = split('-', $C{OSVERSION});
 920     delete $C{OSVERSION};
 921     my ($os_arg) = ($sdk_arg =~ /^([^\.0-9]+)/);
 922     $deployment_arg = "default" if !defined($deployment_arg);
 923     $run_arg = "default" if !defined($run_arg);
 924
 925
 926     die "unknown OS '$os_arg' (expected iphoneos or iphonesimulator or watchos or watchsimulator or macosx)\n" if ($os_arg ne "iphoneos"  &&  $os_arg ne "iphonesimulator"  &&  $os_arg ne "watchos"  &&  $os_arg ne "watchsimulator"  &&  $os_arg ne "macosx");
 927
 928     $C{OS} = $os_arg;
 929
 930     if ($os_arg eq "iphoneos" || $os_arg eq "iphonesimulator") {
 931         $C{TOOLCHAIN} = "ios";
 932     } elsif ($os_arg eq "watchos" || $os_arg eq "watchsimulator") {
 933         $C{TOOLCHAIN} = "watchos";
 934     } elsif ($os_arg eq "macosx") {
 935         $C{TOOLCHAIN} = "osx";
 936     } else {
 937         print "${yellow}WARN: don't know toolchain for OS $C{OS}${nocolor}\n";
 938         $C{TOOLCHAIN} = "default";
 939     }
 940
 941     # Look up SDK
 942     # Try exact match first.
 943     # Then try lexically-last prefix match (so "macosx" => "macosx10.7internal")
 944     my @sdks = getsdks();
 945     if ($VERBOSE) {
 946         print "note: Installed SDKs: @sdks\n";
 947     }
 948     my $exactsdk = undef;
 949     my $prefixsdk = undef;
 950     foreach my $sdk (@sdks) {
 951         $exactsdk = $sdk  if ($sdk eq $sdk_arg);
 952         $prefixsdk = newersdk($sdk, $prefixsdk)  if ($sdk =~ /^$sdk_arg/);
 953     }
 954
 955     my $sdk;
 956     if ($exactsdk) {
 957         $sdk = $exactsdk;
 958     } elsif ($prefixsdk) {
 959         $sdk = $prefixsdk;
 960     } else {
 961         die "unknown SDK '$sdk_arg'\nInstalled SDKs: @sdks\n";
 962     }
 963
 964     # Set deployment target and run target.
 965     # fixme can't enforce version when run_arg eq "default"
 966     # because we don't know it yet
 967     $deployment_arg = versionsuffix($sdk) if $deployment_arg eq "default";
 968     if ($run_arg ne "default") {
 969         die "Deployment target '$deployment_arg' is newer than run target '$run_arg'\n"  if $deployment_arg > $run_arg;
 970     }
 971     $C{DEPLOYMENT_TARGET} = $deployment_arg;
 972     $C{RUN_TARGET} = $run_arg;
 973
 974     # set the config name now, after massaging the language and OS versions,
 975     # but before adding other settings
 976     my $configname = config_name(%C);
 977     die if ($configname =~ /'/);
 978     die if ($configname =~ / /);
 979     ($C{NAME} = $configname) =~ s/~/ /g;
 980     (my $configdir = $configname) =~ s#/##g;
 981     $C{DIR} = "$BUILDDIR/$configdir";
 982
 983     $C{SDK_PATH} = getsdkpath($sdk);
 984
 985     # Look up test library (possible in root or SDK_PATH)
 986
 987     my $rootarg = $root;
 988     my $symroot;
 989     my @sympaths = ( (glob "$root/*~sym")[0],
 990                      (glob "$root/BuildRecords/*_install/Symbols")[0],
 991                      "$root/Symbols" );
 992     my @dstpaths = ( (glob "$root/*~dst")[0],
 993                      (glob "$root/BuildRecords/*_install/Root")[0],
 994                      "$root/Root" );
 995     for(my $i = 0; $i < scalar(@sympaths); $i++) {
 996         if (-e $sympaths[$i]  &&  -e $dstpaths[$i]) {
 997             $symroot = $sympaths[$i];
 998             $root = $dstpaths[$i];
 999             last;
1000         }
1001     }
1002
1003     if ($root ne ""  &&  -e "$root$C{SDK_PATH}$TESTLIBPATH") {
1004         $C{TESTLIB} = "$root$C{SDK_PATH}$TESTLIBPATH";
1005     } elsif (-e "$root$TESTLIBPATH") {
1006         $C{TESTLIB} = "$root$TESTLIBPATH";
1007     } elsif (-e "$root/$TESTLIBNAME") {
1008         $C{TESTLIB} = "$root/$TESTLIBNAME";
1009     } else {
1010         die "No $TESTLIBNAME in root '$rootarg' for sdk '$C{SDK_PATH}'\n"
1011             # . join("\n", @dstpaths) . "\n"
1012             ;
1013     }
1014
1015     if (-e "$symroot/$TESTLIBNAME.dSYM") {
1016         $C{TESTDSYM} = "$symroot/$TESTLIBNAME.dSYM";
1017     }
1018
1019     if ($VERBOSE) {
1020         my @uuids = `/usr/bin/dwarfdump -u '$C{TESTLIB}'`;
1021         while (my $uuid = shift @uuids) {
1022             print "note: $uuid";
1023         }
1024     }
1025
1026     # Look up compilers
1027     my $cc = $C{CC};
1028     my $cxx = cplusplus($C{CC});
1029     my $swift = swift($C{CC});
1030     if (! $BUILD) {
1031         $C{CC} = $cc;
1032         $C{CXX} = $cxx;
1033         $C{SWIFT} = $swift
1034     } else {
1035         $C{CC} = find_compiler($cc, $C{TOOLCHAIN}, $C{SDK_PATH});
1036         $C{CXX} = find_compiler($cxx, $C{TOOLCHAIN}, $C{SDK_PATH});
1037         $C{SWIFT} = find_compiler($swift, $C{TOOLCHAIN}, $C{SDK_PATH});
1038
1039         die "No compiler '$cc' ('$C{CC}') in toolchain '$C{TOOLCHAIN}'\n" if !-e $C{CC};
1040         die "No compiler '$cxx' ('$C{CXX}') in toolchain '$C{TOOLCHAIN}'\n" if !-e $C{CXX};
1041         die "No compiler '$swift' ('$C{SWIFT}') in toolchain '$C{TOOLCHAIN}'\n" if !-e $C{SWIFT};
1042     }
1043
1044     # Populate cflags
1045
1046     # save-temps so dsymutil works so debug info works
1047     my $cflags = "-I$DIR -W -Wall -Wno-deprecated-declarations -Wshorten-64-to-32 -g -save-temps -Os -arch $C{ARCH} ";
1048     my $objcflags = "";
1049     my $swiftflags = "-g ";
1050
1051     $cflags .= " -isysroot '$C{SDK_PATH}'";
1052     $cflags .= " '-Wl,-syslibroot,$C{SDK_PATH}'";
1053     $swiftflags .= " -sdk '$C{SDK_PATH}'";
1054
1055     # Set deployment target cflags
1056     my $target = undef;
1057     die "No deployment target" if $C{DEPLOYMENT_TARGET} eq "";
1058     if ($C{OS} eq "iphoneos") {
1059         $cflags .= " -mios-version-min=$C{DEPLOYMENT_TARGET}";
1060         $target = "$C{ARCH}-apple-ios$C{DEPLOYMENT_TARGET}";
1061     }
1062     elsif ($C{OS} eq "iphonesimulator") {
1063         $cflags .= " -mios-simulator-version-min=$C{DEPLOYMENT_TARGET}";
1064         $target = "$C{ARCH}-apple-ios$C{DEPLOYMENT_TARGET}";
1065     }
1066     elsif ($C{OS} eq "watchos") {
1067         $cflags .= " -mwatchos-version-min=$C{DEPLOYMENT_TARGET}";
1068         $target = "$C{ARCH}-apple-watchos$C{DEPLOYMENT_TARGET}";
1069     }
1070     elsif ($C{OS} eq "watchsimulator") {
1071         $cflags .= " -mwatch-simulator-version-min=$C{DEPLOYMENT_TARGET}";
1072         $target = "$C{ARCH}-apple-watchos$C{DEPLOYMENT_TARGET}";
1073     }
1074     else {
1075         $cflags .= " -mmacosx-version-min=$C{DEPLOYMENT_TARGET}";
1076         $target = "$C{ARCH}-apple-macosx$C{DEPLOYMENT_TARGET}";
1077     }
1078     $swiftflags .= " -target $target";
1079
1080     # fixme still necessary?
1081     if ($C{OS} eq "iphonesimulator"  &&  $C{ARCH} eq "i386") {
1082         $objcflags .= " -fobjc-abi-version=2 -fobjc-legacy-dispatch";
1083     }
1084
1085     if ($root ne "") {
1086         my $library_path = dirname($C{TESTLIB});
1087         $cflags .= " -L$library_path";
1088         $cflags .= " -I '$root/usr/include'";
1089         $cflags .= " -I '$root/usr/local/include'";
1090
1091         if ($C{SDK_PATH} ne "/") {
1092             $cflags .= " -I '$root$C{SDK_PATH}/usr/include'";
1093             $cflags .= " -I '$root$C{SDK_PATH}/usr/local/include'";
1094         }
1095     }
1096
1097     if ($C{CC} =~ /clang/) {
1098         $cflags .= " -Qunused-arguments -fno-caret-diagnostics";
1099         $cflags .= " -stdlib=$C{STDLIB}"; # fixme -fno-objc-link-runtime"
1100         $cflags .= " -Wl,-segalign,0x4000 ";
1101     }
1102
1103
1104     # Populate objcflags
1105
1106     $objcflags .= " -lobjc";
1107     if ($C{MEM} eq "gc") {
1108         $objcflags .= " -fobjc-gc";
1109     }
1110     elsif ($C{MEM} eq "arc") {
1111         $objcflags .= " -fobjc-arc";
1112     }
1113     elsif ($C{MEM} eq "mrc") {
1114         # nothing
1115     }
1116     else {
1117         die "unrecognized MEM '$C{MEM}'\n";
1118     }
1119
1120     if (supportslibauto($C{OS})) {
1121         # do this even for non-GC tests
1122         $objcflags .= " -lauto";
1123     }
1124
1125     # Populate ENV_PREFIX
1126     $C{ENV} = "LANG=C MallocScribble=1";
1127     $C{ENV} .= " VERBOSE=$VERBOSE"  if $VERBOSE;
1128     if ($root ne "") {
1129         die "no spaces allowed in root" if dirname($C{TESTLIB}) =~ /\s+/;
1130     }
1131     if ($C{GUARDMALLOC}) {
1132         $ENV{GUARDMALLOC} = "1";  # checked by tests and errcheck.pl
1133         $C{ENV} .= " DYLD_INSERT_LIBRARIES=/usr/lib/libgmalloc.dylib";
1134         if ($C{GUARDMALLOC} eq "before") {
1135             $C{ENV} .= " MALLOC_PROTECT_BEFORE=1";
1136         } elsif ($C{GUARDMALLOC} eq "after") {
1137             # protect after is the default
1138         } else {
1139             die "Unknown guard malloc mode '$C{GUARDMALLOC}'\n";
1140         }
1141     }
1142
1143     # Populate compiler commands
1144     $C{COMPILE_C}   = "env LANG=C '$C{CC}'  $cflags -x c -std=gnu99";
1145     $C{COMPILE_CXX} = "env LANG=C '$C{CXX}' $cflags -x c++";
1146     $C{COMPILE_M}   = "env LANG=C '$C{CC}'  $cflags $objcflags -x objective-c -std=gnu99";
1147     $C{COMPILE_MM}  = "env LANG=C '$C{CXX}' $cflags $objcflags -x objective-c++";
1148     $C{COMPILE_SWIFT} = "env LANG=C '$C{SWIFT}' $swiftflags";
1149
1150     $C{COMPILE} = $C{COMPILE_C}      if $C{LANGUAGE} eq "c";
1151     $C{COMPILE} = $C{COMPILE_CXX}    if $C{LANGUAGE} eq "c++";
1152     $C{COMPILE} = $C{COMPILE_M}      if $C{LANGUAGE} eq "objective-c";
1153     $C{COMPILE} = $C{COMPILE_MM}     if $C{LANGUAGE} eq "objective-c++";
1154     $C{COMPILE} = $C{COMPILE_SWIFT}  if $C{LANGUAGE} eq "swift";
1155     die "unknown language '$C{LANGUAGE}'\n" if !defined $C{COMPILE};
1156
1157     ($C{COMPILE_NOMEM} = $C{COMPILE}) =~ s/ -fobjc-(?:gc|arc)\S*//g;
1158     ($C{COMPILE_NOLINK} = $C{COMPILE}) =~ s/ '?-(?:Wl,|l)\S*//g;
1159     ($C{COMPILE_NOLINK_NOMEM} = $C{COMPILE_NOMEM}) =~ s/ '?-(?:Wl,|l)\S*//g;
1160
1161
1162     # Reject some self-inconsistent configurations
1163     if ($C{MEM} !~ /^(mrc|arc|gc)$/) {
1164         die "unknown MEM=$C{MEM} (expected one of mrc arc gc)\n";
1165     }
1166
1167     if ($C{MEM} eq "gc"  &&  $C{OS} !~ /^macosx/) {
1168         print "note: skipping configuration $C{NAME}\n";
1169         print "note:   because OS=$C{OS} does not support MEM=$C{MEM}\n";
1170         return 0;
1171     }
1172     if ($C{MEM} eq "gc"  &&  $C{ARCH} eq "x86_64h") {
1173         print "note: skipping configuration $C{NAME}\n";
1174         print "note:   because ARCH=$C{ARCH} does not support MEM=$C{MEM}\n";
1175         return 0;
1176     }
1177     if ($C{MEM} eq "arc"  &&  $C{OS} =~ /^macosx/  &&  $C{ARCH} eq "i386") {
1178         print "note: skipping configuration $C{NAME}\n";
1179         print "note:   because 32-bit Mac does not support MEM=$C{MEM}\n";
1180         return 0;
1181     }
1182     if ($C{MEM} eq "arc"  &&  $C{CC} !~ /clang/) {
1183         print "note: skipping configuration $C{NAME}\n";
1184         print "note:   because CC=$C{CC} does not support MEM=$C{MEM}\n";
1185         return 0;
1186     }
1187
1188     if ($C{STDLIB} ne "libstdc++"  &&  $C{CC} !~ /clang/) {
1189         print "note: skipping configuration $C{NAME}\n";
1190         print "note:   because CC=$C{CC} does not support STDLIB=$C{STDLIB}\n";
1191         return 0;
1192     }
1193
1194     # fixme
1195     if ($C{LANGUAGE} eq "swift"  &&  $C{ARCH} =~ /^arm/) {
1196         print "note: skipping configuration $C{NAME}\n";
1197         print "note:   because ARCH=$C{ARCH} does not support LANGUAGE=SWIFT\n";
1198         return 0;
1199     }
1200
1201     # fixme unimplemented run targets
1202     if ($C{RUN_TARGET} ne "default" &&  $C{OS} !~ /simulator/) {
1203         print "${yellow}WARN: skipping configuration $C{NAME}${nocolor}\n";
1204         print "${yellow}WARN:   because OS=$C{OS} does not yet implement RUN_TARGET=$C{RUN_TARGET}${nocolor}\n";
1205     }
1206
1207     %$configref = %C;
1208 }
1209
1210 sub make_configs {
1211     my ($root, %args) = @_;
1212
1213     my @results = ({});  # start with one empty config
1214
1215     for my $key (keys %args) {
1216         my @newresults;
1217         my @values = @{$args{$key}};
1218         for my $configref (@results) {
1219             my %config = %{$configref};
1220             for my $value (@values) {
1221                 my %newconfig = %config;
1222                 $newconfig{$key} = $value;
1223                 push @newresults, \%newconfig;
1224             }
1225         }
1226         @results = @newresults;
1227     }
1228
1229     my @newresults;
1230     for my $configref(@results) {
1231         if (make_one_config($configref, $root)) {
1232             push @newresults, $configref;
1233         }
1234     }
1235
1236     return @newresults;
1237 }
1238
1239 sub config_name {
1240     my %config = @_;
1241     my $name = "";
1242     for my $key (sort keys %config) {
1243         $name .= '~'  if $name ne "";
1244         $name .= "$key=$config{$key}";
1245     }
1246     return $name;
1247 }
1248
1249 sub run_one_config {
1250     my %C = %{shift()};
1251     my @tests = @_;
1252
1253     # Build and run
1254     my $testcount = 0;
1255     my $failcount = 0;
1256
1257     my @gathertests;
1258     foreach my $test (@tests) {
1259         if ($VERBOSE) {
1260             print "\nGATHER $test\n";
1261         }
1262
1263         if ($ALL_TESTS{$test}) {
1264             gather_simple(\%C, $test) || next;  # not pass, not fail
1265             push @gathertests, $test;
1266         } else {
1267             die "No test named '$test'\n";
1268         }
1269     }
1270
1271     my @builttests;
1272     if (!$BUILD) {
1273         @builttests = @gathertests;
1274         $testcount = scalar(@gathertests);
1275     } else {
1276         my $configdir = $C{DIR};
1277         print $configdir, "\n"  if $VERBOSE;
1278         mkdir $configdir  || die;
1279
1280         foreach my $test (@gathertests) {
1281             if ($VERBOSE) {
1282                 print "\nBUILD $test\n";
1283             }
1284             mkdir "$configdir/$test.build"  || die;
1285
1286             if ($ALL_TESTS{$test}) {
1287                 $testcount++;
1288                 if (!build_simple(\%C, $test)) {
1289                     $failcount++;
1290                 } else {
1291                     push @builttests, $test;
1292                 }
1293             } else {
1294                 die "No test named '$test'\n";
1295             }
1296         }
1297     }
1298
1299     if (!$RUN  ||  !scalar(@builttests)) {
1300         # nothing to do
1301     }
1302     else {
1303         if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) {
1304             # upload all tests to iOS device
1305             make("RSYNC_PASSWORD=alpine rsync -av $C{DIR} rsync://root\@localhost:10873/root/var/root/objctest/");
1306             die "Couldn't rsync tests to device\n" if ($?);
1307
1308             # upload library to iOS device
1309             if ($C{TESTLIB} ne $TESTLIBPATH) {
1310                 make("RSYNC_PASSWORD=alpine rsync -av $C{TESTLIB} rsync://root\@localhost:10873/root/var/root/objctest/");
1311                 die "Couldn't rsync $C{TESTLIB} to device\n" if ($?);
1312                 make("RSYNC_PASSWORD=alpine rsync -av $C{TESTDSYM} rsync://root\@localhost:10873/root/var/root/objctest/");
1313             }
1314         }
1315
1316         foreach my $test (@builttests) {
1317             print "\nRUN $test\n"  if ($VERBOSE);
1318
1319             if ($ALL_TESTS{$test})
1320             {
1321                 if (!run_simple(\%C, $test)) {
1322                     $failcount++;
1323                 }
1324             } else {
1325                 die "No test named '$test'\n";
1326             }
1327         }
1328     }
1329
1330     return ($testcount, $failcount);
1331 }
1332
1333
1334
1335 # Return value if set by "$argname=value" on the command line
1336 # Return $default if not set.
1337 sub getargs {
1338     my ($argname, $default) = @_;
1339
1340     foreach my $arg (@ARGV) {
1341         my ($value) = ($arg =~ /^$argname=(.+)$/);
1342         return [split ',', $value] if defined $value;
1343     }
1344
1345     return [split ',', $default];
1346 }
1347
1348 # Return 1 or 0 if set by "$argname=1" or "$argname=0" on the
1349 # command line. Return $default if not set.
1350 sub getbools {
1351     my ($argname, $default) = @_;
1352
1353     my @values = @{getargs($argname, $default)};
1354     return [( map { ($_ eq "0") ? 0 : 1 } @values )];
1355 }
1356
1357 # Return an integer if set by "$argname=value" on the
1358 # command line. Return $default if not set.
1359 sub getints {
1360     my ($argname, $default) = @_;
1361
1362     my @values = @{getargs($argname, $default)};
1363     return [( map { int($_) } @values )];
1364 }
1365
1366 sub getarg {
1367     my ($argname, $default) = @_;
1368     my @values = @{getargs($argname, $default)};
1369     die "Only one value allowed for $argname\n"  if @values > 1;
1370     return $values[0];
1371 }
1372
1373 sub getbool {
1374     my ($argname, $default) = @_;
1375     my @values = @{getbools($argname, $default)};
1376     die "Only one value allowed for $argname\n"  if @values > 1;
1377     return $values[0];
1378 }
1379
1380 sub getint {
1381     my ($argname, $default) = @_;
1382     my @values = @{getints($argname, $default)};
1383     die "Only one value allowed for $argname\n"  if @values > 1;
1384     return $values[0];
1385 }
1386
1387
1388 # main
1389 my %args;
1390
1391
1392 my $default_arch = (`/usr/sbin/sysctl hw.optional.x86_64` eq "hw.optional.x86_64: 1\n") ? "x86_64" : "i386";
1393 $args{ARCH} = getargs("ARCH", 0);
1394 $args{ARCH} = getargs("ARCHS", $default_arch)  if !@{$args{ARCH}}[0];
1395
1396 $args{OSVERSION} = getargs("OS", "macosx-default-default");
1397
1398 $args{MEM} = getargs("MEM", "mrc");
1399 $args{LANGUAGE} = [ map { lc($_) } @{getargs("LANGUAGE", "objective-c,swift")} ];
1400 $args{STDLIB} = getargs("STDLIB", "libc++");
1401
1402 $args{CC} = getargs("CC", "clang");
1403
1404 {
1405     my $guardmalloc = getargs("GUARDMALLOC", 0);
1406     # GUARDMALLOC=1 is the same as GUARDMALLOC=before,after
1407     my @guardmalloc2 = ();
1408     for my $arg (@$guardmalloc) {
1409         if ($arg == 1) { push @guardmalloc2, "before";
1410                          push @guardmalloc2, "after"; }
1411         else { push @guardmalloc2, $arg }
1412     }
1413     $args{GUARDMALLOC} = \@guardmalloc2;
1414 }
1415
1416 $BUILD = getbool("BUILD", 1);
1417 $RUN = getbool("RUN", 1);
1418 $VERBOSE = getint("VERBOSE", 0);
1419
1420 my $root = getarg("ROOT", "");
1421 $root =~ s#/*$##;
1422
1423 my @tests = gettests();
1424
1425 print "note: -----\n";
1426 print "note: testing root '$root'\n";
1427
1428 my @configs = make_configs($root, %args);
1429
1430 print "note: -----\n";
1431 print "note: testing ", scalar(@configs), " configurations:\n";
1432 for my $configref (@configs) {
1433     my $configname = $$configref{NAME};
1434     print "note: configuration $configname\n";
1435 }
1436
1437 if ($BUILD) {
1438     `rm -rf '$BUILDDIR'`;
1439     mkdir "$BUILDDIR" || die;
1440 }
1441
1442 my $failed = 0;
1443
1444 my $testconfigs = @configs;
1445 my $failconfigs = 0;
1446 my $testcount = 0;
1447 my $failcount = 0;
1448 for my $configref (@configs) {
1449     my $configname = $$configref{NAME};
1450     print "note: -----\n";
1451     print "note: \nnote: $configname\nnote: \n";
1452
1453     (my $t, my $f) = eval { run_one_config($configref, @tests); };
1454     if ($@) {
1455         chomp $@;
1456         print "${red}FAIL: $configname${nocolor}\n";
1457         print "${red}FAIL: $@${nocolor}\n";
1458         $failconfigs++;
1459     } else {
1460         my $color = ($f ? $red : "");
1461         print "note:\n";
1462         print "${color}note: $configname$nocolor\n";
1463         print "${color}note: $t tests, $f failures$nocolor\n";
1464         $testcount += $t;
1465         $failcount += $f;
1466         $failconfigs++ if ($f);
1467     }
1468 }
1469
1470 print "note: -----\n";
1471 my $color = ($failconfigs ? $red : "");
1472 print "${color}note: $testconfigs configurations, $failconfigs with failures$nocolor\n";
1473 print "${color}note: $testcount tests, $failcount failures$nocolor\n";
1474
1475 $failed = ($failconfigs ? 1 : 0);
1476
1477 exit ($failed ? 1 : 0);