test/test.pl

   1 #!/usr/bin/perl
   2
   3 # test.pl
   4 # Run unit tests.
   5
   6 use strict;
   7 use File::Basename;
   8
   9 use Config;
  10 my $supportsParallelBuilds = $Config{useithreads};
  11
  12 if ($supportsParallelBuilds) {
  13     require threads;
  14     import threads;
  15     require Thread::Queue;
  16     import Thread::Queue;
  17 }
  18
  19 # We use encode_json() to write BATS plist files.
  20 # JSON::PP does not exist on iOS devices, but we need not write plists there.
  21 # So we simply load JSON:PP if it exists.
  22 if (eval { require JSON::PP; 1; }) {
  23     JSON::PP->import();
  24 }
  25
  26 # iOS also doesn't have Text::Glob. We don't need it there.
  27 my $has_match_glob = 0;
  28 if (eval { require Text::Glob; 1; }) {
  29     Text::Glob->import();
  30     $has_match_glob = 1;
  31 }
  32
  33
  34 chdir dirname $0;
  35 chomp (my $DIR = `pwd`);
  36
  37 if (scalar(@ARGV) == 1) {
  38     my $arg = $ARGV[0];
  39     if ($arg eq "-h" || $arg eq "-H" || $arg eq "-help" || $arg eq "help") {
  40         print(<<END);
  41 usage: $0 [options] [testname ...]
  42        $0 help
  43
  44 testname:
  45     `testname` runs a specific test. If no testnames are given, runs all tests.
  46
  47 options:
  48     ARCH=<arch>
  49     OS=<sdk name>[sdk version][-<deployment target>[-<run target>]]
  50     ROOT=/path/to/project.roots/
  51     HOST=<test device hostname>
  52     DEVICE=<simulator test device name>
  53
  54     CC=<compiler name>
  55
  56     LANGUAGE=c,c++,objective-c,objective-c++,swift
  57     MEM=mrc,arc
  58     GUARDMALLOC=0|1|before|after
  59
  60     BUILD=0|1      (build the tests?)
  61     RUN=0|1        (run the tests?)
  62     VERBOSE=0|1|2  (0=quieter  1=print commands executed  2=full test output)
  63     BATS=0|1       (build for and/or run in BATS?)
  64     BUILD_SHARED_CACHE=0|1  (build a dyld shared cache with the root and test against that)
  65     DYLD=2|3       (test in dyld 2 or dyld 3 mode)
  66     PARALLELBUILDS=N  (number of parallel builds to run simultaneously)
  67     SHAREDCACHEDIR=/path/to/custom/shared/cache/directory
  68
  69 examples:
  70
  71     test installed library, x86_64
  72     $0
  73
  74     test buildit-built root, i386 and x86_64, MRC and ARC, clang compiler
  75     $0 ARCH=i386,x86_64 ROOT=/tmp/objc4.roots MEM=mrc,arc CC=clang
  76
  77     test buildit-built root with iOS simulator, deploy to iOS 7, run on iOS 8
  78     $0 ARCH=x86_64 ROOT=/tmp/objc4.roots OS=iphonesimulator-7.0-8.0
  79
  80     test buildit-built root on attached iOS device
  81     $0 ARCH=arm64 ROOT=/tmp/objc4.roots OS=iphoneos
  82 END
  83         exit 0;
  84     }
  85 }
  86
  87 #########################################################################
  88 ## Tests
  89
  90 # Maps test name => test's filename extension.
  91 # ex: "msgSend" => "m"
  92 # `keys %ALL_TESTS` is also used as the list of all tests found on disk.
  93 my %ALL_TESTS;
  94
  95 #########################################################################
  96 ## Variables for use in complex build and run rules
  97
  98 # variable         # example value
  99
 100 # things you can multiplex on the command line
 101 # ARCH=i386,x86_64,armv6,armv7
 102 # OS=macosx,iphoneos,iphonesimulator (plus sdk/deployment/run versions)
 103 # LANGUAGE=c,c++,objective-c,objective-c++,swift
 104 # CC=clang
 105 # MEM=mrc,arc
 106 # GUARDMALLOC=0,1,before,after
 107
 108 # things you can set once on the command line
 109 # ROOT=/path/to/project.roots
 110 # BUILD=0|1
 111 # RUN=0|1
 112 # VERBOSE=0|1|2
 113 # BATS=0|1
 114
 115 # environment variables from the command line
 116 # DSTROOT
 117 # OBJROOT
 118 # (SRCROOT is ignored; test sources are assumed to
 119 # be in the same directory as the test script itself.)
 120 # fixme SYMROOT for dsymutil output?
 121
 122
 123 # Some arguments as read from the command line.
 124 my %args;
 125 my $BUILD;
 126 my $RUN;
 127 my $VERBOSE;
 128 my $BATS;
 129
 130 my $HOST;
 131 my $PORT;
 132 my $DEVICE;
 133
 134 my $PARALLELBUILDS;
 135
 136 my $SHAREDCACHEDIR;
 137
 138 my @TESTLIBNAMES = ("libobjc.A.dylib", "libobjc-trampolines.dylib");
 139 my $TESTLIBDIR = "/usr/lib";
 140
 141 # Top level directory for intermediate and final build products.
 142 # Intermediate files must be kept separate for XBS BATS builds.
 143 my $OBJROOT = $ENV{OBJROOT} || "";
 144 my $DSTROOT = $ENV{DSTROOT} || "";
 145
 146 # Build product directory inside DSTROOT and OBJROOT.
 147 # Each test config gets its own build directory inside this.
 148 my $BUILDDIR;
 149
 150 # Local top-level directory.
 151 # This is the default value for $BUILDDIR.
 152 my $LOCALBASE = "/tmp/test-$TESTLIBNAMES[0]-build";
 153
 154 # Device-side top-level directory.
 155 # This replaces $DSTROOT$BUILDDIR/ for on-device execution.
 156 my $REMOTEBASE = "/AppleInternal/objctest";
 157
 158 # BATS top-level directory.
 159 # This replaces $DSTROOT$BUILDDIR/ for BATS execution.
 160 my $BATSBASE = "/AppleInternal/CoreOS/tests/objc4";
 161
 162
 163 my $crashcatch = <<'END';
 164 // interpose-able code to catch crashes, print, and exit cleanly
 165 #include <signal.h>
 166 #include <string.h>
 167 #include <unistd.h>
 168
 169 // from dyld-interposing.h
 170 #define DYLD_INTERPOSE(_replacement,_replacee) __attribute__((used)) static struct{ const void* replacement; const void* replacee; } _interpose_##_replacee __attribute__ ((section ("__DATA,__interpose"))) = { (const void*)(unsigned long)&_replacement, (const void*)(unsigned long)&_replacee };
 171
 172 static void catchcrash(int sig)
 173 {
 174     const char *msg;
 175     switch (sig) {
 176     case SIGILL:  msg = "CRASHED: SIGILL";  break;
 177     case SIGBUS:  msg = "CRASHED: SIGBUS";  break;
 178     case SIGSYS:  msg = "CRASHED: SIGSYS";  break;
 179     case SIGSEGV: msg = "CRASHED: SIGSEGV"; break;
 180     case SIGTRAP: msg = "CRASHED: SIGTRAP"; break;
 181     case SIGABRT: msg = "CRASHED: SIGABRT"; break;
 182     default: msg = "unknown signal"; break;
 183     }
 184     write(STDERR_FILENO, msg, strlen(msg));
 185
 186     // avoid backslash-n newline due to escaping differences somewhere
 187     // in BATS versus local execution (perhaps different perl versions?)
 188     char newline = 0xa;
 189     write(STDERR_FILENO, &newline, 1);
 190
 191     _exit(1);
 192 }
 193
 194 static void setupcrash(void) __attribute__((constructor));
 195 static void setupcrash(void)
 196 {
 197     signal(SIGILL, &catchcrash);
 198     signal(SIGBUS, &catchcrash);
 199     signal(SIGSYS, &catchcrash);
 200     signal(SIGSEGV, &catchcrash);
 201     signal(SIGTRAP, &catchcrash);
 202     signal(SIGABRT, &catchcrash);
 203 }
 204
 205
 206 static int hacked = 0;
 207 ssize_t hacked_write(int fildes, const void *buf, size_t nbyte)
 208 {
 209     if (!hacked) {
 210         setupcrash();
 211         hacked = 1;
 212     }
 213     return write(fildes, buf, nbyte);
 214 }
 215
 216 DYLD_INTERPOSE(hacked_write, write);
 217
 218 END
 219
 220
 221 #########################################################################
 222 ## Harness
 223
 224
 225 # map language to buildable extensions for that language
 226 my %extensions_for_language = (
 227     "c"     => ["c"],
 228     "objective-c" => ["c", "m"],
 229     "c++" => ["c", "cc", "cp", "cpp", "cxx", "c++"],
 230     "objective-c++" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"],
 231     "swift" => ["swift"],
 232
 233     "any" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm", "swift"],
 234     );
 235
 236 # map extension to languages
 237 my %languages_for_extension = (
 238     "c" => ["c", "objective-c", "c++", "objective-c++"],
 239     "m" => ["objective-c", "objective-c++"],
 240     "mm" => ["objective-c++"],
 241     "cc" => ["c++", "objective-c++"],
 242     "cp" => ["c++", "objective-c++"],
 243     "cpp" => ["c++", "objective-c++"],
 244     "cxx" => ["c++", "objective-c++"],
 245     "c++" => ["c++", "objective-c++"],
 246     "swift" => ["swift"],
 247     );
 248
 249 # Run some newline-separated commands like `make` would, stopping if any fail
 250 # run("cmd1 \n cmd2 \n cmd3")
 251 sub make {
 252     my ($cmdstr, $cwd) = @_;
 253     my $output = "";
 254     my @cmds = split("\n", $cmdstr);
 255     die if scalar(@cmds) == 0;
 256     $? = 0;
 257     foreach my $cmd (@cmds) {
 258         chomp $cmd;
 259         next if $cmd =~ /^\s*$/;
 260         $cmd .= " 2>&1";
 261         if (defined $cwd) {
 262             $cmd = "cd $cwd; $cmd";
 263         }
 264         print "$cmd\n" if $VERBOSE;
 265         $output .= `$cmd`;
 266         last if $?;
 267     }
 268     print "$output\n" if $VERBOSE;
 269     return $output;
 270 }
 271
 272 sub chdir_verbose {
 273     my $dir = shift || die;
 274     print "cd $dir\n" if $VERBOSE;
 275     chdir $dir || die "couldn't cd $dir";
 276 }
 277
 278 sub rm_rf_verbose {
 279     my $dir = shift || die;
 280     print "rm -rf $dir\n" if $VERBOSE;
 281     `rm -rf '$dir'`;
 282     die "couldn't rm -rf $dir" if $?;
 283 }
 284
 285 sub mkdir_verbose {
 286     my $dir = shift || die;
 287     print "mkdir -p $dir\n" if $VERBOSE;
 288     `mkdir -p '$dir'`;
 289     die "couldn't mkdir $dir" if $?;
 290 }
 291
 292
 293 # xterm colors
 294 my $red = "\e[41;37m";
 295 my $yellow = "\e[43;30m";
 296 my $nocolor = "\e[0m";
 297 if (! -t STDIN) {
 298     # Not isatty. Don't use colors.
 299     $red = "";
 300     $yellow = "";
 301     $nocolor = "";
 302 }
 303
 304 # print text with a colored prefix on each line
 305 # fixme some callers pass an array of lines and some don't
 306 sub colorprefix {
 307     my $color = shift;
 308     while (defined(my $lines = shift)) {
 309         $lines = "\n" if ($lines eq "");
 310         for my $line (split(/^/, $lines)) {
 311             chomp $line;
 312             print "$color $nocolor$line\n";
 313         }
 314     }
 315 }
 316
 317 # print text colored
 318 # fixme some callers pass an array of lines and some don't
 319 sub colorprint {
 320     my $color = shift;
 321     while (defined(my $lines = shift)) {
 322         $lines = "\n" if ($lines eq "");
 323         for my $line (split(/^/, $lines)) {
 324             chomp $line;
 325             print "$color$line$nocolor\n";
 326         }
 327     }
 328 }
 329
 330 # Return test names from the command line.
 331 # Returns all tests if no tests were named.
 332 sub gettests {
 333     my @tests;
 334
 335     foreach my $arg (@ARGV) {
 336         push @tests, $arg  if ($arg !~ /=/  &&  $arg !~ /^-/);
 337     }
 338
 339     opendir(my $dir, $DIR) || die;
 340     while (my $file = readdir($dir)) {
 341         my ($name, $ext) = ($file =~ /^([^.]+)\.([^.]+)$/);
 342         next if ! $languages_for_extension{$ext};
 343
 344         open(my $in, "< $file") || die "$file";
 345         my $contents = join "", <$in>;
 346         if (defined $ALL_TESTS{$name}) {
 347             colorprint $yellow, "SKIP: multiple tests named '$name'; skipping file '$file'.";
 348         } else {
 349             $ALL_TESTS{$name} = $ext  if ($contents =~ m#^[/*\s]*TEST_#m);
 350         }
 351         close($in);
 352     }
 353     closedir($dir);
 354
 355     if (scalar(@tests) == 0) {
 356         @tests = keys %ALL_TESTS;
 357     }
 358
 359     @tests = sort @tests;
 360
 361     return @tests;
 362 }
 363
 364
 365 # Turn a C compiler name into a C++ compiler name.
 366 sub cplusplus {
 367     my ($c) = @_;
 368     if ($c =~ /cc/) {
 369         $c =~ s/cc/\+\+/;
 370         return $c;
 371     }
 372     return $c . "++";                         # e.g. clang => clang++
 373 }
 374
 375 # Turn a C compiler name into a Swift compiler name
 376 sub swift {
 377     my ($c) = @_;
 378     $c =~ s#[^/]*$#swift#;
 379     return $c;
 380 }
 381
 382 # Returns an array of all sdks from `xcodebuild -showsdks`
 383 my @sdks_memo;
 384 sub getsdks {
 385     if (!@sdks_memo) {
 386         @sdks_memo = (`xcodebuild -showsdks` =~ /-sdk (.+)$/mg);
 387     }
 388     return @sdks_memo;
 389 }
 390
 391 my %sdk_path_memo = {};
 392 sub getsdkpath {
 393     my ($sdk) = @_;
 394     if (!defined $sdk_path_memo{$sdk}) {
 395         ($sdk_path_memo{$sdk}) = (`xcodebuild -version -sdk '$sdk' Path` =~ /^\s*(.+?)\s*$/);
 396     }
 397     return $sdk_path_memo{$sdk};
 398 }
 399
 400 # Extract a version number from a string.
 401 # Ignore trailing "internal".
 402 sub versionsuffix {
 403     my ($str) = @_;
 404     my ($vers) = ($str =~ /([0-9]+\.[0-9]+)(?:\.?internal)?$/);
 405     return $vers;
 406 }
 407 sub majorversionsuffix {
 408     my ($str) = @_;
 409     my ($vers) = ($str =~ /([0-9]+)\.[0-9]+(?:\.?internal)?$/);
 410     return $vers;
 411 }
 412 sub minorversionsuffix {
 413     my ($str) = @_;
 414     my ($vers) = ($str =~ /[0-9]+\.([0-9]+)(?:\.?internal)?$/);
 415     return $vers;
 416 }
 417
 418 # Compares two SDK names and returns the newer one.
 419 # Assumes the two SDKs are the same OS.
 420 sub newersdk {
 421     my ($lhs, $rhs) = @_;
 422
 423     # Major version wins.
 424     my $lhsMajor = majorversionsuffix($lhs);
 425     my $rhsMajor = majorversionsuffix($rhs);
 426     if ($lhsMajor > $rhsMajor) { return $lhs; }
 427     if ($lhsMajor < $rhsMajor) { return $rhs; }
 428
 429     # Minor version wins.
 430     my $lhsMinor = minorversionsuffix($lhs);
 431     my $rhsMinor = minorversionsuffix($rhs);
 432     if ($lhsMinor > $rhsMinor) { return $lhs; }
 433     if ($lhsMinor < $rhsMinor) { return $rhs; }
 434
 435     # Lexically-last wins (i.e. internal is better than not internal)
 436     if ($lhs gt $rhs) { return $lhs; }
 437     return $rhs;
 438 }
 439
 440 sub rewind {
 441     seek($_[0], 0, 0);
 442 }
 443
 444 # parse name=value,value pairs
 445 sub readconditions {
 446     my ($conditionstring) = @_;
 447
 448     my %results;
 449     my @conditions = ($conditionstring =~ /\w+=(?:[^\s,]+,?)+/g);
 450     for my $condition (@conditions) {
 451         my ($name, $values) = ($condition =~ /(\w+)=(.+)/);
 452         $results{$name} = [split ',', $values];
 453     }
 454
 455     return %results;
 456 }
 457
 458 sub check_output {
 459     my %C = %{shift()};
 460     my $name = shift;
 461     my @output = @_;
 462
 463     my %T = %{$C{"TEST_$name"}};
 464
 465     # Quietly strip MallocScribble before saving the "original" output
 466     # because it is distracting.
 467     filter_malloc(\@output);
 468
 469     my @original_output = @output;
 470
 471     # Run result-checking passes, reducing @output each time
 472     my $xit = 1;
 473     my $bad = "";
 474     my $warn = "";
 475     my $runerror = $T{TEST_RUN_OUTPUT};
 476     filter_hax(\@output);
 477     filter_verbose(\@output);
 478     filter_simulator(\@output);
 479     $warn = filter_warn(\@output);
 480     $bad |= filter_guardmalloc(\@output) if ($C{GUARDMALLOC});
 481     $bad |= filter_valgrind(\@output) if ($C{VALGRIND});
 482     $bad = filter_expected(\@output, \%C, $name) if ($bad eq "");
 483     $bad = filter_bad(\@output)  if ($bad eq "");
 484
 485     # OK line should be the only one left
 486     $bad = "(output not 'OK: $name')" if ($bad eq ""  &&  (scalar(@output) != 1  ||  $output[0] !~ /^OK: $name/));
 487
 488     if ($bad ne "") {
 489         colorprint  $red, "FAIL: /// test '$name' \\\\\\";
 490         colorprefix $red, @original_output;
 491         colorprint  $red, "FAIL: \\\\\\ test '$name' ///";
 492         colorprint  $red, "FAIL: $name: $bad";
 493         $xit = 0;
 494     }
 495     elsif ($warn ne "") {
 496         colorprint  $yellow, "PASS: /// test '$name' \\\\\\";
 497         colorprefix $yellow, @original_output;
 498         colorprint  $yellow, "PASS: \\\\\\ test '$name' ///";
 499         print "PASS: $name (with warnings)\n";
 500     }
 501     else {
 502         print "PASS: $name\n";
 503     }
 504     return $xit;
 505 }
 506
 507 sub filter_expected
 508 {
 509     my $outputref = shift;
 510     my %C = %{shift()};
 511     my $name = shift;
 512
 513     my %T = %{$C{"TEST_$name"}};
 514     my $runerror = $T{TEST_RUN_OUTPUT}  ||  return "";
 515
 516     my $bad = "";
 517
 518     my $output = join("\n", @$outputref) . "\n";
 519     if ($output !~ /$runerror/) {
 520         $bad = "(run output does not match TEST_RUN_OUTPUT)";
 521         @$outputref = ("FAIL: $name");
 522     } else {
 523         @$outputref = ("OK: $name");  # pacify later filter
 524     }
 525
 526     return $bad;
 527 }
 528
 529 sub filter_bad
 530 {
 531     my $outputref = shift;
 532     my $bad = "";
 533
 534     my @new_output;
 535     for my $line (@$outputref) {
 536         if ($line =~ /^BAD: (.*)/) {
 537             $bad = "(failed)";
 538         } else {
 539             push @new_output, $line;
 540         }
 541     }
 542
 543     @$outputref = @new_output;
 544     return $bad;
 545 }
 546
 547 sub filter_warn
 548 {
 549     my $outputref = shift;
 550     my $warn = "";
 551
 552     my @new_output;
 553     for my $line (@$outputref) {
 554         if ($line !~ /^WARN: (.*)/) {
 555             push @new_output, $line;
 556         } else {
 557             $warn = "(warned)";
 558         }
 559     }
 560
 561     @$outputref = @new_output;
 562     return $warn;
 563 }
 564
 565 sub filter_verbose
 566 {
 567     my $outputref = shift;
 568
 569     my @new_output;
 570     for my $line (@$outputref) {
 571         if ($line !~ /^VERBOSE: (.*)/) {
 572             push @new_output, $line;
 573         }
 574     }
 575
 576     @$outputref = @new_output;
 577 }
 578
 579 sub filter_simulator
 580 {
 581     my $outputref = shift;
 582
 583     my @new_output;
 584     for my $line (@$outputref) {
 585         if (($line !~ /No simulator devices appear to be running/)  &&
 586             ($line !~ /CoreSimulator is attempting to unload a stale CoreSimulatorService job/)  &&
 587             ($line !~ /Failed to locate a valid instance of CoreSimulatorService/))
 588         {
 589             push @new_output, $line;
 590         }
 591     }
 592
 593     @$outputref = @new_output;
 594 }
 595
 596 sub filter_hax
 597 {
 598     my $outputref = shift;
 599
 600     my @new_output;
 601     for my $line (@$outputref) {
 602         if ($line !~ /Class OS_tcp_/) {
 603             push @new_output, $line;
 604         }
 605     }
 606
 607     @$outputref = @new_output;
 608 }
 609
 610 sub filter_valgrind
 611 {
 612     my $outputref = shift;
 613     my $errors = 0;
 614     my $leaks = 0;
 615
 616     my @new_output;
 617     for my $line (@$outputref) {
 618         if ($line =~ /^Approx: do_origins_Dirty\([RW]\): missed \d bytes$/) {
 619             # --track-origins warning (harmless)
 620             next;
 621         }
 622         if ($line =~ /^UNKNOWN __disable_threadsignal is unsupported. This warning will not be repeated.$/) {
 623             # signals unsupported (harmless)
 624             next;
 625         }
 626         if ($line =~ /^UNKNOWN __pthread_sigmask is unsupported. This warning will not be repeated.$/) {
 627             # signals unsupported (harmless)
 628             next;
 629         }
 630         if ($line !~ /^^\.*==\d+==/) {
 631             # not valgrind output
 632             push @new_output, $line;
 633             next;
 634         }
 635
 636         my ($errcount) = ($line =~ /==\d+== ERROR SUMMARY: (\d+) errors/);
 637         if (defined $errcount  &&  $errcount > 0) {
 638             $errors = 1;
 639         }
 640
 641         (my $leakcount) = ($line =~ /==\d+==\s+(?:definitely|possibly) lost:\s+([0-9,]+)/);
 642         if (defined $leakcount  &&  $leakcount > 0) {
 643             $leaks = 1;
 644         }
 645     }
 646
 647     @$outputref = @new_output;
 648
 649     my $bad = "";
 650     $bad .= "(valgrind errors)" if ($errors);
 651     $bad .= "(valgrind leaks)" if ($leaks);
 652     return $bad;
 653 }
 654
 655
 656
 657 sub filter_malloc
 658 {
 659     my $outputref = shift;
 660     my $errors = 0;
 661
 662     my @new_output;
 663     my $count = 0;
 664     for my $line (@$outputref) {
 665         # Ignore MallocScribble prologue.
 666         # Ignore MallocStackLogging prologue.
 667         if ($line =~ /malloc: enabling scribbling to detect mods to free/  ||
 668             $line =~ /Deleted objects will be dirtied by the collector/  ||
 669             $line =~ /malloc: stack logs being written into/  ||
 670             $line =~ /malloc: stack logs deleted from/  ||
 671             $line =~ /malloc: process \d+ no longer exists/  ||
 672             $line =~ /malloc: recording malloc and VM allocation stacks/)
 673         {
 674             next;
 675         }
 676
 677         # not malloc output
 678         push @new_output, $line;
 679
 680     }
 681
 682     @$outputref = @new_output;
 683 }
 684
 685 sub filter_guardmalloc
 686 {
 687     my $outputref = shift;
 688     my $errors = 0;
 689
 690     my @new_output;
 691     my $count = 0;
 692     for my $line (@$outputref) {
 693         if ($line !~ /^GuardMalloc\[[^\]]+\]: /) {
 694             # not guardmalloc output
 695             push @new_output, $line;
 696             next;
 697         }
 698
 699         # Ignore 4 lines of guardmalloc prologue.
 700         # Anything further is a guardmalloc error.
 701         if (++$count > 4) {
 702             $errors = 1;
 703         }
 704     }
 705
 706     @$outputref = @new_output;
 707
 708     my $bad = "";
 709     $bad .= "(guardmalloc errors)" if ($errors);
 710     return $bad;
 711 }
 712
 713 # TEST_SOMETHING
 714 # text
 715 # text
 716 # END
 717 sub extract_multiline {
 718     my ($flag, $contents, $name) = @_;
 719     if ($contents =~ /$flag\n/) {
 720         my ($output) = ($contents =~ /$flag\n(.*?\n)END[ *\/]*\n/s);
 721         die "$name used $flag without END\n"  if !defined($output);
 722         return $output;
 723     }
 724     return undef;
 725 }
 726
 727
 728 # TEST_SOMETHING
 729 # text
 730 # OR
 731 # text
 732 # END
 733 sub extract_multiple_multiline {
 734     my ($flag, $contents, $name) = @_;
 735     if ($contents =~ /$flag\n/) {
 736         my ($output) = ($contents =~ /$flag\n(.*?\n)END[ *\/]*\n/s);
 737         die "$name used $flag without END\n"  if !defined($output);
 738
 739         $output =~ s/\nOR\n/\n|/sg;
 740         $output = "^(" . $output . ")\$";
 741         return $output;
 742     }
 743     return undef;
 744 }
 745
 746
 747 sub gather_simple {
 748     my $CREF = shift;
 749     my %C = %{$CREF};
 750     my $name = shift;
 751     chdir_verbose $DIR;
 752
 753     my $ext = $ALL_TESTS{$name};
 754     my $file = "$name.$ext";
 755     return 0 if !$file;
 756
 757     # search file for 'TEST_CONFIG' or '#include "test.h"'
 758     # also collect other values:
 759     # TEST_DISABLED disable test with an optional message
 760     # TEST_CRASHES test is expected to crash
 761     # TEST_CONFIG test conditions
 762     # TEST_ENV environment prefix
 763     # TEST_CFLAGS compile flags
 764     # TEST_BUILD build instructions
 765     # TEST_BUILD_OUTPUT expected build stdout/stderr
 766     # TEST_RUN_OUTPUT expected run stdout/stderr
 767     # TEST_ENTITLEMENTS path to entitlements file
 768     open(my $in, "< $file") || die;
 769     my $contents = join "", <$in>;
 770
 771     my $test_h = ($contents =~ /^\s*#\s*(include|import)\s*"test\.h"/m);
 772     my ($disabled) = ($contents =~ /\b(TEST_DISABLED\b.*)$/m);
 773     my $crashes = ($contents =~ /\bTEST_CRASHES\b/m);
 774     my ($conditionstring) = ($contents =~ /\bTEST_CONFIG\b(.*)$/m);
 775     my ($envstring) = ($contents =~ /\bTEST_ENV\b(.*)$/m);
 776     my ($cflags) = ($contents =~ /\bTEST_CFLAGS\b(.*)$/m);
 777     my ($entitlements) = ($contents =~ /\bTEST_ENTITLEMENTS\b(.*)$/m);
 778     $entitlements =~ s/^\s+|\s+$//g;
 779     my ($buildcmd) = extract_multiline("TEST_BUILD", $contents, $name);
 780     my ($builderror) = extract_multiple_multiline("TEST_BUILD_OUTPUT", $contents, $name);
 781     my ($runerror) = extract_multiple_multiline("TEST_RUN_OUTPUT", $contents, $name);
 782
 783     return 0 if !$test_h && !$disabled && !$crashes && !defined($conditionstring)
 784                 && !defined($envstring) && !defined($cflags) && !defined($buildcmd)
 785                 && !defined($builderror) && !defined($runerror) && !defined($entitlements);
 786
 787     if ($disabled) {
 788         colorprint $yellow, "SKIP: $name    (disabled by $disabled)";
 789         return 0;
 790     }
 791
 792     # check test conditions
 793
 794     my $run = 1;
 795     my %conditions = readconditions($conditionstring);
 796     if (! $conditions{LANGUAGE}) {
 797         # implicit language restriction from file extension
 798         $conditions{LANGUAGE} = $languages_for_extension{$ext};
 799     }
 800     for my $condkey (keys %conditions) {
 801         my @condvalues = @{$conditions{$condkey}};
 802
 803         # special case: RUN=0 does not affect build
 804         if ($condkey eq "RUN"  &&  @condvalues == 1  &&  $condvalues[0] == 0) {
 805             $run = 0;
 806             next;
 807         }
 808
 809         my $testvalue = $C{$condkey};
 810         next if !defined($testvalue);
 811         # testvalue is the configuration being run now
 812         # condvalues are the allowed values for this test
 813
 814         my $ok = 0;
 815         for my $condvalue (@condvalues) {
 816
 817             # special case: objc and objc++
 818             if ($condkey eq "LANGUAGE") {
 819                 $condvalue = "objective-c" if $condvalue eq "objc";
 820                 $condvalue = "objective-c++" if $condvalue eq "objc++";
 821             }
 822
 823             $ok = 1  if ($testvalue eq $condvalue);
 824
 825             # special case: CC and CXX allow substring matches
 826             if ($condkey eq "CC"  ||  $condkey eq "CXX") {
 827                 $ok = 1  if ($testvalue =~ /$condvalue/);
 828             }
 829
 830             last if $ok;
 831         }
 832
 833         if (!$ok) {
 834             my $plural = (@condvalues > 1) ? "one of: " : "";
 835             print "SKIP: $name    ($condkey=$testvalue, but test requires $plural", join(' ', @condvalues), ")\n";
 836             return 0;
 837         }
 838     }
 839
 840     # save some results for build and run phases
 841     $$CREF{"TEST_$name"} = {
 842         TEST_BUILD => $buildcmd,
 843         TEST_BUILD_OUTPUT => $builderror,
 844         TEST_CRASHES => $crashes,
 845         TEST_RUN_OUTPUT => $runerror,
 846         TEST_CFLAGS => $cflags,
 847         TEST_ENV => $envstring,
 848         TEST_RUN => $run,
 849         DSTDIR => "$C{DSTDIR}/$name.build",
 850         OBJDIR => "$C{OBJDIR}/$name.build",
 851         ENTITLEMENTS => $entitlements,
 852     };
 853
 854     return 1;
 855 }
 856
 857
 858 # Test description plist to write when building for BATS execution.
 859 my %bats_plist;
 860 $bats_plist{'Project'} = "objc4";
 861 $bats_plist{'Tests'} = [];  # populated by append_bats_test()
 862
 863 # Saves run instructions for a single test in all configurations as a BATS test.
 864 sub append_bats_test {
 865     my $name = shift;
 866
 867     my $arch = join(',', @{$args{ARCH}});
 868     my $os = join(',', @{$args{OSVERSION}});
 869     my $mem = join(',', @{$args{MEM}});
 870     my $language = join(',', @{$args{LANGUAGE}});
 871
 872     push @{$bats_plist{'Tests'}}, {
 873         "TestName" => "$name",
 874         "Command" => [
 875             "/usr/bin/perl",
 876             "$BATSBASE/test/test.pl",
 877             $name,
 878             "ARCH=$arch",
 879             "OS=$os",
 880             "MEM=$mem",
 881             "LANGUAGE=$language",
 882             "BUILD=0",
 883             "RUN=1",
 884             "VERBOSE=1",
 885             "BATS=1",
 886         ]
 887     };
 888 }
 889
 890
 891 # Builds a simple test
 892 sub build_simple {
 893     my %C = %{shift()};
 894     my $name = shift;
 895     my %T = %{$C{"TEST_$name"}};
 896
 897     my $dstdir = $T{DSTDIR};
 898     if (-e "$dstdir/build-succeeded") {
 899         # We delete the whole test directory before building (if it existed),
 900         # so if this file exists now, that means another configuration already
 901         # did an equivalent build.
 902         print "note:    $name is already built at $dstdir, skipping the build\n" if $VERBOSE;
 903         return 1;
 904     }
 905
 906     mkdir_verbose $dstdir;
 907     # we don't mkdir $T{OBJDIR} because most tests don't use it
 908
 909     my $ext = $ALL_TESTS{$name};
 910     my $file = "$DIR/$name.$ext";
 911
 912     if ($T{TEST_CRASHES}) {
 913         `echo '$crashcatch' > $dstdir/crashcatch.c`;
 914         my $output = make("$C{COMPILE_C} -dynamiclib -o libcrashcatch.dylib -x c crashcatch.c", $dstdir);
 915         if ($?) {
 916             colorprint  $red, "FAIL: building crashcatch.c";
 917             colorprefix $red, $output;
 918             return 0;
 919         }
 920     }
 921
 922     my $cmd = $T{TEST_BUILD} ? eval "return \"$T{TEST_BUILD}\"" : "$C{COMPILE}   $T{TEST_CFLAGS} $file -o $name.exe";
 923
 924     my $output = make($cmd, $dstdir);
 925
 926     # ignore out-of-date text-based stubs (caused by ditto into SDK)
 927     $output =~ s/ld: warning: text-based stub file.*\n//g;
 928     # rdar://10163155
 929     $output =~ s/ld: warning: could not create compact unwind for [^\n]+: does not use standard frame\n//g;
 930     # rdar://37937122
 931     $output =~ s/^warning: Cannot lower [^\n]+\n//g;
 932     $output =~ s/^warning:     key: [^\n]+\n//g;
 933     $output =~ s/^warning:     discriminator: [^\n]+\n//g;
 934     $output =~ s/^warning:     callee: [^\n]+\n//g;
 935     # rdar://38710948
 936     $output =~ s/ld: warning: ignoring file [^\n]*libclang_rt\.bridgeos\.a[^\n]*\n//g;
 937     $output =~ s/ld: warning: building for iOS Simulator, but[^\n]*\n//g;
 938     # ignore compiler logging of CCC_OVERRIDE_OPTIONS effects
 939     if (defined $ENV{CCC_OVERRIDE_OPTIONS}) {
 940         $output =~ s/### (CCC_OVERRIDE_OPTIONS:|Adding argument|Deleting argument|Replacing) [^\n]*\n//g;
 941     }
 942
 943     my $ok;
 944     if (my $builderror = $T{TEST_BUILD_OUTPUT}) {
 945         # check for expected output and ignore $?
 946         if ($output =~ /$builderror/) {
 947             $ok = 1;
 948         } elsif (defined $ENV{CCC_OVERRIDE_OPTIONS} && $builderror =~ /warning:/) {
 949             # CCC_OVERRIDE_OPTIONS manipulates compiler diagnostics.
 950             # Don't enforce any TEST_BUILD_OUTPUT that looks for warnings.
 951             colorprint  $yellow, "WARN: /// test '$name' \\\\\\";
 952             colorprefix $yellow, $output;
 953             colorprint  $yellow, "WARN: \\\\\\ test '$name' ///";
 954             colorprint  $yellow, "WARN: $name (build output does not match TEST_BUILD_OUTPUT; not fatal because CCC_OVERRIDE_OPTIONS is set)";
 955             $ok = 1;
 956         } else {
 957             colorprint  $red, "FAIL: /// test '$name' \\\\\\";
 958             colorprefix $red, $output;
 959             colorprint  $red, "FAIL: \\\\\\ test '$name' ///";
 960             colorprint  $red, "FAIL: $name (build output does not match TEST_BUILD_OUTPUT)";
 961             $ok = 0;
 962         }
 963     } elsif ($?) {
 964         colorprint  $red, "FAIL: /// test '$name' \\\\\\";
 965         colorprefix $red, $output;
 966         colorprint  $red, "FAIL: \\\\\\ test '$name' ///";
 967         colorprint  $red, "FAIL: $name (build failed)";
 968         $ok = 0;
 969     } elsif ($output ne "") {
 970         colorprint  $red, "FAIL: /// test '$name' \\\\\\";
 971         colorprefix $red, $output;
 972         colorprint  $red, "FAIL: \\\\\\ test '$name' ///";
 973         colorprint  $red, "FAIL: $name (unexpected build output)";
 974         $ok = 0;
 975     } else {
 976         $ok = 1;
 977     }
 978
 979     if ($ok) {
 980         foreach my $file (glob("$dstdir/*.exe $dstdir/*.dylib $dstdir/*.bundle")) {
 981             if (!$BATS) {
 982                 # not for BATS to save space and build time
 983                 # fixme use SYMROOT?
 984                 make("xcrun dsymutil $file", $dstdir);
 985             }
 986             if ($C{OS} eq "macosx"  ||  $C{OS} =~ /simulator/) {
 987                 # setting any entitlements disables dyld environment variables
 988             } else {
 989                 # get-task-allow entitlement is required
 990                 # to enable dyld environment variables
 991                 if (!$T{ENTITLEMENTS}) {
 992                     $T{ENTITLEMENTS} = "get_task_allow_entitlement.plist";
 993                 }
 994                 my $output = make("xcrun codesign -s - --entitlements $DIR/$T{ENTITLEMENTS} $file", $dstdir);
 995                 if ($?) {
 996                     colorprint  $red, "FAIL: codesign $file";
 997                     colorprefix $red, $output;
 998                     return 0;
 999                 }
1000             }
1001         }
1002     }
1003
1004     # Mark the build as successful so other configs with the same build
1005     # requirements can skip buildiing.
1006     if ($ok) {
1007         make("touch build-succeeded", $dstdir);
1008     }
1009
1010     return $ok;
1011 }
1012
1013 # Run a simple test (testname.exe, with error checking of stdout and stderr)
1014 sub run_simple {
1015     my %C = %{shift()};
1016     my $name = shift;
1017     my %T = %{$C{"TEST_$name"}};
1018
1019     if (! $T{TEST_RUN}) {
1020         print "PASS: $name (build only)\n";
1021         return 1;
1022     }
1023
1024     my $testdir = $T{DSTDIR};
1025     chdir_verbose $testdir;
1026
1027     my $env = "$C{ENV} $T{TEST_ENV}";
1028
1029     if ($T{TEST_CRASHES}) {
1030         $env .= " OBJC_DEBUG_DONT_CRASH=YES";
1031     }
1032
1033     if ($C{DYLD} eq "2") {
1034         $env .= " DYLD_USE_CLOSURES=0";
1035     }
1036     elsif ($C{DYLD} eq "3") {
1037         $env .= " DYLD_USE_CLOSURES=1";
1038     }
1039     else {
1040         die "unknown DYLD setting $C{DYLD}";
1041     }
1042
1043     if ($SHAREDCACHEDIR) {
1044         $env .= " DYLD_SHARED_REGION=private DYLD_SHARED_CACHE_DIR=$SHAREDCACHEDIR";
1045     }
1046
1047     my $output;
1048
1049     if ($C{ARCH} =~ /^arm/ && `uname -p` !~ /^arm/) {
1050         # run on iOS or watchos or tvos device
1051         # fixme device selection and verification
1052         my $remotedir = "$REMOTEBASE/" . basename($C{DSTDIR}) . "/$name.build";
1053
1054         # Add test dir and libobjc's dir to DYLD_LIBRARY_PATH.
1055         # Insert libcrashcatch.dylib if necessary.
1056         $env .= " DYLD_LIBRARY_PATH=$remotedir";
1057         $env .= ":$REMOTEBASE"  if ($C{TESTLIBDIR} ne $TESTLIBDIR);
1058         if ($T{TEST_CRASHES}) {
1059             $env .= " DYLD_INSERT_LIBRARIES=$remotedir/libcrashcatch.dylib";
1060         }
1061
1062         my $cmd = "ssh $PORT $HOST 'cd $remotedir && env $env ./$name.exe'";
1063         $output = make("$cmd");
1064     }
1065     elsif ($C{OS} =~ /simulator/) {
1066         # run locally in a simulator
1067         my $sim = "xcrun -sdk iphonesimulator simctl spawn '$DEVICE'";
1068         # Add test dir and libobjc's dir to DYLD_LIBRARY_PATH.
1069         # Insert libcrashcatch.dylib if necessary.
1070         $env .= " DYLD_LIBRARY_PATH=$testdir";
1071         $env .= ":" . $C{TESTLIBDIR}  if ($C{TESTLIBDIR} ne $TESTLIBDIR);
1072         if ($T{TEST_CRASHES}) {
1073             $env .= " DYLD_INSERT_LIBRARIES=$testdir/libcrashcatch.dylib";
1074         }
1075
1076         my $simenv = "";
1077         foreach my $keyvalue (split(' ', $env)) {
1078             $simenv .= "SIMCTL_CHILD_$keyvalue ";
1079         }
1080         # Use the full path here so hack_cwd in test.h works.
1081         $output = make("env $simenv $sim $testdir/$name.exe");
1082     }
1083     else {
1084         # run locally
1085
1086         # Add test dir and libobjc's dir to DYLD_LIBRARY_PATH.
1087         # Insert libcrashcatch.dylib if necessary.
1088         $env .= " DYLD_LIBRARY_PATH=$testdir";
1089         $env .= ":" . $C{TESTLIBDIR}  if ($C{TESTLIBDIR} ne $TESTLIBDIR);
1090         if ($T{TEST_CRASHES}) {
1091             $env .= " DYLD_INSERT_LIBRARIES=$testdir/libcrashcatch.dylib";
1092         }
1093
1094         $output = make("sh -c '$env ./$name.exe'");
1095     }
1096
1097     return check_output(\%C, $name, split("\n", $output));
1098 }
1099
1100
1101 my %compiler_memo;
1102 sub find_compiler {
1103     my ($cc, $toolchain, $sdk_path) = @_;
1104
1105     # memoize
1106     my $key = $cc . ':' . $toolchain;
1107     my $result = $compiler_memo{$key};
1108     return $result if defined $result;
1109
1110     $result  = make("xcrun -toolchain $toolchain -find $cc 2>/dev/null");
1111
1112     chomp $result;
1113     $compiler_memo{$key} = $result;
1114     return $result;
1115 }
1116
1117 sub dirContainsAllTestLibs {
1118     my $dir = shift;
1119
1120     foreach my $testlib (@TESTLIBNAMES) {
1121         my $found = (-e "$dir/$testlib");
1122         my $foundstr = ($found ? "found" : "didn't find");
1123         print "note: $foundstr $testlib in $dir\n"  if ($VERBOSE);
1124         return 0  if (!$found);
1125     }
1126
1127     return 1;
1128 }
1129
1130 sub findIncludeDir {
1131     my ($root, $includePath) = @_;
1132
1133     foreach my $candidate ("$root/../SDKContentRoot/$includePath", "$root/$includePath") {
1134         my $found = -e $candidate;
1135         my $foundstr = ($found ? "found" : "didn't find");
1136         print "note:    $foundstr $includePath at $candidate\n" if $VERBOSE;
1137         return $candidate if $found;
1138     }
1139
1140     die "Unable to find $includePath in $root.\n";
1141 }
1142
1143 sub buildSharedCache {
1144     my $Cref = shift;
1145     my %C = %$Cref;
1146
1147     make("update_dyld_shared_cache -verbose -cache_dir $BUILDDIR -overlay $C{TESTLIBDIR}/../..");
1148 }
1149
1150 sub make_one_config {
1151     my $configref = shift;
1152     my $root = shift;
1153     my %C = %{$configref};
1154
1155     # Aliases
1156     $C{LANGUAGE} = "objective-c"  if $C{LANGUAGE} eq "objc";
1157     $C{LANGUAGE} = "objective-c++"  if $C{LANGUAGE} eq "objc++";
1158
1159     # Interpret OS version string from command line.
1160     my ($sdk_arg, $deployment_arg, $run_arg, undef) = split('-', $C{OSVERSION});
1161     delete $C{OSVERSION};
1162     my ($os_arg) = ($sdk_arg =~ /^([^\.0-9]+)/);
1163     $deployment_arg = "default" if !defined($deployment_arg);
1164     $run_arg = "default" if !defined($run_arg);
1165
1166     my %allowed_os_args = (
1167         "macosx" => "macosx", "osx" => "macosx", "macos" => "macosx",
1168         "iphoneos" => "iphoneos", "ios" => "iphoneos",
1169         "iphonesimulator" => "iphonesimulator", "iossimulator" => "iphonesimulator",
1170         "watchos" => "watchos",
1171         "watchsimulator" => "watchsimulator", "watchossimulator" => "watchsimulator",
1172         "appletvos" => "appletvos", "tvos" => "appletvos",
1173         "appletvsimulator" => "appletvsimulator", "tvsimulator" => "appletvsimulator",
1174         "bridgeos" => "bridgeos",
1175         );
1176
1177     $C{OS} = $allowed_os_args{$os_arg} || die "unknown OS '$os_arg' (expected " . join(', ', sort keys %allowed_os_args) . ")\n";
1178
1179     # set the config name now, after massaging the language and OS versions,
1180     # but before adding other settings
1181     my $configdirname = config_dir_name(%C);
1182     die if ($configdirname =~ /'/);
1183     die if ($configdirname =~ / /);
1184     ($C{NAME} = $configdirname) =~ s/~/ /g;
1185     (my $configdir = $configdirname) =~ s#/##g;
1186     $C{DSTDIR} = "$DSTROOT$BUILDDIR/$configdir";
1187     $C{OBJDIR} = "$OBJROOT$BUILDDIR/$configdir";
1188
1189     # Allow tests to see BATS-edness in TEST_CONFIG.
1190     $C{BATS} = $BATS;
1191
1192     if ($C{OS} eq "iphoneos" || $C{OS} eq "iphonesimulator") {
1193         $C{TOOLCHAIN} = "ios";
1194     } elsif ($C{OS} eq "watchos" || $C{OS} eq "watchsimulator") {
1195         $C{TOOLCHAIN} = "watchos";
1196     } elsif ($C{OS} eq "appletvos" || $C{OS} eq "appletvsimulator") {
1197         $C{TOOLCHAIN} = "appletvos";
1198     } elsif ($C{OS} eq "bridgeos") {
1199         $C{TOOLCHAIN} = "bridgeos";
1200     } elsif ($C{OS} eq "macosx") {
1201         $C{TOOLCHAIN} = "osx";
1202     } else {
1203         colorprint $yellow, "WARN: don't know toolchain for OS $C{OS}";
1204         $C{TOOLCHAIN} = "default";
1205     }
1206
1207     if ($BUILD) {
1208         # Look up SDK.
1209         # Try exact match first.
1210         # Then try lexically-last prefix match
1211         #   (so "macosx" => "macosx10.7internal")
1212
1213         $sdk_arg =~ s/$os_arg/$C{OS}/;
1214
1215         my @sdks = getsdks();
1216         if ($VERBOSE) {
1217             print "note: Installed SDKs: @sdks\n";
1218         }
1219         my $exactsdk = undef;
1220         my $prefixsdk = undef;
1221         foreach my $sdk (@sdks) {
1222             $exactsdk = $sdk  if ($sdk eq $sdk_arg);
1223             $prefixsdk = newersdk($sdk, $prefixsdk)  if ($sdk =~ /^$sdk_arg/);
1224         }
1225
1226         my $sdk;
1227         if ($exactsdk) {
1228             $sdk = $exactsdk;
1229         } elsif ($prefixsdk) {
1230             $sdk = $prefixsdk;
1231         } else {
1232             die "unknown SDK '$sdk_arg'\nInstalled SDKs: @sdks\n";
1233         }
1234
1235         # Set deployment target.
1236         # fixme can't enforce version when run_arg eq "default"
1237         # because we don't know it yet
1238         $deployment_arg = versionsuffix($sdk) if $deployment_arg eq "default";
1239         if ($run_arg ne "default") {
1240             die "Deployment target '$deployment_arg' is newer than run target '$run_arg'\n"  if $deployment_arg > $run_arg;
1241         }
1242         $C{DEPLOYMENT_TARGET} = $deployment_arg;
1243         $C{SDK_PATH} = getsdkpath($sdk);
1244     } else {
1245         # not $BUILD
1246         $C{DEPLOYMENT_TARGET} = "unknown_deployment_target";
1247         $C{SDK_PATH} = "/unknown/sdk";
1248     }
1249
1250     # Set run target.
1251     $C{RUN_TARGET} = $run_arg;
1252
1253     # Look up test library (possible in root or SDK_PATH)
1254
1255     my $rootarg = $root;
1256     my $symroot;
1257     my @sympaths = ( (glob "$root/*~sym")[0],
1258                      (glob "$root/BuildRecords/*_install/Symbols")[0],
1259                      "$root/Symbols" );
1260     my @dstpaths = ( (glob "$root/*~dst")[0],
1261                      (glob "$root/BuildRecords/*_install/Root")[0],
1262                      "$root/Root" );
1263     for(my $i = 0; $i < scalar(@sympaths); $i++) {
1264         if (-e $sympaths[$i]  &&  -e $dstpaths[$i]) {
1265             $symroot = $sympaths[$i];
1266             $root = $dstpaths[$i];
1267             last;
1268         }
1269     }
1270
1271     if ($root ne "") {
1272         # Root specified. Require that it contain our dylibs.
1273         if (dirContainsAllTestLibs("$root$C{SDK_PATH}$TESTLIBDIR")) {
1274             $C{TESTLIBDIR} = "$root$C{SDK_PATH}$TESTLIBDIR";
1275         } elsif (dirContainsAllTestLibs("$root$TESTLIBDIR")) {
1276             $C{TESTLIBDIR} = "$root$TESTLIBDIR";
1277         } elsif (dirContainsAllTestLibs($root)) {
1278             $C{TESTLIBDIR} = "$root";
1279         } else {
1280             die "Didn't find some libs in root '$rootarg' for sdk '$C{SDK_PATH}'\n";
1281         }
1282     }
1283     else {
1284         # No root specified. Use the SDK or / for our dylibs.
1285         if (dirContainsAllTestLibs("$C{SDK_PATH}$TESTLIBDIR")) {
1286             $C{TESTLIBDIR} = "$C{SDK_PATH}$TESTLIBDIR";
1287         } else {
1288             # We don't actually check in / because on devices
1289             # there are no dylib files there.
1290             $C{TESTLIBDIR} = $TESTLIBDIR;
1291         }
1292     }
1293
1294     @{$C{TESTLIBS}} = map { "$C{TESTLIBDIR}/$_" } @TESTLIBNAMES;
1295     # convenience for tests that want libobjc.dylib's path
1296     $C{TESTLIB} = @{$C{TESTLIBS}}[0];
1297
1298     foreach my $testlibname (@TESTLIBNAMES) {
1299         if (-e "$symroot/$testlibname.dSYM") {
1300             push(@{$C{TESTDSYMS}}, "$symroot/$testlibname.dSYM");
1301         }
1302     }
1303
1304     if ($VERBOSE) {
1305         foreach my $testlib (@{$C{TESTLIBS}}) {
1306             my @uuids = `/usr/bin/dwarfdump -u '$testlib'`;
1307             while (my $uuid = shift @uuids) {
1308                 print "note: $uuid";
1309             }
1310         }
1311     }
1312
1313     # Look up compilers
1314     my $cc = $C{CC};
1315     my $cxx = cplusplus($C{CC});
1316     my $swift = swift($C{CC});
1317     if (! $BUILD) {
1318         $C{CC} = $cc;
1319         $C{CXX} = $cxx;
1320         $C{SWIFT} = $swift
1321     } else {
1322         $C{CC} = find_compiler($cc, $C{TOOLCHAIN}, $C{SDK_PATH});
1323         $C{CXX} = find_compiler($cxx, $C{TOOLCHAIN}, $C{SDK_PATH});
1324         $C{SWIFT} = find_compiler($swift, $C{TOOLCHAIN}, $C{SDK_PATH});
1325
1326         die "No C compiler '$cc' ('$C{CC}') in toolchain '$C{TOOLCHAIN}'\n" if !-e $C{CC};
1327         die "No C++ compiler '$cxx' ('$C{CXX}') in toolchain '$C{TOOLCHAIN}'\n" if !-e $C{CXX};
1328         die "No Swift compiler '$swift' ('$C{SWIFT}') in toolchain '$C{TOOLCHAIN}'\n" if !-e $C{SWIFT};
1329     }
1330
1331     if ($C{ARCH} eq "i386" && $C{OS} eq "macosx") {
1332         # libarclite no longer available on i386
1333         # fixme need an archived copy for bincompat testing
1334         $C{FORCE_LOAD_ARCLITE} = "";
1335     } elsif ($C{OS} eq "bridgeos") {
1336         # no libarclite on bridgeOS
1337         $C{FORCE_LOAD_ARCLITE} = "";
1338     } else {
1339         $C{FORCE_LOAD_ARCLITE} = "-Xlinker -force_load -Xlinker " . dirname($C{CC}) . "/../lib/arc/libarclite_$C{OS}.a";
1340     }
1341
1342     # Populate cflags
1343
1344     my $cflags = "-I$DIR -W -Wall -Wno-objc-weak-compat -Wno-arc-bridge-casts-disallowed-in-nonarc -Wshorten-64-to-32 -Qunused-arguments -fno-caret-diagnostics -Os -arch $C{ARCH} ";
1345     if (!$BATS) {
1346         # save-temps so dsymutil works so debug info works.
1347         # Disabled in BATS to save disk space.
1348         # rdar://45656803 -save-temps causes bad -Wstdlibcxx-not-found warnings
1349         $cflags .= "-g -save-temps -Wno-stdlibcxx-not-found";
1350     }
1351     my $objcflags = "";
1352     my $swiftflags = "-g ";
1353
1354     $cflags .= " -isysroot '$C{SDK_PATH}'";
1355     $cflags .= " '-Wl,-syslibroot,$C{SDK_PATH}'";
1356     $swiftflags .= " -sdk '$C{SDK_PATH}'";
1357
1358     # Set deployment target cflags
1359     my $target = undef;
1360     die "No deployment target" if $C{DEPLOYMENT_TARGET} eq "";
1361     if ($C{OS} eq "iphoneos") {
1362         $cflags .= " -mios-version-min=$C{DEPLOYMENT_TARGET}";
1363         $target = "$C{ARCH}-apple-ios$C{DEPLOYMENT_TARGET}";
1364     }
1365     elsif ($C{OS} eq "iphonesimulator") {
1366         $cflags .= " -mios-simulator-version-min=$C{DEPLOYMENT_TARGET}";
1367         $target = "$C{ARCH}-apple-ios$C{DEPLOYMENT_TARGET}";
1368     }
1369     elsif ($C{OS} eq "watchos") {
1370         $cflags .= " -mwatchos-version-min=$C{DEPLOYMENT_TARGET}";
1371         $target = "$C{ARCH}-apple-watchos$C{DEPLOYMENT_TARGET}";
1372     }
1373     elsif ($C{OS} eq "watchsimulator") {
1374         $cflags .= " -mwatchos-simulator-version-min=$C{DEPLOYMENT_TARGET}";
1375         $target = "$C{ARCH}-apple-watchos$C{DEPLOYMENT_TARGET}";
1376     }
1377     elsif ($C{OS} eq "appletvos") {
1378         $cflags .= " -mtvos-version-min=$C{DEPLOYMENT_TARGET}";
1379         $target = "$C{ARCH}-apple-tvos$C{DEPLOYMENT_TARGET}";
1380     }
1381     elsif ($C{OS} eq "appletvsimulator") {
1382         $cflags .= " -mtvos-simulator-version-min=$C{DEPLOYMENT_TARGET}";
1383         $target = "$C{ARCH}-apple-tvos$C{DEPLOYMENT_TARGET}";
1384     }
1385     elsif ($C{OS} eq "bridgeos") {
1386         $cflags .= " -mbridgeos-version-min=$C{DEPLOYMENT_TARGET}";
1387         $target = "$C{ARCH}-apple-bridgeos$C{DEPLOYMENT_TARGET}";
1388     }
1389     else {
1390         $cflags .= " -mmacosx-version-min=$C{DEPLOYMENT_TARGET}";
1391         $target = "$C{ARCH}-apple-macosx$C{DEPLOYMENT_TARGET}";
1392     }
1393     $swiftflags .= " -target $target";
1394
1395     $C{TESTINCLUDEDIR} = "$C{SDK_PATH}/usr/include";
1396     $C{TESTLOCALINCLUDEDIR} = "$C{SDK_PATH}/usr/local/include";
1397     if ($root ne "") {
1398         if ($C{SDK_PATH} ne "/") {
1399             $cflags .= " -isystem '$root$C{SDK_PATH}/usr/include'";
1400             $cflags .= " -isystem '$root$C{SDK_PATH}/usr/local/include'";
1401         }
1402
1403         my $library_path = $C{TESTLIBDIR};
1404         $cflags .= " -L$library_path";
1405         $C{TESTINCLUDEDIR} = findIncludeDir($root, "usr/include");
1406         $C{TESTLOCALINCLUDEDIR} = findIncludeDir($root, "usr/local/include");
1407         $cflags .= " -isystem '$C{TESTINCLUDEDIR}'";
1408         $cflags .= " -isystem '$C{TESTLOCALINCLUDEDIR}'";
1409     }
1410
1411
1412     # Populate objcflags
1413
1414     $objcflags .= " -lobjc";
1415     if ($C{MEM} eq "arc") {
1416         $objcflags .= " -fobjc-arc";
1417     }
1418     elsif ($C{MEM} eq "mrc") {
1419         # nothing
1420     }
1421     else {
1422         die "unrecognized MEM '$C{MEM}'\n";
1423     }
1424
1425     # Populate ENV_PREFIX
1426     $C{ENV} = "LANG=C MallocScribble=1";
1427     $C{ENV} .= " VERBOSE=$VERBOSE"  if $VERBOSE;
1428     if ($root ne "") {
1429         die "no spaces allowed in root" if $C{TESTLIBDIR} =~ /\s+/;
1430     }
1431     if ($C{GUARDMALLOC}) {
1432         $C{ENV} .= " GUARDMALLOC=1";  # checked by tests and errcheck.pl
1433         $C{ENV} .= " DYLD_INSERT_LIBRARIES=/usr/lib/libgmalloc.dylib";
1434         if ($C{GUARDMALLOC} eq "before") {
1435             $C{ENV} .= " MALLOC_PROTECT_BEFORE=1";
1436         } elsif ($C{GUARDMALLOC} eq "after") {
1437             # protect after is the default
1438         } else {
1439             die "Unknown guard malloc mode '$C{GUARDMALLOC}'\n";
1440         }
1441     }
1442
1443     # Populate compiler commands
1444     $C{XCRUN} = "env LANG=C /usr/bin/xcrun -toolchain '$C{TOOLCHAIN}'";
1445
1446     $C{COMPILE_C}   = "$C{XCRUN} '$C{CC}'  $cflags -x c -std=gnu99";
1447     $C{COMPILE_CXX} = "$C{XCRUN} '$C{CXX}' $cflags -x c++ -std=gnu++17";
1448     $C{COMPILE_M}   = "$C{XCRUN} '$C{CC}'  $cflags $objcflags -x objective-c -std=gnu99";
1449     $C{COMPILE_MM}  = "$C{XCRUN} '$C{CXX}' $cflags $objcflags -x objective-c++ -std=gnu++17";
1450     $C{COMPILE_SWIFT} = "$C{XCRUN} '$C{SWIFT}' $swiftflags";
1451
1452     $C{COMPILE} = $C{COMPILE_C}      if $C{LANGUAGE} eq "c";
1453     $C{COMPILE} = $C{COMPILE_CXX}    if $C{LANGUAGE} eq "c++";
1454     $C{COMPILE} = $C{COMPILE_M}      if $C{LANGUAGE} eq "objective-c";
1455     $C{COMPILE} = $C{COMPILE_MM}     if $C{LANGUAGE} eq "objective-c++";
1456     $C{COMPILE} = $C{COMPILE_SWIFT}  if $C{LANGUAGE} eq "swift";
1457     die "unknown language '$C{LANGUAGE}'\n" if !defined $C{COMPILE};
1458
1459     ($C{COMPILE_NOMEM} = $C{COMPILE}) =~ s/ -fobjc-arc\S*//g;
1460     ($C{COMPILE_NOLINK} = $C{COMPILE}) =~ s/ '?-(?:Wl,|l)\S*//g;
1461     ($C{COMPILE_NOLINK_NOMEM} = $C{COMPILE_NOMEM}) =~ s/ '?-(?:Wl,|l)\S*//g;
1462
1463
1464     # Reject some self-inconsistent and disallowed configurations
1465     if ($C{MEM} !~ /^(mrc|arc)$/) {
1466         die "unknown MEM=$C{MEM} (expected one of mrc arc)\n";
1467     }
1468
1469     if ($C{MEM} eq "arc"  &&  $C{CC} !~ /clang/) {
1470         print "note: skipping configuration $C{NAME}\n";
1471         print "note:   because CC=$C{CC} does not support MEM=$C{MEM}\n";
1472         return 0;
1473     }
1474
1475     if ($C{ARCH} eq "i386"  &&  $C{OS} eq "macosx") {
1476         colorprint $yellow, "WARN: skipping configuration $C{NAME}\n";
1477         colorprint $yellow, "WARN:   because 32-bit Mac is dead\n";
1478         return 0;
1479     }
1480
1481     # fixme
1482     if ($C{LANGUAGE} eq "swift"  &&  $C{ARCH} =~ /^arm/) {
1483         print "note: skipping configuration $C{NAME}\n";
1484         print "note:   because ARCH=$C{ARCH} does not support LANGUAGE=SWIFT\n";
1485         return 0;
1486     }
1487
1488     # fixme unimplemented run targets
1489     if ($C{RUN_TARGET} ne "default" &&  $C{OS} !~ /simulator/) {
1490         colorprint $yellow, "WARN: skipping configuration $C{NAME}";
1491         colorprint $yellow, "WARN:   because OS=$C{OS} does not yet implement RUN_TARGET=$C{RUN_TARGET}";
1492     }
1493
1494     %$configref = %C;
1495 }
1496
1497 sub make_configs {
1498     my ($root, %args) = @_;
1499
1500     my @results = ({});  # start with one empty config
1501
1502     for my $key (keys %args) {
1503         my @newresults;
1504         my @values = @{$args{$key}};
1505         for my $configref (@results) {
1506             my %config = %{$configref};
1507             for my $value (@values) {
1508                 my %newconfig = %config;
1509                 $newconfig{$key} = $value;
1510                 push @newresults, \%newconfig;
1511             }
1512         }
1513         @results = @newresults;
1514     }
1515
1516     my @newresults;
1517     for my $configref(@results) {
1518         if (make_one_config($configref, $root)) {
1519             push @newresults, $configref;
1520         }
1521     }
1522
1523     return @newresults;
1524 }
1525
1526 sub config_dir_name {
1527     my %config = @_;
1528     my $name = "";
1529     for my $key (sort keys %config) {
1530         # Exclude settings that only influence the run, not the build.
1531         next if $key eq "DYLD" || $key eq "GUARDMALLOC";
1532
1533         $name .= '~'  if $name ne "";
1534         $name .= "$key=$config{$key}";
1535     }
1536     return $name;
1537 }
1538
1539 sub rsync_ios {
1540     my ($src, $timeout) = @_;
1541     for (my $i = 0; $i < 10; $i++) {
1542         make("$DIR/timeout.pl $timeout rsync -e 'ssh $PORT' -av $src $HOST:/$REMOTEBASE/");
1543         return if $? == 0;
1544         colorprint $yellow, "WARN: RETRY\n"  if $VERBOSE;
1545     }
1546     die "Couldn't rsync tests to device. Check: device is connected; tcprelay is running; device trusts your Mac; device is unlocked; filesystem is mounted r/w\n";
1547 }
1548
1549 sub build_and_run_one_config {
1550     my %C = %{shift()};
1551     my @tests = @_;
1552
1553     # Build and run
1554     my $testcount = 0;
1555     my $failcount = 0;
1556     my $skipconfig = 0;
1557
1558     my @gathertests;
1559     foreach my $test (@tests) {
1560         if ($VERBOSE) {
1561             print "\nGATHER $test\n";
1562         }
1563
1564         if ($ALL_TESTS{$test}) {
1565             gather_simple(\%C, $test) || next;  # not pass, not fail
1566             push @gathertests, $test;
1567         } elsif ($has_match_glob) {
1568             my @matched = Text::Glob::match_glob($test, (keys %ALL_TESTS));
1569             if (not @matched) {
1570                 die "No test matched '$test'\n";
1571             }
1572             foreach my $match (@matched) {
1573                 gather_simple(\%C, $match) || next;  # not pass, not fail
1574                 push @gathertests, $match;
1575             }
1576         }
1577     }
1578
1579     my @builttests;
1580     if (!$BUILD) {
1581         @builttests = @gathertests;
1582         $testcount = scalar(@gathertests);
1583     } elsif ($PARALLELBUILDS > 1 && $supportsParallelBuilds) {
1584         my $workQueue = Thread::Queue->new();
1585         my $resultsQueue = Thread::Queue->new();
1586         my @threads = map {
1587             threads->create(sub {
1588                 while (defined(my $test = $workQueue->dequeue())) {
1589                     local *STDOUT;
1590                     local *STDERR;
1591                     my $output;
1592                     open STDOUT, '>>', \$output;
1593                     open STDERR, '>>', \$output;
1594
1595                     my $success = build_simple(\%C, $test);
1596                     $resultsQueue->enqueue({ test => $test, success => $success, output => $output });
1597                 }
1598             });
1599         } (1 .. $PARALLELBUILDS);
1600
1601         foreach my $test (@gathertests) {
1602             if ($VERBOSE) {
1603                 print "\nBUILD $test\n";
1604             }
1605             if ($ALL_TESTS{$test}) {
1606                 $testcount++;
1607                 $workQueue->enqueue($test);
1608             } else {
1609                 die "No test named '$test'\n";
1610             }
1611         }
1612         $workQueue->end();
1613         foreach (@gathertests) {
1614             my $result = $resultsQueue->dequeue();
1615             my $test = $result->{test};
1616             my $success = $result->{success};
1617             my $output = $result->{output};
1618
1619             print $output;
1620             if ($success) {
1621                 push @builttests, $test;
1622             } else {
1623                 $failcount++;
1624             }
1625         }
1626         foreach my $thread (@threads) {
1627             $thread->join();
1628         }
1629     } else {
1630         if ($PARALLELBUILDS > 1) {
1631             print "WARNING: requested parallel builds, but this perl interpreter does not support threads. Falling back to sequential builds.\n";
1632         }
1633         foreach my $test (@gathertests) {
1634             if ($VERBOSE) {
1635                 print "\nBUILD $test\n";
1636             }
1637
1638             if ($ALL_TESTS{$test}) {
1639                 $testcount++;
1640                 if (!build_simple(\%C, $test)) {
1641                     $failcount++;
1642                 } else {
1643                     push @builttests, $test;
1644                 }
1645             } else {
1646                 die "No test named '$test'\n";
1647             }
1648         }
1649     }
1650
1651     if (!$RUN  ||  !scalar(@builttests)) {
1652         # nothing to do
1653     }
1654     else {
1655         if ($HOST && $C{ARCH} =~ /^arm/ && `uname -p` !~ /^arm/) {
1656             # upload timeout - longer for slow watch devices
1657             my $timeout = ($C{OS} =~ /watch/) ? 120 : 20;
1658
1659             # upload all tests to iOS device
1660             rsync_ios($C{DSTDIR}, $timeout);
1661
1662             # upload library to iOS device
1663             if ($C{TESTLIBDIR} ne $TESTLIBDIR) {
1664                 foreach my $thing (@{$C{TESTLIBS}}, @{$C{TESTDSYMS}}) {
1665                     rsync_ios($thing, $timeout);
1666                 }
1667             }
1668         }
1669         elsif ($C{OS} =~ /simulator/) {
1670             # run locally in a simulator
1671         }
1672         else {
1673             # run locally
1674             if ($BATS) {
1675                 # BATS execution tries to run architectures that
1676                 # aren't supported by the device. Skip those configs here.
1677                 my $machine = `machine`;
1678                 chomp $machine;
1679                 # unsupported:
1680                 # running arm64e on non-arm64e device
1681                 # running arm64 on non-arm64* device
1682                 # running armv7k on non-armv7k device
1683                 # running arm64_32 on armv7k device
1684                 # We don't need to handle all mismatches here,
1685                 # only mismatches that arise within a single OS.
1686                 $skipconfig =
1687                     (($C{ARCH} eq "arm64e"   && $machine ne "arm64e")  ||
1688                      ($C{ARCH} eq "arm64"    && $machine !~ /^arm64/)  ||
1689                      ($C{ARCH} eq "armv7k"   && $machine ne "armv7k")  ||
1690                      ($C{ARCH} eq "arm64_32" && $machine eq "armv7k"));
1691                 if ($skipconfig) {
1692                     print "note: skipping configuration $C{NAME}\n";
1693                     print "note:   because test arch $C{ARCH} is not " .
1694                         "supported on device arch $machine\n";
1695                     $testcount = 0;
1696                 }
1697             }
1698         }
1699
1700         if (!$skipconfig) {
1701             foreach my $test (@builttests) {
1702                 print "\nRUN $test\n"  if ($VERBOSE);
1703
1704                 if ($ALL_TESTS{$test}) {
1705                     if (!run_simple(\%C, $test)) {
1706                         $failcount++;
1707                     }
1708                 } else {
1709                     die "No test named '$test'\n";
1710                 }
1711             }
1712         }
1713     }
1714
1715     return ($testcount, $failcount, $skipconfig);
1716 }
1717
1718
1719
1720 # Return value if set by "$argname=value" on the command line
1721 # Return $default if not set.
1722 sub getargs {
1723     my ($argname, $default) = @_;
1724
1725     foreach my $arg (@ARGV) {
1726         my ($value) = ($arg =~ /^$argname=(.+)$/);
1727         return [split ',', $value] if defined $value;
1728     }
1729
1730     return [split ',', $default];
1731 }
1732
1733 # Return 1 or 0 if set by "$argname=1" or "$argname=0" on the
1734 # command line. Return $default if not set.
1735 sub getbools {
1736     my ($argname, $default) = @_;
1737
1738     my @values = @{getargs($argname, $default)};
1739     return [( map { ($_ eq "0") ? 0 : 1 } @values )];
1740 }
1741
1742 # Return an integer if set by "$argname=value" on the
1743 # command line. Return $default if not set.
1744 sub getints {
1745     my ($argname, $default) = @_;
1746
1747     my @values = @{getargs($argname, $default)};
1748     return [( map { int($_) } @values )];
1749 }
1750
1751 sub getarg {
1752     my ($argname, $default) = @_;
1753     my @values = @{getargs($argname, $default)};
1754     die "Only one value allowed for $argname\n"  if @values > 1;
1755     return $values[0];
1756 }
1757
1758 sub getbool {
1759     my ($argname, $default) = @_;
1760     my @values = @{getbools($argname, $default)};
1761     die "Only one value allowed for $argname\n"  if @values > 1;
1762     return $values[0];
1763 }
1764
1765 sub getint {
1766     my ($argname, $default) = @_;
1767     my @values = @{getints($argname, $default)};
1768     die "Only one value allowed for $argname\n"  if @values > 1;
1769     return $values[0];
1770 }
1771
1772
1773 my $default_arch = "x86_64";
1774 $args{ARCH} = getargs("ARCH", 0);
1775 $args{ARCH} = getargs("ARCHS", $default_arch)  if !@{$args{ARCH}}[0];
1776
1777 $args{OSVERSION} = getargs("OS", "macosx-default-default");
1778
1779 $args{MEM} = getargs("MEM", "mrc,arc");
1780 $args{LANGUAGE} = [ map { lc($_) } @{getargs("LANGUAGE", "c,objective-c,c++,objective-c++")} ];
1781
1782 $args{BUILD_SHARED_CACHE} = getargs("BUILD_SHARED_CACHE", 0);
1783
1784 $args{DYLD} = getargs("DYLD", "2,3");
1785
1786 $args{CC} = getargs("CC", "clang");
1787
1788 $HOST = getarg("HOST", 0);
1789 $PORT = getarg("PORT", "");
1790 if ($PORT) {
1791   $PORT = "-p $PORT";
1792 }
1793 $DEVICE = getarg("DEVICE", "booted");
1794
1795 $PARALLELBUILDS = getarg("PARALLELBUILDS", `sysctl -n hw.ncpu`);
1796
1797 $SHAREDCACHEDIR = getarg("SHAREDCACHEDIR", "");
1798
1799 {
1800     my $guardmalloc = getargs("GUARDMALLOC", 0);
1801     # GUARDMALLOC=1 is the same as GUARDMALLOC=before,after
1802     my @guardmalloc2 = ();
1803     for my $arg (@$guardmalloc) {
1804         if ($arg == 1) { push @guardmalloc2, "before";
1805                          push @guardmalloc2, "after"; }
1806         else { push @guardmalloc2, $arg }
1807     }
1808     $args{GUARDMALLOC} = \@guardmalloc2;
1809 }
1810
1811 $BUILD = getbool("BUILD", 1);
1812 $RUN = getbool("RUN", 1);
1813 $VERBOSE = getint("VERBOSE", 0);
1814 $BATS = getbool("BATS", 0);
1815 $BUILDDIR = getarg("BUILDDIR", $BATS ? $BATSBASE : $LOCALBASE);
1816
1817 my $root = getarg("ROOT", "");
1818 $root =~ s#/*$##;
1819
1820 my @tests = gettests();
1821
1822 if ($BUILD) {
1823     rm_rf_verbose "$DSTROOT$BUILDDIR";
1824     rm_rf_verbose "$OBJROOT$BUILDDIR";
1825 }
1826
1827 print "note: -----\n";
1828 print "note: testing root '$root'\n";
1829
1830 my @configs = make_configs($root, %args);
1831
1832 print "note: -----\n";
1833 print "note: testing ", scalar(@configs), " configurations:\n";
1834 for my $configref (@configs) {
1835     my $configname = $$configref{NAME};
1836     print "note: configuration $configname\n";
1837 }
1838
1839 my $failed = 0;
1840
1841 my $testconfigs = @configs;
1842 my $failconfigs = 0;
1843 my $skipconfigs = 0;
1844 my $testcount = 0;
1845 my $failcount = 0;
1846 for my $configref (@configs) {
1847     my $configname = $$configref{NAME};
1848     print "note: -----\n";
1849     print "note: \nnote: $configname\nnote: \n";
1850
1851     (my $t, my $f, my $skipconfig) =
1852         eval { build_and_run_one_config($configref, @tests); };
1853     $skipconfigs += $skipconfig;
1854     if ($@) {
1855         chomp $@;
1856         colorprint $red, "FAIL: $configname";
1857         colorprint $red, "FAIL: $@";
1858         $failconfigs++;
1859     } else {
1860         my $color = ($f ? $red : "");
1861         print "note:\n";
1862         colorprint $color, "note: $configname\n";
1863         colorprint $color, "note: $t tests, $f failures";
1864         $testcount += $t;
1865         $failcount += $f;
1866         $failconfigs++ if ($f);
1867     }
1868 }
1869
1870 make("find $DSTROOT$BUILDDIR -name build-succeeded -delete", "/");
1871
1872 print "note: -----\n";
1873 my $color = ($failconfigs ? $red : "");
1874 colorprint $color, "note: $testconfigs configurations, " .
1875     "$failconfigs with failures, $skipconfigs skipped";
1876 colorprint $color, "note: $testcount tests, $failcount failures";
1877
1878 $failed = ($failconfigs ? 1 : 0);
1879
1880
1881 if ($BUILD && $BATS && !$failed) {
1882     # Collect BATS execution instructions for all tests.
1883     # Each BATS "test" is all configurations together of one of our tests.
1884     for my $testname (@tests) {
1885         append_bats_test($testname);
1886     }
1887
1888     # Write the BATS plist to disk.
1889     my $json = encode_json(\%bats_plist);
1890     my $filename = "$DSTROOT$BATSBASE/objc4.plist";
1891     print "note: writing BATS config to $filename\n";
1892     open(my $file, '>', $filename);
1893     print $file $json;
1894     close $file;
1895 }
1896
1897 exit ($failed ? 1 : 0);