test/test.pl

   1 #!/usr/bin/perl
   2
   3 # test.pl
   4 # Run unit tests.
   5
   6 use strict;
   7 use File::Basename;
   8
   9 # We use encode_json() to write BATS plist files.
  10 # JSON::PP does not exist on iOS devices, but we need not write plists there.
  11 # So we simply load JSON:PP if it exists.
  12 if (eval { require JSON::PP; 1; }) {
  13     JSON::PP->import();
  14 }
  15
  16
  17 chdir dirname $0;
  18 chomp (my $DIR = `pwd`);
  19
  20 if (scalar(@ARGV) == 1) {
  21     my $arg = $ARGV[0];
  22     if ($arg eq "-h" || $arg eq "-H" || $arg eq "-help" || $arg eq "help") {
  23         print(<<END);
  24 usage: $0 [options] [testname ...]
  25        $0 help
  26
  27 testname:
  28     `testname` runs a specific test. If no testnames are given, runs all tests.
  29
  30 options:
  31     ARCH=<arch>
  32     OS=<sdk name>[sdk version][-<deployment target>[-<run target>]]
  33     ROOT=/path/to/project.roots/
  34
  35     CC=<compiler name>
  36
  37     LANGUAGE=c,c++,objective-c,objective-c++,swift
  38     MEM=mrc,arc
  39     GUARDMALLOC=0|1|before|after
  40
  41     BUILD=0|1      (build the tests?)
  42     RUN=0|1        (run the tests?)
  43     VERBOSE=0|1|2  (0=quieter  1=print commands executed  2=full test output)
  44     BATS=0|1       (build for and/or run in BATS?)
  45
  46 examples:
  47
  48     test installed library, x86_64
  49     $0
  50
  51     test buildit-built root, i386 and x86_64, MRC and ARC, clang compiler
  52     $0 ARCH=i386,x86_64 ROOT=/tmp/objc4.roots MEM=mrc,arc CC=clang
  53
  54     test buildit-built root with iOS simulator, deploy to iOS 7, run on iOS 8
  55     $0 ARCH=x86_64 ROOT=/tmp/objc4.roots OS=iphonesimulator-7.0-8.0
  56
  57     test buildit-built root on attached iOS device
  58     $0 ARCH=arm64 ROOT=/tmp/objc4.roots OS=iphoneos
  59 END
  60         exit 0;
  61     }
  62 }
  63
  64 #########################################################################
  65 ## Tests
  66
  67 # Maps test name => test's filename extension.
  68 # ex: "msgSend" => "m"
  69 # `keys %ALL_TESTS` is also used as the list of all tests found on disk.
  70 my %ALL_TESTS;
  71
  72 #########################################################################
  73 ## Variables for use in complex build and run rules
  74
  75 # variable         # example value
  76
  77 # things you can multiplex on the command line
  78 # ARCH=i386,x86_64,armv6,armv7
  79 # OS=macosx,iphoneos,iphonesimulator (plus sdk/deployment/run versions)
  80 # LANGUAGE=c,c++,objective-c,objective-c++,swift
  81 # CC=clang
  82 # MEM=mrc,arc
  83 # GUARDMALLOC=0,1,before,after
  84
  85 # things you can set once on the command line
  86 # ROOT=/path/to/project.roots
  87 # BUILD=0|1
  88 # RUN=0|1
  89 # VERBOSE=0|1|2
  90 # BATS=0|1
  91
  92 # environment variables from the command line
  93 # DSTROOT
  94 # OBJROOT
  95 # (SRCROOT is ignored; test sources are assumed to
  96 # be in the same directory as the test script itself.)
  97 # fixme SYMROOT for dsymutil output?
  98
  99
 100 # Some arguments as read from the command line.
 101 my %args;
 102 my $BUILD;
 103 my $RUN;
 104 my $VERBOSE;
 105 my $BATS;
 106
 107 my $HOST;
 108 my $PORT;
 109
 110 my @TESTLIBNAMES = ("libobjc.A.dylib", "libobjc-trampolines.dylib");
 111 my $TESTLIBDIR = "/usr/lib";
 112
 113 # Top level directory for intermediate and final build products.
 114 # Intermediate files must be kept separate for XBS BATS builds.
 115 my $OBJROOT = $ENV{OBJROOT} || "";
 116 my $DSTROOT = $ENV{DSTROOT} || "";
 117
 118 # Build product directory inside DSTROOT and OBJROOT.
 119 # Each test config gets its own build directory inside this.
 120 my $BUILDDIR;
 121
 122 # Local top-level directory.
 123 # This is the default value for $BUILDDIR.
 124 my $LOCALBASE = "/tmp/test-$TESTLIBNAMES[0]-build";
 125
 126 # Device-side top-level directory.
 127 # This replaces $DSTROOT$BUILDDIR/ for on-device execution.
 128 my $REMOTEBASE = "/AppleInternal/objctest";
 129
 130 # BATS top-level directory.
 131 # This replaces $DSTROOT$BUILDDIR/ for BATS execution.
 132 my $BATSBASE = "/AppleInternal/CoreOS/tests/objc4";
 133
 134
 135 my $crashcatch = <<'END';
 136 // interpose-able code to catch crashes, print, and exit cleanly
 137 #include <signal.h>
 138 #include <string.h>
 139 #include <unistd.h>
 140
 141 // from dyld-interposing.h
 142 #define DYLD_INTERPOSE(_replacement,_replacee) __attribute__((used)) static struct{ const void* replacement; const void* replacee; } _interpose_##_replacee __attribute__ ((section ("__DATA,__interpose"))) = { (const void*)(unsigned long)&_replacement, (const void*)(unsigned long)&_replacee };
 143
 144 static void catchcrash(int sig)
 145 {
 146     const char *msg;
 147     switch (sig) {
 148     case SIGILL:  msg = "CRASHED: SIGILL";  break;
 149     case SIGBUS:  msg = "CRASHED: SIGBUS";  break;
 150     case SIGSYS:  msg = "CRASHED: SIGSYS";  break;
 151     case SIGSEGV: msg = "CRASHED: SIGSEGV"; break;
 152     case SIGTRAP: msg = "CRASHED: SIGTRAP"; break;
 153     case SIGABRT: msg = "CRASHED: SIGABRT"; break;
 154     default: msg = "unknown signal"; break;
 155     }
 156     write(STDERR_FILENO, msg, strlen(msg));
 157
 158     // avoid backslash-n newline due to escaping differences somewhere
 159     // in BATS versus local execution (perhaps different perl versions?)
 160     char newline = 0xa;
 161     write(STDERR_FILENO, &newline, 1);
 162
 163     _exit(1);
 164 }
 165
 166 static void setupcrash(void) __attribute__((constructor));
 167 static void setupcrash(void)
 168 {
 169     signal(SIGILL, &catchcrash);
 170     signal(SIGBUS, &catchcrash);
 171     signal(SIGSYS, &catchcrash);
 172     signal(SIGSEGV, &catchcrash);
 173     signal(SIGTRAP, &catchcrash);
 174     signal(SIGABRT, &catchcrash);
 175 }
 176
 177
 178 static int hacked = 0;
 179 ssize_t hacked_write(int fildes, const void *buf, size_t nbyte)
 180 {
 181     if (!hacked) {
 182         setupcrash();
 183         hacked = 1;
 184     }
 185     return write(fildes, buf, nbyte);
 186 }
 187
 188 DYLD_INTERPOSE(hacked_write, write);
 189
 190 END
 191
 192
 193 #########################################################################
 194 ## Harness
 195
 196
 197 # map language to buildable extensions for that language
 198 my %extensions_for_language = (
 199     "c"     => ["c"],
 200     "objective-c" => ["c", "m"],
 201     "c++" => ["c", "cc", "cp", "cpp", "cxx", "c++"],
 202     "objective-c++" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"],
 203     "swift" => ["swift"],
 204
 205     "any" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm", "swift"],
 206     );
 207
 208 # map extension to languages
 209 my %languages_for_extension = (
 210     "c" => ["c", "objective-c", "c++", "objective-c++"],
 211     "m" => ["objective-c", "objective-c++"],
 212     "mm" => ["objective-c++"],
 213     "cc" => ["c++", "objective-c++"],
 214     "cp" => ["c++", "objective-c++"],
 215     "cpp" => ["c++", "objective-c++"],
 216     "cxx" => ["c++", "objective-c++"],
 217     "c++" => ["c++", "objective-c++"],
 218     "swift" => ["swift"],
 219     );
 220
 221 # Run some newline-separated commands like `make` would, stopping if any fail
 222 # run("cmd1 \n cmd2 \n cmd3")
 223 sub make {
 224     my $output = "";
 225     my @cmds = split("\n", $_[0]);
 226     die if scalar(@cmds) == 0;
 227     $? = 0;
 228     foreach my $cmd (@cmds) {
 229         chomp $cmd;
 230         next if $cmd =~ /^\s*$/;
 231         $cmd .= " 2>&1";
 232         print "$cmd\n" if $VERBOSE;
 233         $output .= `$cmd`;
 234         last if $?;
 235     }
 236     print "$output\n" if $VERBOSE;
 237     return $output;
 238 }
 239
 240 sub chdir_verbose {
 241     my $dir = shift || die;
 242     print "cd $dir\n" if $VERBOSE;
 243     chdir $dir || die "couldn't cd $dir";
 244 }
 245
 246 sub rm_rf_verbose {
 247     my $dir = shift || die;
 248     print "mkdir -p $dir\n" if $VERBOSE;
 249     `rm -rf '$dir'`;
 250     die "couldn't rm -rf $dir" if $?;
 251 }
 252
 253 sub mkdir_verbose {
 254     my $dir = shift || die;
 255     print "mkdir -p $dir\n" if $VERBOSE;
 256     `mkdir -p '$dir'`;
 257     die "couldn't mkdir $dir" if $?;
 258 }
 259
 260
 261 # xterm colors
 262 my $red = "\e[41;37m";
 263 my $yellow = "\e[43;30m";
 264 my $nocolor = "\e[0m";
 265 if (! -t STDIN) {
 266     # Not isatty. Don't use colors.
 267     $red = "";
 268     $yellow = "";
 269     $nocolor = "";
 270 }
 271
 272 # print text with a colored prefix on each line
 273 # fixme some callers pass an array of lines and some don't
 274 sub colorprefix {
 275     my $color = shift;
 276     while (defined(my $lines = shift)) {
 277         $lines = "\n" if ($lines eq "");
 278         for my $line (split(/^/, $lines)) {
 279             chomp $line;
 280             print "$color $nocolor$line\n";
 281         }
 282     }
 283 }
 284
 285 # print text colored
 286 # fixme some callers pass an array of lines and some don't
 287 sub colorprint {
 288     my $color = shift;
 289     while (defined(my $lines = shift)) {
 290         $lines = "\n" if ($lines eq "");
 291         for my $line (split(/^/, $lines)) {
 292             chomp $line;
 293             print "$color$line$nocolor\n";
 294         }
 295     }
 296 }
 297
 298 # Return test names from the command line.
 299 # Returns all tests if no tests were named.
 300 sub gettests {
 301     my @tests;
 302
 303     foreach my $arg (@ARGV) {
 304         push @tests, $arg  if ($arg !~ /=/  &&  $arg !~ /^-/);
 305     }
 306
 307     opendir(my $dir, $DIR) || die;
 308     while (my $file = readdir($dir)) {
 309         my ($name, $ext) = ($file =~ /^([^.]+)\.([^.]+)$/);
 310         next if ! $languages_for_extension{$ext};
 311
 312         open(my $in, "< $file") || die "$file";
 313         my $contents = join "", <$in>;
 314         if (defined $ALL_TESTS{$name}) {
 315             colorprint $yellow, "SKIP: multiple tests named '$name'; skipping file '$file'.";
 316         } else {
 317             $ALL_TESTS{$name} = $ext  if ($contents =~ m#^[/*\s]*TEST_#m);
 318         }
 319         close($in);
 320     }
 321     closedir($dir);
 322
 323     if (scalar(@tests) == 0) {
 324         @tests = keys %ALL_TESTS;
 325     }
 326
 327     @tests = sort @tests;
 328
 329     return @tests;
 330 }
 331
 332
 333 # Turn a C compiler name into a C++ compiler name.
 334 sub cplusplus {
 335     my ($c) = @_;
 336     if ($c =~ /cc/) {
 337         $c =~ s/cc/\+\+/;
 338         return $c;
 339     }
 340     return $c . "++";                         # e.g. clang => clang++
 341 }
 342
 343 # Turn a C compiler name into a Swift compiler name
 344 sub swift {
 345     my ($c) = @_;
 346     $c =~ s#[^/]*$#swift#;
 347     return $c;
 348 }
 349
 350 # Returns an array of all sdks from `xcodebuild -showsdks`
 351 my @sdks_memo;
 352 sub getsdks {
 353     if (!@sdks_memo) {
 354         @sdks_memo = (`xcodebuild -showsdks` =~ /-sdk (.+)$/mg);
 355     }
 356     return @sdks_memo;
 357 }
 358
 359 my %sdk_path_memo = {};
 360 sub getsdkpath {
 361     my ($sdk) = @_;
 362     if (!defined $sdk_path_memo{$sdk}) {
 363         ($sdk_path_memo{$sdk}) = (`xcodebuild -version -sdk '$sdk' Path` =~ /^\s*(.+?)\s*$/);
 364     }
 365     return $sdk_path_memo{$sdk};
 366 }
 367
 368 # Extract a version number from a string.
 369 # Ignore trailing "internal".
 370 sub versionsuffix {
 371     my ($str) = @_;
 372     my ($vers) = ($str =~ /([0-9]+\.[0-9]+)(?:\.?internal)?$/);
 373     return $vers;
 374 }
 375 sub majorversionsuffix {
 376     my ($str) = @_;
 377     my ($vers) = ($str =~ /([0-9]+)\.[0-9]+(?:\.?internal)?$/);
 378     return $vers;
 379 }
 380 sub minorversionsuffix {
 381     my ($str) = @_;
 382     my ($vers) = ($str =~ /[0-9]+\.([0-9]+)(?:\.?internal)?$/);
 383     return $vers;
 384 }
 385
 386 # Compares two SDK names and returns the newer one.
 387 # Assumes the two SDKs are the same OS.
 388 sub newersdk {
 389     my ($lhs, $rhs) = @_;
 390
 391     # Major version wins.
 392     my $lhsMajor = majorversionsuffix($lhs);
 393     my $rhsMajor = majorversionsuffix($rhs);
 394     if ($lhsMajor > $rhsMajor) { return $lhs; }
 395     if ($lhsMajor < $rhsMajor) { return $rhs; }
 396
 397     # Minor version wins.
 398     my $lhsMinor = minorversionsuffix($lhs);
 399     my $rhsMinor = minorversionsuffix($rhs);
 400     if ($lhsMinor > $rhsMinor) { return $lhs; }
 401     if ($lhsMinor < $rhsMinor) { return $rhs; }
 402
 403     # Lexically-last wins (i.e. internal is better than not internal)
 404     if ($lhs gt $rhs) { return $lhs; }
 405     return $rhs;
 406 }
 407
 408 sub rewind {
 409     seek($_[0], 0, 0);
 410 }
 411
 412 # parse name=value,value pairs
 413 sub readconditions {
 414     my ($conditionstring) = @_;
 415
 416     my %results;
 417     my @conditions = ($conditionstring =~ /\w+=(?:[^\s,]+,?)+/g);
 418     for my $condition (@conditions) {
 419         my ($name, $values) = ($condition =~ /(\w+)=(.+)/);
 420         $results{$name} = [split ',', $values];
 421     }
 422
 423     return %results;
 424 }
 425
 426 sub check_output {
 427     my %C = %{shift()};
 428     my $name = shift;
 429     my @output = @_;
 430
 431     my %T = %{$C{"TEST_$name"}};
 432
 433     # Quietly strip MallocScribble before saving the "original" output
 434     # because it is distracting.
 435     filter_malloc(\@output);
 436
 437     my @original_output = @output;
 438
 439     # Run result-checking passes, reducing @output each time
 440     my $xit = 1;
 441     my $bad = "";
 442     my $warn = "";
 443     my $runerror = $T{TEST_RUN_OUTPUT};
 444     filter_hax(\@output);
 445     filter_verbose(\@output);
 446     filter_simulator(\@output);
 447     $warn = filter_warn(\@output);
 448     $bad |= filter_guardmalloc(\@output) if ($C{GUARDMALLOC});
 449     $bad |= filter_valgrind(\@output) if ($C{VALGRIND});
 450     $bad = filter_expected(\@output, \%C, $name) if ($bad eq "");
 451     $bad = filter_bad(\@output)  if ($bad eq "");
 452
 453     # OK line should be the only one left
 454     $bad = "(output not 'OK: $name')" if ($bad eq ""  &&  (scalar(@output) != 1  ||  $output[0] !~ /^OK: $name/));
 455
 456     if ($bad ne "") {
 457         colorprint  $red, "FAIL: /// test '$name' \\\\\\";
 458         colorprefix $red, @original_output;
 459         colorprint  $red, "FAIL: \\\\\\ test '$name' ///";
 460         colorprint  $red, "FAIL: $name: $bad";
 461         $xit = 0;
 462     }
 463     elsif ($warn ne "") {
 464         colorprint  $yellow, "PASS: /// test '$name' \\\\\\";
 465         colorprefix $yellow, @original_output;
 466         colorprint  $yellow, "PASS: \\\\\\ test '$name' ///";
 467         print "PASS: $name (with warnings)\n";
 468     }
 469     else {
 470         print "PASS: $name\n";
 471     }
 472     return $xit;
 473 }
 474
 475 sub filter_expected
 476 {
 477     my $outputref = shift;
 478     my %C = %{shift()};
 479     my $name = shift;
 480
 481     my %T = %{$C{"TEST_$name"}};
 482     my $runerror = $T{TEST_RUN_OUTPUT}  ||  return "";
 483
 484     my $bad = "";
 485
 486     my $output = join("\n", @$outputref) . "\n";
 487     if ($output !~ /$runerror/) {
 488         $bad = "(run output does not match TEST_RUN_OUTPUT)";
 489         @$outputref = ("FAIL: $name");
 490     } else {
 491         @$outputref = ("OK: $name");  # pacify later filter
 492     }
 493
 494     return $bad;
 495 }
 496
 497 sub filter_bad
 498 {
 499     my $outputref = shift;
 500     my $bad = "";
 501
 502     my @new_output;
 503     for my $line (@$outputref) {
 504         if ($line =~ /^BAD: (.*)/) {
 505             $bad = "(failed)";
 506         } else {
 507             push @new_output, $line;
 508         }
 509     }
 510
 511     @$outputref = @new_output;
 512     return $bad;
 513 }
 514
 515 sub filter_warn
 516 {
 517     my $outputref = shift;
 518     my $warn = "";
 519
 520     my @new_output;
 521     for my $line (@$outputref) {
 522         if ($line !~ /^WARN: (.*)/) {
 523             push @new_output, $line;
 524         } else {
 525             $warn = "(warned)";
 526         }
 527     }
 528
 529     @$outputref = @new_output;
 530     return $warn;
 531 }
 532
 533 sub filter_verbose
 534 {
 535     my $outputref = shift;
 536
 537     my @new_output;
 538     for my $line (@$outputref) {
 539         if ($line !~ /^VERBOSE: (.*)/) {
 540             push @new_output, $line;
 541         }
 542     }
 543
 544     @$outputref = @new_output;
 545 }
 546
 547 sub filter_simulator
 548 {
 549     my $outputref = shift;
 550
 551     my @new_output;
 552     for my $line (@$outputref) {
 553         if (($line !~ /No simulator devices appear to be running/)  &&
 554             ($line !~ /CoreSimulator is attempting to unload a stale CoreSimulatorService job/)  &&
 555             ($line !~ /Failed to locate a valid instance of CoreSimulatorService/))
 556         {
 557             push @new_output, $line;
 558         }
 559     }
 560
 561     @$outputref = @new_output;
 562 }
 563
 564 sub filter_hax
 565 {
 566     my $outputref = shift;
 567
 568     my @new_output;
 569     for my $line (@$outputref) {
 570         if ($line !~ /Class OS_tcp_/) {
 571             push @new_output, $line;
 572         }
 573     }
 574
 575     @$outputref = @new_output;
 576 }
 577
 578 sub filter_valgrind
 579 {
 580     my $outputref = shift;
 581     my $errors = 0;
 582     my $leaks = 0;
 583
 584     my @new_output;
 585     for my $line (@$outputref) {
 586         if ($line =~ /^Approx: do_origins_Dirty\([RW]\): missed \d bytes$/) {
 587             # --track-origins warning (harmless)
 588             next;
 589         }
 590         if ($line =~ /^UNKNOWN __disable_threadsignal is unsupported. This warning will not be repeated.$/) {
 591             # signals unsupported (harmless)
 592             next;
 593         }
 594         if ($line =~ /^UNKNOWN __pthread_sigmask is unsupported. This warning will not be repeated.$/) {
 595             # signals unsupported (harmless)
 596             next;
 597         }
 598         if ($line !~ /^^\.*==\d+==/) {
 599             # not valgrind output
 600             push @new_output, $line;
 601             next;
 602         }
 603
 604         my ($errcount) = ($line =~ /==\d+== ERROR SUMMARY: (\d+) errors/);
 605         if (defined $errcount  &&  $errcount > 0) {
 606             $errors = 1;
 607         }
 608
 609         (my $leakcount) = ($line =~ /==\d+==\s+(?:definitely|possibly) lost:\s+([0-9,]+)/);
 610         if (defined $leakcount  &&  $leakcount > 0) {
 611             $leaks = 1;
 612         }
 613     }
 614
 615     @$outputref = @new_output;
 616
 617     my $bad = "";
 618     $bad .= "(valgrind errors)" if ($errors);
 619     $bad .= "(valgrind leaks)" if ($leaks);
 620     return $bad;
 621 }
 622
 623
 624
 625 sub filter_malloc
 626 {
 627     my $outputref = shift;
 628     my $errors = 0;
 629
 630     my @new_output;
 631     my $count = 0;
 632     for my $line (@$outputref) {
 633         # Ignore MallocScribble prologue.
 634         # Ignore MallocStackLogging prologue.
 635         if ($line =~ /malloc: enabling scribbling to detect mods to free/  ||
 636             $line =~ /Deleted objects will be dirtied by the collector/  ||
 637             $line =~ /malloc: stack logs being written into/  ||
 638             $line =~ /malloc: stack logs deleted from/  ||
 639             $line =~ /malloc: process \d+ no longer exists/  ||
 640             $line =~ /malloc: recording malloc and VM allocation stacks/)
 641         {
 642             next;
 643         }
 644
 645         # not malloc output
 646         push @new_output, $line;
 647
 648     }
 649
 650     @$outputref = @new_output;
 651 }
 652
 653 sub filter_guardmalloc
 654 {
 655     my $outputref = shift;
 656     my $errors = 0;
 657
 658     my @new_output;
 659     my $count = 0;
 660     for my $line (@$outputref) {
 661         if ($line !~ /^GuardMalloc\[[^\]]+\]: /) {
 662             # not guardmalloc output
 663             push @new_output, $line;
 664             next;
 665         }
 666
 667         # Ignore 4 lines of guardmalloc prologue.
 668         # Anything further is a guardmalloc error.
 669         if (++$count > 4) {
 670             $errors = 1;
 671         }
 672     }
 673
 674     @$outputref = @new_output;
 675
 676     my $bad = "";
 677     $bad .= "(guardmalloc errors)" if ($errors);
 678     return $bad;
 679 }
 680
 681 # TEST_SOMETHING
 682 # text
 683 # text
 684 # END
 685 sub extract_multiline {
 686     my ($flag, $contents, $name) = @_;
 687     if ($contents =~ /$flag\n/) {
 688         my ($output) = ($contents =~ /$flag\n(.*?\n)END[ *\/]*\n/s);
 689         die "$name used $flag without END\n"  if !defined($output);
 690         return $output;
 691     }
 692     return undef;
 693 }
 694
 695
 696 # TEST_SOMETHING
 697 # text
 698 # OR
 699 # text
 700 # END
 701 sub extract_multiple_multiline {
 702     my ($flag, $contents, $name) = @_;
 703     if ($contents =~ /$flag\n/) {
 704         my ($output) = ($contents =~ /$flag\n(.*?\n)END[ *\/]*\n/s);
 705         die "$name used $flag without END\n"  if !defined($output);
 706
 707         $output =~ s/\nOR\n/\n|/sg;
 708         $output = "^(" . $output . ")\$";
 709         return $output;
 710     }
 711     return undef;
 712 }
 713
 714
 715 sub gather_simple {
 716     my $CREF = shift;
 717     my %C = %{$CREF};
 718     my $name = shift;
 719     chdir_verbose $DIR;
 720
 721     my $ext = $ALL_TESTS{$name};
 722     my $file = "$name.$ext";
 723     return 0 if !$file;
 724
 725     # search file for 'TEST_CONFIG' or '#include "test.h"'
 726     # also collect other values:
 727     # TEST_DISABLED disable test with an optional message
 728     # TEST_CRASHES test is expected to crash
 729     # TEST_CONFIG test conditions
 730     # TEST_ENV environment prefix
 731     # TEST_CFLAGS compile flags
 732     # TEST_BUILD build instructions
 733     # TEST_BUILD_OUTPUT expected build stdout/stderr
 734     # TEST_RUN_OUTPUT expected run stdout/stderr
 735     open(my $in, "< $file") || die;
 736     my $contents = join "", <$in>;
 737
 738     my $test_h = ($contents =~ /^\s*#\s*(include|import)\s*"test\.h"/m);
 739     my ($disabled) = ($contents =~ /\b(TEST_DISABLED\b.*)$/m);
 740     my $crashes = ($contents =~ /\bTEST_CRASHES\b/m);
 741     my ($conditionstring) = ($contents =~ /\bTEST_CONFIG\b(.*)$/m);
 742     my ($envstring) = ($contents =~ /\bTEST_ENV\b(.*)$/m);
 743     my ($cflags) = ($contents =~ /\bTEST_CFLAGS\b(.*)$/m);
 744     my ($buildcmd) = extract_multiline("TEST_BUILD", $contents, $name);
 745     my ($builderror) = extract_multiple_multiline("TEST_BUILD_OUTPUT", $contents, $name);
 746     my ($runerror) = extract_multiple_multiline("TEST_RUN_OUTPUT", $contents, $name);
 747
 748     return 0 if !$test_h && !$disabled && !$crashes && !defined($conditionstring) && !defined($envstring) && !defined($cflags) && !defined($buildcmd) && !defined($builderror) && !defined($runerror);
 749
 750     if ($disabled) {
 751         colorprint $yellow, "SKIP: $name    (disabled by $disabled)";
 752         return 0;
 753     }
 754
 755     # check test conditions
 756
 757     my $run = 1;
 758     my %conditions = readconditions($conditionstring);
 759     if (! $conditions{LANGUAGE}) {
 760         # implicit language restriction from file extension
 761         $conditions{LANGUAGE} = $languages_for_extension{$ext};
 762     }
 763     for my $condkey (keys %conditions) {
 764         my @condvalues = @{$conditions{$condkey}};
 765
 766         # special case: RUN=0 does not affect build
 767         if ($condkey eq "RUN"  &&  @condvalues == 1  &&  $condvalues[0] == 0) {
 768             $run = 0;
 769             next;
 770         }
 771
 772         my $testvalue = $C{$condkey};
 773         next if !defined($testvalue);
 774         # testvalue is the configuration being run now
 775         # condvalues are the allowed values for this test
 776
 777         my $ok = 0;
 778         for my $condvalue (@condvalues) {
 779
 780             # special case: objc and objc++
 781             if ($condkey eq "LANGUAGE") {
 782                 $condvalue = "objective-c" if $condvalue eq "objc";
 783                 $condvalue = "objective-c++" if $condvalue eq "objc++";
 784             }
 785
 786             $ok = 1  if ($testvalue eq $condvalue);
 787
 788             # special case: CC and CXX allow substring matches
 789             if ($condkey eq "CC"  ||  $condkey eq "CXX") {
 790                 $ok = 1  if ($testvalue =~ /$condvalue/);
 791             }
 792
 793             last if $ok;
 794         }
 795
 796         if (!$ok) {
 797             my $plural = (@condvalues > 1) ? "one of: " : "";
 798             print "SKIP: $name    ($condkey=$testvalue, but test requires $plural", join(' ', @condvalues), ")\n";
 799             return 0;
 800         }
 801     }
 802
 803     # save some results for build and run phases
 804     $$CREF{"TEST_$name"} = {
 805         TEST_BUILD => $buildcmd,
 806         TEST_BUILD_OUTPUT => $builderror,
 807         TEST_CRASHES => $crashes,
 808         TEST_RUN_OUTPUT => $runerror,
 809         TEST_CFLAGS => $cflags,
 810         TEST_ENV => $envstring,
 811         TEST_RUN => $run,
 812         DSTDIR => "$C{DSTDIR}/$name.build",
 813         OBJDIR => "$C{OBJDIR}/$name.build",
 814     };
 815
 816     return 1;
 817 }
 818
 819
 820 # Test description plist to write when building for BATS execution.
 821 my %bats_plist;
 822 $bats_plist{'Project'} = "objc4";
 823 $bats_plist{'Tests'} = [];  # populated by append_bats_test()
 824
 825 # Saves run instructions for a single test in all configurations as a BATS test.
 826 sub append_bats_test {
 827     my $name = shift;
 828
 829     my $arch = join(',', @{$args{ARCH}});
 830     my $os = join(',', @{$args{OSVERSION}});
 831     my $mem = join(',', @{$args{MEM}});
 832     my $language = join(',', @{$args{LANGUAGE}});
 833
 834     push @{$bats_plist{'Tests'}}, {
 835         "TestName" => "$name",
 836         "Command" => [
 837             "/usr/bin/perl",
 838             "$BATSBASE/test/test.pl",
 839             $name,
 840             "ARCH=$arch",
 841             "OS=$os",
 842             "MEM=$mem",
 843             "LANGUAGE=$language",
 844             "BUILD=0",
 845             "RUN=1",
 846             "VERBOSE=1",
 847             "BATS=1",
 848         ]
 849     };
 850 }
 851
 852
 853 # Builds a simple test
 854 sub build_simple {
 855     my %C = %{shift()};
 856     my $name = shift;
 857     my %T = %{$C{"TEST_$name"}};
 858
 859     mkdir_verbose $T{DSTDIR};
 860     chdir_verbose $T{DSTDIR};
 861     # we don't mkdir $T{OBJDIR} because most tests don't use it
 862
 863     my $ext = $ALL_TESTS{$name};
 864     my $file = "$DIR/$name.$ext";
 865
 866     if ($T{TEST_CRASHES}) {
 867         `echo '$crashcatch' > crashcatch.c`;
 868         make("$C{COMPILE_C} -dynamiclib -o libcrashcatch.dylib -x c crashcatch.c");
 869         die "$?" if $?;
 870     }
 871
 872     my $cmd = $T{TEST_BUILD} ? eval "return \"$T{TEST_BUILD}\"" : "$C{COMPILE}   $T{TEST_CFLAGS} $file -o $name.exe";
 873
 874     my $output = make($cmd);
 875
 876     # ignore out-of-date text-based stubs (caused by ditto into SDK)
 877     $output =~ s/ld: warning: text-based stub file.*\n//g;
 878     # rdar://10163155
 879     $output =~ s/ld: warning: could not create compact unwind for [^\n]+: does not use standard frame\n//g;
 880     # rdar://37937122
 881     $output =~ s/^warning: Cannot lower [^\n]+\n//g;
 882     $output =~ s/^warning:     key: [^\n]+\n//g;
 883     $output =~ s/^warning:     discriminator: [^\n]+\n//g;
 884     $output =~ s/^warning:     callee: [^\n]+\n//g;
 885     # rdar://38710948
 886     $output =~ s/ld: warning: ignoring file [^\n]*libclang_rt\.bridgeos\.a[^\n]*\n//g;
 887     # ignore compiler logging of CCC_OVERRIDE_OPTIONS effects
 888     if (defined $ENV{CCC_OVERRIDE_OPTIONS}) {
 889         $output =~ s/### (CCC_OVERRIDE_OPTIONS:|Adding argument|Deleting argument|Replacing) [^\n]*\n//g;
 890     }
 891
 892     my $ok;
 893     if (my $builderror = $T{TEST_BUILD_OUTPUT}) {
 894         # check for expected output and ignore $?
 895         if ($output =~ /$builderror/) {
 896             $ok = 1;
 897         } elsif (defined $ENV{CCC_OVERRIDE_OPTIONS} && $builderror =~ /warning:/) {
 898             # CCC_OVERRIDE_OPTIONS manipulates compiler diagnostics.
 899             # Don't enforce any TEST_BUILD_OUTPUT that looks for warnings.
 900             colorprint  $yellow, "WARN: /// test '$name' \\\\\\";
 901             colorprefix $yellow, $output;
 902             colorprint  $yellow, "WARN: \\\\\\ test '$name' ///";
 903             colorprint  $yellow, "WARN: $name (build output does not match TEST_BUILD_OUTPUT; not fatal because CCC_OVERRIDE_OPTIONS is set)";
 904             $ok = 1;
 905         } else {
 906             colorprint  $red, "FAIL: /// test '$name' \\\\\\";
 907             colorprefix $red, $output;
 908             colorprint  $red, "FAIL: \\\\\\ test '$name' ///";
 909             colorprint  $red, "FAIL: $name (build output does not match TEST_BUILD_OUTPUT)";
 910             $ok = 0;
 911         }
 912     } elsif ($?) {
 913         colorprint  $red, "FAIL: /// test '$name' \\\\\\";
 914         colorprefix $red, $output;
 915         colorprint  $red, "FAIL: \\\\\\ test '$name' ///";
 916         colorprint  $red, "FAIL: $name (build failed)";
 917         $ok = 0;
 918     } elsif ($output ne "") {
 919         colorprint  $red, "FAIL: /// test '$name' \\\\\\";
 920         colorprefix $red, $output;
 921         colorprint  $red, "FAIL: \\\\\\ test '$name' ///";
 922         colorprint  $red, "FAIL: $name (unexpected build output)";
 923         $ok = 0;
 924     } else {
 925         $ok = 1;
 926     }
 927
 928     if ($ok) {
 929         foreach my $file (glob("*.exe *.dylib *.bundle")) {
 930             if (!$BATS) {
 931                 # not for BATS to save space and build time
 932                 # fixme use SYMROOT?
 933                 make("xcrun dsymutil $file");
 934             }
 935             if ($C{OS} eq "macosx"  ||  $C{OS} =~ /simulator/) {
 936                 # setting any entitlements disables dyld environment variables
 937             } else {
 938                 # get-task-allow entitlement is required
 939                 # to enable dyld environment variables
 940                 make("xcrun codesign -s - --entitlements $DIR/get_task_allow_entitlement.plist $file");
 941                 die "$?" if $?;
 942             }
 943         }
 944     }
 945
 946     return $ok;
 947 }
 948
 949 # Run a simple test (testname.exe, with error checking of stdout and stderr)
 950 sub run_simple {
 951     my %C = %{shift()};
 952     my $name = shift;
 953     my %T = %{$C{"TEST_$name"}};
 954
 955     if (! $T{TEST_RUN}) {
 956         print "PASS: $name (build only)\n";
 957         return 1;
 958     }
 959
 960     my $testdir = $T{DSTDIR};
 961     chdir_verbose $testdir;
 962
 963     my $env = "$C{ENV} $T{TEST_ENV}";
 964
 965     if ($T{TEST_CRASHES}) {
 966         $env .= " OBJC_DEBUG_DONT_CRASH=YES";
 967     }
 968
 969     my $output;
 970
 971     if ($C{ARCH} =~ /^arm/ && `uname -p` !~ /^arm/) {
 972         # run on iOS or watchos or tvos device
 973         # fixme device selection and verification
 974         my $remotedir = "$REMOTEBASE/" . basename($C{DSTDIR}) . "/$name.build";
 975
 976         # Add test dir and libobjc's dir to DYLD_LIBRARY_PATH.
 977         # Insert libcrashcatch.dylib if necessary.
 978         $env .= " DYLD_LIBRARY_PATH=$remotedir";
 979         $env .= ":$REMOTEBASE"  if ($C{TESTLIBDIR} ne $TESTLIBDIR);
 980         if ($T{TEST_CRASHES}) {
 981             $env .= " DYLD_INSERT_LIBRARIES=$remotedir/libcrashcatch.dylib";
 982         }
 983
 984         my $cmd = "ssh -p $PORT $HOST 'cd $remotedir && env $env ./$name.exe'";
 985         $output = make("$cmd");
 986     }
 987     elsif ($C{OS} =~ /simulator/) {
 988         # run locally in a simulator
 989         # fixme selection of simulated OS version
 990         my $simdevice;
 991         if ($C{OS} =~ /iphonesimulator/) {
 992             $simdevice = 'iPhone X';
 993         } elsif ($C{OS} =~ /watchsimulator/) {
 994             $simdevice = 'Apple Watch Series 4 - 40mm';
 995         } elsif ($C{OS} =~ /tvsimulator/) {
 996             $simdevice = 'Apple TV 1080p';
 997         } else {
 998             die "unknown simulator $C{OS}\n";
 999         }
1000         my $sim = "xcrun -sdk iphonesimulator simctl spawn '$simdevice'";
1001         # Add test dir and libobjc's dir to DYLD_LIBRARY_PATH.
1002         # Insert libcrashcatch.dylib if necessary.
1003         $env .= " DYLD_LIBRARY_PATH=$testdir";
1004         $env .= ":" . $C{TESTLIBDIR}  if ($C{TESTLIBDIR} ne $TESTLIBDIR);
1005         if ($T{TEST_CRASHES}) {
1006             $env .= " DYLD_INSERT_LIBRARIES=$testdir/libcrashcatch.dylib";
1007         }
1008
1009         my $simenv = "";
1010         foreach my $keyvalue (split(' ', $env)) {
1011             $simenv .= "SIMCTL_CHILD_$keyvalue ";
1012         }
1013         # Use the full path here so hack_cwd in test.h works.
1014         $output = make("env $simenv $sim $testdir/$name.exe");
1015     }
1016     else {
1017         # run locally
1018
1019         # Add test dir and libobjc's dir to DYLD_LIBRARY_PATH.
1020         # Insert libcrashcatch.dylib if necessary.
1021         $env .= " DYLD_LIBRARY_PATH=$testdir";
1022         $env .= ":" . $C{TESTLIBDIR}  if ($C{TESTLIBDIR} ne $TESTLIBDIR);
1023         if ($T{TEST_CRASHES}) {
1024             $env .= " DYLD_INSERT_LIBRARIES=$testdir/libcrashcatch.dylib";
1025         }
1026
1027         $output = make("sh -c '$env ./$name.exe'");
1028     }
1029
1030     return check_output(\%C, $name, split("\n", $output));
1031 }
1032
1033
1034 my %compiler_memo;
1035 sub find_compiler {
1036     my ($cc, $toolchain, $sdk_path) = @_;
1037
1038     # memoize
1039     my $key = $cc . ':' . $toolchain;
1040     my $result = $compiler_memo{$key};
1041     return $result if defined $result;
1042
1043     $result  = make("xcrun -toolchain $toolchain -find $cc 2>/dev/null");
1044
1045     chomp $result;
1046     $compiler_memo{$key} = $result;
1047     return $result;
1048 }
1049
1050 sub dirContainsAllTestLibs {
1051     my $dir = shift;
1052
1053     foreach my $testlib (@TESTLIBNAMES) {
1054         my $found = (-e "$dir/$testlib");
1055         my $foundstr = ($found ? "found" : "didn't find");
1056         print "note: $foundstr $testlib in $dir\n"  if ($VERBOSE);
1057         return 0  if (!$found);
1058     }
1059
1060     return 1;
1061 }
1062
1063 sub make_one_config {
1064     my $configref = shift;
1065     my $root = shift;
1066     my %C = %{$configref};
1067
1068     # Aliases
1069     $C{LANGUAGE} = "objective-c"  if $C{LANGUAGE} eq "objc";
1070     $C{LANGUAGE} = "objective-c++"  if $C{LANGUAGE} eq "objc++";
1071
1072     # Interpret OS version string from command line.
1073     my ($sdk_arg, $deployment_arg, $run_arg, undef) = split('-', $C{OSVERSION});
1074     delete $C{OSVERSION};
1075     my ($os_arg) = ($sdk_arg =~ /^([^\.0-9]+)/);
1076     $deployment_arg = "default" if !defined($deployment_arg);
1077     $run_arg = "default" if !defined($run_arg);
1078
1079     my %allowed_os_args = (
1080         "macosx" => "macosx", "osx" => "macosx", "macos" => "macosx",
1081         "iphoneos" => "iphoneos", "ios" => "iphoneos",
1082         "iphonesimulator" => "iphonesimulator", "iossimulator" => "iphonesimulator",
1083         "watchos" => "watchos",
1084         "watchsimulator" => "watchsimulator", "watchossimulator" => "watchsimulator",
1085         "appletvos" => "appletvos", "tvos" => "appletvos",
1086         "appletvsimulator" => "appletvsimulator", "tvsimulator" => "appletvsimulator",
1087         "bridgeos" => "bridgeos",
1088         );
1089
1090     $C{OS} = $allowed_os_args{$os_arg} || die "unknown OS '$os_arg' (expected " . join(', ', sort keys %allowed_os_args) . ")\n";
1091
1092     # set the config name now, after massaging the language and OS versions,
1093     # but before adding other settings
1094     my $configname = config_name(%C);
1095     die if ($configname =~ /'/);
1096     die if ($configname =~ / /);
1097     ($C{NAME} = $configname) =~ s/~/ /g;
1098     (my $configdir = $configname) =~ s#/##g;
1099     $C{DSTDIR} = "$DSTROOT$BUILDDIR/$configdir";
1100     $C{OBJDIR} = "$OBJROOT$BUILDDIR/$configdir";
1101
1102     # Allow tests to see BATS-edness in TEST_CONFIG.
1103     $C{BATS} = $BATS;
1104
1105     if ($C{OS} eq "iphoneos" || $C{OS} eq "iphonesimulator") {
1106         $C{TOOLCHAIN} = "ios";
1107     } elsif ($C{OS} eq "watchos" || $C{OS} eq "watchsimulator") {
1108         $C{TOOLCHAIN} = "watchos";
1109     } elsif ($C{OS} eq "appletvos" || $C{OS} eq "appletvsimulator") {
1110         $C{TOOLCHAIN} = "appletvos";
1111     } elsif ($C{OS} eq "bridgeos") {
1112         $C{TOOLCHAIN} = "bridgeos";
1113     } elsif ($C{OS} eq "macosx") {
1114         $C{TOOLCHAIN} = "osx";
1115     } else {
1116         colorprint $yellow, "WARN: don't know toolchain for OS $C{OS}";
1117         $C{TOOLCHAIN} = "default";
1118     }
1119
1120     if ($BUILD) {
1121         # Look up SDK.
1122         # Try exact match first.
1123         # Then try lexically-last prefix match
1124         #   (so "macosx" => "macosx10.7internal")
1125
1126         $sdk_arg =~ s/$os_arg/$C{OS}/;
1127
1128         my @sdks = getsdks();
1129         if ($VERBOSE) {
1130             print "note: Installed SDKs: @sdks\n";
1131         }
1132         my $exactsdk = undef;
1133         my $prefixsdk = undef;
1134         foreach my $sdk (@sdks) {
1135             $exactsdk = $sdk  if ($sdk eq $sdk_arg);
1136             $prefixsdk = newersdk($sdk, $prefixsdk)  if ($sdk =~ /^$sdk_arg/);
1137         }
1138
1139         my $sdk;
1140         if ($exactsdk) {
1141             $sdk = $exactsdk;
1142         } elsif ($prefixsdk) {
1143             $sdk = $prefixsdk;
1144         } else {
1145             die "unknown SDK '$sdk_arg'\nInstalled SDKs: @sdks\n";
1146         }
1147
1148         # Set deployment target.
1149         # fixme can't enforce version when run_arg eq "default"
1150         # because we don't know it yet
1151         $deployment_arg = versionsuffix($sdk) if $deployment_arg eq "default";
1152         if ($run_arg ne "default") {
1153             die "Deployment target '$deployment_arg' is newer than run target '$run_arg'\n"  if $deployment_arg > $run_arg;
1154         }
1155         $C{DEPLOYMENT_TARGET} = $deployment_arg;
1156         $C{SDK_PATH} = getsdkpath($sdk);
1157     } else {
1158         # not $BUILD
1159         $C{DEPLOYMENT_TARGET} = "unknown_deployment_target";
1160         $C{SDK_PATH} = "/unknown/sdk";
1161     }
1162
1163     # Set run target.
1164     $C{RUN_TARGET} = $run_arg;
1165
1166     # Look up test library (possible in root or SDK_PATH)
1167
1168     my $rootarg = $root;
1169     my $symroot;
1170     my @sympaths = ( (glob "$root/*~sym")[0],
1171                      (glob "$root/BuildRecords/*_install/Symbols")[0],
1172                      "$root/Symbols" );
1173     my @dstpaths = ( (glob "$root/*~dst")[0],
1174                      (glob "$root/BuildRecords/*_install/Root")[0],
1175                      "$root/Root" );
1176     for(my $i = 0; $i < scalar(@sympaths); $i++) {
1177         if (-e $sympaths[$i]  &&  -e $dstpaths[$i]) {
1178             $symroot = $sympaths[$i];
1179             $root = $dstpaths[$i];
1180             last;
1181         }
1182     }
1183
1184     if ($root ne "") {
1185         # Root specified. Require that it contain our dylibs.
1186         if (dirContainsAllTestLibs("$root$C{SDK_PATH}$TESTLIBDIR")) {
1187             $C{TESTLIBDIR} = "$root$C{SDK_PATH}$TESTLIBDIR";
1188         } elsif (dirContainsAllTestLibs("$root$TESTLIBDIR")) {
1189             $C{TESTLIBDIR} = "$root$TESTLIBDIR";
1190         } elsif (dirContainsAllTestLibs($root)) {
1191             $C{TESTLIBDIR} = "$root";
1192         } else {
1193             die "Didn't find some libs in root '$rootarg' for sdk '$C{SDK_PATH}'\n";
1194         }
1195     }
1196     else {
1197         # No root specified. Use the SDK or / for our dylibs.
1198         if (dirContainsAllTestLibs("$C{SDK_PATH}$TESTLIBDIR")) {
1199             $C{TESTLIBDIR} = "$C{SDK_PATH}$TESTLIBDIR";
1200         } else {
1201             # We don't actually check in / because on devices
1202             # there are no dylib files there.
1203             $C{TESTLIBDIR} = $TESTLIBDIR;
1204         }
1205     }
1206
1207     @{$C{TESTLIBS}} = map { "$C{TESTLIBDIR}/$_" } @TESTLIBNAMES;
1208     # convenience for tests that want libobjc.dylib's path
1209     $C{TESTLIB} = @{$C{TESTLIBS}}[0];
1210
1211     foreach my $testlibname (@TESTLIBNAMES) {
1212         if (-e "$symroot/$testlibname.dSYM") {
1213             push(@{$C{TESTDSYMS}}, "$symroot/$testlibname.dSYM");
1214         }
1215     }
1216
1217     if ($VERBOSE) {
1218         foreach my $testlib (@{$C{TESTLIBS}}) {
1219             my @uuids = `/usr/bin/dwarfdump -u '$testlib'`;
1220             while (my $uuid = shift @uuids) {
1221                 print "note: $uuid";
1222             }
1223         }
1224     }
1225
1226     # Look up compilers
1227     my $cc = $C{CC};
1228     my $cxx = cplusplus($C{CC});
1229     my $swift = swift($C{CC});
1230     if (! $BUILD) {
1231         $C{CC} = $cc;
1232         $C{CXX} = $cxx;
1233         $C{SWIFT} = $swift
1234     } else {
1235         $C{CC} = find_compiler($cc, $C{TOOLCHAIN}, $C{SDK_PATH});
1236         $C{CXX} = find_compiler($cxx, $C{TOOLCHAIN}, $C{SDK_PATH});
1237         $C{SWIFT} = find_compiler($swift, $C{TOOLCHAIN}, $C{SDK_PATH});
1238
1239         die "No C compiler '$cc' ('$C{CC}') in toolchain '$C{TOOLCHAIN}'\n" if !-e $C{CC};
1240         die "No C++ compiler '$cxx' ('$C{CXX}') in toolchain '$C{TOOLCHAIN}'\n" if !-e $C{CXX};
1241         die "No Swift compiler '$swift' ('$C{SWIFT}') in toolchain '$C{TOOLCHAIN}'\n" if !-e $C{SWIFT};
1242     }
1243
1244     if ($C{ARCH} eq "i386" && $C{OS} eq "macosx") {
1245         # libarclite no longer available on i386
1246         # fixme need an archived copy for bincompat testing
1247         $C{FORCE_LOAD_ARCLITE} = "";
1248     } elsif ($C{OS} eq "bridgeos") {
1249         # no libarclite on bridgeOS
1250         $C{FORCE_LOAD_ARCLITE} = "";
1251     } else {
1252         $C{FORCE_LOAD_ARCLITE} = "-Xlinker -force_load -Xlinker " . dirname($C{CC}) . "/../lib/arc/libarclite_$C{OS}.a";
1253     }
1254
1255     # Populate cflags
1256
1257     my $cflags = "-I$DIR -W -Wall -Wno-objc-weak-compat -Wno-arc-bridge-casts-disallowed-in-nonarc -Wshorten-64-to-32 -Qunused-arguments -fno-caret-diagnostics -Os -arch $C{ARCH} ";
1258     if (!$BATS) {
1259         # save-temps so dsymutil works so debug info works.
1260         # Disabled in BATS to save disk space.
1261         # rdar://45656803 -save-temps causes bad -Wstdlibcxx-not-found warnings
1262         $cflags .= "-g -save-temps -Wno-stdlibcxx-not-found";
1263     }
1264     my $objcflags = "";
1265     my $swiftflags = "-g ";
1266
1267     $cflags .= " -isysroot '$C{SDK_PATH}'";
1268     $cflags .= " '-Wl,-syslibroot,$C{SDK_PATH}'";
1269     $swiftflags .= " -sdk '$C{SDK_PATH}'";
1270
1271     # Set deployment target cflags
1272     my $target = undef;
1273     die "No deployment target" if $C{DEPLOYMENT_TARGET} eq "";
1274     if ($C{OS} eq "iphoneos") {
1275         $cflags .= " -mios-version-min=$C{DEPLOYMENT_TARGET}";
1276         $target = "$C{ARCH}-apple-ios$C{DEPLOYMENT_TARGET}";
1277     }
1278     elsif ($C{OS} eq "iphonesimulator") {
1279         $cflags .= " -mios-simulator-version-min=$C{DEPLOYMENT_TARGET}";
1280         $target = "$C{ARCH}-apple-ios$C{DEPLOYMENT_TARGET}";
1281     }
1282     elsif ($C{OS} eq "watchos") {
1283         $cflags .= " -mwatchos-version-min=$C{DEPLOYMENT_TARGET}";
1284         $target = "$C{ARCH}-apple-watchos$C{DEPLOYMENT_TARGET}";
1285     }
1286     elsif ($C{OS} eq "watchsimulator") {
1287         $cflags .= " -mwatchos-simulator-version-min=$C{DEPLOYMENT_TARGET}";
1288         $target = "$C{ARCH}-apple-watchos$C{DEPLOYMENT_TARGET}";
1289     }
1290     elsif ($C{OS} eq "appletvos") {
1291         $cflags .= " -mtvos-version-min=$C{DEPLOYMENT_TARGET}";
1292         $target = "$C{ARCH}-apple-tvos$C{DEPLOYMENT_TARGET}";
1293     }
1294     elsif ($C{OS} eq "appletvsimulator") {
1295         $cflags .= " -mtvos-simulator-version-min=$C{DEPLOYMENT_TARGET}";
1296         $target = "$C{ARCH}-apple-tvos$C{DEPLOYMENT_TARGET}";
1297     }
1298     elsif ($C{OS} eq "bridgeos") {
1299         $cflags .= " -mbridgeos-version-min=$C{DEPLOYMENT_TARGET}";
1300         $target = "$C{ARCH}-apple-bridgeos$C{DEPLOYMENT_TARGET}";
1301     }
1302     else {
1303         $cflags .= " -mmacosx-version-min=$C{DEPLOYMENT_TARGET}";
1304         $target = "$C{ARCH}-apple-macosx$C{DEPLOYMENT_TARGET}";
1305     }
1306     $swiftflags .= " -target $target";
1307
1308     $C{TESTINCLUDEDIR} = "$C{SDK_PATH}/usr/include";
1309     $C{TESTLOCALINCLUDEDIR} = "$C{SDK_PATH}/usr/local/include";
1310     if ($root ne "") {
1311         if ($C{SDK_PATH} ne "/") {
1312             $cflags .= " -isystem '$root$C{SDK_PATH}/usr/include'";
1313             $cflags .= " -isystem '$root$C{SDK_PATH}/usr/local/include'";
1314         }
1315
1316         my $library_path = $C{TESTLIBDIR};
1317         $cflags .= " -L$library_path";
1318         # fixme Root vs SDKContentRoot
1319         $C{TESTINCLUDEDIR} = "$root/../SDKContentRoot/usr/include";
1320         $C{TESTLOCALINCLUDEDIR} = "$root/../SDKContentRoot/usr/local/include";
1321         $cflags .= " -isystem '$C{TESTINCLUDEDIR}'";
1322         $cflags .= " -isystem '$C{TESTLOCALINCLUDEDIR}'";
1323     }
1324
1325
1326     # Populate objcflags
1327
1328     $objcflags .= " -lobjc";
1329     if ($C{MEM} eq "arc") {
1330         $objcflags .= " -fobjc-arc";
1331     }
1332     elsif ($C{MEM} eq "mrc") {
1333         # nothing
1334     }
1335     else {
1336         die "unrecognized MEM '$C{MEM}'\n";
1337     }
1338
1339     # Populate ENV_PREFIX
1340     $C{ENV} = "LANG=C MallocScribble=1";
1341     $C{ENV} .= " VERBOSE=$VERBOSE"  if $VERBOSE;
1342     if ($root ne "") {
1343         die "no spaces allowed in root" if $C{TESTLIBDIR} =~ /\s+/;
1344     }
1345     if ($C{GUARDMALLOC}) {
1346         $C{ENV} .= " GUARDMALLOC=1";  # checked by tests and errcheck.pl
1347         $C{ENV} .= " DYLD_INSERT_LIBRARIES=/usr/lib/libgmalloc.dylib";
1348         if ($C{GUARDMALLOC} eq "before") {
1349             $C{ENV} .= " MALLOC_PROTECT_BEFORE=1";
1350         } elsif ($C{GUARDMALLOC} eq "after") {
1351             # protect after is the default
1352         } else {
1353             die "Unknown guard malloc mode '$C{GUARDMALLOC}'\n";
1354         }
1355     }
1356
1357     # Populate compiler commands
1358     $C{XCRUN} = "env LANG=C /usr/bin/xcrun -toolchain '$C{TOOLCHAIN}'";
1359
1360     $C{COMPILE_C}   = "$C{XCRUN} '$C{CC}'  $cflags -x c -std=gnu99";
1361     $C{COMPILE_CXX} = "$C{XCRUN} '$C{CXX}' $cflags -x c++";
1362     $C{COMPILE_M}   = "$C{XCRUN} '$C{CC}'  $cflags $objcflags -x objective-c -std=gnu99";
1363     $C{COMPILE_MM}  = "$C{XCRUN} '$C{CXX}' $cflags $objcflags -x objective-c++";
1364     $C{COMPILE_SWIFT} = "$C{XCRUN} '$C{SWIFT}' $swiftflags";
1365
1366     $C{COMPILE} = $C{COMPILE_C}      if $C{LANGUAGE} eq "c";
1367     $C{COMPILE} = $C{COMPILE_CXX}    if $C{LANGUAGE} eq "c++";
1368     $C{COMPILE} = $C{COMPILE_M}      if $C{LANGUAGE} eq "objective-c";
1369     $C{COMPILE} = $C{COMPILE_MM}     if $C{LANGUAGE} eq "objective-c++";
1370     $C{COMPILE} = $C{COMPILE_SWIFT}  if $C{LANGUAGE} eq "swift";
1371     die "unknown language '$C{LANGUAGE}'\n" if !defined $C{COMPILE};
1372
1373     ($C{COMPILE_NOMEM} = $C{COMPILE}) =~ s/ -fobjc-arc\S*//g;
1374     ($C{COMPILE_NOLINK} = $C{COMPILE}) =~ s/ '?-(?:Wl,|l)\S*//g;
1375     ($C{COMPILE_NOLINK_NOMEM} = $C{COMPILE_NOMEM}) =~ s/ '?-(?:Wl,|l)\S*//g;
1376
1377
1378     # Reject some self-inconsistent and disallowed configurations
1379     if ($C{MEM} !~ /^(mrc|arc)$/) {
1380         die "unknown MEM=$C{MEM} (expected one of mrc arc)\n";
1381     }
1382
1383     if ($C{MEM} eq "arc"  &&  $C{CC} !~ /clang/) {
1384         print "note: skipping configuration $C{NAME}\n";
1385         print "note:   because CC=$C{CC} does not support MEM=$C{MEM}\n";
1386         return 0;
1387     }
1388
1389     if ($C{ARCH} eq "i386"  &&  $C{OS} eq "macosx") {
1390         colorprint $yellow, "WARN: skipping configuration $C{NAME}\n";
1391         colorprint $yellow, "WARN:   because 32-bit Mac is dead\n";
1392         return 0;
1393     }
1394
1395     # fixme
1396     if ($C{LANGUAGE} eq "swift"  &&  $C{ARCH} =~ /^arm/) {
1397         print "note: skipping configuration $C{NAME}\n";
1398         print "note:   because ARCH=$C{ARCH} does not support LANGUAGE=SWIFT\n";
1399         return 0;
1400     }
1401
1402     # fixme unimplemented run targets
1403     if ($C{RUN_TARGET} ne "default" &&  $C{OS} !~ /simulator/) {
1404         colorprint $yellow, "WARN: skipping configuration $C{NAME}";
1405         colorprint $yellow, "WARN:   because OS=$C{OS} does not yet implement RUN_TARGET=$C{RUN_TARGET}";
1406     }
1407
1408     %$configref = %C;
1409 }
1410
1411 sub make_configs {
1412     my ($root, %args) = @_;
1413
1414     my @results = ({});  # start with one empty config
1415
1416     for my $key (keys %args) {
1417         my @newresults;
1418         my @values = @{$args{$key}};
1419         for my $configref (@results) {
1420             my %config = %{$configref};
1421             for my $value (@values) {
1422                 my %newconfig = %config;
1423                 $newconfig{$key} = $value;
1424                 push @newresults, \%newconfig;
1425             }
1426         }
1427         @results = @newresults;
1428     }
1429
1430     my @newresults;
1431     for my $configref(@results) {
1432         if (make_one_config($configref, $root)) {
1433             push @newresults, $configref;
1434         }
1435     }
1436
1437     return @newresults;
1438 }
1439
1440 sub config_name {
1441     my %config = @_;
1442     my $name = "";
1443     for my $key (sort keys %config) {
1444         $name .= '~'  if $name ne "";
1445         $name .= "$key=$config{$key}";
1446     }
1447     return $name;
1448 }
1449
1450 sub rsync_ios {
1451     my ($src, $timeout) = @_;
1452     for (my $i = 0; $i < 10; $i++) {
1453         make("$DIR/timeout.pl $timeout rsync -e 'ssh -p $PORT' -av $src $HOST:/$REMOTEBASE/");
1454         return if $? == 0;
1455         colorprint $yellow, "WARN: RETRY\n"  if $VERBOSE;
1456     }
1457     die "Couldn't rsync tests to device. Check: device is connected; tcprelay is running; device trusts your Mac; device is unlocked; filesystem is mounted r/w\n";
1458 }
1459
1460 sub build_and_run_one_config {
1461     my %C = %{shift()};
1462     my @tests = @_;
1463
1464     # Build and run
1465     my $testcount = 0;
1466     my $failcount = 0;
1467     my $skipconfig = 0;
1468
1469     my @gathertests;
1470     foreach my $test (@tests) {
1471         if ($VERBOSE) {
1472             print "\nGATHER $test\n";
1473         }
1474
1475         if ($ALL_TESTS{$test}) {
1476             gather_simple(\%C, $test) || next;  # not pass, not fail
1477             push @gathertests, $test;
1478         } else {
1479             die "No test named '$test'\n";
1480         }
1481     }
1482
1483     my @builttests;
1484     if (!$BUILD) {
1485         @builttests = @gathertests;
1486         $testcount = scalar(@gathertests);
1487     } else {
1488         foreach my $test (@gathertests) {
1489             if ($VERBOSE) {
1490                 print "\nBUILD $test\n";
1491             }
1492
1493             if ($ALL_TESTS{$test}) {
1494                 $testcount++;
1495                 if (!build_simple(\%C, $test)) {
1496                     $failcount++;
1497                 } else {
1498                     push @builttests, $test;
1499                 }
1500             } else {
1501                 die "No test named '$test'\n";
1502             }
1503         }
1504     }
1505
1506     if (!$RUN  ||  !scalar(@builttests)) {
1507         # nothing to do
1508     }
1509     else {
1510         if ($C{ARCH} =~ /^arm/ && `uname -p` !~ /^arm/) {
1511             # upload timeout - longer for slow watch devices
1512             my $timeout = ($C{OS} =~ /watch/) ? 120 : 20;
1513
1514             # upload all tests to iOS device
1515             rsync_ios($C{DSTDIR}, $timeout);
1516
1517             # upload library to iOS device
1518             if ($C{TESTLIBDIR} ne $TESTLIBDIR) {
1519                 foreach my $thing (@{$C{TESTLIBS}}, @{$C{TESTDSYMS}}) {
1520                     rsync_ios($thing, $timeout);
1521                 }
1522             }
1523         }
1524         elsif ($C{OS} =~ /simulator/) {
1525             # run locally in a simulator
1526         }
1527         else {
1528             # run locally
1529             if ($BATS) {
1530                 # BATS execution tries to run architectures that
1531                 # aren't supported by the device. Skip those configs here.
1532                 my $machine = `machine`;
1533                 chomp $machine;
1534                 # unsupported:
1535                 # running arm64e on non-arm64e device
1536                 # running arm64 on non-arm64* device
1537                 # running armv7k on non-armv7k device
1538                 # running arm64_32 on armv7k device
1539                 # We don't need to handle all mismatches here,
1540                 # only mismatches that arise within a single OS.
1541                 $skipconfig =
1542                     (($C{ARCH} eq "arm64e"   && $machine ne "arm64e")  ||
1543                      ($C{ARCH} eq "arm64"    && $machine !~ /^arm64/)  ||
1544                      ($C{ARCH} eq "armv7k"   && $machine ne "armv7k")  ||
1545                      ($C{ARCH} eq "arm64_32" && $machine eq "armv7k"));
1546                 if ($skipconfig) {
1547                     print "note: skipping configuration $C{NAME}\n";
1548                     print "note:   because test arch $C{ARCH} is not " .
1549                         "supported on device arch $machine\n";
1550                     $testcount = 0;
1551                 }
1552             }
1553         }
1554
1555         if (!$skipconfig) {
1556             foreach my $test (@builttests) {
1557                 print "\nRUN $test\n"  if ($VERBOSE);
1558
1559                 if ($ALL_TESTS{$test}) {
1560                     if (!run_simple(\%C, $test)) {
1561                         $failcount++;
1562                     }
1563                 } else {
1564                     die "No test named '$test'\n";
1565                 }
1566             }
1567         }
1568     }
1569
1570     return ($testcount, $failcount, $skipconfig);
1571 }
1572
1573
1574
1575 # Return value if set by "$argname=value" on the command line
1576 # Return $default if not set.
1577 sub getargs {
1578     my ($argname, $default) = @_;
1579
1580     foreach my $arg (@ARGV) {
1581         my ($value) = ($arg =~ /^$argname=(.+)$/);
1582         return [split ',', $value] if defined $value;
1583     }
1584
1585     return [split ',', $default];
1586 }
1587
1588 # Return 1 or 0 if set by "$argname=1" or "$argname=0" on the
1589 # command line. Return $default if not set.
1590 sub getbools {
1591     my ($argname, $default) = @_;
1592
1593     my @values = @{getargs($argname, $default)};
1594     return [( map { ($_ eq "0") ? 0 : 1 } @values )];
1595 }
1596
1597 # Return an integer if set by "$argname=value" on the
1598 # command line. Return $default if not set.
1599 sub getints {
1600     my ($argname, $default) = @_;
1601
1602     my @values = @{getargs($argname, $default)};
1603     return [( map { int($_) } @values )];
1604 }
1605
1606 sub getarg {
1607     my ($argname, $default) = @_;
1608     my @values = @{getargs($argname, $default)};
1609     die "Only one value allowed for $argname\n"  if @values > 1;
1610     return $values[0];
1611 }
1612
1613 sub getbool {
1614     my ($argname, $default) = @_;
1615     my @values = @{getbools($argname, $default)};
1616     die "Only one value allowed for $argname\n"  if @values > 1;
1617     return $values[0];
1618 }
1619
1620 sub getint {
1621     my ($argname, $default) = @_;
1622     my @values = @{getints($argname, $default)};
1623     die "Only one value allowed for $argname\n"  if @values > 1;
1624     return $values[0];
1625 }
1626
1627
1628 my $default_arch = "x86_64";
1629 $args{ARCH} = getargs("ARCH", 0);
1630 $args{ARCH} = getargs("ARCHS", $default_arch)  if !@{$args{ARCH}}[0];
1631
1632 $args{OSVERSION} = getargs("OS", "macosx-default-default");
1633
1634 $args{MEM} = getargs("MEM", "mrc,arc");
1635 $args{LANGUAGE} = [ map { lc($_) } @{getargs("LANGUAGE", "c,objective-c,c++,objective-c++")} ];
1636
1637 $args{CC} = getargs("CC", "clang");
1638
1639 $HOST = getarg("HOST", "iphone");
1640 $PORT = getarg("PORT", "10022");
1641
1642 {
1643     my $guardmalloc = getargs("GUARDMALLOC", 0);
1644     # GUARDMALLOC=1 is the same as GUARDMALLOC=before,after
1645     my @guardmalloc2 = ();
1646     for my $arg (@$guardmalloc) {
1647         if ($arg == 1) { push @guardmalloc2, "before";
1648                          push @guardmalloc2, "after"; }
1649         else { push @guardmalloc2, $arg }
1650     }
1651     $args{GUARDMALLOC} = \@guardmalloc2;
1652 }
1653
1654 $BUILD = getbool("BUILD", 1);
1655 $RUN = getbool("RUN", 1);
1656 $VERBOSE = getint("VERBOSE", 0);
1657 $BATS = getbool("BATS", 0);
1658 $BUILDDIR = getarg("BUILDDIR", $BATS ? $BATSBASE : $LOCALBASE);
1659
1660 my $root = getarg("ROOT", "");
1661 $root =~ s#/*$##;
1662
1663 my @tests = gettests();
1664
1665 if ($BUILD) {
1666     rm_rf_verbose "$DSTROOT$BUILDDIR";
1667     rm_rf_verbose "$OBJROOT$BUILDDIR";
1668 }
1669
1670 print "note: -----\n";
1671 print "note: testing root '$root'\n";
1672
1673 my @configs = make_configs($root, %args);
1674
1675 print "note: -----\n";
1676 print "note: testing ", scalar(@configs), " configurations:\n";
1677 for my $configref (@configs) {
1678     my $configname = $$configref{NAME};
1679     print "note: configuration $configname\n";
1680 }
1681
1682 my $failed = 0;
1683
1684 my $testconfigs = @configs;
1685 my $failconfigs = 0;
1686 my $skipconfigs = 0;
1687 my $testcount = 0;
1688 my $failcount = 0;
1689 for my $configref (@configs) {
1690     my $configname = $$configref{NAME};
1691     print "note: -----\n";
1692     print "note: \nnote: $configname\nnote: \n";
1693
1694     (my $t, my $f, my $skipconfig) =
1695         eval { build_and_run_one_config($configref, @tests); };
1696     $skipconfigs += $skipconfig;
1697     if ($@) {
1698         chomp $@;
1699         colorprint $red, "FAIL: $configname";
1700         colorprint $red, "FAIL: $@";
1701         $failconfigs++;
1702     } else {
1703         my $color = ($f ? $red : "");
1704         print "note:\n";
1705         colorprint $color, "note: $configname\n";
1706         colorprint $color, "note: $t tests, $f failures";
1707         $testcount += $t;
1708         $failcount += $f;
1709         $failconfigs++ if ($f);
1710     }
1711 }
1712
1713 print "note: -----\n";
1714 my $color = ($failconfigs ? $red : "");
1715 colorprint $color, "note: $testconfigs configurations, " .
1716     "$failconfigs with failures, $skipconfigs skipped";
1717 colorprint $color, "note: $testcount tests, $failcount failures";
1718
1719 $failed = ($failconfigs ? 1 : 0);
1720
1721
1722 if ($BUILD && $BATS && !$failed) {
1723     # Collect BATS execution instructions for all tests.
1724     # Each BATS "test" is all configurations together of one of our tests.
1725     for my $testname (@tests) {
1726         append_bats_test($testname);
1727     }
1728
1729     # Write the BATS plist to disk.
1730     my $json = encode_json(\%bats_plist);
1731     my $filename = "$DSTROOT$BATSBASE/objc4.plist";
1732     print "note: writing BATS config to $filename\n";
1733     open(my $file, '>', $filename);
1734     print $file $json;
1735     close $file;
1736 }
1737
1738 exit ($failed ? 1 : 0);