test/test.pl

   1 #!/usr/bin/perl
   2
   3 # test.pl
   4 # Run unit tests.
   5
   6 use strict;
   7 use File::Basename;
   8
   9 # We use encode_json() to write BATS plist files.
  10 # JSON::PP does not exist on iOS devices, but we need not write plists there.
  11 # So we simply load JSON:PP if it exists.
  12 if (eval { require JSON::PP; 1; }) {
  13     JSON::PP->import();
  14 }
  15
  16
  17 chdir dirname $0;
  18 chomp (my $DIR = `pwd`);
  19
  20 if (scalar(@ARGV) == 1) {
  21     my $arg = $ARGV[0];
  22     if ($arg eq "-h" || $arg eq "-H" || $arg eq "-help" || $arg eq "help") {
  23         print(<<END);
  24 usage: $0 [options] [testname ...]
  25        $0 help
  26
  27 testname:
  28     `testname` runs a specific test. If no testnames are given, runs all tests.
  29
  30 options:
  31     ARCH=<arch>
  32     OS=<sdk name>[sdk version][-<deployment target>[-<run target>]]
  33     ROOT=/path/to/project.roots/
  34
  35     CC=<compiler name>
  36
  37     LANGUAGE=c,c++,objective-c,objective-c++,swift
  38     MEM=mrc,arc
  39     GUARDMALLOC=0|1|before|after
  40
  41     BUILD=0|1      (build the tests?)
  42     RUN=0|1        (run the tests?)
  43     VERBOSE=0|1|2  (0=quieter  1=print commands executed  2=full test output)
  44     BATS=0|1       (build for and/or run in BATS?)
  45
  46 examples:
  47
  48     test installed library, x86_64
  49     $0
  50
  51     test buildit-built root, i386 and x86_64, MRC and ARC, clang compiler
  52     $0 ARCH=i386,x86_64 ROOT=/tmp/objc4.roots MEM=mrc,arc CC=clang
  53
  54     test buildit-built root with iOS simulator, deploy to iOS 7, run on iOS 8
  55     $0 ARCH=x86_64 ROOT=/tmp/objc4.roots OS=iphonesimulator-7.0-8.0
  56
  57     test buildit-built root on attached iOS device
  58     $0 ARCH=arm64 ROOT=/tmp/objc4.roots OS=iphoneos
  59 END
  60         exit 0;
  61     }
  62 }
  63
  64 #########################################################################
  65 ## Tests
  66
  67 # Maps test name => test's filename extension.
  68 # ex: "msgSend" => "m"
  69 # `keys %ALL_TESTS` is also used as the list of all tests found on disk.
  70 my %ALL_TESTS;
  71
  72 #########################################################################
  73 ## Variables for use in complex build and run rules
  74
  75 # variable         # example value
  76
  77 # things you can multiplex on the command line
  78 # ARCH=i386,x86_64,armv6,armv7
  79 # OS=macosx,iphoneos,iphonesimulator (plus sdk/deployment/run versions)
  80 # LANGUAGE=c,c++,objective-c,objective-c++,swift
  81 # CC=clang
  82 # MEM=mrc,arc
  83 # GUARDMALLOC=0,1,before,after
  84
  85 # things you can set once on the command line
  86 # ROOT=/path/to/project.roots
  87 # BUILD=0|1
  88 # RUN=0|1
  89 # VERBOSE=0|1|2
  90 # BATS=0|1
  91
  92 # environment variables from the command line
  93 # DSTROOT
  94 # OBJROOT
  95 # (SRCROOT is ignored; test sources are assumed to
  96 # be in the same directory as the test script itself.)
  97 # fixme SYMROOT for dsymutil output?
  98
  99
 100 # Some arguments as read from the command line.
 101 my %args;
 102 my $BUILD;
 103 my $RUN;
 104 my $VERBOSE;
 105 my $BATS;
 106
 107 my @TESTLIBNAMES = ("libobjc.A.dylib", "libobjc-trampolines.dylib");
 108 my $TESTLIBDIR = "/usr/lib";
 109
 110 # Top level directory for intermediate and final build products.
 111 # Intermediate files must be kept separate for XBS BATS builds.
 112 my $OBJROOT = $ENV{OBJROOT} || "";
 113 my $DSTROOT = $ENV{DSTROOT} || "";
 114
 115 # Build product directory inside DSTROOT and OBJROOT.
 116 # Each test config gets its own build directory inside this.
 117 my $BUILDDIR;
 118
 119 # Local top-level directory.
 120 # This is the default value for $BUILDDIR.
 121 my $LOCALBASE = "/tmp/test-$TESTLIBNAMES[0]-build";
 122
 123 # Device-side top-level directory.
 124 # This replaces $DSTROOT$BUILDDIR/ for on-device execution.
 125 my $REMOTEBASE = "/AppleInternal/objctest";
 126
 127 # BATS top-level directory.
 128 # This replaces $DSTROOT$BUILDDIR/ for BATS execution.
 129 my $BATSBASE = "/AppleInternal/CoreOS/tests/objc4";
 130
 131
 132 my $crashcatch = <<'END';
 133 // interpose-able code to catch crashes, print, and exit cleanly
 134 #include <signal.h>
 135 #include <string.h>
 136 #include <unistd.h>
 137
 138 // from dyld-interposing.h
 139 #define DYLD_INTERPOSE(_replacement,_replacee) __attribute__((used)) static struct{ const void* replacement; const void* replacee; } _interpose_##_replacee __attribute__ ((section ("__DATA,__interpose"))) = { (const void*)(unsigned long)&_replacement, (const void*)(unsigned long)&_replacee };
 140
 141 static void catchcrash(int sig)
 142 {
 143     const char *msg;
 144     switch (sig) {
 145     case SIGILL:  msg = "CRASHED: SIGILL";  break;
 146     case SIGBUS:  msg = "CRASHED: SIGBUS";  break;
 147     case SIGSYS:  msg = "CRASHED: SIGSYS";  break;
 148     case SIGSEGV: msg = "CRASHED: SIGSEGV"; break;
 149     case SIGTRAP: msg = "CRASHED: SIGTRAP"; break;
 150     case SIGABRT: msg = "CRASHED: SIGABRT"; break;
 151     default: msg = "unknown signal"; break;
 152     }
 153     write(STDERR_FILENO, msg, strlen(msg));
 154
 155     // avoid backslash-n newline due to escaping differences somewhere
 156     // in BATS versus local execution (perhaps different perl versions?)
 157     char newline = 0xa;
 158     write(STDERR_FILENO, &newline, 1);
 159
 160     _exit(1);
 161 }
 162
 163 static void setupcrash(void) __attribute__((constructor));
 164 static void setupcrash(void)
 165 {
 166     signal(SIGILL, &catchcrash);
 167     signal(SIGBUS, &catchcrash);
 168     signal(SIGSYS, &catchcrash);
 169     signal(SIGSEGV, &catchcrash);
 170     signal(SIGTRAP, &catchcrash);
 171     signal(SIGABRT, &catchcrash);
 172 }
 173
 174
 175 static int hacked = 0;
 176 ssize_t hacked_write(int fildes, const void *buf, size_t nbyte)
 177 {
 178     if (!hacked) {
 179         setupcrash();
 180         hacked = 1;
 181     }
 182     return write(fildes, buf, nbyte);
 183 }
 184
 185 DYLD_INTERPOSE(hacked_write, write);
 186
 187 END
 188
 189
 190 #########################################################################
 191 ## Harness
 192
 193
 194 # map language to buildable extensions for that language
 195 my %extensions_for_language = (
 196     "c"     => ["c"],
 197     "objective-c" => ["c", "m"],
 198     "c++" => ["c", "cc", "cp", "cpp", "cxx", "c++"],
 199     "objective-c++" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"],
 200     "swift" => ["swift"],
 201
 202     "any" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm", "swift"],
 203     );
 204
 205 # map extension to languages
 206 my %languages_for_extension = (
 207     "c" => ["c", "objective-c", "c++", "objective-c++"],
 208     "m" => ["objective-c", "objective-c++"],
 209     "mm" => ["objective-c++"],
 210     "cc" => ["c++", "objective-c++"],
 211     "cp" => ["c++", "objective-c++"],
 212     "cpp" => ["c++", "objective-c++"],
 213     "cxx" => ["c++", "objective-c++"],
 214     "c++" => ["c++", "objective-c++"],
 215     "swift" => ["swift"],
 216     );
 217
 218 # Run some newline-separated commands like `make` would, stopping if any fail
 219 # run("cmd1 \n cmd2 \n cmd3")
 220 sub make {
 221     my $output = "";
 222     my @cmds = split("\n", $_[0]);
 223     die if scalar(@cmds) == 0;
 224     $? = 0;
 225     foreach my $cmd (@cmds) {
 226         chomp $cmd;
 227         next if $cmd =~ /^\s*$/;
 228         $cmd .= " 2>&1";
 229         print "$cmd\n" if $VERBOSE;
 230         $output .= `$cmd`;
 231         last if $?;
 232     }
 233     print "$output\n" if $VERBOSE;
 234     return $output;
 235 }
 236
 237 sub chdir_verbose {
 238     my $dir = shift || die;
 239     print "cd $dir\n" if $VERBOSE;
 240     chdir $dir || die "couldn't cd $dir";
 241 }
 242
 243 sub rm_rf_verbose {
 244     my $dir = shift || die;
 245     print "mkdir -p $dir\n" if $VERBOSE;
 246     `rm -rf '$dir'`;
 247     die "couldn't rm -rf $dir" if $?;
 248 }
 249
 250 sub mkdir_verbose {
 251     my $dir = shift || die;
 252     print "mkdir -p $dir\n" if $VERBOSE;
 253     `mkdir -p '$dir'`;
 254     die "couldn't mkdir $dir" if $?;
 255 }
 256
 257
 258 # xterm colors
 259 my $red = "\e[41;37m";
 260 my $yellow = "\e[43;30m";
 261 my $nocolor = "\e[0m";
 262 if (! -t STDIN) {
 263     # Not isatty. Don't use colors.
 264     $red = "";
 265     $yellow = "";
 266     $nocolor = "";
 267 }
 268
 269 # print text with a colored prefix on each line
 270 # fixme some callers pass an array of lines and some don't
 271 sub colorprefix {
 272     my $color = shift;
 273     while (defined(my $lines = shift)) {
 274         $lines = "\n" if ($lines eq "");
 275         for my $line (split(/^/, $lines)) {
 276             chomp $line;
 277             print "$color $nocolor$line\n";
 278         }
 279     }
 280 }
 281
 282 # print text colored
 283 # fixme some callers pass an array of lines and some don't
 284 sub colorprint {
 285     my $color = shift;
 286     while (defined(my $lines = shift)) {
 287         $lines = "\n" if ($lines eq "");
 288         for my $line (split(/^/, $lines)) {
 289             chomp $line;
 290             print "$color$line$nocolor\n";
 291         }
 292     }
 293 }
 294
 295 # Return test names from the command line.
 296 # Returns all tests if no tests were named.
 297 sub gettests {
 298     my @tests;
 299
 300     foreach my $arg (@ARGV) {
 301         push @tests, $arg  if ($arg !~ /=/  &&  $arg !~ /^-/);
 302     }
 303
 304     opendir(my $dir, $DIR) || die;
 305     while (my $file = readdir($dir)) {
 306         my ($name, $ext) = ($file =~ /^([^.]+)\.([^.]+)$/);
 307         next if ! $languages_for_extension{$ext};
 308
 309         open(my $in, "< $file") || die "$file";
 310         my $contents = join "", <$in>;
 311         if (defined $ALL_TESTS{$name}) {
 312             colorprint $yellow, "SKIP: multiple tests named '$name'; skipping file '$file'.";
 313         } else {
 314             $ALL_TESTS{$name} = $ext  if ($contents =~ m#^[/*\s]*TEST_#m);
 315         }
 316         close($in);
 317     }
 318     closedir($dir);
 319
 320     if (scalar(@tests) == 0) {
 321         @tests = keys %ALL_TESTS;
 322     }
 323
 324     @tests = sort @tests;
 325
 326     return @tests;
 327 }
 328
 329
 330 # Turn a C compiler name into a C++ compiler name.
 331 sub cplusplus {
 332     my ($c) = @_;
 333     if ($c =~ /cc/) {
 334         $c =~ s/cc/\+\+/;
 335         return $c;
 336     }
 337     return $c . "++";                         # e.g. clang => clang++
 338 }
 339
 340 # Turn a C compiler name into a Swift compiler name
 341 sub swift {
 342     my ($c) = @_;
 343     $c =~ s#[^/]*$#swift#;
 344     return $c;
 345 }
 346
 347 # Returns an array of all sdks from `xcodebuild -showsdks`
 348 my @sdks_memo;
 349 sub getsdks {
 350     if (!@sdks_memo) {
 351         @sdks_memo = (`xcodebuild -showsdks` =~ /-sdk (.+)$/mg);
 352     }
 353     return @sdks_memo;
 354 }
 355
 356 my %sdk_path_memo = {};
 357 sub getsdkpath {
 358     my ($sdk) = @_;
 359     if (!defined $sdk_path_memo{$sdk}) {
 360         ($sdk_path_memo{$sdk}) = (`xcodebuild -version -sdk '$sdk' Path` =~ /^\s*(.+?)\s*$/);
 361     }
 362     return $sdk_path_memo{$sdk};
 363 }
 364
 365 # Extract a version number from a string.
 366 # Ignore trailing "internal".
 367 sub versionsuffix {
 368     my ($str) = @_;
 369     my ($vers) = ($str =~ /([0-9]+\.[0-9]+)(?:\.?internal)?$/);
 370     return $vers;
 371 }
 372 sub majorversionsuffix {
 373     my ($str) = @_;
 374     my ($vers) = ($str =~ /([0-9]+)\.[0-9]+(?:\.?internal)?$/);
 375     return $vers;
 376 }
 377 sub minorversionsuffix {
 378     my ($str) = @_;
 379     my ($vers) = ($str =~ /[0-9]+\.([0-9]+)(?:\.?internal)?$/);
 380     return $vers;
 381 }
 382
 383 # Compares two SDK names and returns the newer one.
 384 # Assumes the two SDKs are the same OS.
 385 sub newersdk {
 386     my ($lhs, $rhs) = @_;
 387
 388     # Major version wins.
 389     my $lhsMajor = majorversionsuffix($lhs);
 390     my $rhsMajor = majorversionsuffix($rhs);
 391     if ($lhsMajor > $rhsMajor) { return $lhs; }
 392     if ($lhsMajor < $rhsMajor) { return $rhs; }
 393
 394     # Minor version wins.
 395     my $lhsMinor = minorversionsuffix($lhs);
 396     my $rhsMinor = minorversionsuffix($rhs);
 397     if ($lhsMinor > $rhsMinor) { return $lhs; }
 398     if ($lhsMinor < $rhsMinor) { return $rhs; }
 399
 400     # Lexically-last wins (i.e. internal is better than not internal)
 401     if ($lhs gt $rhs) { return $lhs; }
 402     return $rhs;
 403 }
 404
 405 sub rewind {
 406     seek($_[0], 0, 0);
 407 }
 408
 409 # parse name=value,value pairs
 410 sub readconditions {
 411     my ($conditionstring) = @_;
 412
 413     my %results;
 414     my @conditions = ($conditionstring =~ /\w+=(?:[^\s,]+,?)+/g);
 415     for my $condition (@conditions) {
 416         my ($name, $values) = ($condition =~ /(\w+)=(.+)/);
 417         $results{$name} = [split ',', $values];
 418     }
 419
 420     return %results;
 421 }
 422
 423 sub check_output {
 424     my %C = %{shift()};
 425     my $name = shift;
 426     my @output = @_;
 427
 428     my %T = %{$C{"TEST_$name"}};
 429
 430     # Quietly strip MallocScribble before saving the "original" output
 431     # because it is distracting.
 432     filter_malloc(\@output);
 433
 434     my @original_output = @output;
 435
 436     # Run result-checking passes, reducing @output each time
 437     my $xit = 1;
 438     my $bad = "";
 439     my $warn = "";
 440     my $runerror = $T{TEST_RUN_OUTPUT};
 441     filter_hax(\@output);
 442     filter_verbose(\@output);
 443     filter_simulator(\@output);
 444     $warn = filter_warn(\@output);
 445     $bad |= filter_guardmalloc(\@output) if ($C{GUARDMALLOC});
 446     $bad |= filter_valgrind(\@output) if ($C{VALGRIND});
 447     $bad = filter_expected(\@output, \%C, $name) if ($bad eq "");
 448     $bad = filter_bad(\@output)  if ($bad eq "");
 449
 450     # OK line should be the only one left
 451     $bad = "(output not 'OK: $name')" if ($bad eq ""  &&  (scalar(@output) != 1  ||  $output[0] !~ /^OK: $name/));
 452
 453     if ($bad ne "") {
 454         colorprint  $red, "FAIL: /// test '$name' \\\\\\";
 455         colorprefix $red, @original_output;
 456         colorprint  $red, "FAIL: \\\\\\ test '$name' ///";
 457         colorprint  $red, "FAIL: $name: $bad";
 458         $xit = 0;
 459     }
 460     elsif ($warn ne "") {
 461         colorprint  $yellow, "PASS: /// test '$name' \\\\\\";
 462         colorprefix $yellow, @original_output;
 463         colorprint  $yellow, "PASS: \\\\\\ test '$name' ///";
 464         print "PASS: $name (with warnings)\n";
 465     }
 466     else {
 467         print "PASS: $name\n";
 468     }
 469     return $xit;
 470 }
 471
 472 sub filter_expected
 473 {
 474     my $outputref = shift;
 475     my %C = %{shift()};
 476     my $name = shift;
 477
 478     my %T = %{$C{"TEST_$name"}};
 479     my $runerror = $T{TEST_RUN_OUTPUT}  ||  return "";
 480
 481     my $bad = "";
 482
 483     my $output = join("\n", @$outputref) . "\n";
 484     if ($output !~ /$runerror/) {
 485         $bad = "(run output does not match TEST_RUN_OUTPUT)";
 486         @$outputref = ("FAIL: $name");
 487     } else {
 488         @$outputref = ("OK: $name");  # pacify later filter
 489     }
 490
 491     return $bad;
 492 }
 493
 494 sub filter_bad
 495 {
 496     my $outputref = shift;
 497     my $bad = "";
 498
 499     my @new_output;
 500     for my $line (@$outputref) {
 501         if ($line =~ /^BAD: (.*)/) {
 502             $bad = "(failed)";
 503         } else {
 504             push @new_output, $line;
 505         }
 506     }
 507
 508     @$outputref = @new_output;
 509     return $bad;
 510 }
 511
 512 sub filter_warn
 513 {
 514     my $outputref = shift;
 515     my $warn = "";
 516
 517     my @new_output;
 518     for my $line (@$outputref) {
 519         if ($line !~ /^WARN: (.*)/) {
 520             push @new_output, $line;
 521         } else {
 522             $warn = "(warned)";
 523         }
 524     }
 525
 526     @$outputref = @new_output;
 527     return $warn;
 528 }
 529
 530 sub filter_verbose
 531 {
 532     my $outputref = shift;
 533
 534     my @new_output;
 535     for my $line (@$outputref) {
 536         if ($line !~ /^VERBOSE: (.*)/) {
 537             push @new_output, $line;
 538         }
 539     }
 540
 541     @$outputref = @new_output;
 542 }
 543
 544 sub filter_simulator
 545 {
 546     my $outputref = shift;
 547
 548     my @new_output;
 549     for my $line (@$outputref) {
 550         if (($line !~ /No simulator devices appear to be running/)  &&
 551             ($line !~ /CoreSimulator is attempting to unload a stale CoreSimulatorService job/)  &&
 552             ($line !~ /Failed to locate a valid instance of CoreSimulatorService/))
 553         {
 554             push @new_output, $line;
 555         }
 556     }
 557
 558     @$outputref = @new_output;
 559 }
 560
 561 sub filter_hax
 562 {
 563     my $outputref = shift;
 564
 565     my @new_output;
 566     for my $line (@$outputref) {
 567         if ($line !~ /Class OS_tcp_/) {
 568             push @new_output, $line;
 569         }
 570     }
 571
 572     @$outputref = @new_output;
 573 }
 574
 575 sub filter_valgrind
 576 {
 577     my $outputref = shift;
 578     my $errors = 0;
 579     my $leaks = 0;
 580
 581     my @new_output;
 582     for my $line (@$outputref) {
 583         if ($line =~ /^Approx: do_origins_Dirty\([RW]\): missed \d bytes$/) {
 584             # --track-origins warning (harmless)
 585             next;
 586         }
 587         if ($line =~ /^UNKNOWN __disable_threadsignal is unsupported. This warning will not be repeated.$/) {
 588             # signals unsupported (harmless)
 589             next;
 590         }
 591         if ($line =~ /^UNKNOWN __pthread_sigmask is unsupported. This warning will not be repeated.$/) {
 592             # signals unsupported (harmless)
 593             next;
 594         }
 595         if ($line !~ /^^\.*==\d+==/) {
 596             # not valgrind output
 597             push @new_output, $line;
 598             next;
 599         }
 600
 601         my ($errcount) = ($line =~ /==\d+== ERROR SUMMARY: (\d+) errors/);
 602         if (defined $errcount  &&  $errcount > 0) {
 603             $errors = 1;
 604         }
 605
 606         (my $leakcount) = ($line =~ /==\d+==\s+(?:definitely|possibly) lost:\s+([0-9,]+)/);
 607         if (defined $leakcount  &&  $leakcount > 0) {
 608             $leaks = 1;
 609         }
 610     }
 611
 612     @$outputref = @new_output;
 613
 614     my $bad = "";
 615     $bad .= "(valgrind errors)" if ($errors);
 616     $bad .= "(valgrind leaks)" if ($leaks);
 617     return $bad;
 618 }
 619
 620
 621
 622 sub filter_malloc
 623 {
 624     my $outputref = shift;
 625     my $errors = 0;
 626
 627     my @new_output;
 628     my $count = 0;
 629     for my $line (@$outputref) {
 630         # Ignore MallocScribble prologue.
 631         # Ignore MallocStackLogging prologue.
 632         if ($line =~ /malloc: enabling scribbling to detect mods to free/  ||
 633             $line =~ /Deleted objects will be dirtied by the collector/  ||
 634             $line =~ /malloc: stack logs being written into/  ||
 635             $line =~ /malloc: stack logs deleted from/  ||
 636             $line =~ /malloc: process \d+ no longer exists/  ||
 637             $line =~ /malloc: recording malloc and VM allocation stacks/)
 638         {
 639             next;
 640         }
 641
 642         # not malloc output
 643         push @new_output, $line;
 644
 645     }
 646
 647     @$outputref = @new_output;
 648 }
 649
 650 sub filter_guardmalloc
 651 {
 652     my $outputref = shift;
 653     my $errors = 0;
 654
 655     my @new_output;
 656     my $count = 0;
 657     for my $line (@$outputref) {
 658         if ($line !~ /^GuardMalloc\[[^\]]+\]: /) {
 659             # not guardmalloc output
 660             push @new_output, $line;
 661             next;
 662         }
 663
 664         # Ignore 4 lines of guardmalloc prologue.
 665         # Anything further is a guardmalloc error.
 666         if (++$count > 4) {
 667             $errors = 1;
 668         }
 669     }
 670
 671     @$outputref = @new_output;
 672
 673     my $bad = "";
 674     $bad .= "(guardmalloc errors)" if ($errors);
 675     return $bad;
 676 }
 677
 678 # TEST_SOMETHING
 679 # text
 680 # text
 681 # END
 682 sub extract_multiline {
 683     my ($flag, $contents, $name) = @_;
 684     if ($contents =~ /$flag\n/) {
 685         my ($output) = ($contents =~ /$flag\n(.*?\n)END[ *\/]*\n/s);
 686         die "$name used $flag without END\n"  if !defined($output);
 687         return $output;
 688     }
 689     return undef;
 690 }
 691
 692
 693 # TEST_SOMETHING
 694 # text
 695 # OR
 696 # text
 697 # END
 698 sub extract_multiple_multiline {
 699     my ($flag, $contents, $name) = @_;
 700     if ($contents =~ /$flag\n/) {
 701         my ($output) = ($contents =~ /$flag\n(.*?\n)END[ *\/]*\n/s);
 702         die "$name used $flag without END\n"  if !defined($output);
 703
 704         $output =~ s/\nOR\n/\n|/sg;
 705         $output = "^(" . $output . ")\$";
 706         return $output;
 707     }
 708     return undef;
 709 }
 710
 711
 712 sub gather_simple {
 713     my $CREF = shift;
 714     my %C = %{$CREF};
 715     my $name = shift;
 716     chdir_verbose $DIR;
 717
 718     my $ext = $ALL_TESTS{$name};
 719     my $file = "$name.$ext";
 720     return 0 if !$file;
 721
 722     # search file for 'TEST_CONFIG' or '#include "test.h"'
 723     # also collect other values:
 724     # TEST_DISABLED disable test with an optional message
 725     # TEST_CRASHES test is expected to crash
 726     # TEST_CONFIG test conditions
 727     # TEST_ENV environment prefix
 728     # TEST_CFLAGS compile flags
 729     # TEST_BUILD build instructions
 730     # TEST_BUILD_OUTPUT expected build stdout/stderr
 731     # TEST_RUN_OUTPUT expected run stdout/stderr
 732     open(my $in, "< $file") || die;
 733     my $contents = join "", <$in>;
 734
 735     my $test_h = ($contents =~ /^\s*#\s*(include|import)\s*"test\.h"/m);
 736     my ($disabled) = ($contents =~ /\b(TEST_DISABLED\b.*)$/m);
 737     my $crashes = ($contents =~ /\bTEST_CRASHES\b/m);
 738     my ($conditionstring) = ($contents =~ /\bTEST_CONFIG\b(.*)$/m);
 739     my ($envstring) = ($contents =~ /\bTEST_ENV\b(.*)$/m);
 740     my ($cflags) = ($contents =~ /\bTEST_CFLAGS\b(.*)$/m);
 741     my ($buildcmd) = extract_multiline("TEST_BUILD", $contents, $name);
 742     my ($builderror) = extract_multiple_multiline("TEST_BUILD_OUTPUT", $contents, $name);
 743     my ($runerror) = extract_multiple_multiline("TEST_RUN_OUTPUT", $contents, $name);
 744
 745     return 0 if !$test_h && !$disabled && !$crashes && !defined($conditionstring) && !defined($envstring) && !defined($cflags) && !defined($buildcmd) && !defined($builderror) && !defined($runerror);
 746
 747     if ($disabled) {
 748         colorprint $yellow, "SKIP: $name    (disabled by $disabled)";
 749         return 0;
 750     }
 751
 752     # check test conditions
 753
 754     my $run = 1;
 755     my %conditions = readconditions($conditionstring);
 756     if (! $conditions{LANGUAGE}) {
 757         # implicit language restriction from file extension
 758         $conditions{LANGUAGE} = $languages_for_extension{$ext};
 759     }
 760     for my $condkey (keys %conditions) {
 761         my @condvalues = @{$conditions{$condkey}};
 762
 763         # special case: RUN=0 does not affect build
 764         if ($condkey eq "RUN"  &&  @condvalues == 1  &&  $condvalues[0] == 0) {
 765             $run = 0;
 766             next;
 767         }
 768
 769         my $testvalue = $C{$condkey};
 770         next if !defined($testvalue);
 771         # testvalue is the configuration being run now
 772         # condvalues are the allowed values for this test
 773
 774         my $ok = 0;
 775         for my $condvalue (@condvalues) {
 776
 777             # special case: objc and objc++
 778             if ($condkey eq "LANGUAGE") {
 779                 $condvalue = "objective-c" if $condvalue eq "objc";
 780                 $condvalue = "objective-c++" if $condvalue eq "objc++";
 781             }
 782
 783             $ok = 1  if ($testvalue eq $condvalue);
 784
 785             # special case: CC and CXX allow substring matches
 786             if ($condkey eq "CC"  ||  $condkey eq "CXX") {
 787                 $ok = 1  if ($testvalue =~ /$condvalue/);
 788             }
 789
 790             last if $ok;
 791         }
 792
 793         if (!$ok) {
 794             my $plural = (@condvalues > 1) ? "one of: " : "";
 795             print "SKIP: $name    ($condkey=$testvalue, but test requires $plural", join(' ', @condvalues), ")\n";
 796             return 0;
 797         }
 798     }
 799
 800     # save some results for build and run phases
 801     $$CREF{"TEST_$name"} = {
 802         TEST_BUILD => $buildcmd,
 803         TEST_BUILD_OUTPUT => $builderror,
 804         TEST_CRASHES => $crashes,
 805         TEST_RUN_OUTPUT => $runerror,
 806         TEST_CFLAGS => $cflags,
 807         TEST_ENV => $envstring,
 808         TEST_RUN => $run,
 809         DSTDIR => "$C{DSTDIR}/$name.build",
 810         OBJDIR => "$C{OBJDIR}/$name.build",
 811     };
 812
 813     return 1;
 814 }
 815
 816
 817 # Test description plist to write when building for BATS execution.
 818 my %bats_plist;
 819 $bats_plist{'Project'} = "objc4";
 820 $bats_plist{'Tests'} = [];  # populated by append_bats_test()
 821
 822 # Saves run instructions for a single test in all configurations as a BATS test.
 823 sub append_bats_test {
 824     my $name = shift;
 825
 826     my $arch = join(',', @{$args{ARCH}});
 827     my $os = join(',', @{$args{OSVERSION}});
 828     my $mem = join(',', @{$args{MEM}});
 829     my $language = join(',', @{$args{LANGUAGE}});
 830
 831     push @{$bats_plist{'Tests'}}, {
 832         "TestName" => "$name",
 833         "Command" => [
 834             "/usr/bin/perl",
 835             "$BATSBASE/test/test.pl",
 836             $name,
 837             "ARCH=$arch",
 838             "OS=$os",
 839             "MEM=$mem",
 840             "LANGUAGE=$language",
 841             "BUILD=0",
 842             "RUN=1",
 843             "VERBOSE=1",
 844             "BATS=1",
 845         ]
 846     };
 847 }
 848
 849
 850 # Builds a simple test
 851 sub build_simple {
 852     my %C = %{shift()};
 853     my $name = shift;
 854     my %T = %{$C{"TEST_$name"}};
 855
 856     mkdir_verbose $T{DSTDIR};
 857     chdir_verbose $T{DSTDIR};
 858     # we don't mkdir $T{OBJDIR} because most tests don't use it
 859
 860     my $ext = $ALL_TESTS{$name};
 861     my $file = "$DIR/$name.$ext";
 862
 863     if ($T{TEST_CRASHES}) {
 864         `echo '$crashcatch' > crashcatch.c`;
 865         make("$C{COMPILE_C} -dynamiclib -o libcrashcatch.dylib -x c crashcatch.c");
 866         die "$?" if $?;
 867     }
 868
 869     my $cmd = $T{TEST_BUILD} ? eval "return \"$T{TEST_BUILD}\"" : "$C{COMPILE}   $T{TEST_CFLAGS} $file -o $name.exe";
 870
 871     my $output = make($cmd);
 872
 873     # ignore out-of-date text-based stubs (caused by ditto into SDK)
 874     $output =~ s/ld: warning: text-based stub file.*\n//g;
 875     # rdar://10163155
 876     $output =~ s/ld: warning: could not create compact unwind for [^\n]+: does not use standard frame\n//g;
 877     # rdar://37937122
 878     $output =~ s/^warning: Cannot lower [^\n]+\n//g;
 879     $output =~ s/^warning:     key: [^\n]+\n//g;
 880     $output =~ s/^warning:     discriminator: [^\n]+\n//g;
 881     $output =~ s/^warning:     callee: [^\n]+\n//g;
 882
 883     my $ok;
 884     if (my $builderror = $T{TEST_BUILD_OUTPUT}) {
 885         # check for expected output and ignore $?
 886         if ($output =~ /$builderror/) {
 887             $ok = 1;
 888         } else {
 889             colorprint  $red, "FAIL: /// test '$name' \\\\\\";
 890             colorprefix $red, $output;
 891             colorprint  $red, "FAIL: \\\\\\ test '$name' ///";
 892             colorprint  $red, "FAIL: $name (build output does not match TEST_BUILD_OUTPUT)";
 893             $ok = 0;
 894         }
 895     } elsif ($?) {
 896         colorprint  $red, "FAIL: /// test '$name' \\\\\\";
 897         colorprefix $red, $output;
 898         colorprint  $red, "FAIL: \\\\\\ test '$name' ///";
 899         colorprint  $red, "FAIL: $name (build failed)";
 900         $ok = 0;
 901     } elsif ($output ne "") {
 902         colorprint  $red, "FAIL: /// test '$name' \\\\\\";
 903         colorprefix $red, $output;
 904         colorprint  $red, "FAIL: \\\\\\ test '$name' ///";
 905         colorprint  $red, "FAIL: $name (unexpected build output)";
 906         $ok = 0;
 907     } else {
 908         $ok = 1;
 909     }
 910
 911     if ($ok) {
 912         foreach my $file (glob("*.exe *.dylib *.bundle")) {
 913             if (!$BATS) {
 914                 # not for BATS to save space and build time
 915                 # fixme use SYMROOT?
 916                 make("xcrun dsymutil $file");
 917             }
 918             if ($C{OS} eq "macosx"  ||  $C{OS} =~ /simulator/) {
 919                 # setting any entitlements disables dyld environment variables
 920             } else {
 921                 # get-task-allow entitlement is required
 922                 # to enable dyld environment variables
 923                 make("xcrun codesign -s - --entitlements $DIR/get_task_allow_entitlement.plist $file");
 924                 die "$?" if $?;
 925             }
 926         }
 927     }
 928
 929     return $ok;
 930 }
 931
 932 # Run a simple test (testname.exe, with error checking of stdout and stderr)
 933 sub run_simple {
 934     my %C = %{shift()};
 935     my $name = shift;
 936     my %T = %{$C{"TEST_$name"}};
 937
 938     if (! $T{TEST_RUN}) {
 939         print "PASS: $name (build only)\n";
 940         return 1;
 941     }
 942
 943     my $testdir = $T{DSTDIR};
 944     chdir_verbose $testdir;
 945
 946     my $env = "$C{ENV} $T{TEST_ENV}";
 947
 948     if ($T{TEST_CRASHES}) {
 949         $env .= " OBJC_DEBUG_DONT_CRASH=YES";
 950     }
 951
 952     my $output;
 953
 954     if ($C{ARCH} =~ /^arm/ && `uname -p` !~ /^arm/) {
 955         # run on iOS or watchos or tvos device
 956         # fixme device selection and verification
 957         my $remotedir = "$REMOTEBASE/" . basename($C{DSTDIR}) . "/$name.build";
 958
 959         # Add test dir and libobjc's dir to DYLD_LIBRARY_PATH.
 960         # Insert libcrashcatch.dylib if necessary.
 961         $env .= " DYLD_LIBRARY_PATH=$remotedir";
 962         $env .= ":$REMOTEBASE"  if ($C{TESTLIBDIR} ne $TESTLIBDIR);
 963         if ($T{TEST_CRASHES}) {
 964             $env .= " DYLD_INSERT_LIBRARIES=$remotedir/libcrashcatch.dylib";
 965         }
 966
 967         my $cmd = "ssh iphone 'cd $remotedir && env $env ./$name.exe'";
 968         $output = make("$cmd");
 969     }
 970     elsif ($C{OS} =~ /simulator/) {
 971         # run locally in a simulator
 972         # fixme selection of simulated OS version
 973         my $simdevice;
 974         if ($C{OS} =~ /iphonesimulator/) {
 975             $simdevice = 'iPhone 6';
 976         } elsif ($C{OS} =~ /watchsimulator/) {
 977             $simdevice = 'Apple Watch Series 4 - 40mm';
 978         } elsif ($C{OS} =~ /tvsimulator/) {
 979             $simdevice = 'Apple TV 1080p';
 980         } else {
 981             die "unknown simulator $C{OS}\n";
 982         }
 983         my $sim = "xcrun -sdk iphonesimulator simctl spawn '$simdevice'";
 984         # Add test dir and libobjc's dir to DYLD_LIBRARY_PATH.
 985         # Insert libcrashcatch.dylib if necessary.
 986         $env .= " DYLD_LIBRARY_PATH=$testdir";
 987         $env .= ":" . $C{TESTLIBDIR}  if ($C{TESTLIBDIR} ne $TESTLIBDIR);
 988         if ($T{TEST_CRASHES}) {
 989             $env .= " DYLD_INSERT_LIBRARIES=$testdir/libcrashcatch.dylib";
 990         }
 991
 992         my $simenv = "";
 993         foreach my $keyvalue (split(' ', $env)) {
 994             $simenv .= "SIMCTL_CHILD_$keyvalue ";
 995         }
 996         # Use the full path here so hack_cwd in test.h works.
 997         $output = make("env $simenv $sim $testdir/$name.exe");
 998     }
 999     else {
1000         # run locally
1001
1002         # Add test dir and libobjc's dir to DYLD_LIBRARY_PATH.
1003         # Insert libcrashcatch.dylib if necessary.
1004         $env .= " DYLD_LIBRARY_PATH=$testdir";
1005         $env .= ":" . $C{TESTLIBDIR}  if ($C{TESTLIBDIR} ne $TESTLIBDIR);
1006         if ($T{TEST_CRASHES}) {
1007             $env .= " DYLD_INSERT_LIBRARIES=$testdir/libcrashcatch.dylib";
1008         }
1009
1010         $output = make("sh -c '$env ./$name.exe'");
1011     }
1012
1013     return check_output(\%C, $name, split("\n", $output));
1014 }
1015
1016
1017 my %compiler_memo;
1018 sub find_compiler {
1019     my ($cc, $toolchain, $sdk_path) = @_;
1020
1021     # memoize
1022     my $key = $cc . ':' . $toolchain;
1023     my $result = $compiler_memo{$key};
1024     return $result if defined $result;
1025
1026     $result  = make("xcrun -toolchain $toolchain -find $cc 2>/dev/null");
1027
1028     chomp $result;
1029     $compiler_memo{$key} = $result;
1030     return $result;
1031 }
1032
1033 sub dirContainsAllTestLibs {
1034     my $dir = shift;
1035
1036     foreach my $testlib (@TESTLIBNAMES) {
1037         my $found = (-e "$dir/$testlib");
1038         my $foundstr = ($found ? "found" : "didn't find");
1039         print "note: $foundstr $testlib in $dir\n"  if ($VERBOSE);
1040         return 0  if (!$found);
1041     }
1042
1043     return 1;
1044 }
1045
1046 sub make_one_config {
1047     my $configref = shift;
1048     my $root = shift;
1049     my %C = %{$configref};
1050
1051     # Aliases
1052     $C{LANGUAGE} = "objective-c"  if $C{LANGUAGE} eq "objc";
1053     $C{LANGUAGE} = "objective-c++"  if $C{LANGUAGE} eq "objc++";
1054
1055     # Interpret OS version string from command line.
1056     my ($sdk_arg, $deployment_arg, $run_arg, undef) = split('-', $C{OSVERSION});
1057     delete $C{OSVERSION};
1058     my ($os_arg) = ($sdk_arg =~ /^([^\.0-9]+)/);
1059     $deployment_arg = "default" if !defined($deployment_arg);
1060     $run_arg = "default" if !defined($run_arg);
1061
1062     my %allowed_os_args = (
1063         "macosx" => "macosx", "osx" => "macosx", "macos" => "macosx",
1064         "iphoneos" => "iphoneos", "ios" => "iphoneos",
1065         "iphonesimulator" => "iphonesimulator", "iossimulator" => "iphonesimulator",
1066         "watchos" => "watchos",
1067         "watchsimulator" => "watchsimulator", "watchossimulator" => "watchsimulator",
1068         "appletvos" => "appletvos", "tvos" => "appletvos",
1069         "appletvsimulator" => "appletvsimulator", "tvsimulator" => "appletvsimulator",
1070         "bridgeos" => "bridgeos",
1071         );
1072
1073     $C{OS} = $allowed_os_args{$os_arg} || die "unknown OS '$os_arg' (expected " . join(', ', sort keys %allowed_os_args) . ")\n";
1074
1075     # set the config name now, after massaging the language and OS versions,
1076     # but before adding other settings
1077     my $configname = config_name(%C);
1078     die if ($configname =~ /'/);
1079     die if ($configname =~ / /);
1080     ($C{NAME} = $configname) =~ s/~/ /g;
1081     (my $configdir = $configname) =~ s#/##g;
1082     $C{DSTDIR} = "$DSTROOT$BUILDDIR/$configdir";
1083     $C{OBJDIR} = "$OBJROOT$BUILDDIR/$configdir";
1084
1085     # Allow tests to see BATS-edness in TEST_CONFIG.
1086     $C{BATS} = $BATS;
1087
1088     if ($C{OS} eq "iphoneos" || $C{OS} eq "iphonesimulator") {
1089         $C{TOOLCHAIN} = "ios";
1090     } elsif ($C{OS} eq "watchos" || $C{OS} eq "watchsimulator") {
1091         $C{TOOLCHAIN} = "watchos";
1092     } elsif ($C{OS} eq "appletvos" || $C{OS} eq "appletvsimulator") {
1093         $C{TOOLCHAIN} = "appletvos";
1094     } elsif ($C{OS} eq "bridgeos") {
1095         $C{TOOLCHAIN} = "bridgeos";
1096     } elsif ($C{OS} eq "macosx") {
1097         $C{TOOLCHAIN} = "osx";
1098     } else {
1099         colorprint $yellow, "WARN: don't know toolchain for OS $C{OS}";
1100         $C{TOOLCHAIN} = "default";
1101     }
1102
1103     if ($BUILD) {
1104         # Look up SDK.
1105         # Try exact match first.
1106         # Then try lexically-last prefix match
1107         #   (so "macosx" => "macosx10.7internal")
1108
1109         $sdk_arg =~ s/$os_arg/$C{OS}/;
1110
1111         my @sdks = getsdks();
1112         if ($VERBOSE) {
1113             print "note: Installed SDKs: @sdks\n";
1114         }
1115         my $exactsdk = undef;
1116         my $prefixsdk = undef;
1117         foreach my $sdk (@sdks) {
1118             $exactsdk = $sdk  if ($sdk eq $sdk_arg);
1119             $prefixsdk = newersdk($sdk, $prefixsdk)  if ($sdk =~ /^$sdk_arg/);
1120         }
1121
1122         my $sdk;
1123         if ($exactsdk) {
1124             $sdk = $exactsdk;
1125         } elsif ($prefixsdk) {
1126             $sdk = $prefixsdk;
1127         } else {
1128             die "unknown SDK '$sdk_arg'\nInstalled SDKs: @sdks\n";
1129         }
1130
1131         # Set deployment target.
1132         # fixme can't enforce version when run_arg eq "default"
1133         # because we don't know it yet
1134         $deployment_arg = versionsuffix($sdk) if $deployment_arg eq "default";
1135         if ($run_arg ne "default") {
1136             die "Deployment target '$deployment_arg' is newer than run target '$run_arg'\n"  if $deployment_arg > $run_arg;
1137         }
1138         $C{DEPLOYMENT_TARGET} = $deployment_arg;
1139         $C{SDK_PATH} = getsdkpath($sdk);
1140     } else {
1141         # not $BUILD
1142         $C{DEPLOYMENT_TARGET} = "unknown_deployment_target";
1143         $C{SDK_PATH} = "/unknown/sdk";
1144     }
1145
1146     # Set run target.
1147     $C{RUN_TARGET} = $run_arg;
1148
1149     # Look up test library (possible in root or SDK_PATH)
1150
1151     my $rootarg = $root;
1152     my $symroot;
1153     my @sympaths = ( (glob "$root/*~sym")[0],
1154                      (glob "$root/BuildRecords/*_install/Symbols")[0],
1155                      "$root/Symbols" );
1156     my @dstpaths = ( (glob "$root/*~dst")[0],
1157                      (glob "$root/BuildRecords/*_install/Root")[0],
1158                      "$root/Root" );
1159     for(my $i = 0; $i < scalar(@sympaths); $i++) {
1160         if (-e $sympaths[$i]  &&  -e $dstpaths[$i]) {
1161             $symroot = $sympaths[$i];
1162             $root = $dstpaths[$i];
1163             last;
1164         }
1165     }
1166
1167     if ($root ne "") {
1168         # Root specified. Require that it contain our dylibs.
1169         if (dirContainsAllTestLibs("$root$C{SDK_PATH}$TESTLIBDIR")) {
1170             $C{TESTLIBDIR} = "$root$C{SDK_PATH}$TESTLIBDIR";
1171         } elsif (dirContainsAllTestLibs("$root$TESTLIBDIR")) {
1172             $C{TESTLIBDIR} = "$root$TESTLIBDIR";
1173         } elsif (dirContainsAllTestLibs($root)) {
1174             $C{TESTLIBDIR} = "$root";
1175         } else {
1176             die "Didn't find some libs in root '$rootarg' for sdk '$C{SDK_PATH}'\n";
1177         }
1178     }
1179     else {
1180         # No root specified. Use the SDK or / for our dylibs.
1181         if (dirContainsAllTestLibs("$C{SDK_PATH}$TESTLIBDIR")) {
1182             $C{TESTLIBDIR} = "$C{SDK_PATH}$TESTLIBDIR";
1183         } else {
1184             # We don't actually check in / because on devices
1185             # there are no dylib files there.
1186             $C{TESTLIBDIR} = $TESTLIBDIR;
1187         }
1188     }
1189
1190     @{$C{TESTLIBS}} = map { "$C{TESTLIBDIR}/$_" } @TESTLIBNAMES;
1191     # convenience for tests that want libobjc.dylib's path
1192     $C{TESTLIB} = @{$C{TESTLIBS}}[0];
1193
1194     foreach my $testlibname (@TESTLIBNAMES) {
1195         if (-e "$symroot/$testlibname.dSYM") {
1196             push(@{$C{TESTDSYMS}}, "$symroot/$testlibname.dSYM");
1197         }
1198     }
1199
1200     if ($VERBOSE) {
1201         foreach my $testlib (@{$C{TESTLIBS}}) {
1202             my @uuids = `/usr/bin/dwarfdump -u '$testlib'`;
1203             while (my $uuid = shift @uuids) {
1204                 print "note: $uuid";
1205             }
1206         }
1207     }
1208
1209     # Look up compilers
1210     my $cc = $C{CC};
1211     my $cxx = cplusplus($C{CC});
1212     my $swift = swift($C{CC});
1213     if (! $BUILD) {
1214         $C{CC} = $cc;
1215         $C{CXX} = $cxx;
1216         $C{SWIFT} = $swift
1217     } else {
1218         $C{CC} = find_compiler($cc, $C{TOOLCHAIN}, $C{SDK_PATH});
1219         $C{CXX} = find_compiler($cxx, $C{TOOLCHAIN}, $C{SDK_PATH});
1220         $C{SWIFT} = find_compiler($swift, $C{TOOLCHAIN}, $C{SDK_PATH});
1221
1222         die "No C compiler '$cc' ('$C{CC}') in toolchain '$C{TOOLCHAIN}'\n" if !-e $C{CC};
1223         die "No C++ compiler '$cxx' ('$C{CXX}') in toolchain '$C{TOOLCHAIN}'\n" if !-e $C{CXX};
1224         die "No Swift compiler '$swift' ('$C{SWIFT}') in toolchain '$C{TOOLCHAIN}'\n" if !-e $C{SWIFT};
1225     }
1226
1227     if ($C{ARCH} eq "i386" && $C{OS} eq "macosx") {
1228         # libarclite no longer available on i386
1229         # fixme need an archived copy for bincompat testing
1230         $C{FORCE_LOAD_ARCLITE} = "";
1231     } else {
1232         $C{FORCE_LOAD_ARCLITE} = "-Xlinker -force_load -Xlinker " . dirname($C{CC}) . "/../lib/arc/libarclite_$C{OS}.a";
1233     }
1234
1235     # Populate cflags
1236
1237     my $cflags = "-I$DIR -W -Wall -Wno-objc-weak-compat -Wno-arc-bridge-casts-disallowed-in-nonarc -Wshorten-64-to-32 -Qunused-arguments -fno-caret-diagnostics -Os -arch $C{ARCH} ";
1238     if (!$BATS) {
1239         # save-temps so dsymutil works so debug info works.
1240         # Disabled in BATS to save disk space.
1241         # rdar://45656803 -save-temps causes bad -Wstdlibcxx-not-found warnings
1242         $cflags .= "-g -save-temps -Wno-stdlibcxx-not-found";
1243     }
1244     my $objcflags = "";
1245     my $swiftflags = "-g ";
1246
1247     $cflags .= " -isysroot '$C{SDK_PATH}'";
1248     $cflags .= " '-Wl,-syslibroot,$C{SDK_PATH}'";
1249     $swiftflags .= " -sdk '$C{SDK_PATH}'";
1250
1251     # Set deployment target cflags
1252     my $target = undef;
1253     die "No deployment target" if $C{DEPLOYMENT_TARGET} eq "";
1254     if ($C{OS} eq "iphoneos") {
1255         $cflags .= " -mios-version-min=$C{DEPLOYMENT_TARGET}";
1256         $target = "$C{ARCH}-apple-ios$C{DEPLOYMENT_TARGET}";
1257     }
1258     elsif ($C{OS} eq "iphonesimulator") {
1259         $cflags .= " -mios-simulator-version-min=$C{DEPLOYMENT_TARGET}";
1260         $target = "$C{ARCH}-apple-ios$C{DEPLOYMENT_TARGET}";
1261     }
1262     elsif ($C{OS} eq "watchos") {
1263         $cflags .= " -mwatchos-version-min=$C{DEPLOYMENT_TARGET}";
1264         $target = "$C{ARCH}-apple-watchos$C{DEPLOYMENT_TARGET}";
1265     }
1266     elsif ($C{OS} eq "watchsimulator") {
1267         $cflags .= " -mwatchos-simulator-version-min=$C{DEPLOYMENT_TARGET}";
1268         $target = "$C{ARCH}-apple-watchos$C{DEPLOYMENT_TARGET}";
1269     }
1270     elsif ($C{OS} eq "appletvos") {
1271         $cflags .= " -mtvos-version-min=$C{DEPLOYMENT_TARGET}";
1272         $target = "$C{ARCH}-apple-tvos$C{DEPLOYMENT_TARGET}";
1273     }
1274     elsif ($C{OS} eq "appletvsimulator") {
1275         $cflags .= " -mtvos-simulator-version-min=$C{DEPLOYMENT_TARGET}";
1276         $target = "$C{ARCH}-apple-tvos$C{DEPLOYMENT_TARGET}";
1277     }
1278     elsif ($C{OS} eq "bridgeos") {
1279         $cflags .= " -mbridgeos-version-min=$C{DEPLOYMENT_TARGET}";
1280         $target = "$C{ARCH}-apple-bridgeos$C{DEPLOYMENT_TARGET}";
1281     }
1282     else {
1283         $cflags .= " -mmacosx-version-min=$C{DEPLOYMENT_TARGET}";
1284         $target = "$C{ARCH}-apple-macosx$C{DEPLOYMENT_TARGET}";
1285     }
1286     $swiftflags .= " -target $target";
1287
1288     $C{TESTINCLUDEDIR} = "$C{SDK_PATH}/usr/include";
1289     $C{TESTLOCALINCLUDEDIR} = "$C{SDK_PATH}/usr/local/include";
1290     if ($root ne "") {
1291         if ($C{SDK_PATH} ne "/") {
1292             $cflags .= " -isystem '$root$C{SDK_PATH}/usr/include'";
1293             $cflags .= " -isystem '$root$C{SDK_PATH}/usr/local/include'";
1294         }
1295
1296         my $library_path = $C{TESTLIBDIR};
1297         $cflags .= " -L$library_path";
1298         # fixme Root vs SDKContentRoot
1299         $C{TESTINCLUDEDIR} = "$root/../SDKContentRoot/usr/include";
1300         $C{TESTLOCALINCLUDEDIR} = "$root/../SDKContentRoot/usr/local/include";
1301         $cflags .= " -isystem '$C{TESTINCLUDEDIR}'";
1302         $cflags .= " -isystem '$C{TESTLOCALINCLUDEDIR}'";
1303     }
1304
1305
1306     # Populate objcflags
1307
1308     $objcflags .= " -lobjc";
1309     if ($C{MEM} eq "arc") {
1310         $objcflags .= " -fobjc-arc";
1311     }
1312     elsif ($C{MEM} eq "mrc") {
1313         # nothing
1314     }
1315     else {
1316         die "unrecognized MEM '$C{MEM}'\n";
1317     }
1318
1319     # Populate ENV_PREFIX
1320     $C{ENV} = "LANG=C MallocScribble=1";
1321     $C{ENV} .= " VERBOSE=$VERBOSE"  if $VERBOSE;
1322     if ($root ne "") {
1323         die "no spaces allowed in root" if $C{TESTLIBDIR} =~ /\s+/;
1324     }
1325     if ($C{GUARDMALLOC}) {
1326         $C{ENV} .= " GUARDMALLOC=1";  # checked by tests and errcheck.pl
1327         $C{ENV} .= " DYLD_INSERT_LIBRARIES=/usr/lib/libgmalloc.dylib";
1328         if ($C{GUARDMALLOC} eq "before") {
1329             $C{ENV} .= " MALLOC_PROTECT_BEFORE=1";
1330         } elsif ($C{GUARDMALLOC} eq "after") {
1331             # protect after is the default
1332         } else {
1333             die "Unknown guard malloc mode '$C{GUARDMALLOC}'\n";
1334         }
1335     }
1336
1337     # Populate compiler commands
1338     $C{XCRUN} = "env LANG=C /usr/bin/xcrun -toolchain '$C{TOOLCHAIN}'";
1339
1340     $C{COMPILE_C}   = "$C{XCRUN} '$C{CC}'  $cflags -x c -std=gnu99";
1341     $C{COMPILE_CXX} = "$C{XCRUN} '$C{CXX}' $cflags -x c++";
1342     $C{COMPILE_M}   = "$C{XCRUN} '$C{CC}'  $cflags $objcflags -x objective-c -std=gnu99";
1343     $C{COMPILE_MM}  = "$C{XCRUN} '$C{CXX}' $cflags $objcflags -x objective-c++";
1344     $C{COMPILE_SWIFT} = "$C{XCRUN} '$C{SWIFT}' $swiftflags";
1345
1346     $C{COMPILE} = $C{COMPILE_C}      if $C{LANGUAGE} eq "c";
1347     $C{COMPILE} = $C{COMPILE_CXX}    if $C{LANGUAGE} eq "c++";
1348     $C{COMPILE} = $C{COMPILE_M}      if $C{LANGUAGE} eq "objective-c";
1349     $C{COMPILE} = $C{COMPILE_MM}     if $C{LANGUAGE} eq "objective-c++";
1350     $C{COMPILE} = $C{COMPILE_SWIFT}  if $C{LANGUAGE} eq "swift";
1351     die "unknown language '$C{LANGUAGE}'\n" if !defined $C{COMPILE};
1352
1353     ($C{COMPILE_NOMEM} = $C{COMPILE}) =~ s/ -fobjc-arc\S*//g;
1354     ($C{COMPILE_NOLINK} = $C{COMPILE}) =~ s/ '?-(?:Wl,|l)\S*//g;
1355     ($C{COMPILE_NOLINK_NOMEM} = $C{COMPILE_NOMEM}) =~ s/ '?-(?:Wl,|l)\S*//g;
1356
1357
1358     # Reject some self-inconsistent and disallowed configurations
1359     if ($C{MEM} !~ /^(mrc|arc)$/) {
1360         die "unknown MEM=$C{MEM} (expected one of mrc arc)\n";
1361     }
1362
1363     if ($C{MEM} eq "arc"  &&  $C{CC} !~ /clang/) {
1364         print "note: skipping configuration $C{NAME}\n";
1365         print "note:   because CC=$C{CC} does not support MEM=$C{MEM}\n";
1366         return 0;
1367     }
1368
1369     if ($C{ARCH} eq "i386"  &&  $C{OS} eq "macosx") {
1370         colorprint $yellow, "WARN: skipping configuration $C{NAME}\n";
1371         colorprint $yellow, "WARN:   because 32-bit Mac is dead\n";
1372         return 0;
1373     }
1374
1375     # fixme
1376     if ($C{LANGUAGE} eq "swift"  &&  $C{ARCH} =~ /^arm/) {
1377         print "note: skipping configuration $C{NAME}\n";
1378         print "note:   because ARCH=$C{ARCH} does not support LANGUAGE=SWIFT\n";
1379         return 0;
1380     }
1381
1382     # fixme unimplemented run targets
1383     if ($C{RUN_TARGET} ne "default" &&  $C{OS} !~ /simulator/) {
1384         colorprint $yellow, "WARN: skipping configuration $C{NAME}";
1385         colorprint $yellow, "WARN:   because OS=$C{OS} does not yet implement RUN_TARGET=$C{RUN_TARGET}";
1386     }
1387
1388     %$configref = %C;
1389 }
1390
1391 sub make_configs {
1392     my ($root, %args) = @_;
1393
1394     my @results = ({});  # start with one empty config
1395
1396     for my $key (keys %args) {
1397         my @newresults;
1398         my @values = @{$args{$key}};
1399         for my $configref (@results) {
1400             my %config = %{$configref};
1401             for my $value (@values) {
1402                 my %newconfig = %config;
1403                 $newconfig{$key} = $value;
1404                 push @newresults, \%newconfig;
1405             }
1406         }
1407         @results = @newresults;
1408     }
1409
1410     my @newresults;
1411     for my $configref(@results) {
1412         if (make_one_config($configref, $root)) {
1413             push @newresults, $configref;
1414         }
1415     }
1416
1417     return @newresults;
1418 }
1419
1420 sub config_name {
1421     my %config = @_;
1422     my $name = "";
1423     for my $key (sort keys %config) {
1424         $name .= '~'  if $name ne "";
1425         $name .= "$key=$config{$key}";
1426     }
1427     return $name;
1428 }
1429
1430 sub rsync_ios {
1431     my ($src, $timeout) = @_;
1432     for (my $i = 0; $i < 10; $i++) {
1433         make("$DIR/timeout.pl $timeout env RSYNC_PASSWORD=alpine rsync -av $src rsync://root\@localhost:10873/root/$REMOTEBASE/");
1434         return if $? == 0;
1435         colorprint $yellow, "WARN: RETRY\n"  if $VERBOSE;
1436     }
1437     die "Couldn't rsync tests to device. Check: device is connected; tcprelay is running; device trusts your Mac; device is unlocked; filesystem is mounted r/w\n";
1438 }
1439
1440 sub build_and_run_one_config {
1441     my %C = %{shift()};
1442     my @tests = @_;
1443
1444     # Build and run
1445     my $testcount = 0;
1446     my $failcount = 0;
1447     my $skipconfig = 0;
1448
1449     my @gathertests;
1450     foreach my $test (@tests) {
1451         if ($VERBOSE) {
1452             print "\nGATHER $test\n";
1453         }
1454
1455         if ($ALL_TESTS{$test}) {
1456             gather_simple(\%C, $test) || next;  # not pass, not fail
1457             push @gathertests, $test;
1458         } else {
1459             die "No test named '$test'\n";
1460         }
1461     }
1462
1463     my @builttests;
1464     if (!$BUILD) {
1465         @builttests = @gathertests;
1466         $testcount = scalar(@gathertests);
1467     } else {
1468         foreach my $test (@gathertests) {
1469             if ($VERBOSE) {
1470                 print "\nBUILD $test\n";
1471             }
1472
1473             if ($ALL_TESTS{$test}) {
1474                 $testcount++;
1475                 if (!build_simple(\%C, $test)) {
1476                     $failcount++;
1477                 } else {
1478                     push @builttests, $test;
1479                 }
1480             } else {
1481                 die "No test named '$test'\n";
1482             }
1483         }
1484     }
1485
1486     if (!$RUN  ||  !scalar(@builttests)) {
1487         # nothing to do
1488     }
1489     else {
1490         if ($C{ARCH} =~ /^arm/ && `uname -p` !~ /^arm/) {
1491             # upload timeout - longer for slow watch devices
1492             my $timeout = ($C{OS} =~ /watch/) ? 120 : 20;
1493
1494             # upload all tests to iOS device
1495             rsync_ios($C{DSTDIR}, $timeout);
1496
1497             # upload library to iOS device
1498             if ($C{TESTLIBDIR} ne $TESTLIBDIR) {
1499                 foreach my $thing (@{$C{TESTLIBS}}, @{$C{TESTDSYMS}}) {
1500                     rsync_ios($thing, $timeout);
1501                 }
1502             }
1503         }
1504         elsif ($C{OS} =~ /simulator/) {
1505             # run locally in a simulator
1506         }
1507         else {
1508             # run locally
1509             if ($BATS) {
1510                 # BATS execution tries to run architectures that
1511                 # aren't supported by the device. Skip those configs here.
1512                 my $machine = `machine`;
1513                 chomp $machine;
1514                 # unsupported:
1515                 # running arm64e on non-arm64e device
1516                 # running arm64 on non-arm64* device
1517                 # running armv7k on non-armv7k device
1518                 # running arm64_32 on armv7k device
1519                 # We don't need to handle all mismatches here,
1520                 # only mismatches that arise within a single OS.
1521                 $skipconfig =
1522                     (($C{ARCH} eq "arm64e"   && $machine ne "arm64e")  ||
1523                      ($C{ARCH} eq "arm64"    && $machine !~ /^arm64/)  ||
1524                      ($C{ARCH} eq "armv7k"   && $machine ne "armv7k")  ||
1525                      ($C{ARCH} eq "arm64_32" && $machine eq "armv7k"));
1526                 if ($skipconfig) {
1527                     print "note: skipping configuration $C{NAME}\n";
1528                     print "note:   because test arch $C{ARCH} is not " .
1529                         "supported on device arch $machine\n";
1530                     $testcount = 0;
1531                 }
1532             }
1533         }
1534
1535         if (!$skipconfig) {
1536             foreach my $test (@builttests) {
1537                 print "\nRUN $test\n"  if ($VERBOSE);
1538
1539                 if ($ALL_TESTS{$test}) {
1540                     if (!run_simple(\%C, $test)) {
1541                         $failcount++;
1542                     }
1543                 } else {
1544                     die "No test named '$test'\n";
1545                 }
1546             }
1547         }
1548     }
1549
1550     return ($testcount, $failcount, $skipconfig);
1551 }
1552
1553
1554
1555 # Return value if set by "$argname=value" on the command line
1556 # Return $default if not set.
1557 sub getargs {
1558     my ($argname, $default) = @_;
1559
1560     foreach my $arg (@ARGV) {
1561         my ($value) = ($arg =~ /^$argname=(.+)$/);
1562         return [split ',', $value] if defined $value;
1563     }
1564
1565     return [split ',', $default];
1566 }
1567
1568 # Return 1 or 0 if set by "$argname=1" or "$argname=0" on the
1569 # command line. Return $default if not set.
1570 sub getbools {
1571     my ($argname, $default) = @_;
1572
1573     my @values = @{getargs($argname, $default)};
1574     return [( map { ($_ eq "0") ? 0 : 1 } @values )];
1575 }
1576
1577 # Return an integer if set by "$argname=value" on the
1578 # command line. Return $default if not set.
1579 sub getints {
1580     my ($argname, $default) = @_;
1581
1582     my @values = @{getargs($argname, $default)};
1583     return [( map { int($_) } @values )];
1584 }
1585
1586 sub getarg {
1587     my ($argname, $default) = @_;
1588     my @values = @{getargs($argname, $default)};
1589     die "Only one value allowed for $argname\n"  if @values > 1;
1590     return $values[0];
1591 }
1592
1593 sub getbool {
1594     my ($argname, $default) = @_;
1595     my @values = @{getbools($argname, $default)};
1596     die "Only one value allowed for $argname\n"  if @values > 1;
1597     return $values[0];
1598 }
1599
1600 sub getint {
1601     my ($argname, $default) = @_;
1602     my @values = @{getints($argname, $default)};
1603     die "Only one value allowed for $argname\n"  if @values > 1;
1604     return $values[0];
1605 }
1606
1607
1608 my $default_arch = "x86_64";
1609 $args{ARCH} = getargs("ARCH", 0);
1610 $args{ARCH} = getargs("ARCHS", $default_arch)  if !@{$args{ARCH}}[0];
1611
1612 $args{OSVERSION} = getargs("OS", "macosx-default-default");
1613
1614 $args{MEM} = getargs("MEM", "mrc");
1615 $args{LANGUAGE} = [ map { lc($_) } @{getargs("LANGUAGE", "objective-c")} ];
1616
1617 $args{CC} = getargs("CC", "clang");
1618
1619 {
1620     my $guardmalloc = getargs("GUARDMALLOC", 0);
1621     # GUARDMALLOC=1 is the same as GUARDMALLOC=before,after
1622     my @guardmalloc2 = ();
1623     for my $arg (@$guardmalloc) {
1624         if ($arg == 1) { push @guardmalloc2, "before";
1625                          push @guardmalloc2, "after"; }
1626         else { push @guardmalloc2, $arg }
1627     }
1628     $args{GUARDMALLOC} = \@guardmalloc2;
1629 }
1630
1631 $BUILD = getbool("BUILD", 1);
1632 $RUN = getbool("RUN", 1);
1633 $VERBOSE = getint("VERBOSE", 0);
1634 $BATS = getbool("BATS", 0);
1635 $BUILDDIR = getarg("BUILDDIR", $BATS ? $BATSBASE : $LOCALBASE);
1636
1637 my $root = getarg("ROOT", "");
1638 $root =~ s#/*$##;
1639
1640 my @tests = gettests();
1641
1642 if ($BUILD) {
1643     rm_rf_verbose "$DSTROOT$BUILDDIR";
1644     rm_rf_verbose "$OBJROOT$BUILDDIR";
1645 }
1646
1647 print "note: -----\n";
1648 print "note: testing root '$root'\n";
1649
1650 my @configs = make_configs($root, %args);
1651
1652 print "note: -----\n";
1653 print "note: testing ", scalar(@configs), " configurations:\n";
1654 for my $configref (@configs) {
1655     my $configname = $$configref{NAME};
1656     print "note: configuration $configname\n";
1657 }
1658
1659 my $failed = 0;
1660
1661 my $testconfigs = @configs;
1662 my $failconfigs = 0;
1663 my $skipconfigs = 0;
1664 my $testcount = 0;
1665 my $failcount = 0;
1666 for my $configref (@configs) {
1667     my $configname = $$configref{NAME};
1668     print "note: -----\n";
1669     print "note: \nnote: $configname\nnote: \n";
1670
1671     (my $t, my $f, my $skipconfig) =
1672         eval { build_and_run_one_config($configref, @tests); };
1673     $skipconfigs += $skipconfig;
1674     if ($@) {
1675         chomp $@;
1676         colorprint $red, "FAIL: $configname";
1677         colorprint $red, "FAIL: $@";
1678         $failconfigs++;
1679     } else {
1680         my $color = ($f ? $red : "");
1681         print "note:\n";
1682         colorprint $color, "note: $configname\n";
1683         colorprint $color, "note: $t tests, $f failures";
1684         $testcount += $t;
1685         $failcount += $f;
1686         $failconfigs++ if ($f);
1687     }
1688 }
1689
1690 print "note: -----\n";
1691 my $color = ($failconfigs ? $red : "");
1692 colorprint $color, "note: $testconfigs configurations, " .
1693     "$failconfigs with failures, $skipconfigs skipped";
1694 colorprint $color, "note: $testcount tests, $failcount failures";
1695
1696 $failed = ($failconfigs ? 1 : 0);
1697
1698
1699 if ($BUILD && $BATS && !$failed) {
1700     # Collect BATS execution instructions for all tests.
1701     # Each BATS "test" is all configurations together of one of our tests.
1702     for my $testname (@tests) {
1703         append_bats_test($testname);
1704     }
1705
1706     # Write the BATS plist to disk.
1707     my $json = encode_json(\%bats_plist);
1708     my $filename = "$DSTROOT$BATSBASE/objc4.plist";
1709     print "note: writing BATS config to $filename\n";
1710     open(my $file, '>', $filename);
1711     print $file $json;
1712     close $file;
1713 }
1714
1715 exit ($failed ? 1 : 0);