test/test.pl

   1 #!/usr/bin/perl
   2
   3 # test.pl
   4 # Run unit tests.
   5
   6 use strict;
   7 use File::Basename;
   8
   9 # We use encode_json() to write BATS plist files.
  10 # JSON::PP does not exist on iOS devices, but we need not write plists there.
  11 # So we simply load JSON:PP if it exists.
  12 if (eval { require JSON::PP; 1; }) {
  13     JSON::PP->import();
  14 }
  15
  16
  17 chdir dirname $0;
  18 chomp (my $DIR = `pwd`);
  19
  20 if (scalar(@ARGV) == 1) {
  21     my $arg = $ARGV[0];
  22     if ($arg eq "-h" || $arg eq "-H" || $arg eq "-help" || $arg eq "help") {
  23         print(<<END);
  24 usage: $0 [options] [testname ...]
  25        $0 help
  26
  27 testname:
  28     `testname` runs a specific test. If no testnames are given, runs all tests.
  29
  30 options:
  31     ARCH=<arch>
  32     OS=<sdk name>[sdk version][-<deployment target>[-<run target>]]
  33     ROOT=/path/to/project.roots/
  34
  35     CC=<compiler name>
  36
  37     LANGUAGE=c,c++,objective-c,objective-c++,swift
  38     MEM=mrc,arc
  39     GUARDMALLOC=0|1|before|after
  40
  41     BUILD=0|1      (build the tests?)
  42     RUN=0|1        (run the tests?)
  43     VERBOSE=0|1|2  (0=quieter  1=print commands executed  2=full test output)
  44     BATS=0|1       (build for and/or run in BATS?)
  45     BUILD_SHARED_CACHE=0|1  (build a dyld shared cache with the root and test against that)
  46     DYLD=2|3       (test in dyld 2 or dyld 3 mode)
  47
  48 examples:
  49
  50     test installed library, x86_64
  51     $0
  52
  53     test buildit-built root, i386 and x86_64, MRC and ARC, clang compiler
  54     $0 ARCH=i386,x86_64 ROOT=/tmp/objc4.roots MEM=mrc,arc CC=clang
  55
  56     test buildit-built root with iOS simulator, deploy to iOS 7, run on iOS 8
  57     $0 ARCH=x86_64 ROOT=/tmp/objc4.roots OS=iphonesimulator-7.0-8.0
  58
  59     test buildit-built root on attached iOS device
  60     $0 ARCH=arm64 ROOT=/tmp/objc4.roots OS=iphoneos
  61 END
  62         exit 0;
  63     }
  64 }
  65
  66 #########################################################################
  67 ## Tests
  68
  69 # Maps test name => test's filename extension.
  70 # ex: "msgSend" => "m"
  71 # `keys %ALL_TESTS` is also used as the list of all tests found on disk.
  72 my %ALL_TESTS;
  73
  74 #########################################################################
  75 ## Variables for use in complex build and run rules
  76
  77 # variable         # example value
  78
  79 # things you can multiplex on the command line
  80 # ARCH=i386,x86_64,armv6,armv7
  81 # OS=macosx,iphoneos,iphonesimulator (plus sdk/deployment/run versions)
  82 # LANGUAGE=c,c++,objective-c,objective-c++,swift
  83 # CC=clang
  84 # MEM=mrc,arc
  85 # GUARDMALLOC=0,1,before,after
  86
  87 # things you can set once on the command line
  88 # ROOT=/path/to/project.roots
  89 # BUILD=0|1
  90 # RUN=0|1
  91 # VERBOSE=0|1|2
  92 # BATS=0|1
  93
  94 # environment variables from the command line
  95 # DSTROOT
  96 # OBJROOT
  97 # (SRCROOT is ignored; test sources are assumed to
  98 # be in the same directory as the test script itself.)
  99 # fixme SYMROOT for dsymutil output?
 100
 101
 102 # Some arguments as read from the command line.
 103 my %args;
 104 my $BUILD;
 105 my $RUN;
 106 my $VERBOSE;
 107 my $BATS;
 108
 109 my $HOST;
 110 my $PORT;
 111
 112 my @TESTLIBNAMES = ("libobjc.A.dylib", "libobjc-trampolines.dylib");
 113 my $TESTLIBDIR = "/usr/lib";
 114
 115 # Top level directory for intermediate and final build products.
 116 # Intermediate files must be kept separate for XBS BATS builds.
 117 my $OBJROOT = $ENV{OBJROOT} || "";
 118 my $DSTROOT = $ENV{DSTROOT} || "";
 119
 120 # Build product directory inside DSTROOT and OBJROOT.
 121 # Each test config gets its own build directory inside this.
 122 my $BUILDDIR;
 123
 124 # Local top-level directory.
 125 # This is the default value for $BUILDDIR.
 126 my $LOCALBASE = "/tmp/test-$TESTLIBNAMES[0]-build";
 127
 128 # Device-side top-level directory.
 129 # This replaces $DSTROOT$BUILDDIR/ for on-device execution.
 130 my $REMOTEBASE = "/AppleInternal/objctest";
 131
 132 # BATS top-level directory.
 133 # This replaces $DSTROOT$BUILDDIR/ for BATS execution.
 134 my $BATSBASE = "/AppleInternal/CoreOS/tests/objc4";
 135
 136
 137 my $crashcatch = <<'END';
 138 // interpose-able code to catch crashes, print, and exit cleanly
 139 #include <signal.h>
 140 #include <string.h>
 141 #include <unistd.h>
 142
 143 // from dyld-interposing.h
 144 #define DYLD_INTERPOSE(_replacement,_replacee) __attribute__((used)) static struct{ const void* replacement; const void* replacee; } _interpose_##_replacee __attribute__ ((section ("__DATA,__interpose"))) = { (const void*)(unsigned long)&_replacement, (const void*)(unsigned long)&_replacee };
 145
 146 static void catchcrash(int sig)
 147 {
 148     const char *msg;
 149     switch (sig) {
 150     case SIGILL:  msg = "CRASHED: SIGILL";  break;
 151     case SIGBUS:  msg = "CRASHED: SIGBUS";  break;
 152     case SIGSYS:  msg = "CRASHED: SIGSYS";  break;
 153     case SIGSEGV: msg = "CRASHED: SIGSEGV"; break;
 154     case SIGTRAP: msg = "CRASHED: SIGTRAP"; break;
 155     case SIGABRT: msg = "CRASHED: SIGABRT"; break;
 156     default: msg = "unknown signal"; break;
 157     }
 158     write(STDERR_FILENO, msg, strlen(msg));
 159
 160     // avoid backslash-n newline due to escaping differences somewhere
 161     // in BATS versus local execution (perhaps different perl versions?)
 162     char newline = 0xa;
 163     write(STDERR_FILENO, &newline, 1);
 164
 165     _exit(1);
 166 }
 167
 168 static void setupcrash(void) __attribute__((constructor));
 169 static void setupcrash(void)
 170 {
 171     signal(SIGILL, &catchcrash);
 172     signal(SIGBUS, &catchcrash);
 173     signal(SIGSYS, &catchcrash);
 174     signal(SIGSEGV, &catchcrash);
 175     signal(SIGTRAP, &catchcrash);
 176     signal(SIGABRT, &catchcrash);
 177 }
 178
 179
 180 static int hacked = 0;
 181 ssize_t hacked_write(int fildes, const void *buf, size_t nbyte)
 182 {
 183     if (!hacked) {
 184         setupcrash();
 185         hacked = 1;
 186     }
 187     return write(fildes, buf, nbyte);
 188 }
 189
 190 DYLD_INTERPOSE(hacked_write, write);
 191
 192 END
 193
 194
 195 #########################################################################
 196 ## Harness
 197
 198
 199 # map language to buildable extensions for that language
 200 my %extensions_for_language = (
 201     "c"     => ["c"],
 202     "objective-c" => ["c", "m"],
 203     "c++" => ["c", "cc", "cp", "cpp", "cxx", "c++"],
 204     "objective-c++" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"],
 205     "swift" => ["swift"],
 206
 207     "any" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm", "swift"],
 208     );
 209
 210 # map extension to languages
 211 my %languages_for_extension = (
 212     "c" => ["c", "objective-c", "c++", "objective-c++"],
 213     "m" => ["objective-c", "objective-c++"],
 214     "mm" => ["objective-c++"],
 215     "cc" => ["c++", "objective-c++"],
 216     "cp" => ["c++", "objective-c++"],
 217     "cpp" => ["c++", "objective-c++"],
 218     "cxx" => ["c++", "objective-c++"],
 219     "c++" => ["c++", "objective-c++"],
 220     "swift" => ["swift"],
 221     );
 222
 223 # Run some newline-separated commands like `make` would, stopping if any fail
 224 # run("cmd1 \n cmd2 \n cmd3")
 225 sub make {
 226     my $output = "";
 227     my @cmds = split("\n", $_[0]);
 228     die if scalar(@cmds) == 0;
 229     $? = 0;
 230     foreach my $cmd (@cmds) {
 231         chomp $cmd;
 232         next if $cmd =~ /^\s*$/;
 233         $cmd .= " 2>&1";
 234         print "$cmd\n" if $VERBOSE;
 235         eval {
 236             local $SIG{ALRM} = sub { die "alarm\n" };
 237             # Timeout after 600 seconds so a deadlocked test doesn't wedge the
 238             # entire test suite. Increase to an hour for B&I builds.
 239             if (exists $ENV{"RC_XBS"}) {
 240                 alarm 3600;
 241             } else {
 242                 alarm 600;
 243             }
 244             $output .= `$cmd`;
 245             alarm 0;
 246         };
 247         if ($@) {
 248             die unless $@ eq "alarm\n";
 249             $output .= "\nTIMED OUT";
 250         }
 251         last if $?;
 252     }
 253     print "$output\n" if $VERBOSE;
 254     return $output;
 255 }
 256
 257 sub chdir_verbose {
 258     my $dir = shift || die;
 259     print "cd $dir\n" if $VERBOSE;
 260     chdir $dir || die "couldn't cd $dir";
 261 }
 262
 263 sub rm_rf_verbose {
 264     my $dir = shift || die;
 265     print "mkdir -p $dir\n" if $VERBOSE;
 266     `rm -rf '$dir'`;
 267     die "couldn't rm -rf $dir" if $?;
 268 }
 269
 270 sub mkdir_verbose {
 271     my $dir = shift || die;
 272     print "mkdir -p $dir\n" if $VERBOSE;
 273     `mkdir -p '$dir'`;
 274     die "couldn't mkdir $dir" if $?;
 275 }
 276
 277
 278 # xterm colors
 279 my $red = "\e[41;37m";
 280 my $yellow = "\e[43;30m";
 281 my $nocolor = "\e[0m";
 282 if (! -t STDIN) {
 283     # Not isatty. Don't use colors.
 284     $red = "";
 285     $yellow = "";
 286     $nocolor = "";
 287 }
 288
 289 # print text with a colored prefix on each line
 290 # fixme some callers pass an array of lines and some don't
 291 sub colorprefix {
 292     my $color = shift;
 293     while (defined(my $lines = shift)) {
 294         $lines = "\n" if ($lines eq "");
 295         for my $line (split(/^/, $lines)) {
 296             chomp $line;
 297             print "$color $nocolor$line\n";
 298         }
 299     }
 300 }
 301
 302 # print text colored
 303 # fixme some callers pass an array of lines and some don't
 304 sub colorprint {
 305     my $color = shift;
 306     while (defined(my $lines = shift)) {
 307         $lines = "\n" if ($lines eq "");
 308         for my $line (split(/^/, $lines)) {
 309             chomp $line;
 310             print "$color$line$nocolor\n";
 311         }
 312     }
 313 }
 314
 315 # Return test names from the command line.
 316 # Returns all tests if no tests were named.
 317 sub gettests {
 318     my @tests;
 319
 320     foreach my $arg (@ARGV) {
 321         push @tests, $arg  if ($arg !~ /=/  &&  $arg !~ /^-/);
 322     }
 323
 324     opendir(my $dir, $DIR) || die;
 325     while (my $file = readdir($dir)) {
 326         my ($name, $ext) = ($file =~ /^([^.]+)\.([^.]+)$/);
 327         next if ! $languages_for_extension{$ext};
 328
 329         open(my $in, "< $file") || die "$file";
 330         my $contents = join "", <$in>;
 331         if (defined $ALL_TESTS{$name}) {
 332             colorprint $yellow, "SKIP: multiple tests named '$name'; skipping file '$file'.";
 333         } else {
 334             $ALL_TESTS{$name} = $ext  if ($contents =~ m#^[/*\s]*TEST_#m);
 335         }
 336         close($in);
 337     }
 338     closedir($dir);
 339
 340     if (scalar(@tests) == 0) {
 341         @tests = keys %ALL_TESTS;
 342     }
 343
 344     @tests = sort @tests;
 345
 346     return @tests;
 347 }
 348
 349
 350 # Turn a C compiler name into a C++ compiler name.
 351 sub cplusplus {
 352     my ($c) = @_;
 353     if ($c =~ /cc/) {
 354         $c =~ s/cc/\+\+/;
 355         return $c;
 356     }
 357     return $c . "++";                         # e.g. clang => clang++
 358 }
 359
 360 # Turn a C compiler name into a Swift compiler name
 361 sub swift {
 362     my ($c) = @_;
 363     $c =~ s#[^/]*$#swift#;
 364     return $c;
 365 }
 366
 367 # Returns an array of all sdks from `xcodebuild -showsdks`
 368 my @sdks_memo;
 369 sub getsdks {
 370     if (!@sdks_memo) {
 371         @sdks_memo = (`xcodebuild -showsdks` =~ /-sdk (.+)$/mg);
 372     }
 373     return @sdks_memo;
 374 }
 375
 376 my %sdk_path_memo = {};
 377 sub getsdkpath {
 378     my ($sdk) = @_;
 379     if (!defined $sdk_path_memo{$sdk}) {
 380         ($sdk_path_memo{$sdk}) = (`xcodebuild -version -sdk '$sdk' Path` =~ /^\s*(.+?)\s*$/);
 381     }
 382     return $sdk_path_memo{$sdk};
 383 }
 384
 385 # Extract a version number from a string.
 386 # Ignore trailing "internal".
 387 sub versionsuffix {
 388     my ($str) = @_;
 389     my ($vers) = ($str =~ /([0-9]+\.[0-9]+)(?:\.?internal)?$/);
 390     return $vers;
 391 }
 392 sub majorversionsuffix {
 393     my ($str) = @_;
 394     my ($vers) = ($str =~ /([0-9]+)\.[0-9]+(?:\.?internal)?$/);
 395     return $vers;
 396 }
 397 sub minorversionsuffix {
 398     my ($str) = @_;
 399     my ($vers) = ($str =~ /[0-9]+\.([0-9]+)(?:\.?internal)?$/);
 400     return $vers;
 401 }
 402
 403 # Compares two SDK names and returns the newer one.
 404 # Assumes the two SDKs are the same OS.
 405 sub newersdk {
 406     my ($lhs, $rhs) = @_;
 407
 408     # Major version wins.
 409     my $lhsMajor = majorversionsuffix($lhs);
 410     my $rhsMajor = majorversionsuffix($rhs);
 411     if ($lhsMajor > $rhsMajor) { return $lhs; }
 412     if ($lhsMajor < $rhsMajor) { return $rhs; }
 413
 414     # Minor version wins.
 415     my $lhsMinor = minorversionsuffix($lhs);
 416     my $rhsMinor = minorversionsuffix($rhs);
 417     if ($lhsMinor > $rhsMinor) { return $lhs; }
 418     if ($lhsMinor < $rhsMinor) { return $rhs; }
 419
 420     # Lexically-last wins (i.e. internal is better than not internal)
 421     if ($lhs gt $rhs) { return $lhs; }
 422     return $rhs;
 423 }
 424
 425 sub rewind {
 426     seek($_[0], 0, 0);
 427 }
 428
 429 # parse name=value,value pairs
 430 sub readconditions {
 431     my ($conditionstring) = @_;
 432
 433     my %results;
 434     my @conditions = ($conditionstring =~ /\w+=(?:[^\s,]+,?)+/g);
 435     for my $condition (@conditions) {
 436         my ($name, $values) = ($condition =~ /(\w+)=(.+)/);
 437         $results{$name} = [split ',', $values];
 438     }
 439
 440     return %results;
 441 }
 442
 443 sub check_output {
 444     my %C = %{shift()};
 445     my $name = shift;
 446     my @output = @_;
 447
 448     my %T = %{$C{"TEST_$name"}};
 449
 450     # Quietly strip MallocScribble before saving the "original" output
 451     # because it is distracting.
 452     filter_malloc(\@output);
 453
 454     my @original_output = @output;
 455
 456     # Run result-checking passes, reducing @output each time
 457     my $xit = 1;
 458     my $bad = "";
 459     my $warn = "";
 460     my $runerror = $T{TEST_RUN_OUTPUT};
 461     filter_hax(\@output);
 462     filter_verbose(\@output);
 463     filter_simulator(\@output);
 464     $warn = filter_warn(\@output);
 465     $bad |= filter_guardmalloc(\@output) if ($C{GUARDMALLOC});
 466     $bad |= filter_valgrind(\@output) if ($C{VALGRIND});
 467     $bad = filter_expected(\@output, \%C, $name) if ($bad eq "");
 468     $bad = filter_bad(\@output)  if ($bad eq "");
 469
 470     # OK line should be the only one left
 471     $bad = "(output not 'OK: $name')" if ($bad eq ""  &&  (scalar(@output) != 1  ||  $output[0] !~ /^OK: $name/));
 472
 473     if ($bad ne "") {
 474         colorprint  $red, "FAIL: /// test '$name' \\\\\\";
 475         colorprefix $red, @original_output;
 476         colorprint  $red, "FAIL: \\\\\\ test '$name' ///";
 477         colorprint  $red, "FAIL: $name: $bad";
 478         $xit = 0;
 479     }
 480     elsif ($warn ne "") {
 481         colorprint  $yellow, "PASS: /// test '$name' \\\\\\";
 482         colorprefix $yellow, @original_output;
 483         colorprint  $yellow, "PASS: \\\\\\ test '$name' ///";
 484         print "PASS: $name (with warnings)\n";
 485     }
 486     else {
 487         print "PASS: $name\n";
 488     }
 489     return $xit;
 490 }
 491
 492 sub filter_expected
 493 {
 494     my $outputref = shift;
 495     my %C = %{shift()};
 496     my $name = shift;
 497
 498     my %T = %{$C{"TEST_$name"}};
 499     my $runerror = $T{TEST_RUN_OUTPUT}  ||  return "";
 500
 501     my $bad = "";
 502
 503     my $output = join("\n", @$outputref) . "\n";
 504     if ($output !~ /$runerror/) {
 505         $bad = "(run output does not match TEST_RUN_OUTPUT)";
 506         @$outputref = ("FAIL: $name");
 507     } else {
 508         @$outputref = ("OK: $name");  # pacify later filter
 509     }
 510
 511     return $bad;
 512 }
 513
 514 sub filter_bad
 515 {
 516     my $outputref = shift;
 517     my $bad = "";
 518
 519     my @new_output;
 520     for my $line (@$outputref) {
 521         if ($line =~ /^BAD: (.*)/) {
 522             $bad = "(failed)";
 523         } else {
 524             push @new_output, $line;
 525         }
 526     }
 527
 528     @$outputref = @new_output;
 529     return $bad;
 530 }
 531
 532 sub filter_warn
 533 {
 534     my $outputref = shift;
 535     my $warn = "";
 536
 537     my @new_output;
 538     for my $line (@$outputref) {
 539         if ($line !~ /^WARN: (.*)/) {
 540             push @new_output, $line;
 541         } else {
 542             $warn = "(warned)";
 543         }
 544     }
 545
 546     @$outputref = @new_output;
 547     return $warn;
 548 }
 549
 550 sub filter_verbose
 551 {
 552     my $outputref = shift;
 553
 554     my @new_output;
 555     for my $line (@$outputref) {
 556         if ($line !~ /^VERBOSE: (.*)/) {
 557             push @new_output, $line;
 558         }
 559     }
 560
 561     @$outputref = @new_output;
 562 }
 563
 564 sub filter_simulator
 565 {
 566     my $outputref = shift;
 567
 568     my @new_output;
 569     for my $line (@$outputref) {
 570         if (($line !~ /No simulator devices appear to be running/)  &&
 571             ($line !~ /CoreSimulator is attempting to unload a stale CoreSimulatorService job/)  &&
 572             ($line !~ /Failed to locate a valid instance of CoreSimulatorService/))
 573         {
 574             push @new_output, $line;
 575         }
 576     }
 577
 578     @$outputref = @new_output;
 579 }
 580
 581 sub filter_hax
 582 {
 583     my $outputref = shift;
 584
 585     my @new_output;
 586     for my $line (@$outputref) {
 587         if ($line !~ /Class OS_tcp_/) {
 588             push @new_output, $line;
 589         }
 590     }
 591
 592     @$outputref = @new_output;
 593 }
 594
 595 sub filter_valgrind
 596 {
 597     my $outputref = shift;
 598     my $errors = 0;
 599     my $leaks = 0;
 600
 601     my @new_output;
 602     for my $line (@$outputref) {
 603         if ($line =~ /^Approx: do_origins_Dirty\([RW]\): missed \d bytes$/) {
 604             # --track-origins warning (harmless)
 605             next;
 606         }
 607         if ($line =~ /^UNKNOWN __disable_threadsignal is unsupported. This warning will not be repeated.$/) {
 608             # signals unsupported (harmless)
 609             next;
 610         }
 611         if ($line =~ /^UNKNOWN __pthread_sigmask is unsupported. This warning will not be repeated.$/) {
 612             # signals unsupported (harmless)
 613             next;
 614         }
 615         if ($line !~ /^^\.*==\d+==/) {
 616             # not valgrind output
 617             push @new_output, $line;
 618             next;
 619         }
 620
 621         my ($errcount) = ($line =~ /==\d+== ERROR SUMMARY: (\d+) errors/);
 622         if (defined $errcount  &&  $errcount > 0) {
 623             $errors = 1;
 624         }
 625
 626         (my $leakcount) = ($line =~ /==\d+==\s+(?:definitely|possibly) lost:\s+([0-9,]+)/);
 627         if (defined $leakcount  &&  $leakcount > 0) {
 628             $leaks = 1;
 629         }
 630     }
 631
 632     @$outputref = @new_output;
 633
 634     my $bad = "";
 635     $bad .= "(valgrind errors)" if ($errors);
 636     $bad .= "(valgrind leaks)" if ($leaks);
 637     return $bad;
 638 }
 639
 640
 641
 642 sub filter_malloc
 643 {
 644     my $outputref = shift;
 645     my $errors = 0;
 646
 647     my @new_output;
 648     my $count = 0;
 649     for my $line (@$outputref) {
 650         # Ignore MallocScribble prologue.
 651         # Ignore MallocStackLogging prologue.
 652         if ($line =~ /malloc: enabling scribbling to detect mods to free/  ||
 653             $line =~ /Deleted objects will be dirtied by the collector/  ||
 654             $line =~ /malloc: stack logs being written into/  ||
 655             $line =~ /malloc: stack logs deleted from/  ||
 656             $line =~ /malloc: process \d+ no longer exists/  ||
 657             $line =~ /malloc: recording malloc and VM allocation stacks/)
 658         {
 659             next;
 660         }
 661
 662         # not malloc output
 663         push @new_output, $line;
 664
 665     }
 666
 667     @$outputref = @new_output;
 668 }
 669
 670 sub filter_guardmalloc
 671 {
 672     my $outputref = shift;
 673     my $errors = 0;
 674
 675     my @new_output;
 676     my $count = 0;
 677     for my $line (@$outputref) {
 678         if ($line !~ /^GuardMalloc\[[^\]]+\]: /) {
 679             # not guardmalloc output
 680             push @new_output, $line;
 681             next;
 682         }
 683
 684         # Ignore 4 lines of guardmalloc prologue.
 685         # Anything further is a guardmalloc error.
 686         if (++$count > 4) {
 687             $errors = 1;
 688         }
 689     }
 690
 691     @$outputref = @new_output;
 692
 693     my $bad = "";
 694     $bad .= "(guardmalloc errors)" if ($errors);
 695     return $bad;
 696 }
 697
 698 # TEST_SOMETHING
 699 # text
 700 # text
 701 # END
 702 sub extract_multiline {
 703     my ($flag, $contents, $name) = @_;
 704     if ($contents =~ /$flag\n/) {
 705         my ($output) = ($contents =~ /$flag\n(.*?\n)END[ *\/]*\n/s);
 706         die "$name used $flag without END\n"  if !defined($output);
 707         return $output;
 708     }
 709     return undef;
 710 }
 711
 712
 713 # TEST_SOMETHING
 714 # text
 715 # OR
 716 # text
 717 # END
 718 sub extract_multiple_multiline {
 719     my ($flag, $contents, $name) = @_;
 720     if ($contents =~ /$flag\n/) {
 721         my ($output) = ($contents =~ /$flag\n(.*?\n)END[ *\/]*\n/s);
 722         die "$name used $flag without END\n"  if !defined($output);
 723
 724         $output =~ s/\nOR\n/\n|/sg;
 725         $output = "^(" . $output . ")\$";
 726         return $output;
 727     }
 728     return undef;
 729 }
 730
 731
 732 sub gather_simple {
 733     my $CREF = shift;
 734     my %C = %{$CREF};
 735     my $name = shift;
 736     chdir_verbose $DIR;
 737
 738     my $ext = $ALL_TESTS{$name};
 739     my $file = "$name.$ext";
 740     return 0 if !$file;
 741
 742     # search file for 'TEST_CONFIG' or '#include "test.h"'
 743     # also collect other values:
 744     # TEST_DISABLED disable test with an optional message
 745     # TEST_CRASHES test is expected to crash
 746     # TEST_CONFIG test conditions
 747     # TEST_ENV environment prefix
 748     # TEST_CFLAGS compile flags
 749     # TEST_BUILD build instructions
 750     # TEST_BUILD_OUTPUT expected build stdout/stderr
 751     # TEST_RUN_OUTPUT expected run stdout/stderr
 752     open(my $in, "< $file") || die;
 753     my $contents = join "", <$in>;
 754
 755     my $test_h = ($contents =~ /^\s*#\s*(include|import)\s*"test\.h"/m);
 756     my ($disabled) = ($contents =~ /\b(TEST_DISABLED\b.*)$/m);
 757     my $crashes = ($contents =~ /\bTEST_CRASHES\b/m);
 758     my ($conditionstring) = ($contents =~ /\bTEST_CONFIG\b(.*)$/m);
 759     my ($envstring) = ($contents =~ /\bTEST_ENV\b(.*)$/m);
 760     my ($cflags) = ($contents =~ /\bTEST_CFLAGS\b(.*)$/m);
 761     my ($buildcmd) = extract_multiline("TEST_BUILD", $contents, $name);
 762     my ($builderror) = extract_multiple_multiline("TEST_BUILD_OUTPUT", $contents, $name);
 763     my ($runerror) = extract_multiple_multiline("TEST_RUN_OUTPUT", $contents, $name);
 764
 765     return 0 if !$test_h && !$disabled && !$crashes && !defined($conditionstring) && !defined($envstring) && !defined($cflags) && !defined($buildcmd) && !defined($builderror) && !defined($runerror);
 766
 767     if ($disabled) {
 768         colorprint $yellow, "SKIP: $name    (disabled by $disabled)";
 769         return 0;
 770     }
 771
 772     # check test conditions
 773
 774     my $run = 1;
 775     my %conditions = readconditions($conditionstring);
 776     if (! $conditions{LANGUAGE}) {
 777         # implicit language restriction from file extension
 778         $conditions{LANGUAGE} = $languages_for_extension{$ext};
 779     }
 780     for my $condkey (keys %conditions) {
 781         my @condvalues = @{$conditions{$condkey}};
 782
 783         # special case: RUN=0 does not affect build
 784         if ($condkey eq "RUN"  &&  @condvalues == 1  &&  $condvalues[0] == 0) {
 785             $run = 0;
 786             next;
 787         }
 788
 789         my $testvalue = $C{$condkey};
 790         next if !defined($testvalue);
 791         # testvalue is the configuration being run now
 792         # condvalues are the allowed values for this test
 793
 794         my $ok = 0;
 795         for my $condvalue (@condvalues) {
 796
 797             # special case: objc and objc++
 798             if ($condkey eq "LANGUAGE") {
 799                 $condvalue = "objective-c" if $condvalue eq "objc";
 800                 $condvalue = "objective-c++" if $condvalue eq "objc++";
 801             }
 802
 803             $ok = 1  if ($testvalue eq $condvalue);
 804
 805             # special case: CC and CXX allow substring matches
 806             if ($condkey eq "CC"  ||  $condkey eq "CXX") {
 807                 $ok = 1  if ($testvalue =~ /$condvalue/);
 808             }
 809
 810             last if $ok;
 811         }
 812
 813         if (!$ok) {
 814             my $plural = (@condvalues > 1) ? "one of: " : "";
 815             print "SKIP: $name    ($condkey=$testvalue, but test requires $plural", join(' ', @condvalues), ")\n";
 816             return 0;
 817         }
 818     }
 819
 820     # save some results for build and run phases
 821     $$CREF{"TEST_$name"} = {
 822         TEST_BUILD => $buildcmd,
 823         TEST_BUILD_OUTPUT => $builderror,
 824         TEST_CRASHES => $crashes,
 825         TEST_RUN_OUTPUT => $runerror,
 826         TEST_CFLAGS => $cflags,
 827         TEST_ENV => $envstring,
 828         TEST_RUN => $run,
 829         DSTDIR => "$C{DSTDIR}/$name.build",
 830         OBJDIR => "$C{OBJDIR}/$name.build",
 831     };
 832
 833     return 1;
 834 }
 835
 836
 837 # Test description plist to write when building for BATS execution.
 838 my %bats_plist;
 839 $bats_plist{'Project'} = "objc4";
 840 $bats_plist{'Tests'} = [];  # populated by append_bats_test()
 841
 842 # Saves run instructions for a single test in all configurations as a BATS test.
 843 sub append_bats_test {
 844     my $name = shift;
 845
 846     my $arch = join(',', @{$args{ARCH}});
 847     my $os = join(',', @{$args{OSVERSION}});
 848     my $mem = join(',', @{$args{MEM}});
 849     my $language = join(',', @{$args{LANGUAGE}});
 850
 851     push @{$bats_plist{'Tests'}}, {
 852         "TestName" => "$name",
 853         "Command" => [
 854             "/usr/bin/perl",
 855             "$BATSBASE/test/test.pl",
 856             $name,
 857             "ARCH=$arch",
 858             "OS=$os",
 859             "MEM=$mem",
 860             "LANGUAGE=$language",
 861             "BUILD=0",
 862             "RUN=1",
 863             "VERBOSE=1",
 864             "BATS=1",
 865         ]
 866     };
 867 }
 868
 869
 870 # Builds a simple test
 871 sub build_simple {
 872     my %C = %{shift()};
 873     my $name = shift;
 874     my %T = %{$C{"TEST_$name"}};
 875
 876     mkdir_verbose $T{DSTDIR};
 877     chdir_verbose $T{DSTDIR};
 878     # we don't mkdir $T{OBJDIR} because most tests don't use it
 879
 880     my $ext = $ALL_TESTS{$name};
 881     my $file = "$DIR/$name.$ext";
 882
 883     if ($T{TEST_CRASHES}) {
 884         `echo '$crashcatch' > crashcatch.c`;
 885         make("$C{COMPILE_C} -dynamiclib -o libcrashcatch.dylib -x c crashcatch.c");
 886         die "$?" if $?;
 887     }
 888
 889     my $cmd = $T{TEST_BUILD} ? eval "return \"$T{TEST_BUILD}\"" : "$C{COMPILE}   $T{TEST_CFLAGS} $file -o $name.exe";
 890
 891     my $output = make($cmd);
 892
 893     # ignore out-of-date text-based stubs (caused by ditto into SDK)
 894     $output =~ s/ld: warning: text-based stub file.*\n//g;
 895     # rdar://10163155
 896     $output =~ s/ld: warning: could not create compact unwind for [^\n]+: does not use standard frame\n//g;
 897     # rdar://37937122
 898     $output =~ s/^warning: Cannot lower [^\n]+\n//g;
 899     $output =~ s/^warning:     key: [^\n]+\n//g;
 900     $output =~ s/^warning:     discriminator: [^\n]+\n//g;
 901     $output =~ s/^warning:     callee: [^\n]+\n//g;
 902     # rdar://38710948
 903     $output =~ s/ld: warning: ignoring file [^\n]*libclang_rt\.bridgeos\.a[^\n]*\n//g;
 904     # ignore compiler logging of CCC_OVERRIDE_OPTIONS effects
 905     if (defined $ENV{CCC_OVERRIDE_OPTIONS}) {
 906         $output =~ s/### (CCC_OVERRIDE_OPTIONS:|Adding argument|Deleting argument|Replacing) [^\n]*\n//g;
 907     }
 908
 909     my $ok;
 910     if (my $builderror = $T{TEST_BUILD_OUTPUT}) {
 911         # check for expected output and ignore $?
 912         if ($output =~ /$builderror/) {
 913             $ok = 1;
 914         } elsif (defined $ENV{CCC_OVERRIDE_OPTIONS} && $builderror =~ /warning:/) {
 915             # CCC_OVERRIDE_OPTIONS manipulates compiler diagnostics.
 916             # Don't enforce any TEST_BUILD_OUTPUT that looks for warnings.
 917             colorprint  $yellow, "WARN: /// test '$name' \\\\\\";
 918             colorprefix $yellow, $output;
 919             colorprint  $yellow, "WARN: \\\\\\ test '$name' ///";
 920             colorprint  $yellow, "WARN: $name (build output does not match TEST_BUILD_OUTPUT; not fatal because CCC_OVERRIDE_OPTIONS is set)";
 921             $ok = 1;
 922         } else {
 923             colorprint  $red, "FAIL: /// test '$name' \\\\\\";
 924             colorprefix $red, $output;
 925             colorprint  $red, "FAIL: \\\\\\ test '$name' ///";
 926             colorprint  $red, "FAIL: $name (build output does not match TEST_BUILD_OUTPUT)";
 927             $ok = 0;
 928         }
 929     } elsif ($?) {
 930         colorprint  $red, "FAIL: /// test '$name' \\\\\\";
 931         colorprefix $red, $output;
 932         colorprint  $red, "FAIL: \\\\\\ test '$name' ///";
 933         colorprint  $red, "FAIL: $name (build failed)";
 934         $ok = 0;
 935     } elsif ($output ne "") {
 936         colorprint  $red, "FAIL: /// test '$name' \\\\\\";
 937         colorprefix $red, $output;
 938         colorprint  $red, "FAIL: \\\\\\ test '$name' ///";
 939         colorprint  $red, "FAIL: $name (unexpected build output)";
 940         $ok = 0;
 941     } else {
 942         $ok = 1;
 943     }
 944
 945     if ($ok) {
 946         foreach my $file (glob("*.exe *.dylib *.bundle")) {
 947             if (!$BATS) {
 948                 # not for BATS to save space and build time
 949                 # fixme use SYMROOT?
 950                 make("xcrun dsymutil $file");
 951             }
 952             if ($C{OS} eq "macosx"  ||  $C{OS} =~ /simulator/) {
 953                 # setting any entitlements disables dyld environment variables
 954             } else {
 955                 # get-task-allow entitlement is required
 956                 # to enable dyld environment variables
 957                 make("xcrun codesign -s - --entitlements $DIR/get_task_allow_entitlement.plist $file");
 958                 die "$?" if $?;
 959             }
 960         }
 961     }
 962
 963     return $ok;
 964 }
 965
 966 # Run a simple test (testname.exe, with error checking of stdout and stderr)
 967 sub run_simple {
 968     my %C = %{shift()};
 969     my $name = shift;
 970     my %T = %{$C{"TEST_$name"}};
 971
 972     if (! $T{TEST_RUN}) {
 973         print "PASS: $name (build only)\n";
 974         return 1;
 975     }
 976
 977     my $testdir = $T{DSTDIR};
 978     chdir_verbose $testdir;
 979
 980     my $env = "$C{ENV} $T{TEST_ENV}";
 981
 982     if ($T{TEST_CRASHES}) {
 983         $env .= " OBJC_DEBUG_DONT_CRASH=YES";
 984     }
 985
 986     if ($C{DYLD} eq "2") {
 987         $env .= " DYLD_USE_CLOSURES=0";
 988     }
 989     elsif ($C{DYLD} eq "3") {
 990         $env .= " DYLD_USE_CLOSURES=1";
 991     }
 992     else {
 993         die "unknown DYLD setting $C{DYLD}";
 994     }
 995
 996     my $output;
 997
 998     if ($C{ARCH} =~ /^arm/ && `uname -p` !~ /^arm/) {
 999         # run on iOS or watchos or tvos device
1000         # fixme device selection and verification
1001         my $remotedir = "$REMOTEBASE/" . basename($C{DSTDIR}) . "/$name.build";
1002
1003         # Add test dir and libobjc's dir to DYLD_LIBRARY_PATH.
1004         # Insert libcrashcatch.dylib if necessary.
1005         $env .= " DYLD_LIBRARY_PATH=$remotedir";
1006         $env .= ":$REMOTEBASE"  if ($C{TESTLIBDIR} ne $TESTLIBDIR);
1007         if ($T{TEST_CRASHES}) {
1008             $env .= " DYLD_INSERT_LIBRARIES=$remotedir/libcrashcatch.dylib";
1009         }
1010
1011         my $cmd = "ssh -p $PORT $HOST 'cd $remotedir && env $env ./$name.exe'";
1012         $output = make("$cmd");
1013     }
1014     elsif ($C{OS} =~ /simulator/) {
1015         # run locally in a simulator
1016         # fixme selection of simulated OS version
1017         my $simdevice;
1018         if ($C{OS} =~ /iphonesimulator/) {
1019             $simdevice = 'iPhone X';
1020         } elsif ($C{OS} =~ /watchsimulator/) {
1021             $simdevice = 'Apple Watch Series 4 - 40mm';
1022         } elsif ($C{OS} =~ /tvsimulator/) {
1023             $simdevice = 'Apple TV 1080p';
1024         } else {
1025             die "unknown simulator $C{OS}\n";
1026         }
1027         my $sim = "xcrun -sdk iphonesimulator simctl spawn '$simdevice'";
1028         # Add test dir and libobjc's dir to DYLD_LIBRARY_PATH.
1029         # Insert libcrashcatch.dylib if necessary.
1030         $env .= " DYLD_LIBRARY_PATH=$testdir";
1031         $env .= ":" . $C{TESTLIBDIR}  if ($C{TESTLIBDIR} ne $TESTLIBDIR);
1032         if ($T{TEST_CRASHES}) {
1033             $env .= " DYLD_INSERT_LIBRARIES=$testdir/libcrashcatch.dylib";
1034         }
1035
1036         my $simenv = "";
1037         foreach my $keyvalue (split(' ', $env)) {
1038             $simenv .= "SIMCTL_CHILD_$keyvalue ";
1039         }
1040         # Use the full path here so hack_cwd in test.h works.
1041         $output = make("env $simenv $sim $testdir/$name.exe");
1042     }
1043     else {
1044         # run locally
1045
1046         # Add test dir and libobjc's dir to DYLD_LIBRARY_PATH.
1047         # Insert libcrashcatch.dylib if necessary.
1048         $env .= " DYLD_LIBRARY_PATH=$testdir";
1049         $env .= ":" . $C{TESTLIBDIR}  if ($C{TESTLIBDIR} ne $TESTLIBDIR);
1050         if ($T{TEST_CRASHES}) {
1051             $env .= " DYLD_INSERT_LIBRARIES=$testdir/libcrashcatch.dylib";
1052         }
1053
1054         $output = make("sh -c '$env ./$name.exe'");
1055     }
1056
1057     return check_output(\%C, $name, split("\n", $output));
1058 }
1059
1060
1061 my %compiler_memo;
1062 sub find_compiler {
1063     my ($cc, $toolchain, $sdk_path) = @_;
1064
1065     # memoize
1066     my $key = $cc . ':' . $toolchain;
1067     my $result = $compiler_memo{$key};
1068     return $result if defined $result;
1069
1070     $result  = make("xcrun -toolchain $toolchain -find $cc 2>/dev/null");
1071
1072     chomp $result;
1073     $compiler_memo{$key} = $result;
1074     return $result;
1075 }
1076
1077 sub dirContainsAllTestLibs {
1078     my $dir = shift;
1079
1080     foreach my $testlib (@TESTLIBNAMES) {
1081         my $found = (-e "$dir/$testlib");
1082         my $foundstr = ($found ? "found" : "didn't find");
1083         print "note: $foundstr $testlib in $dir\n"  if ($VERBOSE);
1084         return 0  if (!$found);
1085     }
1086
1087     return 1;
1088 }
1089
1090 sub findIncludeDir {
1091     my ($root, $includePath) = @_;
1092
1093     foreach my $candidate ("$root/../SDKContentRoot/$includePath", "$root/$includePath") {
1094         my $found = -e $candidate;
1095         my $foundstr = ($found ? "found" : "didn't find");
1096         print "note:    $foundstr $includePath at $candidate\n" if $VERBOSE;
1097         return $candidate if $found;
1098     }
1099
1100     die "Unable to find $includePath in $root.\n";
1101 }
1102
1103 sub buildSharedCache {
1104     my $Cref = shift;
1105     my %C = %$Cref;
1106
1107     make("update_dyld_shared_cache -verbose -cache_dir $BUILDDIR -overlay $C{TESTLIBDIR}/../..");
1108 }
1109
1110 sub make_one_config {
1111     my $configref = shift;
1112     my $root = shift;
1113     my %C = %{$configref};
1114
1115     # Aliases
1116     $C{LANGUAGE} = "objective-c"  if $C{LANGUAGE} eq "objc";
1117     $C{LANGUAGE} = "objective-c++"  if $C{LANGUAGE} eq "objc++";
1118
1119     # Interpret OS version string from command line.
1120     my ($sdk_arg, $deployment_arg, $run_arg, undef) = split('-', $C{OSVERSION});
1121     delete $C{OSVERSION};
1122     my ($os_arg) = ($sdk_arg =~ /^([^\.0-9]+)/);
1123     $deployment_arg = "default" if !defined($deployment_arg);
1124     $run_arg = "default" if !defined($run_arg);
1125
1126     my %allowed_os_args = (
1127         "macosx" => "macosx", "osx" => "macosx", "macos" => "macosx",
1128         "iphoneos" => "iphoneos", "ios" => "iphoneos",
1129         "iphonesimulator" => "iphonesimulator", "iossimulator" => "iphonesimulator",
1130         "watchos" => "watchos",
1131         "watchsimulator" => "watchsimulator", "watchossimulator" => "watchsimulator",
1132         "appletvos" => "appletvos", "tvos" => "appletvos",
1133         "appletvsimulator" => "appletvsimulator", "tvsimulator" => "appletvsimulator",
1134         "bridgeos" => "bridgeos",
1135         );
1136
1137     $C{OS} = $allowed_os_args{$os_arg} || die "unknown OS '$os_arg' (expected " . join(', ', sort keys %allowed_os_args) . ")\n";
1138
1139     # set the config name now, after massaging the language and OS versions,
1140     # but before adding other settings
1141     my $configname = config_name(%C);
1142     die if ($configname =~ /'/);
1143     die if ($configname =~ / /);
1144     ($C{NAME} = $configname) =~ s/~/ /g;
1145     (my $configdir = $configname) =~ s#/##g;
1146     $C{DSTDIR} = "$DSTROOT$BUILDDIR/$configdir";
1147     $C{OBJDIR} = "$OBJROOT$BUILDDIR/$configdir";
1148
1149     # Allow tests to see BATS-edness in TEST_CONFIG.
1150     $C{BATS} = $BATS;
1151
1152     if ($C{OS} eq "iphoneos" || $C{OS} eq "iphonesimulator") {
1153         $C{TOOLCHAIN} = "ios";
1154     } elsif ($C{OS} eq "watchos" || $C{OS} eq "watchsimulator") {
1155         $C{TOOLCHAIN} = "watchos";
1156     } elsif ($C{OS} eq "appletvos" || $C{OS} eq "appletvsimulator") {
1157         $C{TOOLCHAIN} = "appletvos";
1158     } elsif ($C{OS} eq "bridgeos") {
1159         $C{TOOLCHAIN} = "bridgeos";
1160     } elsif ($C{OS} eq "macosx") {
1161         $C{TOOLCHAIN} = "osx";
1162     } else {
1163         colorprint $yellow, "WARN: don't know toolchain for OS $C{OS}";
1164         $C{TOOLCHAIN} = "default";
1165     }
1166
1167     if ($BUILD) {
1168         # Look up SDK.
1169         # Try exact match first.
1170         # Then try lexically-last prefix match
1171         #   (so "macosx" => "macosx10.7internal")
1172
1173         $sdk_arg =~ s/$os_arg/$C{OS}/;
1174
1175         my @sdks = getsdks();
1176         if ($VERBOSE) {
1177             print "note: Installed SDKs: @sdks\n";
1178         }
1179         my $exactsdk = undef;
1180         my $prefixsdk = undef;
1181         foreach my $sdk (@sdks) {
1182             $exactsdk = $sdk  if ($sdk eq $sdk_arg);
1183             $prefixsdk = newersdk($sdk, $prefixsdk)  if ($sdk =~ /^$sdk_arg/);
1184         }
1185
1186         my $sdk;
1187         if ($exactsdk) {
1188             $sdk = $exactsdk;
1189         } elsif ($prefixsdk) {
1190             $sdk = $prefixsdk;
1191         } else {
1192             die "unknown SDK '$sdk_arg'\nInstalled SDKs: @sdks\n";
1193         }
1194
1195         # Set deployment target.
1196         # fixme can't enforce version when run_arg eq "default"
1197         # because we don't know it yet
1198         $deployment_arg = versionsuffix($sdk) if $deployment_arg eq "default";
1199         if ($run_arg ne "default") {
1200             die "Deployment target '$deployment_arg' is newer than run target '$run_arg'\n"  if $deployment_arg > $run_arg;
1201         }
1202         $C{DEPLOYMENT_TARGET} = $deployment_arg;
1203         $C{SDK_PATH} = getsdkpath($sdk);
1204     } else {
1205         # not $BUILD
1206         $C{DEPLOYMENT_TARGET} = "unknown_deployment_target";
1207         $C{SDK_PATH} = "/unknown/sdk";
1208     }
1209
1210     # Set run target.
1211     $C{RUN_TARGET} = $run_arg;
1212
1213     # Look up test library (possible in root or SDK_PATH)
1214
1215     my $rootarg = $root;
1216     my $symroot;
1217     my @sympaths = ( (glob "$root/*~sym")[0],
1218                      (glob "$root/BuildRecords/*_install/Symbols")[0],
1219                      "$root/Symbols" );
1220     my @dstpaths = ( (glob "$root/*~dst")[0],
1221                      (glob "$root/BuildRecords/*_install/Root")[0],
1222                      "$root/Root" );
1223     for(my $i = 0; $i < scalar(@sympaths); $i++) {
1224         if (-e $sympaths[$i]  &&  -e $dstpaths[$i]) {
1225             $symroot = $sympaths[$i];
1226             $root = $dstpaths[$i];
1227             last;
1228         }
1229     }
1230
1231     if ($root ne "") {
1232         # Root specified. Require that it contain our dylibs.
1233         if (dirContainsAllTestLibs("$root$C{SDK_PATH}$TESTLIBDIR")) {
1234             $C{TESTLIBDIR} = "$root$C{SDK_PATH}$TESTLIBDIR";
1235         } elsif (dirContainsAllTestLibs("$root$TESTLIBDIR")) {
1236             $C{TESTLIBDIR} = "$root$TESTLIBDIR";
1237         } elsif (dirContainsAllTestLibs($root)) {
1238             $C{TESTLIBDIR} = "$root";
1239         } else {
1240             die "Didn't find some libs in root '$rootarg' for sdk '$C{SDK_PATH}'\n";
1241         }
1242     }
1243     else {
1244         # No root specified. Use the SDK or / for our dylibs.
1245         if (dirContainsAllTestLibs("$C{SDK_PATH}$TESTLIBDIR")) {
1246             $C{TESTLIBDIR} = "$C{SDK_PATH}$TESTLIBDIR";
1247         } else {
1248             # We don't actually check in / because on devices
1249             # there are no dylib files there.
1250             $C{TESTLIBDIR} = $TESTLIBDIR;
1251         }
1252     }
1253
1254     @{$C{TESTLIBS}} = map { "$C{TESTLIBDIR}/$_" } @TESTLIBNAMES;
1255     # convenience for tests that want libobjc.dylib's path
1256     $C{TESTLIB} = @{$C{TESTLIBS}}[0];
1257
1258     foreach my $testlibname (@TESTLIBNAMES) {
1259         if (-e "$symroot/$testlibname.dSYM") {
1260             push(@{$C{TESTDSYMS}}, "$symroot/$testlibname.dSYM");
1261         }
1262     }
1263
1264     if ($VERBOSE) {
1265         foreach my $testlib (@{$C{TESTLIBS}}) {
1266             my @uuids = `/usr/bin/dwarfdump -u '$testlib'`;
1267             while (my $uuid = shift @uuids) {
1268                 print "note: $uuid";
1269             }
1270         }
1271     }
1272
1273     # Look up compilers
1274     my $cc = $C{CC};
1275     my $cxx = cplusplus($C{CC});
1276     my $swift = swift($C{CC});
1277     if (! $BUILD) {
1278         $C{CC} = $cc;
1279         $C{CXX} = $cxx;
1280         $C{SWIFT} = $swift
1281     } else {
1282         $C{CC} = find_compiler($cc, $C{TOOLCHAIN}, $C{SDK_PATH});
1283         $C{CXX} = find_compiler($cxx, $C{TOOLCHAIN}, $C{SDK_PATH});
1284         $C{SWIFT} = find_compiler($swift, $C{TOOLCHAIN}, $C{SDK_PATH});
1285
1286         die "No C compiler '$cc' ('$C{CC}') in toolchain '$C{TOOLCHAIN}'\n" if !-e $C{CC};
1287         die "No C++ compiler '$cxx' ('$C{CXX}') in toolchain '$C{TOOLCHAIN}'\n" if !-e $C{CXX};
1288         die "No Swift compiler '$swift' ('$C{SWIFT}') in toolchain '$C{TOOLCHAIN}'\n" if !-e $C{SWIFT};
1289     }
1290
1291     if ($C{ARCH} eq "i386" && $C{OS} eq "macosx") {
1292         # libarclite no longer available on i386
1293         # fixme need an archived copy for bincompat testing
1294         $C{FORCE_LOAD_ARCLITE} = "";
1295     } elsif ($C{OS} eq "bridgeos") {
1296         # no libarclite on bridgeOS
1297         $C{FORCE_LOAD_ARCLITE} = "";
1298     } else {
1299         $C{FORCE_LOAD_ARCLITE} = "-Xlinker -force_load -Xlinker " . dirname($C{CC}) . "/../lib/arc/libarclite_$C{OS}.a";
1300     }
1301
1302     # Populate cflags
1303
1304     my $cflags = "-I$DIR -W -Wall -Wno-objc-weak-compat -Wno-arc-bridge-casts-disallowed-in-nonarc -Wshorten-64-to-32 -Qunused-arguments -fno-caret-diagnostics -Os -arch $C{ARCH} ";
1305     if (!$BATS) {
1306         # save-temps so dsymutil works so debug info works.
1307         # Disabled in BATS to save disk space.
1308         # rdar://45656803 -save-temps causes bad -Wstdlibcxx-not-found warnings
1309         $cflags .= "-g -save-temps -Wno-stdlibcxx-not-found";
1310     }
1311     my $objcflags = "";
1312     my $swiftflags = "-g ";
1313
1314     $cflags .= " -isysroot '$C{SDK_PATH}'";
1315     $cflags .= " '-Wl,-syslibroot,$C{SDK_PATH}'";
1316     $swiftflags .= " -sdk '$C{SDK_PATH}'";
1317
1318     # Set deployment target cflags
1319     my $target = undef;
1320     die "No deployment target" if $C{DEPLOYMENT_TARGET} eq "";
1321     if ($C{OS} eq "iphoneos") {
1322         $cflags .= " -mios-version-min=$C{DEPLOYMENT_TARGET}";
1323         $target = "$C{ARCH}-apple-ios$C{DEPLOYMENT_TARGET}";
1324     }
1325     elsif ($C{OS} eq "iphonesimulator") {
1326         $cflags .= " -mios-simulator-version-min=$C{DEPLOYMENT_TARGET}";
1327         $target = "$C{ARCH}-apple-ios$C{DEPLOYMENT_TARGET}";
1328     }
1329     elsif ($C{OS} eq "watchos") {
1330         $cflags .= " -mwatchos-version-min=$C{DEPLOYMENT_TARGET}";
1331         $target = "$C{ARCH}-apple-watchos$C{DEPLOYMENT_TARGET}";
1332     }
1333     elsif ($C{OS} eq "watchsimulator") {
1334         $cflags .= " -mwatchos-simulator-version-min=$C{DEPLOYMENT_TARGET}";
1335         $target = "$C{ARCH}-apple-watchos$C{DEPLOYMENT_TARGET}";
1336     }
1337     elsif ($C{OS} eq "appletvos") {
1338         $cflags .= " -mtvos-version-min=$C{DEPLOYMENT_TARGET}";
1339         $target = "$C{ARCH}-apple-tvos$C{DEPLOYMENT_TARGET}";
1340     }
1341     elsif ($C{OS} eq "appletvsimulator") {
1342         $cflags .= " -mtvos-simulator-version-min=$C{DEPLOYMENT_TARGET}";
1343         $target = "$C{ARCH}-apple-tvos$C{DEPLOYMENT_TARGET}";
1344     }
1345     elsif ($C{OS} eq "bridgeos") {
1346         $cflags .= " -mbridgeos-version-min=$C{DEPLOYMENT_TARGET}";
1347         $target = "$C{ARCH}-apple-bridgeos$C{DEPLOYMENT_TARGET}";
1348     }
1349     else {
1350         $cflags .= " -mmacosx-version-min=$C{DEPLOYMENT_TARGET}";
1351         $target = "$C{ARCH}-apple-macosx$C{DEPLOYMENT_TARGET}";
1352     }
1353     $swiftflags .= " -target $target";
1354
1355     $C{TESTINCLUDEDIR} = "$C{SDK_PATH}/usr/include";
1356     $C{TESTLOCALINCLUDEDIR} = "$C{SDK_PATH}/usr/local/include";
1357     if ($root ne "") {
1358         if ($C{SDK_PATH} ne "/") {
1359             $cflags .= " -isystem '$root$C{SDK_PATH}/usr/include'";
1360             $cflags .= " -isystem '$root$C{SDK_PATH}/usr/local/include'";
1361         }
1362
1363         my $library_path = $C{TESTLIBDIR};
1364         $cflags .= " -L$library_path";
1365         $C{TESTINCLUDEDIR} = findIncludeDir($root, "usr/include");
1366         $C{TESTLOCALINCLUDEDIR} = findIncludeDir($root, "usr/local/include");
1367         $cflags .= " -isystem '$C{TESTINCLUDEDIR}'";
1368         $cflags .= " -isystem '$C{TESTLOCALINCLUDEDIR}'";
1369     }
1370
1371
1372     # Populate objcflags
1373
1374     $objcflags .= " -lobjc";
1375     if ($C{MEM} eq "arc") {
1376         $objcflags .= " -fobjc-arc";
1377     }
1378     elsif ($C{MEM} eq "mrc") {
1379         # nothing
1380     }
1381     else {
1382         die "unrecognized MEM '$C{MEM}'\n";
1383     }
1384
1385     # Populate ENV_PREFIX
1386     $C{ENV} = "LANG=C MallocScribble=1";
1387     $C{ENV} .= " VERBOSE=$VERBOSE"  if $VERBOSE;
1388     if ($root ne "") {
1389         die "no spaces allowed in root" if $C{TESTLIBDIR} =~ /\s+/;
1390     }
1391     if ($C{GUARDMALLOC}) {
1392         $C{ENV} .= " GUARDMALLOC=1";  # checked by tests and errcheck.pl
1393         $C{ENV} .= " DYLD_INSERT_LIBRARIES=/usr/lib/libgmalloc.dylib";
1394         if ($C{GUARDMALLOC} eq "before") {
1395             $C{ENV} .= " MALLOC_PROTECT_BEFORE=1";
1396         } elsif ($C{GUARDMALLOC} eq "after") {
1397             # protect after is the default
1398         } else {
1399             die "Unknown guard malloc mode '$C{GUARDMALLOC}'\n";
1400         }
1401     }
1402
1403     # Populate compiler commands
1404     $C{XCRUN} = "env LANG=C /usr/bin/xcrun -toolchain '$C{TOOLCHAIN}'";
1405
1406     $C{COMPILE_C}   = "$C{XCRUN} '$C{CC}'  $cflags -x c -std=gnu99";
1407     $C{COMPILE_CXX} = "$C{XCRUN} '$C{CXX}' $cflags -x c++";
1408     $C{COMPILE_M}   = "$C{XCRUN} '$C{CC}'  $cflags $objcflags -x objective-c -std=gnu99";
1409     $C{COMPILE_MM}  = "$C{XCRUN} '$C{CXX}' $cflags $objcflags -x objective-c++";
1410     $C{COMPILE_SWIFT} = "$C{XCRUN} '$C{SWIFT}' $swiftflags";
1411
1412     $C{COMPILE} = $C{COMPILE_C}      if $C{LANGUAGE} eq "c";
1413     $C{COMPILE} = $C{COMPILE_CXX}    if $C{LANGUAGE} eq "c++";
1414     $C{COMPILE} = $C{COMPILE_M}      if $C{LANGUAGE} eq "objective-c";
1415     $C{COMPILE} = $C{COMPILE_MM}     if $C{LANGUAGE} eq "objective-c++";
1416     $C{COMPILE} = $C{COMPILE_SWIFT}  if $C{LANGUAGE} eq "swift";
1417     die "unknown language '$C{LANGUAGE}'\n" if !defined $C{COMPILE};
1418
1419     ($C{COMPILE_NOMEM} = $C{COMPILE}) =~ s/ -fobjc-arc\S*//g;
1420     ($C{COMPILE_NOLINK} = $C{COMPILE}) =~ s/ '?-(?:Wl,|l)\S*//g;
1421     ($C{COMPILE_NOLINK_NOMEM} = $C{COMPILE_NOMEM}) =~ s/ '?-(?:Wl,|l)\S*//g;
1422
1423
1424     # Reject some self-inconsistent and disallowed configurations
1425     if ($C{MEM} !~ /^(mrc|arc)$/) {
1426         die "unknown MEM=$C{MEM} (expected one of mrc arc)\n";
1427     }
1428
1429     if ($C{MEM} eq "arc"  &&  $C{CC} !~ /clang/) {
1430         print "note: skipping configuration $C{NAME}\n";
1431         print "note:   because CC=$C{CC} does not support MEM=$C{MEM}\n";
1432         return 0;
1433     }
1434
1435     if ($C{ARCH} eq "i386"  &&  $C{OS} eq "macosx") {
1436         colorprint $yellow, "WARN: skipping configuration $C{NAME}\n";
1437         colorprint $yellow, "WARN:   because 32-bit Mac is dead\n";
1438         return 0;
1439     }
1440
1441     # fixme
1442     if ($C{LANGUAGE} eq "swift"  &&  $C{ARCH} =~ /^arm/) {
1443         print "note: skipping configuration $C{NAME}\n";
1444         print "note:   because ARCH=$C{ARCH} does not support LANGUAGE=SWIFT\n";
1445         return 0;
1446     }
1447
1448     # fixme unimplemented run targets
1449     if ($C{RUN_TARGET} ne "default" &&  $C{OS} !~ /simulator/) {
1450         colorprint $yellow, "WARN: skipping configuration $C{NAME}";
1451         colorprint $yellow, "WARN:   because OS=$C{OS} does not yet implement RUN_TARGET=$C{RUN_TARGET}";
1452     }
1453
1454     %$configref = %C;
1455 }
1456
1457 sub make_configs {
1458     my ($root, %args) = @_;
1459
1460     my @results = ({});  # start with one empty config
1461
1462     for my $key (keys %args) {
1463         my @newresults;
1464         my @values = @{$args{$key}};
1465         for my $configref (@results) {
1466             my %config = %{$configref};
1467             for my $value (@values) {
1468                 my %newconfig = %config;
1469                 $newconfig{$key} = $value;
1470                 push @newresults, \%newconfig;
1471             }
1472         }
1473         @results = @newresults;
1474     }
1475
1476     my @newresults;
1477     for my $configref(@results) {
1478         if (make_one_config($configref, $root)) {
1479             push @newresults, $configref;
1480         }
1481     }
1482
1483     return @newresults;
1484 }
1485
1486 sub config_name {
1487     my %config = @_;
1488     my $name = "";
1489     for my $key (sort keys %config) {
1490         $name .= '~'  if $name ne "";
1491         $name .= "$key=$config{$key}";
1492     }
1493     return $name;
1494 }
1495
1496 sub rsync_ios {
1497     my ($src, $timeout) = @_;
1498     for (my $i = 0; $i < 10; $i++) {
1499         make("$DIR/timeout.pl $timeout rsync -e 'ssh -p $PORT' -av $src $HOST:/$REMOTEBASE/");
1500         return if $? == 0;
1501         colorprint $yellow, "WARN: RETRY\n"  if $VERBOSE;
1502     }
1503     die "Couldn't rsync tests to device. Check: device is connected; tcprelay is running; device trusts your Mac; device is unlocked; filesystem is mounted r/w\n";
1504 }
1505
1506 sub build_and_run_one_config {
1507     my %C = %{shift()};
1508     my @tests = @_;
1509
1510     # Build and run
1511     my $testcount = 0;
1512     my $failcount = 0;
1513     my $skipconfig = 0;
1514
1515     my @gathertests;
1516     foreach my $test (@tests) {
1517         if ($VERBOSE) {
1518             print "\nGATHER $test\n";
1519         }
1520
1521         if ($ALL_TESTS{$test}) {
1522             gather_simple(\%C, $test) || next;  # not pass, not fail
1523             push @gathertests, $test;
1524         } else {
1525             die "No test named '$test'\n";
1526         }
1527     }
1528
1529     my @builttests;
1530     if (!$BUILD) {
1531         @builttests = @gathertests;
1532         $testcount = scalar(@gathertests);
1533     } else {
1534         foreach my $test (@gathertests) {
1535             if ($VERBOSE) {
1536                 print "\nBUILD $test\n";
1537             }
1538
1539             if ($ALL_TESTS{$test}) {
1540                 $testcount++;
1541                 if (!build_simple(\%C, $test)) {
1542                     $failcount++;
1543                 } else {
1544                     push @builttests, $test;
1545                 }
1546             } else {
1547                 die "No test named '$test'\n";
1548             }
1549         }
1550     }
1551
1552     if (!$RUN  ||  !scalar(@builttests)) {
1553         # nothing to do
1554     }
1555     else {
1556         if ($C{ARCH} =~ /^arm/ && `uname -p` !~ /^arm/) {
1557             # upload timeout - longer for slow watch devices
1558             my $timeout = ($C{OS} =~ /watch/) ? 120 : 20;
1559
1560             # upload all tests to iOS device
1561             rsync_ios($C{DSTDIR}, $timeout);
1562
1563             # upload library to iOS device
1564             if ($C{TESTLIBDIR} ne $TESTLIBDIR) {
1565                 foreach my $thing (@{$C{TESTLIBS}}, @{$C{TESTDSYMS}}) {
1566                     rsync_ios($thing, $timeout);
1567                 }
1568             }
1569         }
1570         elsif ($C{OS} =~ /simulator/) {
1571             # run locally in a simulator
1572         }
1573         else {
1574             # run locally
1575             if ($BATS) {
1576                 # BATS execution tries to run architectures that
1577                 # aren't supported by the device. Skip those configs here.
1578                 my $machine = `machine`;
1579                 chomp $machine;
1580                 # unsupported:
1581                 # running arm64e on non-arm64e device
1582                 # running arm64 on non-arm64* device
1583                 # running armv7k on non-armv7k device
1584                 # running arm64_32 on armv7k device
1585                 # We don't need to handle all mismatches here,
1586                 # only mismatches that arise within a single OS.
1587                 $skipconfig =
1588                     (($C{ARCH} eq "arm64e"   && $machine ne "arm64e")  ||
1589                      ($C{ARCH} eq "arm64"    && $machine !~ /^arm64/)  ||
1590                      ($C{ARCH} eq "armv7k"   && $machine ne "armv7k")  ||
1591                      ($C{ARCH} eq "arm64_32" && $machine eq "armv7k"));
1592                 if ($skipconfig) {
1593                     print "note: skipping configuration $C{NAME}\n";
1594                     print "note:   because test arch $C{ARCH} is not " .
1595                         "supported on device arch $machine\n";
1596                     $testcount = 0;
1597                 }
1598             }
1599         }
1600
1601         if (!$skipconfig) {
1602             foreach my $test (@builttests) {
1603                 print "\nRUN $test\n"  if ($VERBOSE);
1604
1605                 if ($ALL_TESTS{$test}) {
1606                     if (!run_simple(\%C, $test)) {
1607                         $failcount++;
1608                     }
1609                 } else {
1610                     die "No test named '$test'\n";
1611                 }
1612             }
1613         }
1614     }
1615
1616     return ($testcount, $failcount, $skipconfig);
1617 }
1618
1619
1620
1621 # Return value if set by "$argname=value" on the command line
1622 # Return $default if not set.
1623 sub getargs {
1624     my ($argname, $default) = @_;
1625
1626     foreach my $arg (@ARGV) {
1627         my ($value) = ($arg =~ /^$argname=(.+)$/);
1628         return [split ',', $value] if defined $value;
1629     }
1630
1631     return [split ',', $default];
1632 }
1633
1634 # Return 1 or 0 if set by "$argname=1" or "$argname=0" on the
1635 # command line. Return $default if not set.
1636 sub getbools {
1637     my ($argname, $default) = @_;
1638
1639     my @values = @{getargs($argname, $default)};
1640     return [( map { ($_ eq "0") ? 0 : 1 } @values )];
1641 }
1642
1643 # Return an integer if set by "$argname=value" on the
1644 # command line. Return $default if not set.
1645 sub getints {
1646     my ($argname, $default) = @_;
1647
1648     my @values = @{getargs($argname, $default)};
1649     return [( map { int($_) } @values )];
1650 }
1651
1652 sub getarg {
1653     my ($argname, $default) = @_;
1654     my @values = @{getargs($argname, $default)};
1655     die "Only one value allowed for $argname\n"  if @values > 1;
1656     return $values[0];
1657 }
1658
1659 sub getbool {
1660     my ($argname, $default) = @_;
1661     my @values = @{getbools($argname, $default)};
1662     die "Only one value allowed for $argname\n"  if @values > 1;
1663     return $values[0];
1664 }
1665
1666 sub getint {
1667     my ($argname, $default) = @_;
1668     my @values = @{getints($argname, $default)};
1669     die "Only one value allowed for $argname\n"  if @values > 1;
1670     return $values[0];
1671 }
1672
1673
1674 my $default_arch = "x86_64";
1675 $args{ARCH} = getargs("ARCH", 0);
1676 $args{ARCH} = getargs("ARCHS", $default_arch)  if !@{$args{ARCH}}[0];
1677
1678 $args{OSVERSION} = getargs("OS", "macosx-default-default");
1679
1680 $args{MEM} = getargs("MEM", "mrc,arc");
1681 $args{LANGUAGE} = [ map { lc($_) } @{getargs("LANGUAGE", "c,objective-c,c++,objective-c++")} ];
1682
1683 $args{BUILD_SHARED_CACHE} = getargs("BUILD_SHARED_CACHE", 0);
1684
1685 $args{DYLD} = getargs("DYLD", "2,3");
1686
1687 $args{CC} = getargs("CC", "clang");
1688
1689 $HOST = getarg("HOST", "iphone");
1690 $PORT = getarg("PORT", "10022");
1691
1692 {
1693     my $guardmalloc = getargs("GUARDMALLOC", 0);
1694     # GUARDMALLOC=1 is the same as GUARDMALLOC=before,after
1695     my @guardmalloc2 = ();
1696     for my $arg (@$guardmalloc) {
1697         if ($arg == 1) { push @guardmalloc2, "before";
1698                          push @guardmalloc2, "after"; }
1699         else { push @guardmalloc2, $arg }
1700     }
1701     $args{GUARDMALLOC} = \@guardmalloc2;
1702 }
1703
1704 $BUILD = getbool("BUILD", 1);
1705 $RUN = getbool("RUN", 1);
1706 $VERBOSE = getint("VERBOSE", 0);
1707 $BATS = getbool("BATS", 0);
1708 $BUILDDIR = getarg("BUILDDIR", $BATS ? $BATSBASE : $LOCALBASE);
1709
1710 my $root = getarg("ROOT", "");
1711 $root =~ s#/*$##;
1712
1713 my @tests = gettests();
1714
1715 if ($BUILD) {
1716     rm_rf_verbose "$DSTROOT$BUILDDIR";
1717     rm_rf_verbose "$OBJROOT$BUILDDIR";
1718 }
1719
1720 print "note: -----\n";
1721 print "note: testing root '$root'\n";
1722
1723 my @configs = make_configs($root, %args);
1724
1725 print "note: -----\n";
1726 print "note: testing ", scalar(@configs), " configurations:\n";
1727 for my $configref (@configs) {
1728     my $configname = $$configref{NAME};
1729     print "note: configuration $configname\n";
1730 }
1731
1732 my $failed = 0;
1733
1734 my $testconfigs = @configs;
1735 my $failconfigs = 0;
1736 my $skipconfigs = 0;
1737 my $testcount = 0;
1738 my $failcount = 0;
1739 for my $configref (@configs) {
1740     my $configname = $$configref{NAME};
1741     print "note: -----\n";
1742     print "note: \nnote: $configname\nnote: \n";
1743
1744     (my $t, my $f, my $skipconfig) =
1745         eval { build_and_run_one_config($configref, @tests); };
1746     $skipconfigs += $skipconfig;
1747     if ($@) {
1748         chomp $@;
1749         colorprint $red, "FAIL: $configname";
1750         colorprint $red, "FAIL: $@";
1751         $failconfigs++;
1752     } else {
1753         my $color = ($f ? $red : "");
1754         print "note:\n";
1755         colorprint $color, "note: $configname\n";
1756         colorprint $color, "note: $t tests, $f failures";
1757         $testcount += $t;
1758         $failcount += $f;
1759         $failconfigs++ if ($f);
1760     }
1761 }
1762
1763 print "note: -----\n";
1764 my $color = ($failconfigs ? $red : "");
1765 colorprint $color, "note: $testconfigs configurations, " .
1766     "$failconfigs with failures, $skipconfigs skipped";
1767 colorprint $color, "note: $testcount tests, $failcount failures";
1768
1769 $failed = ($failconfigs ? 1 : 0);
1770
1771
1772 if ($BUILD && $BATS && !$failed) {
1773     # Collect BATS execution instructions for all tests.
1774     # Each BATS "test" is all configurations together of one of our tests.
1775     for my $testname (@tests) {
1776         append_bats_test($testname);
1777     }
1778
1779     # Write the BATS plist to disk.
1780     my $json = encode_json(\%bats_plist);
1781     my $filename = "$DSTROOT$BATSBASE/objc4.plist";
1782     print "note: writing BATS config to $filename\n";
1783     open(my $file, '>', $filename);
1784     print $file $json;
1785     close $file;
1786 }
1787
1788 exit ($failed ? 1 : 0);