test/test.pl

   1 #!/usr/bin/perl
   2
   3 # test.pl
   4 # Run unit tests.
   5
   6 use strict;
   7 use File::Basename;
   8
   9 chdir dirname $0;
  10 chomp (my $DIR = `pwd`);
  11
  12 my $TESTLIBNAME = "libobjc.A.dylib";
  13 my $TESTLIBPATH = "/usr/lib/$TESTLIBNAME";
  14
  15 my $BUILDDIR = "/tmp/test-$TESTLIBNAME-build";
  16
  17 # xterm colors
  18 my $red = "\e[41;37m";
  19 my $yellow = "\e[43;37m";
  20 my $def = "\e[0m";
  21
  22 # clean, help
  23 if (scalar(@ARGV) == 1) {
  24     my $arg = $ARGV[0];
  25     if ($arg eq "clean") {
  26         my $cmd = "rm -rf $BUILDDIR *~";
  27         print "$cmd\n";
  28         `$cmd`;
  29         exit 0;
  30     }
  31     elsif ($arg eq "-h" || $arg eq "-H" || $arg eq "-help" || $arg eq "help") {
  32         print(<<END);
  33 usage: $0 [options] [testname ...]
  34        $0 clean
  35        $0 help
  36
  37 testname:
  38     `testname` runs a specific test. If no testnames are given, runs all tests.
  39
  40 options:
  41     ARCH=<arch>
  42     SDK=<sdk name>
  43     ROOT=/path/to/project.roots/
  44
  45     CC=<compiler name>
  46
  47     MEM=mrc,arc,gc
  48     STDLIB=libc++,libstdc++
  49     GUARDMALLOC=0|1
  50
  51     BUILD=0|1
  52     RUN=0|1
  53     VERBOSE=0|1
  54
  55 examples:
  56
  57     test installed library, x86_64, no gc
  58     $0
  59
  60     test buildit-built root, i386 and x86_64, MRC and ARC and GC, clang compiler
  61     $0 ARCH=i386,x86_64 ROOT=/tmp/libclosure.roots MEM=mrc,arc,gc CC=clang
  62
  63     test buildit-built root with iOS simulator
  64     $0 ARCH=i386 ROOT=/tmp/libclosure.roots SDK=iphonesimulator
  65
  66     test buildit-built root on attached iOS device
  67     $0 ARCH=armv7 ROOT=/tmp/libclosure.roots SDK=iphoneos
  68 END
  69         exit 0;
  70     }
  71 }
  72
  73 #########################################################################
  74 ## Tests
  75
  76 my %ALL_TESTS;
  77
  78 #########################################################################
  79 ## Variables for use in complex build and run rules
  80
  81 # variable         # example value
  82
  83 # things you can multiplex on the command line
  84 # ARCH=i386,x86_64,armv6,armv7
  85 # SDK=system,macosx,iphoneos,iphonesimulator
  86 # LANGUAGE=c,c++,objective-c,objective-c++
  87 # CC=clang,gcc-4.2,llvm-gcc-4.2
  88 # MEM=mrc,arc,gc
  89 # STDLIB=libc++,libstdc++
  90 # GUARDMALLOC=0,1
  91
  92 # things you can set once on the command line
  93 # ROOT=/path/to/project.roots
  94 # BUILD=0|1
  95 # RUN=0|1
  96 # VERBOSE=0|1
  97
  98
  99
 100 my $BUILD;
 101 my $RUN;
 102 my $VERBOSE;
 103
 104 my $crashcatch = <<'END';
 105 // interpose-able code to catch crashes, print, and exit cleanly
 106 #include <signal.h>
 107 #include <string.h>
 108 #include <unistd.h>
 109
 110 // from dyld-interposing.h
 111 #define DYLD_INTERPOSE(_replacement,_replacee) __attribute__((used)) static struct{ const void* replacement; const void* replacee; } _interpose_##_replacee __attribute__ ((section ("__DATA,__interpose"))) = { (const void*)(unsigned long)&_replacement, (const void*)(unsigned long)&_replacee };
 112
 113 static void catchcrash(int sig)
 114 {
 115     const char *msg;
 116     switch (sig) {
 117     case SIGILL:  msg = "CRASHED: SIGILL\\n";  break;
 118     case SIGBUS:  msg = "CRASHED: SIGBUS\\n";  break;
 119     case SIGSYS:  msg = "CRASHED: SIGSYS\\n";  break;
 120     case SIGSEGV: msg = "CRASHED: SIGSEGV\\n"; break;
 121     case SIGTRAP: msg = "CRASHED: SIGTRAP\\n"; break;
 122     case SIGABRT: msg = "CRASHED: SIGABRT\\n"; break;
 123     default: msg = "SIG\?\?\?\?\\n"; break;
 124     }
 125     write(STDERR_FILENO, msg, strlen(msg));
 126     _exit(0);
 127 }
 128
 129 static void setupcrash(void) __attribute__((constructor));
 130 static void setupcrash(void)
 131 {
 132     signal(SIGILL, &catchcrash);
 133     signal(SIGBUS, &catchcrash);
 134     signal(SIGSYS, &catchcrash);
 135     signal(SIGSEGV, &catchcrash);
 136     signal(SIGTRAP, &catchcrash);
 137     signal(SIGABRT, &catchcrash);
 138 }
 139
 140
 141 static int hacked = 0;
 142 ssize_t hacked_write(int fildes, const void *buf, size_t nbyte)
 143 {
 144     if (!hacked) {
 145         setupcrash();
 146         hacked = 1;
 147     }
 148     return write(fildes, buf, nbyte);
 149 }
 150
 151 DYLD_INTERPOSE(hacked_write, write);
 152
 153 END
 154
 155
 156 #########################################################################
 157 ## Harness
 158
 159
 160 # map language to buildable extensions for that language
 161 my %extensions_for_language = (
 162     "c"     => ["c"],
 163     "objective-c" => ["c", "m"],
 164     "c++" => ["c", "cc", "cp", "cpp", "cxx", "c++"],
 165     "objective-c++" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"],
 166
 167     "any" => ["c", "m", "cc", "cp", "cpp", "cxx", "c++", "mm"],
 168     );
 169
 170 # map extension to languages
 171 my %languages_for_extension = (
 172     "c" => ["c", "objective-c", "c++", "objective-c++"],
 173     "m" => ["objective-c", "objective-c++"],
 174     "mm" => ["objective-c++"],
 175     "cc" => ["c++", "objective-c++"],
 176     "cp" => ["c++", "objective-c++"],
 177     "cpp" => ["c++", "objective-c++"],
 178     "cxx" => ["c++", "objective-c++"],
 179     "c++" => ["c++", "objective-c++"],
 180     );
 181
 182 # Run some newline-separated commands like `make` would, stopping if any fail
 183 # run("cmd1 \n cmd2 \n cmd3")
 184 sub make {
 185     my $output = "";
 186     my @cmds = split("\n", $_[0]);
 187     die if scalar(@cmds) == 0;
 188     $? = 0;
 189     foreach my $cmd (@cmds) {
 190         chomp $cmd;
 191         next if $cmd =~ /^\s*$/;
 192         $cmd .= " 2>&1";
 193         print "$cmd\n" if $VERBOSE;
 194         $output .= `$cmd`;
 195         last if $?;
 196     }
 197     print "$output\n" if $VERBOSE;
 198     return $output;
 199 }
 200
 201 sub chdir_verbose {
 202     my $dir = shift;
 203     chdir $dir || die;
 204     print "cd $dir\n" if $VERBOSE;
 205 }
 206
 207
 208 # Return test names from the command line.
 209 # Returns all tests if no tests were named.
 210 sub gettests {
 211     my @tests;
 212
 213     foreach my $arg (@ARGV) {
 214         push @tests, $arg  if ($arg !~ /=/  &&  $arg !~ /^-/);
 215     }
 216
 217     opendir(my $dir, $DIR) || die;
 218     while (my $file = readdir($dir)) {
 219         my ($name, $ext) = ($file =~ /^([^.]+)\.([^.]+)$/);
 220         next if ! $languages_for_extension{$ext};
 221
 222         open(my $in, "< $file") || die "$file";
 223         my $contents = join "", <$in>;
 224         if (defined $ALL_TESTS{$name}) {
 225             print "${yellow}SKIP: multiple tests named '$name'; skipping file '$file'.${def}\n";
 226         } else {
 227             $ALL_TESTS{$name} = $ext  if ($contents =~ m#^[/*\s]*TEST_#m);
 228         }
 229         close($in);
 230     }
 231     closedir($dir);
 232
 233     if (scalar(@tests) == 0) {
 234         @tests = keys %ALL_TESTS;
 235     }
 236
 237     @tests = sort @tests;
 238
 239     return @tests;
 240 }
 241
 242
 243 # Turn a C compiler name into a C++ compiler name.
 244 sub cplusplus {
 245     my ($c) = @_;
 246     if ($c =~ /cc/) {
 247         $c =~ s/cc/\+\+/;
 248         return $c;
 249     }
 250     return $c . "++";                         # e.g. clang => clang++
 251 }
 252
 253 # Returns an array of all sdks from `xcodebuild -showsdks`
 254 my @sdks_memo;
 255 sub getsdks {
 256     if (!@sdks_memo) {
 257         @sdks_memo = ("system", `xcodebuild -showsdks` =~ /-sdk (.+)$/mg);
 258     }
 259     return @sdks_memo;
 260 }
 261
 262 # Returns whether the given sdk supports -lauto
 263 sub supportslibauto {
 264     my ($sdk) = @_;
 265     return 1 if $sdk eq "system";
 266     return 1 if $sdk =~ /^macosx/;
 267     return 0 if $sdk =~ /^iphone/;
 268     die;
 269 }
 270
 271 # print text with a colored prefix on each line
 272 sub colorprint {
 273     my $color = shift;
 274     while (my @lines = split("\n", shift)) {
 275         for my $line (@lines) {
 276             chomp $line;
 277             print "$color $def$line\n";
 278         }
 279     }
 280 }
 281
 282 sub rewind {
 283     seek($_[0], 0, 0);
 284 }
 285
 286 # parse name=value,value pairs
 287 sub readconditions {
 288     my ($conditionstring) = @_;
 289
 290     my %results;
 291     my @conditions = ($conditionstring =~ /\w+=(?:[^\s,]+,?)+/g);
 292     for my $condition (@conditions) {
 293         my ($name, $values) = ($condition =~ /(\w+)=(.+)/);
 294         $results{$name} = [split ',', $values];
 295     }
 296
 297     return %results;
 298 }
 299
 300 # Get the name of the system SDK from sw_vers
 301 sub systemsdkname {
 302     my @lines = `/usr/bin/sw_vers`;
 303     my $name;
 304     my $vers;
 305     for my $line (@lines) {
 306         ($name) = ($line =~ /^ProductName:\s+(.*)/)  if !$name;
 307         ($vers) = ($line =~ /^ProductVersion:\s+(.*)/)  if !$vers;
 308     }
 309
 310     $name =~ s/ //g;
 311     $name = lc($name);
 312     my $internal = "";
 313     if (-d "/usr/local/include/objc") {
 314         if ($name eq "macosx") {
 315             $internal = "internal";
 316         } else {
 317             $internal = ".internal";
 318         }
 319     }
 320     return $name . $vers . $internal;
 321 }
 322
 323 sub check_output {
 324     my %C = %{shift()};
 325     my $name = shift;
 326     my @output = @_;
 327
 328     my %T = %{$C{"TEST_$name"}};
 329
 330     # Quietly strip MallocScribble before saving the "original" output
 331     # because it is distracting.
 332     filter_malloc(\@output);
 333
 334     my @original_output = @output;
 335
 336     # Run result-checking passes, reducing @output each time
 337     my $xit = 1;
 338     my $bad = "";
 339     my $warn = "";
 340     my $runerror = $T{TEST_RUN_OUTPUT};
 341     filter_verbose(\@output);
 342     $warn = filter_warn(\@output);
 343     $bad |= filter_guardmalloc(\@output) if ($C{GUARDMALLOC});
 344     $bad |= filter_valgrind(\@output) if ($C{VALGRIND});
 345     $bad = filter_expected(\@output, \%C, $name) if ($bad eq "");
 346     $bad = filter_bad(\@output)  if ($bad eq "");
 347
 348     # OK line should be the only one left
 349     $bad = "(output not 'OK: $name')" if ($bad eq ""  &&  (scalar(@output) != 1  ||  $output[0] !~ /^OK: $name/));
 350
 351     if ($bad ne "") {
 352         print "${red}FAIL: /// test '$name' \\\\\\$def\n";
 353         colorprint($red, @original_output);
 354         print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
 355         print "${red}FAIL: $name: $bad$def\n";
 356         $xit = 0;
 357     }
 358     elsif ($warn ne "") {
 359         print "${yellow}PASS: /// test '$name' \\\\\\$def\n";
 360         colorprint($yellow, @original_output);
 361         print "${yellow}PASS: \\\\\\ test '$name' ///$def\n";
 362         print "PASS: $name (with warnings)\n";
 363     }
 364     else {
 365         print "PASS: $name\n";
 366     }
 367     return $xit;
 368 }
 369
 370 sub filter_expected
 371 {
 372     my $outputref = shift;
 373     my %C = %{shift()};
 374     my $name = shift;
 375
 376     my %T = %{$C{"TEST_$name"}};
 377     my $runerror = $T{TEST_RUN_OUTPUT}  ||  return "";
 378
 379     my $bad = "";
 380
 381     my $output = join("\n", @$outputref) . "\n";
 382     if ($output !~ /$runerror/) {
 383         $bad = "(run output does not match TEST_RUN_OUTPUT)";
 384         @$outputref = ("FAIL: $name");
 385     } else {
 386         @$outputref = ("OK: $name");  # pacify later filter
 387     }
 388
 389     return $bad;
 390 }
 391
 392 sub filter_bad
 393 {
 394     my $outputref = shift;
 395     my $bad = "";
 396
 397     my @new_output;
 398     for my $line (@$outputref) {
 399         if ($line =~ /^BAD: (.*)/) {
 400             $bad = "(failed)";
 401         } else {
 402             push @new_output, $line;
 403         }
 404     }
 405
 406     @$outputref = @new_output;
 407     return $bad;
 408 }
 409
 410 sub filter_warn
 411 {
 412     my $outputref = shift;
 413     my $warn = "";
 414
 415     my @new_output;
 416     for my $line (@$outputref) {
 417         if ($line !~ /^WARN: (.*)/) {
 418             push @new_output, $line;
 419         } else {
 420             $warn = "(warned)";
 421         }
 422     }
 423
 424     @$outputref = @new_output;
 425     return $warn;
 426 }
 427
 428 sub filter_verbose
 429 {
 430     my $outputref = shift;
 431
 432     my @new_output;
 433     for my $line (@$outputref) {
 434         if ($line !~ /^VERBOSE: (.*)/) {
 435             push @new_output, $line;
 436         }
 437     }
 438
 439     @$outputref = @new_output;
 440 }
 441
 442 sub filter_valgrind
 443 {
 444     my $outputref = shift;
 445     my $errors = 0;
 446     my $leaks = 0;
 447
 448     my @new_output;
 449     for my $line (@$outputref) {
 450         if ($line =~ /^Approx: do_origins_Dirty\([RW]\): missed \d bytes$/) {
 451             # --track-origins warning (harmless)
 452             next;
 453         }
 454         if ($line =~ /^UNKNOWN __disable_threadsignal is unsupported. This warning will not be repeated.$/) {
 455             # signals unsupported (harmless)
 456             next;
 457         }
 458         if ($line =~ /^UNKNOWN __pthread_sigmask is unsupported. This warning will not be repeated.$/) {
 459             # signals unsupported (harmless)
 460             next;
 461         }
 462         if ($line !~ /^^\.*==\d+==/) {
 463             # not valgrind output
 464             push @new_output, $line;
 465             next;
 466         }
 467
 468         my ($errcount) = ($line =~ /==\d+== ERROR SUMMARY: (\d+) errors/);
 469         if (defined $errcount  &&  $errcount > 0) {
 470             $errors = 1;
 471         }
 472
 473         (my $leakcount) = ($line =~ /==\d+==\s+(?:definitely|possibly) lost:\s+([0-9,]+)/);
 474         if (defined $leakcount  &&  $leakcount > 0) {
 475             $leaks = 1;
 476         }
 477     }
 478
 479     @$outputref = @new_output;
 480
 481     my $bad = "";
 482     $bad .= "(valgrind errors)" if ($errors);
 483     $bad .= "(valgrind leaks)" if ($leaks);
 484     return $bad;
 485 }
 486
 487
 488
 489 sub filter_malloc
 490 {
 491     my $outputref = shift;
 492     my $errors = 0;
 493
 494     my @new_output;
 495     my $count = 0;
 496     for my $line (@$outputref) {
 497         # Ignore MallocScribble prologue.
 498         # Ignore MallocStackLogging prologue.
 499         if ($line =~ /malloc: enabling scribbling to detect mods to free/  ||
 500             $line =~ /Deleted objects will be dirtied by the collector/  ||
 501             $line =~ /malloc: stack logs being written into/  ||
 502             $line =~ /malloc: recording malloc and VM allocation stacks/)
 503         {
 504             next;
 505         }
 506
 507         # not malloc output
 508         push @new_output, $line;
 509
 510     }
 511
 512     @$outputref = @new_output;
 513 }
 514
 515 sub filter_guardmalloc
 516 {
 517     my $outputref = shift;
 518     my $errors = 0;
 519
 520     my @new_output;
 521     my $count = 0;
 522     for my $line (@$outputref) {
 523         if ($line !~ /^GuardMalloc\[[^\]]+\]: /) {
 524             # not guardmalloc output
 525             push @new_output, $line;
 526             next;
 527         }
 528
 529         # Ignore 4 lines of guardmalloc prologue.
 530         # Anything further is a guardmalloc error.
 531         if (++$count > 4) {
 532             $errors = 1;
 533         }
 534     }
 535
 536     @$outputref = @new_output;
 537
 538     my $bad = "";
 539     $bad .= "(guardmalloc errors)" if ($errors);
 540     return $bad;
 541 }
 542
 543 sub gather_simple {
 544     my $CREF = shift;
 545     my %C = %{$CREF};
 546     my $name = shift;
 547     chdir_verbose $DIR;
 548
 549     my $ext = $ALL_TESTS{$name};
 550     my $file = "$name.$ext";
 551     return 0 if !$file;
 552
 553     # search file for 'TEST_CONFIG' or '#include "test.h"'
 554     # also collect other values:
 555     # TEST_CONFIG test conditions
 556     # TEST_ENV environment prefix
 557     # TEST_CFLAGS compile flags
 558     # TEST_BUILD build instructions
 559     # TEST_BUILD_OUTPUT expected build stdout/stderr
 560     # TEST_RUN_OUTPUT expected run stdout/stderr
 561     open(my $in, "< $file") || die;
 562     my $contents = join "", <$in>;
 563
 564     my $test_h = ($contents =~ /^\s*#\s*(include|import)\s*"test\.h"/m);
 565     my $disabled = ($contents =~ /\bTEST_DISABLED\b/m);
 566     my $crashes = ($contents =~ /\bTEST_CRASHES\b/m);
 567     my ($conditionstring) = ($contents =~ /\bTEST_CONFIG\b(.*)$/m);
 568     my ($envstring) = ($contents =~ /\bTEST_ENV\b(.*)$/m);
 569     my ($cflags) = ($contents =~ /\bTEST_CFLAGS\b(.*)$/m);
 570     my ($buildcmd) = ($contents =~ /TEST_BUILD\n(.*?\n)END[ *\/]*\n/s);
 571     my ($builderror) = ($contents =~ /TEST_BUILD_OUTPUT\n(.*?\n)END[ *\/]*\n/s);
 572     my ($runerror) = ($contents =~ /TEST_RUN_OUTPUT\n(.*?\n)END[ *\/]*\n/s);
 573
 574     return 0 if !$test_h && !$disabled && !$crashes && !defined($conditionstring) && !defined($envstring) && !defined($cflags) && !defined($buildcmd) && !defined($builderror) && !defined($runerror);
 575
 576     if ($disabled) {
 577         print "${yellow}SKIP: $name    (disabled by TEST_DISABLED)$def\n";
 578         return 0;
 579     }
 580
 581     # check test conditions
 582
 583     my $run = 1;
 584     my %conditions = readconditions($conditionstring);
 585     if (! $conditions{LANGUAGE}) {
 586         # implicit language restriction from file extension
 587         $conditions{LANGUAGE} = $languages_for_extension{$ext};
 588     }
 589     for my $condkey (keys %conditions) {
 590         my @condvalues = @{$conditions{$condkey}};
 591
 592         # special case: RUN=0 does not affect build
 593         if ($condkey eq "RUN"  &&  @condvalues == 1  &&  $condvalues[0] == 0) {
 594             $run = 0;
 595             next;
 596         }
 597
 598         my $testvalue = $C{$condkey};
 599         next if !defined($testvalue);
 600         # testvalue is the configuration being run now
 601         # condvalues are the allowed values for this test
 602
 603         # special case: look up the name of SDK "system"
 604         if ($condkey eq "SDK"  &&  $testvalue eq "system") {
 605             $testvalue = systemsdkname();
 606         }
 607
 608         my $ok = 0;
 609         for my $condvalue (@condvalues) {
 610
 611             # special case: objc and objc++
 612             if ($condkey eq "LANGUAGE") {
 613                 $condvalue = "objective-c" if $condvalue eq "objc";
 614                 $condvalue = "objective-c++" if $condvalue eq "objc++";
 615             }
 616
 617             $ok = 1  if ($testvalue eq $condvalue);
 618
 619             # special case: SDK allows prefixes, and "system" is "macosx"
 620             if ($condkey eq "SDK") {
 621                 $ok = 1  if ($testvalue =~ /^$condvalue/);
 622                 $ok = 1  if ($testvalue eq "system"  &&  "macosx" =~ /^$condvalue/);
 623             }
 624
 625             # special case: CC and CXX allow substring matches
 626             if ($condkey eq "CC"  ||  $condkey eq "CXX") {
 627                 $ok = 1  if ($testvalue =~ /$condvalue/);
 628             }
 629
 630             last if $ok;
 631         }
 632
 633         if (!$ok) {
 634             my $plural = (@condvalues > 1) ? "one of: " : "";
 635             print "SKIP: $name    ($condkey=$testvalue, but test requires $plural", join(' ', @condvalues), ")\n";
 636             return 0;
 637         }
 638     }
 639
 640     # builderror is multiple REs separated by OR
 641     if (defined $builderror) {
 642         $builderror =~ s/\nOR\n/\n|/sg;
 643         $builderror = "^(" . $builderror . ")\$";
 644     }
 645     # runerror is multiple REs separated by OR
 646     if (defined $runerror) {
 647         $runerror =~ s/\nOR\n/\n|/sg;
 648         $runerror = "^(" . $runerror . ")\$";
 649     }
 650
 651     # save some results for build and run phases
 652     $$CREF{"TEST_$name"} = {
 653         TEST_BUILD => $buildcmd,
 654         TEST_BUILD_OUTPUT => $builderror,
 655         TEST_CRASHES => $crashes,
 656         TEST_RUN_OUTPUT => $runerror,
 657         TEST_CFLAGS => $cflags,
 658         TEST_ENV => $envstring,
 659         TEST_RUN => $run,
 660     };
 661
 662     return 1;
 663 }
 664
 665 # Builds a simple test
 666 sub build_simple {
 667     my %C = %{shift()};
 668     my $name = shift;
 669     my %T = %{$C{"TEST_$name"}};
 670     chdir_verbose "$C{DIR}/$name.build";
 671
 672     my $ext = $ALL_TESTS{$name};
 673     my $file = "$DIR/$name.$ext";
 674
 675     if ($T{TEST_CRASHES}) {
 676         `echo '$crashcatch' > crashcatch.c`;
 677         make("$C{COMPILE_C} -dynamiclib -o libcrashcatch.dylib -x c crashcatch.c");
 678         die "$?" if $?;
 679     }
 680
 681     my $cmd = $T{TEST_BUILD} ? eval "return \"$T{TEST_BUILD}\"" : "$C{COMPILE}   $T{TEST_CFLAGS} $file -o $name.out";
 682
 683     my $output = make($cmd);
 684
 685     my $ok;
 686     if (my $builderror = $T{TEST_BUILD_OUTPUT}) {
 687         # check for expected output and ignore $?
 688         if ($output =~ /$builderror/) {
 689             $ok = 1;
 690         } else {
 691             print "${red}FAIL: /// test '$name' \\\\\\$def\n";
 692             colorprint $red, $output;
 693             print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
 694             print "${red}FAIL: $name (build output does not match TEST_BUILD_OUTPUT)$def\n";
 695             $ok = 0;
 696         }
 697     } elsif ($?) {
 698         print "${red}FAIL: /// test '$name' \\\\\\$def\n";
 699         colorprint $red, $output;
 700         print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
 701         print "${red}FAIL: $name (build failed)$def\n";
 702         $ok = 0;
 703     } elsif ($output ne "") {
 704         print "${red}FAIL: /// test '$name' \\\\\\$def\n";
 705         colorprint $red, $output;
 706         print "${red}FAIL: \\\\\\ test '$name' ///$def\n";
 707         print "${red}FAIL: $name (unexpected build output)$def\n";
 708         $ok = 0;
 709     } else {
 710         $ok = 1;
 711     }
 712
 713
 714     if ($ok) {
 715         foreach my $file (glob("*.out *.dylib *.bundle")) {
 716             make("dsymutil $file");
 717         }
 718     }
 719
 720     return $ok;
 721 }
 722
 723 # Run a simple test (testname.out, with error checking of stdout and stderr)
 724 sub run_simple {
 725     my %C = %{shift()};
 726     my $name = shift;
 727     my %T = %{$C{"TEST_$name"}};
 728
 729     if (! $T{TEST_RUN}) {
 730         print "PASS: $name (build only)\n";
 731         return 1;
 732     }
 733     else {
 734         chdir_verbose "$C{DIR}/$name.build";
 735     }
 736
 737     my $env = "$C{ENV} $T{TEST_ENV}";
 738     if ($T{TEST_CRASHES}) {
 739         $env .= " DYLD_INSERT_LIBRARIES=libcrashcatch.dylib";
 740     }
 741
 742     my $output;
 743
 744     if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) {
 745         # run on iOS device
 746
 747         my $remotedir = "/var/root/test/" . basename($C{DIR}) . "/$name.build";
 748         my $remotedyld = " DYLD_LIBRARY_PATH=$remotedir";
 749         $remotedyld .= ":/var/root/test/"  if ($C{TESTLIB} ne $TESTLIBPATH);
 750
 751         # elide host-specific paths
 752         $env =~ s/DYLD_LIBRARY_PATH=\S+//;
 753         $env =~ s/DYLD_ROOT_PATH=\S+//;
 754
 755         my $cmd = "ssh iphone 'cd $remotedir && env $env $remotedyld ./$name.out'";
 756         $output = make("$cmd");
 757     }
 758     else {
 759         # run locally
 760
 761         my $cmd = "env $env ./$name.out";
 762         $output = make("sh -c '$cmd 2>&1' 2>&1");
 763         # need extra sh level to capture "sh: Illegal instruction" after crash
 764         # fixme fail if $? except tests that expect to crash
 765     }
 766
 767     return check_output(\%C, $name, split("\n", $output));
 768 }
 769
 770
 771 my %compiler_memo;
 772 sub find_compiler {
 773     my ($cc, $sdk, $sdk_path) = @_;
 774
 775     # memoize
 776     my $key = $cc . ':' . $sdk;
 777     my $result = $compiler_memo{$key};
 778     return $result if defined $result;
 779
 780     if ($sdk eq "system") {
 781         $result  = `xcrun -find $cc 2>/dev/null`;
 782     } else {
 783         $result  = `xcrun -sdk $sdk -find $cc 2>/dev/null`;
 784     }
 785
 786     chomp $result;
 787     $compiler_memo{$key} = $result;
 788     return $result;
 789 }
 790
 791 sub make_one_config {
 792     my $configref = shift;
 793     my $root = shift;
 794     my %C = %{$configref};
 795
 796     # Aliases
 797     $C{LANGUAGE} = "objective-c"  if $C{LANGUAGE} eq "objc";
 798     $C{LANGUAGE} = "objective-c++"  if $C{LANGUAGE} eq "objc++";
 799
 800     # Look up SDK
 801     # Try exact match first.
 802     # Then try lexically-last prefix match (so "macosx" => "macosx10.7internal").
 803     my @sdks = getsdks();
 804     if ($VERBOSE) {
 805         print "Installed SDKs: @sdks\n";
 806     }
 807     my $exactsdk = undef;
 808     my $prefixsdk = undef;
 809     foreach my $sdk (@sdks) {
 810         my $SDK = $C{SDK};
 811         $exactsdk = $sdk  if ($sdk eq $SDK);
 812         # check for digits to prevent e.g. "iphone" => "iphonesimulator4.2"
 813         $prefixsdk = $sdk  if ($sdk =~ /^$SDK[0-9]/  &&  $sdk gt $prefixsdk);
 814     }
 815     if ($exactsdk) {
 816         $C{SDK} = $exactsdk;
 817     } elsif ($prefixsdk) {
 818         $C{SDK} = $prefixsdk;
 819     } else {
 820         die "unknown SDK '$C{SDK}'\nInstalled SDKs: @sdks\n";
 821     }
 822
 823     # set the config name now, after massaging the language and sdk,
 824     # but before adding other settings
 825     my $configname = config_name(%C);
 826     die if ($configname =~ /'/);
 827     die if ($configname =~ / /);
 828     ($C{NAME} = $configname) =~ s/~/ /g;
 829     (my $configdir = $configname) =~ s#/##g;
 830     $C{DIR} = "$BUILDDIR/$configdir";
 831
 832     $C{SDK_PATH} = "/";
 833     if ($C{SDK} ne "system") {
 834         ($C{SDK_PATH}) = (`xcodebuild -version -sdk $C{SDK} Path` =~ /^\s*(.+?)\s*$/);
 835     }
 836
 837     # Look up test library (possible in root or SDK_PATH)
 838
 839     my $rootarg = $root;
 840     my $symroot;
 841     my @sympaths = ( (glob "$root/*~sym")[0],
 842                      (glob "$root/BuildRecords/*_install/Symbols")[0],
 843                      "$root/Symbols" );
 844     my @dstpaths = ( (glob "$root/*~dst")[0],
 845                      (glob "$root/BuildRecords/*_install/Root")[0],
 846                      "$root/Root" );
 847     for(my $i = 0; $i < scalar(@sympaths); $i++) {
 848         if (-e $sympaths[$i]  &&  -e $dstpaths[$i]) {
 849             $symroot = $sympaths[$i];
 850             $root = $dstpaths[$i];
 851             last;
 852         }
 853     }
 854
 855     if ($root ne ""  &&  -e "$root$C{SDK_PATH}$TESTLIBPATH") {
 856         $C{TESTLIB} = "$root$C{SDK_PATH}$TESTLIBPATH";
 857     } elsif (-e "$root$TESTLIBPATH") {
 858         $C{TESTLIB} = "$root$TESTLIBPATH";
 859     } elsif (-e "$root/$TESTLIBNAME") {
 860         $C{TESTLIB} = "$root/$TESTLIBNAME";
 861     } else {
 862         die "No $TESTLIBNAME in root '$rootarg' for sdk '$C{SDK_PATH}'\n"
 863             # . join("\n", @dstpaths) . "\n"
 864             ;
 865     }
 866
 867     if (-e "$symroot/$TESTLIBNAME.dSYM") {
 868         $C{TESTDSYM} = "$symroot/$TESTLIBNAME.dSYM";
 869     }
 870
 871     if ($VERBOSE) {
 872         my @uuids = `/usr/bin/dwarfdump -u '$C{TESTLIB}'`;
 873         while (my $uuid = shift @uuids) {
 874             print "note: $uuid";
 875         }
 876     }
 877
 878     # Look up compilers
 879     my $cc = $C{CC};
 880     my $cxx = cplusplus($C{CC});
 881     if (! $BUILD) {
 882         $C{CC} = $cc;
 883         $C{CXX} = $cxx;
 884     } else {
 885         $C{CC} = find_compiler($cc, $C{SDK}, $C{SDK_PATH});
 886         $C{CXX} = find_compiler($cxx, $C{SDK}, $C{SDK_PATH});
 887
 888         die "No compiler '$cc' ('$C{CC}') in SDK '$C{SDK}'\n" if !-e $C{CC};
 889         die "No compiler '$cxx' ('$C{CXX}') in SDK '$C{SDK}'\n" if !-e $C{CXX};
 890     }
 891
 892     # Populate cflags
 893
 894     # save-temps so dsymutil works so debug info works
 895     my $cflags = "-I$DIR -W -Wall -Wno-deprecated-declarations -Wshorten-64-to-32 -g -save-temps -Os -arch $C{ARCH} ";
 896     my $objcflags = "";
 897
 898     if ($C{SDK} ne "system") {
 899         $cflags .= " -isysroot '$C{SDK_PATH}'";
 900         $cflags .= " '-Wl,-syslibroot,$C{SDK_PATH}'";
 901     }
 902
 903     if ($C{SDK} =~ /^iphoneos[0-9]/  &&  $cflags !~ /-mios-version-min/) {
 904         my ($vers) = ($C{SDK} =~ /^iphoneos([0-9]+\.[0-9+])/);
 905         $cflags .= " -mios-version-min=$vers";
 906     }
 907     if ($C{SDK} =~ /^iphonesimulator[0-9]/  &&  $cflags !~ /-mios-simulator-version-min/) {
 908         my ($vers) = ($C{SDK} =~ /^iphonesimulator([0-9]+\.[0-9+])/);
 909         $cflags .= " -mios-simulator-version-min=$vers";
 910     }
 911     if ($C{SDK} =~ /^iphonesimulator/) {
 912         $objcflags .= " -fobjc-abi-version=2 -fobjc-legacy-dispatch";
 913     }
 914
 915     if ($root ne "") {
 916         my $library_path = dirname($C{TESTLIB});
 917         $cflags .= " -L$library_path";
 918         $cflags .= " -I '$root/usr/include'";
 919         $cflags .= " -I '$root/usr/local/include'";
 920
 921         if ($C{SDK_PATH} ne "/") {
 922             $cflags .= " -I '$root$C{SDK_PATH}/usr/include'";
 923             $cflags .= " -I '$root$C{SDK_PATH}/usr/local/include'";
 924         }
 925     }
 926
 927     if ($C{CC} =~ /clang/) {
 928         $cflags .= " -Qunused-arguments -fno-caret-diagnostics";
 929         $cflags .= " -stdlib=$C{STDLIB} -fno-objc-link-runtime";
 930     }
 931
 932
 933     # Populate objcflags
 934
 935     $objcflags .= " -lobjc";
 936     if ($C{MEM} eq "gc") {
 937         $objcflags .= " -fobjc-gc";
 938     }
 939     elsif ($C{MEM} eq "arc") {
 940         $objcflags .= " -fobjc-arc";
 941     }
 942     elsif ($C{MEM} eq "mrc") {
 943         # nothing
 944     }
 945     else {
 946         die "unrecognized MEM '$C{MEM}'\n";
 947     }
 948
 949     if (supportslibauto($C{SDK})) {
 950         # do this even for non-GC tests
 951         $objcflags .= " -lauto";
 952     }
 953
 954     # Populate ENV_PREFIX
 955     $C{ENV} = "LANG=C MallocScribble=1";
 956     $C{ENV} .= " VERBOSE=1"  if $VERBOSE;
 957     if ($root ne "") {
 958         my $library_path = dirname($C{TESTLIB});
 959         die "no spaces allowed in root" if $library_path =~ /\s+/;
 960         $C{ENV} .= " DYLD_LIBRARY_PATH=$library_path"  if ($library_path ne "/usr/lib");
 961     }
 962     if ($C{SDK_PATH} ne "/") {
 963         die "no spaces allowed in sdk" if $C{SDK_PATH} =~ /\s+/;
 964         $C{ENV} .= " DYLD_ROOT_PATH=$C{SDK_PATH}";
 965     }
 966     if ($C{GUARDMALLOC}) {
 967         $ENV{GUARDMALLOC} = "1";  # checked by tests and errcheck.pl
 968         $C{ENV} .= " DYLD_INSERT_LIBRARIES=/usr/lib/libgmalloc.dylib";
 969     }
 970     if ($C{SDK} =~ /^iphonesimulator[0-9]/) {
 971         my ($vers) = ($C{SDK} =~ /^iphonesimulator([0-9]+\.[0-9+])/);
 972         $C{ENV} .=
 973             " CFFIXED_USER_HOME=$ENV{HOME}/Library/Application\\ Support/iPhone\\ Simulator/$vers" .
 974             " IPHONE_SIMULATOR_ROOT=$C{SDK_PATH}" .
 975             " IPHONE_SHARED_RESOURCES_DIRECTORY=$ENV{HOME}/Library/Application\\ Support/iPhone\\ Simulator/$vers";
 976     }
 977
 978     # Populate compiler commands
 979     $C{COMPILE_C}   = "env LANG=C '$C{CC}'  $cflags -x c -std=gnu99";
 980     $C{COMPILE_CXX} = "env LANG=C '$C{CXX}' $cflags -x c++";
 981     $C{COMPILE_M}   = "env LANG=C '$C{CC}'  $cflags $objcflags -x objective-c -std=gnu99";
 982     $C{COMPILE_MM}  = "env LANG=C '$C{CXX}' $cflags $objcflags -x objective-c++";
 983
 984     $C{COMPILE} = $C{COMPILE_C}    if $C{LANGUAGE} eq "c";
 985     $C{COMPILE} = $C{COMPILE_CXX}  if $C{LANGUAGE} eq "c++";
 986     $C{COMPILE} = $C{COMPILE_M}    if $C{LANGUAGE} eq "objective-c";
 987     $C{COMPILE} = $C{COMPILE_MM}   if $C{LANGUAGE} eq "objective-c++";
 988     die "unknown language '$C{LANGUAGE}'\n" if !defined $C{COMPILE};
 989
 990     ($C{COMPILE_NOMEM} = $C{COMPILE}) =~ s/ -fobjc-(?:gc|arc)\S*//g;
 991     ($C{COMPILE_NOLINK} = $C{COMPILE}) =~ s/ '?-(?:Wl,|l)\S*//g;
 992     ($C{COMPILE_NOLINK_NOMEM} = $C{COMPILE_NOMEM}) =~ s/ '?-(?:Wl,|l)\S*//g;
 993
 994
 995     # Reject some self-inconsistent configurations
 996     if ($C{MEM} !~ /^(mrc|arc|gc)$/) {
 997         die "unknown MEM=$C{MEM} (expected one of mrc arc gc)\n";
 998     }
 999
1000     if ($C{MEM} eq "gc"  &&  $C{SDK} =~ /^iphone/) {
1001         print "note: skipping configuration $C{NAME}\n";
1002         print "note:   because SDK=$C{SDK} does not support MEM=$C{MEM}\n";
1003         return 0;
1004     }
1005     if ($C{MEM} eq "arc"  &&  $C{SDK} !~ /^iphone/  &&  $C{ARCH} eq "i386") {
1006         print "note: skipping configuration $C{NAME}\n";
1007         print "note:   because 32-bit Mac does not support MEM=$C{MEM}\n";
1008         return 0;
1009     }
1010     if ($C{MEM} eq "arc"  &&  $C{CC} !~ /clang/) {
1011         print "note: skipping configuration $C{NAME}\n";
1012         print "note:   because CC=$C{CC} does not support MEM=$C{MEM}\n";
1013         return 0;
1014     }
1015
1016     if ($C{STDLIB} ne "libstdc++"  &&  $C{CC} !~ /clang/) {
1017         print "note: skipping configuration $C{NAME}\n";
1018         print "note:   because CC=$C{CC} does not support STDLIB=$C{STDLIB}\n";
1019         return 0;
1020     }
1021
1022     %$configref = %C;
1023 }
1024
1025 sub make_configs {
1026     my ($root, %args) = @_;
1027
1028     my @results = ({});  # start with one empty config
1029
1030     for my $key (keys %args) {
1031         my @newresults;
1032         my @values = @{$args{$key}};
1033         for my $configref (@results) {
1034             my %config = %{$configref};
1035             for my $value (@values) {
1036                 my %newconfig = %config;
1037                 $newconfig{$key} = $value;
1038                 push @newresults, \%newconfig;
1039             }
1040         }
1041         @results = @newresults;
1042     }
1043
1044     my @newresults;
1045     for my $configref(@results) {
1046         if (make_one_config($configref, $root)) {
1047             push @newresults, $configref;
1048         }
1049     }
1050
1051     return @newresults;
1052 }
1053
1054 sub config_name {
1055     my %config = @_;
1056     my $name = "";
1057     for my $key (sort keys %config) {
1058         $name .= '~'  if $name ne "";
1059         $name .= "$key=$config{$key}";
1060     }
1061     return $name;
1062 }
1063
1064 sub run_one_config {
1065     my %C = %{shift()};
1066     my @tests = @_;
1067
1068     # Build and run
1069     my $testcount = 0;
1070     my $failcount = 0;
1071
1072     my @gathertests;
1073     foreach my $test (@tests) {
1074         if ($VERBOSE) {
1075             print "\nGATHER $test\n";
1076         }
1077
1078         if ($ALL_TESTS{$test}) {
1079             gather_simple(\%C, $test) || next;  # not pass, not fail
1080             push @gathertests, $test;
1081         } else {
1082             die "No test named '$test'\n";
1083         }
1084     }
1085
1086     my @builttests;
1087     if (!$BUILD) {
1088         @builttests = @gathertests;
1089         $testcount = scalar(@gathertests);
1090     } else {
1091         my $configdir = $C{DIR};
1092         print $configdir, "\n"  if $VERBOSE;
1093         mkdir $configdir  || die;
1094
1095         foreach my $test (@gathertests) {
1096             if ($VERBOSE) {
1097                 print "\nBUILD $test\n";
1098             }
1099             mkdir "$configdir/$test.build"  || die;
1100
1101             if ($ALL_TESTS{$test}) {
1102                 $testcount++;
1103                 if (!build_simple(\%C, $test)) {
1104                     $failcount++;
1105                 } else {
1106                     push @builttests, $test;
1107                 }
1108             } else {
1109                 die "No test named '$test'\n";
1110             }
1111         }
1112     }
1113
1114     if (!$RUN  ||  !scalar(@builttests)) {
1115         # nothing to do
1116     }
1117     else {
1118         if ($C{ARCH} =~ /^arm/ && `unamep -p` !~ /^arm/) {
1119             # upload all tests to iOS device
1120             make("RSYNC_PASSWORD=alpine rsync -av $C{DIR} rsync://root\@localhost:10873/root/var/root/test/");
1121             die "Couldn't rsync tests to device\n" if ($?);
1122
1123             # upload library to iOS device
1124             if ($C{TESTLIB} ne $TESTLIBPATH) {
1125                 # hack - send thin library because device may use lib=armv7
1126                 # even though app=armv6, and we want to set the lib's arch
1127                 make("xcrun -sdk $C{SDK} lipo -output /tmp/$TESTLIBNAME -thin $C{ARCH} $C{TESTLIB}  ||  cp $C{TESTLIB} /tmp/$TESTLIBNAME");
1128                 die "Couldn't thin $C{TESTLIB} to $C{ARCH}\n" if ($?);
1129                 make("RSYNC_PASSWORD=alpine rsync -av /tmp/$TESTLIBNAME rsync://root\@localhost:10873/root/var/root/test/");
1130                 die "Couldn't rsync $C{TESTLIB} to device\n" if ($?);
1131                 make("RSYNC_PASSWORD=alpine rsync -av $C{TESTDSYM} rsync://root\@localhost:10873/root/var/root/test/");
1132             }
1133         }
1134
1135         foreach my $test (@builttests) {
1136             print "\nRUN $test\n"  if ($VERBOSE);
1137
1138             if ($ALL_TESTS{$test})
1139             {
1140                 if (!run_simple(\%C, $test)) {
1141                     $failcount++;
1142                 }
1143             } else {
1144                 die "No test named '$test'\n";
1145             }
1146         }
1147     }
1148
1149     return ($testcount, $failcount);
1150 }
1151
1152
1153
1154 # Return value if set by "$argname=value" on the command line
1155 # Return $default if not set.
1156 sub getargs {
1157     my ($argname, $default) = @_;
1158
1159     foreach my $arg (@ARGV) {
1160         my ($value) = ($arg =~ /^$argname=(.+)$/);
1161         return [split ',', $value] if defined $value;
1162     }
1163
1164     return [$default];
1165 }
1166
1167 # Return 1 or 0 if set by "$argname=1" or "$argname=0" on the
1168 # command line. Return $default if not set.
1169 sub getbools {
1170     my ($argname, $default) = @_;
1171
1172     my @values = @{getargs($argname, $default)};
1173     return [( map { ($_ eq "0") ? 0 : 1 } @values )];
1174 }
1175
1176 sub getarg {
1177     my ($argname, $default) = @_;
1178     my @values = @{getargs($argname, $default)};
1179     die "Only one value allowed for $argname\n"  if @values > 1;
1180     return $values[0];
1181 }
1182
1183 sub getbool {
1184     my ($argname, $default) = @_;
1185     my @values = @{getbools($argname, $default)};
1186     die "Only one value allowed for $argname\n"  if @values > 1;
1187     return $values[0];
1188 }
1189
1190
1191 # main
1192 my %args;
1193
1194
1195 my $default_arch = (`/usr/sbin/sysctl hw.optional.x86_64` eq "hw.optional.x86_64: 1\n") ? "x86_64" : "i386";
1196 $args{ARCH} = getargs("ARCH", 0);
1197 $args{ARCH} = getargs("ARCHS", $default_arch)  if !@{$args{ARCH}}[0];
1198
1199 $args{SDK} = getargs("SDK", "system");
1200
1201 $args{MEM} = getargs("MEM", "mrc");
1202 $args{LANGUAGE} = [ map { lc($_) } @{getargs("LANGUAGE", "objective-c")} ];
1203 $args{STDLIB} = getargs("STDLIB", "libstdc++");
1204
1205 $args{CC} = getargs("CC", "clang");
1206
1207 $args{GUARDMALLOC} = getbools("GUARDMALLOC", 0);
1208
1209 $BUILD = getbool("BUILD", 1);
1210 $RUN = getbool("RUN", 1);
1211 $VERBOSE = getbool("VERBOSE", 0);
1212
1213 my $root = getarg("ROOT", "");
1214 $root =~ s#/*$##;
1215
1216 my @tests = gettests();
1217
1218 print "note: -----\n";
1219 print "note: testing root '$root'\n";
1220
1221 my @configs = make_configs($root, %args);
1222
1223 print "note: -----\n";
1224 print "note: testing ", scalar(@configs), " configurations:\n";
1225 for my $configref (@configs) {
1226     my $configname = $$configref{NAME};
1227     print "note: configuration $configname\n";
1228 }
1229
1230 if ($BUILD) {
1231     `rm -rf '$BUILDDIR'`;
1232     mkdir "$BUILDDIR" || die;
1233 }
1234
1235 my $failed = 0;
1236
1237 my $testconfigs = @configs;
1238 my $failconfigs = 0;
1239 my $testcount = 0;
1240 my $failcount = 0;
1241 for my $configref (@configs) {
1242     my $configname = $$configref{NAME};
1243     print "note: -----\n";
1244     print "note: \nnote: $configname\nnote: \n";
1245
1246     (my $t, my $f) = eval { run_one_config($configref, @tests); };
1247     if ($@) {
1248         chomp $@;
1249         print "${red}FAIL: $configname${def}\n";
1250         print "${red}FAIL: $@${def}\n";
1251         $failconfigs++;
1252     } else {
1253         my $color = ($f ? $red : "");
1254         print "note:\n";
1255         print "${color}note: $configname$def\n";
1256         print "${color}note: $t tests, $f failures$def\n";
1257         $testcount += $t;
1258         $failcount += $f;
1259         $failconfigs++ if ($f);
1260     }
1261 }
1262
1263 print "note: -----\n";
1264 my $color = ($failconfigs ? $red : "");
1265 print "${color}note: $testconfigs configurations, $failconfigs with failures$def\n";
1266 print "${color}note: $testcount tests, $failcount failures$def\n";
1267
1268 $failed = ($failconfigs ? 1 : 0);
1269
1270 exit ($failed ? 1 : 0);