testing/build_tests.py

   1 #!/usr/bin/python2.7
   2
   3 import plistlib
   4 import string
   5 import argparse
   6 import sys
   7 import os
   8 import tempfile
   9 import shutil
  10 import subprocess
  11 import uuid
  12
  13
  14 #
  15 # Scan files in .dtest directory looking for BUILD: or RUN: directives.
  16 # Return a dictionary of all directives.
  17 #
  18 def parseDirectives(testCaseSourceDir):
  19     onlyLines = []
  20     buildLines = []
  21     extractLines = []
  22     runLines = []
  23     minOS = ""
  24     timeout = ""
  25     noCrashLogs = []
  26     bootArgs = []
  27     for file in os.listdir(testCaseSourceDir):
  28         if file.endswith((".c", ".cpp", ".cxx", ".m", ".mm")):
  29             with open(testCaseSourceDir + "/" + file) as f:
  30                 for line in f.read().splitlines():
  31                     buildIndex = string.find(line, "BUILD:")
  32                     if buildIndex != -1:
  33                         buildLines.append(line[buildIndex + 6:].lstrip())
  34                     runIndex = string.find(line, "RUN:")
  35                     if runIndex != -1:
  36                         runLines.append(line[runIndex+4:].lstrip())
  37                     onlyIndex = string.find(line, "BUILD_ONLY:")
  38                     if onlyIndex != -1:
  39                         onlyLines.append(line[onlyIndex+11:].lstrip())
  40                     minOsIndex = string.find(line, "BUILD_MIN_OS:")
  41                     if minOsIndex != -1:
  42                         minOS = line[minOsIndex+13:].lstrip()
  43                     timeoutIndex = string.find(line, "RUN_TIMEOUT:")
  44                     if timeoutIndex != -1:
  45                         timeout = line[timeoutIndex+12:].lstrip()
  46                     noCrashLogsIndex = string.find(line, "NO_CRASH_LOG:")
  47                     if noCrashLogsIndex != -1:
  48                         noCrashLogs.append(line[noCrashLogsIndex+13:].lstrip())
  49                     bootArgsIndex = string.find(line, "BOOT_ARGS:")
  50                     if bootArgsIndex != -1:
  51                         bootArgs.append(line[bootArgsIndex+10:].lstrip())
  52     return {
  53         "BUILD":        buildLines,
  54         "BUILD_ONLY":   onlyLines,
  55         "BUILD_MIN_OS": minOS,
  56         "RUN":          runLines,
  57         "RUN_TIMEOUT":  timeout,
  58         "NO_CRASH_LOG": noCrashLogs,
  59         "BOOT_ARGS":    bootArgs,
  60    }
  61
  62
  63 #
  64 # Look at directives dictionary to see if this test should be skipped for this platform
  65 #
  66 def useTestCase(testName, testCaseDirectives, platformName):
  67     onlyLines = testCaseDirectives["BUILD_ONLY"]
  68     for only in onlyLines:
  69         if only == "MacOSX" and platformName != "macosx":
  70             return False
  71         if only == "iOS" and platformName != "iphoneos":
  72             return False
  73     return True
  74
  75
  76 #
  77 # Use BUILD directives to construct the test case
  78 # Use RUN directives to generate a shell script to run test(s)
  79 #
  80 def buildTestCase(testCaseDirectives, testCaseSourceDir, toolsDir, sdkDir, dyldIncludesDir, minOsOptionsName, defaultMinOS, archOptions, testCaseDestDirBuild, testCaseDestDirRun, plistDir):
  81     scratchDir = tempfile.mkdtemp()
  82     if testCaseDirectives["BUILD_MIN_OS"]:
  83         minOS = testCaseDirectives["BUILD_MIN_OS"]
  84     else:
  85         minOS = defaultMinOS
  86     compilerSearchOptions = " -isysroot " + sdkDir + " -I" + sdkDir + "/System/Library/Frameworks/System.framework/PrivateHeaders" + " -I" + dyldIncludesDir + " -I" + testsSrcTopDir + "../include/"
  87     defines = " -DINSTALL_PATH=\"" + testCaseDestDirRun + "\""
  88     if minOsOptionsName == "mmacosx-version-min":
  89         taskForPidCommand = "touch "
  90         envEnableCommand  = "touch "
  91     else:
  92         taskForPidCommand = "codesign --force --sign - --entitlements " + testCaseSourceDir + "/../../task_for_pid_entitlement.plist "
  93         envEnableCommand  = "codesign --force --sign - --entitlements " + testCaseSourceDir + "/../../get_task_allow_entitlement.plist "
  94     buildSubs = {
  95         "CC":                   toolsDir + "/usr/bin/clang "   + archOptions + " -" + minOsOptionsName + "=" + str(minOS) + compilerSearchOptions + defines,
  96         "CXX":                  toolsDir + "/usr/bin/clang++ " + archOptions + " -" + minOsOptionsName + "=" + str(minOS) + compilerSearchOptions + defines,
  97         "BUILD_DIR":            testCaseDestDirBuild,
  98         "RUN_DIR":              testCaseDestDirRun,
  99         "TEMP_DIR":             scratchDir,
 100         "TASK_FOR_PID_ENABLE":  taskForPidCommand,
 101         "DYLD_ENV_VARS_ENABLE": envEnableCommand
 102     }
 103     os.makedirs(testCaseDestDirBuild)
 104     os.chdir(testCaseSourceDir)
 105     outputfiles = []
 106     alreadySigned = []
 107     print >> sys.stderr, "cd " + testCaseSourceDir
 108     for line in testCaseDirectives["BUILD"]:
 109         cmd = string.Template(line).safe_substitute(buildSubs)
 110         if "codesign" in cmd:
 111             alreadySigned.append(string.split(cmd).pop())
 112         print >> sys.stderr, cmd
 113         if "&&" in cmd:
 114             result = subprocess.call(cmd, shell=True)
 115         else:
 116             cmdList = []
 117             cmdList = string.split(cmd)
 118             result = subprocess.call(cmdList)
 119         if result:
 120             return result
 121         args = cmd.split()
 122         for index, arg in enumerate(args):
 123             if arg == "-o":
 124                 outputfiles.append(args[index+1])
 125                 break
 126     print >> sys.stderr, "outfiles: " +  ' '.join(outputfiles)  + "already signed: " + ' '.join(alreadySigned)
 127     for outfile in outputfiles:
 128         if outfile not in alreadySigned:
 129             cmd = "codesign --force --sign - " + outfile
 130             print >> sys.stderr, cmd
 131             subprocess.call(string.split(cmd))
 132     shutil.rmtree(scratchDir, ignore_errors=True)
 133     sudoSub = ""
 134     if minOsOptionsName == "mmacosx-version-min":
 135         sudoSub = "sudo"
 136     runSubs = {
 137         "RUN_DIR":        testCaseDestDirRun,
 138         "REQUIRE_CRASH":  "nocr -require_crash",
 139         "SUDO":           sudoSub,
 140     }
 141     runFilePath = testCaseDestDirBuild + "/run.sh"
 142     with open(runFilePath, "a") as runFile:
 143         runFile.write("#!/bin/sh\n")
 144         runFile.write("cd " + testCaseDestDirRun + "\n")
 145         os.chmod(runFilePath, 0755)
 146
 147         runFile.write("echo \"run in dyld2 mode\" \n");
 148         for runline in testCaseDirectives["RUN"]:
 149           subLine = string.Template(runline).safe_substitute(runSubs)
 150           subLine = "TEST_DYLD_MODE=2 DYLD_USE_CLOSURES=0 " + subLine
 151           runFile.write(subLine + "\n")
 152
 153         if minOsOptionsName == "mmacosx-version-min":
 154             runFile.write("echo \"run in dyld2 mode with no shared cache\" \n");
 155             for runline in testCaseDirectives["RUN"]:
 156                 subLine = string.Template(runline).safe_substitute(runSubs)
 157                 subLine = "TEST_DYLD_MODE=2 DYLD_SHARED_REGION=avoid " + subLine
 158                 runFile.write(subLine + "\n")
 159
 160         runFile.write("echo \"run in dyld3 mode\" \n");
 161         for runline in testCaseDirectives["RUN"]:
 162             subLine = string.Template(runline).safe_substitute(runSubs)
 163             if subLine.startswith("sudo "):
 164                 subLine = "sudo TEST_DYLD_MODE=3 DYLD_USE_CLOSURES=1 " + subLine[5:]
 165             else:
 166                 subLine = "TEST_DYLD_MODE=3 DYLD_USE_CLOSURES=1 " + subLine
 167             runFile.write(subLine + "\n")
 168
 169         if minOsOptionsName == "mmacosx-version-min":
 170             runFile.write("echo \"run in dyld3 mode with no shared cache\" \n");
 171             for runline in testCaseDirectives["RUN"]:
 172                 subLine = string.Template(runline).safe_substitute(runSubs)
 173                 if subLine.startswith("sudo "):
 174                     subLine = "sudo TEST_DYLD_MODE=3 DYLD_SHARED_REGION=avoid DYLD_USE_CLOSURES=1 " + subLine[5:]
 175                 else:
 176                     subLine = "TEST_DYLD_MODE=3 DYLD_SHARED_REGION=avoid DYLD_USE_CLOSURES=1 " + subLine
 177                 runFile.write(subLine + "\n")
 178
 179         runFile.write("\n")
 180         runFile.close()
 181     for runline in testCaseDirectives["RUN"]:
 182         runTarget = runline.split().pop()
 183         os.system("xcrun dt_extractmeta extract -i " + testCaseDestDirRun + "/" + runTarget + " -b " + testCaseDestDirBuild + "/" + runTarget + " -o " + plistDir + "/" + str(uuid.uuid4()) + ".plist 2> /dev/null")
 184     return 0
 185
 186
 187
 188 #
 189 # Use XCode build settings to build all unit tests for specified platform
 190 # Generate a .plist for BATS to use to run all tests
 191 #
 192 if __name__ == "__main__":
 193     dstDir = os.getenv("DSTROOT", "/tmp/dyld_tests/")
 194     testsRunDstTopDir = "/AppleInternal/CoreOS/tests/dyld/"
 195     testsBuildDstTopDir = dstDir + testsRunDstTopDir
 196     # If we want to run directly from the dstroot then override that now
 197     runFromDstRoot = os.getenv("RUN_FROM_DSTROOT", "")
 198     if runFromDstRoot:
 199         testsRunDstTopDir = testsBuildDstTopDir
 200     shutil.rmtree(testsBuildDstTopDir, ignore_errors=True)
 201     dyldSrcDir = os.getenv("SRCROOT", "")
 202     if not dyldSrcDir:
 203         dyldSrcDir = os.getcwd()
 204     testsSrcTopDir = dyldSrcDir + "/testing/test-cases/"
 205     dyldIncludesDir = dyldSrcDir + "/include/"
 206     sdkDir = os.getenv("SDKROOT", "")
 207     if not sdkDir:
 208         sdkDir = subprocess.check_output(["xcrun", "-sdk", "macosx.internal", "--show-sdk-path"]).rstrip()
 209     toolsDir = os.getenv("TOOLCHAIN_DIR", "/")
 210     defaultMinOS = ""
 211     minVersNum = "10.14"
 212     minOSOption = os.getenv("DEPLOYMENT_TARGET_CLANG_FLAG_NAME", "")
 213     if minOSOption:
 214         minOSVersName = os.getenv("DEPLOYMENT_TARGET_CLANG_ENV_NAME", "")
 215         if minOSVersName:
 216             minVersNum = os.getenv(minOSVersName, "")
 217     else:
 218         minOSOption = "mmacosx-version-min"
 219     platformName = os.getenv("PLATFORM_NAME", "macosx")
 220     archOptions = ""
 221     archList = os.getenv("RC_ARCHS", "")
 222     if archList:
 223         for arch in string.split(archList, " "):
 224             archOptions = archOptions + " -arch " + arch
 225     else:
 226         if platformName == "watchos":
 227             archOptions = "-arch armv7k"
 228         elif platformName == "appletvos":
 229             archOptions = "-arch arm64"
 230         elif platformName == "macosx":
 231             archList = os.getenv("ARCHS_STANDARD_64_BIT", "")
 232             if archList:
 233                 for arch in string.split(archList, " "):
 234                     archOptions = archOptions + " -arch " + arch
 235             else:
 236                 archOptions = "-arch x86_64"
 237         else:
 238             archList = os.getenv("ARCHS_STANDARD_32_64_BIT", "")
 239             if archList:
 240                 for arch in string.split(archList, " "):
 241                     archOptions = archOptions + " -arch " + arch
 242             else:
 243                 archOptions = "-arch x86_64"
 244     allTests = []
 245     suppressCrashLogs = []
 246     plistDir = tempfile.mkdtemp()
 247     for f in sorted(os.listdir(testsSrcTopDir)):
 248         if f.endswith(".dtest"):
 249             testName = f[0:-6]
 250             outDirBuild = testsBuildDstTopDir + testName
 251             outDirRun = testsRunDstTopDir + testName
 252             testCaseDir = testsSrcTopDir + f
 253             onlyTestDir = os.getenv("ONLY_BUILD_TEST", "")
 254             if onlyTestDir:
 255                 if onlyTestDir != testName:
 256                     continue
 257                 print >> sys.stderr, "Going to build " + testName
 258             testCaseDirectives = parseDirectives(testCaseDir)
 259             if useTestCase(testName, testCaseDirectives, platformName):
 260                 result = buildTestCase(testCaseDirectives, testCaseDir, toolsDir, sdkDir, dyldIncludesDir, minOSOption, minVersNum, archOptions, outDirBuild, outDirRun, plistDir)
 261                 if result:
 262                     sys.exit(result)
 263                 mytest = {}
 264                 mytest["TestName"] = testName
 265                 mytest["Arch"] = "platform-native"
 266                 mytest["WorkingDirectory"] = testsRunDstTopDir + testName
 267                 mytest["Command"] = []
 268                 mytest["Command"].append("./run.sh")
 269                 for runline in testCaseDirectives["RUN"]:
 270                     if "$SUDO" in runline:
 271                         mytest["AsRoot"] = True
 272                 if testCaseDirectives["RUN_TIMEOUT"]:
 273                     mytest["Timeout"] = testCaseDirectives["RUN_TIMEOUT"]
 274                 if testCaseDirectives["BOOT_ARGS"]:
 275                     mytest["BootArgsSet"] = ",".join(testCaseDirectives["BOOT_ARGS"]);
 276                 allTests.append(mytest)
 277             if testCaseDirectives["NO_CRASH_LOG"]:
 278                 for skipCrash in testCaseDirectives["NO_CRASH_LOG"]:
 279                     suppressCrashLogs.append(skipCrash)
 280     batsInfo = { "BATSConfigVersion": "0.1.0",
 281                  "Project":           "dyld_tests",
 282                  "Tests":             allTests }
 283     if suppressCrashLogs:
 284         batsInfo["IgnoreCrashes"] = []
 285         for skipCrash in suppressCrashLogs:
 286             batsInfo["IgnoreCrashes"].append(skipCrash)
 287     batsFileDir = dstDir + "/AppleInternal/CoreOS/BATS/unit_tests/"
 288     shutil.rmtree(batsFileDir, ignore_errors=True)
 289     os.makedirs(batsFileDir)
 290     batsFilePath = batsFileDir + "dyld.plist"
 291     with open(batsFilePath, "w") as batsFile:
 292         batsFile.write(plistlib.writePlistToString(batsInfo))
 293         batsFile.close()
 294     os.system('plutil -convert binary1 ' + batsFilePath) # convert the plist in place to binary
 295     runHelper = dstDir + "/AppleInternal/CoreOS/tests/dyld/run_all_dyld_tests.sh"
 296     print runHelper
 297     with open(runHelper, "w") as shFile:
 298         shFile.write("#!/bin/sh\n")
 299         for test in allTests:
 300             shFile.write(test["WorkingDirectory"] + "/run.sh\n")
 301         shFile.close()
 302     os.chmod(runHelper, 0755)
 303     if not os.path.exists(dstDir + "/AppleInternal/CoreOS/tests/metadata/dyld/"): os.makedirs(dstDir + "/AppleInternal/CoreOS/tests/metadata/dyld/")
 304     os.system("xcrun dt_extractmeta merge -o " + dstDir + "/AppleInternal/CoreOS/tests/metadata/dyld/dyld.plist " + plistDir + "/*")
 305 #   FIXME: Enable this once all tests move to darwintest
 306 #    os.system("xcrun dt_convertmeta " + dstDir + "/AppleInternal/CoreOS/BATS/unit_tests/dyld.plist dyld_tests " + dstDir + "/AppleInternal/CoreOS/tests/metadata/dyld/dyld.plist")
 307     shutil.rmtree(plistDir, ignore_errors=True)
 308