testing/build_ninja.py

   1 #!/usr/bin/python2.7
   2
   3 import plistlib
   4 import string
   5 import argparse
   6 import sys
   7 import os
   8 import tempfile
   9 import shutil
  10 import subprocess
  11 import re
  12 import hashlib
  13 import textwrap
  14 from string import Template
  15
  16 class BufferedFile:
  17     def __init__(self, fileName):
  18         self.data = ""
  19         self.fileName = os.path.abspath(fileName)
  20     def __enter__(self):
  21         return self
  22     def __exit__(self, type, value, traceback):
  23         if os.path.exists(self.fileName):
  24             with open(self.fileName, "r") as file:
  25                 fileData = file.read()
  26                 if fileData == self.data: return
  27         else:
  28             dir = os.path.dirname(self.fileName)
  29             if not os.path.exists(dir):
  30                 os.makedirs(dir)
  31         with open(self.fileName, "w") as file:
  32             file.write(self.data)
  33     def write(self, str):
  34         self.data += str
  35
  36 class NinjaFile:
  37     class Variable:
  38         def __init__(self, name, value):
  39             self.name = name
  40             self.value = value
  41         def __lt__(self, other):
  42             return self.name.__lt__(other.name)
  43         def __str__(self):
  44             return NinjaFile.lineWrap("{} = {}".format(self.name, self.value))
  45
  46     class Rule:
  47         def __init__(self, name, command, depfile):
  48             self.name = name
  49             self.command = command
  50             self.depfile = depfile
  51         def __lt__(self, other):
  52             return self.name.__lt__(other.name)
  53         def __str__(self):
  54             result = NinjaFile.lineWrap("rule {}".format(self.name))
  55             if self.command: result += ("\n"+ NinjaFile.lineWrap("    command = {}".format(self.command)))
  56             if self.depfile:
  57                 result += ("\n" + NinjaFile.lineWrap("    deps = gcc"))
  58                 result += ("\n" + NinjaFile.lineWrap("    depfile = {}".format(self.depfile)))
  59             return result
  60     class Target:
  61         def __init__(self, rule):
  62             self.rule = rule
  63             self.output = ""
  64             self.inputs = []
  65             self.variables = []
  66             self.dependencies = []
  67         def __lt__(self, other):
  68             return self.output.__lt__(other.output)
  69         def __str__(self):
  70             self.inputs.sort()
  71             self.variables.sort()
  72             self.dependencies.sort()
  73             buildLine = "build {}: {}".format(self.output, self.rule)
  74             if self.inputs: buildLine += " {}".format(" ".join(self.inputs))
  75             if self.dependencies: buildLine += " | {}".format(" ".join(self.dependencies))
  76             result = NinjaFile.lineWrap(buildLine)
  77             for variable in self.variables: result += ("\n" + NinjaFile.lineWrap("    " + str(variable)))
  78             return result
  79         def addVariable(self, name, value): self.variables.append(NinjaFile.Variable(name, value))
  80         def addDependency(self, dependency):
  81             if isinstance(dependency, str):
  82                 self.dependencies.append(dependency)
  83             elif isinstance(dependency, NinjaFile.Target):
  84                 self.dependencies.append(dependency.output)
  85             else:
  86                 raise ValueError("dependency must be a string or NinjaFile.Target")
  87         def addInput(self, input):
  88             if isinstance(input, str):
  89                 self.inputs.append(input)
  90             elif isinstance(input, NinjaFile.Target):
  91                 self.inputs.append(input.output)
  92             else:
  93                 raise ValueError("input must be a string or NinjaFile.Target")
  94     class Include:
  95         def __init__(self, file):
  96             self.file = file
  97         def __lt__(self, other):
  98             return self.file.__lt__(other.file)
  99         def __str__(self):
 100             return NinjaFile.lineWrap("include {}".format(self.file))
 101
 102     def __init__(self, fileName):
 103         self.fileName = os.path.abspath(fileName)
 104         self.rules = []
 105         self.variables = []
 106         self.targets = []
 107         self.includes = []
 108     def __enter__(self):
 109         return self
 110     def __exit__(self, type, value, traceback):
 111         with BufferedFile(self.fileName) as file:
 112             file.write(str(self))
 113     def addRule(self, name, command, deps): self.rules.append(NinjaFile.Rule(name, command, deps))
 114     def addVariable(self, name, value): self.variables.append(NinjaFile.Variable(name, value))
 115     def addInclude(self, file): self.includes.append(NinjaFile.Include(file))
 116     def newTarget(self, type, name):
 117         target = NinjaFile.Target(type)
 118         target.output = name
 119         self.targets.append(target)
 120         return target
 121     def findTarget(self, name):
 122         #PERF If this gets to be significant we can sort the array and binary search it
 123         for target in self.targets:
 124             if target.output == name: return target
 125         raise ValueError("Target \"{}\"  not found".format(name))
 126     def deleteTarget(self, name): self.targets.remove(self.findTarget(name))
 127     def __str__(self):
 128         self.variables.sort()
 129         self.rules.sort()
 130         self.targets.sort()
 131         self.includes.sort()
 132         subs = {
 133             "VARIABLES": "\n".join(map(str, self.variables)),
 134             "RULES": "\n\n".join(map(str, self.rules)),
 135             "TARGETS": "\n\n".join(map(str, self.targets)),
 136             "INCLUDES": "\n\n".join(map(str, self.includes))
 137         }
 138         return string.Template(
 139 """ninja_required_version = 1.6
 140
 141 $INCLUDES
 142
 143 $VARIABLES
 144
 145 $RULES
 146
 147 $TARGETS
 148
 149 """).safe_substitute(subs)
 150 #    wrapper = textwrap.TextWrapper(width = 130, subsequent_indent = "    ", break_long_words = False)
 151     @classmethod
 152     def lineWrap(cls, line):
 153         if len(line) <= 132: return line
 154         result = ""
 155         currentIdx = 0
 156         wrappedLineLeadingSpace = "  "
 157         firstLineIndent = 0
 158         if line[0].isspace():
 159             firstLineIndent = 4
 160             result = "    "
 161             wrappedLineLeadingSpace = "      "
 162         trailer = " $"
 163         wrappedLineLeadingSpaceLen = len(wrappedLineLeadingSpace)
 164         lineSpaceAvailable = 132-(firstLineIndent+wrappedLineLeadingSpaceLen)
 165         words = line.split()
 166         wordsCount = len(words)-1
 167         for idx, word in enumerate(words):
 168             wordLen = len(word)
 169             if (wordLen <= lineSpaceAvailable and idx == wordsCount):
 170                 result += word
 171             elif wordLen <= lineSpaceAvailable+2:
 172                 result += "{} ".format(word)
 173                 lineSpaceAvailable -= (wordLen)
 174             else:
 175                 result += "$\n{}{} ".format(wrappedLineLeadingSpace, word)
 176                 lineSpaceAvailable = 132-(wrappedLineLeadingSpaceLen+wordLen)
 177         return result
 178
 179 def processBuildLines(ninja, buildLines, testName, platform, osFlag, forceArchs, testDstDir, testSrcDir):
 180     testInstallTarget = ninja.newTarget("phony", "install-{}".format(testName))
 181     testTarget = ninja.newTarget("phony", testName)
 182     ninja.findTarget("all").addInput(testTarget)
 183     ninja.findTarget("install").addInput(testInstallTarget)
 184     for buildLine in buildLines:
 185         minOS = None
 186         args = buildLine.split()
 187         if args[0] == "$DTRACE":
 188             target = None
 189             for idx, arg in enumerate(args):
 190                 if arg == "-o": target = ninja.newTarget("dtrace", args[idx+1])
 191             for idx, arg in enumerate(args):
 192                 if arg == "-s": target.addInput(testSrcDir + "/" + args[idx+1])
 193         elif args[0] == "$CP":
 194             target = ninja.newTarget("cp", args[2])
 195             target.addInput(testSrcDir + "/" + args[1])
 196             testTarget.addInput(target)
 197             installTarget = ninja.newTarget("install", "$INSTALL_DIR/AppleInternal/CoreOS/tests/dyld/{}".format(args[2][9:]))
 198             installTarget.addInput(target.output)
 199             installTarget.addVariable("mode", "0644")
 200             testInstallTarget.addInput(installTarget)
 201         elif args[0] == "$SYMLINK":
 202             target = ninja.newTarget("symlink", args[2])
 203             target.addVariable("source", args[1])
 204             testTarget.addInput(target)
 205             installTarget = ninja.newTarget("symlink", "$INSTALL_DIR/AppleInternal/CoreOS/tests/dyld/{}".format(args[2][9:]))
 206             installTarget.addVariable("source", args[1])
 207             testInstallTarget.addInput(installTarget)
 208         elif args[0] == "$STRIP":
 209             target = ninja.findTarget(args[1])
 210             target.addVariable("extraCmds", "&& strip {}".format(target.output))
 211         elif args[0] == "$SKIP_INSTALL":
 212             target = "$INSTALL_DIR/AppleInternal/CoreOS/tests/dyld/{}".format(args[1][9:])
 213             ninja.deleteTarget(target)
 214             testInstallTarget.inputs.remove(target)
 215         elif args[0] == "$DYLD_ENV_VARS_ENABLE":
 216             if platform != "macos":
 217                 target = ninja.findTarget(args[1])
 218                 target.addVariable("entitlements", "--entitlements $SRCROOT/testing/get_task_allow_entitlement.plist")
 219         elif args[0] == "$TASK_FOR_PID_ENABLE":
 220             if platform != "macos":
 221                 target = ninja.findTarget(args[1])
 222                 target.addVariable("entitlements", "--entitlements $SRCROOT/testing/task_for_pid_entitlement.plist")
 223         elif args[0] in ["$CC", "$CXX"]:
 224             tool = args[0][1:].lower()
 225             sources = []
 226             cflags = []
 227             ldflags = []
 228             dependencies = []
 229             skipCount = 0
 230             linkTarget = None
 231             linkTestSupport = True
 232             platformVersion = None
 233             targetNames = [target.output for target in ninja.targets]
 234             args = [escapedArg.replace("\"", "\\\"") for escapedArg in args[1:]]
 235             #First find the target
 236             for idx, arg in enumerate(args):
 237                 if arg == "-o":
 238                     linkTarget = ninja.newTarget("{}-link".format(tool), args[idx+1])
 239                     linkTarget.addDependency("$BUILT_PRODUCTS_DIR/libtest_support.a")
 240                     testTarget.addInput(linkTarget);
 241                     break
 242             skipCount = 0
 243             for idx, arg in enumerate(args):
 244                 if skipCount: skipCount -= 1
 245                 elif arg == "-o":
 246                     skipCount = 1
 247                 elif arg == "$DEPENDS_ON":
 248                     skipCount = 1
 249                     dependencies.append(args[idx+1])
 250                 elif arg in ["-arch"]:
 251                     skipCount = 1
 252                     nextArg = args[idx+1]
 253                     ldflags.append(arg)
 254                     ldflags.append(nextArg)
 255                     cflags.append(arg)
 256                     cflags.append(nextArg)
 257                 elif arg in ["-target"]:
 258                     skipCount = 1
 259                     nextArg = args[idx+1]
 260                     ldflags.append(arg)
 261                     ldflags.append(nextArg)
 262                     cflags.append(arg)
 263                     cflags.append(nextArg)
 264                 elif arg in ["-install_name","-framework", "-rpath","-compatibility_version","-sub_library", "-undefined", "-current_version"]:
 265                     skipCount = 1
 266                     nextArg = args[idx+1]
 267                     ldflags.append(arg)
 268                     ldflags.append(nextArg)
 269                 elif arg == "-sectcreate":
 270                     skipCount = 3
 271                     ldflags.append(arg)
 272                     ldflags.append(args[idx+1])
 273                     ldflags.append(args[idx+2])
 274                     ldflags.append(args[idx+3])
 275                 elif arg[:2] == "-L": ldflags.append(arg)
 276                 elif arg[:2] == "-F": ldflags.append(arg)
 277                 elif arg == "-nostdlib":
 278                     ldflags.append(arg)
 279                     # Kernel tests pass -nostdlib so don't link test support
 280                     linkTestSupport = False
 281                 elif arg == "-flat_namespace":
 282                     ldflags.append(arg)
 283                 elif arg in ["-dynamiclib","-bundle"]:
 284                     ldflags.append(arg)
 285                     linkTestSupport = False
 286                 elif arg.endswith((".s", ".c", ".cpp", ".cxx", ".m", ".mm")):
 287                     if not arg.startswith("$SRCROOT"): sources.append(testSrcDir + "/" + arg)
 288                     else: sources.append(arg)
 289                 elif arg in targetNames:
 290                     linkTarget.addInput(arg)
 291                 elif osFlag in arg:
 292                     minOS = arg[arg.find('=')+1:]
 293                 elif arg[:4] == "-Wl,":
 294                     linkerArgs = arg.split(",")
 295                     if linkerArgs[1] == "-platform_version":
 296                         minOS = linkerArgs[3]
 297                         platformVersion = arg
 298                     else:
 299                         for linkerArg in linkerArgs[1:]:
 300                             if linkerArg in targetNames: linkTarget.addDependency(linkerArg)
 301                         ldflags.append(arg)
 302                 elif arg[:2] == "-l":
 303                     candidate = "{}/lib{}.dylib".format(testDstDir, arg[2:])
 304                     if candidate in targetNames: linkTarget.addDependency(candidate)
 305                     ldflags.append(arg)
 306                 elif arg[:7] == "-weak-l":
 307                     candidate = "{}/lib{}.dylib".format(testDstDir, arg[7:])
 308                     if candidate in targetNames: linkTarget.addDependency(candidate)
 309                     ldflags.append(arg)
 310                 elif arg[:9] == "-upward-l":
 311                     candidate = "{}/lib{}.dylib".format(testDstDir, arg[9:])
 312                     if candidate in targetNames: linkTarget.addDependency(candidate)
 313                     ldflags.append(arg)
 314                 elif arg[:8] == "-fuse-ld":
 315                     # This is not typically used, but if we ever wanted to try a new ld64, it can be useful
 316                     ldflags.append(arg)
 317                 else:
 318                     cflags.append(arg)
 319             if linkTestSupport:
 320                 ldflags.append("-force_load $BUILT_PRODUCTS_DIR/libtest_support.a")
 321             for source in sources:
 322                 objectHash = hashlib.sha1(linkTarget.output+source+tool+"".join(cflags)).hexdigest()
 323                 target = ninja.newTarget(tool, "$OBJROOT/dyld_tests.build/Objects-normal/" + objectHash + ".o")
 324                 target.addInput(source)
 325                 target.dependencies = dependencies
 326                 if cflags: target.addVariable("cflags", " ".join(cflags))
 327                 if minOS: target.addVariable("minOS", "-" + osFlag + "=" + minOS)
 328                 if forceArchs: target.addVariable("archs", " ".join(["-arch {}".format(arch) for arch in forceArchs]))
 329                 linkTarget.addInput(target)
 330             if ldflags: linkTarget.addVariable("ldflags", " ".join(ldflags))
 331             if platformVersion: linkTarget.addVariable("minOS", platformVersion)
 332             elif minOS: linkTarget.addVariable("minOS", "-" + osFlag + "=" + minOS)
 333             if forceArchs: linkTarget.addVariable("archs", " ".join(["-arch {}".format(arch) for arch in forceArchs]))
 334             installTarget = ninja.newTarget("install", "$INSTALL_DIR/AppleInternal/CoreOS/tests/dyld/{}".format(linkTarget.output[9:]))
 335             installTarget.addInput(linkTarget)
 336             testTarget.addInput(linkTarget)
 337             testInstallTarget.addInput(installTarget)
 338         elif args[0] == "$APP_CACHE_UTIL":
 339             tool = args[0][1:].lower()
 340             sources = []
 341             flags = []
 342             dependencies = []
 343             skipCount = 0
 344             linkTarget = None
 345             args = [escapedArg.replace("\"", "\\\"") for escapedArg in args[1:]]
 346             skipCount = 0
 347             for idx, arg in enumerate(args):
 348                 if skipCount: skipCount -= 1
 349                 elif arg == "$DEPENDS_ON":
 350                     skipCount = 1
 351                     dependencies.append(args[idx+1])
 352                 elif arg == "-create-kernel-collection":
 353                     skipCount = 1
 354                     linkTarget = ninja.newTarget("app-cache-util", args[idx+1])
 355                     linkTarget.addVariable("create_kind", arg)
 356                     testTarget.addInput(linkTarget)
 357                     dependencies.append("$BUILT_PRODUCTS_DIR/host_tools/dyld_app_cache_util")
 358                 elif arg == "-kernel":
 359                     skipCount = 1
 360                     linkTarget.addInput(args[idx+1])
 361                     flags.append(arg)
 362                     flags.append(args[idx+1])
 363                 elif arg == "-create-aux-kernel-collection":
 364                     skipCount = 1
 365                     linkTarget = ninja.newTarget("app-cache-util", args[idx+1])
 366                     linkTarget.addVariable("create_kind", arg)
 367                     testTarget.addInput(linkTarget)
 368                     dependencies.append("$BUILT_PRODUCTS_DIR/host_tools/dyld_app_cache_util")
 369                 elif arg == "-create-pageable-kernel-collection":
 370                     skipCount = 1
 371                     linkTarget = ninja.newTarget("app-cache-util", args[idx+1])
 372                     linkTarget.addVariable("create_kind", arg)
 373                     testTarget.addInput(linkTarget)
 374                     dependencies.append("$BUILT_PRODUCTS_DIR/host_tools/dyld_app_cache_util")
 375                 elif arg == "-kernel-collection":
 376                     skipCount = 1
 377                     linkTarget.addInput(args[idx+1])
 378                     flags.append(arg)
 379                     flags.append(args[idx+1])
 380                 elif arg == "-pageable-collection":
 381                     skipCount = 1
 382                     linkTarget.addInput(args[idx+1])
 383                     flags.append(arg)
 384                     flags.append(args[idx+1])
 385                 else:
 386                     flags.append(arg)
 387             linkTarget.dependencies = dependencies
 388             if flags: linkTarget.addVariable("flags", " ".join(flags))
 389             if forceArchs: linkTarget.addVariable("archs", " ".join(["-arch {}".format(arch) for arch in forceArchs]))
 390             installTarget = ninja.newTarget("install", "$INSTALL_DIR/AppleInternal/CoreOS/tests/dyld/{}".format(linkTarget.output[9:]))
 391             installTarget.addInput(linkTarget)
 392             testTarget.addInput(linkTarget)
 393             testInstallTarget.addInput(installTarget)
 394         else: raise ValueError("Unknown Command: {}".format(args[0]))
 395
 396
 397 def processRunLine(runFile, runLine, environment):
 398     if runLine.startswith("sudo "):
 399         runFile.write("sudo {} {}\n".format(environment, runLine[5:]))
 400     else:
 401         runFile.write("{} {}\n".format(environment, runLine))
 402
 403 def processRunLines(ninja, runLines, testName, platform, runStatic, symRoot, xcTestInvocations):
 404     runFilePath = "{}/{}/run.sh".format(symRoot, testName)
 405     for runLine in runLines:
 406         xcTestInvocations.append("{{ \"{}\", \"{}\" }}".format(testName, runLine.replace("\"","\\\"").replace("sudo","")))
 407     with BufferedFile(runFilePath) as runFile:
 408         runFile.write("#!/bin/sh\n")
 409         runFile.write("cd  {}\n".format(testRunDir))
 410
 411         if runStatic:
 412             runFile.write("echo \"run static\" \n");
 413             for runLine in runLines:
 414                 processRunLine(runFile, runLine, "TEST_OUTPUT=BATS")
 415         else:
 416             runFile.write("echo \"run in dyld2 mode\" \n");
 417             for runLine in runLines:
 418                 processRunLine(runFile, runLine, "TEST_OUTPUT=BATS DYLD_USE_CLOSURES=0")
 419
 420             runFile.write("echo \"run in dyld3 mode\" \n");
 421             for runLine in runLines:
 422                 processRunLine(runFile, runLine, "TEST_OUTPUT=BATS DYLD_USE_CLOSURES=1")
 423
 424             runFile.write("echo \"run in dyld3s mode\" \n");
 425             for runLine in runLines:
 426                 if runLine.startswith("sudo "):
 427                     runFile.write("sudo TEST_OUTPUT=BATS DYLD_USE_CLOSURES=2 {}\n".format(runLine[5:]))
 428                 else:
 429                     runFile.write("TEST_OUTPUT=BATS DYLD_USE_CLOSURES=2 {}\n".format(runLine))
 430     os.chmod(runFilePath, 0755)
 431     installPath = "$INSTALL_DIR/AppleInternal/CoreOS/tests/dyld/{}/run.sh".format(testName)
 432     target = ninja.newTarget("install", installPath)
 433     target.addInput(runFilePath)
 434     ninja.findTarget("install-{}".format(testName)).addInput(installPath)
 435
 436 # returns a tuple of:
 437 # 1. Idx after end of directive
 438 # 2. Set of platforms directive is restricted to
 439 # 3. Bool indicatin if the directive has a platform specifier
 440 def parseDirective(line, directive, platform, archs):
 441     idx = string.find(line, directive)
 442     if idx == -1: return -1, archs, False
 443     if line[idx + len(directive)] == ':': return idx+len(directive)+1, archs, False
 444     match = re.match("\((.*?)\)?(?:\|(.*?))?\:", line[idx + len(directive):]);
 445     if match:
 446         foundPlatform = False
 447         platforms = []
 448         restrictedArchs = []
 449         if match.group(1):
 450             foundPlatform = True
 451             platforms = match.group(1).split(",");
 452         if match.group(2): restrictedArchs = match.group(2).split(",");
 453         if platforms and platform not in platforms: return -1, archs, foundPlatform
 454         effectiveArchs = list(set(archs) & set(restrictedArchs))
 455         if effectiveArchs: return idx + len(directive) + len(match.group()), effectiveArchs, foundPlatform
 456         return len(line), archs, foundPlatform
 457     return -1, archs, False
 458
 459 if __name__ == "__main__":
 460     configPath = sys.argv[1]
 461     configMap = {}
 462     with open(configPath) as configFile:
 463         for line in configFile.read().splitlines():
 464             args = line.split()
 465             configMap[args[0]] = args[2:]
 466     sys.stderr.write("CONFIG: {}\n".format(configMap))
 467     srcRoot = configMap["SRCROOT"][0]
 468     symRoot = configMap["SYMROOT"][0]
 469     sdkRoot = configMap["SDKROOT"][0]
 470     objRoot = configMap["OBJROOT"][0]
 471     osFlag = configMap["OSFLAG"][0]
 472     osVers = configMap["OSVERSION"][0]
 473     linkerFlags = configMap["LDFLAGS"][0];
 474     installOwner = configMap["INSTALL_OWNER"][0];
 475     installGroup = configMap["INSTALL_GROUP"][0];
 476     installMode = configMap["INSTALL_MODE_FLAG"][0];
 477     installDir = configMap["INSTALL_DIR"][0];
 478     userHeaderSearchPaths = configMap["USER_HEADER_SEARCH_PATHS"]
 479     systemHeaderSearchPaths = configMap["SYSTEM_HEADER_SEARCH_PATHS"]
 480
 481     derivedFilesDir = configMap["DERIVED_FILES_DIR"][0]
 482     archs = configMap["ARCHS"]
 483
 484     if not os.path.exists(derivedFilesDir): os.makedirs(derivedFilesDir)
 485     if not os.path.exists(objRoot): os.makedirs(objRoot)
 486
 487     sys.stderr.write("srcRoot = {}\n".format(srcRoot))
 488     sys.stderr.write("sdkRoot = {}\n".format(sdkRoot))
 489     sys.stderr.write("objRoot = {}\n".format(objRoot))
 490     sys.stderr.write("osFlag = {}\n".format(osFlag))
 491     sys.stderr.write("osVers = {}\n".format(osVers))
 492     sys.stderr.write("archs = {}\n".format(archs))
 493     sys.stderr.write("derivedFilesDir = {}\n".format(derivedFilesDir))
 494
 495     testSrcRoot = os.path.abspath(srcRoot + "/testing/test-cases")
 496     ccTool = os.popen("xcrun --sdk " + sdkRoot + " --find clang").read().rstrip()
 497     cxxTool = os.popen("xcrun --sdk " + sdkRoot + " --find clang++").read().rstrip()
 498     headerPaths = " -isysroot " + sdkRoot
 499     for headerPath in userHeaderSearchPaths:  headerPaths += " -I{}".format(headerPath)
 500     for headerPath in systemHeaderSearchPaths:  headerPaths += " -I{}".format(headerPath)
 501     sudoCmd = ""
 502     platform = ""
 503     if osFlag == "mmacosx-version-min":
 504         platform = "macos"
 505         sudoCmd = "sudo"
 506     elif osFlag == "miphoneos-version-min": platform = "ios"
 507     elif osFlag == "mtvos-version-min": platform = "tvos"
 508     elif osFlag == "mwatchos-version-min": platform = "watchos"
 509     elif osFlag == "mbridgeos-version-min": platform = "bridgeos"
 510     else:
 511         sys.stderr.write("Unknown platform\n")
 512         sys.exit(-1)
 513
 514     with NinjaFile(derivedFilesDir + "/build.ninja") as ninja:
 515         extraCmds = "$extraCmds"
 516         if "RC_XBS" in os.environ and os.environ["RC_XBS"] == "YES":
 517             extraCmds = "&& dsymutil -o $out.dSYM $out $extraCmds"
 518         ninja.addInclude("config.ninja")
 519         ninja.addVariable("minOS", "-" + osFlag + "=" + osVers)
 520         ninja.addVariable("archs", " ".join(["-arch {}".format(arch) for arch in archs]))
 521         ninja.addVariable("mode", "0755")
 522         ninja.addVariable("headerpaths", headerPaths)
 523
 524         ninja.addRule("cc", "{} -g -MMD -MF $out.d $archs -o $out -c $in $minOS $headerpaths $cflags".format(ccTool), "$out.d")
 525         ninja.addRule("cxx", "{} -g -MMD -MF $out.d $archs -o $out -c $in $minOS $headerpaths $cflags".format(cxxTool), "$out.d")
 526         ninja.addRule("cc-link", "{}  -g $archs -o $out -ltest_support $in $minOS -isysroot {} {} $ldflags {} && codesign --force --sign - $entitlements $out".format(ccTool, sdkRoot, linkerFlags, extraCmds), False)
 527         ninja.addRule("cxx-link", "{}  -g $archs -o $out -ltest_support $in $minOS -isysroot {} {} $ldflags {} && codesign --force --sign - $entitlements $out".format(cxxTool, sdkRoot, linkerFlags, extraCmds), False)
 528         ninja.addRule("app-cache-util", "$BUILT_PRODUCTS_DIR/host_tools/dyld_app_cache_util $archs $create_kind $out $flags", False)
 529         ninja.addRule("dtrace", "/usr/sbin/dtrace -h -s $in -o $out", False)
 530         ninja.addRule("cp", "/bin/cp -p $in $out", False)
 531         ninja.addRule("install", "/usr/bin/install -m $mode -o {} -g {} $install_flags $in $out".format(installOwner, installGroup), False)
 532         ninja.addRule("symlink", "ln -sfh $source $out", False)
 533
 534         allTarget = ninja.newTarget("phony", "all")
 535         masterInstallTarget = ninja.newTarget("phony", "install")
 536
 537         runAllScriptPath = "{}/run_all_dyld_tests.sh".format(derivedFilesDir)
 538         xctestPath = "{}/dyld_xctest.h".format(derivedFilesDir)
 539         if "XCTestGenPath" in os.environ: xctestPath = os.environ["XCTestGenPath"]
 540         batsTests = []
 541         batsSuppressedCrashes = []
 542         xctestInvocations = []
 543         with BufferedFile(runAllScriptPath) as runAllScript:
 544             missingPlatformDirectives = False
 545             runAllScript.write("#!/bin/sh\n")
 546             for entry in os.listdir(testSrcRoot):
 547                 if entry.endswith((".dtest")):
 548                     testName = entry[:-6]
 549                     sys.stdout.write("Processing " + testName + "\n")
 550                     runLines = []
 551                     runStaticLines = []
 552                     buildLines = []
 553
 554                     testSrcDir = "$SRCROOT/testing/test-cases/{}.dtest".format(testName)
 555                     testDstDir = "$SYMROOT/{}".format(testName)
 556                     testRunDir = "/AppleInternal/CoreOS/tests/dyld/{}".format(testName)
 557
 558                     batsTest = {}
 559                     batsTest["TestName"] = testName
 560                     batsTest["Arch"] = "platform-native"
 561                     batsTest["WorkingDirectory"] = testRunDir
 562                     batsTest["ShowSubtestResults"] = True
 563                     batsTest["Command"] = []
 564                     batsTest["Command"].append("./run.sh")
 565                     for file in os.listdir(testSrcRoot + "/" + entry):
 566                         buildSubs = {
 567                             "BUILD_DIR":            testDstDir,
 568                             "RUN_DIR":              testRunDir,
 569                             "SRC_DIR":              testSrcDir
 570                         }
 571                         runSubs = {
 572                             "RUN_DIR":        testRunDir,
 573                             "SUDO":           sudoCmd,
 574                             "RUN_STATIC":     "/AppleInternal/CoreOS/tests/dyld/run-static",
 575                         }
 576                         if file.endswith((".c", ".cpp", ".cxx", ".m", ".mm")):
 577                             with open(testSrcRoot + "/" + entry + "/" + file) as f:
 578                                 requiresPlatformDirective = False
 579                                 foundPlatformDirective = False
 580                                 for line in f.read().splitlines():
 581                                     idx, forceArchs, foundPlatform = parseDirective(line, "BUILD", platform, archs);
 582                                     if foundPlatform: requiresPlatformDirective = True
 583                                     if idx != -1:
 584                                         foundPlatformDirective = True
 585                                         if line[idx:]: buildLines.append(string.Template(line[idx:]).safe_substitute(buildSubs))
 586                                         continue
 587                                     idx, _, _ = parseDirective(line, "RUN", platform, archs);
 588                                     if idx != -1:
 589                                         if "$SUDO" in line: batsTest["AsRoot"] = True
 590                                         runLines.append(string.Template(line[idx:]).safe_substitute(runSubs).lstrip())
 591                                         continue
 592                                     idx, _, _ = parseDirective(line,"RUN_STATIC", platform, archs)
 593                                     if idx != -1:
 594                                         runStaticLines.append(string.Template(line[idx:]).safe_substitute(runSubs).lstrip())
 595                                         continue
 596                                     idx, _, _ = parseDirective(line,"RUN_TIMEOUT", platform, archs)
 597                                     if idx != -1:
 598                                         batsTest["Timeout"] = line[idx:].lstrip()
 599                                         continue
 600                                     idx, _, _ = parseDirective(line,"BOOT_ARGS", platform, archs)
 601                                     if idx != -1:
 602                                         batsTest["BootArgsSet"] = ",".join(line[idx:].split())
 603                                         continue
 604                                     idx, _, _ = parseDirective(line,"NO_CRASH_LOG", platform, archs)
 605                                     if idx != -1:
 606                                         batsSuppressedCrashes.append(line[idx:].lstrip())
 607                                         continue
 608                                 if requiresPlatformDirective and not foundPlatformDirective:
 609                                     missingPlatformDirectives = True
 610                                     sys.stderr.write("Did not find platform({}) BUILD directive for {}\n".format(platform, testName))
 611                     if buildLines and (runLines or runStaticLines):
 612                         processBuildLines(ninja, buildLines, testName, platform, osFlag, forceArchs, testDstDir, testSrcDir)
 613                         if runLines:
 614                             processRunLines(ninja, runLines, testName, platform, False, symRoot, xctestInvocations)
 615                         if runStaticLines:
 616                             processRunLines(ninja, runStaticLines, testName, platform, True, symRoot, xctestInvocations)
 617                         runAllScript.write("/AppleInternal/CoreOS/tests/dyld/{}/run.sh\n".format(testName))
 618                         batsTests.append(batsTest)
 619             if missingPlatformDirectives: sys.exit(-1)
 620             sys.stderr.write("Wrote test config to: {}\n".format(xctestPath))
 621             with BufferedFile(xctestPath) as xcTestFile:
 622                 xcTestFile.write("static const TestInfo sTests[] = {\n")
 623                 xcTestFile.write(",\n".join(xctestInvocations))
 624                 xcTestFile.write("\n};")
 625         os.chmod(runAllScriptPath, 0755)
 626         runAllFilesInstallTarget = ninja.newTarget("install", "$INSTALL_DIR/AppleInternal/CoreOS/tests/dyld/run_all_dyld_tests.sh")
 627         runAllFilesInstallTarget.addInput("$DERIVED_FILES_DIR/run_all_dyld_tests.sh")
 628         masterInstallTarget.addInput(runAllFilesInstallTarget)
 629         batsFilePath = derivedFilesDir + "/dyld.plist"
 630         batsTests.sort(key=lambda test: test["TestName"])
 631         with BufferedFile(batsFilePath) as batsFile:
 632             batsConfig = { "BATSConfigVersion": "0.1.0",
 633                          "Project":           "dyld_tests",
 634                          "Tests":             batsTests }
 635             if batsSuppressedCrashes: batsConfig["IgnoreCrashes"] = batsSuppressedCrashes
 636             batsFile.write(plistlib.writePlistToString(batsConfig))
 637         os.system('plutil -convert binary1 ' + batsFilePath) # convert the plist in place to binary
 638         batsConfigInstallTarget = ninja.newTarget("install", "$INSTALL_DIR/AppleInternal/CoreOS/BATS/unit_tests/dyld.plist")
 639         batsConfigInstallTarget.addInput(batsFilePath)
 640         batsConfigInstallTarget.addVariable("mode", "0644")
 641         masterInstallTarget.addInput(batsConfigInstallTarget)
 642     sys.stdout.write("DONE\n")
 643