testing/build_ninja.py

   1 #!/usr/bin/python2.7
   2
   3 import plistlib
   4 import string
   5 import argparse
   6 import sys
   7 import os
   8 import tempfile
   9 import shutil
  10 import subprocess
  11 import re
  12 import hashlib
  13 import textwrap
  14 from string import Template
  15
  16 class BufferedFile:
  17     def __init__(self, fileName):
  18         self.data = ""
  19         self.fileName = os.path.abspath(fileName)
  20     def __enter__(self):
  21         return self
  22     def __exit__(self, type, value, traceback):
  23         if os.path.exists(self.fileName):
  24             with open(self.fileName, "r") as file:
  25                 fileData = file.read()
  26                 if fileData == self.data: return
  27         else:
  28             dir = os.path.dirname(self.fileName)
  29             if not os.path.exists(dir):
  30                 os.makedirs(dir)
  31         with open(self.fileName, "w") as file:
  32             file.write(self.data)
  33     def write(self, str):
  34         self.data += str
  35
  36 class NinjaFile:
  37     class Variable:
  38         def __init__(self, name, value):
  39             self.name = name
  40             self.value = value
  41         def __lt__(self, other):
  42             return self.name.__lt__(other.name)
  43         def __str__(self):
  44             return NinjaFile.lineWrap("{} = {}".format(self.name, self.value))
  45
  46     class Rule:
  47         def __init__(self, name, command, depfile):
  48             self.name = name
  49             self.command = command
  50             self.depfile = depfile
  51         def __lt__(self, other):
  52             return self.name.__lt__(other.name)
  53         def __str__(self):
  54             result = NinjaFile.lineWrap("rule {}".format(self.name))
  55             if self.command: result += ("\n"+ NinjaFile.lineWrap("    command = {}".format(self.command)))
  56             if self.depfile:
  57                 result += ("\n" + NinjaFile.lineWrap("    deps = gcc"))
  58                 result += ("\n" + NinjaFile.lineWrap("    depfile = {}".format(self.depfile)))
  59             return result
  60     class Target:
  61         def __init__(self, rule):
  62             self.rule = rule
  63             self.output = ""
  64             self.inputs = []
  65             self.variables = []
  66             self.dependencies = []
  67         def __lt__(self, other):
  68             return self.output.__lt__(other.output)
  69         def __str__(self):
  70             self.inputs.sort()
  71             self.variables.sort()
  72             self.dependencies.sort()
  73             buildLine = "build {}: {}".format(self.output, self.rule)
  74             if self.inputs: buildLine += " {}".format(" ".join(self.inputs))
  75             if self.dependencies: buildLine += " | {}".format(" ".join(self.dependencies))
  76             result = NinjaFile.lineWrap(buildLine)
  77             for variable in self.variables: result += ("\n" + NinjaFile.lineWrap("    " + str(variable)))
  78             return result
  79         def addVariable(self, name, value): self.variables.append(NinjaFile.Variable(name, value))
  80         def addDependency(self, dependency):
  81             if isinstance(dependency, str):
  82                 self.dependencies.append(dependency)
  83             elif isinstance(dependency, NinjaFile.Target):
  84                 self.dependencies.append(dependency.output)
  85             else:
  86                 raise ValueError("dependency must be a string or NinjaFile.Target")
  87         def addInput(self, input):
  88             if isinstance(input, str):
  89                 self.inputs.append(input)
  90             elif isinstance(input, NinjaFile.Target):
  91                 self.inputs.append(input.output)
  92             else:
  93                 raise ValueError("input must be a string or NinjaFile.Target")
  94     class Include:
  95         def __init__(self, file):
  96             self.file = file
  97         def __lt__(self, other):
  98             return self.file.__lt__(other.file)
  99         def __str__(self):
 100             return NinjaFile.lineWrap("include {}".format(self.file))
 101
 102     def __init__(self, fileName):
 103         self.fileName = os.path.abspath(fileName)
 104         self.rules = []
 105         self.variables = []
 106         self.targets = []
 107         self.includes = []
 108     def __enter__(self):
 109         return self
 110     def __exit__(self, type, value, traceback):
 111         with BufferedFile(self.fileName) as file:
 112             file.write(str(self))
 113     def addRule(self, name, command, deps): self.rules.append(NinjaFile.Rule(name, command, deps))
 114     def addVariable(self, name, value): self.variables.append(NinjaFile.Variable(name, value))
 115     def addInclude(self, file): self.includes.append(NinjaFile.Include(file))
 116     def newTarget(self, type, name):
 117         target = NinjaFile.Target(type)
 118         target.output = name
 119         self.targets.append(target)
 120         return target
 121     def findTarget(self, name):
 122         #PERF If this gets to be significant we can sort the array and binary search it
 123         for target in self.targets:
 124             if target.output == name: return target
 125         raise ValueError("Target \"{}\"  not found".format(name))
 126     def deleteTarget(self, name): self.targets.remove(self.findTarget(name))
 127     def __str__(self):
 128         self.variables.sort()
 129         self.rules.sort()
 130         self.targets.sort()
 131         self.includes.sort()
 132         subs = {
 133             "VARIABLES": "\n".join(map(str, self.variables)),
 134             "RULES": "\n\n".join(map(str, self.rules)),
 135             "TARGETS": "\n\n".join(map(str, self.targets)),
 136             "INCLUDES": "\n\n".join(map(str, self.includes))
 137         }
 138         return string.Template(
 139 """ninja_required_version = 1.6
 140
 141 $INCLUDES
 142
 143 $VARIABLES
 144
 145 $RULES
 146
 147 $TARGETS
 148
 149 """).safe_substitute(subs)
 150 #    wrapper = textwrap.TextWrapper(width = 130, subsequent_indent = "    ", break_long_words = False)
 151     @classmethod
 152     def lineWrap(cls, line):
 153         if len(line) <= 132: return line
 154         result = ""
 155         currentIdx = 0
 156         wrappedLineLeadingSpace = "  "
 157         firstLineIndent = 0
 158         if line[0].isspace():
 159             firstLineIndent = 4
 160             result = "    "
 161             wrappedLineLeadingSpace = "      "
 162         trailer = " $"
 163         wrappedLineLeadingSpaceLen = len(wrappedLineLeadingSpace)
 164         lineSpaceAvailable = 132-(firstLineIndent+wrappedLineLeadingSpaceLen)
 165         words = line.split()
 166         wordsCount = len(words)-1
 167         for idx, word in enumerate(words):
 168             wordLen = len(word)
 169             if (wordLen <= lineSpaceAvailable and idx == wordsCount):
 170                 result += word
 171             elif wordLen <= lineSpaceAvailable+2:
 172                 result += "{} ".format(word)
 173                 lineSpaceAvailable -= (wordLen)
 174             else:
 175                 result += "$\n{}{} ".format(wrappedLineLeadingSpace, word)
 176                 lineSpaceAvailable = 132-(wrappedLineLeadingSpaceLen+wordLen)
 177         return result
 178
 179 def processBuildLines(ninja, buildLines, testName, platform, osFlag, forceArchs, testDstDir, testSrcDir):
 180     testInstallTarget = ninja.newTarget("phony", "install-{}".format(testName))
 181     testTarget = ninja.newTarget("phony", testName)
 182     ninja.findTarget("all").addInput(testTarget)
 183     ninja.findTarget("install").addInput(testInstallTarget)
 184     for buildLine in buildLines:
 185         minOS = None
 186         args = buildLine.split()
 187         if args[0] == "$DTRACE":
 188             target = None
 189             for idx, arg in enumerate(args):
 190                 if arg == "-o": target = ninja.newTarget("dtrace", args[idx+1])
 191             for idx, arg in enumerate(args):
 192                 if arg == "-s": target.addInput(testSrcDir + "/" + args[idx+1])
 193         elif args[0] == "$CP":
 194             target = ninja.newTarget("cp", args[2])
 195             target.addInput(testSrcDir + "/" + args[1])
 196             testTarget.addInput(target)
 197             installTarget = ninja.newTarget("install", "$INSTALL_DIR/AppleInternal/CoreOS/tests/dyld/{}".format(args[2][9:]))
 198             installTarget.addInput(target.output)
 199             installTarget.addVariable("mode", "0644")
 200             testInstallTarget.addInput(installTarget)
 201         elif args[0] == "$SYMLINK":
 202             target = ninja.newTarget("symlink", args[2])
 203             target.addVariable("source", args[1])
 204             testTarget.addInput(target)
 205             installTarget = ninja.newTarget("symlink", "$INSTALL_DIR/AppleInternal/CoreOS/tests/dyld/{}".format(args[2][9:]))
 206             installTarget.addVariable("source", args[1])
 207             testInstallTarget.addInput(installTarget)
 208         elif args[0] == "$STRIP":
 209             target = ninja.findTarget(args[1])
 210             target.addVariable("extraCmds", "&& strip {}".format(target.output))
 211         elif args[0] == "$SKIP_INSTALL":
 212             target = "$INSTALL_DIR/AppleInternal/CoreOS/tests/dyld/{}".format(args[1][9:])
 213             ninja.deleteTarget(target)
 214             testInstallTarget.inputs.remove(target)
 215         elif args[0] == "$DYLD_ENV_VARS_ENABLE":
 216             if platform != "macos":
 217                 target = ninja.findTarget(args[1])
 218                 target.addVariable("entitlements", "--entitlements $SRCROOT/testing/get_task_allow_entitlement.plist")
 219         elif args[0] == "$TASK_FOR_PID_ENABLE":
 220             target = ninja.findTarget(args[1])
 221             target.addVariable("entitlements", "--entitlements $SRCROOT/testing/task_read_for_pid_entitlement.plist")
 222         elif args[0] in ["$CC", "$CXX"]:
 223             tool = args[0][1:].lower()
 224             sources = []
 225             cflags = []
 226             ldflags = []
 227             dependencies = []
 228             skipCount = 0
 229             linkTarget = None
 230             linkTestSupport = True
 231             platformVersion = None
 232             targetNames = [target.output for target in ninja.targets]
 233             args = [escapedArg.replace("\"", "\\\"") for escapedArg in args[1:]]
 234             #First find the target
 235             for idx, arg in enumerate(args):
 236                 if arg == "-o":
 237                     linkTarget = ninja.newTarget("{}-link".format(tool), args[idx+1])
 238                     linkTarget.addDependency("$BUILT_PRODUCTS_DIR/libtest_support.a")
 239                     testTarget.addInput(linkTarget);
 240                     break
 241             skipCount = 0
 242             for idx, arg in enumerate(args):
 243                 if skipCount: skipCount -= 1
 244                 elif arg == "-o":
 245                     skipCount = 1
 246                 elif arg == "$DEPENDS_ON":
 247                     skipCount = 1
 248                     dependencies.append(args[idx+1])
 249                 elif arg in ["-arch"]:
 250                     skipCount = 1
 251                     nextArg = args[idx+1]
 252                     ldflags.append(arg)
 253                     ldflags.append(nextArg)
 254                     cflags.append(arg)
 255                     cflags.append(nextArg)
 256                 elif arg in ["-target"]:
 257                     skipCount = 1
 258                     nextArg = args[idx+1]
 259                     ldflags.append(arg)
 260                     ldflags.append(nextArg)
 261                     cflags.append(arg)
 262                     cflags.append(nextArg)
 263                 elif arg in ["-install_name","-framework", "-rpath","-compatibility_version","-sub_library", "-undefined", "-current_version"]:
 264                     skipCount = 1
 265                     nextArg = args[idx+1]
 266                     ldflags.append(arg)
 267                     ldflags.append(nextArg)
 268                 elif arg == "-sectcreate":
 269                     skipCount = 3
 270                     ldflags.append(arg)
 271                     ldflags.append(args[idx+1])
 272                     ldflags.append(args[idx+2])
 273                     ldflags.append(args[idx+3])
 274                 elif arg[:2] == "-L": ldflags.append(arg)
 275                 elif arg[:2] == "-F": ldflags.append(arg)
 276                 elif arg == "-nostdlib":
 277                     ldflags.append(arg)
 278                     # Kernel tests pass -nostdlib so don't link test support
 279                     linkTestSupport = False
 280                 elif arg == "-flat_namespace":
 281                     ldflags.append(arg)
 282                 elif arg in ["-dynamiclib","-bundle"]:
 283                     ldflags.append(arg)
 284                     linkTestSupport = False
 285                 elif arg.endswith((".s", ".c", ".cpp", ".cxx", ".m", ".mm")):
 286                     if not arg.startswith("$SRCROOT"): sources.append(testSrcDir + "/" + arg)
 287                     else: sources.append(arg)
 288                 elif arg in targetNames:
 289                     linkTarget.addInput(arg)
 290                 elif osFlag in arg:
 291                     minOS = arg[arg.find('=')+1:]
 292                 elif arg[:4] == "-Wl,":
 293                     linkerArgs = arg.split(",")
 294                     if linkerArgs[1] == "-platform_version":
 295                         minOS = linkerArgs[3]
 296                         platformVersion = arg
 297                     else:
 298                         for linkerArg in linkerArgs[1:]:
 299                             if linkerArg in targetNames: linkTarget.addDependency(linkerArg)
 300                         ldflags.append(arg)
 301                 elif arg[:2] == "-l":
 302                     candidate = "{}/lib{}.dylib".format(testDstDir, arg[2:])
 303                     if candidate in targetNames: linkTarget.addDependency(candidate)
 304                     ldflags.append(arg)
 305                 elif arg[:7] == "-weak-l":
 306                     candidate = "{}/lib{}.dylib".format(testDstDir, arg[7:])
 307                     if candidate in targetNames: linkTarget.addDependency(candidate)
 308                     ldflags.append(arg)
 309                 elif arg[:9] == "-upward-l":
 310                     candidate = "{}/lib{}.dylib".format(testDstDir, arg[9:])
 311                     if candidate in targetNames: linkTarget.addDependency(candidate)
 312                     ldflags.append(arg)
 313                 elif arg[:8] == "-fuse-ld":
 314                     # This is not typically used, but if we ever wanted to try a new ld64, it can be useful
 315                     ldflags.append(arg)
 316                 else:
 317                     cflags.append(arg)
 318             if linkTestSupport:
 319                 ldflags.append("-force_load $BUILT_PRODUCTS_DIR/libtest_support.a")
 320             for source in sources:
 321                 objectHash = hashlib.sha1(linkTarget.output+source+tool+"".join(cflags)).hexdigest()
 322                 target = ninja.newTarget(tool, "$OBJROOT/dyld_tests.build/Objects-normal/" + objectHash + ".o")
 323                 target.addInput(source)
 324                 target.dependencies = dependencies
 325                 if cflags: target.addVariable("cflags", " ".join(cflags))
 326                 if minOS: target.addVariable("minOS", "-" + osFlag + "=" + minOS)
 327                 if forceArchs: target.addVariable("archs", " ".join(["-arch {}".format(arch) for arch in forceArchs]))
 328                 linkTarget.addInput(target)
 329             if ldflags: linkTarget.addVariable("ldflags", " ".join(ldflags))
 330             if platformVersion: linkTarget.addVariable("minOS", platformVersion)
 331             elif minOS: linkTarget.addVariable("minOS", "-" + osFlag + "=" + minOS)
 332             if forceArchs: linkTarget.addVariable("archs", " ".join(["-arch {}".format(arch) for arch in forceArchs]))
 333             installTarget = ninja.newTarget("install", "$INSTALL_DIR/AppleInternal/CoreOS/tests/dyld/{}".format(linkTarget.output[9:]))
 334             installTarget.addInput(linkTarget)
 335             testTarget.addInput(linkTarget)
 336             testInstallTarget.addInput(installTarget)
 337         elif args[0] == "$APP_CACHE_UTIL":
 338             tool = args[0][1:].lower()
 339             sources = []
 340             flags = []
 341             dependencies = []
 342             skipCount = 0
 343             linkTarget = None
 344             args = [escapedArg.replace("\"", "\\\"") for escapedArg in args[1:]]
 345             skipCount = 0
 346             for idx, arg in enumerate(args):
 347                 if skipCount: skipCount -= 1
 348                 elif arg == "$DEPENDS_ON":
 349                     skipCount = 1
 350                     dependencies.append(args[idx+1])
 351                 elif arg == "-create-kernel-collection":
 352                     skipCount = 1
 353                     linkTarget = ninja.newTarget("app-cache-util", args[idx+1])
 354                     linkTarget.addVariable("create_kind", arg)
 355                     testTarget.addInput(linkTarget)
 356                     dependencies.append("$BUILT_PRODUCTS_DIR/host_tools/dyld_app_cache_util")
 357                 elif arg == "-kernel":
 358                     skipCount = 1
 359                     linkTarget.addInput(args[idx+1])
 360                     flags.append(arg)
 361                     flags.append(args[idx+1])
 362                 elif arg == "-create-aux-kernel-collection":
 363                     skipCount = 1
 364                     linkTarget = ninja.newTarget("app-cache-util", args[idx+1])
 365                     linkTarget.addVariable("create_kind", arg)
 366                     testTarget.addInput(linkTarget)
 367                     dependencies.append("$BUILT_PRODUCTS_DIR/host_tools/dyld_app_cache_util")
 368                 elif arg == "-create-pageable-kernel-collection":
 369                     skipCount = 1
 370                     linkTarget = ninja.newTarget("app-cache-util", args[idx+1])
 371                     linkTarget.addVariable("create_kind", arg)
 372                     testTarget.addInput(linkTarget)
 373                     dependencies.append("$BUILT_PRODUCTS_DIR/host_tools/dyld_app_cache_util")
 374                 elif arg == "-kernel-collection":
 375                     skipCount = 1
 376                     linkTarget.addInput(args[idx+1])
 377                     flags.append(arg)
 378                     flags.append(args[idx+1])
 379                 elif arg == "-pageable-collection":
 380                     skipCount = 1
 381                     linkTarget.addInput(args[idx+1])
 382                     flags.append(arg)
 383                     flags.append(args[idx+1])
 384                 else:
 385                     flags.append(arg)
 386             linkTarget.dependencies = dependencies
 387             if flags: linkTarget.addVariable("flags", " ".join(flags))
 388             if forceArchs: linkTarget.addVariable("archs", " ".join(["-arch {}".format(arch) for arch in forceArchs]))
 389             installTarget = ninja.newTarget("install", "$INSTALL_DIR/AppleInternal/CoreOS/tests/dyld/{}".format(linkTarget.output[9:]))
 390             installTarget.addInput(linkTarget)
 391             testTarget.addInput(linkTarget)
 392             testInstallTarget.addInput(installTarget)
 393         else: raise ValueError("Unknown Command: {}".format(args[0]))
 394
 395
 396 def processRunLine(runFile, runLine, environment):
 397     if runLine.startswith("sudo "):
 398         runFile.write("sudo {} {}\n".format(environment, runLine[5:]))
 399     else:
 400         runFile.write("{} {}\n".format(environment, runLine))
 401
 402 def processRunLines(ninja, runLines, testName, platform, runStatic, symRoot, xcTestInvocations):
 403     runFilePath = "{}/{}/run.sh".format(symRoot, testName)
 404     for runLine in runLines:
 405         xcTestInvocations.append("{{ \"{}\", \"{}\" }}".format(testName, runLine.replace("\"","\\\"").replace("sudo","")))
 406     with BufferedFile(runFilePath) as runFile:
 407         runFile.write("#!/bin/sh\n")
 408         runFile.write("cd  {}\n".format(testRunDir))
 409
 410         if runStatic:
 411             runFile.write("echo \"run static\" \n");
 412             for runLine in runLines:
 413                 processRunLine(runFile, runLine, "TEST_OUTPUT=BATS")
 414         else:
 415             runFile.write("echo \"run in dyld2 mode\" \n");
 416             for runLine in runLines:
 417                 processRunLine(runFile, runLine, "TEST_OUTPUT=BATS DYLD_USE_CLOSURES=0")
 418
 419             runFile.write("echo \"run in dyld3 mode\" \n");
 420             for runLine in runLines:
 421                 processRunLine(runFile, runLine, "TEST_OUTPUT=BATS DYLD_USE_CLOSURES=1")
 422
 423             runFile.write("echo \"run in dyld3s mode\" \n");
 424             for runLine in runLines:
 425                 if runLine.startswith("sudo "):
 426                     runFile.write("sudo TEST_OUTPUT=BATS DYLD_USE_CLOSURES=2 {}\n".format(runLine[5:]))
 427                 else:
 428                     runFile.write("TEST_OUTPUT=BATS DYLD_USE_CLOSURES=2 {}\n".format(runLine))
 429     os.chmod(runFilePath, 0755)
 430     installPath = "$INSTALL_DIR/AppleInternal/CoreOS/tests/dyld/{}/run.sh".format(testName)
 431     target = ninja.newTarget("install", installPath)
 432     target.addInput(runFilePath)
 433     ninja.findTarget("install-{}".format(testName)).addInput(installPath)
 434
 435 # returns a tuple of:
 436 # 1. Idx after end of directive
 437 # 2. Set of platforms directive is restricted to
 438 # 3. Bool indicatin if the directive has a platform specifier
 439 def parseDirective(line, directive, platform, archs):
 440     idx = string.find(line, directive)
 441     if idx == -1: return -1, archs, False
 442     if line[idx + len(directive)] == ':': return idx+len(directive)+1, archs, False
 443     match = re.match("\((.*?)\)?(?:\|(.*?))?\:", line[idx + len(directive):]);
 444     if match:
 445         foundPlatform = False
 446         platforms = []
 447         restrictedArchs = []
 448         if match.group(1):
 449             foundPlatform = True
 450             platforms = match.group(1).split(",");
 451         if match.group(2): restrictedArchs = match.group(2).split(",");
 452         if platforms and platform not in platforms: return -1, archs, foundPlatform
 453         effectiveArchs = list(set(archs) & set(restrictedArchs))
 454         if effectiveArchs: return idx + len(directive) + len(match.group()), effectiveArchs, foundPlatform
 455         return line.find(':')+1, archs, foundPlatform
 456     return -1, archs, False
 457
 458 if __name__ == "__main__":
 459     configPath = sys.argv[1]
 460     configMap = {}
 461     with open(configPath) as configFile:
 462         for line in configFile.read().splitlines():
 463             args = line.split()
 464             configMap[args[0]] = args[2:]
 465     sys.stderr.write("CONFIG: {}\n".format(configMap))
 466     srcRoot = configMap["SRCROOT"][0]
 467     symRoot = configMap["SYMROOT"][0]
 468     sdkRoot = configMap["SDKROOT"][0]
 469     objRoot = configMap["OBJROOT"][0]
 470     osFlag = configMap["OSFLAG"][0]
 471     osVers = configMap["OSVERSION"][0]
 472     linkerFlags = configMap["LDFLAGS"][0];
 473     installOwner = configMap["INSTALL_OWNER"][0];
 474     installGroup = configMap["INSTALL_GROUP"][0];
 475     installMode = configMap["INSTALL_MODE_FLAG"][0];
 476     installDir = configMap["INSTALL_DIR"][0];
 477     userHeaderSearchPaths = configMap["USER_HEADER_SEARCH_PATHS"]
 478     systemHeaderSearchPaths = configMap["SYSTEM_HEADER_SEARCH_PATHS"]
 479
 480     derivedFilesDir = configMap["DERIVED_FILES_DIR"][0]
 481     archs = configMap["ARCHS"]
 482
 483     if not os.path.exists(derivedFilesDir): os.makedirs(derivedFilesDir)
 484     if not os.path.exists(objRoot): os.makedirs(objRoot)
 485
 486     sys.stderr.write("srcRoot = {}\n".format(srcRoot))
 487     sys.stderr.write("sdkRoot = {}\n".format(sdkRoot))
 488     sys.stderr.write("objRoot = {}\n".format(objRoot))
 489     sys.stderr.write("osFlag = {}\n".format(osFlag))
 490     sys.stderr.write("osVers = {}\n".format(osVers))
 491     sys.stderr.write("archs = {}\n".format(archs))
 492     sys.stderr.write("derivedFilesDir = {}\n".format(derivedFilesDir))
 493
 494     testSrcRoot = os.path.abspath(srcRoot + "/testing/test-cases")
 495     ccTool = os.popen("xcrun --sdk " + sdkRoot + " --find clang").read().rstrip()
 496     cxxTool = os.popen("xcrun --sdk " + sdkRoot + " --find clang++").read().rstrip()
 497     headerPaths = " -isysroot " + sdkRoot
 498     for headerPath in userHeaderSearchPaths:  headerPaths += " -I{}".format(headerPath)
 499     for headerPath in systemHeaderSearchPaths:  headerPaths += " -I{}".format(headerPath)
 500     sudoCmd = ""
 501     platform = ""
 502     if osFlag == "mmacosx-version-min":
 503         platform = "macos"
 504         sudoCmd = "sudo"
 505     elif osFlag == "miphoneos-version-min": platform = "ios"
 506     elif osFlag == "mtvos-version-min": platform = "tvos"
 507     elif osFlag == "mwatchos-version-min": platform = "watchos"
 508     elif osFlag == "mbridgeos-version-min": platform = "bridgeos"
 509     else:
 510         sys.stderr.write("Unknown platform\n")
 511         sys.exit(-1)
 512
 513     with NinjaFile(derivedFilesDir + "/build.ninja") as ninja:
 514         extraCmds = "$extraCmds"
 515         if "RC_XBS" in os.environ and os.environ["RC_XBS"] == "YES":
 516             extraCmds = "&& dsymutil -o $out.dSYM $out $extraCmds"
 517         ninja.addInclude("config.ninja")
 518         ninja.addVariable("minOS", "-" + osFlag + "=" + osVers)
 519         ninja.addVariable("archs", " ".join(["-arch {}".format(arch) for arch in archs]))
 520         ninja.addVariable("mode", "0755")
 521         ninja.addVariable("headerpaths", headerPaths)
 522
 523         ninja.addRule("cc", "{} -g -MMD -MF $out.d $archs -o $out -c $in $minOS $headerpaths $cflags".format(ccTool), "$out.d")
 524         ninja.addRule("cxx", "{} -g -MMD -MF $out.d $archs -o $out -c $in $minOS $headerpaths $cflags".format(cxxTool), "$out.d")
 525         ninja.addRule("cc-link", "{}  -g $archs -o $out -ltest_support $in $minOS -isysroot {} {} $ldflags {} && codesign --force --sign - $entitlements $out".format(ccTool, sdkRoot, linkerFlags, extraCmds), False)
 526         ninja.addRule("cxx-link", "{}  -g $archs -o $out -ltest_support $in $minOS -isysroot {} {} $ldflags {} && codesign --force --sign - $entitlements $out".format(cxxTool, sdkRoot, linkerFlags, extraCmds), False)
 527         ninja.addRule("app-cache-util", "$BUILT_PRODUCTS_DIR/host_tools/dyld_app_cache_util $archs $create_kind $out $flags", False)
 528         ninja.addRule("dtrace", "/usr/sbin/dtrace -h -s $in -o $out", False)
 529         ninja.addRule("cp", "/bin/cp -p $in $out", False)
 530         ninja.addRule("install", "/usr/bin/install -m $mode -o {} -g {} $install_flags $in $out".format(installOwner, installGroup), False)
 531         ninja.addRule("symlink", "ln -sfh $source $out", False)
 532
 533         allTarget = ninja.newTarget("phony", "all")
 534         masterInstallTarget = ninja.newTarget("phony", "install")
 535
 536         runAllScriptPath = "{}/run_all_dyld_tests.sh".format(derivedFilesDir)
 537         xctestPath = "{}/dyld_xctest.h".format(derivedFilesDir)
 538         if "XCTestGenPath" in os.environ: xctestPath = os.environ["XCTestGenPath"]
 539         batsTests = []
 540         batsSuppressedCrashes = []
 541         xctestInvocations = []
 542         with BufferedFile(runAllScriptPath) as runAllScript:
 543             missingPlatformDirectives = False
 544             runAllScript.write("#!/bin/sh\n")
 545             for entry in os.listdir(testSrcRoot):
 546                 if entry.endswith((".dtest")):
 547                     testName = entry[:-6]
 548                     sys.stdout.write("Processing " + testName + "\n")
 549                     runLines = []
 550                     runStaticLines = []
 551                     buildLines = []
 552
 553                     testSrcDir = "$SRCROOT/testing/test-cases/{}.dtest".format(testName)
 554                     testDstDir = "$SYMROOT/{}".format(testName)
 555                     testRunDir = "/AppleInternal/CoreOS/tests/dyld/{}".format(testName)
 556
 557                     batsTest = {}
 558                     batsTest["TestName"] = testName
 559                     batsTest["Arch"] = "platform-native"
 560                     batsTest["WorkingDirectory"] = testRunDir
 561                     batsTest["ShowSubtestResults"] = True
 562                     batsTest["Command"] = []
 563                     batsTest["Command"].append("./run.sh")
 564                     for file in os.listdir(testSrcRoot + "/" + entry):
 565                         buildSubs = {
 566                             "BUILD_DIR":            testDstDir,
 567                             "RUN_DIR":              testRunDir,
 568                             "SRC_DIR":              testSrcDir
 569                         }
 570                         runSubs = {
 571                             "RUN_DIR":        testRunDir,
 572                             "SUDO":           sudoCmd,
 573                             "RUN_STATIC":     "/AppleInternal/CoreOS/tests/dyld/run-static",
 574                         }
 575                         if file.endswith((".c", ".cpp", ".cxx", ".m", ".mm")):
 576                             with open(testSrcRoot + "/" + entry + "/" + file) as f:
 577                                 requiresPlatformDirective = False
 578                                 foundPlatformDirective = False
 579                                 for line in f.read().splitlines():
 580                                     idx, forceArchs, foundPlatform = parseDirective(line, "BUILD", platform, archs);
 581                                     if foundPlatform: requiresPlatformDirective = True
 582                                     if idx != -1:
 583                                         foundPlatformDirective = True
 584                                         if line[idx:]: buildLines.append(string.Template(line[idx:]).safe_substitute(buildSubs))
 585                                         continue
 586                                     idx, _, _ = parseDirective(line, "RUN", platform, archs);
 587                                     if idx != -1:
 588                                         if "$SUDO" in line: batsTest["AsRoot"] = True
 589                                         runLines.append(string.Template(line[idx:]).safe_substitute(runSubs).lstrip())
 590                                         continue
 591                                     idx, _, _ = parseDirective(line,"RUN_STATIC", platform, archs)
 592                                     if idx != -1:
 593                                         runStaticLines.append(string.Template(line[idx:]).safe_substitute(runSubs).lstrip())
 594                                         continue
 595                                     idx, _, _ = parseDirective(line,"RUN_TIMEOUT", platform, archs)
 596                                     if idx != -1:
 597                                         batsTest["Timeout"] = line[idx:].lstrip()
 598                                         continue
 599                                     idx, _, _ = parseDirective(line,"BOOT_ARGS", platform, archs)
 600                                     if idx != -1:
 601                                         batsTest["BootArgsSet"] = ",".join(line[idx:].split())
 602                                         continue
 603                                     idx, _, _ = parseDirective(line,"NO_CRASH_LOG", platform, archs)
 604                                     if idx != -1:
 605                                         batsSuppressedCrashes.append(line[idx:].lstrip())
 606                                         continue
 607                                 if requiresPlatformDirective and not foundPlatformDirective:
 608                                     missingPlatformDirectives = True
 609                                     sys.stderr.write("Did not find platform({}) BUILD directive for {}\n".format(platform, testName))
 610                     if buildLines and (runLines or runStaticLines):
 611                         processBuildLines(ninja, buildLines, testName, platform, osFlag, forceArchs, testDstDir, testSrcDir)
 612                         if runLines:
 613                             processRunLines(ninja, runLines, testName, platform, False, symRoot, xctestInvocations)
 614                         if runStaticLines:
 615                             processRunLines(ninja, runStaticLines, testName, platform, True, symRoot, xctestInvocations)
 616                         runAllScript.write("/AppleInternal/CoreOS/tests/dyld/{}/run.sh\n".format(testName))
 617                         batsTests.append(batsTest)
 618             if missingPlatformDirectives: sys.exit(-1)
 619             sys.stderr.write("Wrote test config to: {}\n".format(xctestPath))
 620             with BufferedFile(xctestPath) as xcTestFile:
 621                 xcTestFile.write("static const TestInfo sTests[] = {\n")
 622                 xcTestFile.write(",\n".join(xctestInvocations))
 623                 xcTestFile.write("\n};")
 624         os.chmod(runAllScriptPath, 0755)
 625         runAllFilesInstallTarget = ninja.newTarget("install", "$INSTALL_DIR/AppleInternal/CoreOS/tests/dyld/run_all_dyld_tests.sh")
 626         runAllFilesInstallTarget.addInput("$DERIVED_FILES_DIR/run_all_dyld_tests.sh")
 627         masterInstallTarget.addInput(runAllFilesInstallTarget)
 628         batsFilePath = derivedFilesDir + "/dyld.plist"
 629         batsTests.sort(key=lambda test: test["TestName"])
 630         with BufferedFile(batsFilePath) as batsFile:
 631             batsConfig = { "BATSConfigVersion": "0.1.0",
 632                          "Project":           "dyld_tests",
 633                          "Tests":             batsTests }
 634             if batsSuppressedCrashes: batsConfig["IgnoreCrashes"] = batsSuppressedCrashes
 635             batsFile.write(plistlib.writePlistToString(batsConfig))
 636         os.system('plutil -convert binary1 ' + batsFilePath) # convert the plist in place to binary
 637         batsConfigInstallTarget = ninja.newTarget("install", "$INSTALL_DIR/AppleInternal/CoreOS/BATS/unit_tests/dyld.plist")
 638         batsConfigInstallTarget.addInput(batsFilePath)
 639         batsConfigInstallTarget.addVariable("mode", "0644")
 640         masterInstallTarget.addInput(batsConfigInstallTarget)
 641     sys.stdout.write("DONE\n")
 642