tools/tests/unit_tests/xnu_raft_tests.py

   1 #!/usr/bin/python
   2 # -*- coding: utf-8 -*-
   3 #
   4 # Python Imports
   5 import os
   6 import sys
   7 import re
   8
   9 """
  10 xnu_raft_tests
  11 Automate testing of unit tests for xnu.
  12
  13 2012/02/23
  14 """
  15
  16 # this needs to be first thing for raft to load its environment correctly
  17 if __name__ == '__main__':
  18         # The following code allows this test to be invoked outside the harness and should be left unchanged
  19         args = [os.path.realpath(os.path.expanduser("/usr/local/bin/raft")), "-f"] + sys.argv
  20         os.execv(args[0], args)
  21
  22
  23 # Library Imports
  24 from raftlibs.coreos import crashReporterStop, crashReporterStart, doPrivileged, runFunctionWithTestReRun
  25 from raftlibs.coreos import runUniversalLogProcess, spotlightStopSubtest, spotlightStartSubtest, svnCheckoutTestTool, svnCheckoutToPath, runSimpleProcess
  26
  27 from raft.core.logging import log_note
  28
  29 # Raft Imports
  30 from __test__ import __path__
  31
  32 # This is a Raft test. For more information see http://raft.apple.com
  33 testDescription  = "Runs all tests defined as targets in Makefile"                 # Add a brief description of test functionality
  34 testVersion      = "0.1"              # Used to differentiate between results for different versions of the test
  35 testState        = DevelopmentState   # Possible values: DevelopmentState, ProductionState
  36
  37
  38 # class definitions
  39 class xnuTest:
  40         """ A container to hold test and its result """
  41         def __init__(self,testName):
  42                 self.name = str(testName)
  43                 self.buildStatus = False
  44                 self.executeStatus = False
  45                 self.exitValue = None
  46                 self.comments = ''
  47
  48         def getName(self):
  49                 return self.name
  50
  51         @staticmethod
  52         def getSummaryHeader():
  53                 return "| {0: ^40s} |{1: >6s} |{2: >5s} |{3: >10s} |{4}".format("Test Name", "Build", "Run", "ExitVal", "Comments")
  54
  55         def getSummary(self):
  56                 formatString ="| {0: <40s} |{1: >6s} |{2: >5s} |{3: >10s} |{4}"
  57                 nameVal = str(self.name)
  58                 buildVal = str(self.buildStatus)
  59                 execVal = str(self.executeStatus)
  60                 exitVal = str(self.exitValue)
  61                 commentsVal = str(self.comments)
  62                 return formatString.format(nameVal, buildVal, execVal, exitVal, commentsVal)
  63
  64 # global functions
  65 def getTestsFromMakeFile(makeFilePath):
  66         makeTargets=[]
  67         targetRegex = re.compile("^\s*([a-zA-Z0-9_.]+)\s*:\s*([a-zA-Z0-9_.]*).*",re.IGNORECASE|re.DOTALL)
  68         fh = open(makeFilePath,"r");
  69         for line in fh:
  70                 tmp_res = targetRegex.findall(line)
  71                 if len(tmp_res) == 1:
  72                         makeTargets.append(xnuTest(tmp_res[0][0]))
  73         fh.close()
  74         return makeTargets
  75
  76
  77 def buildTest(test, path):
  78         os.chdir(path)
  79         result = doCommand("/usr/bin/make",test)
  80         if result['status'] != 0:
  81                 print "Failed to Build %s" % test
  82                 print "**STDOUT**\n%s" % result['stdout']
  83                 print "**STDERR**\n%s" % result['stderr']
  84                 raise StandardError
  85         log_note("Built %s successfully" % test)
  86
  87 def executeTest(testObject,path):
  88         os.chdir(path)
  89         test = testObject.getName()
  90         executable_path = os.path.join(path, test)
  91         print "[TEST] %s" % test
  92         print "[BEGIN] %s" % test
  93         try:
  94                 result = runSimpleProcess(executable_path,testName()+"_"+test, wait_time=120)
  95                 testObject.exitValue = result['status']
  96                 if result['status'] == 0:
  97                         print "[PASS] %s returned %d" % (test,result['status'])
  98         except:
  99                 print "[FAIL] %s returned %d" % (test, result['status'])
 100                 testObject.comments = "Failed due to timeout or file not found error"
 101         log_note("Completed running test %s" % test)
 102
 103 def removeTestExecutable(test,path):
 104         os.chdir(path)
 105         doCommand("/bin/rm",test)
 106
 107 def runTest(params):
 108         # Change to /tmp, because make doesn't support directory paths with spaces
 109         os.chdir("/private/tmp")
 110         output= {'status': 1 }
 111         try:
 112                 output = svnCheckoutTestTool("unit_tests")
 113         except:
 114                 pass
 115         if output['status'] != 0 :
 116                 # since we are not fully published yet. lets get data from a branch
 117                 print "Fetching unit_test roots from Branch instead of trunk"
 118                 baseURL = "http://src.apple.com/svn/xnu/branches/PR-10938974/tools/tests/unit_tests/"
 119                 output = svnCheckoutToPath(baseURL)
 120                 if output['status'] != 0 :
 121                         logFail("[FAIL] error in checkout from branch")
 122                         sys.exit(1)
 123
 124         local_path = os.path.join(os.getcwd(), "unit_tests")
 125         makefile_path = os.path.join(local_path, "Makefile")
 126         build_path = os.path.join(local_path, "BUILD")
 127
 128
 129         tests_to_run = getTestsFromMakeFile(makefile_path)
 130         log_note("Starting raft tests for XNU")
 131         stats = {"total":len(tests_to_run) , "pass":0, "fail":0}
 132         for testObject in tests_to_run:
 133                 test = testObject.getName()
 134                 if test == "clean":
 135                         stats["pass"]+=1
 136                         testObject.buildStatus = True
 137                         testObject.executeStatus = True
 138                         testObject.exitValue = 0
 139                         continue
 140
 141                 log_note("Running test :%s" % test)
 142                 try:
 143                         buildTest(test,local_path)
 144                         testObject.buildStatus = True
 145                         res = executeTest(testObject,build_path)
 146                         testObject.executeStatus = True
 147                         if testObject.exitValue == 0 :
 148                                 stats["pass"]+=1
 149                         else:
 150                                 stats["fail"]+=1
 151                         removeTestExecutable(test,build_path)
 152                         logPass(test)
 153                 except:
 154                         logFail("[FAIL] %s failed." % test)
 155         print "Finished running tests. Cleaning up"
 156         doCommand("/usr/bin/make","clean")
 157         #Now to print the Summary and statistics
 158         print "\n\n Test Summary \n"
 159         print xnuTest.getSummaryHeader()
 160         for testObject in tests_to_run:
 161                 print testObject.getSummary()
 162         print "\n===============================\n"
 163         print "[SUMMARY]"
 164         print "Total tests: %d" % stats["total"]
 165         print "Passed     : %d" % stats["pass"]
 166         print "Failed     : %d" % stats["fail"]
 167         print "================================\n\n"
 168
 169         logPass() # This line is implicit and can be removed