icuSources/test/depstest/depstest.py

   1 #! /usr/bin/python -B
   2 # -*- coding: utf-8 -*-
   3 #
   4 # Copyright (C) 2016 and later: Unicode, Inc. and others.
   5 # License & terms of use: http://www.unicode.org/copyright.html
   6 # Copyright (C) 2011-2015, International Business Machines
   7 # Corporation and others. All Rights Reserved.
   8 #
   9 # file name: depstest.py
  10 #
  11 # created on: 2011may24
  12
  13 """ICU dependency tester.
  14
  15 This probably works only on Linux.
  16
  17 The exit code is 0 if everything is fine, 1 for errors, 2 for only warnings.
  18
  19 Sample invocation with an in-source build:
  20   ~/icu/icu4c/source/test/depstest$ ./depstest.py ../../
  21
  22 Sample invocation with an out-of-source build:
  23   ~/icu/icu4c/source/test/depstest$ ./depstest.py ~/build/
  24 """
  25
  26 from __future__ import print_function
  27
  28 __author__ = "Markus W. Scherer"
  29
  30 import glob
  31 import os.path
  32 import subprocess
  33 import sys
  34
  35 import dependencies
  36
  37 _ignored_symbols = set()
  38 _obj_files = {}
  39 _symbols_to_files = {}
  40 _return_value = 0
  41
  42 # Classes with vtables (and thus virtual methods).
  43 _virtual_classes = set()
  44 # Classes with weakly defined destructors.
  45 # nm shows a symbol class of "W" rather than "T".
  46 _weak_destructors = set()
  47
  48 def iteritems(items):
  49   """Python 2/3-compatible iteritems"""
  50   try:
  51     for v in items.iteritems():
  52       yield v
  53   except AttributeError:
  54     for v in items.items():
  55       yield v
  56
  57 def _ReadObjFile(root_path, library_name, obj_name):
  58   global _ignored_symbols, _obj_files, _symbols_to_files
  59   global _virtual_classes, _weak_destructors
  60   lib_obj_name = library_name + "/" + obj_name
  61   if lib_obj_name in _obj_files:
  62     print("Warning: duplicate .o file " + lib_obj_name)
  63     _return_value = 2
  64     return
  65
  66   path = os.path.join(root_path, library_name, obj_name)
  67   nm_result = subprocess.Popen(["nm", "--demangle", "--format=sysv",
  68                                 "--extern-only", "--no-sort", path],
  69                                stdout=subprocess.PIPE).communicate()[0]
  70   obj_imports = set()
  71   obj_exports = set()
  72   for line in nm_result.splitlines():
  73     fields = line.decode().split("|")
  74     if len(fields) == 1: continue
  75     name = fields[0].strip()
  76     # Ignore symbols like '__cxa_pure_virtual',
  77     # 'vtable for __cxxabiv1::__si_class_type_info' or
  78     # 'DW.ref.__gxx_personality_v0'.
  79     # '__dso_handle' belongs to __cxa_atexit().
  80     if (name.startswith("__cxa") or "__cxxabi" in name or "__gxx" in name or
  81         name == "__dso_handle"):
  82       _ignored_symbols.add(name)
  83       continue
  84     type = fields[2].strip()
  85     if type == "U":
  86       obj_imports.add(name)
  87     else:
  88       obj_exports.add(name)
  89       _symbols_to_files[name] = lib_obj_name
  90       # Is this a vtable? E.g., "vtable for icu_49::ByteSink".
  91       if name.startswith("vtable for icu"):
  92         _virtual_classes.add(name[name.index("::") + 2:])
  93       # Is this a destructor? E.g., "icu_49::ByteSink::~ByteSink()".
  94       index = name.find("::~")
  95       if index >= 0 and type == "W":
  96         _weak_destructors.add(name[index + 3:name.index("(", index)])
  97   _obj_files[lib_obj_name] = {"imports": obj_imports, "exports": obj_exports}
  98
  99 def _ReadLibrary(root_path, library_name):
 100   obj_paths = glob.glob(os.path.join(root_path, library_name, "*.o"))
 101   for path in obj_paths:
 102     _ReadObjFile(root_path, library_name, os.path.basename(path))
 103
 104 # Dependencies that would otherwise be errors, but that are to be allowed
 105 # in a limited (not transitive) context.  List of (file_name, symbol)
 106 # TODO: Move this data to dependencies.txt?
 107 allowed_errors = (
 108   ("common/umutex.o", "std::__throw_system_error(int)"),
 109   ("common/umutex.o", "std::uncaught_exception()"),
 110   ("common/umutex.o", "std::__once_callable"),
 111   ("common/umutex.o", "std::__once_call"),
 112   ("common/umutex.o", "__once_proxy"),
 113   ("common/umutex.o", "__tls_get_addr"),
 114   ("common/unifiedcache.o", "std::__throw_system_error(int)"),
 115 )
 116
 117 def _Resolve(name, parents):
 118   global _ignored_symbols, _obj_files, _symbols_to_files, _return_value
 119   item = dependencies.items[name]
 120   item_type = item["type"]
 121   if name in parents:
 122     sys.exit("Error: %s %s has a circular dependency on itself: %s" %
 123              (item_type, name, parents))
 124   # Check if already cached.
 125   exports = item.get("exports")
 126   if exports != None: return item
 127   # Calculcate recursively.
 128   parents.append(name)
 129   imports = set()
 130   exports = set()
 131   system_symbols = item.get("system_symbols")
 132   if system_symbols == None: system_symbols = item["system_symbols"] = set()
 133   files = item.get("files")
 134   if files:
 135     for file_name in files:
 136       obj_file = _obj_files[file_name]
 137       imports |= obj_file["imports"]
 138       exports |= obj_file["exports"]
 139   imports -= exports | _ignored_symbols
 140   deps = item.get("deps")
 141   if deps:
 142     for dep in deps:
 143       dep_item = _Resolve(dep, parents)
 144       # Detect whether this item needs to depend on dep,
 145       # except when this item has no files, that is, when it is just
 146       # a deliberate umbrella group or library.
 147       dep_exports = dep_item["exports"]
 148       dep_system_symbols = dep_item["system_symbols"]
 149       if files and imports.isdisjoint(dep_exports) and imports.isdisjoint(dep_system_symbols):
 150         print("Info:  %s %s  does not need to depend on  %s\n" % (item_type, name, dep))
 151       # We always include the dependency's exports, even if we do not need them
 152       # to satisfy local imports.
 153       exports |= dep_exports
 154       system_symbols |= dep_system_symbols
 155   item["exports"] = exports
 156   item["system_symbols"] = system_symbols
 157   imports -= exports | system_symbols
 158   for symbol in imports:
 159     for file_name in files:
 160       if (file_name, symbol) in allowed_errors:
 161          sys.stderr.write("Info:  ignoring %s imports %s\n\n" % (file_name, symbol))
 162          continue
 163       if symbol in _obj_files[file_name]["imports"]:
 164         neededFile = _symbols_to_files.get(symbol)
 165         if neededFile in dependencies.file_to_item:
 166           neededItem = "but %s does not depend on %s (for %s)" % (name, dependencies.file_to_item[neededFile], neededFile)
 167         else:
 168           neededItem = "- is this a new system symbol?"
 169         sys.stderr.write("Error: in %s %s: %s imports %s %s\n" %
 170                          (item_type, name, file_name, symbol, neededItem))
 171         _return_value = 1
 172   del parents[-1]
 173   return item
 174
 175 def Process(root_path):
 176   """Loads dependencies.txt, reads the libraries' .o files, and processes them.
 177
 178   Modifies dependencies.items: Recursively builds each item's system_symbols and exports.
 179   """
 180   global _ignored_symbols, _obj_files, _return_value
 181   global _virtual_classes, _weak_destructors
 182   dependencies.Load()
 183   for name_and_item in iteritems(dependencies.items):
 184     name = name_and_item[0]
 185     item = name_and_item[1]
 186     system_symbols = item.get("system_symbols")
 187     if system_symbols:
 188       for symbol in system_symbols:
 189         _symbols_to_files[symbol] = name
 190   for library_name in dependencies.libraries:
 191     _ReadLibrary(root_path, library_name)
 192   o_files_set = set(_obj_files.keys())
 193   files_missing_from_deps = o_files_set - dependencies.files
 194   files_missing_from_build = dependencies.files - o_files_set
 195   if files_missing_from_deps:
 196     sys.stderr.write("Error: files missing from dependencies.txt:\n%s\n" %
 197                      sorted(files_missing_from_deps))
 198     _return_value = 1
 199   if files_missing_from_build:
 200     sys.stderr.write("Error: files in dependencies.txt but not built:\n%s\n" %
 201                      sorted(files_missing_from_build))
 202     _return_value = 1
 203   if not _return_value:
 204     for library_name in dependencies.libraries:
 205       _Resolve(library_name, [])
 206   if not _return_value:
 207     virtual_classes_with_weak_destructors = _virtual_classes & _weak_destructors
 208     if virtual_classes_with_weak_destructors:
 209       sys.stderr.write("Error: Some classes have virtual methods, and "
 210                        "an implicit or inline destructor "
 211                        "(see ICU ticket #8454 for details):\n%s\n" %
 212                        sorted(virtual_classes_with_weak_destructors))
 213       _return_value = 1
 214
 215 def main():
 216   global _return_value
 217   if len(sys.argv) <= 1:
 218     sys.exit(("Command line error: " +
 219              "need one argument with the root path to the built ICU libraries/*.o files."))
 220   Process(sys.argv[1])
 221   if _ignored_symbols:
 222     print("Info: ignored symbols:\n%s" % sorted(_ignored_symbols))
 223   if not _return_value:
 224     print("OK: Specified and actual dependencies match.")
 225   else:
 226     print("Error: There were errors, please fix them and re-run. Processing may have terminated abnormally.")
 227   return _return_value
 228
 229 if __name__ == "__main__":
 230   sys.exit(main())