--- /dev/null
+# Copyright (C) 2018 and later: Unicode, Inc. and others.
+# License & terms of use: http://www.unicode.org/copyright.html
+
+# Python 2/3 Compatibility (ICU-20299)
+# TODO(ICU-20301): Remove this.
+from __future__ import print_function
+
+import argparse
+import glob as pyglob
+import io as pyio
+import json
+import os
+import sys
+
+from . import *
+from .comment_stripper import CommentStripper
+from .request_types import CopyRequest
+from .renderers import makefile, common_exec
+from . import filtration, utils
+
+flag_parser = argparse.ArgumentParser(
+ description = """Generates rules for building ICU binary data files from text
+and other input files in source control.
+
+Use the --mode option to declare how to execute those rules, either exporting
+the rules to a Makefile or spawning child processes to run them immediately:
+
+ --mode=gnumake prints a Makefile to standard out.
+ --mode=unix-exec spawns child processes in a Unix-like environment.
+ --mode=windows-exec spawns child processes in a Windows-like environment.
+
+Tips for --mode=unix-exec
+=========================
+
+Create two empty directories for out_dir and tmp_dir. They will get filled
+with a lot of intermediate files.
+
+Set LD_LIBRARY_PATH to include the lib directory. e.g., from icu4c/source:
+
+ $ LD_LIBRARY_PATH=lib PYTHONPATH=python python3 -m icutools.databuilder ...
+
+Once icutools.databuilder finishes, you have compiled the data, but you have
+not packaged it into a .dat or .so file. This is done by the separate pkgdata
+tool in bin. Read the docs of pkgdata:
+
+ $ LD_LIBRARY_PATH=lib ./bin/pkgdata --help
+
+Example command line to call pkgdata:
+
+ $ LD_LIBRARY_PATH=lib ./bin/pkgdata -m common -p icudt63l -c \\
+ -O data/icupkg.inc -s $OUTDIR -d $TMPDIR $TMPDIR/icudata.lst
+
+where $OUTDIR and $TMPDIR are your out and tmp directories, respectively.
+The above command will create icudt63l.dat in the tmpdir.
+
+Command-Line Arguments
+======================
+""",
+ formatter_class = argparse.RawDescriptionHelpFormatter
+)
+
+arg_group_required = flag_parser.add_argument_group("required arguments")
+arg_group_required.add_argument(
+ "--mode",
+ help = "What to do with the generated rules.",
+ choices = ["gnumake", "unix-exec", "windows-exec", "bazel-exec"],
+ required = True
+)
+
+flag_parser.add_argument(
+ "--src_dir",
+ help = "Path to data source folder (icu4c/source/data).",
+ default = "."
+)
+flag_parser.add_argument(
+ "--filter_file",
+ metavar = "PATH",
+ help = "Path to an ICU data filter JSON file.",
+ default = None
+)
+flag_parser.add_argument(
+ "--include_uni_core_data",
+ help = "Include the full Unicode core data in the dat file.",
+ default = False,
+ action = "store_true"
+)
+flag_parser.add_argument(
+ "--seqmode",
+ help = "Whether to optimize rules to be run sequentially (fewer threads) or in parallel (many threads). Defaults to 'sequential', which is better for unix-exec and windows-exec modes. 'parallel' is often better for massively parallel build systems.",
+ choices = ["sequential", "parallel"],
+ default = "sequential"
+)
+flag_parser.add_argument(
+ "--verbose",
+ help = "Print more verbose output (default false).",
+ default = False,
+ action = "store_true"
+)
+
+arg_group_exec = flag_parser.add_argument_group("arguments for unix-exec and windows-exec modes")
+arg_group_exec.add_argument(
+ "--out_dir",
+ help = "Path to where to save output data files.",
+ default = "icudata"
+)
+arg_group_exec.add_argument(
+ "--tmp_dir",
+ help = "Path to where to save temporary files.",
+ default = "icutmp"
+)
+arg_group_exec.add_argument(
+ "--tool_dir",
+ help = "Path to where to find binary tools (genrb, etc).",
+ default = "../bin"
+)
+arg_group_exec.add_argument(
+ "--tool_cfg",
+ help = "The build configuration of the tools. Used in 'windows-exec' mode only.",
+ default = "x86/Debug"
+)
+
+
+class Config(object):
+
+ def __init__(self, args):
+ # Process arguments
+ self.max_parallel = (args.seqmode == "parallel")
+
+ # Boolean: Whether to include core Unicode data files in the .dat file
+ self.include_uni_core_data = args.include_uni_core_data
+
+ # Default fields before processing filter file
+ self.filters_json_data = {}
+ self.filter_dir = "ERROR_NO_FILTER_FILE"
+
+ # Process filter file
+ if args.filter_file:
+ try:
+ with open(args.filter_file, "r") as f:
+ print("Note: Applying filters from %s." % args.filter_file, file=sys.stderr)
+ self._parse_filter_file(f)
+ except IOError:
+ print("Error: Could not read filter file %s." % args.filter_file, file=sys.stderr)
+ exit(1)
+ self.filter_dir = os.path.abspath(os.path.dirname(args.filter_file))
+
+ # Either "unihan" or "implicithan"
+ self.coll_han_type = "unihan"
+ if "collationUCAData" in self.filters_json_data:
+ self.coll_han_type = self.filters_json_data["collationUCAData"]
+
+ # Either "additive" or "subtractive"
+ self.strategy = "subtractive"
+ if "strategy" in self.filters_json_data:
+ self.strategy = self.filters_json_data["strategy"]
+
+ # True or False (could be extended later to support enum/list)
+ self.use_pool_bundle = True
+ if "usePoolBundle" in self.filters_json_data:
+ self.use_pool_bundle = self.filters_json_data["usePoolBundle"]
+
+ def _parse_filter_file(self, f):
+ # Use the Hjson parser if it is available; otherwise, use vanilla JSON.
+ try:
+ import hjson
+ self.filters_json_data = hjson.load(f)
+ except ImportError:
+ self.filters_json_data = json.load(CommentStripper(f))
+
+ # Optionally pre-validate the JSON schema before further processing.
+ # Some schema errors will be caught later, but this step ensures
+ # maximal validity.
+ try:
+ import jsonschema
+ schema_path = os.path.join(os.path.dirname(__file__), "filtration_schema.json")
+ with open(schema_path) as schema_f:
+ schema = json.load(CommentStripper(schema_f))
+ validator = jsonschema.Draft4Validator(schema)
+ for error in validator.iter_errors(self.filters_json_data, schema):
+ print("WARNING: ICU data filter JSON file:", error.message,
+ "at", "".join(
+ "[%d]" % part if isinstance(part, int) else ".%s" % part
+ for part in error.absolute_path
+ ),
+ file=sys.stderr)
+ except ImportError:
+ print("Tip: to validate your filter file, install the Pip package 'jsonschema'", file=sys.stderr)
+ pass
+
+
+def add_copy_input_requests(requests, config, common_vars):
+ files_to_copy = set()
+ for request in requests:
+ request_files = request.all_input_files()
+ # Also add known dependency txt files as possible inputs.
+ # This is required for translit rule files.
+ if hasattr(request, "dep_targets"):
+ request_files += [
+ f for f in request.dep_targets if isinstance(f, InFile)
+ ]
+ for f in request_files:
+ if isinstance(f, InFile):
+ files_to_copy.add(f)
+
+ result = []
+ id = 0
+
+ json_data = config.filters_json_data["fileReplacements"]
+ dirname = json_data["directory"]
+ for directive in json_data["replacements"]:
+ if type(directive) == str:
+ input_file = LocalFile(dirname, directive)
+ output_file = InFile(directive)
+ else:
+ input_file = LocalFile(dirname, directive["src"])
+ output_file = InFile(directive["dest"])
+ result += [
+ CopyRequest(
+ name = "input_copy_%d" % id,
+ input_file = input_file,
+ output_file = output_file
+ )
+ ]
+ files_to_copy.remove(output_file)
+ id += 1
+
+ for f in files_to_copy:
+ result += [
+ CopyRequest(
+ name = "input_copy_%d" % id,
+ input_file = SrcFile(f.filename),
+ output_file = f
+ )
+ ]
+ id += 1
+
+ result += requests
+ return result
+
+
+class IO(object):
+ """I/O operations required when computing the build actions"""
+
+ def __init__(self, src_dir):
+ self.src_dir = src_dir
+
+ def glob(self, pattern):
+ absolute_paths = pyglob.glob(os.path.join(self.src_dir, pattern))
+ # Strip off the absolute path suffix so we are left with a relative path.
+ relative_paths = [v[len(self.src_dir)+1:] for v in sorted(absolute_paths)]
+ # For the purposes of icutools.databuilder, force Unix-style directory separators.
+ # Within the Python code, including BUILDRULES.py and user-provided config files,
+ # directory separators are normalized to '/', including on Windows platforms.
+ return [v.replace("\\", "/") for v in relative_paths]
+
+ def read_locale_deps(self, tree):
+ return self._read_json("%s/LOCALE_DEPS.json" % tree)
+
+ def _read_json(self, filename):
+ with pyio.open(os.path.join(self.src_dir, filename), "r", encoding="utf-8-sig") as f:
+ return json.load(CommentStripper(f))
+
+
+def main(argv):
+ args = flag_parser.parse_args(argv)
+ config = Config(args)
+
+ if args.mode == "gnumake":
+ makefile_vars = {
+ "SRC_DIR": "$(srcdir)",
+ "IN_DIR": "$(srcdir)",
+ "INDEX_NAME": "res_index"
+ }
+ makefile_env = ["ICUDATA_CHAR", "OUT_DIR", "TMP_DIR"]
+ common = {
+ key: "$(%s)" % key
+ for key in list(makefile_vars.keys()) + makefile_env
+ }
+ common["FILTERS_DIR"] = config.filter_dir
+ common["CWD_DIR"] = os.getcwd()
+ else:
+ makefile_vars = None
+ common = {
+ "SRC_DIR": args.src_dir,
+ "IN_DIR": args.src_dir,
+ "OUT_DIR": args.out_dir,
+ "TMP_DIR": args.tmp_dir,
+ "FILTERS_DIR": config.filter_dir,
+ "CWD_DIR": os.getcwd(),
+ "INDEX_NAME": "res_index",
+ # TODO: Pull this from configure script:
+ "ICUDATA_CHAR": "l"
+ }
+
+ # Automatically load BUILDRULES from the src_dir
+ sys.path.append(args.src_dir)
+ try:
+ import BUILDRULES
+ except ImportError:
+ print("Cannot find BUILDRULES! Did you set your --src_dir?", file=sys.stderr)
+ sys.exit(1)
+
+ io = IO(args.src_dir)
+ requests = BUILDRULES.generate(config, io, common)
+
+ if "fileReplacements" in config.filters_json_data:
+ tmp_in_dir = "{TMP_DIR}/in".format(**common)
+ if makefile_vars:
+ makefile_vars["IN_DIR"] = tmp_in_dir
+ else:
+ common["IN_DIR"] = tmp_in_dir
+ requests = add_copy_input_requests(requests, config, common)
+
+ requests = filtration.apply_filters(requests, config, io)
+ requests = utils.flatten_requests(requests, config, common)
+
+ build_dirs = utils.compute_directories(requests)
+
+ if args.mode == "gnumake":
+ print(makefile.get_gnumake_rules(
+ build_dirs,
+ requests,
+ makefile_vars,
+ common_vars = common
+ ))
+ elif args.mode == "windows-exec":
+ return common_exec.run(
+ platform = "windows",
+ build_dirs = build_dirs,
+ requests = requests,
+ common_vars = common,
+ tool_dir = args.tool_dir,
+ tool_cfg = args.tool_cfg,
+ verbose = args.verbose,
+ )
+ elif args.mode == "unix-exec":
+ return common_exec.run(
+ platform = "unix",
+ build_dirs = build_dirs,
+ requests = requests,
+ common_vars = common,
+ tool_dir = args.tool_dir,
+ verbose = args.verbose,
+ )
+ elif args.mode == "bazel-exec":
+ return common_exec.run(
+ platform = "bazel",
+ build_dirs = build_dirs,
+ requests = requests,
+ common_vars = common,
+ tool_dir = args.tool_dir,
+ verbose = args.verbose,
+ )
+ else:
+ print("Mode not supported: %s" % args.mode)
+ return 1
+ return 0
+
+if __name__ == "__main__":
+ exit(main(sys.argv[1:]))
projects / apple / icu.git / blobdiff