icuSources/python/icutools/databuilder/renderers/common_exec.py

   1 # Copyright (C) 2018 and later: Unicode, Inc. and others.
   2 # License & terms of use: http://www.unicode.org/copyright.html
   3
   4 from . import *
   5 from .. import *
   6 from .. import utils
   7 from ..request_types import *
   8
   9 import pathlib
  10 import os
  11 import shutil
  12 import subprocess
  13 import sys
  14
  15 def run(build_dirs, requests, common_vars, verbose=True, **kwargs):
  16     for bd in build_dirs:
  17         makedirs(bd.format(**common_vars))
  18     for request in requests:
  19         status = run_helper(request, common_vars, verbose=verbose, **kwargs)
  20         if status != 0:
  21             print("!!! ERROR executing above command line: exit code %d" % status)
  22             return 1
  23     if verbose:
  24         print("All data build commands executed")
  25     return 0
  26
  27 def makedirs(dirs):
  28     """makedirs compatible between Python 2 and 3"""
  29     try:
  30         # Python 3 version
  31         os.makedirs(dirs, exist_ok=True)
  32     except TypeError as e:
  33         # Python 2 version
  34         try:
  35             os.makedirs(dirs)
  36         except OSError as e:
  37             if e.errno != errno.EEXIST:
  38                 raise e
  39
  40 def run_helper(request, common_vars, platform, tool_dir, verbose, tool_cfg=None, **kwargs):
  41     # [The block below was added for APPLE_XCODE_BUILD]
  42     # The block of code below checks the mod dates of the files involved in the request--
  43     # if all of the output files exist and all of them are newer than the newest input file,
  44     # we don't run the tool.  If the request's input or output file list is empty, we run the
  45     # tool.  This applies to index and resource-list requests, which depend on other requests
  46     # and strictly don't need to run when the things they depend on haven't been updated, but
  47     # they're quick, so I didn't worry about this.
  48     if len(request.all_input_files()) > 0 and len(request.all_output_files()) > 0:
  49         input_paths = [pathlib.Path(utils.dir_for(input_file).format(**common_vars), input_file.filename) for input_file in request.all_input_files()]
  50         newest_input_mod_date = max([os.path.getmtime(input_path) for input_path in input_paths])
  51         output_paths = [pathlib.Path(utils.dir_for(output_file).format(**common_vars), output_file.filename) for output_file in request.all_output_files()]
  52         oldest_output_mod_date = min([(os.path.getmtime(output_path) if os.path.exists(output_path) else 0) for output_path in output_paths])
  53         if newest_input_mod_date <= oldest_output_mod_date:
  54             return 0
  55
  56     if isinstance(request, PrintFileRequest):
  57         output_path = "{DIRNAME}/{FILENAME}".format(
  58             DIRNAME = utils.dir_for(request.output_file).format(**common_vars),
  59             FILENAME = request.output_file.filename,
  60         )
  61         if verbose:
  62             print("Printing to file: %s" % output_path)
  63         with open(output_path, "w") as f:
  64             f.write(request.content)
  65         return 0
  66     if isinstance(request, CopyRequest):
  67         input_path = "{DIRNAME}/{FILENAME}".format(
  68             DIRNAME = utils.dir_for(request.input_file).format(**common_vars),
  69             FILENAME = request.input_file.filename,
  70         )
  71         output_path = "{DIRNAME}/{FILENAME}".format(
  72             DIRNAME = utils.dir_for(request.output_file).format(**common_vars),
  73             FILENAME = request.output_file.filename,
  74         )
  75         if verbose:
  76             print("Copying file to: %s" % output_path)
  77         shutil.copyfile(input_path, output_path)
  78         return 0
  79     if isinstance(request, VariableRequest):
  80         # No-op
  81         return 0
  82
  83     assert isinstance(request.tool, IcuTool)
  84     if platform == "windows":
  85         cmd_template = "{TOOL_DIR}/{TOOL}/{TOOL_CFG}/{TOOL}.exe {{ARGS}}".format(
  86             TOOL_DIR = tool_dir,
  87             TOOL_CFG = tool_cfg,
  88             TOOL = request.tool.name,
  89             **common_vars
  90         )
  91     elif platform == "unix":
  92         cmd_template = "{TOOL_DIR}/{TOOL} {{ARGS}}".format(
  93             TOOL_DIR = tool_dir,
  94             TOOL = request.tool.name,
  95             **common_vars
  96         )
  97     elif platform == "bazel":
  98         cmd_template = "{TOOL_DIR}/{TOOL}/{TOOL} {{ARGS}}".format(
  99             TOOL_DIR = tool_dir,
 100             TOOL = request.tool.name,
 101             **common_vars
 102         )
 103     else:
 104         raise ValueError("Unknown platform: %s" % platform)
 105
 106     if isinstance(request, RepeatedExecutionRequest):
 107         can_skip_files = not("EXTRA_OPTION" in request.format_with and "--usePoolBundle" in request.format_with["EXTRA_OPTION"])
 108         can_skip_files = can_skip_files and request.category != "translit"
 109         for loop_vars in utils.repeated_execution_request_looper(request):
 110                 # [The block below was added for APPLE_XCODE_BUILD]
 111             # The mod-date checks at the top of this function determine whether to execute an entire request--
 112             # if the request is a repeated execution request, it'll still allow all iterations of the request
 113             # to execute.  So we do an extra check here to skip any iterations of the repeated request where the
 114             # output file exists and is newer than the input file.  (This logic doesn't work if we're creating resource
 115             # bundles using the --write-pool-bundle/--use-pool-bundle options-- when we're using the pool bundle, we
 116             # have to rebuild EVERYTHING if any of the input files have changed.  It also doesn't work for the translit
 117             # rules, which have many input files.)
 118             if can_skip_files:
 119                 (_, _, input_file, output_file) = loop_vars
 120                 input_path = pathlib.Path(utils.dir_for(input_file).format(**common_vars), input_file.filename)
 121                 output_path = pathlib.Path(utils.dir_for(output_file).format(**common_vars), output_file.filename)
 122                 if os.path.exists(output_path) and os.path.getmtime(output_path) > os.path.getmtime(input_path):
 123                     continue
 124
 125             command_line = utils.format_repeated_request_command(
 126                 request,
 127                 cmd_template,
 128                 loop_vars,
 129                 common_vars
 130             )
 131             if platform == "windows":
 132                 # Note: this / to \ substitution may be too aggressive?
 133                 command_line = command_line.replace("/", "\\")
 134             returncode = run_shell_command(command_line, platform, verbose)
 135             if returncode != 0:
 136                 return returncode
 137         return 0
 138     if isinstance(request, SingleExecutionRequest):
 139         command_line = utils.format_single_request_command(
 140             request,
 141             cmd_template,
 142             common_vars
 143         )
 144         if platform == "windows":
 145             # Note: this / to \ substitution may be too aggressive?
 146             command_line = command_line.replace("/", "\\")
 147         returncode = run_shell_command(command_line, platform, verbose)
 148         return returncode
 149     assert False
 150
 151 def run_shell_command(command_line, platform, verbose):
 152     changed_windows_comspec = False
 153     # If the command line length on Windows exceeds the absolute maximum that CMD supports (8191), then
 154     # we temporarily switch over to use PowerShell for the command, and then switch back to CMD.
 155     # We don't want to use PowerShell for everything though, as it tends to be slower.
 156     if (platform == "windows"):
 157         previous_comspec = os.environ["COMSPEC"]
 158         # Add 7 to the length for the argument /c with quotes.
 159         # For example:  C:\WINDOWS\system32\cmd.exe /c "<command_line>"
 160         if ((len(previous_comspec) + len(command_line) + 7) > 8190):
 161             if verbose:
 162                 print("Command length exceeds the max length for CMD on Windows, using PowerShell instead.")
 163             os.environ["COMSPEC"] = 'powershell'
 164             changed_windows_comspec = True
 165     if verbose:
 166         print("Running: %s" % command_line)
 167         returncode = subprocess.call(
 168             command_line,
 169             shell = True
 170         )
 171     else:
 172         # Pipe output to /dev/null in quiet mode
 173         with open(os.devnull, "w") as devnull:
 174             returncode = subprocess.call(
 175                 command_line,
 176                 shell = True,
 177                 stdout = devnull,
 178                 stderr = devnull
 179             )
 180     if changed_windows_comspec:
 181         os.environ["COMSPEC"] = previous_comspec
 182     return returncode