icuSources/data/buildtool/request_types.py

   1 # Copyright (C) 2018 and later: Unicode, Inc. and others.
   2 # License & terms of use: http://www.unicode.org/copyright.html
   3
   4 # Python 2/3 Compatibility (ICU-20299)
   5 # TODO(ICU-20301): Remove this.
   6 from __future__ import print_function
   7
   8 from abc import abstractmethod
   9 import copy
  10 import sys
  11
  12 from . import *
  13 from . import utils
  14
  15
  16 # TODO(ICU-20301): Remove arguments from all instances of super() in this file
  17
  18 # Note: for this to be a proper abstract class, it should extend abc.ABC.
  19 # There is no nice way to do this that works in both Python 2 and 3.
  20 # TODO(ICU-20301): Make this inherit from abc.ABC.
  21 class AbstractRequest(object):
  22     def __init__(self, **kwargs):
  23
  24         # Used for identification purposes
  25         self.name = None
  26
  27         # The filter category that applies to this request
  28         self.category = None
  29
  30         self._set_fields(kwargs)
  31
  32     def _set_fields(self, kwargs):
  33         for key, value in list(kwargs.items()):
  34             if hasattr(self, key):
  35                 if isinstance(value, list):
  36                     value = copy.copy(value)
  37                 elif isinstance(value, dict):
  38                     value = copy.deepcopy(value)
  39                 setattr(self, key, value)
  40             else:
  41                 raise ValueError("Unknown argument: %s" % key)
  42
  43     def apply_file_filter(self, filter):
  44         """
  45         Returns True if this request still has input files after filtering,
  46         or False if the request is "empty" after filtering.
  47         """
  48         return True
  49
  50     def flatten(self, config, all_requests, common_vars):
  51         return [self]
  52
  53     def all_input_files(self):
  54         return []
  55
  56     def all_output_files(self):
  57         return []
  58
  59
  60 class AbstractExecutionRequest(AbstractRequest):
  61     def __init__(self, **kwargs):
  62
  63         # Names of targets (requests) or files that this request depends on.
  64         # The entries of dep_targets may be any of the following types:
  65         #
  66         #   1. DepTarget, for the output of an execution request.
  67         #   2. InFile, TmpFile, etc., for a specific file.
  68         #   3. A list of InFile, TmpFile, etc., where the list is the same
  69         #      length as self.input_files and self.output_files.
  70         #
  71         # In cases 1 and 2, the dependency is added to all rules that the
  72         # request generates. In case 3, the dependency is added only to the
  73         # rule that generates the output file at the same array index.
  74         self.dep_targets = []
  75
  76         # Computed during self.flatten(); don't edit directly.
  77         self.common_dep_files = []
  78
  79         # Primary input files
  80         self.input_files = []
  81
  82         # Output files; for some subclasses, this must be the same length
  83         # as input_files
  84         self.output_files = []
  85
  86         # What tool to execute
  87         self.tool = None
  88
  89         # Argument string to pass to the tool with optional placeholders
  90         self.args = ""
  91
  92         # Placeholders to substitute into the argument string; if any of these
  93         # have a list type, the list must be equal in length to input_files
  94         self.format_with = {}
  95
  96         super(AbstractExecutionRequest, self).__init__(**kwargs)
  97
  98     def apply_file_filter(self, filter):
  99         i = 0
 100         while i < len(self.input_files):
 101             if filter.match(self.input_files[i]):
 102                 i += 1
 103                 continue
 104             self._del_at(i)
 105         return i > 0
 106
 107     def _del_at(self, i):
 108         del self.input_files[i]
 109         for _, v in self.format_with.items():
 110             if isinstance(v, list):
 111                 assert len(v) == len(self.input_files) + 1
 112                 del v[i]
 113         for v in self.dep_targets:
 114             if isinstance(v, list):
 115                 assert len(v) == len(self.input_files) + 1
 116                 del v[i]
 117
 118     def flatten(self, config, all_requests, common_vars):
 119         self._dep_targets_to_files(all_requests)
 120         return super(AbstractExecutionRequest, self).flatten(config, all_requests, common_vars)
 121
 122     def _dep_targets_to_files(self, all_requests):
 123         if not self.dep_targets:
 124             return
 125         for dep_target in self.dep_targets:
 126             if isinstance(dep_target, list):
 127                 if hasattr(self, "specific_dep_files"):
 128                     assert len(dep_target) == len(self.specific_dep_files)
 129                     for file, out_list in zip(dep_target, self.specific_dep_files):
 130                         assert hasattr(file, "filename")
 131                         out_list.append(file)
 132                 else:
 133                     self.common_dep_files += dep_target
 134                 continue
 135             if not isinstance(dep_target, DepTarget):
 136                 # Copy file entries directly to dep_files.
 137                 assert hasattr(dep_target, "filename")
 138                 self.common_dep_files.append(dep_target)
 139                 continue
 140             # For DepTarget entries, search for the target.
 141             for request in all_requests:
 142                 if request.name == dep_target.name:
 143                     self.common_dep_files += request.all_output_files()
 144                     break
 145             else:
 146                 print("Warning: Unable to find target %s, a dependency of %s" % (
 147                     dep_target.name,
 148                     self.name
 149                 ), file=sys.stderr)
 150
 151     def all_input_files(self):
 152         return self.common_dep_files + self.input_files
 153
 154     def all_output_files(self):
 155         return self.output_files
 156
 157
 158 class SingleExecutionRequest(AbstractExecutionRequest):
 159     def __init__(self, **kwargs):
 160         super(SingleExecutionRequest, self).__init__(**kwargs)
 161
 162
 163 class RepeatedExecutionRequest(AbstractExecutionRequest):
 164     def __init__(self, **kwargs):
 165
 166         # Placeholders to substitute into the argument string unique to each
 167         # iteration; all values must be lists equal in length to input_files
 168         self.repeat_with = {}
 169
 170         # Lists for dep files that are specific to individual resource bundle files
 171         self.specific_dep_files = [[] for _ in range(len(kwargs["input_files"]))]
 172
 173         super(RepeatedExecutionRequest, self).__init__(**kwargs)
 174
 175     def _del_at(self, i):
 176         super(RepeatedExecutionRequest, self)._del_at(i)
 177         del self.output_files[i]
 178         for _, v in self.repeat_with.items():
 179             if isinstance(v, list):
 180                 del v[i]
 181
 182     def all_input_files(self):
 183         files = super(RepeatedExecutionRequest, self).all_input_files()
 184         for specific_file_list in self.specific_dep_files:
 185             files += specific_file_list
 186         return files
 187
 188
 189 class RepeatedOrSingleExecutionRequest(AbstractExecutionRequest):
 190     def __init__(self, **kwargs):
 191         self.repeat_with = {}
 192         super(RepeatedOrSingleExecutionRequest, self).__init__(**kwargs)
 193
 194     def flatten(self, config, all_requests, common_vars):
 195         if config.max_parallel:
 196             new_request = RepeatedExecutionRequest(
 197                 name = self.name,
 198                 category = self.category,
 199                 dep_targets = self.dep_targets,
 200                 input_files = self.input_files,
 201                 output_files = self.output_files,
 202                 tool = self.tool,
 203                 args = self.args,
 204                 format_with = self.format_with,
 205                 repeat_with = self.repeat_with
 206             )
 207         else:
 208             new_request = SingleExecutionRequest(
 209                 name = self.name,
 210                 category = self.category,
 211                 dep_targets = self.dep_targets,
 212                 input_files = self.input_files,
 213                 output_files = self.output_files,
 214                 tool = self.tool,
 215                 args = self.args,
 216                 format_with = utils.concat_dicts(self.format_with, self.repeat_with)
 217             )
 218         return new_request.flatten(config, all_requests, common_vars)
 219
 220     def _del_at(self, i):
 221         super(RepeatedOrSingleExecutionRequest, self)._del_at(i)
 222         del self.output_files[i]
 223         for _, v in self.repeat_with.items():
 224             if isinstance(v, list):
 225                 del v[i]
 226
 227
 228 class PrintFileRequest(AbstractRequest):
 229     def __init__(self, **kwargs):
 230         self.output_file = None
 231         self.content = None
 232         super(PrintFileRequest, self).__init__(**kwargs)
 233
 234     def all_output_files(self):
 235         return [self.output_file]
 236
 237
 238 class CopyRequest(AbstractRequest):
 239     def __init__(self, **kwargs):
 240         self.input_file = None
 241         self.output_file = None
 242         super(CopyRequest, self).__init__(**kwargs)
 243
 244     def all_input_files(self):
 245         return [self.input_file]
 246
 247     def all_output_files(self):
 248         return [self.output_file]
 249
 250
 251 class VariableRequest(AbstractRequest):
 252     def __init__(self, **kwargs):
 253         self.input_files = []
 254         super(VariableRequest, self).__init__(**kwargs)
 255
 256     def all_input_files(self):
 257         return self.input_files
 258
 259
 260 class ListRequest(AbstractRequest):
 261     def __init__(self, **kwargs):
 262         self.variable_name = None
 263         self.output_file = None
 264         self.include_tmp = None
 265         super(ListRequest, self).__init__(**kwargs)
 266
 267     def flatten(self, config, all_requests, common_vars):
 268         list_files = list(sorted(utils.get_all_output_files(all_requests)))
 269         if self.include_tmp:
 270             variable_files = list(sorted(utils.get_all_output_files(all_requests, include_tmp=True)))
 271         else:
 272             # Always include the list file itself
 273             variable_files = list_files + [self.output_file]
 274         return PrintFileRequest(
 275             name = self.name,
 276             output_file = self.output_file,
 277             content = "\n".join(file.filename for file in list_files)
 278         ).flatten(config, all_requests, common_vars) + VariableRequest(
 279             name = self.variable_name,
 280             input_files = variable_files
 281         ).flatten(config, all_requests, common_vars)
 282
 283     def all_output_files(self):
 284         return [self.output_file]
 285
 286
 287 class IndexTxtRequest(AbstractRequest):
 288     def __init__(self, **kwargs):
 289         self.input_files = []
 290         self.output_file = None
 291         self.cldr_version = ""
 292         super(IndexTxtRequest, self).__init__(**kwargs)
 293
 294     def apply_file_filter(self, filter):
 295         i = 0
 296         while i < len(self.input_files):
 297             if filter.match(self.input_files[i]):
 298                 i += 1
 299                 continue
 300             del self.input_files[i]
 301         return i > 0
 302
 303     def flatten(self, config, all_requests, common_vars):
 304         return PrintFileRequest(
 305             name = self.name,
 306             output_file = self.output_file,
 307             content = self._generate_index_file(common_vars)
 308         ).flatten(config, all_requests, common_vars)
 309
 310     def _generate_index_file(self, common_vars):
 311         locales = [f.filename[f.filename.rfind("/")+1:-4] for f in self.input_files]
 312         formatted_version = "    CLDRVersion { \"%s\" }\n" % self.cldr_version if self.cldr_version else ""
 313         formatted_locales = "\n".join(["        %s {\"\"}" % v for v in locales])
 314         # TODO: CLDRVersion is required only in the base file
 315         return ("// Warning this file is automatically generated\n"
 316                 "{INDEX_NAME}:table(nofallback) {{\n"
 317                 "{FORMATTED_VERSION}"
 318                 "    InstalledLocales {{\n"
 319                 "{FORMATTED_LOCALES}\n"
 320                 "    }}\n"
 321                 "}}").format(
 322                     FORMATTED_VERSION = formatted_version,
 323                     FORMATTED_LOCALES = formatted_locales,
 324                     **common_vars
 325                 )
 326
 327     def all_input_files(self):
 328         return self.input_files
 329
 330     def all_output_files(self):
 331         return [self.output_file]