icuSources/tools/genrb/reslist.h

   1 // © 2016 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3 /*
   4 *******************************************************************************
   5 *
   6 *   Copyright (C) 2000-2015, International Business Machines
   7 *   Corporation and others.  All Rights Reserved.
   8 *
   9 *******************************************************************************
  10 *
  11 * File reslist.h
  12 *
  13 * Modification History:
  14 *
  15 *   Date        Name        Description
  16 *   02/21/00    weiv        Creation.
  17 *******************************************************************************
  18 */
  19
  20 #ifndef RESLIST_H
  21 #define RESLIST_H
  22
  23 #define KEY_SPACE_SIZE 65536
  24 #define RESLIST_MAX_INT_VECTOR 2048
  25
  26 #include <functional>
  27
  28 #include "unicode/utypes.h"
  29 #include "unicode/unistr.h"
  30 #include "unicode/ures.h"
  31 #include "unicode/ustring.h"
  32 #include "cmemory.h"
  33 #include "cstring.h"
  34 #include "uhash.h"
  35 #include "unewdata.h"
  36 #include "uresdata.h"
  37 #include "ustr.h"
  38
  39 U_CDECL_BEGIN
  40
  41 class PathFilter;
  42 class PseudoListResource;
  43 class ResKeyPath;
  44
  45 struct ResFile {
  46     ResFile()
  47             : fBytes(NULL), fIndexes(NULL),
  48               fKeys(NULL), fKeysLength(0), fKeysCount(0),
  49               fStrings(NULL), fStringIndexLimit(0),
  50               fChecksum(0) {}
  51     ~ResFile() { close(); }
  52
  53     void close();
  54
  55     uint8_t *fBytes;
  56     const int32_t *fIndexes;
  57     const char *fKeys;
  58     int32_t fKeysLength;
  59     int32_t fKeysCount;
  60
  61     PseudoListResource *fStrings;
  62     int32_t fStringIndexLimit;
  63
  64     int32_t fChecksum;
  65 };
  66
  67 struct SResource;
  68
  69 typedef struct KeyMapEntry {
  70     int32_t oldpos, newpos;
  71 } KeyMapEntry;
  72
  73 /* Resource bundle root table */
  74 struct SRBRoot {
  75     SRBRoot(const UString *comment, UBool isPoolBundle, UErrorCode &errorCode);
  76     ~SRBRoot();
  77
  78     void write(const char *outputDir, const char *outputPkg,
  79                char *writtenFilename, int writtenFilenameLen, UErrorCode &errorCode);
  80
  81     void setLocale(UChar *locale, UErrorCode &errorCode);
  82     int32_t addTag(const char *tag, UErrorCode &errorCode);
  83
  84     const char *getKeyString(int32_t key) const;
  85     const char *getKeyBytes(int32_t *pLength) const;
  86
  87     int32_t addKeyBytes(const char *keyBytes, int32_t length, UErrorCode &errorCode);
  88
  89     void compactKeys(UErrorCode &errorCode);
  90
  91     int32_t makeRes16(uint32_t resWord) const;
  92     int32_t mapKey(int32_t oldpos) const;
  93
  94 private:
  95     void compactStringsV2(UHashtable *stringSet, UErrorCode &errorCode);
  96
  97 public:
  98     // TODO: private
  99
 100   SResource *fRoot;  // Normally a TableResource.
 101   char *fLocale;
 102   int32_t fIndexLength;
 103   int32_t fMaxTableLength;
 104   UBool fNoFallback; /* see URES_ATT_NO_FALLBACK */
 105   int8_t fStringsForm; /* default STRINGS_UTF16_V1 */
 106   UBool fIsPoolBundle;
 107
 108   char *fKeys;
 109   KeyMapEntry *fKeyMap;
 110   int32_t fKeysBottom, fKeysTop;
 111   int32_t fKeysCapacity;
 112   int32_t fKeysCount;
 113   int32_t fLocalKeyLimit; /* key offset < limit fits into URES_TABLE */
 114
 115   icu::UnicodeString f16BitUnits;
 116   int32_t f16BitStringsLength;
 117
 118   const ResFile *fUsePoolBundle;
 119   int32_t fPoolStringIndexLimit;
 120   int32_t fPoolStringIndex16Limit;
 121   int32_t fLocalStringIndexLimit;
 122   SRBRoot *fWritePoolBundle;
 123 };
 124
 125 /* write a java resource file */
 126 // TODO: C++ify
 127 void bundle_write_java(struct SRBRoot *bundle, const char *outputDir, const char* outputEnc, char *writtenFilename,
 128                        int writtenFilenameLen, const char* packageName, const char* bundleName, UErrorCode *status);
 129
 130 /* write a xml resource file */
 131 // TODO: C++ify
 132 void bundle_write_xml(struct SRBRoot *bundle, const char *outputDir,const char* outputEnc, const char* rbname,
 133                   char *writtenFilename, int writtenFilenameLen, const char* language, const char* package, UErrorCode *status);
 134
 135 /* Various resource types */
 136
 137 /*
 138  * Return a unique pointer to a dummy object,
 139  * for use in non-error cases when no resource is to be added to the bundle.
 140  * (NULL is used in error cases.)
 141  */
 142 struct SResource* res_none(void);
 143
 144 class ArrayResource;
 145 class TableResource;
 146 class IntVectorResource;
 147
 148 TableResource *table_open(struct SRBRoot *bundle, const char *tag, const struct UString* comment, UErrorCode *status);
 149
 150 ArrayResource *array_open(struct SRBRoot *bundle, const char *tag, const struct UString* comment, UErrorCode *status);
 151
 152 struct SResource *string_open(struct SRBRoot *bundle, const char *tag, const UChar *value, int32_t len, const struct UString* comment, UErrorCode *status);
 153
 154 struct SResource *alias_open(struct SRBRoot *bundle, const char *tag, UChar *value, int32_t len, const struct UString* comment, UErrorCode *status);
 155
 156 IntVectorResource *intvector_open(struct SRBRoot *bundle, const char *tag,  const struct UString* comment, UErrorCode *status);
 157
 158 struct SResource *int_open(struct SRBRoot *bundle, const char *tag, int32_t value, const struct UString* comment, UErrorCode *status);
 159
 160 struct SResource *bin_open(struct SRBRoot *bundle, const char *tag, uint32_t length, uint8_t *data, const char* fileName, const struct UString* comment, UErrorCode *status);
 161
 162 /* Resource place holder */
 163
 164 struct SResource {
 165     SResource();
 166     SResource(SRBRoot *bundle, const char *tag, int8_t type, const UString* comment,
 167               UErrorCode &errorCode);
 168     virtual ~SResource();
 169
 170     UBool isTable() const { return fType == URES_TABLE; }
 171     UBool isString() const { return fType == URES_STRING; }
 172
 173     const char *getKeyString(const SRBRoot *bundle) const;
 174
 175     /**
 176      * Preflights strings.
 177      * Finds duplicates and counts the total number of string code units
 178      * so that they can be written first to the 16-bit array,
 179      * for minimal string and container storage.
 180      *
 181      * We walk the final parse tree, rather than collecting this information while building it,
 182      * so that we need not deal with changes to the parse tree (especially removing resources).
 183      */
 184     void preflightStrings(SRBRoot *bundle, UHashtable *stringSet, UErrorCode &errorCode);
 185     virtual void handlePreflightStrings(SRBRoot *bundle, UHashtable *stringSet, UErrorCode &errorCode);
 186
 187     /**
 188      * Writes resource values into f16BitUnits
 189      * and determines the resource item word, if possible.
 190      */
 191     void write16(SRBRoot *bundle);
 192     virtual void handleWrite16(SRBRoot *bundle);
 193
 194     /**
 195      * Calculates ("preflights") and advances the *byteOffset
 196      * by the size of the resource's data in the binary file and
 197      * determines the resource item word.
 198      *
 199      * Most handlePreWrite() functions may add any number of bytes, but preWrite()
 200      * will always pad it to a multiple of 4.
 201      * The resource item type may be a related subtype of the fType.
 202      *
 203      * The preWrite() and write() functions start and end at the same
 204      * byteOffset values.
 205      * Prewriting allows bundle.write() to determine the root resource item word,
 206      * before actually writing the bundle contents to the file,
 207      * which is necessary because the root item is stored at the beginning.
 208      */
 209     void preWrite(uint32_t *byteOffset);
 210     virtual void handlePreWrite(uint32_t *byteOffset);
 211
 212     /**
 213      * Writes the resource's data to mem and updates the byteOffset
 214      * in parallel.
 215      */
 216     void write(UNewDataMemory *mem, uint32_t *byteOffset);
 217     virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset);
 218
 219     /**
 220      * Applies the given filter with the given base path to this resource.
 221      * Removes child resources rejected by the filter recursively.
 222      *
 223      * @param bundle Needed in order to access the key for this and child resources.
 224      */
 225     virtual void applyFilter(const PathFilter& filter, ResKeyPath& path, const SRBRoot* bundle);
 226
 227     /**
 228      * Calls the given function for every key ID present in this tree.
 229      */
 230     virtual void collectKeys(std::function<void(int32_t)> collector) const;
 231
 232     int8_t   fType;     /* nominal type: fRes (when != 0xffffffff) may use subtype */
 233     UBool    fWritten;  /* res_write() can exit early */
 234     uint32_t fRes;      /* resource item word; RES_BOGUS=0xffffffff if not known yet */
 235     int32_t  fRes16;    /* Res16 version of fRes for Table, Table16, Array16; -1 if it does not fit. */
 236     int32_t  fKey;      /* Index into bundle->fKeys; -1 if no key. */
 237     int32_t  fKey16;    /* Key16 version of fKey for Table & Table16; -1 if no key or it does not fit. */
 238     int      line;      /* used internally to report duplicate keys in tables */
 239     SResource *fNext;   /* This is for internal chaining while building */
 240     struct UString fComment;
 241 };
 242
 243 class ContainerResource : public SResource {
 244 public:
 245     ContainerResource(SRBRoot *bundle, const char *tag, int8_t type,
 246                       const UString* comment, UErrorCode &errorCode)
 247             : SResource(bundle, tag, type, comment, errorCode),
 248               fCount(0), fFirst(NULL) {}
 249     virtual ~ContainerResource();
 250
 251     void handlePreflightStrings(SRBRoot *bundle, UHashtable *stringSet, UErrorCode &errorCode) override;
 252
 253     void collectKeys(std::function<void(int32_t)> collector) const override;
 254
 255 protected:
 256     void writeAllRes16(SRBRoot *bundle);
 257     void preWriteAllRes(uint32_t *byteOffset);
 258     void writeAllRes(UNewDataMemory *mem, uint32_t *byteOffset);
 259     void writeAllRes32(UNewDataMemory *mem, uint32_t *byteOffset);
 260
 261 public:
 262     // TODO: private with getter?
 263     uint32_t fCount;
 264     SResource *fFirst;
 265 };
 266
 267 class TableResource : public ContainerResource {
 268 public:
 269     TableResource(SRBRoot *bundle, const char *tag,
 270                   const UString* comment, UErrorCode &errorCode)
 271             : ContainerResource(bundle, tag, URES_TABLE, comment, errorCode),
 272               fTableType(URES_TABLE), fRoot(bundle) {}
 273     virtual ~TableResource();
 274
 275     void add(SResource *res, int linenumber, UErrorCode &errorCode);
 276
 277     void handleWrite16(SRBRoot *bundle) override;
 278     void handlePreWrite(uint32_t *byteOffset) override;
 279     void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset) override;
 280
 281     void applyFilter(const PathFilter& filter, ResKeyPath& path, const SRBRoot* bundle) override;
 282
 283     int8_t fTableType;  // determined by table_write16() for table_preWrite() & table_write()
 284     SRBRoot *fRoot;
 285 };
 286
 287 class ArrayResource : public ContainerResource {
 288 public:
 289     ArrayResource(SRBRoot *bundle, const char *tag,
 290                   const UString* comment, UErrorCode &errorCode)
 291             : ContainerResource(bundle, tag, URES_ARRAY, comment, errorCode),
 292               fLast(NULL) {}
 293     virtual ~ArrayResource();
 294
 295     void add(SResource *res);
 296
 297     virtual void handleWrite16(SRBRoot *bundle);
 298     virtual void handlePreWrite(uint32_t *byteOffset);
 299     virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset);
 300
 301     SResource *fLast;
 302 };
 303
 304 /**
 305  * List of resources for a pool bundle.
 306  * Writes an empty table resource, rather than a container structure.
 307  */
 308 class PseudoListResource : public ContainerResource {
 309 public:
 310     PseudoListResource(SRBRoot *bundle, UErrorCode &errorCode)
 311             : ContainerResource(bundle, NULL, URES_TABLE, NULL, errorCode) {}
 312     virtual ~PseudoListResource();
 313
 314     void add(SResource *res);
 315
 316     virtual void handleWrite16(SRBRoot *bundle);
 317 };
 318
 319 class StringBaseResource : public SResource {
 320 public:
 321     StringBaseResource(SRBRoot *bundle, const char *tag, int8_t type,
 322                        const UChar *value, int32_t len,
 323                        const UString* comment, UErrorCode &errorCode);
 324     StringBaseResource(SRBRoot *bundle, int8_t type,
 325                        const icu::UnicodeString &value, UErrorCode &errorCode);
 326     StringBaseResource(int8_t type, const UChar *value, int32_t len, UErrorCode &errorCode);
 327     virtual ~StringBaseResource();
 328
 329     const UChar *getBuffer() const { return icu::toUCharPtr(fString.getBuffer()); }
 330     int32_t length() const { return fString.length(); }
 331
 332     virtual void handlePreWrite(uint32_t *byteOffset);
 333     virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset);
 334
 335     // TODO: private with getter?
 336     icu::UnicodeString fString;
 337 };
 338
 339 class StringResource : public StringBaseResource {
 340 public:
 341     StringResource(SRBRoot *bundle, const char *tag, const UChar *value, int32_t len,
 342                    const UString* comment, UErrorCode &errorCode)
 343             : StringBaseResource(bundle, tag, URES_STRING, value, len, comment, errorCode),
 344               fSame(NULL), fSuffixOffset(0),
 345               fNumCopies(0), fNumUnitsSaved(0), fNumCharsForLength(0) {}
 346     StringResource(SRBRoot *bundle, const icu::UnicodeString &value, UErrorCode &errorCode)
 347             : StringBaseResource(bundle, URES_STRING, value, errorCode),
 348               fSame(NULL), fSuffixOffset(0),
 349               fNumCopies(0), fNumUnitsSaved(0), fNumCharsForLength(0) {}
 350     StringResource(int32_t poolStringIndex, int8_t numCharsForLength,
 351                    const UChar *value, int32_t length,
 352                    UErrorCode &errorCode)
 353             : StringBaseResource(URES_STRING, value, length, errorCode),
 354               fSame(NULL), fSuffixOffset(0),
 355               fNumCopies(0), fNumUnitsSaved(0), fNumCharsForLength(numCharsForLength) {
 356         // v3 pool string encoded as string-v2 with low offset
 357         fRes = URES_MAKE_RESOURCE(URES_STRING_V2, poolStringIndex);
 358         fWritten = TRUE;
 359     }
 360     virtual ~StringResource();
 361
 362     int32_t get16BitStringsLength() const {
 363         return fNumCharsForLength + length() + 1;  // +1 for the NUL
 364     }
 365
 366     virtual void handlePreflightStrings(SRBRoot *bundle, UHashtable *stringSet, UErrorCode &errorCode);
 367     virtual void handleWrite16(SRBRoot *bundle);
 368
 369     void writeUTF16v2(int32_t base, icu::UnicodeString &dest);
 370
 371     StringResource *fSame;  // used for duplicates
 372     int32_t fSuffixOffset;  // this string is a suffix of fSame at this offset
 373     int32_t fNumCopies;     // number of equal strings represented by one stringSet element
 374     int32_t fNumUnitsSaved;  // from not writing duplicates and suffixes
 375     int8_t fNumCharsForLength;
 376 };
 377
 378 class AliasResource : public StringBaseResource {
 379 public:
 380     AliasResource(SRBRoot *bundle, const char *tag, const UChar *value, int32_t len,
 381                   const UString* comment, UErrorCode &errorCode)
 382             : StringBaseResource(bundle, tag, URES_ALIAS, value, len, comment, errorCode) {}
 383     virtual ~AliasResource();
 384 };
 385
 386 class IntResource : public SResource {
 387 public:
 388     IntResource(SRBRoot *bundle, const char *tag, int32_t value,
 389                 const UString* comment, UErrorCode &errorCode);
 390     virtual ~IntResource();
 391
 392     // TODO: private with getter?
 393     int32_t fValue;
 394 };
 395
 396 class IntVectorResource : public SResource {
 397 public:
 398     IntVectorResource(SRBRoot *bundle, const char *tag,
 399                       const UString* comment, UErrorCode &errorCode);
 400     virtual ~IntVectorResource();
 401
 402     void add(int32_t value, UErrorCode &errorCode);
 403
 404     virtual void handlePreWrite(uint32_t *byteOffset);
 405     virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset);
 406
 407     // TODO: UVector32
 408     uint32_t fCount;
 409     uint32_t *fArray;
 410 };
 411
 412 class BinaryResource : public SResource {
 413 public:
 414     BinaryResource(SRBRoot *bundle, const char *tag,
 415                    uint32_t length, uint8_t *data, const char* fileName,
 416                    const UString* comment, UErrorCode &errorCode);
 417     virtual ~BinaryResource();
 418
 419     virtual void handlePreWrite(uint32_t *byteOffset);
 420     virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset);
 421
 422     // TODO: CharString?
 423     uint32_t fLength;
 424     uint8_t *fData;
 425     // TODO: CharString
 426     char* fFileName;  // file name for binary or import binary tags if any
 427 };
 428
 429 // TODO: use LocalPointer or delete
 430 void res_close(struct SResource *res);
 431
 432 void setIncludeCopyright(UBool val);
 433 UBool getIncludeCopyright(void);
 434
 435 void setFormatVersion(int32_t formatVersion);
 436
 437 int32_t getFormatVersion();
 438
 439 void setUsePoolBundle(UBool use);
 440
 441 /* in wrtxml.cpp */
 442 uint32_t computeCRC(const char *ptr, uint32_t len, uint32_t lastcrc);
 443
 444 U_CDECL_END
 445 #endif /* #ifndef RESLIST_H */