icuSources/tools/gendict/gendict.cpp

   1 /*
   2 **********************************************************************
   3 *   Copyright (C) 2002-2012, International Business Machines
   4 *   Corporation and others.  All Rights Reserved.
   5 **********************************************************************
   6 *
   7 * File gendict.cpp
   8 */
   9
  10 #include "unicode/utypes.h"
  11 #include "unicode/uchar.h"
  12 #include "unicode/ucnv.h"
  13 #include "unicode/uniset.h"
  14 #include "unicode/unistr.h"
  15 #include "unicode/uclean.h"
  16 #include "unicode/udata.h"
  17 #include "unicode/putil.h"
  18 #include "unicode/ucharstriebuilder.h"
  19 #include "unicode/bytestriebuilder.h"
  20 #include "unicode/ucharstrie.h"
  21 #include "unicode/bytestrie.h"
  22 #include "unicode/ucnv.h"
  23 #include "unicode/utf16.h"
  24
  25 #include "charstr.h"
  26 #include "dictionarydata.h"
  27 #include "uoptions.h"
  28 #include "unewdata.h"
  29 #include "cmemory.h"
  30 #include "uassert.h"
  31 #include "ucbuf.h"
  32 #include "toolutil.h"
  33 #include "cstring.h"
  34
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38
  39 #include "putilimp.h"
  40 UDate startTime = -1.0;
  41
  42 static int elapsedTime() {
  43   return (int)uprv_floor((uprv_getRawUTCtime()-startTime)/1000.0);
  44 }
  45
  46 #if U_PLATFORM_IMPLEMENTS_POSIX && !U_PLATFORM_HAS_WIN32_API
  47 #include <signal.h>
  48 #include <unistd.h>
  49
  50 const char *wToolname="gendict";
  51 const char *wOutname="(some file)";
  52
  53 const int firstSeconds = 5; /* seconds between notices*/
  54 const int nextSeconds = 15; /* seconds between notices*/
  55
  56 static void alarm_fn(int /*n*/) {
  57   printf("%s: still writing\t%s (%ds)\t...\n",    wToolname, wOutname, elapsedTime());
  58
  59   signal(SIGALRM, &alarm_fn);
  60   alarm(nextSeconds); // reset the alarm
  61 }
  62
  63 static void install_watchdog(const char *toolName, const char *outFileName) {
  64   wToolname=toolName;
  65   wOutname=outFileName;
  66
  67   if(startTime<0) { // uninitialized
  68     startTime = uprv_getRawUTCtime();
  69   }
  70   signal(SIGALRM, &alarm_fn);
  71
  72   alarm(firstSeconds); // set the alarm
  73 }
  74
  75 #else
  76 static void install_watchdog(const char*, const char*) {
  77   // not implemented
  78 }
  79 #endif
  80
  81
  82
  83
  84 U_NAMESPACE_USE
  85
  86 static char *progName;
  87 static UOption options[]={
  88     UOPTION_HELP_H,             /* 0 */
  89     UOPTION_HELP_QUESTION_MARK, /* 1 */
  90     UOPTION_VERBOSE,            /* 2 */
  91     UOPTION_ICUDATADIR,         /* 4 */
  92     UOPTION_COPYRIGHT,          /* 5 */
  93     { "uchars", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0}, /* 6 */
  94     { "bytes", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0}, /* 7 */
  95     { "transform", NULL, NULL, NULL, '\1', UOPT_REQUIRES_ARG, 0}, /* 8 */
  96 };
  97
  98 enum arguments {
  99     ARG_HELP = 0,
 100     ARG_QMARK,
 101     ARG_VERBOSE,
 102     ARG_ICUDATADIR,
 103     ARG_COPYRIGHT,
 104     ARG_UCHARS,
 105     ARG_BYTES,
 106     ARG_TRANSFORM
 107 };
 108
 109 // prints out the standard usage method describing command line arguments,
 110 // then bails out with the desired exit code
 111 static void usageAndDie(UErrorCode retCode) {
 112     fprintf((U_SUCCESS(retCode) ? stdout : stderr), "Usage: %s -trietype [-options] input-dictionary-file output-file\n", progName);
 113     fprintf((U_SUCCESS(retCode) ? stdout : stderr),
 114            "\tRead in a word list and write out a string trie dictionary\n"
 115            "options:\n"
 116            "\t-h or -? or --help  this usage text\n"
 117            "\t-V or --version     show a version message\n"
 118            "\t-c or --copyright   include a copyright notice\n"
 119            "\t-v or --verbose     turn on verbose output\n"
 120            "\t-i or --icudatadir  directory for locating any needed intermediate data files,\n" // TODO: figure out if we need this option
 121            "\t                    followed by path, defaults to %s\n"
 122            "\t--uchars            output a UCharsTrie (mutually exclusive with -b!)\n"
 123            "\t--bytes             output a BytesTrie (mutually exclusive with -u!)\n"
 124            "\t--transform         the kind of transform to use (eg --transform offset-40A3,\n"
 125            "\t                    which specifies an offset transform with constant 0x40A3)\n",
 126             u_getDataDirectory());
 127     exit(retCode);
 128 }
 129
 130
 131 /* UDataInfo cf. udata.h */
 132 static UDataInfo dataInfo = {
 133     sizeof(UDataInfo),
 134     0,
 135
 136     U_IS_BIG_ENDIAN,
 137     U_CHARSET_FAMILY,
 138     U_SIZEOF_UCHAR,
 139     0,
 140
 141     { 0x44, 0x69, 0x63, 0x74 },     /* "Dict" */
 142     { 1, 0, 0, 0 },                 /* format version */
 143     { 0, 0, 0, 0 }                  /* data version */
 144 };
 145
 146 #if !UCONFIG_NO_BREAK_ITERATION
 147
 148 // A wrapper for both BytesTrieBuilder and UCharsTrieBuilder.
 149 // may want to put this somewhere in ICU, as it could be useful outside
 150 // of this tool?
 151 class DataDict {
 152 private:
 153     BytesTrieBuilder *bt;
 154     UCharsTrieBuilder *ut;
 155     UChar32 transformConstant;
 156     int32_t transformType;
 157 public:
 158     // constructs a new data dictionary. if there is an error,
 159     // it will be returned in status
 160     // isBytesTrie != 0 will produce a BytesTrieBuilder,
 161     // isBytesTrie == 0 will produce a UCharsTrieBuilder
 162     DataDict(UBool isBytesTrie, UErrorCode &status) : bt(NULL), ut(NULL),
 163         transformConstant(0), transformType(DictionaryData::TRANSFORM_NONE) {
 164         if (isBytesTrie) {
 165             bt = new BytesTrieBuilder(status);
 166         } else {
 167             ut = new UCharsTrieBuilder(status);
 168         }
 169     }
 170
 171     ~DataDict() {
 172         delete bt;
 173         delete ut;
 174     }
 175
 176 private:
 177     char transform(UChar32 c, UErrorCode &status) {
 178         if (transformType == DictionaryData::TRANSFORM_TYPE_OFFSET) {
 179             if (c == 0x200D) { return (char)0xFF; }
 180             else if (c == 0x200C) { return (char)0xFE; }
 181             int32_t delta = c - transformConstant;
 182             if (delta < 0 || 0xFD < delta) {
 183                 fprintf(stderr, "Codepoint U+%04lx out of range for --transform offset-%04lx!\n",
 184                         (long)c, (long)transformConstant);
 185                 exit(U_ILLEGAL_ARGUMENT_ERROR); // TODO: should return and print the line number
 186             }
 187             return (char)delta;
 188         } else { // no such transform type
 189             status = U_INTERNAL_PROGRAM_ERROR;
 190             return (char)c; // it should be noted this transform type will not generally work
 191         }
 192     }
 193
 194     void transform(const UnicodeString &word, CharString &buf, UErrorCode &errorCode) {
 195         UChar32 c = 0;
 196         int32_t len = word.length();
 197         for (int32_t i = 0; i < len; i += U16_LENGTH(c)) {
 198             c = word.char32At(i);
 199             buf.append(transform(c, errorCode), errorCode);
 200         }
 201     }
 202
 203 public:
 204     // sets the desired transformation data.
 205     // should be populated from a command line argument
 206     // so far the only acceptable format is offset-<hex constant>
 207     // eventually others (mask-<hex constant>?) may be enabled
 208     // more complex functions may be more difficult
 209     void setTransform(const char *t) {
 210         if (strncmp(t, "offset-", 7) == 0) {
 211             char *end;
 212             unsigned long base = uprv_strtoul(t + 7, &end, 16);
 213             if (end == (t + 7) || *end != 0 || base > 0x10FF80) {
 214                 fprintf(stderr, "Syntax for offset value in --transform offset-%s invalid!\n", t + 7);
 215                 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
 216             }
 217             transformType = DictionaryData::TRANSFORM_TYPE_OFFSET;
 218             transformConstant = (UChar32)base;
 219         }
 220         else {
 221             fprintf(stderr, "Invalid transform specified: %s\n", t);
 222             usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
 223         }
 224     }
 225
 226     // add a word to the trie
 227     void addWord(const UnicodeString &word, int32_t value, UErrorCode &status) {
 228         if (bt) {
 229             CharString buf;
 230             transform(word, buf, status);
 231             bt->add(buf.toStringPiece(), value, status);
 232         }
 233         if (ut) { ut->add(word, value, status); }
 234     }
 235
 236     // if we are a bytestrie, give back the StringPiece representing the serialized version of us
 237     StringPiece serializeBytes(UErrorCode &status) {
 238         return bt->buildStringPiece(USTRINGTRIE_BUILD_SMALL, status);
 239     }
 240
 241     // if we are a ucharstrie, produce the UnicodeString representing the serialized version of us
 242     void serializeUChars(UnicodeString &s, UErrorCode &status) {
 243         ut->buildUnicodeString(USTRINGTRIE_BUILD_SMALL, s, status);
 244     }
 245
 246     int32_t getTransform() {
 247         return (int32_t)(transformType | transformConstant);
 248     }
 249 };
 250 #endif
 251
 252 static const UChar LINEFEED_CHARACTER = 0x000A;
 253 static const UChar CARRIAGE_RETURN_CHARACTER = 0x000D;
 254
 255 static UBool readLine(UCHARBUF *f, UnicodeString &fileLine, IcuToolErrorCode &errorCode) {
 256     int32_t lineLength;
 257     const UChar *line = ucbuf_readline(f, &lineLength, errorCode);
 258     if(line == NULL || errorCode.isFailure()) { return FALSE; }
 259     // Strip trailing CR/LF, comments, and spaces.
 260     const UChar *comment = u_memchr(line, 0x23, lineLength);  // '#'
 261     if(comment != NULL) {
 262         lineLength = (int32_t)(comment - line);
 263     } else {
 264         while(lineLength > 0 && (line[lineLength - 1] == CARRIAGE_RETURN_CHARACTER || line[lineLength - 1] == LINEFEED_CHARACTER)) { --lineLength; }
 265     }
 266     while(lineLength > 0 && u_isspace(line[lineLength - 1])) { --lineLength; }
 267     fileLine.setTo(FALSE, line, lineLength);
 268     return TRUE;
 269 }
 270
 271 //----------------------------------------------------------------------------
 272 //
 273 //  main      for gendict
 274 //
 275 //----------------------------------------------------------------------------
 276 int  main(int argc, char **argv) {
 277     //
 278     // Pick up and check the command line arguments,
 279     //    using the standard ICU tool utils option handling.
 280     //
 281     U_MAIN_INIT_ARGS(argc, argv);
 282     progName = argv[0];
 283     argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
 284     if(argc<0) {
 285         // Unrecognized option
 286         fprintf(stderr, "error in command line argument \"%s\"\n", argv[-argc]);
 287         usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
 288     }
 289
 290     if(options[ARG_HELP].doesOccur || options[ARG_QMARK].doesOccur) {
 291         //  -? or -h for help.
 292         usageAndDie(U_ZERO_ERROR);
 293     }
 294
 295     UBool verbose = options[ARG_VERBOSE].doesOccur;
 296
 297     if (argc < 3) {
 298         fprintf(stderr, "input and output file must both be specified.\n");
 299         usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
 300     }
 301     const char *outFileName  = argv[2];
 302     const char *wordFileName = argv[1];
 303
 304     // set up the watchdog
 305     install_watchdog(progName, outFileName);
 306
 307     if (options[ARG_ICUDATADIR].doesOccur) {
 308         u_setDataDirectory(options[ARG_ICUDATADIR].value);
 309     }
 310
 311     const char *copyright = NULL;
 312     if (options[ARG_COPYRIGHT].doesOccur) {
 313         copyright = U_COPYRIGHT_STRING;
 314     }
 315
 316     if (options[ARG_UCHARS].doesOccur == options[ARG_BYTES].doesOccur) {
 317         fprintf(stderr, "you must specify exactly one type of trie to output!\n");
 318         usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
 319     }
 320     UBool isBytesTrie = options[ARG_BYTES].doesOccur;
 321     if (isBytesTrie != options[ARG_TRANSFORM].doesOccur) {
 322         fprintf(stderr, "you must provide a transformation for a bytes trie, and must not provide one for a uchars trie!\n");
 323         usageAndDie(U_ILLEGAL_ARGUMENT_ERROR);
 324     }
 325
 326     IcuToolErrorCode status("gendict/main()");
 327
 328 #if UCONFIG_NO_BREAK_ITERATION || UCONFIG_NO_FILE_IO
 329     const char* outDir=NULL;
 330
 331     UNewDataMemory *pData;
 332     char msg[1024];
 333     UErrorCode tempstatus = U_ZERO_ERROR;
 334
 335     /* write message with just the name */ // potential for a buffer overflow here...
 336     sprintf(msg, "gendict writes dummy %s because of UCONFIG_NO_BREAK_ITERATION and/or UCONFIG_NO_FILE_IO, see uconfig.h", outFileName);
 337     fprintf(stderr, "%s\n", msg);
 338
 339     /* write the dummy data file */
 340     pData = udata_create(outDir, NULL, outFileName, &dataInfo, NULL, &tempstatus);
 341     udata_writeBlock(pData, msg, strlen(msg));
 342     udata_finish(pData, &tempstatus);
 343     return (int)tempstatus;
 344
 345 #else
 346     //  Read in the dictionary source file
 347     if (verbose) { printf("Opening file %s...\n", wordFileName); }
 348     const char *codepage = "UTF-8";
 349     UCHARBUF *f = ucbuf_open(wordFileName, &codepage, TRUE, FALSE, status);
 350     if (status.isFailure()) {
 351         fprintf(stderr, "error opening input file: ICU Error \"%s\"\n", status.errorName());
 352         exit(status.reset());
 353     }
 354     if (verbose) { printf("Initializing dictionary builder of type %s...\n", (isBytesTrie ? "BytesTrie" : "UCharsTrie")); }
 355     DataDict dict(isBytesTrie, status);
 356     if (status.isFailure()) {
 357         fprintf(stderr, "new DataDict: ICU Error \"%s\"\n", status.errorName());
 358         exit(status.reset());
 359     }
 360     if (options[ARG_TRANSFORM].doesOccur) {
 361         dict.setTransform(options[ARG_TRANSFORM].value);
 362     }
 363
 364     UnicodeString fileLine;
 365     if (verbose) { puts("Adding words to dictionary..."); }
 366     UBool hasValues = FALSE;
 367     UBool hasValuelessContents = FALSE;
 368     int lineCount = 0;
 369     UBool isOk = TRUE;
 370     while (readLine(f, fileLine, status)) {
 371         lineCount++;
 372         if (fileLine.isEmpty()) continue;
 373
 374         // Parse word [spaces value].
 375         int32_t keyLen;
 376         for (keyLen = 0; keyLen < fileLine.length() && !u_isspace(fileLine[keyLen]); ++keyLen) {}
 377         if (keyLen == 0) {
 378             fprintf(stderr, "Error: no word on line %i!\n", lineCount);
 379             isOk = FALSE;
 380             continue;
 381         }
 382         int32_t valueStart;
 383         for (valueStart = keyLen;
 384             valueStart < fileLine.length() && u_isspace(fileLine[valueStart]);
 385             ++valueStart) {}
 386
 387         if (keyLen < valueStart) {
 388             int32_t valueLength = fileLine.length() - valueStart;
 389             if (valueLength > 15) {
 390                 fprintf(stderr, "Error: value too long on line %i!\n", lineCount);
 391                 isOk = FALSE;
 392                 continue;
 393             }
 394             char s[16];
 395             fileLine.extract(valueStart, valueLength, s, 16, US_INV);
 396             char *end;
 397             unsigned long value = uprv_strtoul(s, &end, 0);
 398             if (end == s || *end != 0 || (int32_t)uprv_strlen(s) != valueLength || value > 0xffffffff) {
 399                 fprintf(stderr, "Error: value syntax error or value too large on line %i!\n", lineCount);
 400                 isOk = FALSE;
 401                 continue;
 402             }
 403             dict.addWord(fileLine.tempSubString(0, keyLen), (int32_t)value, status);
 404             hasValues = TRUE;
 405         } else {
 406             dict.addWord(fileLine.tempSubString(0, keyLen), 0, status);
 407             hasValuelessContents = FALSE;
 408         }
 409
 410         if (status.isFailure()) {
 411             fprintf(stderr, "ICU Error \"%s\": Failed to add word to trie at input line %d in input file\n",
 412                 status.errorName(), lineCount);
 413             exit(status.reset());
 414         }
 415     }
 416
 417     if (!isOk && status.isSuccess()) {
 418         status.set(U_ILLEGAL_ARGUMENT_ERROR);
 419     }
 420     if (hasValues && hasValuelessContents) {
 421         fprintf(stderr, "warning: file contained both valued and unvalued strings!\n");
 422     }
 423
 424     if (verbose) { puts("Serializing data..."); }
 425     int32_t outDataSize;
 426     const void *outData;
 427     UnicodeString usp;
 428     if (isBytesTrie) {
 429         StringPiece sp = dict.serializeBytes(status);
 430         outDataSize = sp.size();
 431         outData = sp.data();
 432     } else {
 433         dict.serializeUChars(usp, status);
 434         outDataSize = usp.length() * U_SIZEOF_UCHAR;
 435         outData = usp.getBuffer();
 436     }
 437     if (status.isFailure()) {
 438         fprintf(stderr, "gendict: got failure of type %s while serializing\n", status.errorName());
 439         exit(status.reset());
 440     }
 441     if (verbose) { puts("Opening output file..."); }
 442     UNewDataMemory *pData = udata_create(NULL, NULL, outFileName, &dataInfo, copyright, status);
 443     if (status.isFailure()) {
 444         fprintf(stderr, "gendict: could not open output file \"%s\", \"%s\"\n", outFileName, status.errorName());
 445         exit(status.reset());
 446     }
 447
 448     if (verbose) { puts("Writing to output file..."); }
 449     int32_t indexes[DictionaryData::IX_COUNT] = {
 450         DictionaryData::IX_COUNT * sizeof(int32_t), 0, 0, 0, 0, 0, 0, 0
 451     };
 452     int32_t size = outDataSize + indexes[DictionaryData::IX_STRING_TRIE_OFFSET];
 453     indexes[DictionaryData::IX_RESERVED1_OFFSET] = size;
 454     indexes[DictionaryData::IX_RESERVED2_OFFSET] = size;
 455     indexes[DictionaryData::IX_TOTAL_SIZE] = size;
 456
 457     indexes[DictionaryData::IX_TRIE_TYPE] = isBytesTrie ? DictionaryData::TRIE_TYPE_BYTES : DictionaryData::TRIE_TYPE_UCHARS;
 458     if (hasValues) {
 459         indexes[DictionaryData::IX_TRIE_TYPE] |= DictionaryData::TRIE_HAS_VALUES;
 460     }
 461
 462     indexes[DictionaryData::IX_TRANSFORM] = dict.getTransform();
 463     udata_writeBlock(pData, indexes, sizeof(indexes));
 464     udata_writeBlock(pData, outData, outDataSize);
 465     size_t bytesWritten = udata_finish(pData, status);
 466     if (status.isFailure()) {
 467         fprintf(stderr, "gendict: error \"%s\" writing the output file\n", status.errorName());
 468         exit(status.reset());
 469     }
 470
 471     if (bytesWritten != (size_t)size) {
 472         fprintf(stderr, "Error writing to output file \"%s\"\n", outFileName);
 473         exit(U_INTERNAL_PROGRAM_ERROR);
 474     }
 475
 476     printf("%s: done writing\t%s (%ds).\n", progName, outFileName, elapsedTime());
 477
 478 #ifdef TEST_GENDICT
 479     if (isBytesTrie) {
 480         BytesTrie::Iterator it(outData, outDataSize, status);
 481         while (it.hasNext()) {
 482             it.next(status);
 483             const StringPiece s = it.getString();
 484             int32_t val = it.getValue();
 485             printf("%s -> %i\n", s.data(), val);
 486         }
 487     } else {
 488         UCharsTrie::Iterator it((const UChar *)outData, outDataSize, status);
 489         while (it.hasNext()) {
 490             it.next(status);
 491             const UnicodeString s = it.getString();
 492             int32_t val = it.getValue();
 493             char tmp[1024];
 494             s.extract(0, s.length(), tmp, 1024);
 495             printf("%s -> %i\n", tmp, val);
 496         }
 497     }
 498 #endif
 499
 500     return 0;
 501 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */
 502 }