icuSources/tools/gencnval/gencnval.c

   1 /*
   2 *******************************************************************************
   3 *
   4 *   Copyright (C) 1999-2004, International Business Machines
   5 *   Corporation and others.  All Rights Reserved.
   6 *
   7 *******************************************************************************
   8 *   file name:  gencnval.c
   9 *   encoding:   US-ASCII
  10 *   tab size:   8 (not used)
  11 *   indentation:4
  12 *
  13 *   created on: 1999nov05
  14 *   created by: Markus W. Scherer
  15 *
  16 *   This program reads convrtrs.txt and writes a memory-mappable
  17 *   converter name alias table to cnvalias.dat .
  18 *
  19 *   This program currently writes version 2.1 of the data format. See
  20 *   ucnv_io.c for more details on the format. Note that version 2.1
  21 *   is written in such a way that a 2.0 reader will be able to use it,
  22 *   and a 2.1 reader will be able to read 2.0.
  23 */
  24
  25 #include "unicode/utypes.h"
  26 #include "unicode/putil.h"
  27 #include "unicode/ucnv.h" /* ucnv_compareNames() */
  28 #include "ucnv_io.h"
  29 #include "cmemory.h"
  30 #include "cstring.h"
  31 #include "uinvchar.h"
  32 #include "filestrm.h"
  33 #include "unicode/uclean.h"
  34 #include "unewdata.h"
  35 #include "uoptions.h"
  36
  37 #include <stdio.h>
  38 #include <stdlib.h>
  39 #include <ctype.h>
  40
  41 /* TODO: Need to check alias name length is less than UCNV_MAX_CONVERTER_NAME_LENGTH */
  42
  43 /* STRING_STORE_SIZE + TAG_STORE_SIZE <= ((2^16 - 1) * 2)
  44  That is the maximum size for the string stores combined
  45  because the strings are index at 16-bit boundries by a
  46  16-bit index, and there is only one section for the
  47  strings.
  48  */
  49 #define STRING_STORE_SIZE 0x1FBFE   /* 130046 */
  50 #define TAG_STORE_SIZE      0x400   /* 1024 */
  51
  52 /* The combined tag and converter count can affect the number of lists
  53  created.  The size of all lists must be less than (2^17 - 1)
  54  because the lists are indexed as a 16-bit array with a 16-bit index.
  55  */
  56 #define MAX_TAG_COUNT 0x3F      /* 63 */
  57 #define MAX_CONV_COUNT UCNV_CONVERTER_INDEX_MASK
  58 #define MAX_ALIAS_COUNT 0xFFFF  /* 65535 */
  59
  60 /* The maximum number of aliases that a standard tag/converter combination can have.
  61  At this moment 6/18/2002, IANA has 12 names for ASCII. Don't go below 15 for
  62  this value. I don't recommend more than 31 for this value.
  63  */
  64 #define MAX_TC_ALIAS_COUNT 0x1F    /* 31 */
  65
  66 #define MAX_LINE_SIZE 0x7FFF    /* 32767 */
  67 #define MAX_LIST_SIZE 0xFFFF    /* 65535 */
  68
  69 #define DATA_NAME "cnvalias"
  70 #define DATA_TYPE "icu" /* ICU alias table */
  71
  72 #define ALL_TAG_STR "ALL"
  73 #define ALL_TAG_NUM 1
  74 #define EMPTY_TAG_NUM 0
  75
  76 /* UDataInfo cf. udata.h */
  77 static const UDataInfo dataInfo={
  78     sizeof(UDataInfo),
  79     0,
  80
  81     U_IS_BIG_ENDIAN,
  82     U_CHARSET_FAMILY,
  83     sizeof(UChar),
  84     0,
  85
  86     {0x43, 0x76, 0x41, 0x6c},     /* dataFormat="CvAl" */
  87     {3, 0, 0, 0},                 /* formatVersion */
  88     {1, 4, 2, 0}                  /* dataVersion */
  89 };
  90
  91 typedef struct {
  92     char *store;
  93     uint32_t top;
  94     uint32_t max;
  95 } StringBlock;
  96
  97 static char stringStore[STRING_STORE_SIZE];
  98 static StringBlock stringBlock = { stringStore, 0, STRING_STORE_SIZE };
  99
 100 typedef struct {
 101     uint16_t    aliasCount;
 102     uint16_t    *aliases;     /* Index into stringStore */
 103 } AliasList;
 104
 105 typedef struct {
 106     uint16_t converter;     /* Index into stringStore */
 107     uint16_t totalAliasCount;    /* Total aliases in this column */
 108 } Converter;
 109
 110 static Converter converters[MAX_CONV_COUNT];
 111 static uint16_t converterCount=0;
 112
 113 static char tagStore[TAG_STORE_SIZE];
 114 static StringBlock tagBlock = { tagStore, 0, TAG_STORE_SIZE };
 115
 116 typedef struct {
 117     uint16_t    tag;        /* Index into tagStore */
 118     uint16_t    totalAliasCount; /* Total aliases in this row */
 119     AliasList   aliasList[MAX_CONV_COUNT];
 120 } Tag;
 121
 122 /* Think of this as a 3D array. It's tagCount by converterCount by aliasCount */
 123 static Tag tags[MAX_TAG_COUNT];
 124 static uint16_t tagCount = 0;
 125
 126 /* Used for storing all aliases  */
 127 static uint16_t knownAliases[MAX_ALIAS_COUNT];
 128 static uint16_t knownAliasesCount = 0;
 129 /*static uint16_t duplicateKnownAliasesCount = 0;*/
 130
 131 /* Used for storing the lists section that point to aliases */
 132 static uint16_t aliasLists[MAX_LIST_SIZE];
 133 static uint16_t aliasListsSize = 0;
 134
 135 /* Were the standard tags declared before the aliases. */
 136 static UBool standardTagsUsed = FALSE;
 137 static UBool verbose = FALSE;
 138 static int lineNum = 1;
 139
 140 /* prototypes --------------------------------------------------------------- */
 141
 142 static void
 143 parseLine(const char *line);
 144
 145 static void
 146 parseFile(FileStream *in);
 147
 148 static int32_t
 149 chomp(char *line);
 150
 151 static void
 152 addOfficialTaggedStandards(char *line, int32_t lineLen);
 153
 154 static uint16_t
 155 addAlias(const char *alias, uint16_t standard, uint16_t converter, UBool defaultName);
 156
 157 static uint16_t
 158 addConverter(const char *converter);
 159
 160 static char *
 161 allocString(StringBlock *block, const char *s, int32_t length);
 162
 163 static uint16_t
 164 addToKnownAliases(const char *alias);
 165
 166 static int
 167 compareAliases(const void *alias1, const void *alias2);
 168
 169 static uint16_t
 170 getTagNumber(const char *tag, uint16_t tagLen);
 171
 172 /*static void
 173 addTaggedAlias(uint16_t tag, const char *alias, uint16_t converter);*/
 174
 175 static void
 176 writeAliasTable(UNewDataMemory *out);
 177
 178 /* -------------------------------------------------------------------------- */
 179
 180 /* Presumes that you used allocString() */
 181 #define GET_ALIAS_STR(index) (stringStore + ((size_t)(index) << 1))
 182 #define GET_TAG_STR(index) (tagStore + ((size_t)(index) << 1))
 183
 184 /* Presumes that you used allocString() */
 185 #define GET_ALIAS_NUM(str) ((uint16_t)((str - stringStore) >> 1))
 186 #define GET_TAG_NUM(str) ((uint16_t)((str - tagStore) >> 1))
 187
 188 enum
 189 {
 190     HELP1,
 191     HELP2,
 192     VERBOSE,
 193     COPYRIGHT,
 194     DESTDIR,
 195     SOURCEDIR
 196 };
 197
 198 static UOption options[]={
 199     UOPTION_HELP_H,
 200     UOPTION_HELP_QUESTION_MARK,
 201     UOPTION_VERBOSE,
 202     UOPTION_COPYRIGHT,
 203     UOPTION_DESTDIR,
 204     UOPTION_SOURCEDIR
 205 };
 206
 207 extern int
 208 main(int argc, char* argv[]) {
 209     char pathBuf[512];
 210     const char *path;
 211     FileStream *in;
 212     UNewDataMemory *out;
 213     UErrorCode errorCode=U_ZERO_ERROR;
 214
 215     U_MAIN_INIT_ARGS(argc, argv);
 216
 217     /* preset then read command line options */
 218     options[DESTDIR].value=options[SOURCEDIR].value=u_getDataDirectory();
 219     argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
 220
 221     /* error handling, printing usage message */
 222     if(argc<0) {
 223         fprintf(stderr,
 224             "error in command line argument \"%s\"\n",
 225             argv[-argc]);
 226     }
 227     if(argc<0 || options[HELP1].doesOccur || options[HELP2].doesOccur) {
 228         fprintf(stderr,
 229             "usage: %s [-options] [convrtrs.txt]\n"
 230             "\tread convrtrs.txt and create " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE "\n"
 231             "options:\n"
 232             "\t-h or -? or --help  this usage text\n"
 233             "\t-v or --verbose     prints out extra information about the alias table\n"
 234             "\t-c or --copyright   include a copyright notice\n"
 235             "\t-d or --destdir     destination directory, followed by the path\n"
 236             "\t-s or --sourcedir   source directory, followed by the path\n",
 237             argv[0]);
 238         return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
 239     }
 240
 241     if(options[VERBOSE].doesOccur) {
 242         verbose = TRUE;
 243     }
 244
 245     if(argc>=2) {
 246         path=argv[1];
 247     } else {
 248         path=options[SOURCEDIR].value;
 249         if(path!=NULL && *path!=0) {
 250             char *end;
 251
 252             uprv_strcpy(pathBuf, path);
 253             end = uprv_strchr(pathBuf, 0);
 254             if(*(end-1)!=U_FILE_SEP_CHAR) {
 255                 *(end++)=U_FILE_SEP_CHAR;
 256             }
 257             uprv_strcpy(end, "convrtrs.txt");
 258             path=pathBuf;
 259         } else {
 260             path = "convrtrs.txt";
 261         }
 262     }
 263
 264     uprv_memset(stringStore, 0, sizeof(stringStore));
 265     uprv_memset(tagStore, 0, sizeof(tagStore));
 266     uprv_memset(converters, 0, sizeof(converters));
 267     uprv_memset(tags, 0, sizeof(tags));
 268     uprv_memset(aliasLists, 0, sizeof(aliasLists));
 269     uprv_memset(knownAliases, 0, sizeof(aliasLists));
 270
 271
 272     in=T_FileStream_open(path, "r");
 273     if(in==NULL) {
 274         fprintf(stderr, "gencnval: unable to open input file convrtrs.txt\n");
 275         exit(U_FILE_ACCESS_ERROR);
 276     }
 277     parseFile(in);
 278     T_FileStream_close(in);
 279
 280     /* create the output file */
 281     out=udata_create(options[DESTDIR].value, DATA_TYPE, DATA_NAME, &dataInfo,
 282                      options[COPYRIGHT].doesOccur ? U_COPYRIGHT_STRING : NULL, &errorCode);
 283     if(U_FAILURE(errorCode)) {
 284         fprintf(stderr, "gencnval: unable to open output file - error %s\n", u_errorName(errorCode));
 285         exit(errorCode);
 286     }
 287
 288     /* write the table of aliases based on a tag/converter name combination */
 289     writeAliasTable(out);
 290
 291     /* finish */
 292     udata_finish(out, &errorCode);
 293     if(U_FAILURE(errorCode)) {
 294         fprintf(stderr, "gencnval: error finishing output file - %s\n", u_errorName(errorCode));
 295         exit(errorCode);
 296     }
 297
 298     return 0;
 299 }
 300
 301 static void
 302 parseFile(FileStream *in) {
 303     char line[MAX_LINE_SIZE];
 304     char lastLine[MAX_LINE_SIZE];
 305     int32_t lineSize = 0;
 306     int32_t lastLineSize = 0;
 307     UBool validParse = TRUE;
 308
 309     lineNum = 0;
 310
 311     /* Add the empty tag, which is for untagged aliases */
 312     getTagNumber("", 0);
 313     getTagNumber(ALL_TAG_STR, 3);
 314     allocString(&stringBlock, "", 0);
 315
 316     /* read the list of aliases */
 317     while (validParse) {
 318         validParse = FALSE;
 319
 320         /* Read non-empty lines that don't start with a space character. */
 321         while (T_FileStream_readLine(in, lastLine, MAX_LINE_SIZE) != NULL) {
 322             lastLineSize = chomp(lastLine);
 323             if (lineSize == 0 || (lastLineSize > 0 && isspace(*lastLine))) {
 324                 uprv_strcpy(line + lineSize, lastLine);
 325                 lineSize += lastLineSize;
 326             } else if (lineSize > 0) {
 327                 validParse = TRUE;
 328                 break;
 329             }
 330             lineNum++;
 331         }
 332
 333         if (validParse || lineSize > 0) {
 334             if (isspace(*line)) {
 335                 fprintf(stderr, "error(line %d): cannot start an alias with a space\n", lineNum-1);
 336                 exit(U_PARSE_ERROR);
 337             } else if (line[0] == '{') {
 338                 if (!standardTagsUsed && line[lineSize - 1] != '}') {
 339                     fprintf(stderr, "error(line %d): alias needs to start with a converter name\n", lineNum);
 340                     exit(U_PARSE_ERROR);
 341                 }
 342                 addOfficialTaggedStandards(line, lineSize);
 343                 standardTagsUsed = TRUE;
 344             } else {
 345                 if (standardTagsUsed) {
 346                     parseLine(line);
 347                 }
 348                 else {
 349                     fprintf(stderr, "error(line %d): alias table needs to start a list of standard tags\n", lineNum);
 350                     exit(U_PARSE_ERROR);
 351                 }
 352             }
 353             /* Was the last line consumed */
 354             if (lastLineSize > 0) {
 355                 uprv_strcpy(line, lastLine);
 356                 lineSize = lastLineSize;
 357             }
 358             else {
 359                 lineSize = 0;
 360             }
 361         }
 362         lineNum++;
 363     }
 364 }
 365
 366 /* This works almost like the Perl chomp.
 367  It removes the newlines, comments and trailing whitespace (not preceding whitespace).
 368 */
 369 static int32_t
 370 chomp(char *line) {
 371     char *s = line;
 372     char *lastNonSpace = line;
 373     while(*s!=0) {
 374         /* truncate at a newline or a comment */
 375         if(*s == '\r' || *s == '\n' || *s == '#') {
 376             *s = 0;
 377             break;
 378         }
 379         if (!isspace(*s)) {
 380             lastNonSpace = s;
 381         }
 382         ++s;
 383     }
 384     if (lastNonSpace++ > line) {
 385         *lastNonSpace = 0;
 386         s = lastNonSpace;
 387     }
 388     return (int32_t)(s - line);
 389 }
 390
 391 static void
 392 parseLine(const char *line) {
 393     uint16_t pos=0, start, limit, length, cnv;
 394     char *converter, *alias;
 395
 396     /* skip leading white space */
 397     /* There is no whitespace at the beginning anymore */
 398 /*    while(line[pos]!=0 && isspace(line[pos])) {
 399         ++pos;
 400     }
 401 */
 402
 403     /* is there nothing on this line? */
 404     if(line[pos]==0) {
 405         return;
 406     }
 407
 408     /* get the converter name */
 409     start=pos;
 410     while(line[pos]!=0 && !isspace(line[pos])) {
 411         ++pos;
 412     }
 413     limit=pos;
 414
 415     /* store the converter name */
 416     length=(uint16_t)(limit-start);
 417     converter=allocString(&stringBlock, line+start, length);
 418
 419     /* add the converter to the converter table */
 420     cnv=addConverter(converter);
 421
 422     /* The name itself may be tagged, so let's added it to the aliases list properly */
 423     pos = start;
 424
 425     /* get all the real aliases */
 426     for(;;) {
 427
 428         /* skip white space */
 429         while(line[pos]!=0 && isspace(line[pos])) {
 430             ++pos;
 431         }
 432
 433         /* is there no more alias name on this line? */
 434         if(line[pos]==0) {
 435             break;
 436         }
 437
 438         /* get an alias name */
 439         start=pos;
 440         while(line[pos]!=0 && line[pos]!='{' && !isspace(line[pos])) {
 441             ++pos;
 442         }
 443         limit=pos;
 444
 445         /* store the alias name */
 446         length=(uint16_t)(limit-start);
 447         if (start == 0) {
 448             /* add the converter as its own alias to the alias table */
 449             alias = converter;
 450             addAlias(alias, ALL_TAG_NUM, cnv, TRUE);
 451         }
 452         else {
 453             alias=allocString(&stringBlock, line+start, length);
 454             addAlias(alias, ALL_TAG_NUM, cnv, FALSE);
 455         }
 456         addToKnownAliases(alias);
 457
 458         /* add the alias/converter pair to the alias table */
 459         /* addAlias(alias, 0, cnv, FALSE);*/
 460
 461         /* skip whitespace */
 462         while (line[pos] && isspace(line[pos])) {
 463             ++pos;
 464         }
 465
 466         /* handle tags if they are present */
 467         if (line[pos] == '{') {
 468             ++pos;
 469             do {
 470                 start = pos;
 471                 while (line[pos] && line[pos] != '}' && !isspace( line[pos])) {
 472                     ++pos;
 473                 }
 474                 limit = pos;
 475
 476                 if (start != limit) {
 477                     /* add the tag to the tag table */
 478                     uint16_t tag = getTagNumber(line + start, (uint16_t)(limit - start));
 479                     addAlias(alias, tag, cnv, (UBool)(line[limit-1] == '*'));
 480                 }
 481
 482                 while (line[pos] && isspace(line[pos])) {
 483                     ++pos;
 484                 }
 485             } while (line[pos] && line[pos] != '}');
 486
 487             if (line[pos] == '}') {
 488                 ++pos;
 489             } else {
 490                 fprintf(stderr, "error(line %d): Unterminated tag list\n", lineNum);
 491                 exit(U_UNMATCHED_BRACES);
 492             }
 493         } else {
 494             addAlias(alias, EMPTY_TAG_NUM, cnv, (UBool)(tags[0].aliasList[cnv].aliasCount == 0));
 495         }
 496     }
 497 }
 498
 499 static uint16_t
 500 getTagNumber(const char *tag, uint16_t tagLen) {
 501     char *atag;
 502     uint16_t t;
 503     UBool preferredName = ((tagLen > 0) ? (tag[tagLen - 1] == '*') : (FALSE));
 504
 505     if (tagCount >= MAX_TAG_COUNT) {
 506         fprintf(stderr, "error(line %d): too many tags\n", lineNum);
 507         exit(U_BUFFER_OVERFLOW_ERROR);
 508     }
 509
 510     if (preferredName) {
 511 /*        puts(tag);*/
 512         tagLen--;
 513     }
 514
 515     for (t = 0; t < tagCount; ++t) {
 516         const char *currTag = GET_TAG_STR(tags[t].tag);
 517         if (uprv_strlen(currTag) == tagLen && !uprv_strnicmp(currTag, tag, tagLen)) {
 518             return t;
 519         }
 520     }
 521
 522     /* we need to add this tag */
 523     if (tagCount >= MAX_TAG_COUNT) {
 524         fprintf(stderr, "error(line %d): too many tags\n", lineNum);
 525         exit(U_BUFFER_OVERFLOW_ERROR);
 526     }
 527
 528     /* allocate a new entry in the tag table */
 529     atag = allocString(&tagBlock, tag, tagLen);
 530
 531     if (standardTagsUsed) {
 532         fprintf(stderr, "error(line %d): Tag \"%s\" is not declared at the beginning of the alias table.\n",
 533             lineNum, atag);
 534         exit(1);
 535     }
 536     else if (tagLen > 0 && strcmp(tag, ALL_TAG_STR) != 0) {
 537         fprintf(stderr, "warning(line %d): Tag \"%s\" was added to the list of standards because it was not declared at beginning of the alias table.\n",
 538             lineNum, atag);
 539     }
 540
 541     /* add the tag to the tag table */
 542     tags[tagCount].tag = GET_TAG_NUM(atag);
 543     /* The aliasList should be set to 0's already */
 544
 545     return tagCount++;
 546 }
 547
 548 /*static void
 549 addTaggedAlias(uint16_t tag, const char *alias, uint16_t converter) {
 550     tags[tag].aliases[converter] = alias;
 551 }
 552 */
 553
 554 static void
 555 addOfficialTaggedStandards(char *line, int32_t lineLen) {
 556     char *atag;
 557     char *tag = strchr(line, '{') + 1;
 558     static const char WHITESPACE[] = " \t";
 559
 560     if (tagCount > UCNV_NUM_RESERVED_TAGS) {
 561         fprintf(stderr, "error(line %d): official tags already added\n", lineNum);
 562         exit(U_BUFFER_OVERFLOW_ERROR);
 563     }
 564     strchr(tag, '}')[0] = 0;
 565
 566     tag = strtok(tag, WHITESPACE);
 567     while (tag != NULL) {
 568 /*        printf("Adding original tag \"%s\"\n", tag);*/
 569
 570         /* allocate a new entry in the tag table */
 571         atag = allocString(&tagBlock, tag, -1);
 572
 573         /* add the tag to the tag table */
 574         tags[tagCount++].tag = (uint16_t)((atag - tagStore) >> 1);
 575
 576         /* The aliasList should already be set to 0's */
 577
 578         /* Get next tag */
 579         tag = strtok(NULL, WHITESPACE);
 580     }
 581 }
 582
 583 static uint16_t
 584 addToKnownAliases(const char *alias) {
 585 /*    uint32_t idx; */
 586     /* strict matching */
 587 /*    for (idx = 0; idx < knownAliasesCount; idx++) {
 588         uint16_t num = GET_ALIAS_NUM(alias);
 589         if (knownAliases[idx] != num
 590             && uprv_strcmp(alias, GET_ALIAS_STR(knownAliases[idx])) == 0)
 591         {
 592             fprintf(stderr, "warning(line %d): duplicate alias %s and %s found\n",
 593                 lineNum, alias, GET_ALIAS_STR(knownAliases[idx]));
 594             duplicateKnownAliasesCount++;
 595             break;
 596         }
 597         else if (knownAliases[idx] != num
 598             && ucnv_compareNames(alias, GET_ALIAS_STR(knownAliases[idx])) == 0)
 599         {
 600             if (verbose) {
 601                 fprintf(stderr, "information(line %d): duplicate alias %s and %s found\n",
 602                     lineNum, alias, GET_ALIAS_STR(knownAliases[idx]));
 603             }
 604             duplicateKnownAliasesCount++;
 605             break;
 606         }
 607     }
 608 */
 609     if (knownAliasesCount >= MAX_ALIAS_COUNT) {
 610         fprintf(stderr, "warning(line %d): Too many aliases defined for all converters\n",
 611             lineNum);
 612         exit(U_BUFFER_OVERFLOW_ERROR);
 613     }
 614     /* TODO: We could try to unlist exact duplicates. */
 615     return knownAliases[knownAliasesCount++] = GET_ALIAS_NUM(alias);
 616 }
 617
 618 /*
 619 @param standard When standard is 0, then it's the "empty" tag.
 620 */
 621 static uint16_t
 622 addAlias(const char *alias, uint16_t standard, uint16_t converter, UBool defaultName) {
 623     uint32_t idx, idx2;
 624     UBool dupFound = FALSE;
 625     UBool startEmptyWithoutDefault = FALSE;
 626     AliasList *aliasList;
 627
 628     if(standard>=MAX_TAG_COUNT) {
 629         fprintf(stderr, "error(line %d): too many standard tags\n", lineNum);
 630         exit(U_BUFFER_OVERFLOW_ERROR);
 631     }
 632     if(converter>=MAX_CONV_COUNT) {
 633         fprintf(stderr, "error(line %d): too many converter names\n", lineNum);
 634         exit(U_BUFFER_OVERFLOW_ERROR);
 635     }
 636     aliasList = &tags[standard].aliasList[converter];
 637
 638     if (strchr(alias, '}')) {
 639         fprintf(stderr, "error(line %d): unmatched } found\n",
 640             lineNum);
 641     }
 642
 643     if(aliasList->aliasCount + 1 >= MAX_TC_ALIAS_COUNT) {
 644         fprintf(stderr, "error(line %d): too many aliases for alias %s and converter %s\n",
 645             lineNum, alias, GET_ALIAS_STR(converters[converter].converter));
 646         exit(U_BUFFER_OVERFLOW_ERROR);
 647     }
 648
 649     /* Show this warning only once. All aliases are added to the "ALL" tag. */
 650     if (standard == ALL_TAG_NUM && GET_ALIAS_STR(converters[converter].converter) != alias) {
 651         /* Normally these option values are parsed at runtime, and they can
 652            be discarded when the alias is a default converter. Options should
 653            only be on a converter and not an alias. */
 654         if (uprv_strchr(alias, UCNV_OPTION_SEP_CHAR) != 0)
 655         {
 656             fprintf(stderr, "warning(line %d): alias %s contains a \""UCNV_OPTION_SEP_STRING"\". Options are parsed at run-time and do not need to be in the alias table.\n",
 657                 lineNum, alias);
 658         }
 659         if (uprv_strchr(alias, UCNV_VALUE_SEP_CHAR) != 0)
 660         {
 661             fprintf(stderr, "warning(line %d): alias %s contains an \""UCNV_VALUE_SEP_STRING"\". Options are parsed at run-time and do not need to be in the alias table.\n",
 662                 lineNum, alias);
 663         }
 664     }
 665
 666     /* Check for duplicates in a tag/converter combination */
 667     for (idx = 0; idx < aliasList->aliasCount; idx++) {
 668         uint16_t aliasNum = tags[standard].aliasList[converter].aliases[idx];
 669         if (aliasNum && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0 && standard != ALL_TAG_NUM)
 670         {
 671             fprintf(stderr, "warning(line %d): duplicate alias %s and %s found for standard %s\n",
 672                 lineNum, alias, GET_ALIAS_STR(aliasNum), GET_TAG_STR(tags[standard].tag));
 673             dupFound = TRUE;
 674             break;
 675         }
 676     }
 677
 678     if (!dupFound && standard != ALL_TAG_NUM) {
 679         /* Check for duplicate aliases for this tag on all converters */
 680         for (idx = 0; idx < converterCount; idx++) {
 681             for (idx2 = 0; idx2 < tags[standard].aliasList[idx].aliasCount; idx2++) {
 682                 uint16_t aliasNum = tags[standard].aliasList[idx].aliases[idx2];
 683                 if (aliasNum
 684                     && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0)
 685                 {
 686                     fprintf(stderr, "warning(line %d): duplicate alias %s found for standard tag %s between converter %s and converter %s\n",
 687                         lineNum, alias, GET_TAG_STR(tags[standard].tag), GET_ALIAS_STR(converters[converter].converter), GET_ALIAS_STR(converters[idx].converter));
 688                     dupFound = TRUE;
 689                     break;
 690                 }
 691             }
 692         }
 693
 694         /* Check for duplicate default aliases for this converter on all tags */
 695         /* It's okay to have multiple standards prefer the same name */
 696 /*        if (verbose && !dupFound) {
 697             for (idx = 0; idx < tagCount; idx++) {
 698                 if (tags[idx].aliasList[converter].aliases) {
 699                     uint16_t aliasNum = tags[idx].aliasList[converter].aliases[0];
 700                     if (aliasNum
 701                         && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0)
 702                     {
 703                         fprintf(stderr, "warning(line %d): duplicate alias %s found for converter %s and standard tag %s\n",
 704                             lineNum, alias, GET_ALIAS_STR(converters[converter].converter), GET_TAG_STR(tags[standard].tag));
 705                         break;
 706                     }
 707                 }
 708             }
 709         }*/
 710     }
 711
 712     if (aliasList->aliasCount <= 0) {
 713         aliasList->aliasCount++;
 714         startEmptyWithoutDefault = TRUE;
 715     }
 716     aliasList->aliases = (uint16_t *)uprv_realloc(aliasList->aliases, (aliasList->aliasCount + 1) * sizeof(aliasList->aliases[0]));
 717     if (startEmptyWithoutDefault) {
 718         aliasList->aliases[0] = 0;
 719     }
 720     if (defaultName) {
 721         if (aliasList->aliases[0] != 0) {
 722             fprintf(stderr, "error(line %d): Alias %s and %s cannot both be the default alias for standard tag %s and converter %s\n",
 723                 lineNum,
 724                 alias,
 725                 GET_ALIAS_STR(aliasList->aliases[0]),
 726                 GET_TAG_STR(tags[standard].tag),
 727                 GET_ALIAS_STR(converters[converter].converter));
 728             exit(U_PARSE_ERROR);
 729         }
 730         aliasList->aliases[0] = GET_ALIAS_NUM(alias);
 731     } else {
 732         aliasList->aliases[aliasList->aliasCount++] = GET_ALIAS_NUM(alias);
 733     }
 734 /*    aliasList->converter = converter;*/
 735
 736     converters[converter].totalAliasCount++; /* One more to the column */
 737     tags[standard].totalAliasCount++; /* One more to the row */
 738
 739     return aliasList->aliasCount;
 740 }
 741
 742 static uint16_t
 743 addConverter(const char *converter) {
 744     uint32_t idx;
 745     if(converterCount>=MAX_CONV_COUNT) {
 746         fprintf(stderr, "error(line %d): too many converters\n", lineNum);
 747         exit(U_BUFFER_OVERFLOW_ERROR);
 748     }
 749
 750     for (idx = 0; idx < converterCount; idx++) {
 751         if (ucnv_compareNames(converter, GET_ALIAS_STR(converters[idx].converter)) == 0) {
 752             fprintf(stderr, "error(line %d): duplicate converter %s found!\n", lineNum, converter);
 753             exit(U_PARSE_ERROR);
 754             break;
 755         }
 756     }
 757
 758     converters[converterCount].converter = GET_ALIAS_NUM(converter);
 759     converters[converterCount].totalAliasCount = 0;
 760
 761     return converterCount++;
 762 }
 763
 764 /* resolve this alias based on the prioritization of the standard tags. */
 765 static void
 766 resolveAliasToConverter(uint16_t alias, uint16_t *tagNum, uint16_t *converterNum) {
 767     uint16_t idx, idx2, idx3;
 768
 769     for (idx = UCNV_NUM_RESERVED_TAGS; idx < tagCount; idx++) {
 770         for (idx2 = 0; idx2 < converterCount; idx2++) {
 771             for (idx3 = 0; idx3 < tags[idx].aliasList[idx2].aliasCount; idx3++) {
 772                 uint16_t aliasNum = tags[idx].aliasList[idx2].aliases[idx3];
 773                 if (aliasNum == alias) {
 774                     *tagNum = idx;
 775                     *converterNum = idx2;
 776                     return;
 777                 }
 778             }
 779         }
 780     }
 781     /* Do the leftovers last, just in case */
 782     /* There is no need to do the ALL tag */
 783     idx = 0;
 784     for (idx2 = 0; idx2 < converterCount; idx2++) {
 785         for (idx3 = 0; idx3 < tags[idx].aliasList[idx2].aliasCount; idx3++) {
 786             uint16_t aliasNum = tags[idx].aliasList[idx2].aliases[idx3];
 787             if (aliasNum == alias) {
 788                 *tagNum = idx;
 789                 *converterNum = idx2;
 790                 return;
 791             }
 792         }
 793     }
 794     *tagNum = UINT16_MAX;
 795     *converterNum = UINT16_MAX;
 796     fprintf(stderr, "warning: alias %s not found\n",
 797         GET_ALIAS_STR(alias));
 798     return;
 799 }
 800
 801 /* The knownAliases should be sorted before calling this function */
 802 static uint32_t
 803 resolveAliases(uint16_t *uniqueAliasArr, uint16_t *uniqueAliasToConverterArr, uint16_t aliasOffset) {
 804     uint32_t uniqueAliasIdx = 0;
 805     uint32_t idx;
 806     uint16_t currTagNum, oldTagNum;
 807     uint16_t currConvNum, oldConvNum;
 808     const char *lastName;
 809
 810     resolveAliasToConverter(knownAliases[0], &oldTagNum, &currConvNum);
 811     uniqueAliasToConverterArr[uniqueAliasIdx] = currConvNum;
 812     oldConvNum = currConvNum;
 813     uniqueAliasArr[uniqueAliasIdx] = knownAliases[0] + aliasOffset;
 814     uniqueAliasIdx++;
 815     lastName = GET_ALIAS_STR(knownAliases[0]);
 816
 817     for (idx = 1; idx < knownAliasesCount; idx++) {
 818         resolveAliasToConverter(knownAliases[idx], &currTagNum, &currConvNum);
 819         if (ucnv_compareNames(lastName, GET_ALIAS_STR(knownAliases[idx])) == 0) {
 820             /* duplicate found */
 821             if ((currTagNum < oldTagNum && currTagNum >= UCNV_NUM_RESERVED_TAGS)
 822                 || oldTagNum == 0) {
 823                 oldTagNum = currTagNum;
 824                 uniqueAliasToConverterArr[uniqueAliasIdx - 1] = currConvNum;
 825                 uniqueAliasArr[uniqueAliasIdx - 1] = knownAliases[idx] + aliasOffset;
 826                 if (verbose) {
 827                     printf("using %s instead of %s -> %s",
 828                         GET_ALIAS_STR(knownAliases[idx]),
 829                         lastName,
 830                         GET_ALIAS_STR(converters[currConvNum].converter));
 831                     if (oldConvNum != currConvNum) {
 832                         printf(" (alias conflict)");
 833                     }
 834                     puts("");
 835                 }
 836             }
 837             else {
 838                 /* else ignore it */
 839                 if (verbose) {
 840                     printf("folding %s into %s -> %s",
 841                         GET_ALIAS_STR(knownAliases[idx]),
 842                         lastName,
 843                         GET_ALIAS_STR(converters[oldConvNum].converter));
 844                     if (oldConvNum != currConvNum) {
 845                         printf(" (alias conflict)");
 846                     }
 847                     puts("");
 848                 }
 849             }
 850             if (oldConvNum != currConvNum) {
 851                 uniqueAliasToConverterArr[uniqueAliasIdx - 1] |= UCNV_AMBIGUOUS_ALIAS_MAP_BIT;
 852             }
 853         }
 854         else {
 855             uniqueAliasToConverterArr[uniqueAliasIdx] = currConvNum;
 856             oldConvNum = currConvNum;
 857             uniqueAliasArr[uniqueAliasIdx] = knownAliases[idx] + aliasOffset;
 858             uniqueAliasIdx++;
 859             lastName = GET_ALIAS_STR(knownAliases[idx]);
 860             oldTagNum = currTagNum;
 861             /*printf("%s -> %s\n", GET_ALIAS_STR(knownAliases[idx]), GET_ALIAS_STR(converters[currConvNum].converter));*/
 862         }
 863     }
 864     return uniqueAliasIdx;
 865 }
 866
 867 static void
 868 createOneAliasList(uint16_t *aliasArrLists, uint32_t tag, uint32_t converter, uint16_t offset) {
 869     uint32_t aliasNum;
 870     AliasList *aliasList = &tags[tag].aliasList[converter];
 871
 872     if (aliasList->aliasCount == 0) {
 873         aliasArrLists[tag*converterCount + converter] = 0;
 874     }
 875     else {
 876         aliasLists[aliasListsSize++] = aliasList->aliasCount;
 877
 878         /* write into the array area a 1's based index. */
 879         aliasArrLists[tag*converterCount + converter] = aliasListsSize;
 880
 881 /*        printf("tag %s converter %s\n",
 882             GET_TAG_STR(tags[tag].tag),
 883             GET_ALIAS_STR(converters[converter].converter));*/
 884         for (aliasNum = 0; aliasNum < aliasList->aliasCount; aliasNum++) {
 885             uint16_t value;
 886 /*            printf("   %s\n",
 887                 GET_ALIAS_STR(aliasList->aliases[aliasNum]));*/
 888             if (aliasList->aliases[aliasNum]) {
 889                 value = aliasList->aliases[aliasNum] + offset;
 890             } else {
 891                 value = 0;
 892                 if (tag != 0) { /* Only show the warning when it's not the leftover tag. */
 893                     printf("warning: tag %s does not have a default alias for %s\n",
 894                             GET_TAG_STR(tags[tag].tag),
 895                             GET_ALIAS_STR(converters[converter].converter));
 896                 }
 897             }
 898             aliasLists[aliasListsSize++] = value;
 899             if (aliasListsSize >= MAX_LIST_SIZE) {
 900                 fprintf(stderr, "error: Too many alias lists\n");
 901                 exit(U_BUFFER_OVERFLOW_ERROR);
 902             }
 903
 904         }
 905     }
 906 }
 907
 908 static void
 909 writeAliasTable(UNewDataMemory *out) {
 910     uint32_t i, j;
 911     uint32_t uniqueAliasesSize;
 912     uint16_t aliasOffset = (uint16_t)(tagBlock.top/sizeof(uint16_t));
 913     uint16_t *aliasArrLists = (uint16_t *)uprv_malloc(tagCount * converterCount * sizeof(uint16_t));
 914     uint16_t *uniqueAliases = (uint16_t *)uprv_malloc(knownAliasesCount * sizeof(uint16_t));
 915     uint16_t *uniqueAliasesToConverter = (uint16_t *)uprv_malloc(knownAliasesCount * sizeof(uint16_t));
 916
 917     qsort(knownAliases, knownAliasesCount, sizeof(knownAliases[0]), compareAliases);
 918     uniqueAliasesSize = resolveAliases(uniqueAliases, uniqueAliasesToConverter, aliasOffset);
 919
 920     /* Array index starts at 1. aliasLists[0] is the size of the lists section. */
 921     aliasListsSize = 0;
 922
 923     /* write the offsets of all the aliases lists in a 2D array, and create the lists. */
 924     for (i = 0; i < tagCount; ++i) {
 925         for (j = 0; j < converterCount; ++j) {
 926             createOneAliasList(aliasArrLists, i, j, aliasOffset);
 927         }
 928     }
 929
 930     /* Write the size of the TOC */
 931     udata_write32(out, 8);
 932
 933     /* Write the sizes of each section */
 934     /* All sizes are the number of uint16_t units, not bytes */
 935     udata_write32(out, converterCount);
 936     udata_write32(out, tagCount);
 937     udata_write32(out, uniqueAliasesSize);  /* list of aliases */
 938     udata_write32(out, uniqueAliasesSize);  /* The preresolved form of mapping an untagged the alias to a converter */
 939     udata_write32(out, tagCount * converterCount);
 940     udata_write32(out, aliasListsSize + 1);
 941     udata_write32(out, 0);  /* Reserved space. */
 942     udata_write32(out, (tagBlock.top + stringBlock.top) / sizeof(uint16_t));
 943
 944     /* write the table of converters */
 945     /* Think of this as the column headers */
 946     for(i=0; i<converterCount; ++i) {
 947         udata_write16(out, (uint16_t)(converters[i].converter + aliasOffset));
 948     }
 949
 950     /* write the table of tags */
 951     /* Think of this as the row headers */
 952     for(i=UCNV_NUM_RESERVED_TAGS; i<tagCount; ++i) {
 953         udata_write16(out, tags[i].tag);
 954     }
 955     /* The empty tag is considered the leftover list, and put that at the end of the priority list. */
 956     udata_write16(out, tags[EMPTY_TAG_NUM].tag);
 957     udata_write16(out, tags[ALL_TAG_NUM].tag);
 958
 959     /* Write the unique list of aliases */
 960     udata_writeBlock(out, uniqueAliases, uniqueAliasesSize * sizeof(uint16_t));
 961
 962     /* Write the unique list of aliases */
 963     udata_writeBlock(out, uniqueAliasesToConverter, uniqueAliasesSize * sizeof(uint16_t));
 964
 965     /* Write the array to the lists */
 966     udata_writeBlock(out, (const void *)(aliasArrLists + (2*converterCount)), (((tagCount - 2) * converterCount) * sizeof(uint16_t)));
 967     /* Now write the leftover part of the array for the EMPTY and ALL lists */
 968     udata_writeBlock(out, (const void *)aliasArrLists, (2 * converterCount * sizeof(uint16_t)));
 969
 970     /* Offset the next array to make the index start at 1. */
 971     udata_write16(out, 0xDEAD);
 972
 973     /* Write the lists */
 974     udata_writeBlock(out, (const void *)aliasLists, aliasListsSize * sizeof(uint16_t));
 975
 976     /* write the tags strings */
 977     udata_writeString(out, tagBlock.store, tagBlock.top);
 978
 979     /* write the aliases strings */
 980     udata_writeString(out, stringBlock.store, stringBlock.top);
 981
 982     uprv_free(aliasArrLists);
 983     uprv_free(uniqueAliases);
 984 }
 985
 986 static char *
 987 allocString(StringBlock *block, const char *s, int32_t length) {
 988     uint32_t top;
 989     char *p;
 990
 991     if(length<0) {
 992         length=(int32_t)uprv_strlen(s);
 993     }
 994
 995     /*
 996      * add 1 for the terminating NUL
 997      * and round up (+1 &~1)
 998      * to keep the addresses on a 16-bit boundary
 999      */
1000     top=block->top + (uint32_t)((length + 1 + 1) & ~1);
1001
1002     if(top >= block->max) {
1003         fprintf(stderr, "error(line %d): out of memory\n", lineNum);
1004         exit(U_MEMORY_ALLOCATION_ERROR);
1005     }
1006
1007     /* get the pointer and copy the string */
1008     p = block->store + block->top;
1009     uprv_memcpy(p, s, length);
1010     p[length] = 0; /* NUL-terminate it */
1011     if((length & 1) == 0) {
1012         p[length + 1] = 0; /* set the padding byte */
1013     }
1014
1015     /* check for invariant characters now that we have a NUL-terminated string for easy output */
1016     if(!uprv_isInvariantString(p, length)) {
1017         fprintf(stderr, "error(line %d): the name %s contains not just invariant characters\n", lineNum, p);
1018         exit(U_INVALID_TABLE_FORMAT);
1019     }
1020
1021     block->top = top;
1022     return p;
1023 }
1024
1025 static int
1026 compareAliases(const void *alias1, const void *alias2) {
1027     /* Names like IBM850 and ibm-850 need to be sorted together */
1028     int result = ucnv_compareNames(GET_ALIAS_STR(*(uint16_t*)alias1), GET_ALIAS_STR(*(uint16_t*)alias2));
1029     if (!result) {
1030         /* Sort the shortest first */
1031         return (int)uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias1)) - (int)uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias2));
1032     }
1033     return result;
1034 }
1035
1036 /*
1037  * Hey, Emacs, please set the following:
1038  *
1039  * Local Variables:
1040  * indent-tabs-mode: nil
1041  * End:
1042  *
1043  */
1044