ICU-491.11.1.tar.gz

[apple/icu.git] / icuSources / tools / gencnval / gencnval.c

diff --git a/icuSources/tools/gencnval/gencnval.c b/icuSources/tools/gencnval/gencnval.c
index a81ee51e78d9c7bdb9cfb42e28ede42b32571ce8..0f0a915887b0004f8801dc1b9d04395edb3cdd99 100644 (file)
--- a/icuSources/tools/gencnval/gencnval.c
+++ b/icuSources/tools/gencnval/gencnval.c
@@ -1,7 +1,7 @@
 /*
 *******************************************************************************
 *
 /*
 *******************************************************************************
 *

-*   Copyright (C) 1999-2003, International Business Machines
+*   Copyright (C) 1999-2011, International Business Machines

 *   Corporation and others.  All Rights Reserved.
 *
 *******************************************************************************
 *   Corporation and others.  All Rights Reserved.
 *
 *******************************************************************************


@@ -28,7 +28,9 @@
 #include "ucnv_io.h"
 #include "cmemory.h"
 #include "cstring.h"
 #include "ucnv_io.h"
 #include "cmemory.h"
 #include "cstring.h"

+#include "uinvchar.h"

 #include "filestrm.h"
 #include "filestrm.h"

+#include "unicode/uclean.h"

 #include "unewdata.h"
 #include "uoptions.h"
 
 #include "unewdata.h"
 #include "uoptions.h"
 


@@ -82,7 +84,7 @@ static const UDataInfo dataInfo={
     0,
 
     {0x43, 0x76, 0x41, 0x6c},     /* dataFormat="CvAl" */
     0,
 
     {0x43, 0x76, 0x41, 0x6c},     /* dataFormat="CvAl" */

-    {3, 0, 0, 0},                 /* formatVersion */
+    {3, 0, 1, 0},                 /* formatVersion */

     {1, 4, 2, 0}                  /* dataVersion */
 };
 
     {1, 4, 2, 0}                  /* dataVersion */
 };
 


@@ -133,7 +135,18 @@ static uint16_t aliasListsSize = 0;
 /* Were the standard tags declared before the aliases. */
 static UBool standardTagsUsed = FALSE;
 static UBool verbose = FALSE;
 /* Were the standard tags declared before the aliases. */
 static UBool standardTagsUsed = FALSE;
 static UBool verbose = FALSE;

-static int32_t lineNum = 1;
+static int lineNum = 1;
+
+static UConverterAliasOptions tableOptions = {
+    UCNV_IO_STD_NORMALIZED,
+    1 /* containsCnvOptionInfo */
+};
+
+
+/**
+ * path to convrtrs.txt
+ */
+const char *path;

 
 /* prototypes --------------------------------------------------------------- */
 
 
 /* prototypes --------------------------------------------------------------- */
 


@@ -156,7 +169,7 @@ static uint16_t
 addConverter(const char *converter);
 
 static char *
 addConverter(const char *converter);
 
 static char *

-allocString(StringBlock *block, uint32_t length);
+allocString(StringBlock *block, const char *s, int32_t length);

 
 static uint16_t
 addToKnownAliases(const char *alias);
 
 static uint16_t
 addToKnownAliases(const char *alias);


@@ -205,7 +218,6 @@ static UOption options[]={
 extern int
 main(int argc, char* argv[]) {
     char pathBuf[512];
 extern int
 main(int argc, char* argv[]) {
     char pathBuf[512];

-    const char *path;

     FileStream *in;
     UNewDataMemory *out;
     UErrorCode errorCode=U_ZERO_ERROR;
     FileStream *in;
     UNewDataMemory *out;
     UErrorCode errorCode=U_ZERO_ERROR;


@@ -213,7 +225,7 @@ main(int argc, char* argv[]) {
     U_MAIN_INIT_ARGS(argc, argv);
 
     /* preset then read command line options */
     U_MAIN_INIT_ARGS(argc, argv);
 
     /* preset then read command line options */

-    options[3].value=options[4].value=u_getDataDirectory();
+    options[DESTDIR].value=options[SOURCEDIR].value=u_getDataDirectory();

     argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
 
     /* error handling, printing usage message */
     argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
 
     /* error handling, printing usage message */


@@ -243,10 +255,12 @@ main(int argc, char* argv[]) {
     if(argc>=2) {
         path=argv[1];
     } else {
     if(argc>=2) {
         path=argv[1];
     } else {

-        path=options[4].value;
+        path=options[SOURCEDIR].value;

         if(path!=NULL && *path!=0) {
         if(path!=NULL && *path!=0) {

-            char *end = pathBuf+uprv_strlen(pathBuf);
+            char *end;
+

             uprv_strcpy(pathBuf, path);
             uprv_strcpy(pathBuf, path);

+            end = uprv_strchr(pathBuf, 0);

             if(*(end-1)!=U_FILE_SEP_CHAR) {
                 *(end++)=U_FILE_SEP_CHAR;
             }
             if(*(end-1)!=U_FILE_SEP_CHAR) {
                 *(end++)=U_FILE_SEP_CHAR;
             }


@@ -267,15 +281,15 @@ main(int argc, char* argv[]) {
 
     in=T_FileStream_open(path, "r");
     if(in==NULL) {
 
     in=T_FileStream_open(path, "r");
     if(in==NULL) {

-        fprintf(stderr, "gencnval: unable to open input file convrtrs.txt\n");
+        fprintf(stderr, "gencnval: unable to open input file %s\n", path);

         exit(U_FILE_ACCESS_ERROR);
     }
     parseFile(in);
     T_FileStream_close(in);
 
     /* create the output file */
         exit(U_FILE_ACCESS_ERROR);
     }
     parseFile(in);
     T_FileStream_close(in);
 
     /* create the output file */

-    out=udata_create(options[3].value, DATA_TYPE, U_ICUDATA_NAME "_" DATA_NAME, &dataInfo,
-                     options[2].doesOccur ? U_COPYRIGHT_STRING : NULL, &errorCode);
+    out=udata_create(options[DESTDIR].value, DATA_TYPE, DATA_NAME, &dataInfo,
+                     options[COPYRIGHT].doesOccur ? U_COPYRIGHT_STRING : NULL, &errorCode);

     if(U_FAILURE(errorCode)) {
         fprintf(stderr, "gencnval: unable to open output file - error %s\n", u_errorName(errorCode));
         exit(errorCode);
     if(U_FAILURE(errorCode)) {
         fprintf(stderr, "gencnval: unable to open output file - error %s\n", u_errorName(errorCode));
         exit(errorCode);


@@ -307,7 +321,7 @@ parseFile(FileStream *in) {
     /* Add the empty tag, which is for untagged aliases */
     getTagNumber("", 0);
     getTagNumber(ALL_TAG_STR, 3);
     /* Add the empty tag, which is for untagged aliases */
     getTagNumber("", 0);
     getTagNumber(ALL_TAG_STR, 3);

-    allocString(&stringBlock, 1);
+    allocString(&stringBlock, "", 0);

 
     /* read the list of aliases */
     while (validParse) {
 
     /* read the list of aliases */
     while (validParse) {


@@ -316,7 +330,7 @@ parseFile(FileStream *in) {
         /* Read non-empty lines that don't start with a space character. */
         while (T_FileStream_readLine(in, lastLine, MAX_LINE_SIZE) != NULL) {
             lastLineSize = chomp(lastLine);
         /* Read non-empty lines that don't start with a space character. */
         while (T_FileStream_readLine(in, lastLine, MAX_LINE_SIZE) != NULL) {
             lastLineSize = chomp(lastLine);

-            if (lineSize == 0 || (lastLineSize > 0 && isspace(*lastLine))) {
+            if (lineSize == 0 || (lastLineSize > 0 && isspace((int)*lastLine))) {

                 uprv_strcpy(line + lineSize, lastLine);
                 lineSize += lastLineSize;
             } else if (lineSize > 0) {
                 uprv_strcpy(line + lineSize, lastLine);
                 lineSize += lastLineSize;
             } else if (lineSize > 0) {


@@ -327,12 +341,12 @@ parseFile(FileStream *in) {
         }
 
         if (validParse || lineSize > 0) {
         }
 
         if (validParse || lineSize > 0) {

-            if (isspace(*line)) {
-                fprintf(stderr, "error(line %d): cannot start an alias with a space\n", lineNum-1);
+            if (isspace((int)*line)) {
+                fprintf(stderr, "%s:%d: error: cannot start an alias with a space\n", path, lineNum-1);

                 exit(U_PARSE_ERROR);
             } else if (line[0] == '{') {
                 if (!standardTagsUsed && line[lineSize - 1] != '}') {
                 exit(U_PARSE_ERROR);
             } else if (line[0] == '{') {
                 if (!standardTagsUsed && line[lineSize - 1] != '}') {

-                    fprintf(stderr, "error(line %d): alias needs to start with a converter name\n", lineNum);
+                    fprintf(stderr, "%s:%d: error: alias needs to start with a converter name\n", path, lineNum);

                     exit(U_PARSE_ERROR);
                 }
                 addOfficialTaggedStandards(line, lineSize);
                     exit(U_PARSE_ERROR);
                 }
                 addOfficialTaggedStandards(line, lineSize);


@@ -342,7 +356,7 @@ parseFile(FileStream *in) {
                     parseLine(line);
                 }
                 else {
                     parseLine(line);
                 }
                 else {

-                    fprintf(stderr, "error(line %d): alias table needs to start a list of standard tags\n", lineNum);
+                    fprintf(stderr, "%s:%d: error: alias table needs to start a list of standard tags\n", path, lineNum);

                     exit(U_PARSE_ERROR);
                 }
             }
                     exit(U_PARSE_ERROR);
                 }
             }


@@ -372,7 +386,7 @@ chomp(char *line) {
             *s = 0;
             break;
         }
             *s = 0;
             break;
         }

-        if (!isspace(*s)) {
+        if (!isspace((int)*s)) {

             lastNonSpace = s;
         }
         ++s;
             lastNonSpace = s;
         }
         ++s;


@@ -403,16 +417,14 @@ parseLine(const char *line) {
 
     /* get the converter name */
     start=pos;
 
     /* get the converter name */
     start=pos;

-    while(line[pos]!=0 && !isspace(line[pos])) {
+    while(line[pos]!=0 && !isspace((int)line[pos])) {

         ++pos;
     }
     limit=pos;
 
     /* store the converter name */
     length=(uint16_t)(limit-start);
         ++pos;
     }
     limit=pos;
 
     /* store the converter name */
     length=(uint16_t)(limit-start);

-    converter=allocString(&stringBlock, length+1);
-    uprv_memcpy(converter, line+start, length);
-    converter[length]=0;
+    converter=allocString(&stringBlock, line+start, length);

 
     /* add the converter to the converter table */
     cnv=addConverter(converter);
 
     /* add the converter to the converter table */
     cnv=addConverter(converter);


@@ -424,7 +436,7 @@ parseLine(const char *line) {
     for(;;) {
 
         /* skip white space */
     for(;;) {
 
         /* skip white space */

-        while(line[pos]!=0 && isspace(line[pos])) {
+        while(line[pos]!=0 && isspace((int)line[pos])) {

             ++pos;
         }
 
             ++pos;
         }
 


@@ -435,7 +447,7 @@ parseLine(const char *line) {
 
         /* get an alias name */
         start=pos;
 
         /* get an alias name */
         start=pos;

-        while(line[pos]!=0 && line[pos]!='{' && !isspace(line[pos])) {
+        while(line[pos]!=0 && line[pos]!='{' && !isspace((int)line[pos])) {

             ++pos;
         }
         limit=pos;
             ++pos;
         }
         limit=pos;


@@ -448,9 +460,7 @@ parseLine(const char *line) {
             addAlias(alias, ALL_TAG_NUM, cnv, TRUE);
         }
         else {
             addAlias(alias, ALL_TAG_NUM, cnv, TRUE);
         }
         else {

-            alias=allocString(&stringBlock, length+1);
-            uprv_memcpy(alias, line+start, length);
-            alias[length]=0;
+            alias=allocString(&stringBlock, line+start, length);

             addAlias(alias, ALL_TAG_NUM, cnv, FALSE);
         }
         addToKnownAliases(alias);
             addAlias(alias, ALL_TAG_NUM, cnv, FALSE);
         }
         addToKnownAliases(alias);


@@ -459,7 +469,7 @@ parseLine(const char *line) {
         /* addAlias(alias, 0, cnv, FALSE);*/
 
         /* skip whitespace */
         /* addAlias(alias, 0, cnv, FALSE);*/
 
         /* skip whitespace */

-        while (line[pos] && isspace(line[pos])) {
+        while (line[pos] && isspace((int)line[pos])) {

             ++pos;
         }
 
             ++pos;
         }
 


@@ -468,7 +478,7 @@ parseLine(const char *line) {
             ++pos;
             do {
                 start = pos;
             ++pos;
             do {
                 start = pos;

-                while (line[pos] && line[pos] != '}' && !isspace( line[pos])) {
+                while (line[pos] && line[pos] != '}' && !isspace((int)line[pos])) {

                     ++pos;
                 }
                 limit = pos;
                     ++pos;
                 }
                 limit = pos;


@@ -479,7 +489,7 @@ parseLine(const char *line) {
                     addAlias(alias, tag, cnv, (UBool)(line[limit-1] == '*'));
                 }
 
                     addAlias(alias, tag, cnv, (UBool)(line[limit-1] == '*'));
                 }
 

-                while (line[pos] && isspace(line[pos])) {
+                while (line[pos] && isspace((int)line[pos])) {

                     ++pos;
                 }
             } while (line[pos] && line[pos] != '}');
                     ++pos;
                 }
             } while (line[pos] && line[pos] != '}');


@@ -487,7 +497,7 @@ parseLine(const char *line) {
             if (line[pos] == '}') {
                 ++pos;
             } else {
             if (line[pos] == '}') {
                 ++pos;
             } else {

-                fprintf(stderr, "error(line %d): Unterminated tag list\n", lineNum);
+                fprintf(stderr, "%s:%d: Unterminated tag list\n", path, lineNum);

                 exit(U_UNMATCHED_BRACES);
             }
         } else {
                 exit(U_UNMATCHED_BRACES);
             }
         } else {


@@ -503,7 +513,7 @@ getTagNumber(const char *tag, uint16_t tagLen) {
     UBool preferredName = ((tagLen > 0) ? (tag[tagLen - 1] == '*') : (FALSE));
 
     if (tagCount >= MAX_TAG_COUNT) {
     UBool preferredName = ((tagLen > 0) ? (tag[tagLen - 1] == '*') : (FALSE));
 
     if (tagCount >= MAX_TAG_COUNT) {

-        fprintf(stderr, "error(line %d): too many tags\n", lineNum);
+        fprintf(stderr, "%s:%d: too many tags\n", path, lineNum);

         exit(U_BUFFER_OVERFLOW_ERROR);
     }
 
         exit(U_BUFFER_OVERFLOW_ERROR);
     }
 


@@ -521,23 +531,21 @@ getTagNumber(const char *tag, uint16_t tagLen) {
 
     /* we need to add this tag */
     if (tagCount >= MAX_TAG_COUNT) {
 
     /* we need to add this tag */
     if (tagCount >= MAX_TAG_COUNT) {

-        fprintf(stderr, "error(line %d): too many tags\n", lineNum);
+        fprintf(stderr, "%s:%d: error: too many tags\n", path, lineNum);

         exit(U_BUFFER_OVERFLOW_ERROR);
     }
 
     /* allocate a new entry in the tag table */
         exit(U_BUFFER_OVERFLOW_ERROR);
     }
 
     /* allocate a new entry in the tag table */

-    atag = allocString(&tagBlock, tagLen + 1);
-    uprv_memcpy(atag, tag, tagLen);
-    atag[tagLen] = 0;
+    atag = allocString(&tagBlock, tag, tagLen);

 
     if (standardTagsUsed) {
 
     if (standardTagsUsed) {

-        fprintf(stderr, "error(line %d): Tag \"%s\" is not declared at the beginning of the alias table.\n",
-            lineNum, atag);
+        fprintf(stderr, "%s:%d: error: Tag \"%s\" is not declared at the beginning of the alias table.\n",
+            path, lineNum, atag);

         exit(1);
     }
     else if (tagLen > 0 && strcmp(tag, ALL_TAG_STR) != 0) {
         exit(1);
     }
     else if (tagLen > 0 && strcmp(tag, ALL_TAG_STR) != 0) {

-        fprintf(stderr, "warning(line %d): Tag \"%s\" was added to the list of standards because it was not declared at beginning of the alias table.\n",
-            lineNum, atag);
+        fprintf(stderr, "%s:%d: warning: Tag \"%s\" was added to the list of standards because it was not declared at beginning of the alias table.\n",
+            path, lineNum, atag);

     }
 
     /* add the tag to the tag table */
     }
 
     /* add the tag to the tag table */


@@ -556,25 +564,34 @@ addTaggedAlias(uint16_t tag, const char *alias, uint16_t converter) {
 static void
 addOfficialTaggedStandards(char *line, int32_t lineLen) {
     char *atag;
 static void
 addOfficialTaggedStandards(char *line, int32_t lineLen) {
     char *atag;

-    char *tag = strchr(line, '{') + 1;
-    uint16_t tagSize;
+    char *endTagExp;
+    char *tag;

     static const char WHITESPACE[] = " \t";
 
     if (tagCount > UCNV_NUM_RESERVED_TAGS) {
     static const char WHITESPACE[] = " \t";
 
     if (tagCount > UCNV_NUM_RESERVED_TAGS) {

-        fprintf(stderr, "error(line %d): official tags already added\n", lineNum);
+        fprintf(stderr, "%s:%d: error: official tags already added\n", path, lineNum);

         exit(U_BUFFER_OVERFLOW_ERROR);
     }
         exit(U_BUFFER_OVERFLOW_ERROR);
     }

-    strchr(tag, '}')[0] = 0;
+    tag = strchr(line, '{');
+    if (tag == NULL) {
+        /* Why were we called? */
+        fprintf(stderr, "%s:%d: error: Missing start of tag group\n", path, lineNum);
+        exit(U_PARSE_ERROR);
+    }
+    tag++;
+    endTagExp = strchr(tag, '}');
+    if (endTagExp == NULL) {
+        fprintf(stderr, "%s:%d: error: Missing end of tag group\n", path, lineNum);
+        exit(U_PARSE_ERROR);
+    }
+    endTagExp[0] = 0;

 
     tag = strtok(tag, WHITESPACE);
     while (tag != NULL) {
 /*        printf("Adding original tag \"%s\"\n", tag);*/
 
 
     tag = strtok(tag, WHITESPACE);
     while (tag != NULL) {
 /*        printf("Adding original tag \"%s\"\n", tag);*/
 

-        tagSize = strlen(tag) + 1;

         /* allocate a new entry in the tag table */
         /* allocate a new entry in the tag table */

-
-        atag = allocString(&tagBlock, tagSize);
-        uprv_memcpy(atag, tag, tagSize);
+        atag = allocString(&tagBlock, tag, -1);

 
         /* add the tag to the tag table */
         tags[tagCount++].tag = (uint16_t)((atag - tagStore) >> 1);
 
         /* add the tag to the tag table */
         tags[tagCount++].tag = (uint16_t)((atag - tagStore) >> 1);


@@ -595,7 +612,7 @@ addToKnownAliases(const char *alias) {
         if (knownAliases[idx] != num
             && uprv_strcmp(alias, GET_ALIAS_STR(knownAliases[idx])) == 0)
         {
         if (knownAliases[idx] != num
             && uprv_strcmp(alias, GET_ALIAS_STR(knownAliases[idx])) == 0)
         {

-            fprintf(stderr, "warning(line %d): duplicate alias %s and %s found\n",
+            fprintf(stderr, "%s:%d: warning: duplicate alias %s and %s found\n", path, 

                 lineNum, alias, GET_ALIAS_STR(knownAliases[idx]));
             duplicateKnownAliasesCount++;
             break;
                 lineNum, alias, GET_ALIAS_STR(knownAliases[idx]));
             duplicateKnownAliasesCount++;
             break;


@@ -604,7 +621,7 @@ addToKnownAliases(const char *alias) {
             && ucnv_compareNames(alias, GET_ALIAS_STR(knownAliases[idx])) == 0)
         {
             if (verbose) {
             && ucnv_compareNames(alias, GET_ALIAS_STR(knownAliases[idx])) == 0)
         {
             if (verbose) {

-                fprintf(stderr, "information(line %d): duplicate alias %s and %s found\n",
+                fprintf(stderr, "%s:%d: information: duplicate alias %s and %s found\n", path, 

                     lineNum, alias, GET_ALIAS_STR(knownAliases[idx]));
             }
             duplicateKnownAliasesCount++;
                     lineNum, alias, GET_ALIAS_STR(knownAliases[idx]));
             }
             duplicateKnownAliasesCount++;


@@ -613,8 +630,8 @@ addToKnownAliases(const char *alias) {
     }
 */
     if (knownAliasesCount >= MAX_ALIAS_COUNT) {
     }
 */
     if (knownAliasesCount >= MAX_ALIAS_COUNT) {

-        fprintf(stderr, "warning(line %d): Too many aliases defined for all converters\n",
-            lineNum);
+        fprintf(stderr, "%s:%d: warning: Too many aliases defined for all converters\n",
+            path, lineNum);

         exit(U_BUFFER_OVERFLOW_ERROR);
     }
     /* TODO: We could try to unlist exact duplicates. */
         exit(U_BUFFER_OVERFLOW_ERROR);
     }
     /* TODO: We could try to unlist exact duplicates. */


@@ -632,22 +649,22 @@ addAlias(const char *alias, uint16_t standard, uint16_t converter, UBool default
     AliasList *aliasList;
 
     if(standard>=MAX_TAG_COUNT) {
     AliasList *aliasList;
 
     if(standard>=MAX_TAG_COUNT) {

-        fprintf(stderr, "error(line %d): too many standard tags\n", lineNum);
+        fprintf(stderr, "%s:%d: error: too many standard tags\n", path, lineNum);

         exit(U_BUFFER_OVERFLOW_ERROR);
     }
     if(converter>=MAX_CONV_COUNT) {
         exit(U_BUFFER_OVERFLOW_ERROR);
     }
     if(converter>=MAX_CONV_COUNT) {

-        fprintf(stderr, "error(line %d): too many converter names\n", lineNum);
+        fprintf(stderr, "%s:%d: error: too many converter names\n", path, lineNum);

         exit(U_BUFFER_OVERFLOW_ERROR);
     }
     aliasList = &tags[standard].aliasList[converter];
 
     if (strchr(alias, '}')) {
         exit(U_BUFFER_OVERFLOW_ERROR);
     }
     aliasList = &tags[standard].aliasList[converter];
 
     if (strchr(alias, '}')) {

-        fprintf(stderr, "error(line %d): unmatched } found\n",
+        fprintf(stderr, "%s:%d: error: unmatched } found\n", path, 

             lineNum);
     }
 
     if(aliasList->aliasCount + 1 >= MAX_TC_ALIAS_COUNT) {
             lineNum);
     }
 
     if(aliasList->aliasCount + 1 >= MAX_TC_ALIAS_COUNT) {

-        fprintf(stderr, "error(line %d): too many aliases for alias %s and converter %s\n",
+        fprintf(stderr, "%s:%d: error: too many aliases for alias %s and converter %s\n", path, 

             lineNum, alias, GET_ALIAS_STR(converters[converter].converter));
         exit(U_BUFFER_OVERFLOW_ERROR);
     }
             lineNum, alias, GET_ALIAS_STR(converters[converter].converter));
         exit(U_BUFFER_OVERFLOW_ERROR);
     }


@@ -669,19 +686,7 @@ addAlias(const char *alias, uint16_t standard, uint16_t converter, UBool default
         }
     }
 
         }
     }
 

-    /* Check for duplicates in a tag/converter combination */
-    for (idx = 0; idx < aliasList->aliasCount; idx++) {
-        uint16_t aliasNum = tags[standard].aliasList[converter].aliases[idx];
-        if (aliasNum && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0 && standard != ALL_TAG_NUM)
-        {
-            fprintf(stderr, "warning(line %d): duplicate alias %s and %s found for standard %s\n",
-                lineNum, alias, GET_ALIAS_STR(aliasNum), GET_TAG_STR(tags[standard].tag));
-            dupFound = TRUE;
-            break;
-        }
-    }
-
-    if (!dupFound && standard != ALL_TAG_NUM) {
+    if (standard != ALL_TAG_NUM) {

         /* Check for duplicate aliases for this tag on all converters */
         for (idx = 0; idx < converterCount; idx++) {
             for (idx2 = 0; idx2 < tags[standard].aliasList[idx].aliasCount; idx2++) {
         /* Check for duplicate aliases for this tag on all converters */
         for (idx = 0; idx < converterCount; idx++) {
             for (idx2 = 0; idx2 < tags[standard].aliasList[idx].aliasCount; idx2++) {


@@ -689,8 +694,25 @@ addAlias(const char *alias, uint16_t standard, uint16_t converter, UBool default
                 if (aliasNum
                     && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0)
                 {
                 if (aliasNum
                     && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0)
                 {

-                    fprintf(stderr, "warning(line %d): duplicate alias %s found for standard tag %s between converter %s and converter %s\n",
-                        lineNum, alias, GET_TAG_STR(tags[standard].tag), GET_ALIAS_STR(converters[converter].converter), GET_ALIAS_STR(converters[idx].converter));
+                    if (idx == converter) {
+                        /*
+                         * (alias, standard) duplicates are harmless if they map to the same converter.
+                         * Only print a warning in verbose mode, or if the alias is a precise duplicate,
+                         * not just a lenient-match duplicate.
+                         */
+                        if (verbose || 0 == uprv_strcmp(alias, GET_ALIAS_STR(aliasNum))) {
+                            fprintf(stderr, "%s:%d: warning: duplicate aliases %s and %s found for standard %s and converter %s\n", path, 
+                                lineNum, alias, GET_ALIAS_STR(aliasNum),
+                                GET_TAG_STR(tags[standard].tag),
+                                GET_ALIAS_STR(converters[converter].converter));
+                        }
+                    } else {
+                        fprintf(stderr, "%s:%d: warning: duplicate aliases %s and %s found for standard tag %s between converter %s and converter %s\n", path, 
+                            lineNum, alias, GET_ALIAS_STR(aliasNum),
+                            GET_TAG_STR(tags[standard].tag),
+                            GET_ALIAS_STR(converters[converter].converter),
+                            GET_ALIAS_STR(converters[idx].converter));
+                    }

                     dupFound = TRUE;
                     break;
                 }
                     dupFound = TRUE;
                     break;
                 }


@@ -706,7 +728,7 @@ addAlias(const char *alias, uint16_t standard, uint16_t converter, UBool default
                     if (aliasNum
                         && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0)
                     {
                     if (aliasNum
                         && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0)
                     {

-                        fprintf(stderr, "warning(line %d): duplicate alias %s found for converter %s and standard tag %s\n",
+                        fprintf(stderr, "%s:%d: warning: duplicate alias %s found for converter %s and standard tag %s\n", path, 

                             lineNum, alias, GET_ALIAS_STR(converters[converter].converter), GET_TAG_STR(tags[standard].tag));
                         break;
                     }
                             lineNum, alias, GET_ALIAS_STR(converters[converter].converter), GET_TAG_STR(tags[standard].tag));
                         break;
                     }


@@ -725,7 +747,7 @@ addAlias(const char *alias, uint16_t standard, uint16_t converter, UBool default
     }
     if (defaultName) {
         if (aliasList->aliases[0] != 0) {
     }
     if (defaultName) {
         if (aliasList->aliases[0] != 0) {

-            fprintf(stderr, "error(line %d): Alias %s and %s cannot both be the default alias for standard tag %s and converter %s\n",
+            fprintf(stderr, "%s:%d: error: Alias %s and %s cannot both be the default alias for standard tag %s and converter %s\n", path, 

                 lineNum,
                 alias,
                 GET_ALIAS_STR(aliasList->aliases[0]),
                 lineNum,
                 alias,
                 GET_ALIAS_STR(aliasList->aliases[0]),


@@ -749,13 +771,13 @@ static uint16_t
 addConverter(const char *converter) {
     uint32_t idx;
     if(converterCount>=MAX_CONV_COUNT) {
 addConverter(const char *converter) {
     uint32_t idx;
     if(converterCount>=MAX_CONV_COUNT) {

-        fprintf(stderr, "error(line %d): too many converters\n", lineNum);
+        fprintf(stderr, "%s:%d: error: too many converters\n", path, lineNum);

         exit(U_BUFFER_OVERFLOW_ERROR);
     }
 
     for (idx = 0; idx < converterCount; idx++) {
         if (ucnv_compareNames(converter, GET_ALIAS_STR(converters[idx].converter)) == 0) {
         exit(U_BUFFER_OVERFLOW_ERROR);
     }
 
     for (idx = 0; idx < converterCount; idx++) {
         if (ucnv_compareNames(converter, GET_ALIAS_STR(converters[idx].converter)) == 0) {

-            fprintf(stderr, "error(line %d): duplicate converter %s found!\n", lineNum, converter);
+            fprintf(stderr, "%s:%d: error: duplicate converter %s found!\n", path, lineNum, converter);

             exit(U_PARSE_ERROR);
             break;
         }
             exit(U_PARSE_ERROR);
             break;
         }


@@ -799,7 +821,8 @@ resolveAliasToConverter(uint16_t alias, uint16_t *tagNum, uint16_t *converterNum
     }
     *tagNum = UINT16_MAX;
     *converterNum = UINT16_MAX;
     }
     *tagNum = UINT16_MAX;
     *converterNum = UINT16_MAX;

-    fprintf(stderr, "warning: alias %s not found\n",
+    fprintf(stderr, "%s: warning: alias %s not found\n",
+        path,

         GET_ALIAS_STR(alias));
     return;
 }
         GET_ALIAS_STR(alias));
     return;
 }


@@ -866,6 +889,9 @@ resolveAliases(uint16_t *uniqueAliasArr, uint16_t *uniqueAliasToConverterArr, ui
             oldTagNum = currTagNum;
             /*printf("%s -> %s\n", GET_ALIAS_STR(knownAliases[idx]), GET_ALIAS_STR(converters[currConvNum].converter));*/
         }
             oldTagNum = currTagNum;
             /*printf("%s -> %s\n", GET_ALIAS_STR(knownAliases[idx]), GET_ALIAS_STR(converters[currConvNum].converter));*/
         }

+        if (uprv_strchr(GET_ALIAS_STR(converters[currConvNum].converter), UCNV_OPTION_SEP_CHAR) != NULL) {
+            uniqueAliasToConverterArr[uniqueAliasIdx-1] |= UCNV_CONTAINS_OPTION_BIT;
+        }

     }
     return uniqueAliasIdx;
 }
     }
     return uniqueAliasIdx;
 }


@@ -896,14 +922,15 @@ createOneAliasList(uint16_t *aliasArrLists, uint32_t tag, uint32_t converter, ui
             } else {
                 value = 0;
                 if (tag != 0) { /* Only show the warning when it's not the leftover tag. */
             } else {
                 value = 0;
                 if (tag != 0) { /* Only show the warning when it's not the leftover tag. */

-                    printf("warning: tag %s does not have a default alias for %s\n",
+                    fprintf(stderr, "%s: warning: tag %s does not have a default alias for %s\n",
+                            path,

                             GET_TAG_STR(tags[tag].tag),
                             GET_ALIAS_STR(converters[converter].converter));
                 }
             }
             aliasLists[aliasListsSize++] = value;
             if (aliasListsSize >= MAX_LIST_SIZE) {
                             GET_TAG_STR(tags[tag].tag),
                             GET_ALIAS_STR(converters[converter].converter));
                 }
             }
             aliasLists[aliasListsSize++] = value;
             if (aliasListsSize >= MAX_LIST_SIZE) {

-                fprintf(stderr, "error: Too many alias lists\n");
+                fprintf(stderr, "%s: error: Too many alias lists\n", path);

                 exit(U_BUFFER_OVERFLOW_ERROR);
             }
 
                 exit(U_BUFFER_OVERFLOW_ERROR);
             }
 


@@ -911,6 +938,26 @@ createOneAliasList(uint16_t *aliasArrLists, uint32_t tag, uint32_t converter, ui
     }
 }
 
     }
 }
 

+static void
+createNormalizedAliasStrings(char *normalizedStrings, const char *origStringBlock, int32_t stringBlockLength) {
+    int32_t currStrLen;
+    uprv_memcpy(normalizedStrings, origStringBlock, stringBlockLength);
+    while ((currStrLen = (int32_t)uprv_strlen(origStringBlock)) < stringBlockLength) {
+        int32_t currStrSize = currStrLen + 1;
+        if (currStrLen > 0) {
+            int32_t normStrLen;
+            ucnv_io_stripForCompare(normalizedStrings, origStringBlock);
+            normStrLen = uprv_strlen(normalizedStrings);
+            if (normStrLen > 0) {
+                uprv_memset(normalizedStrings + normStrLen, 0, currStrSize - normStrLen);
+            }
+        }
+        stringBlockLength -= currStrSize;
+        normalizedStrings += currStrSize;
+        origStringBlock += currStrSize;
+    }
+}
+

 static void
 writeAliasTable(UNewDataMemory *out) {
     uint32_t i, j;
 static void
 writeAliasTable(UNewDataMemory *out) {
     uint32_t i, j;


@@ -934,7 +981,12 @@ writeAliasTable(UNewDataMemory *out) {
     }
 
     /* Write the size of the TOC */
     }
 
     /* Write the size of the TOC */

-    udata_write32(out, 8);
+    if (tableOptions.stringNormalizationType == UCNV_IO_UNNORMALIZED) {
+        udata_write32(out, 8);
+    }
+    else {
+        udata_write32(out, 9);
+    }

 
     /* Write the sizes of each section */
     /* All sizes are the number of uint16_t units, not bytes */
 
     /* Write the sizes of each section */
     /* All sizes are the number of uint16_t units, not bytes */


@@ -944,8 +996,11 @@ writeAliasTable(UNewDataMemory *out) {
     udata_write32(out, uniqueAliasesSize);  /* The preresolved form of mapping an untagged the alias to a converter */
     udata_write32(out, tagCount * converterCount);
     udata_write32(out, aliasListsSize + 1);
     udata_write32(out, uniqueAliasesSize);  /* The preresolved form of mapping an untagged the alias to a converter */
     udata_write32(out, tagCount * converterCount);
     udata_write32(out, aliasListsSize + 1);

-    udata_write32(out, 0);  /* Reserved space. */
+    udata_write32(out, sizeof(tableOptions) / sizeof(uint16_t));

     udata_write32(out, (tagBlock.top + stringBlock.top) / sizeof(uint16_t));
     udata_write32(out, (tagBlock.top + stringBlock.top) / sizeof(uint16_t));

+    if (tableOptions.stringNormalizationType != UCNV_IO_UNNORMALIZED) {
+        udata_write32(out, (tagBlock.top + stringBlock.top) / sizeof(uint16_t));
+    }

 
     /* write the table of converters */
     /* Think of this as the column headers */
 
     /* write the table of converters */
     /* Think of this as the column headers */


@@ -979,27 +1034,65 @@ writeAliasTable(UNewDataMemory *out) {
     /* Write the lists */
     udata_writeBlock(out, (const void *)aliasLists, aliasListsSize * sizeof(uint16_t));
 
     /* Write the lists */
     udata_writeBlock(out, (const void *)aliasLists, aliasListsSize * sizeof(uint16_t));
 

+    /* Write any options for the alias table. */
+    udata_writeBlock(out, (const void *)&tableOptions, sizeof(tableOptions));
+

     /* write the tags strings */
     udata_writeString(out, tagBlock.store, tagBlock.top);
 
     /* write the aliases strings */
     udata_writeString(out, stringBlock.store, stringBlock.top);
 
     /* write the tags strings */
     udata_writeString(out, tagBlock.store, tagBlock.top);
 
     /* write the aliases strings */
     udata_writeString(out, stringBlock.store, stringBlock.top);
 

+    /* write the normalized aliases strings */
+    if (tableOptions.stringNormalizationType != UCNV_IO_UNNORMALIZED) {
+        char *normalizedStrings = (char *)uprv_malloc(tagBlock.top + stringBlock.top);
+        createNormalizedAliasStrings(normalizedStrings, tagBlock.store, tagBlock.top);
+        createNormalizedAliasStrings(normalizedStrings + tagBlock.top, stringBlock.store, stringBlock.top);
+
+        /* Write out the complete normalized array. */
+        udata_writeString(out, normalizedStrings, tagBlock.top + stringBlock.top);
+        uprv_free(normalizedStrings);
+    }
+

     uprv_free(aliasArrLists);
     uprv_free(uniqueAliases);
 }
 
 static char *
     uprv_free(aliasArrLists);
     uprv_free(uniqueAliases);
 }
 
 static char *

-allocString(StringBlock *block, uint32_t length) {
-    /* The (length&1) is used to keep the addresses on a 16-bit boundary */
-    uint32_t top=block->top + length + (length&1);
+allocString(StringBlock *block, const char *s, int32_t length) {
+    uint32_t top;

     char *p;
 
     char *p;
 

+    if(length<0) {
+        length=(int32_t)uprv_strlen(s);
+    }
+
+    /*
+     * add 1 for the terminating NUL
+     * and round up (+1 &~1)
+     * to keep the addresses on a 16-bit boundary
+     */
+    top=block->top + (uint32_t)((length + 1 + 1) & ~1);
+

     if(top >= block->max) {
     if(top >= block->max) {

-        fprintf(stderr, "error(line %d): out of memory\n", lineNum);
+        fprintf(stderr, "%s:%d: error: out of memory\n", path, lineNum);

         exit(U_MEMORY_ALLOCATION_ERROR);
     }
         exit(U_MEMORY_ALLOCATION_ERROR);
     }

+
+    /* get the pointer and copy the string */

     p = block->store + block->top;
     p = block->store + block->top;

+    uprv_memcpy(p, s, length);
+    p[length] = 0; /* NUL-terminate it */
+    if((length & 1) == 0) {
+        p[length + 1] = 0; /* set the padding byte */
+    }
+
+    /* check for invariant characters now that we have a NUL-terminated string for easy output */
+    if(!uprv_isInvariantString(p, length)) {
+        fprintf(stderr, "%s:%d: error: the name %s contains not just invariant characters\n", path, lineNum, p);
+        exit(U_INVALID_TABLE_FORMAT);
+    }
+

     block->top = top;
     return p;
 }
     block->top = top;
     return p;
 }


@@ -1010,7 +1103,7 @@ compareAliases(const void *alias1, const void *alias2) {
     int result = ucnv_compareNames(GET_ALIAS_STR(*(uint16_t*)alias1), GET_ALIAS_STR(*(uint16_t*)alias2));
     if (!result) {
         /* Sort the shortest first */
     int result = ucnv_compareNames(GET_ALIAS_STR(*(uint16_t*)alias1), GET_ALIAS_STR(*(uint16_t*)alias2));
     if (!result) {
         /* Sort the shortest first */

-        return uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias1)) - uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias2));
+        return (int)uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias1)) - (int)uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias2));

     }
     return result;
 }
     }
     return result;
 }