/*
*******************************************************************************
*
-* Copyright (C) 1999-2003, International Business Machines
+* Copyright (C) 1999-2011, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
#include "ucnv_io.h"
#include "cmemory.h"
#include "cstring.h"
+#include "uinvchar.h"
#include "filestrm.h"
+#include "unicode/uclean.h"
#include "unewdata.h"
#include "uoptions.h"
0,
{0x43, 0x76, 0x41, 0x6c}, /* dataFormat="CvAl" */
- {3, 0, 0, 0}, /* formatVersion */
+ {3, 0, 1, 0}, /* formatVersion */
{1, 4, 2, 0} /* dataVersion */
};
/* Were the standard tags declared before the aliases. */
static UBool standardTagsUsed = FALSE;
static UBool verbose = FALSE;
-static int32_t lineNum = 1;
+static int lineNum = 1;
+
+static UConverterAliasOptions tableOptions = {
+ UCNV_IO_STD_NORMALIZED,
+ 1 /* containsCnvOptionInfo */
+};
+
+
+/**
+ * path to convrtrs.txt
+ */
+const char *path;
/* prototypes --------------------------------------------------------------- */
addConverter(const char *converter);
static char *
-allocString(StringBlock *block, uint32_t length);
+allocString(StringBlock *block, const char *s, int32_t length);
static uint16_t
addToKnownAliases(const char *alias);
extern int
main(int argc, char* argv[]) {
char pathBuf[512];
- const char *path;
FileStream *in;
UNewDataMemory *out;
UErrorCode errorCode=U_ZERO_ERROR;
U_MAIN_INIT_ARGS(argc, argv);
/* preset then read command line options */
- options[3].value=options[4].value=u_getDataDirectory();
+ options[DESTDIR].value=options[SOURCEDIR].value=u_getDataDirectory();
argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
/* error handling, printing usage message */
if(argc>=2) {
path=argv[1];
} else {
- path=options[4].value;
+ path=options[SOURCEDIR].value;
if(path!=NULL && *path!=0) {
- char *end = pathBuf+uprv_strlen(pathBuf);
+ char *end;
+
uprv_strcpy(pathBuf, path);
+ end = uprv_strchr(pathBuf, 0);
if(*(end-1)!=U_FILE_SEP_CHAR) {
*(end++)=U_FILE_SEP_CHAR;
}
in=T_FileStream_open(path, "r");
if(in==NULL) {
- fprintf(stderr, "gencnval: unable to open input file convrtrs.txt\n");
+ fprintf(stderr, "gencnval: unable to open input file %s\n", path);
exit(U_FILE_ACCESS_ERROR);
}
parseFile(in);
T_FileStream_close(in);
/* create the output file */
- out=udata_create(options[3].value, DATA_TYPE, U_ICUDATA_NAME "_" DATA_NAME, &dataInfo,
- options[2].doesOccur ? U_COPYRIGHT_STRING : NULL, &errorCode);
+ out=udata_create(options[DESTDIR].value, DATA_TYPE, DATA_NAME, &dataInfo,
+ options[COPYRIGHT].doesOccur ? U_COPYRIGHT_STRING : NULL, &errorCode);
if(U_FAILURE(errorCode)) {
fprintf(stderr, "gencnval: unable to open output file - error %s\n", u_errorName(errorCode));
exit(errorCode);
/* Add the empty tag, which is for untagged aliases */
getTagNumber("", 0);
getTagNumber(ALL_TAG_STR, 3);
- allocString(&stringBlock, 1);
+ allocString(&stringBlock, "", 0);
/* read the list of aliases */
while (validParse) {
/* Read non-empty lines that don't start with a space character. */
while (T_FileStream_readLine(in, lastLine, MAX_LINE_SIZE) != NULL) {
lastLineSize = chomp(lastLine);
- if (lineSize == 0 || (lastLineSize > 0 && isspace(*lastLine))) {
+ if (lineSize == 0 || (lastLineSize > 0 && isspace((int)*lastLine))) {
uprv_strcpy(line + lineSize, lastLine);
lineSize += lastLineSize;
} else if (lineSize > 0) {
}
if (validParse || lineSize > 0) {
- if (isspace(*line)) {
- fprintf(stderr, "error(line %d): cannot start an alias with a space\n", lineNum-1);
+ if (isspace((int)*line)) {
+ fprintf(stderr, "%s:%d: error: cannot start an alias with a space\n", path, lineNum-1);
exit(U_PARSE_ERROR);
} else if (line[0] == '{') {
if (!standardTagsUsed && line[lineSize - 1] != '}') {
- fprintf(stderr, "error(line %d): alias needs to start with a converter name\n", lineNum);
+ fprintf(stderr, "%s:%d: error: alias needs to start with a converter name\n", path, lineNum);
exit(U_PARSE_ERROR);
}
addOfficialTaggedStandards(line, lineSize);
parseLine(line);
}
else {
- fprintf(stderr, "error(line %d): alias table needs to start a list of standard tags\n", lineNum);
+ fprintf(stderr, "%s:%d: error: alias table needs to start a list of standard tags\n", path, lineNum);
exit(U_PARSE_ERROR);
}
}
*s = 0;
break;
}
- if (!isspace(*s)) {
+ if (!isspace((int)*s)) {
lastNonSpace = s;
}
++s;
/* get the converter name */
start=pos;
- while(line[pos]!=0 && !isspace(line[pos])) {
+ while(line[pos]!=0 && !isspace((int)line[pos])) {
++pos;
}
limit=pos;
/* store the converter name */
length=(uint16_t)(limit-start);
- converter=allocString(&stringBlock, length+1);
- uprv_memcpy(converter, line+start, length);
- converter[length]=0;
+ converter=allocString(&stringBlock, line+start, length);
/* add the converter to the converter table */
cnv=addConverter(converter);
for(;;) {
/* skip white space */
- while(line[pos]!=0 && isspace(line[pos])) {
+ while(line[pos]!=0 && isspace((int)line[pos])) {
++pos;
}
/* get an alias name */
start=pos;
- while(line[pos]!=0 && line[pos]!='{' && !isspace(line[pos])) {
+ while(line[pos]!=0 && line[pos]!='{' && !isspace((int)line[pos])) {
++pos;
}
limit=pos;
addAlias(alias, ALL_TAG_NUM, cnv, TRUE);
}
else {
- alias=allocString(&stringBlock, length+1);
- uprv_memcpy(alias, line+start, length);
- alias[length]=0;
+ alias=allocString(&stringBlock, line+start, length);
addAlias(alias, ALL_TAG_NUM, cnv, FALSE);
}
addToKnownAliases(alias);
/* addAlias(alias, 0, cnv, FALSE);*/
/* skip whitespace */
- while (line[pos] && isspace(line[pos])) {
+ while (line[pos] && isspace((int)line[pos])) {
++pos;
}
++pos;
do {
start = pos;
- while (line[pos] && line[pos] != '}' && !isspace( line[pos])) {
+ while (line[pos] && line[pos] != '}' && !isspace((int)line[pos])) {
++pos;
}
limit = pos;
addAlias(alias, tag, cnv, (UBool)(line[limit-1] == '*'));
}
- while (line[pos] && isspace(line[pos])) {
+ while (line[pos] && isspace((int)line[pos])) {
++pos;
}
} while (line[pos] && line[pos] != '}');
if (line[pos] == '}') {
++pos;
} else {
- fprintf(stderr, "error(line %d): Unterminated tag list\n", lineNum);
+ fprintf(stderr, "%s:%d: Unterminated tag list\n", path, lineNum);
exit(U_UNMATCHED_BRACES);
}
} else {
UBool preferredName = ((tagLen > 0) ? (tag[tagLen - 1] == '*') : (FALSE));
if (tagCount >= MAX_TAG_COUNT) {
- fprintf(stderr, "error(line %d): too many tags\n", lineNum);
+ fprintf(stderr, "%s:%d: too many tags\n", path, lineNum);
exit(U_BUFFER_OVERFLOW_ERROR);
}
/* we need to add this tag */
if (tagCount >= MAX_TAG_COUNT) {
- fprintf(stderr, "error(line %d): too many tags\n", lineNum);
+ fprintf(stderr, "%s:%d: error: too many tags\n", path, lineNum);
exit(U_BUFFER_OVERFLOW_ERROR);
}
/* allocate a new entry in the tag table */
- atag = allocString(&tagBlock, tagLen + 1);
- uprv_memcpy(atag, tag, tagLen);
- atag[tagLen] = 0;
+ atag = allocString(&tagBlock, tag, tagLen);
if (standardTagsUsed) {
- fprintf(stderr, "error(line %d): Tag \"%s\" is not declared at the beginning of the alias table.\n",
- lineNum, atag);
+ fprintf(stderr, "%s:%d: error: Tag \"%s\" is not declared at the beginning of the alias table.\n",
+ path, lineNum, atag);
exit(1);
}
else if (tagLen > 0 && strcmp(tag, ALL_TAG_STR) != 0) {
- fprintf(stderr, "warning(line %d): Tag \"%s\" was added to the list of standards because it was not declared at beginning of the alias table.\n",
- lineNum, atag);
+ fprintf(stderr, "%s:%d: warning: Tag \"%s\" was added to the list of standards because it was not declared at beginning of the alias table.\n",
+ path, lineNum, atag);
}
/* add the tag to the tag table */
static void
addOfficialTaggedStandards(char *line, int32_t lineLen) {
char *atag;
- char *tag = strchr(line, '{') + 1;
- uint16_t tagSize;
+ char *endTagExp;
+ char *tag;
static const char WHITESPACE[] = " \t";
if (tagCount > UCNV_NUM_RESERVED_TAGS) {
- fprintf(stderr, "error(line %d): official tags already added\n", lineNum);
+ fprintf(stderr, "%s:%d: error: official tags already added\n", path, lineNum);
exit(U_BUFFER_OVERFLOW_ERROR);
}
- strchr(tag, '}')[0] = 0;
+ tag = strchr(line, '{');
+ if (tag == NULL) {
+ /* Why were we called? */
+ fprintf(stderr, "%s:%d: error: Missing start of tag group\n", path, lineNum);
+ exit(U_PARSE_ERROR);
+ }
+ tag++;
+ endTagExp = strchr(tag, '}');
+ if (endTagExp == NULL) {
+ fprintf(stderr, "%s:%d: error: Missing end of tag group\n", path, lineNum);
+ exit(U_PARSE_ERROR);
+ }
+ endTagExp[0] = 0;
tag = strtok(tag, WHITESPACE);
while (tag != NULL) {
/* printf("Adding original tag \"%s\"\n", tag);*/
- tagSize = strlen(tag) + 1;
/* allocate a new entry in the tag table */
-
- atag = allocString(&tagBlock, tagSize);
- uprv_memcpy(atag, tag, tagSize);
+ atag = allocString(&tagBlock, tag, -1);
/* add the tag to the tag table */
tags[tagCount++].tag = (uint16_t)((atag - tagStore) >> 1);
if (knownAliases[idx] != num
&& uprv_strcmp(alias, GET_ALIAS_STR(knownAliases[idx])) == 0)
{
- fprintf(stderr, "warning(line %d): duplicate alias %s and %s found\n",
+ fprintf(stderr, "%s:%d: warning: duplicate alias %s and %s found\n", path,
lineNum, alias, GET_ALIAS_STR(knownAliases[idx]));
duplicateKnownAliasesCount++;
break;
&& ucnv_compareNames(alias, GET_ALIAS_STR(knownAliases[idx])) == 0)
{
if (verbose) {
- fprintf(stderr, "information(line %d): duplicate alias %s and %s found\n",
+ fprintf(stderr, "%s:%d: information: duplicate alias %s and %s found\n", path,
lineNum, alias, GET_ALIAS_STR(knownAliases[idx]));
}
duplicateKnownAliasesCount++;
}
*/
if (knownAliasesCount >= MAX_ALIAS_COUNT) {
- fprintf(stderr, "warning(line %d): Too many aliases defined for all converters\n",
- lineNum);
+ fprintf(stderr, "%s:%d: warning: Too many aliases defined for all converters\n",
+ path, lineNum);
exit(U_BUFFER_OVERFLOW_ERROR);
}
/* TODO: We could try to unlist exact duplicates. */
AliasList *aliasList;
if(standard>=MAX_TAG_COUNT) {
- fprintf(stderr, "error(line %d): too many standard tags\n", lineNum);
+ fprintf(stderr, "%s:%d: error: too many standard tags\n", path, lineNum);
exit(U_BUFFER_OVERFLOW_ERROR);
}
if(converter>=MAX_CONV_COUNT) {
- fprintf(stderr, "error(line %d): too many converter names\n", lineNum);
+ fprintf(stderr, "%s:%d: error: too many converter names\n", path, lineNum);
exit(U_BUFFER_OVERFLOW_ERROR);
}
aliasList = &tags[standard].aliasList[converter];
if (strchr(alias, '}')) {
- fprintf(stderr, "error(line %d): unmatched } found\n",
+ fprintf(stderr, "%s:%d: error: unmatched } found\n", path,
lineNum);
}
if(aliasList->aliasCount + 1 >= MAX_TC_ALIAS_COUNT) {
- fprintf(stderr, "error(line %d): too many aliases for alias %s and converter %s\n",
+ fprintf(stderr, "%s:%d: error: too many aliases for alias %s and converter %s\n", path,
lineNum, alias, GET_ALIAS_STR(converters[converter].converter));
exit(U_BUFFER_OVERFLOW_ERROR);
}
}
}
- /* Check for duplicates in a tag/converter combination */
- for (idx = 0; idx < aliasList->aliasCount; idx++) {
- uint16_t aliasNum = tags[standard].aliasList[converter].aliases[idx];
- if (aliasNum && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0 && standard != ALL_TAG_NUM)
- {
- fprintf(stderr, "warning(line %d): duplicate alias %s and %s found for standard %s\n",
- lineNum, alias, GET_ALIAS_STR(aliasNum), GET_TAG_STR(tags[standard].tag));
- dupFound = TRUE;
- break;
- }
- }
-
- if (!dupFound && standard != ALL_TAG_NUM) {
+ if (standard != ALL_TAG_NUM) {
/* Check for duplicate aliases for this tag on all converters */
for (idx = 0; idx < converterCount; idx++) {
for (idx2 = 0; idx2 < tags[standard].aliasList[idx].aliasCount; idx2++) {
if (aliasNum
&& ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0)
{
- fprintf(stderr, "warning(line %d): duplicate alias %s found for standard tag %s between converter %s and converter %s\n",
- lineNum, alias, GET_TAG_STR(tags[standard].tag), GET_ALIAS_STR(converters[converter].converter), GET_ALIAS_STR(converters[idx].converter));
+ if (idx == converter) {
+ /*
+ * (alias, standard) duplicates are harmless if they map to the same converter.
+ * Only print a warning in verbose mode, or if the alias is a precise duplicate,
+ * not just a lenient-match duplicate.
+ */
+ if (verbose || 0 == uprv_strcmp(alias, GET_ALIAS_STR(aliasNum))) {
+ fprintf(stderr, "%s:%d: warning: duplicate aliases %s and %s found for standard %s and converter %s\n", path,
+ lineNum, alias, GET_ALIAS_STR(aliasNum),
+ GET_TAG_STR(tags[standard].tag),
+ GET_ALIAS_STR(converters[converter].converter));
+ }
+ } else {
+ fprintf(stderr, "%s:%d: warning: duplicate aliases %s and %s found for standard tag %s between converter %s and converter %s\n", path,
+ lineNum, alias, GET_ALIAS_STR(aliasNum),
+ GET_TAG_STR(tags[standard].tag),
+ GET_ALIAS_STR(converters[converter].converter),
+ GET_ALIAS_STR(converters[idx].converter));
+ }
dupFound = TRUE;
break;
}
if (aliasNum
&& ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0)
{
- fprintf(stderr, "warning(line %d): duplicate alias %s found for converter %s and standard tag %s\n",
+ fprintf(stderr, "%s:%d: warning: duplicate alias %s found for converter %s and standard tag %s\n", path,
lineNum, alias, GET_ALIAS_STR(converters[converter].converter), GET_TAG_STR(tags[standard].tag));
break;
}
}
if (defaultName) {
if (aliasList->aliases[0] != 0) {
- fprintf(stderr, "error(line %d): Alias %s and %s cannot both be the default alias for standard tag %s and converter %s\n",
+ fprintf(stderr, "%s:%d: error: Alias %s and %s cannot both be the default alias for standard tag %s and converter %s\n", path,
lineNum,
alias,
GET_ALIAS_STR(aliasList->aliases[0]),
addConverter(const char *converter) {
uint32_t idx;
if(converterCount>=MAX_CONV_COUNT) {
- fprintf(stderr, "error(line %d): too many converters\n", lineNum);
+ fprintf(stderr, "%s:%d: error: too many converters\n", path, lineNum);
exit(U_BUFFER_OVERFLOW_ERROR);
}
for (idx = 0; idx < converterCount; idx++) {
if (ucnv_compareNames(converter, GET_ALIAS_STR(converters[idx].converter)) == 0) {
- fprintf(stderr, "error(line %d): duplicate converter %s found!\n", lineNum, converter);
+ fprintf(stderr, "%s:%d: error: duplicate converter %s found!\n", path, lineNum, converter);
exit(U_PARSE_ERROR);
break;
}
}
*tagNum = UINT16_MAX;
*converterNum = UINT16_MAX;
- fprintf(stderr, "warning: alias %s not found\n",
+ fprintf(stderr, "%s: warning: alias %s not found\n",
+ path,
GET_ALIAS_STR(alias));
return;
}
oldTagNum = currTagNum;
/*printf("%s -> %s\n", GET_ALIAS_STR(knownAliases[idx]), GET_ALIAS_STR(converters[currConvNum].converter));*/
}
+ if (uprv_strchr(GET_ALIAS_STR(converters[currConvNum].converter), UCNV_OPTION_SEP_CHAR) != NULL) {
+ uniqueAliasToConverterArr[uniqueAliasIdx-1] |= UCNV_CONTAINS_OPTION_BIT;
+ }
}
return uniqueAliasIdx;
}
} else {
value = 0;
if (tag != 0) { /* Only show the warning when it's not the leftover tag. */
- printf("warning: tag %s does not have a default alias for %s\n",
+ fprintf(stderr, "%s: warning: tag %s does not have a default alias for %s\n",
+ path,
GET_TAG_STR(tags[tag].tag),
GET_ALIAS_STR(converters[converter].converter));
}
}
aliasLists[aliasListsSize++] = value;
if (aliasListsSize >= MAX_LIST_SIZE) {
- fprintf(stderr, "error: Too many alias lists\n");
+ fprintf(stderr, "%s: error: Too many alias lists\n", path);
exit(U_BUFFER_OVERFLOW_ERROR);
}
}
}
+static void
+createNormalizedAliasStrings(char *normalizedStrings, const char *origStringBlock, int32_t stringBlockLength) {
+ int32_t currStrLen;
+ uprv_memcpy(normalizedStrings, origStringBlock, stringBlockLength);
+ while ((currStrLen = (int32_t)uprv_strlen(origStringBlock)) < stringBlockLength) {
+ int32_t currStrSize = currStrLen + 1;
+ if (currStrLen > 0) {
+ int32_t normStrLen;
+ ucnv_io_stripForCompare(normalizedStrings, origStringBlock);
+ normStrLen = uprv_strlen(normalizedStrings);
+ if (normStrLen > 0) {
+ uprv_memset(normalizedStrings + normStrLen, 0, currStrSize - normStrLen);
+ }
+ }
+ stringBlockLength -= currStrSize;
+ normalizedStrings += currStrSize;
+ origStringBlock += currStrSize;
+ }
+}
+
static void
writeAliasTable(UNewDataMemory *out) {
uint32_t i, j;
}
/* Write the size of the TOC */
- udata_write32(out, 8);
+ if (tableOptions.stringNormalizationType == UCNV_IO_UNNORMALIZED) {
+ udata_write32(out, 8);
+ }
+ else {
+ udata_write32(out, 9);
+ }
/* Write the sizes of each section */
/* All sizes are the number of uint16_t units, not bytes */
udata_write32(out, uniqueAliasesSize); /* The preresolved form of mapping an untagged the alias to a converter */
udata_write32(out, tagCount * converterCount);
udata_write32(out, aliasListsSize + 1);
- udata_write32(out, 0); /* Reserved space. */
+ udata_write32(out, sizeof(tableOptions) / sizeof(uint16_t));
udata_write32(out, (tagBlock.top + stringBlock.top) / sizeof(uint16_t));
+ if (tableOptions.stringNormalizationType != UCNV_IO_UNNORMALIZED) {
+ udata_write32(out, (tagBlock.top + stringBlock.top) / sizeof(uint16_t));
+ }
/* write the table of converters */
/* Think of this as the column headers */
/* Write the lists */
udata_writeBlock(out, (const void *)aliasLists, aliasListsSize * sizeof(uint16_t));
+ /* Write any options for the alias table. */
+ udata_writeBlock(out, (const void *)&tableOptions, sizeof(tableOptions));
+
/* write the tags strings */
udata_writeString(out, tagBlock.store, tagBlock.top);
/* write the aliases strings */
udata_writeString(out, stringBlock.store, stringBlock.top);
+ /* write the normalized aliases strings */
+ if (tableOptions.stringNormalizationType != UCNV_IO_UNNORMALIZED) {
+ char *normalizedStrings = (char *)uprv_malloc(tagBlock.top + stringBlock.top);
+ createNormalizedAliasStrings(normalizedStrings, tagBlock.store, tagBlock.top);
+ createNormalizedAliasStrings(normalizedStrings + tagBlock.top, stringBlock.store, stringBlock.top);
+
+ /* Write out the complete normalized array. */
+ udata_writeString(out, normalizedStrings, tagBlock.top + stringBlock.top);
+ uprv_free(normalizedStrings);
+ }
+
uprv_free(aliasArrLists);
uprv_free(uniqueAliases);
}
static char *
-allocString(StringBlock *block, uint32_t length) {
- /* The (length&1) is used to keep the addresses on a 16-bit boundary */
- uint32_t top=block->top + length + (length&1);
+allocString(StringBlock *block, const char *s, int32_t length) {
+ uint32_t top;
char *p;
+ if(length<0) {
+ length=(int32_t)uprv_strlen(s);
+ }
+
+ /*
+ * add 1 for the terminating NUL
+ * and round up (+1 &~1)
+ * to keep the addresses on a 16-bit boundary
+ */
+ top=block->top + (uint32_t)((length + 1 + 1) & ~1);
+
if(top >= block->max) {
- fprintf(stderr, "error(line %d): out of memory\n", lineNum);
+ fprintf(stderr, "%s:%d: error: out of memory\n", path, lineNum);
exit(U_MEMORY_ALLOCATION_ERROR);
}
+
+ /* get the pointer and copy the string */
p = block->store + block->top;
+ uprv_memcpy(p, s, length);
+ p[length] = 0; /* NUL-terminate it */
+ if((length & 1) == 0) {
+ p[length + 1] = 0; /* set the padding byte */
+ }
+
+ /* check for invariant characters now that we have a NUL-terminated string for easy output */
+ if(!uprv_isInvariantString(p, length)) {
+ fprintf(stderr, "%s:%d: error: the name %s contains not just invariant characters\n", path, lineNum, p);
+ exit(U_INVALID_TABLE_FORMAT);
+ }
+
block->top = top;
return p;
}
int result = ucnv_compareNames(GET_ALIAS_STR(*(uint16_t*)alias1), GET_ALIAS_STR(*(uint16_t*)alias2));
if (!result) {
/* Sort the shortest first */
- return uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias1)) - uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias2));
+ return (int)uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias1)) - (int)uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias2));
}
return result;
}
projects / apple / icu.git / blobdiff