]> git.saurik.com Git - apple/icu.git/blob - icuSources/tools/gencnval/gencnval.c
ICU-6.2.14.tar.gz
[apple/icu.git] / icuSources / tools / gencnval / gencnval.c
1 /*
2 *******************************************************************************
3 *
4 * Copyright (C) 1999-2004, International Business Machines
5 * Corporation and others. All Rights Reserved.
6 *
7 *******************************************************************************
8 * file name: gencnval.c
9 * encoding: US-ASCII
10 * tab size: 8 (not used)
11 * indentation:4
12 *
13 * created on: 1999nov05
14 * created by: Markus W. Scherer
15 *
16 * This program reads convrtrs.txt and writes a memory-mappable
17 * converter name alias table to cnvalias.dat .
18 *
19 * This program currently writes version 2.1 of the data format. See
20 * ucnv_io.c for more details on the format. Note that version 2.1
21 * is written in such a way that a 2.0 reader will be able to use it,
22 * and a 2.1 reader will be able to read 2.0.
23 */
24
25 #include "unicode/utypes.h"
26 #include "unicode/putil.h"
27 #include "unicode/ucnv.h" /* ucnv_compareNames() */
28 #include "ucnv_io.h"
29 #include "cmemory.h"
30 #include "cstring.h"
31 #include "uinvchar.h"
32 #include "filestrm.h"
33 #include "unicode/uclean.h"
34 #include "unewdata.h"
35 #include "uoptions.h"
36
37 #include <stdio.h>
38 #include <stdlib.h>
39 #include <ctype.h>
40
41 /* TODO: Need to check alias name length is less than UCNV_MAX_CONVERTER_NAME_LENGTH */
42
43 /* STRING_STORE_SIZE + TAG_STORE_SIZE <= ((2^16 - 1) * 2)
44 That is the maximum size for the string stores combined
45 because the strings are index at 16-bit boundries by a
46 16-bit index, and there is only one section for the
47 strings.
48 */
49 #define STRING_STORE_SIZE 0x1FBFE /* 130046 */
50 #define TAG_STORE_SIZE 0x400 /* 1024 */
51
52 /* The combined tag and converter count can affect the number of lists
53 created. The size of all lists must be less than (2^17 - 1)
54 because the lists are indexed as a 16-bit array with a 16-bit index.
55 */
56 #define MAX_TAG_COUNT 0x3F /* 63 */
57 #define MAX_CONV_COUNT UCNV_CONVERTER_INDEX_MASK
58 #define MAX_ALIAS_COUNT 0xFFFF /* 65535 */
59
60 /* The maximum number of aliases that a standard tag/converter combination can have.
61 At this moment 6/18/2002, IANA has 12 names for ASCII. Don't go below 15 for
62 this value. I don't recommend more than 31 for this value.
63 */
64 #define MAX_TC_ALIAS_COUNT 0x1F /* 31 */
65
66 #define MAX_LINE_SIZE 0x7FFF /* 32767 */
67 #define MAX_LIST_SIZE 0xFFFF /* 65535 */
68
69 #define DATA_NAME "cnvalias"
70 #define DATA_TYPE "icu" /* ICU alias table */
71
72 #define ALL_TAG_STR "ALL"
73 #define ALL_TAG_NUM 1
74 #define EMPTY_TAG_NUM 0
75
76 /* UDataInfo cf. udata.h */
77 static const UDataInfo dataInfo={
78 sizeof(UDataInfo),
79 0,
80
81 U_IS_BIG_ENDIAN,
82 U_CHARSET_FAMILY,
83 sizeof(UChar),
84 0,
85
86 {0x43, 0x76, 0x41, 0x6c}, /* dataFormat="CvAl" */
87 {3, 0, 0, 0}, /* formatVersion */
88 {1, 4, 2, 0} /* dataVersion */
89 };
90
91 typedef struct {
92 char *store;
93 uint32_t top;
94 uint32_t max;
95 } StringBlock;
96
97 static char stringStore[STRING_STORE_SIZE];
98 static StringBlock stringBlock = { stringStore, 0, STRING_STORE_SIZE };
99
100 typedef struct {
101 uint16_t aliasCount;
102 uint16_t *aliases; /* Index into stringStore */
103 } AliasList;
104
105 typedef struct {
106 uint16_t converter; /* Index into stringStore */
107 uint16_t totalAliasCount; /* Total aliases in this column */
108 } Converter;
109
110 static Converter converters[MAX_CONV_COUNT];
111 static uint16_t converterCount=0;
112
113 static char tagStore[TAG_STORE_SIZE];
114 static StringBlock tagBlock = { tagStore, 0, TAG_STORE_SIZE };
115
116 typedef struct {
117 uint16_t tag; /* Index into tagStore */
118 uint16_t totalAliasCount; /* Total aliases in this row */
119 AliasList aliasList[MAX_CONV_COUNT];
120 } Tag;
121
122 /* Think of this as a 3D array. It's tagCount by converterCount by aliasCount */
123 static Tag tags[MAX_TAG_COUNT];
124 static uint16_t tagCount = 0;
125
126 /* Used for storing all aliases */
127 static uint16_t knownAliases[MAX_ALIAS_COUNT];
128 static uint16_t knownAliasesCount = 0;
129 /*static uint16_t duplicateKnownAliasesCount = 0;*/
130
131 /* Used for storing the lists section that point to aliases */
132 static uint16_t aliasLists[MAX_LIST_SIZE];
133 static uint16_t aliasListsSize = 0;
134
135 /* Were the standard tags declared before the aliases. */
136 static UBool standardTagsUsed = FALSE;
137 static UBool verbose = FALSE;
138 static int lineNum = 1;
139
140 /* prototypes --------------------------------------------------------------- */
141
142 static void
143 parseLine(const char *line);
144
145 static void
146 parseFile(FileStream *in);
147
148 static int32_t
149 chomp(char *line);
150
151 static void
152 addOfficialTaggedStandards(char *line, int32_t lineLen);
153
154 static uint16_t
155 addAlias(const char *alias, uint16_t standard, uint16_t converter, UBool defaultName);
156
157 static uint16_t
158 addConverter(const char *converter);
159
160 static char *
161 allocString(StringBlock *block, const char *s, int32_t length);
162
163 static uint16_t
164 addToKnownAliases(const char *alias);
165
166 static int
167 compareAliases(const void *alias1, const void *alias2);
168
169 static uint16_t
170 getTagNumber(const char *tag, uint16_t tagLen);
171
172 /*static void
173 addTaggedAlias(uint16_t tag, const char *alias, uint16_t converter);*/
174
175 static void
176 writeAliasTable(UNewDataMemory *out);
177
178 /* -------------------------------------------------------------------------- */
179
180 /* Presumes that you used allocString() */
181 #define GET_ALIAS_STR(index) (stringStore + ((size_t)(index) << 1))
182 #define GET_TAG_STR(index) (tagStore + ((size_t)(index) << 1))
183
184 /* Presumes that you used allocString() */
185 #define GET_ALIAS_NUM(str) ((uint16_t)((str - stringStore) >> 1))
186 #define GET_TAG_NUM(str) ((uint16_t)((str - tagStore) >> 1))
187
188 enum
189 {
190 HELP1,
191 HELP2,
192 VERBOSE,
193 COPYRIGHT,
194 DESTDIR,
195 SOURCEDIR
196 };
197
198 static UOption options[]={
199 UOPTION_HELP_H,
200 UOPTION_HELP_QUESTION_MARK,
201 UOPTION_VERBOSE,
202 UOPTION_COPYRIGHT,
203 UOPTION_DESTDIR,
204 UOPTION_SOURCEDIR
205 };
206
207 extern int
208 main(int argc, char* argv[]) {
209 char pathBuf[512];
210 const char *path;
211 FileStream *in;
212 UNewDataMemory *out;
213 UErrorCode errorCode=U_ZERO_ERROR;
214
215 U_MAIN_INIT_ARGS(argc, argv);
216
217 /* preset then read command line options */
218 options[DESTDIR].value=options[SOURCEDIR].value=u_getDataDirectory();
219 argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
220
221 /* error handling, printing usage message */
222 if(argc<0) {
223 fprintf(stderr,
224 "error in command line argument \"%s\"\n",
225 argv[-argc]);
226 }
227 if(argc<0 || options[HELP1].doesOccur || options[HELP2].doesOccur) {
228 fprintf(stderr,
229 "usage: %s [-options] [convrtrs.txt]\n"
230 "\tread convrtrs.txt and create " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE "\n"
231 "options:\n"
232 "\t-h or -? or --help this usage text\n"
233 "\t-v or --verbose prints out extra information about the alias table\n"
234 "\t-c or --copyright include a copyright notice\n"
235 "\t-d or --destdir destination directory, followed by the path\n"
236 "\t-s or --sourcedir source directory, followed by the path\n",
237 argv[0]);
238 return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
239 }
240
241 if(options[VERBOSE].doesOccur) {
242 verbose = TRUE;
243 }
244
245 if(argc>=2) {
246 path=argv[1];
247 } else {
248 path=options[SOURCEDIR].value;
249 if(path!=NULL && *path!=0) {
250 char *end;
251
252 uprv_strcpy(pathBuf, path);
253 end = uprv_strchr(pathBuf, 0);
254 if(*(end-1)!=U_FILE_SEP_CHAR) {
255 *(end++)=U_FILE_SEP_CHAR;
256 }
257 uprv_strcpy(end, "convrtrs.txt");
258 path=pathBuf;
259 } else {
260 path = "convrtrs.txt";
261 }
262 }
263
264 uprv_memset(stringStore, 0, sizeof(stringStore));
265 uprv_memset(tagStore, 0, sizeof(tagStore));
266 uprv_memset(converters, 0, sizeof(converters));
267 uprv_memset(tags, 0, sizeof(tags));
268 uprv_memset(aliasLists, 0, sizeof(aliasLists));
269 uprv_memset(knownAliases, 0, sizeof(aliasLists));
270
271
272 in=T_FileStream_open(path, "r");
273 if(in==NULL) {
274 fprintf(stderr, "gencnval: unable to open input file convrtrs.txt\n");
275 exit(U_FILE_ACCESS_ERROR);
276 }
277 parseFile(in);
278 T_FileStream_close(in);
279
280 /* create the output file */
281 out=udata_create(options[DESTDIR].value, DATA_TYPE, DATA_NAME, &dataInfo,
282 options[COPYRIGHT].doesOccur ? U_COPYRIGHT_STRING : NULL, &errorCode);
283 if(U_FAILURE(errorCode)) {
284 fprintf(stderr, "gencnval: unable to open output file - error %s\n", u_errorName(errorCode));
285 exit(errorCode);
286 }
287
288 /* write the table of aliases based on a tag/converter name combination */
289 writeAliasTable(out);
290
291 /* finish */
292 udata_finish(out, &errorCode);
293 if(U_FAILURE(errorCode)) {
294 fprintf(stderr, "gencnval: error finishing output file - %s\n", u_errorName(errorCode));
295 exit(errorCode);
296 }
297
298 return 0;
299 }
300
301 static void
302 parseFile(FileStream *in) {
303 char line[MAX_LINE_SIZE];
304 char lastLine[MAX_LINE_SIZE];
305 int32_t lineSize = 0;
306 int32_t lastLineSize = 0;
307 UBool validParse = TRUE;
308
309 lineNum = 0;
310
311 /* Add the empty tag, which is for untagged aliases */
312 getTagNumber("", 0);
313 getTagNumber(ALL_TAG_STR, 3);
314 allocString(&stringBlock, "", 0);
315
316 /* read the list of aliases */
317 while (validParse) {
318 validParse = FALSE;
319
320 /* Read non-empty lines that don't start with a space character. */
321 while (T_FileStream_readLine(in, lastLine, MAX_LINE_SIZE) != NULL) {
322 lastLineSize = chomp(lastLine);
323 if (lineSize == 0 || (lastLineSize > 0 && isspace(*lastLine))) {
324 uprv_strcpy(line + lineSize, lastLine);
325 lineSize += lastLineSize;
326 } else if (lineSize > 0) {
327 validParse = TRUE;
328 break;
329 }
330 lineNum++;
331 }
332
333 if (validParse || lineSize > 0) {
334 if (isspace(*line)) {
335 fprintf(stderr, "error(line %d): cannot start an alias with a space\n", lineNum-1);
336 exit(U_PARSE_ERROR);
337 } else if (line[0] == '{') {
338 if (!standardTagsUsed && line[lineSize - 1] != '}') {
339 fprintf(stderr, "error(line %d): alias needs to start with a converter name\n", lineNum);
340 exit(U_PARSE_ERROR);
341 }
342 addOfficialTaggedStandards(line, lineSize);
343 standardTagsUsed = TRUE;
344 } else {
345 if (standardTagsUsed) {
346 parseLine(line);
347 }
348 else {
349 fprintf(stderr, "error(line %d): alias table needs to start a list of standard tags\n", lineNum);
350 exit(U_PARSE_ERROR);
351 }
352 }
353 /* Was the last line consumed */
354 if (lastLineSize > 0) {
355 uprv_strcpy(line, lastLine);
356 lineSize = lastLineSize;
357 }
358 else {
359 lineSize = 0;
360 }
361 }
362 lineNum++;
363 }
364 }
365
366 /* This works almost like the Perl chomp.
367 It removes the newlines, comments and trailing whitespace (not preceding whitespace).
368 */
369 static int32_t
370 chomp(char *line) {
371 char *s = line;
372 char *lastNonSpace = line;
373 while(*s!=0) {
374 /* truncate at a newline or a comment */
375 if(*s == '\r' || *s == '\n' || *s == '#') {
376 *s = 0;
377 break;
378 }
379 if (!isspace(*s)) {
380 lastNonSpace = s;
381 }
382 ++s;
383 }
384 if (lastNonSpace++ > line) {
385 *lastNonSpace = 0;
386 s = lastNonSpace;
387 }
388 return (int32_t)(s - line);
389 }
390
391 static void
392 parseLine(const char *line) {
393 uint16_t pos=0, start, limit, length, cnv;
394 char *converter, *alias;
395
396 /* skip leading white space */
397 /* There is no whitespace at the beginning anymore */
398 /* while(line[pos]!=0 && isspace(line[pos])) {
399 ++pos;
400 }
401 */
402
403 /* is there nothing on this line? */
404 if(line[pos]==0) {
405 return;
406 }
407
408 /* get the converter name */
409 start=pos;
410 while(line[pos]!=0 && !isspace(line[pos])) {
411 ++pos;
412 }
413 limit=pos;
414
415 /* store the converter name */
416 length=(uint16_t)(limit-start);
417 converter=allocString(&stringBlock, line+start, length);
418
419 /* add the converter to the converter table */
420 cnv=addConverter(converter);
421
422 /* The name itself may be tagged, so let's added it to the aliases list properly */
423 pos = start;
424
425 /* get all the real aliases */
426 for(;;) {
427
428 /* skip white space */
429 while(line[pos]!=0 && isspace(line[pos])) {
430 ++pos;
431 }
432
433 /* is there no more alias name on this line? */
434 if(line[pos]==0) {
435 break;
436 }
437
438 /* get an alias name */
439 start=pos;
440 while(line[pos]!=0 && line[pos]!='{' && !isspace(line[pos])) {
441 ++pos;
442 }
443 limit=pos;
444
445 /* store the alias name */
446 length=(uint16_t)(limit-start);
447 if (start == 0) {
448 /* add the converter as its own alias to the alias table */
449 alias = converter;
450 addAlias(alias, ALL_TAG_NUM, cnv, TRUE);
451 }
452 else {
453 alias=allocString(&stringBlock, line+start, length);
454 addAlias(alias, ALL_TAG_NUM, cnv, FALSE);
455 }
456 addToKnownAliases(alias);
457
458 /* add the alias/converter pair to the alias table */
459 /* addAlias(alias, 0, cnv, FALSE);*/
460
461 /* skip whitespace */
462 while (line[pos] && isspace(line[pos])) {
463 ++pos;
464 }
465
466 /* handle tags if they are present */
467 if (line[pos] == '{') {
468 ++pos;
469 do {
470 start = pos;
471 while (line[pos] && line[pos] != '}' && !isspace( line[pos])) {
472 ++pos;
473 }
474 limit = pos;
475
476 if (start != limit) {
477 /* add the tag to the tag table */
478 uint16_t tag = getTagNumber(line + start, (uint16_t)(limit - start));
479 addAlias(alias, tag, cnv, (UBool)(line[limit-1] == '*'));
480 }
481
482 while (line[pos] && isspace(line[pos])) {
483 ++pos;
484 }
485 } while (line[pos] && line[pos] != '}');
486
487 if (line[pos] == '}') {
488 ++pos;
489 } else {
490 fprintf(stderr, "error(line %d): Unterminated tag list\n", lineNum);
491 exit(U_UNMATCHED_BRACES);
492 }
493 } else {
494 addAlias(alias, EMPTY_TAG_NUM, cnv, (UBool)(tags[0].aliasList[cnv].aliasCount == 0));
495 }
496 }
497 }
498
499 static uint16_t
500 getTagNumber(const char *tag, uint16_t tagLen) {
501 char *atag;
502 uint16_t t;
503 UBool preferredName = ((tagLen > 0) ? (tag[tagLen - 1] == '*') : (FALSE));
504
505 if (tagCount >= MAX_TAG_COUNT) {
506 fprintf(stderr, "error(line %d): too many tags\n", lineNum);
507 exit(U_BUFFER_OVERFLOW_ERROR);
508 }
509
510 if (preferredName) {
511 /* puts(tag);*/
512 tagLen--;
513 }
514
515 for (t = 0; t < tagCount; ++t) {
516 const char *currTag = GET_TAG_STR(tags[t].tag);
517 if (uprv_strlen(currTag) == tagLen && !uprv_strnicmp(currTag, tag, tagLen)) {
518 return t;
519 }
520 }
521
522 /* we need to add this tag */
523 if (tagCount >= MAX_TAG_COUNT) {
524 fprintf(stderr, "error(line %d): too many tags\n", lineNum);
525 exit(U_BUFFER_OVERFLOW_ERROR);
526 }
527
528 /* allocate a new entry in the tag table */
529 atag = allocString(&tagBlock, tag, tagLen);
530
531 if (standardTagsUsed) {
532 fprintf(stderr, "error(line %d): Tag \"%s\" is not declared at the beginning of the alias table.\n",
533 lineNum, atag);
534 exit(1);
535 }
536 else if (tagLen > 0 && strcmp(tag, ALL_TAG_STR) != 0) {
537 fprintf(stderr, "warning(line %d): Tag \"%s\" was added to the list of standards because it was not declared at beginning of the alias table.\n",
538 lineNum, atag);
539 }
540
541 /* add the tag to the tag table */
542 tags[tagCount].tag = GET_TAG_NUM(atag);
543 /* The aliasList should be set to 0's already */
544
545 return tagCount++;
546 }
547
548 /*static void
549 addTaggedAlias(uint16_t tag, const char *alias, uint16_t converter) {
550 tags[tag].aliases[converter] = alias;
551 }
552 */
553
554 static void
555 addOfficialTaggedStandards(char *line, int32_t lineLen) {
556 char *atag;
557 char *tag = strchr(line, '{') + 1;
558 static const char WHITESPACE[] = " \t";
559
560 if (tagCount > UCNV_NUM_RESERVED_TAGS) {
561 fprintf(stderr, "error(line %d): official tags already added\n", lineNum);
562 exit(U_BUFFER_OVERFLOW_ERROR);
563 }
564 strchr(tag, '}')[0] = 0;
565
566 tag = strtok(tag, WHITESPACE);
567 while (tag != NULL) {
568 /* printf("Adding original tag \"%s\"\n", tag);*/
569
570 /* allocate a new entry in the tag table */
571 atag = allocString(&tagBlock, tag, -1);
572
573 /* add the tag to the tag table */
574 tags[tagCount++].tag = (uint16_t)((atag - tagStore) >> 1);
575
576 /* The aliasList should already be set to 0's */
577
578 /* Get next tag */
579 tag = strtok(NULL, WHITESPACE);
580 }
581 }
582
583 static uint16_t
584 addToKnownAliases(const char *alias) {
585 /* uint32_t idx; */
586 /* strict matching */
587 /* for (idx = 0; idx < knownAliasesCount; idx++) {
588 uint16_t num = GET_ALIAS_NUM(alias);
589 if (knownAliases[idx] != num
590 && uprv_strcmp(alias, GET_ALIAS_STR(knownAliases[idx])) == 0)
591 {
592 fprintf(stderr, "warning(line %d): duplicate alias %s and %s found\n",
593 lineNum, alias, GET_ALIAS_STR(knownAliases[idx]));
594 duplicateKnownAliasesCount++;
595 break;
596 }
597 else if (knownAliases[idx] != num
598 && ucnv_compareNames(alias, GET_ALIAS_STR(knownAliases[idx])) == 0)
599 {
600 if (verbose) {
601 fprintf(stderr, "information(line %d): duplicate alias %s and %s found\n",
602 lineNum, alias, GET_ALIAS_STR(knownAliases[idx]));
603 }
604 duplicateKnownAliasesCount++;
605 break;
606 }
607 }
608 */
609 if (knownAliasesCount >= MAX_ALIAS_COUNT) {
610 fprintf(stderr, "warning(line %d): Too many aliases defined for all converters\n",
611 lineNum);
612 exit(U_BUFFER_OVERFLOW_ERROR);
613 }
614 /* TODO: We could try to unlist exact duplicates. */
615 return knownAliases[knownAliasesCount++] = GET_ALIAS_NUM(alias);
616 }
617
618 /*
619 @param standard When standard is 0, then it's the "empty" tag.
620 */
621 static uint16_t
622 addAlias(const char *alias, uint16_t standard, uint16_t converter, UBool defaultName) {
623 uint32_t idx, idx2;
624 UBool dupFound = FALSE;
625 UBool startEmptyWithoutDefault = FALSE;
626 AliasList *aliasList;
627
628 if(standard>=MAX_TAG_COUNT) {
629 fprintf(stderr, "error(line %d): too many standard tags\n", lineNum);
630 exit(U_BUFFER_OVERFLOW_ERROR);
631 }
632 if(converter>=MAX_CONV_COUNT) {
633 fprintf(stderr, "error(line %d): too many converter names\n", lineNum);
634 exit(U_BUFFER_OVERFLOW_ERROR);
635 }
636 aliasList = &tags[standard].aliasList[converter];
637
638 if (strchr(alias, '}')) {
639 fprintf(stderr, "error(line %d): unmatched } found\n",
640 lineNum);
641 }
642
643 if(aliasList->aliasCount + 1 >= MAX_TC_ALIAS_COUNT) {
644 fprintf(stderr, "error(line %d): too many aliases for alias %s and converter %s\n",
645 lineNum, alias, GET_ALIAS_STR(converters[converter].converter));
646 exit(U_BUFFER_OVERFLOW_ERROR);
647 }
648
649 /* Show this warning only once. All aliases are added to the "ALL" tag. */
650 if (standard == ALL_TAG_NUM && GET_ALIAS_STR(converters[converter].converter) != alias) {
651 /* Normally these option values are parsed at runtime, and they can
652 be discarded when the alias is a default converter. Options should
653 only be on a converter and not an alias. */
654 if (uprv_strchr(alias, UCNV_OPTION_SEP_CHAR) != 0)
655 {
656 fprintf(stderr, "warning(line %d): alias %s contains a \""UCNV_OPTION_SEP_STRING"\". Options are parsed at run-time and do not need to be in the alias table.\n",
657 lineNum, alias);
658 }
659 if (uprv_strchr(alias, UCNV_VALUE_SEP_CHAR) != 0)
660 {
661 fprintf(stderr, "warning(line %d): alias %s contains an \""UCNV_VALUE_SEP_STRING"\". Options are parsed at run-time and do not need to be in the alias table.\n",
662 lineNum, alias);
663 }
664 }
665
666 /* Check for duplicates in a tag/converter combination */
667 for (idx = 0; idx < aliasList->aliasCount; idx++) {
668 uint16_t aliasNum = tags[standard].aliasList[converter].aliases[idx];
669 if (aliasNum && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0 && standard != ALL_TAG_NUM)
670 {
671 fprintf(stderr, "warning(line %d): duplicate alias %s and %s found for standard %s\n",
672 lineNum, alias, GET_ALIAS_STR(aliasNum), GET_TAG_STR(tags[standard].tag));
673 dupFound = TRUE;
674 break;
675 }
676 }
677
678 if (!dupFound && standard != ALL_TAG_NUM) {
679 /* Check for duplicate aliases for this tag on all converters */
680 for (idx = 0; idx < converterCount; idx++) {
681 for (idx2 = 0; idx2 < tags[standard].aliasList[idx].aliasCount; idx2++) {
682 uint16_t aliasNum = tags[standard].aliasList[idx].aliases[idx2];
683 if (aliasNum
684 && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0)
685 {
686 fprintf(stderr, "warning(line %d): duplicate alias %s found for standard tag %s between converter %s and converter %s\n",
687 lineNum, alias, GET_TAG_STR(tags[standard].tag), GET_ALIAS_STR(converters[converter].converter), GET_ALIAS_STR(converters[idx].converter));
688 dupFound = TRUE;
689 break;
690 }
691 }
692 }
693
694 /* Check for duplicate default aliases for this converter on all tags */
695 /* It's okay to have multiple standards prefer the same name */
696 /* if (verbose && !dupFound) {
697 for (idx = 0; idx < tagCount; idx++) {
698 if (tags[idx].aliasList[converter].aliases) {
699 uint16_t aliasNum = tags[idx].aliasList[converter].aliases[0];
700 if (aliasNum
701 && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0)
702 {
703 fprintf(stderr, "warning(line %d): duplicate alias %s found for converter %s and standard tag %s\n",
704 lineNum, alias, GET_ALIAS_STR(converters[converter].converter), GET_TAG_STR(tags[standard].tag));
705 break;
706 }
707 }
708 }
709 }*/
710 }
711
712 if (aliasList->aliasCount <= 0) {
713 aliasList->aliasCount++;
714 startEmptyWithoutDefault = TRUE;
715 }
716 aliasList->aliases = (uint16_t *)uprv_realloc(aliasList->aliases, (aliasList->aliasCount + 1) * sizeof(aliasList->aliases[0]));
717 if (startEmptyWithoutDefault) {
718 aliasList->aliases[0] = 0;
719 }
720 if (defaultName) {
721 if (aliasList->aliases[0] != 0) {
722 fprintf(stderr, "error(line %d): Alias %s and %s cannot both be the default alias for standard tag %s and converter %s\n",
723 lineNum,
724 alias,
725 GET_ALIAS_STR(aliasList->aliases[0]),
726 GET_TAG_STR(tags[standard].tag),
727 GET_ALIAS_STR(converters[converter].converter));
728 exit(U_PARSE_ERROR);
729 }
730 aliasList->aliases[0] = GET_ALIAS_NUM(alias);
731 } else {
732 aliasList->aliases[aliasList->aliasCount++] = GET_ALIAS_NUM(alias);
733 }
734 /* aliasList->converter = converter;*/
735
736 converters[converter].totalAliasCount++; /* One more to the column */
737 tags[standard].totalAliasCount++; /* One more to the row */
738
739 return aliasList->aliasCount;
740 }
741
742 static uint16_t
743 addConverter(const char *converter) {
744 uint32_t idx;
745 if(converterCount>=MAX_CONV_COUNT) {
746 fprintf(stderr, "error(line %d): too many converters\n", lineNum);
747 exit(U_BUFFER_OVERFLOW_ERROR);
748 }
749
750 for (idx = 0; idx < converterCount; idx++) {
751 if (ucnv_compareNames(converter, GET_ALIAS_STR(converters[idx].converter)) == 0) {
752 fprintf(stderr, "error(line %d): duplicate converter %s found!\n", lineNum, converter);
753 exit(U_PARSE_ERROR);
754 break;
755 }
756 }
757
758 converters[converterCount].converter = GET_ALIAS_NUM(converter);
759 converters[converterCount].totalAliasCount = 0;
760
761 return converterCount++;
762 }
763
764 /* resolve this alias based on the prioritization of the standard tags. */
765 static void
766 resolveAliasToConverter(uint16_t alias, uint16_t *tagNum, uint16_t *converterNum) {
767 uint16_t idx, idx2, idx3;
768
769 for (idx = UCNV_NUM_RESERVED_TAGS; idx < tagCount; idx++) {
770 for (idx2 = 0; idx2 < converterCount; idx2++) {
771 for (idx3 = 0; idx3 < tags[idx].aliasList[idx2].aliasCount; idx3++) {
772 uint16_t aliasNum = tags[idx].aliasList[idx2].aliases[idx3];
773 if (aliasNum == alias) {
774 *tagNum = idx;
775 *converterNum = idx2;
776 return;
777 }
778 }
779 }
780 }
781 /* Do the leftovers last, just in case */
782 /* There is no need to do the ALL tag */
783 idx = 0;
784 for (idx2 = 0; idx2 < converterCount; idx2++) {
785 for (idx3 = 0; idx3 < tags[idx].aliasList[idx2].aliasCount; idx3++) {
786 uint16_t aliasNum = tags[idx].aliasList[idx2].aliases[idx3];
787 if (aliasNum == alias) {
788 *tagNum = idx;
789 *converterNum = idx2;
790 return;
791 }
792 }
793 }
794 *tagNum = UINT16_MAX;
795 *converterNum = UINT16_MAX;
796 fprintf(stderr, "warning: alias %s not found\n",
797 GET_ALIAS_STR(alias));
798 return;
799 }
800
801 /* The knownAliases should be sorted before calling this function */
802 static uint32_t
803 resolveAliases(uint16_t *uniqueAliasArr, uint16_t *uniqueAliasToConverterArr, uint16_t aliasOffset) {
804 uint32_t uniqueAliasIdx = 0;
805 uint32_t idx;
806 uint16_t currTagNum, oldTagNum;
807 uint16_t currConvNum, oldConvNum;
808 const char *lastName;
809
810 resolveAliasToConverter(knownAliases[0], &oldTagNum, &currConvNum);
811 uniqueAliasToConverterArr[uniqueAliasIdx] = currConvNum;
812 oldConvNum = currConvNum;
813 uniqueAliasArr[uniqueAliasIdx] = knownAliases[0] + aliasOffset;
814 uniqueAliasIdx++;
815 lastName = GET_ALIAS_STR(knownAliases[0]);
816
817 for (idx = 1; idx < knownAliasesCount; idx++) {
818 resolveAliasToConverter(knownAliases[idx], &currTagNum, &currConvNum);
819 if (ucnv_compareNames(lastName, GET_ALIAS_STR(knownAliases[idx])) == 0) {
820 /* duplicate found */
821 if ((currTagNum < oldTagNum && currTagNum >= UCNV_NUM_RESERVED_TAGS)
822 || oldTagNum == 0) {
823 oldTagNum = currTagNum;
824 uniqueAliasToConverterArr[uniqueAliasIdx - 1] = currConvNum;
825 uniqueAliasArr[uniqueAliasIdx - 1] = knownAliases[idx] + aliasOffset;
826 if (verbose) {
827 printf("using %s instead of %s -> %s",
828 GET_ALIAS_STR(knownAliases[idx]),
829 lastName,
830 GET_ALIAS_STR(converters[currConvNum].converter));
831 if (oldConvNum != currConvNum) {
832 printf(" (alias conflict)");
833 }
834 puts("");
835 }
836 }
837 else {
838 /* else ignore it */
839 if (verbose) {
840 printf("folding %s into %s -> %s",
841 GET_ALIAS_STR(knownAliases[idx]),
842 lastName,
843 GET_ALIAS_STR(converters[oldConvNum].converter));
844 if (oldConvNum != currConvNum) {
845 printf(" (alias conflict)");
846 }
847 puts("");
848 }
849 }
850 if (oldConvNum != currConvNum) {
851 uniqueAliasToConverterArr[uniqueAliasIdx - 1] |= UCNV_AMBIGUOUS_ALIAS_MAP_BIT;
852 }
853 }
854 else {
855 uniqueAliasToConverterArr[uniqueAliasIdx] = currConvNum;
856 oldConvNum = currConvNum;
857 uniqueAliasArr[uniqueAliasIdx] = knownAliases[idx] + aliasOffset;
858 uniqueAliasIdx++;
859 lastName = GET_ALIAS_STR(knownAliases[idx]);
860 oldTagNum = currTagNum;
861 /*printf("%s -> %s\n", GET_ALIAS_STR(knownAliases[idx]), GET_ALIAS_STR(converters[currConvNum].converter));*/
862 }
863 }
864 return uniqueAliasIdx;
865 }
866
867 static void
868 createOneAliasList(uint16_t *aliasArrLists, uint32_t tag, uint32_t converter, uint16_t offset) {
869 uint32_t aliasNum;
870 AliasList *aliasList = &tags[tag].aliasList[converter];
871
872 if (aliasList->aliasCount == 0) {
873 aliasArrLists[tag*converterCount + converter] = 0;
874 }
875 else {
876 aliasLists[aliasListsSize++] = aliasList->aliasCount;
877
878 /* write into the array area a 1's based index. */
879 aliasArrLists[tag*converterCount + converter] = aliasListsSize;
880
881 /* printf("tag %s converter %s\n",
882 GET_TAG_STR(tags[tag].tag),
883 GET_ALIAS_STR(converters[converter].converter));*/
884 for (aliasNum = 0; aliasNum < aliasList->aliasCount; aliasNum++) {
885 uint16_t value;
886 /* printf(" %s\n",
887 GET_ALIAS_STR(aliasList->aliases[aliasNum]));*/
888 if (aliasList->aliases[aliasNum]) {
889 value = aliasList->aliases[aliasNum] + offset;
890 } else {
891 value = 0;
892 if (tag != 0) { /* Only show the warning when it's not the leftover tag. */
893 printf("warning: tag %s does not have a default alias for %s\n",
894 GET_TAG_STR(tags[tag].tag),
895 GET_ALIAS_STR(converters[converter].converter));
896 }
897 }
898 aliasLists[aliasListsSize++] = value;
899 if (aliasListsSize >= MAX_LIST_SIZE) {
900 fprintf(stderr, "error: Too many alias lists\n");
901 exit(U_BUFFER_OVERFLOW_ERROR);
902 }
903
904 }
905 }
906 }
907
908 static void
909 writeAliasTable(UNewDataMemory *out) {
910 uint32_t i, j;
911 uint32_t uniqueAliasesSize;
912 uint16_t aliasOffset = (uint16_t)(tagBlock.top/sizeof(uint16_t));
913 uint16_t *aliasArrLists = (uint16_t *)uprv_malloc(tagCount * converterCount * sizeof(uint16_t));
914 uint16_t *uniqueAliases = (uint16_t *)uprv_malloc(knownAliasesCount * sizeof(uint16_t));
915 uint16_t *uniqueAliasesToConverter = (uint16_t *)uprv_malloc(knownAliasesCount * sizeof(uint16_t));
916
917 qsort(knownAliases, knownAliasesCount, sizeof(knownAliases[0]), compareAliases);
918 uniqueAliasesSize = resolveAliases(uniqueAliases, uniqueAliasesToConverter, aliasOffset);
919
920 /* Array index starts at 1. aliasLists[0] is the size of the lists section. */
921 aliasListsSize = 0;
922
923 /* write the offsets of all the aliases lists in a 2D array, and create the lists. */
924 for (i = 0; i < tagCount; ++i) {
925 for (j = 0; j < converterCount; ++j) {
926 createOneAliasList(aliasArrLists, i, j, aliasOffset);
927 }
928 }
929
930 /* Write the size of the TOC */
931 udata_write32(out, 8);
932
933 /* Write the sizes of each section */
934 /* All sizes are the number of uint16_t units, not bytes */
935 udata_write32(out, converterCount);
936 udata_write32(out, tagCount);
937 udata_write32(out, uniqueAliasesSize); /* list of aliases */
938 udata_write32(out, uniqueAliasesSize); /* The preresolved form of mapping an untagged the alias to a converter */
939 udata_write32(out, tagCount * converterCount);
940 udata_write32(out, aliasListsSize + 1);
941 udata_write32(out, 0); /* Reserved space. */
942 udata_write32(out, (tagBlock.top + stringBlock.top) / sizeof(uint16_t));
943
944 /* write the table of converters */
945 /* Think of this as the column headers */
946 for(i=0; i<converterCount; ++i) {
947 udata_write16(out, (uint16_t)(converters[i].converter + aliasOffset));
948 }
949
950 /* write the table of tags */
951 /* Think of this as the row headers */
952 for(i=UCNV_NUM_RESERVED_TAGS; i<tagCount; ++i) {
953 udata_write16(out, tags[i].tag);
954 }
955 /* The empty tag is considered the leftover list, and put that at the end of the priority list. */
956 udata_write16(out, tags[EMPTY_TAG_NUM].tag);
957 udata_write16(out, tags[ALL_TAG_NUM].tag);
958
959 /* Write the unique list of aliases */
960 udata_writeBlock(out, uniqueAliases, uniqueAliasesSize * sizeof(uint16_t));
961
962 /* Write the unique list of aliases */
963 udata_writeBlock(out, uniqueAliasesToConverter, uniqueAliasesSize * sizeof(uint16_t));
964
965 /* Write the array to the lists */
966 udata_writeBlock(out, (const void *)(aliasArrLists + (2*converterCount)), (((tagCount - 2) * converterCount) * sizeof(uint16_t)));
967 /* Now write the leftover part of the array for the EMPTY and ALL lists */
968 udata_writeBlock(out, (const void *)aliasArrLists, (2 * converterCount * sizeof(uint16_t)));
969
970 /* Offset the next array to make the index start at 1. */
971 udata_write16(out, 0xDEAD);
972
973 /* Write the lists */
974 udata_writeBlock(out, (const void *)aliasLists, aliasListsSize * sizeof(uint16_t));
975
976 /* write the tags strings */
977 udata_writeString(out, tagBlock.store, tagBlock.top);
978
979 /* write the aliases strings */
980 udata_writeString(out, stringBlock.store, stringBlock.top);
981
982 uprv_free(aliasArrLists);
983 uprv_free(uniqueAliases);
984 }
985
986 static char *
987 allocString(StringBlock *block, const char *s, int32_t length) {
988 uint32_t top;
989 char *p;
990
991 if(length<0) {
992 length=(int32_t)uprv_strlen(s);
993 }
994
995 /*
996 * add 1 for the terminating NUL
997 * and round up (+1 &~1)
998 * to keep the addresses on a 16-bit boundary
999 */
1000 top=block->top + (uint32_t)((length + 1 + 1) & ~1);
1001
1002 if(top >= block->max) {
1003 fprintf(stderr, "error(line %d): out of memory\n", lineNum);
1004 exit(U_MEMORY_ALLOCATION_ERROR);
1005 }
1006
1007 /* get the pointer and copy the string */
1008 p = block->store + block->top;
1009 uprv_memcpy(p, s, length);
1010 p[length] = 0; /* NUL-terminate it */
1011 if((length & 1) == 0) {
1012 p[length + 1] = 0; /* set the padding byte */
1013 }
1014
1015 /* check for invariant characters now that we have a NUL-terminated string for easy output */
1016 if(!uprv_isInvariantString(p, length)) {
1017 fprintf(stderr, "error(line %d): the name %s contains not just invariant characters\n", lineNum, p);
1018 exit(U_INVALID_TABLE_FORMAT);
1019 }
1020
1021 block->top = top;
1022 return p;
1023 }
1024
1025 static int
1026 compareAliases(const void *alias1, const void *alias2) {
1027 /* Names like IBM850 and ibm-850 need to be sorted together */
1028 int result = ucnv_compareNames(GET_ALIAS_STR(*(uint16_t*)alias1), GET_ALIAS_STR(*(uint16_t*)alias2));
1029 if (!result) {
1030 /* Sort the shortest first */
1031 return (int)uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias1)) - (int)uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias2));
1032 }
1033 return result;
1034 }
1035
1036 /*
1037 * Hey, Emacs, please set the following:
1038 *
1039 * Local Variables:
1040 * indent-tabs-mode: nil
1041 * End:
1042 *
1043 */
1044