]>
Commit | Line | Data |
---|---|---|
b75a7d8f A |
1 | /* |
2 | ******************************************************************************* | |
3 | * | |
73c04bcf | 4 | * Copyright (C) 2002-2006, International Business Machines |
b75a7d8f A |
5 | * Corporation and others. All Rights Reserved. |
6 | * | |
7 | ******************************************************************************* | |
8 | * file name: props2.c | |
9 | * encoding: US-ASCII | |
10 | * tab size: 8 (not used) | |
11 | * indentation:4 | |
12 | * | |
13 | * created on: 2002feb24 | |
14 | * created by: Markus W. Scherer | |
15 | * | |
16 | * Parse more Unicode Character Database files and store | |
17 | * additional Unicode character properties in bit set vectors. | |
18 | */ | |
19 | ||
20 | #include <stdio.h> | |
21 | #include "unicode/utypes.h" | |
22 | #include "unicode/uchar.h" | |
23 | #include "unicode/uscript.h" | |
24 | #include "cstring.h" | |
25 | #include "cmemory.h" | |
26 | #include "utrie.h" | |
27 | #include "uprops.h" | |
28 | #include "propsvec.h" | |
29 | #include "uparse.h" | |
73c04bcf | 30 | #include "writesrc.h" |
b75a7d8f A |
31 | #include "genprops.h" |
32 | ||
33 | #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) | |
34 | ||
35 | /* data --------------------------------------------------------------------- */ | |
36 | ||
37 | static UNewTrie *trie; | |
38 | uint32_t *pv; | |
39 | static int32_t pvCount; | |
40 | ||
41 | /* miscellaneous ------------------------------------------------------------ */ | |
42 | ||
43 | static char * | |
44 | trimTerminateField(char *s, char *limit) { | |
45 | /* trim leading whitespace */ | |
46 | s=(char *)u_skipWhitespace(s); | |
47 | ||
48 | /* trim trailing whitespace */ | |
49 | while(s<limit && (*(limit-1)==' ' || *(limit-1)=='\t')) { | |
50 | --limit; | |
51 | } | |
52 | *limit=0; | |
53 | ||
54 | return s; | |
55 | } | |
56 | ||
57 | static void | |
58 | parseTwoFieldFile(char *filename, char *basename, | |
59 | const char *ucdFile, const char *suffix, | |
60 | UParseLineFn *lineFn, | |
61 | UErrorCode *pErrorCode) { | |
62 | char *fields[2][2]; | |
63 | ||
64 | if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { | |
65 | return; | |
66 | } | |
67 | ||
68 | writeUCDFilename(basename, ucdFile, suffix); | |
69 | ||
70 | u_parseDelimitedFile(filename, ';', fields, 2, lineFn, NULL, pErrorCode); | |
71 | if(U_FAILURE(*pErrorCode)) { | |
72 | fprintf(stderr, "error parsing %s.txt: %s\n", ucdFile, u_errorName(*pErrorCode)); | |
73 | } | |
74 | } | |
75 | ||
76 | static void U_CALLCONV | |
77 | ageLineFn(void *context, | |
78 | char *fields[][2], int32_t fieldCount, | |
79 | UErrorCode *pErrorCode); | |
80 | ||
81 | static void | |
82 | parseMultiFieldFile(char *filename, char *basename, | |
83 | const char *ucdFile, const char *suffix, | |
84 | int32_t fieldCount, | |
85 | UParseLineFn *lineFn, | |
86 | UErrorCode *pErrorCode) { | |
87 | char *fields[20][2]; | |
88 | ||
89 | if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { | |
90 | return; | |
91 | } | |
92 | ||
93 | writeUCDFilename(basename, ucdFile, suffix); | |
94 | ||
95 | u_parseDelimitedFile(filename, ';', fields, fieldCount, lineFn, NULL, pErrorCode); | |
96 | if(U_FAILURE(*pErrorCode)) { | |
97 | fprintf(stderr, "error parsing %s.txt: %s\n", ucdFile, u_errorName(*pErrorCode)); | |
98 | } | |
99 | } | |
100 | ||
101 | static void U_CALLCONV | |
102 | numericLineFn(void *context, | |
103 | char *fields[][2], int32_t fieldCount, | |
104 | UErrorCode *pErrorCode); | |
105 | ||
b75a7d8f A |
106 | /* parse files with single enumerated properties ---------------------------- */ |
107 | ||
108 | struct SingleEnum { | |
109 | const char *ucdFile, *propName; | |
110 | UProperty prop; | |
111 | int32_t vecWord, vecShift; | |
112 | uint32_t vecMask; | |
113 | }; | |
114 | typedef struct SingleEnum SingleEnum; | |
115 | ||
116 | static void | |
117 | parseSingleEnumFile(char *filename, char *basename, const char *suffix, | |
118 | const SingleEnum *sen, | |
119 | UErrorCode *pErrorCode); | |
120 | ||
121 | static const SingleEnum scriptSingleEnum={ | |
122 | "Scripts", "script", | |
123 | UCHAR_SCRIPT, | |
124 | 0, 0, UPROPS_SCRIPT_MASK | |
125 | }; | |
126 | ||
127 | static const SingleEnum blockSingleEnum={ | |
128 | "Blocks", "block", | |
129 | UCHAR_BLOCK, | |
130 | 0, UPROPS_BLOCK_SHIFT, UPROPS_BLOCK_MASK | |
131 | }; | |
132 | ||
73c04bcf A |
133 | static const SingleEnum graphemeClusterBreakSingleEnum={ |
134 | "GraphemeBreakProperty", "Grapheme_Cluster_Break", | |
135 | UCHAR_GRAPHEME_CLUSTER_BREAK, | |
136 | 2, UPROPS_GCB_SHIFT, UPROPS_GCB_MASK | |
137 | }; | |
138 | ||
139 | static const SingleEnum wordBreakSingleEnum={ | |
140 | "WordBreakProperty", "Word_Break", | |
141 | UCHAR_WORD_BREAK, | |
142 | 2, UPROPS_WB_SHIFT, UPROPS_WB_MASK | |
143 | }; | |
144 | ||
145 | static const SingleEnum sentenceBreakSingleEnum={ | |
146 | "SentenceBreakProperty", "Sentence_Break", | |
147 | UCHAR_SENTENCE_BREAK, | |
148 | 2, UPROPS_SB_SHIFT, UPROPS_SB_MASK | |
149 | }; | |
150 | ||
b75a7d8f A |
151 | static const SingleEnum lineBreakSingleEnum={ |
152 | "LineBreak", "line break", | |
153 | UCHAR_LINE_BREAK, | |
154 | 0, UPROPS_LB_SHIFT, UPROPS_LB_MASK | |
155 | }; | |
156 | ||
157 | static const SingleEnum eawSingleEnum={ | |
158 | "EastAsianWidth", "east asian width", | |
159 | UCHAR_EAST_ASIAN_WIDTH, | |
160 | 0, UPROPS_EA_SHIFT, UPROPS_EA_MASK | |
161 | }; | |
162 | ||
b75a7d8f A |
163 | static void U_CALLCONV |
164 | singleEnumLineFn(void *context, | |
165 | char *fields[][2], int32_t fieldCount, | |
166 | UErrorCode *pErrorCode) { | |
167 | const SingleEnum *sen; | |
168 | char *s; | |
169 | uint32_t start, limit, uv; | |
170 | int32_t value; | |
171 | ||
172 | sen=(const SingleEnum *)context; | |
173 | ||
174 | u_parseCodePointRange(fields[0][0], &start, &limit, pErrorCode); | |
175 | if(U_FAILURE(*pErrorCode)) { | |
176 | fprintf(stderr, "genprops: syntax error in %s.txt field 0 at %s\n", sen->ucdFile, fields[0][0]); | |
177 | exit(*pErrorCode); | |
178 | } | |
179 | ++limit; | |
180 | ||
181 | /* parse property alias */ | |
182 | s=trimTerminateField(fields[1][0], fields[1][1]); | |
183 | value=u_getPropertyValueEnum(sen->prop, s); | |
184 | if(value<0) { | |
185 | if(sen->prop==UCHAR_BLOCK) { | |
186 | if(isToken("Greek", s)) { | |
187 | value=UBLOCK_GREEK; /* Unicode 3.2 renames this to "Greek and Coptic" */ | |
188 | } else if(isToken("Combining Marks for Symbols", s)) { | |
189 | value=UBLOCK_COMBINING_MARKS_FOR_SYMBOLS; /* Unicode 3.2 renames this to "Combining Diacritical Marks for Symbols" */ | |
190 | } else if(isToken("Private Use", s)) { | |
191 | value=UBLOCK_PRIVATE_USE; /* Unicode 3.2 renames this to "Private Use Area" */ | |
192 | } | |
193 | } | |
194 | } | |
195 | if(value<0) { | |
196 | fprintf(stderr, "genprops error: unknown %s name in %s.txt field 1 at %s\n", | |
197 | sen->propName, sen->ucdFile, s); | |
198 | exit(U_PARSE_ERROR); | |
199 | } | |
200 | ||
201 | uv=(uint32_t)(value<<sen->vecShift); | |
202 | if((uv&sen->vecMask)!=uv) { | |
203 | fprintf(stderr, "genprops error: %s value overflow (0x%x) at %s\n", | |
374ca955 | 204 | sen->propName, (int)uv, s); |
b75a7d8f A |
205 | exit(U_INTERNAL_PROGRAM_ERROR); |
206 | } | |
207 | ||
208 | if(!upvec_setValue(pv, start, limit, sen->vecWord, uv, sen->vecMask, pErrorCode)) { | |
209 | fprintf(stderr, "genprops error: unable to set %s code: %s\n", | |
210 | sen->propName, u_errorName(*pErrorCode)); | |
211 | exit(*pErrorCode); | |
212 | } | |
213 | } | |
214 | ||
215 | static void | |
216 | parseSingleEnumFile(char *filename, char *basename, const char *suffix, | |
217 | const SingleEnum *sen, | |
218 | UErrorCode *pErrorCode) { | |
219 | char *fields[2][2]; | |
220 | ||
221 | if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { | |
222 | return; | |
223 | } | |
224 | ||
225 | writeUCDFilename(basename, sen->ucdFile, suffix); | |
226 | ||
227 | u_parseDelimitedFile(filename, ';', fields, 2, singleEnumLineFn, (void *)sen, pErrorCode); | |
228 | if(U_FAILURE(*pErrorCode)) { | |
229 | fprintf(stderr, "error parsing %s.txt: %s\n", sen->ucdFile, u_errorName(*pErrorCode)); | |
230 | } | |
231 | } | |
232 | ||
233 | /* parse files with multiple binary properties ------------------------------ */ | |
234 | ||
235 | struct Binary { | |
236 | const char *propName; | |
237 | int32_t vecWord, vecShift; | |
238 | }; | |
239 | typedef struct Binary Binary; | |
240 | ||
241 | struct Binaries { | |
242 | const char *ucdFile; | |
243 | const Binary *binaries; | |
244 | int32_t binariesCount; | |
245 | }; | |
246 | typedef struct Binaries Binaries; | |
247 | ||
248 | static const Binary | |
249 | propListNames[]={ | |
250 | { "White_Space", 1, UPROPS_WHITE_SPACE }, | |
b75a7d8f A |
251 | { "Dash", 1, UPROPS_DASH }, |
252 | { "Hyphen", 1, UPROPS_HYPHEN }, | |
253 | { "Quotation_Mark", 1, UPROPS_QUOTATION_MARK }, | |
254 | { "Terminal_Punctuation", 1, UPROPS_TERMINAL_PUNCTUATION }, | |
255 | { "Hex_Digit", 1, UPROPS_HEX_DIGIT }, | |
256 | { "ASCII_Hex_Digit", 1, UPROPS_ASCII_HEX_DIGIT }, | |
257 | { "Ideographic", 1, UPROPS_IDEOGRAPHIC }, | |
258 | { "Diacritic", 1, UPROPS_DIACRITIC }, | |
259 | { "Extender", 1, UPROPS_EXTENDER }, | |
260 | { "Noncharacter_Code_Point", 1, UPROPS_NONCHARACTER_CODE_POINT }, | |
261 | { "Grapheme_Link", 1, UPROPS_GRAPHEME_LINK }, | |
262 | { "IDS_Binary_Operator", 1, UPROPS_IDS_BINARY_OPERATOR }, | |
263 | { "IDS_Trinary_Operator", 1, UPROPS_IDS_TRINARY_OPERATOR }, | |
264 | { "Radical", 1, UPROPS_RADICAL }, | |
265 | { "Unified_Ideograph", 1, UPROPS_UNIFIED_IDEOGRAPH }, | |
266 | { "Deprecated", 1, UPROPS_DEPRECATED }, | |
374ca955 A |
267 | { "Logical_Order_Exception", 1, UPROPS_LOGICAL_ORDER_EXCEPTION }, |
268 | ||
269 | /* new properties in Unicode 4.0.1 */ | |
270 | { "STerm", 2, UPROPS_V2_S_TERM }, | |
73c04bcf A |
271 | { "Variation_Selector", 2, UPROPS_V2_VARIATION_SELECTOR }, |
272 | ||
273 | /* new properties in Unicode 4.1 */ | |
274 | { "Pattern_Syntax", 2, UPROPS_V2_PATTERN_SYNTAX }, | |
275 | { "Pattern_White_Space", 2, UPROPS_V2_PATTERN_WHITE_SPACE } | |
b75a7d8f A |
276 | }; |
277 | ||
278 | static const Binaries | |
279 | propListBinaries={ | |
280 | "PropList", propListNames, LENGTHOF(propListNames) | |
281 | }; | |
282 | ||
283 | static const Binary | |
284 | derCorePropsNames[]={ | |
285 | { "XID_Start", 1, UPROPS_XID_START }, | |
286 | { "XID_Continue", 1, UPROPS_XID_CONTINUE }, | |
287 | ||
288 | /* before Unicode 4/ICU 2.6/format version 3.2, these used to be Other_XYZ from PropList.txt */ | |
289 | { "Math", 1, UPROPS_MATH }, | |
290 | { "Alphabetic", 1, UPROPS_ALPHABETIC }, | |
b75a7d8f A |
291 | { "Grapheme_Extend", 1, UPROPS_GRAPHEME_EXTEND }, |
292 | { "Default_Ignorable_Code_Point", 1, UPROPS_DEFAULT_IGNORABLE_CODE_POINT }, | |
293 | ||
294 | /* new properties bits in ICU 2.6/format version 3.2 */ | |
295 | { "ID_Start", 1, UPROPS_ID_START }, | |
296 | { "ID_Continue", 1, UPROPS_ID_CONTINUE }, | |
73c04bcf A |
297 | { "Grapheme_Base", 1, UPROPS_GRAPHEME_BASE }, |
298 | ||
299 | /* | |
300 | * Unicode 5/ICU 3.6 moves Grapheme_Link from PropList.txt | |
301 | * to DerivedCoreProperties.txt and deprecates it. | |
302 | */ | |
303 | { "Grapheme_Link", 1, UPROPS_GRAPHEME_LINK } | |
b75a7d8f A |
304 | }; |
305 | ||
306 | static const Binaries | |
307 | derCorePropsBinaries={ | |
308 | "DerivedCoreProperties", derCorePropsNames, LENGTHOF(derCorePropsNames) | |
309 | }; | |
310 | ||
311 | static char ignoredProps[100][64]; | |
312 | static int32_t ignoredPropsCount; | |
313 | ||
314 | static void | |
315 | addIgnoredProp(char *s, char *limit) { | |
316 | int32_t i; | |
317 | ||
318 | s=trimTerminateField(s, limit); | |
319 | for(i=0; i<ignoredPropsCount; ++i) { | |
320 | if(0==uprv_strcmp(ignoredProps[i], s)) { | |
321 | return; | |
322 | } | |
323 | } | |
324 | uprv_strcpy(ignoredProps[ignoredPropsCount++], s); | |
325 | } | |
326 | ||
327 | static void U_CALLCONV | |
328 | binariesLineFn(void *context, | |
329 | char *fields[][2], int32_t fieldCount, | |
330 | UErrorCode *pErrorCode) { | |
331 | const Binaries *bin; | |
332 | char *s; | |
333 | uint32_t start, limit, uv; | |
334 | int32_t i; | |
335 | ||
336 | bin=(const Binaries *)context; | |
337 | ||
338 | u_parseCodePointRange(fields[0][0], &start, &limit, pErrorCode); | |
339 | if(U_FAILURE(*pErrorCode)) { | |
340 | fprintf(stderr, "genprops: syntax error in %s.txt field 0 at %s\n", bin->ucdFile, fields[0][0]); | |
341 | exit(*pErrorCode); | |
342 | } | |
343 | ++limit; | |
344 | ||
345 | /* parse binary property name */ | |
346 | s=(char *)u_skipWhitespace(fields[1][0]); | |
347 | for(i=0;; ++i) { | |
348 | if(i==bin->binariesCount) { | |
349 | /* ignore unrecognized properties */ | |
73c04bcf A |
350 | if(beVerbose) { |
351 | addIgnoredProp(s, fields[1][1]); | |
352 | } | |
b75a7d8f A |
353 | return; |
354 | } | |
355 | if(isToken(bin->binaries[i].propName, s)) { | |
356 | break; | |
357 | } | |
358 | } | |
359 | ||
360 | if(bin->binaries[i].vecShift>=32) { | |
361 | fprintf(stderr, "genprops error: shift value %d>=32 for %s %s\n", | |
374ca955 | 362 | (int)bin->binaries[i].vecShift, bin->ucdFile, bin->binaries[i].propName); |
b75a7d8f A |
363 | exit(U_INTERNAL_PROGRAM_ERROR); |
364 | } | |
365 | uv=U_MASK(bin->binaries[i].vecShift); | |
366 | ||
367 | if(!upvec_setValue(pv, start, limit, bin->binaries[i].vecWord, uv, uv, pErrorCode)) { | |
368 | fprintf(stderr, "genprops error: unable to set %s code: %s\n", | |
369 | bin->binaries[i].propName, u_errorName(*pErrorCode)); | |
370 | exit(*pErrorCode); | |
371 | } | |
372 | } | |
373 | ||
374 | static void | |
375 | parseBinariesFile(char *filename, char *basename, const char *suffix, | |
376 | const Binaries *bin, | |
377 | UErrorCode *pErrorCode) { | |
378 | char *fields[2][2]; | |
379 | int32_t i; | |
380 | ||
381 | if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { | |
382 | return; | |
383 | } | |
384 | ||
385 | writeUCDFilename(basename, bin->ucdFile, suffix); | |
386 | ||
387 | ignoredPropsCount=0; | |
388 | ||
389 | u_parseDelimitedFile(filename, ';', fields, 2, binariesLineFn, (void *)bin, pErrorCode); | |
390 | if(U_FAILURE(*pErrorCode)) { | |
391 | fprintf(stderr, "error parsing %s.txt: %s\n", bin->ucdFile, u_errorName(*pErrorCode)); | |
392 | } | |
393 | ||
73c04bcf A |
394 | if(beVerbose) { |
395 | for(i=0; i<ignoredPropsCount; ++i) { | |
396 | printf("genprops: ignoring property %s in %s.txt\n", ignoredProps[i], bin->ucdFile); | |
397 | } | |
b75a7d8f A |
398 | } |
399 | } | |
400 | ||
401 | /* -------------------------------------------------------------------------- */ | |
402 | ||
403 | U_CFUNC void | |
404 | initAdditionalProperties() { | |
405 | pv=upvec_open(UPROPS_VECTOR_WORDS, 20000); | |
406 | } | |
407 | ||
73c04bcf A |
408 | U_CFUNC void |
409 | exitAdditionalProperties() { | |
410 | utrie_close(trie); | |
411 | upvec_close(pv); | |
412 | } | |
413 | ||
b75a7d8f A |
414 | U_CFUNC void |
415 | generateAdditionalProperties(char *filename, const char *suffix, UErrorCode *pErrorCode) { | |
416 | char *basename; | |
417 | ||
418 | basename=filename+uprv_strlen(filename); | |
419 | ||
420 | /* process various UCD .txt files */ | |
421 | ||
422 | /* add Han numeric types & values */ | |
374ca955 | 423 | parseMultiFieldFile(filename, basename, "DerivedNumericValues", suffix, 2, numericLineFn, pErrorCode); |
b75a7d8f | 424 | |
b75a7d8f A |
425 | parseTwoFieldFile(filename, basename, "DerivedAge", suffix, ageLineFn, pErrorCode); |
426 | ||
427 | /* | |
428 | * UTR 24 says: | |
429 | * Section 2: | |
430 | * "Common - For characters that may be used | |
431 | * within multiple scripts, | |
432 | * or any unassigned code points." | |
433 | * | |
434 | * Section 4: | |
435 | * "The value COMMON is the default value, | |
436 | * given to all code points that are not | |
437 | * explicitly mentioned in the data file." | |
438 | * | |
439 | * COMMON==USCRIPT_COMMON==0 - nothing to do | |
440 | */ | |
441 | parseSingleEnumFile(filename, basename, suffix, &scriptSingleEnum, pErrorCode); | |
442 | ||
443 | parseSingleEnumFile(filename, basename, suffix, &blockSingleEnum, pErrorCode); | |
444 | ||
445 | parseBinariesFile(filename, basename, suffix, &propListBinaries, pErrorCode); | |
446 | ||
447 | parseBinariesFile(filename, basename, suffix, &derCorePropsBinaries, pErrorCode); | |
448 | ||
73c04bcf A |
449 | parseSingleEnumFile(filename, basename, suffix, &graphemeClusterBreakSingleEnum, pErrorCode); |
450 | ||
451 | parseSingleEnumFile(filename, basename, suffix, &wordBreakSingleEnum, pErrorCode); | |
452 | ||
453 | parseSingleEnumFile(filename, basename, suffix, &sentenceBreakSingleEnum, pErrorCode); | |
454 | ||
b75a7d8f A |
455 | /* |
456 | * LineBreak-4.0.0.txt: | |
457 | * - All code points, assigned and unassigned, that are not listed | |
458 | * explicitly are given the value "XX". | |
459 | * | |
460 | * XX==U_LB_UNKNOWN==0 - nothing to do | |
461 | */ | |
462 | parseSingleEnumFile(filename, basename, suffix, &lineBreakSingleEnum, pErrorCode); | |
463 | ||
b75a7d8f A |
464 | /* |
465 | * Preset East Asian Width defaults: | |
466 | * | |
467 | * http://www.unicode.org/reports/tr11/#Unassigned | |
468 | * 7.1 Unassigned and Private Use characters | |
469 | * | |
470 | * All unassigned characters are by default classified as non-East Asian neutral, | |
471 | * except for the range U+20000 to U+2FFFD, | |
472 | * since all code positions from U+20000 to U+2FFFD are intended for CJK ideographs (W). | |
473 | * All Private use characters are by default classified as ambiguous, | |
474 | * since their definition depends on context. | |
475 | * | |
476 | * N for all ==0 - nothing to do | |
477 | * A for Private Use | |
478 | * W for plane 2 | |
479 | */ | |
480 | *pErrorCode=U_ZERO_ERROR; | |
481 | if( !upvec_setValue(pv, 0xe000, 0xf900, 0, (uint32_t)(U_EA_AMBIGUOUS<<UPROPS_EA_SHIFT), UPROPS_EA_MASK, pErrorCode) || | |
482 | !upvec_setValue(pv, 0xf0000, 0xffffe, 0, (uint32_t)(U_EA_AMBIGUOUS<<UPROPS_EA_SHIFT), UPROPS_EA_MASK, pErrorCode) || | |
483 | !upvec_setValue(pv, 0x100000, 0x10fffe, 0, (uint32_t)(U_EA_AMBIGUOUS<<UPROPS_EA_SHIFT), UPROPS_EA_MASK, pErrorCode) || | |
484 | !upvec_setValue(pv, 0x20000, 0x2fffe, 0, (uint32_t)(U_EA_WIDE<<UPROPS_EA_SHIFT), UPROPS_EA_MASK, pErrorCode) | |
485 | ) { | |
486 | fprintf(stderr, "genprops: unable to set default East Asian Widths: %s\n", u_errorName(*pErrorCode)); | |
487 | exit(*pErrorCode); | |
488 | } | |
489 | ||
490 | /* parse EastAsianWidth.txt */ | |
491 | parseSingleEnumFile(filename, basename, suffix, &eawSingleEnum, pErrorCode); | |
492 | ||
374ca955 | 493 | trie=utrie_open(NULL, NULL, 50000, 0, 0, TRUE); |
b75a7d8f A |
494 | if(trie==NULL) { |
495 | *pErrorCode=U_MEMORY_ALLOCATION_ERROR; | |
496 | upvec_close(pv); | |
497 | return; | |
498 | } | |
499 | ||
73c04bcf | 500 | pvCount=upvec_compact(pv, upvec_compactToTrieHandler, trie, pErrorCode); |
b75a7d8f A |
501 | if(U_FAILURE(*pErrorCode)) { |
502 | fprintf(stderr, "genprops error: unable to build trie for additional properties: %s\n", u_errorName(*pErrorCode)); | |
503 | exit(*pErrorCode); | |
504 | } | |
505 | } | |
506 | ||
507 | /* DerivedAge.txt ----------------------------------------------------------- */ | |
508 | ||
509 | static void U_CALLCONV | |
510 | ageLineFn(void *context, | |
511 | char *fields[][2], int32_t fieldCount, | |
512 | UErrorCode *pErrorCode) { | |
513 | char *s, *end; | |
514 | uint32_t value, start, limit, version; | |
515 | ||
516 | u_parseCodePointRange(fields[0][0], &start, &limit, pErrorCode); | |
517 | if(U_FAILURE(*pErrorCode)) { | |
518 | fprintf(stderr, "genprops: syntax error in DerivedAge.txt field 0 at %s\n", fields[0][0]); | |
519 | exit(*pErrorCode); | |
520 | } | |
521 | ++limit; | |
522 | ||
73c04bcf | 523 | /* ignore "unassigned" (the default is already set to 0.0) */ |
b75a7d8f | 524 | s=(char *)u_skipWhitespace(fields[1][0]); |
73c04bcf A |
525 | if(0==uprv_strncmp(s, "unassigned", 10)) { |
526 | return; | |
527 | } | |
528 | ||
529 | /* parse version number */ | |
b75a7d8f A |
530 | value=(uint32_t)uprv_strtoul(s, &end, 10); |
531 | if(s==end || value==0 || value>15 || (*end!='.' && *end!=' ' && *end!='\t' && *end!=0)) { | |
532 | fprintf(stderr, "genprops: syntax error in DerivedAge.txt field 1 at %s\n", fields[1][0]); | |
533 | *pErrorCode=U_PARSE_ERROR; | |
534 | exit(U_PARSE_ERROR); | |
535 | } | |
536 | version=value<<4; | |
537 | ||
538 | /* parse minor version number */ | |
539 | if(*end=='.') { | |
540 | s=(char *)u_skipWhitespace(end+1); | |
541 | value=(uint32_t)uprv_strtoul(s, &end, 10); | |
542 | if(s==end || value>15 || (*end!=' ' && *end!='\t' && *end!=0)) { | |
543 | fprintf(stderr, "genprops: syntax error in DerivedAge.txt field 1 at %s\n", fields[1][0]); | |
544 | *pErrorCode=U_PARSE_ERROR; | |
545 | exit(U_PARSE_ERROR); | |
546 | } | |
547 | version|=value; | |
548 | } | |
549 | ||
550 | if(!upvec_setValue(pv, start, limit, 0, version<<UPROPS_AGE_SHIFT, UPROPS_AGE_MASK, pErrorCode)) { | |
551 | fprintf(stderr, "genprops error: unable to set character age: %s\n", u_errorName(*pErrorCode)); | |
552 | exit(*pErrorCode); | |
553 | } | |
554 | } | |
555 | ||
556 | /* DerivedNumericValues.txt ------------------------------------------------- */ | |
557 | ||
558 | static void U_CALLCONV | |
559 | numericLineFn(void *context, | |
560 | char *fields[][2], int32_t fieldCount, | |
561 | UErrorCode *pErrorCode) { | |
73c04bcf | 562 | Props newProps={ 0 }; |
b75a7d8f A |
563 | char *s, *end; |
564 | uint32_t start, limit, value, oldProps32; | |
374ca955 | 565 | int32_t oldType; |
b75a7d8f A |
566 | char c; |
567 | UBool isFraction; | |
568 | ||
569 | /* get the code point range */ | |
570 | u_parseCodePointRange(fields[0][0], &start, &limit, pErrorCode); | |
571 | if(U_FAILURE(*pErrorCode)) { | |
572 | fprintf(stderr, "genprops: syntax error in DerivedNumericValues.txt field 0 at %s\n", fields[0][0]); | |
573 | exit(*pErrorCode); | |
574 | } | |
575 | ++limit; | |
576 | ||
577 | /* check if the numeric value is a fraction (this code does not handle any) */ | |
578 | isFraction=FALSE; | |
579 | s=uprv_strchr(fields[1][0], '.'); | |
580 | if(s!=NULL) { | |
581 | end=s+1; | |
582 | while('0'<=(c=*end++) && c<='9') { | |
583 | if(c!='0') { | |
584 | isFraction=TRUE; | |
585 | break; | |
586 | } | |
587 | } | |
588 | } | |
589 | ||
590 | if(isFraction) { | |
591 | value=0; | |
592 | } else { | |
593 | /* parse numeric value */ | |
594 | s=(char *)u_skipWhitespace(fields[1][0]); | |
595 | ||
596 | /* try large powers of 10 first, may otherwise overflow strtoul() */ | |
597 | if(0==uprv_strncmp(s, "10000000000", 11)) { | |
598 | /* large powers of 10 are encoded in a special way, see store.c */ | |
73c04bcf A |
599 | uint8_t exp=0; |
600 | ||
b75a7d8f A |
601 | end=s; |
602 | while(*(++end)=='0') { | |
73c04bcf | 603 | ++exp; |
b75a7d8f | 604 | } |
73c04bcf A |
605 | value=1; |
606 | newProps.exponent=exp; | |
b75a7d8f A |
607 | } else { |
608 | /* normal number parsing */ | |
609 | value=(uint32_t)uprv_strtoul(s, &end, 10); | |
610 | } | |
611 | if(end<=s || (*end!='.' && u_skipWhitespace(end)!=fields[1][1]) || value>=0x80000000) { | |
612 | fprintf(stderr, "genprops: syntax error in DerivedNumericValues.txt field 1 at %s\n", fields[0][0]); | |
613 | exit(U_PARSE_ERROR); | |
614 | } | |
615 | } | |
616 | ||
374ca955 A |
617 | /* |
618 | * Unicode 4.0.1 removes the third column that used to list the numeric type. | |
619 | * Assume that either the data is the same as in UnicodeData.txt, | |
620 | * or else that the numeric type is "numeric". | |
621 | * This should work because we only expect to add numeric values for | |
622 | * Han characters; for those, UnicodeData.txt lists only ranges without | |
623 | * specific properties for single characters. | |
624 | */ | |
b75a7d8f | 625 | |
73c04bcf A |
626 | /* set the new numeric type and value */ |
627 | newProps.numericType=(uint8_t)U_NT_NUMERIC; /* assumed numeric type, see Unicode 4.0.1 comment */ | |
628 | newProps.numericValue=(int32_t)value; /* newly parsed numeric value */ | |
629 | /* the exponent may have been set above */ | |
630 | value=makeProps(&newProps); | |
631 | ||
b75a7d8f A |
632 | for(; start<limit; ++start) { |
633 | oldProps32=getProps(start); | |
634 | oldType=(int32_t)GET_NUMERIC_TYPE(oldProps32); | |
73c04bcf A |
635 | |
636 | if(isFraction) { | |
637 | if(oldType!=0) { | |
638 | /* this code point was already listed with its numeric value in UnicodeData.txt */ | |
639 | continue; | |
640 | } else { | |
641 | fprintf(stderr, "genprops: not prepared for new fractions in DerivedNumericValues.txt field 1 at %s\n", fields[1][0]); | |
642 | exit(U_PARSE_ERROR); | |
643 | } | |
b75a7d8f | 644 | } |
b75a7d8f A |
645 | |
646 | /* | |
b75a7d8f A |
647 | * For simplicity, and because we only expect to set numeric values for Han characters, |
648 | * for now we only allow to set these values for Lo characters. | |
649 | */ | |
73c04bcf A |
650 | if(oldType==0 && GET_CATEGORY(oldProps32)!=U_OTHER_LETTER) { |
651 | fprintf(stderr, "genprops error: new numeric value for a character other than Lo in DerivedNumericValues.txt at %s\n", fields[0][0]); | |
b75a7d8f A |
652 | exit(U_PARSE_ERROR); |
653 | } | |
654 | ||
73c04bcf A |
655 | /* verify that we do not change an existing value (fractions were excluded above) */ |
656 | if(oldType!=0) { | |
657 | /* the code point already has a value stored */ | |
658 | if((oldProps32&0xff00)!=(value&0xff00)) { | |
659 | fprintf(stderr, "genprops error: new numeric value differs from old one for U+%04lx\n", (long)start); | |
660 | exit(U_PARSE_ERROR); | |
661 | } | |
662 | /* same value, continue */ | |
663 | } else { | |
664 | /* the code point is getting a new numeric value */ | |
665 | if(beVerbose) { | |
666 | printf("adding U+%04x numeric type %d value 0x%04x from %s\n", (int)start, U_NT_NUMERIC, (int)value, fields[0][0]); | |
667 | } | |
b75a7d8f | 668 | |
73c04bcf | 669 | addProps(start, value|GET_CATEGORY(oldProps32)); |
b75a7d8f | 670 | } |
b75a7d8f A |
671 | } |
672 | } | |
673 | ||
674 | /* data serialization ------------------------------------------------------- */ | |
675 | ||
676 | U_CFUNC int32_t | |
73c04bcf | 677 | writeAdditionalData(FILE *f, uint8_t *p, int32_t capacity, int32_t indexes[UPROPS_INDEX_COUNT]) { |
b75a7d8f A |
678 | int32_t length; |
679 | UErrorCode errorCode; | |
680 | ||
681 | errorCode=U_ZERO_ERROR; | |
73c04bcf | 682 | length=utrie_serialize(trie, p, capacity, NULL, TRUE, &errorCode); |
b75a7d8f A |
683 | if(U_FAILURE(errorCode)) { |
684 | fprintf(stderr, "genprops error: unable to serialize trie for additional properties: %s\n", u_errorName(errorCode)); | |
685 | exit(errorCode); | |
686 | } | |
687 | if(p!=NULL) { | |
b75a7d8f | 688 | if(beVerbose) { |
374ca955 | 689 | printf("size in bytes of additional props trie:%5u\n", (int)length); |
b75a7d8f | 690 | } |
73c04bcf A |
691 | if(f!=NULL) { |
692 | UTrie trie2={ NULL }; | |
693 | utrie_unserialize(&trie2, p, length, &errorCode); | |
694 | if(U_FAILURE(errorCode)) { | |
695 | fprintf( | |
696 | stderr, | |
697 | "genprops error: failed to utrie_unserialize(trie for additional properties) - %s\n", | |
698 | u_errorName(errorCode)); | |
699 | exit(errorCode); | |
700 | } | |
701 | usrc_writeUTrieArrays(f, | |
702 | "static const uint16_t propsVectorsTrie_index[%ld]={\n", NULL, | |
703 | &trie2, | |
704 | "\n};\n\n"); | |
705 | usrc_writeUTrieStruct(f, | |
706 | "static const UTrie propsVectorsTrie={\n", | |
707 | &trie2, "propsVectorsTrie_index", NULL, NULL, | |
708 | "};\n\n"); | |
709 | } | |
710 | ||
711 | p+=length; | |
712 | capacity-=length; | |
b75a7d8f A |
713 | |
714 | /* set indexes */ | |
715 | indexes[UPROPS_ADDITIONAL_VECTORS_INDEX]= | |
716 | indexes[UPROPS_ADDITIONAL_TRIE_INDEX]+length/4; | |
717 | indexes[UPROPS_ADDITIONAL_VECTORS_COLUMNS_INDEX]=UPROPS_VECTOR_WORDS; | |
718 | indexes[UPROPS_RESERVED_INDEX]= | |
719 | indexes[UPROPS_ADDITIONAL_VECTORS_INDEX]+pvCount; | |
720 | ||
721 | indexes[UPROPS_MAX_VALUES_INDEX]= | |
722 | (((int32_t)U_LB_COUNT-1)<<UPROPS_LB_SHIFT)| | |
723 | (((int32_t)U_EA_COUNT-1)<<UPROPS_EA_SHIFT)| | |
724 | (((int32_t)UBLOCK_COUNT-1)<<UPROPS_BLOCK_SHIFT)| | |
725 | ((int32_t)USCRIPT_CODE_LIMIT-1); | |
726 | indexes[UPROPS_MAX_VALUES_2_INDEX]= | |
73c04bcf A |
727 | (((int32_t)U_SB_COUNT-1)<<UPROPS_SB_SHIFT)| |
728 | (((int32_t)U_WB_COUNT-1)<<UPROPS_WB_SHIFT)| | |
729 | (((int32_t)U_GCB_COUNT-1)<<UPROPS_GCB_SHIFT)| | |
b75a7d8f A |
730 | ((int32_t)U_DT_COUNT-1); |
731 | } | |
732 | ||
733 | if(p!=NULL && (pvCount*4)<=capacity) { | |
73c04bcf A |
734 | if(f!=NULL) { |
735 | usrc_writeArray(f, | |
736 | "static const uint32_t propsVectors[%ld]={\n", | |
737 | pv, 32, pvCount, | |
738 | "};\n\n"); | |
739 | fprintf(f, "static const int32_t countPropsVectors=%ld;\n", (long)pvCount); | |
740 | fprintf(f, "static const int32_t propsVectorsColumns=%ld;\n", (long)indexes[UPROPS_ADDITIONAL_VECTORS_COLUMNS_INDEX]); | |
741 | } else { | |
742 | uprv_memcpy(p, pv, pvCount*4); | |
743 | } | |
b75a7d8f | 744 | if(beVerbose) { |
374ca955 | 745 | printf("number of additional props vectors: %5u\n", (int)pvCount/UPROPS_VECTOR_WORDS); |
b75a7d8f A |
746 | printf("number of 32-bit words per vector: %5u\n", UPROPS_VECTOR_WORDS); |
747 | } | |
748 | } | |
749 | length+=pvCount*4; | |
750 | ||
b75a7d8f A |
751 | return length; |
752 | } |