]> git.saurik.com Git - apple/icu.git/blob - icuSources/tools/genprops/store.c
ICU-6.2.8.tar.gz
[apple/icu.git] / icuSources / tools / genprops / store.c
1 /*
2 *******************************************************************************
3 *
4 * Copyright (C) 1999-2004, International Business Machines
5 * Corporation and others. All Rights Reserved.
6 *
7 *******************************************************************************
8 * file name: store.c
9 * encoding: US-ASCII
10 * tab size: 8 (not used)
11 * indentation:4
12 *
13 * created on: 1999dec11
14 * created by: Markus W. Scherer
15 *
16 * Store Unicode character properties efficiently for
17 * random access.
18 */
19
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include "unicode/utypes.h"
23 #include "unicode/uchar.h"
24 #include "cmemory.h"
25 #include "cstring.h"
26 #include "filestrm.h"
27 #include "utrie.h"
28 #include "unicode/udata.h"
29 #include "unewdata.h"
30 #include "uprops.h"
31 #include "genprops.h"
32
33 #define DO_DEBUG_OUT 0
34
35 /* Unicode character properties file format ------------------------------------
36
37 The file format prepared and written here contains several data
38 structures that store indexes or data.
39
40 Before the data contents described below, there are the headers required by
41 the udata API for loading ICU data. Especially, a UDataInfo structure
42 precedes the actual data. It contains platform properties values and the
43 file format version.
44
45 The following is a description of format version 3 .
46
47 Data contents:
48
49 The contents is a parsed, binary form of several Unicode character
50 database files, most prominently UnicodeData.txt.
51
52 Any Unicode code point from 0 to 0x10ffff can be looked up to get
53 the properties, if any, for that code point. This means that the input
54 to the lookup are 21-bit unsigned integers, with not all of the
55 21-bit range used.
56
57 It is assumed that client code keeps a uint32_t pointer
58 to the beginning of the data:
59
60 const uint32_t *p32;
61
62 Formally, the file contains the following structures:
63
64 const int32_t indexes[16] with values i0..i15:
65
66 i0 propsIndex; -- 32-bit unit index to the table of 32-bit properties words
67 i1 exceptionsIndex; -- 32-bit unit index to the table of 32-bit exception words
68 i2 exceptionsTopIndex; -- 32-bit unit index to the array of UChars for special mappings
69
70 i3 additionalTrieIndex; -- 32-bit unit index to the additional trie for more properties
71 i4 additionalVectorsIndex; -- 32-bit unit index to the table of properties vectors
72 i5 additionalVectorsColumns; -- number of 32-bit words per properties vector
73
74 i6 reservedItemIndex; -- 32-bit unit index to the top of the properties vectors table
75 i7..i9 reservedIndexes; -- reserved values; 0 for now
76
77 i10 maxValues; -- maximum code values for vector word 0, see uprops.h (format version 3.1+)
78 i11 maxValues2; -- maximum code values for vector word 2, see uprops.h (format version 3.2)
79 i12..i15 reservedIndexes; -- reserved values; 0 for now
80
81 PT serialized properties trie, see utrie.h (byte size: 4*(i0-16))
82
83 P const uint32_t props32[i1-i0];
84 E const uint32_t exceptions[i2-i1];
85 U const UChar uchars[2*(i3-i2)];
86
87 AT serialized trie for additional properties (byte size: 4*(i4-i3))
88 PV const uint32_t propsVectors[(i6-i4)/i5][i5]==uint32_t propsVectors[i6-i4];
89
90 Trie lookup and properties:
91
92 In order to condense the data for the 21-bit code space, several properties of
93 the Unicode code assignment are exploited:
94 - The code space is sparse.
95 - There are several 10k of consecutive codes with the same properties.
96 - Characters and scripts are allocated in groups of 16 code points.
97 - Inside blocks for scripts the properties are often repetitive.
98 - The 21-bit space is not fully used for Unicode.
99
100 The lookup of properties for a given code point is done with a trie lookup,
101 using the UTrie implementation.
102 The trie lookup result is a 16-bit index in the props32[] table where the
103 actual 32-bit properties word is stored. This is done to save space.
104
105 (There are thousands of 16-bit entries in the trie data table, but
106 only a few hundred unique 32-bit properties words.
107 If the trie data table contained 32-bit words directly, then that would be
108 larger because the length of the table would be the same as now but the
109 width would be 32 bits instead of 16. This saves more than 10kB.)
110
111 With a given Unicode code point
112
113 UChar32 c;
114
115 and 0<=c<0x110000, the lookup is done like this:
116
117 uint16_t i;
118 UTRIE_GET16(c, i);
119 uint32_t props=p32[i];
120
121 For some characters, not all of the properties can be efficiently encoded
122 using 32 bits. For them, the 32-bit word contains an index into the exceptions[]
123 array:
124
125 if(props&EXCEPTION_BIT)) {
126 uint16_t e=(uint16_t)(props>>VALUE_SHIFT);
127 ...
128 }
129
130 The exception values are a variable number of uint32_t starting at
131
132 const uint32_t *pe=p32+exceptionsIndex+e;
133
134 The first uint32_t there contains flags about what values actually follow it.
135 Some of the exception values are UChar32 code points for the case mappings,
136 others are numeric values etc.
137
138 32-bit properties sets:
139
140 Each 32-bit properties word contains:
141
142 0.. 4 general category
143 5 has exception values
144 6..10 BiDi category
145 11 is mirrored
146 12..14 numericType:
147 0 no numeric value
148 1 decimal digit value
149 2 digit value
150 3 numeric value
151 ### TODO: type 4 for Han digits & numbers?!
152 15..19 reserved
153 20..31 value according to bits 0..5:
154 if(has exception) {
155 exception index;
156 } else switch(general category) {
157 case Ll: delta to uppercase; -- same as titlecase
158 case Lu: -delta to lowercase; -- titlecase is same as c
159 case Lt: -delta to lowercase; -- uppercase is same as c
160 default:
161 if(is mirrored) {
162 delta to mirror;
163 } else if(numericType!=0) {
164 numericValue;
165 } else {
166 0;
167 };
168 }
169
170 Exception values:
171
172 In the first uint32_t exception word for a code point,
173 bits
174 31..16 reserved
175 15..0 flags that indicate which values follow:
176
177 bit
178 0 has uppercase mapping
179 1 has lowercase mapping
180 2 has titlecase mapping
181 3 unused
182 4 has numeric value (numerator)
183 if numericValue=0x7fffff00+x then numericValue=10^x
184 5 has denominator value
185 6 has a mirror-image Unicode code point
186 7 has SpecialCasing.txt entries
187 8 has CaseFolding.txt entries
188
189 According to the flags in this word, one or more uint32_t words follow it
190 in the sequence of the bit flags in the flags word; if a flag is not set,
191 then the value is missing or 0:
192
193 For the case mappings and the mirror-image Unicode code point,
194 one uint32_t or UChar32 each is the code point.
195 If the titlecase mapping is missing, then it is the same as the uppercase mapping.
196
197 For the digit values, bits 31..16 contain the decimal digit value, and
198 bits 15..0 contain the digit value. A value of -1 indicates that
199 this value is missing.
200
201 For the numeric/numerator value, an int32_t word contains the value directly,
202 except for when there is no numerator but a denominator, then the numerator
203 is implicitly 1. This means:
204 numerator denominator result
205 none none none
206 x none x
207 none y 1/y
208 x y x/y
209
210 If the numerator value is 0x7fffff00+x then it is replaced with 10^x.
211
212 For the denominator value, a uint32_t word contains the value directly.
213
214 For special casing mappings, the 32-bit exception word contains:
215 31 if set, this character has complex, conditional mappings
216 that are not stored;
217 otherwise, the mappings are stored according to the following bits
218 30..24 number of UChars used for mappings
219 23..16 reserved
220 15.. 0 UChar offset from the beginning of the UChars array where the
221 UChars for the special case mappings are stored in the following format:
222
223 Format of special casing UChars:
224 One UChar value with lengths as follows:
225 14..10 number of UChars for titlecase mapping
226 9.. 5 number of UChars for uppercase mapping
227 4.. 0 number of UChars for lowercase mapping
228
229 Followed by the UChars for lowercase, uppercase, titlecase mappings in this order.
230
231 For case folding mappings, the 32-bit exception word contains:
232 31..24 number of UChars used for the full mapping
233 23..16 reserved
234 15.. 0 UChar offset from the beginning of the UChars array where the
235 UChars for the special case mappings are stored in the following format:
236
237 Format of case folding UChars:
238 Two UChars contain the simple mapping as follows:
239 0, 0 no simple mapping
240 BMP,0 a simple mapping to a BMP code point
241 s1, s2 a simple mapping to a supplementary code point stored as two surrogates
242 This is followed by the UChars for the full case folding mappings.
243
244 Example:
245 U+2160, ROMAN NUMERAL ONE, needs an exception because it has a lowercase
246 mapping and a numeric value.
247 Its exception values would be stored as 3 uint32_t words:
248
249 - flags=0x0a (see above) with combining class 0
250 - lowercase mapping 0x2170
251 - numeric value=1
252
253 --- Additional properties (new in format version 2.1) ---
254
255 The second trie for additional properties (AT) is also a UTrie with 16-bit data.
256 The data words consist of 32-bit unit indexes (not row indexes!) into the
257 table of unique properties vectors (PV).
258 Each vector contains a set of properties.
259 The width of a vector (number of uint32_t per row) may change
260 with the formatVersion, it is stored in i5.
261
262 Current properties: see icu/source/common/uprops.h
263
264 --- Changes in format version 3.1 ---
265
266 See i10 maxValues above, contains only UBLOCK_COUNT and USCRIPT_CODE_LIMIT.
267
268 --- Changes in format version 3.2 ---
269
270 - The tries use linear Latin-1 ranges.
271 - The additional properties bits store full properties XYZ instead
272 of partial Other_XYZ, so that changes in the derivation formulas
273 need not be tracked in runtime library code.
274 - Joining Type and Line Break are also stored completely, so that uprops.c
275 needs no runtime formulas for enumerated properties either.
276 - Store the case-sensitive flag in the main properties word.
277 - i10 also contains U_LB_COUNT and U_EA_COUNT.
278 - i11 contains maxValues2 for vector word 2.
279
280 ----------------------------------------------------------------------------- */
281
282 /* UDataInfo cf. udata.h */
283 static UDataInfo dataInfo={
284 sizeof(UDataInfo),
285 0,
286
287 U_IS_BIG_ENDIAN,
288 U_CHARSET_FAMILY,
289 U_SIZEOF_UCHAR,
290 0,
291
292 { 0x55, 0x50, 0x72, 0x6f }, /* dataFormat="UPro" */
293 { 3, 2, UTRIE_SHIFT, UTRIE_INDEX_SHIFT }, /* formatVersion */
294 { 4, 0, 1, 0 } /* dataVersion */
295 };
296
297 /* definitions of expected data size limits */
298 enum {
299 MAX_PROPS_COUNT=26000,
300 MAX_UCHAR_COUNT=10000
301 };
302
303 static UNewTrie *pTrie=NULL;
304
305 /* props32[] contains unique properties words after compacting the array of properties */
306 static uint32_t props32[MAX_PROPS_COUNT];
307
308 /* context pointer for compareProps() - temporarily holds a pointer to the trie data */
309 static uint32_t *props;
310
311 /* length of props32[] after compaction */
312 static int32_t propsTop;
313
314 /* exceptions values */
315 static uint32_t exceptions[UPROPS_MAX_EXCEPTIONS_COUNT+20];
316 static uint16_t exceptionsTop=0;
317
318 /* Unicode characters, e.g. for special casing or decomposition */
319 static UChar uchars[MAX_UCHAR_COUNT+20];
320 static uint32_t ucharsTop=0;
321
322 /* statistics */
323 static uint16_t exceptionsCount=0;
324
325 /* prototypes --------------------------------------------------------------- */
326
327 static int
328 compareProps(const void *l, const void *r);
329
330 static uint32_t
331 addUChars(const UChar *s, uint32_t length);
332
333 /* -------------------------------------------------------------------------- */
334
335 extern void
336 setUnicodeVersion(const char *v) {
337 UVersionInfo version;
338 u_versionFromString(version, v);
339 uprv_memcpy(dataInfo.dataVersion, version, 4);
340 }
341
342 extern void
343 initStore() {
344 pTrie=utrie_open(NULL, NULL, MAX_PROPS_COUNT, 0, 0, TRUE);
345 if(pTrie==NULL) {
346 fprintf(stderr, "error: unable to create a UNewTrie\n");
347 exit(U_MEMORY_ALLOCATION_ERROR);
348 }
349
350 uprv_memset(props32, 0, sizeof(props32));
351 initAdditionalProperties();
352 }
353
354 /* store a character's properties ------------------------------------------- */
355
356 extern uint32_t
357 makeProps(Props *p) {
358 uint32_t x;
359 int32_t value;
360 uint16_t count;
361 UBool isNumber;
362
363 /*
364 * Simple ideas for reducing the number of bits for one character's
365 * properties:
366 *
367 * Some fields are only used for characters of certain
368 * general categories:
369 * - casing fields for letters and others, not for
370 * numbers & Mn
371 * + uppercase not for uppercase letters
372 * + lowercase not for lowercase letters
373 * + titlecase not for titlecase letters
374 *
375 * * most of the time, uppercase=titlecase
376 * - numeric fields for various digit & other types
377 * - canonical combining classes for non-spacing marks (Mn)
378 * * the above is not always true, for all three cases
379 *
380 * Using the same bits for alternate fields saves some space.
381 *
382 * For the canonical categories, there are only few actually used
383 * most of the time.
384 * They can be stored using 5 bits.
385 *
386 * In the BiDi categories, the 5 explicit codes are only ever
387 * assigned 1:1 to 5 well-known code points. Storing only one
388 * value for all "explicit codes" gets this down to 4 bits.
389 * Client code then needs to check for this special value
390 * and replace it by the real one using a 5-element table.
391 *
392 * The general categories Mn & Me, non-spacing & enclosing marks,
393 * are always NSM, and NSM are always of those categories.
394 *
395 * Digit values can often be derived from the code point value
396 * itself in a simple way.
397 *
398 */
399
400 /* count the case mappings and other values competing for the value bit field */
401 x=0;
402 value=0;
403 count=0;
404 isNumber= (UBool)(genCategoryNames[p->generalCategory][0]=='N');
405
406 if(p->upperCase!=0) {
407 /* verify that no numbers and no Mn have case mappings */
408 if(p->generalCategory==U_LOWERCASE_LETTER) {
409 value=(int32_t)p->code-(int32_t)p->upperCase;
410 } else {
411 x=UPROPS_EXCEPTION_BIT;
412 }
413 ++count;
414 }
415 if(p->lowerCase!=0) {
416 /* verify that no numbers and no Mn have case mappings */
417 if(p->generalCategory==U_UPPERCASE_LETTER || p->generalCategory==U_TITLECASE_LETTER) {
418 value=(int32_t)p->lowerCase-(int32_t)p->code;
419 } else {
420 x=UPROPS_EXCEPTION_BIT;
421 }
422 ++count;
423 }
424 if(p->upperCase!=p->titleCase) {
425 x=UPROPS_EXCEPTION_BIT;
426 ++count;
427 }
428 if(p->numericType!=0) {
429 value=p->numericValue;
430 ++count;
431 }
432 if(p->denominator!=0) {
433 x=UPROPS_EXCEPTION_BIT;
434 ++count;
435 }
436 if(p->isMirrored) {
437 if(p->mirrorMapping!=0) {
438 value=(int32_t)p->mirrorMapping-(int32_t)p->code;
439 }
440 ++count;
441 }
442 if(p->specialCasing!=NULL) {
443 x=UPROPS_EXCEPTION_BIT;
444 ++count;
445 }
446 if(p->caseFolding!=NULL) {
447 x=UPROPS_EXCEPTION_BIT;
448 ++count;
449 }
450
451 /* handle exceptions */
452 if(count>1 || x!=0 || value<UPROPS_MIN_VALUE || UPROPS_MAX_VALUE<value) {
453 /* this code point needs exception values */
454 if(beVerbose) {
455 if(x!=0) {
456 /* do not print - many code points because of SpecialCasing & CaseFolding
457 printf("*** code 0x%06x needs an exception because it is irregular\n", p->code);
458 */
459 } else if(value<UPROPS_MIN_VALUE || UPROPS_MAX_VALUE<value) {
460 printf("*** U+%04x needs an exception because its value is out-of-bounds at %ld (not [%ld..%ld]\n",
461 (int)p->code, (long)value, (long)UPROPS_MIN_VALUE, (long)UPROPS_MAX_VALUE);
462 } else {
463 printf("*** U+%04x needs an exception because it has %u values\n",
464 (int)p->code, count);
465 }
466 }
467
468 ++exceptionsCount;
469 x=UPROPS_EXCEPTION_BIT;
470
471 /* allocate and create exception values */
472 value=exceptionsTop;
473 if(value>=UPROPS_MAX_EXCEPTIONS_COUNT) {
474 fprintf(stderr, "genprops: out of exceptions memory at U+%06x. (%d exceeds allocated space)\n",
475 (int)p->code, (int)value);
476 exit(U_MEMORY_ALLOCATION_ERROR);
477 } else {
478 uint32_t first=0;
479 uint16_t length=1;
480
481 if(p->upperCase!=0) {
482 first|=1;
483 exceptions[value+length++]=p->upperCase;
484 }
485 if(p->lowerCase!=0) {
486 first|=2;
487 exceptions[value+length++]=p->lowerCase;
488 }
489 if(p->upperCase!=p->titleCase) {
490 first|=4;
491 if(p->titleCase!=0) {
492 exceptions[value+length++]=p->titleCase;
493 } else {
494 exceptions[value+length++]=p->code;
495 }
496 }
497 if(p->numericType!=0) {
498 if(p->denominator==0) {
499 first|=0x10;
500 exceptions[value+length++]=(uint32_t)p->numericValue;
501 } else {
502 if(p->numericValue!=1) {
503 first|=0x10;
504 exceptions[value+length++]=(uint32_t)p->numericValue;
505 }
506 first|=0x20;
507 exceptions[value+length++]=p->denominator;
508 }
509 }
510 if(p->isMirrored) {
511 first|=0x40;
512 exceptions[value+length++]=p->mirrorMapping;
513 }
514 if(p->specialCasing!=NULL) {
515 first|=0x80;
516 if(p->specialCasing->isComplex) {
517 /* complex special casing */
518 exceptions[value+length++]=0x80000000;
519 } else {
520 /* unconditional special casing */
521 UChar u[128];
522 uint32_t i;
523 uint16_t j, entry;
524
525 i=1;
526 entry=0;
527 j=p->specialCasing->lowerCase[0];
528 if(j>0) {
529 uprv_memcpy(u+1, p->specialCasing->lowerCase+1, 2*j);
530 i+=j;
531 entry=j;
532 }
533 j=p->specialCasing->upperCase[0];
534 if(j>0) {
535 uprv_memcpy(u+i, p->specialCasing->upperCase+1, 2*j);
536 i+=j;
537 entry|=j<<5;
538 }
539 j=p->specialCasing->titleCase[0];
540 if(j>0) {
541 uprv_memcpy(u+i, p->specialCasing->titleCase+1, 2*j);
542 i+=j;
543 entry|=j<<10;
544 }
545 u[0]=entry;
546
547 exceptions[value+length++]=(i<<24)|addUChars(u, i);
548 }
549 }
550 if(p->caseFolding!=NULL) {
551 first|=0x100;
552 if(p->caseFolding->simple==0 && p->caseFolding->full[0]==0) {
553 /* special case folding, store only a marker */
554 exceptions[value+length++]=0;
555 } else {
556 /* normal case folding with a simple and a full mapping */
557 UChar u[128];
558 uint16_t i;
559
560 /* store the simple mapping into the first two UChars */
561 i=0;
562 u[1]=0;
563 UTF_APPEND_CHAR_UNSAFE(u, i, p->caseFolding->simple);
564
565 /* store the full mapping after that */
566 i=p->caseFolding->full[0];
567 if(i>0) {
568 uprv_memcpy(u+2, p->caseFolding->full+1, 2*i);
569 }
570
571 exceptions[value+length++]=(i<<24)|addUChars(u, 2+i);
572 }
573 }
574 exceptions[value]=first;
575 exceptionsTop+=length;
576 }
577 }
578
579 /* put together the 32-bit word of encoded properties */
580 x|=
581 (uint32_t)p->generalCategory |
582 (uint32_t)p->bidi<<UPROPS_BIDI_SHIFT |
583 (uint32_t)p->isMirrored<<UPROPS_MIRROR_SHIFT |
584 (uint32_t)p->numericType<<UPROPS_NUMERIC_TYPE_SHIFT |
585 (uint32_t)value<<UPROPS_VALUE_SHIFT;
586
587 return x;
588
589 /*
590 * "Higher-hanging fruit" (not implemented):
591 *
592 * For some sets of fields, there are fewer sets of values
593 * than the product of the numbers of values per field.
594 * This means that storing one single value for more than
595 * one field and later looking up both field values in a table
596 * saves space.
597 * Examples:
598 * - general category & BiDi
599 *
600 * There are only few common displacements between a code point
601 * and its case mappings. Store deltas. Store codes for few
602 * occuring deltas.
603 */
604 }
605
606 extern void
607 addProps(uint32_t c, uint32_t x) {
608 if(!utrie_set32(pTrie, (UChar32)c, x)) {
609 fprintf(stderr, "error: too many entries for the properties trie\n");
610 exit(U_BUFFER_OVERFLOW_ERROR);
611 }
612 }
613
614 extern void
615 addCaseSensitive(UChar32 first, UChar32 last) {
616 uint32_t x, cs;
617
618 cs=U_MASK(UPROPS_CASE_SENSITIVE_SHIFT);
619 while(first<=last) {
620 x=utrie_get32(pTrie, first, NULL);
621 if(!utrie_set32(pTrie, first, x|cs)) {
622 fprintf(stderr, "error: too many entries for the properties trie\n");
623 exit(U_BUFFER_OVERFLOW_ERROR);
624 }
625 ++first;
626 }
627 }
628
629 extern uint32_t
630 getProps(uint32_t c) {
631 return utrie_get32(pTrie, (UChar32)c, NULL);
632 }
633
634 /* areas of same properties ------------------------------------------------- */
635
636 extern void
637 repeatProps(uint32_t first, uint32_t last, uint32_t x) {
638 if(!utrie_setRange32(pTrie, (UChar32)first, (UChar32)(last+1), x, FALSE)) {
639 fprintf(stderr, "error: too many entries for the properties trie\n");
640 exit(U_BUFFER_OVERFLOW_ERROR);
641 }
642 }
643
644 /* compacting --------------------------------------------------------------- */
645
646 static void
647 compactProps(void) {
648 /*
649 * At this point, all the propsTop properties are in props[], but they
650 * are not all unique.
651 * Now we sort them, reduce them to unique ones in props32[], and
652 * build an index in stage3[] from the old to the new indexes.
653 * (The quick sort averages at N*log(N) with N=propsTop. The inverting
654 * yields linear performance.)
655 */
656
657 /*
658 * We are going to sort only an index table in map[] because we need this
659 * index table anyway and qsort() does not allow to sort two tables together
660 * directly. This will thus also reduce the amount of data moved around.
661 */
662 uint32_t x;
663 int32_t i, oldIndex, newIndex;
664
665 static uint16_t map[MAX_PROPS_COUNT];
666
667 #if DO_DEBUG_OUT
668 {
669 /* debug output */
670 uint16_t i1, i2, i3;
671 uint32_t c;
672 for(c=0; c<0xffff; c+=307) {
673 printf("properties(0x%06x)=0x%06x\n", c, getProps(c, &i1, &i2, &i3));
674 }
675 }
676 #endif
677
678 props=utrie_getData(pTrie, &propsTop);
679
680 /* build the index table */
681 for(i=propsTop; i>0;) {
682 --i;
683 map[i]=(uint16_t)i;
684 }
685
686 /* reorder */
687 qsort(map, propsTop, 2, compareProps);
688
689 /*
690 * Now invert the reordered table and compact it in the same step.
691 * The result will be props32[] having only unique properties words
692 * and stage3[] having indexes to them.
693 */
694 newIndex=0;
695 for(i=0; i<propsTop;) {
696 /* set the first of a possible series of the same properties */
697 oldIndex=map[i];
698 props32[newIndex]=x=props[oldIndex];
699 props[oldIndex]=newIndex;
700
701 /* set the following same properties only in stage3 */
702 while(++i<propsTop && x==props[map[i]]) {
703 props[map[i]]=newIndex;
704 }
705
706 ++newIndex;
707 }
708
709 /* we saved some space */
710 if(beVerbose) {
711 printf("compactProps() reduced propsTop from %u to %u\n",
712 (int)propsTop, (int)newIndex);
713 }
714 propsTop=newIndex;
715
716 #if DO_DEBUG_OUT
717 {
718 /* debug output */
719 uint16_t i1, i2, i3, i4;
720 uint32_t c;
721 for(c=0; c<0xffff; c+=307) {
722 printf("properties(0x%06x)=0x%06x\n", c, getProps2(c, &i1, &i2, &i3, &i4));
723 }
724 }
725 #endif
726 }
727
728 static int
729 compareProps(const void *l, const void *r) {
730 uint32_t left=props[*(const uint16_t *)l], right=props[*(const uint16_t *)r];
731
732 /* compare general categories first */
733 int rc=(int)(left&0x1f)-(int)(right&0x1f);
734 if(rc==0 && left!=right) {
735 rc= left<right ? -1 : 1;
736 }
737 return rc;
738 }
739
740 /* generate output data ----------------------------------------------------- */
741
742 /* folding value: just store the offset (16 bits) if there is any non-0 entry */
743 U_CFUNC uint32_t U_EXPORT2
744 getFoldedPropsValue(UNewTrie *trie, UChar32 start, int32_t offset) {
745 uint32_t value;
746 UChar32 limit;
747 UBool inBlockZero;
748
749 limit=start+0x400;
750 while(start<limit) {
751 value=utrie_get32(trie, start, &inBlockZero);
752 if(inBlockZero) {
753 start+=UTRIE_DATA_BLOCK_LENGTH;
754 } else if(value!=0) {
755 return (uint32_t)(offset|0x8000);
756 } else {
757 ++start;
758 }
759 }
760 return 0;
761 }
762
763 extern void
764 generateData(const char *dataDir) {
765 static int32_t indexes[UPROPS_INDEX_COUNT]={
766 0, 0, 0, 0,
767 0, 0, 0, 0,
768 0, 0, 0, 0,
769 0, 0, 0, 0
770 };
771 static uint8_t trieBlock[40000];
772 static uint8_t additionalProps[120000];
773
774 UNewDataMemory *pData;
775 UErrorCode errorCode=U_ZERO_ERROR;
776 uint32_t size;
777 int32_t trieSize, additionalPropsSize, offset;
778 long dataLength;
779
780 compactProps();
781
782 trieSize=utrie_serialize(pTrie, trieBlock, sizeof(trieBlock), getFoldedPropsValue, TRUE, &errorCode);
783 if(U_FAILURE(errorCode)) {
784 fprintf(stderr, "error: utrie_serialize failed: %s (length %ld)\n", u_errorName(errorCode), (long)trieSize);
785 exit(errorCode);
786 }
787
788 offset=sizeof(indexes)/4; /* uint32_t offset to the properties trie */
789
790 /* round up trie size to 4-alignement */
791 trieSize=(trieSize+3)&~3;
792 offset+=trieSize>>2;
793 indexes[UPROPS_PROPS32_INDEX]=offset; /* uint32_t offset to props[] */
794
795 offset+=propsTop;
796 indexes[UPROPS_EXCEPTIONS_INDEX]=offset;/* uint32_t offset to exceptions[] */
797
798 offset+=exceptionsTop; /* uint32_t offset to the first unit after exceptions[] */
799 indexes[UPROPS_EXCEPTIONS_TOP_INDEX]=offset;
800
801 /* round up UChar count to 4-alignement */
802 ucharsTop=(ucharsTop+1)&~1;
803 offset+=(uint16_t)(ucharsTop/2); /* uint32_t offset to the first unit after uchars[] */
804 indexes[UPROPS_ADDITIONAL_TRIE_INDEX]=offset;
805
806 if(beVerbose) {
807 printf("trie size in bytes: %5u\n", (int)trieSize);
808 printf("number of unique properties values: %5u\n", (int)propsTop);
809 printf("number of code points with exceptions: %5u\n", exceptionsCount);
810 printf("size in bytes of exceptions: %5u\n", 4*exceptionsTop);
811 printf("number of UChars for special mappings: %5u\n", (int)ucharsTop);
812 }
813
814 additionalPropsSize=writeAdditionalData(additionalProps, sizeof(additionalProps), indexes);
815
816 size=4*offset+additionalPropsSize; /* total size of data */
817 if(beVerbose) {
818 printf("data size: %6lu\n", (unsigned long)size);
819 }
820
821 /* write the data */
822 pData=udata_create(dataDir, DATA_TYPE, DATA_NAME, &dataInfo,
823 haveCopyright ? U_COPYRIGHT_STRING : NULL, &errorCode);
824 if(U_FAILURE(errorCode)) {
825 fprintf(stderr, "genprops: unable to create data memory, %s\n", u_errorName(errorCode));
826 exit(errorCode);
827 }
828
829 udata_writeBlock(pData, indexes, sizeof(indexes));
830 udata_writeBlock(pData, trieBlock, trieSize);
831 udata_writeBlock(pData, props32, 4*propsTop);
832 udata_writeBlock(pData, exceptions, 4*exceptionsTop);
833 udata_writeBlock(pData, uchars, 2*ucharsTop);
834 udata_writeBlock(pData, additionalProps, additionalPropsSize);
835
836 /* finish up */
837 dataLength=udata_finish(pData, &errorCode);
838 if(U_FAILURE(errorCode)) {
839 fprintf(stderr, "genprops: error %d writing the output file\n", errorCode);
840 exit(errorCode);
841 }
842
843 if(dataLength!=(long)size) {
844 fprintf(stderr, "genprops: data length %ld != calculated size %lu\n",
845 dataLength, (unsigned long)size);
846 exit(U_INTERNAL_PROGRAM_ERROR);
847 }
848
849 utrie_close(pTrie);
850 }
851
852 /* helpers ------------------------------------------------------------------ */
853
854 static uint32_t
855 addUChars(const UChar *s, uint32_t length) {
856 uint32_t top=(uint16_t)(ucharsTop+length);
857 UChar *p;
858
859 if(top>=MAX_UCHAR_COUNT) {
860 fprintf(stderr, "genprops: out of UChars memory\n");
861 exit(U_MEMORY_ALLOCATION_ERROR);
862 }
863 p=uchars+ucharsTop;
864 uprv_memcpy(p, s, 2*length);
865 ucharsTop=top;
866 return (uint32_t)(p-uchars);
867 }
868
869 /*
870 * Hey, Emacs, please set the following:
871 *
872 * Local Variables:
873 * indent-tabs-mode: nil
874 * End:
875 *
876 */