2 *******************************************************************************
4 * Copyright (C) 2003-2006, International Business Machines
5 * Corporation and others. All Rights Reserved.
7 *******************************************************************************
8 * file name: gencnvex.c
10 * tab size: 8 (not used)
13 * created on: 2003oct12
14 * created by: Markus W. Scherer
18 #include "unicode/utypes.h"
19 #include "unicode/ustring.h"
30 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
34 CnvExtClose(NewConverter
*cnvData
);
37 CnvExtIsValid(NewConverter
*cnvData
,
38 const uint8_t *bytes
, int32_t length
);
41 CnvExtAddTable(NewConverter
*cnvData
, UCMTable
*table
, UConverterStaticData
*staticData
);
44 CnvExtWrite(NewConverter
*cnvData
, const UConverterStaticData
*staticData
,
45 UNewDataMemory
*pData
, int32_t tableType
);
47 typedef struct CnvExtData
{
48 NewConverter newConverter
;
52 /* toUnicode (state table in ucm->states) */
53 UToolMemory
*toUTable
, *toUUChars
;
56 UToolMemory
*fromUTableUChars
, *fromUTableValues
, *fromUBytes
;
58 uint16_t stage1
[MBCS_STAGE_1_SIZE
];
59 uint16_t stage2
[MBCS_STAGE_2_SIZE
];
60 uint16_t stage3
[0x10000<<UCNV_EXT_STAGE_2_LEFT_SHIFT
]; /* 0x10000 because of 16-bit stage 2/3 indexes */
61 uint32_t stage3b
[0x10000];
63 int32_t stage1Top
, stage2Top
, stage3Top
, stage3bTop
;
65 /* for stage3 compaction of <subchar1> |2 mappings */
66 uint16_t stage3Sub1Block
;
70 maxInBytes
, maxOutBytes
, maxBytesPerUChar
,
71 maxInUChars
, maxOutUChars
, maxUCharsPerByte
;
75 CnvExtOpen(UCMFile
*ucm
) {
78 extData
=(CnvExtData
*)uprv_malloc(sizeof(CnvExtData
));
80 uprv_memset(extData
, 0, sizeof(CnvExtData
));
82 extData
->ucm
=ucm
; /* aliased, not owned */
84 extData
->newConverter
.close
=CnvExtClose
;
85 extData
->newConverter
.isValid
=CnvExtIsValid
;
86 extData
->newConverter
.addTable
=CnvExtAddTable
;
87 extData
->newConverter
.write
=CnvExtWrite
;
89 return &extData
->newConverter
;
93 CnvExtClose(NewConverter
*cnvData
) {
94 CnvExtData
*extData
=(CnvExtData
*)cnvData
;
96 utm_close(extData
->toUTable
);
97 utm_close(extData
->toUUChars
);
98 utm_close(extData
->fromUTableUChars
);
99 utm_close(extData
->fromUTableValues
);
100 utm_close(extData
->fromUBytes
);
104 /* we do not expect this to be called */
106 CnvExtIsValid(NewConverter
*cnvData
,
107 const uint8_t *bytes
, int32_t length
) {
112 CnvExtWrite(NewConverter
*cnvData
, const UConverterStaticData
*staticData
,
113 UNewDataMemory
*pData
, int32_t tableType
) {
114 CnvExtData
*extData
=(CnvExtData
*)cnvData
;
115 int32_t length
, top
, headerSize
;
117 int32_t indexes
[UCNV_EXT_INDEXES_MIN_LENGTH
]={ 0 };
119 if(tableType
&TABLE_BASE
) {
122 _MBCSHeader header
={ { 0, 0, 0, 0 }, 0, 0, 0, 0, 0, 0, 0 };
124 /* write the header and base table name for an extension-only table */
125 length
=(int32_t)uprv_strlen(extData
->ucm
->baseName
)+1;
128 extData
->ucm
->baseName
[length
++]=0;
131 headerSize
=sizeof(header
)+length
;
133 /* fill the header */
136 header
.flags
=(uint32_t)((headerSize
<<8)|MBCS_OUTPUT_EXT_ONLY
);
138 /* write the header and the base table name */
139 udata_writeBlock(pData
, &header
, sizeof(header
));
140 udata_writeBlock(pData
, extData
->ucm
->baseName
, length
);
143 /* fill indexes[] - offsets/indexes are in units of the target array */
146 indexes
[UCNV_EXT_INDEXES_LENGTH
]=length
=UCNV_EXT_INDEXES_MIN_LENGTH
;
149 indexes
[UCNV_EXT_TO_U_INDEX
]=top
;
150 indexes
[UCNV_EXT_TO_U_LENGTH
]=length
=utm_countItems(extData
->toUTable
);
153 indexes
[UCNV_EXT_TO_U_UCHARS_INDEX
]=top
;
154 indexes
[UCNV_EXT_TO_U_UCHARS_LENGTH
]=length
=utm_countItems(extData
->toUUChars
);
157 indexes
[UCNV_EXT_FROM_U_UCHARS_INDEX
]=top
;
158 length
=utm_countItems(extData
->fromUTableUChars
);
163 *((UChar
*)utm_alloc(extData
->fromUTableUChars
))=0;
164 *((uint32_t *)utm_alloc(extData
->fromUTableValues
))=0;
168 indexes
[UCNV_EXT_FROM_U_LENGTH
]=length
;
170 indexes
[UCNV_EXT_FROM_U_VALUES_INDEX
]=top
;
173 indexes
[UCNV_EXT_FROM_U_BYTES_INDEX
]=top
;
174 length
=utm_countItems(extData
->fromUBytes
);
179 *((uint8_t *)utm_alloc(extData
->fromUBytes
))=0;
183 indexes
[UCNV_EXT_FROM_U_BYTES_LENGTH
]=length
;
185 indexes
[UCNV_EXT_FROM_U_STAGE_12_INDEX
]=top
;
186 indexes
[UCNV_EXT_FROM_U_STAGE_1_LENGTH
]=length
=extData
->stage1Top
;
187 indexes
[UCNV_EXT_FROM_U_STAGE_12_LENGTH
]=length
+=extData
->stage2Top
;
190 indexes
[UCNV_EXT_FROM_U_STAGE_3_INDEX
]=top
;
191 length
=extData
->stage3Top
;
196 extData
->stage3
[extData
->stage3Top
++]=0;
200 indexes
[UCNV_EXT_FROM_U_STAGE_3_LENGTH
]=length
;
202 indexes
[UCNV_EXT_FROM_U_STAGE_3B_INDEX
]=top
;
203 indexes
[UCNV_EXT_FROM_U_STAGE_3B_LENGTH
]=length
=extData
->stage3bTop
;
206 indexes
[UCNV_EXT_SIZE
]=top
;
209 indexes
[UCNV_EXT_COUNT_BYTES
]=
210 (extData
->maxInBytes
<<16)|
211 (extData
->maxOutBytes
<<8)|
212 extData
->maxBytesPerUChar
;
213 indexes
[UCNV_EXT_COUNT_UCHARS
]=
214 (extData
->maxInUChars
<<16)|
215 (extData
->maxOutUChars
<<8)|
216 extData
->maxUCharsPerByte
;
218 indexes
[UCNV_EXT_FLAGS
]=extData
->ucm
->ext
->unicodeMask
;
220 /* write the extension data */
221 udata_writeBlock(pData
, indexes
, sizeof(indexes
));
222 udata_writeBlock(pData
, utm_getStart(extData
->toUTable
), indexes
[UCNV_EXT_TO_U_LENGTH
]*4);
223 udata_writeBlock(pData
, utm_getStart(extData
->toUUChars
), indexes
[UCNV_EXT_TO_U_UCHARS_LENGTH
]*2);
225 udata_writeBlock(pData
, utm_getStart(extData
->fromUTableUChars
), indexes
[UCNV_EXT_FROM_U_LENGTH
]*2);
226 udata_writeBlock(pData
, utm_getStart(extData
->fromUTableValues
), indexes
[UCNV_EXT_FROM_U_LENGTH
]*4);
227 udata_writeBlock(pData
, utm_getStart(extData
->fromUBytes
), indexes
[UCNV_EXT_FROM_U_BYTES_LENGTH
]);
229 udata_writeBlock(pData
, extData
->stage1
, extData
->stage1Top
*2);
230 udata_writeBlock(pData
, extData
->stage2
, extData
->stage2Top
*2);
231 udata_writeBlock(pData
, extData
->stage3
, extData
->stage3Top
*2);
232 udata_writeBlock(pData
, extData
->stage3b
, extData
->stage3bTop
*4);
238 length
=extData
->stage1Top
;
239 printf("\nstage1[%x]:\n", length
);
241 for(i
=0; i
<length
; ++i
) {
242 if(extData
->stage1
[i
]!=length
) {
243 printf("stage1[%04x]=%04x\n", i
, extData
->stage1
[i
]);
248 length
=extData
->stage2Top
;
249 printf("\nstage2[%x]:\n", length
);
251 for(i
=0; i
<length
; ++j
, ++i
) {
252 if(extData
->stage2
[i
]!=0) {
253 printf("stage12[%04x]=%04x\n", j
, extData
->stage2
[i
]);
257 length
=extData
->stage3Top
;
258 printf("\nstage3[%x]:\n", length
);
260 for(i
=0; i
<length
; ++i
) {
261 if(extData
->stage3
[i
]!=0) {
262 printf("stage3[%04x]=%04x\n", i
, extData
->stage3
[i
]);
266 length
=extData
->stage3bTop
;
267 printf("\nstage3b[%x]:\n", length
);
269 for(i
=0; i
<length
; ++i
) {
270 if(extData
->stage3b
[i
]!=0) {
271 printf("stage3b[%04x]=%08x\n", i
, extData
->stage3b
[i
]);
278 printf("size of extension data: %ld\n", (long)top
);
281 /* return the number of bytes that should have been written */
282 return (uint32_t)(headerSize
+top
);
285 /* to Unicode --------------------------------------------------------------- */
288 * Remove fromUnicode fallbacks and SUB mappings which are irrelevant for
289 * the toUnicode table.
290 * The table must be sorted.
291 * Destroys previous data in the reverseMap.
294 reduceToUMappings(UCMTable
*table
) {
300 mappings
=table
->mappings
;
301 map
=table
->reverseMap
;
302 count
=table
->mappingsLength
;
304 /* leave the map alone for the initial mappings with desired flags */
305 for(i
=j
=0; i
<count
; ++i
) {
306 flag
=mappings
[map
[i
]].f
;
307 if(flag
!=0 && flag
!=3) {
312 /* reduce from here to the rest */
313 for(j
=i
; i
<count
; ++i
) {
314 flag
=mappings
[map
[i
]].f
;
315 if(flag
==0 || flag
==3) {
324 getToUnicodeValue(CnvExtData
*extData
, UCMTable
*table
, UCMapping
*m
) {
328 int32_t u16Length
, ratio
;
329 UErrorCode errorCode
;
331 /* write the Unicode result code point or string index */
333 u16Length
=U16_LENGTH(m
->u
);
334 value
=(uint32_t)(UCNV_EXT_TO_U_MIN_CODE_POINT
+m
->u
);
336 /* the parser enforces m->uLen<=UCNV_EXT_MAX_UCHARS */
338 /* get the result code point string and its 16-bit string length */
339 u32
=UCM_GET_CODE_POINTS(table
, m
);
340 errorCode
=U_ZERO_ERROR
;
341 u_strFromUTF32(NULL
, 0, &u16Length
, u32
, m
->uLen
, &errorCode
);
342 if(U_FAILURE(errorCode
) && errorCode
!=U_BUFFER_OVERFLOW_ERROR
) {
346 /* allocate it and put its length and index into the value */
348 (((uint32_t)m
->uLen
+UCNV_EXT_TO_U_LENGTH_OFFSET
)<<UCNV_EXT_TO_U_LENGTH_SHIFT
)|
349 ((uint32_t)utm_countItems(extData
->toUUChars
));
350 u
=utm_allocN(extData
->toUUChars
, u16Length
);
352 /* write the result 16-bit string */
353 errorCode
=U_ZERO_ERROR
;
354 u_strFromUTF32(u
, u16Length
, NULL
, u32
, m
->uLen
, &errorCode
);
355 if(U_FAILURE(errorCode
) && errorCode
!=U_BUFFER_OVERFLOW_ERROR
) {
360 value
|=UCNV_EXT_TO_U_ROUNDTRIP_FLAG
;
363 /* update statistics */
364 if(m
->bLen
>extData
->maxInBytes
) {
365 extData
->maxInBytes
=m
->bLen
;
367 if(u16Length
>extData
->maxOutUChars
) {
368 extData
->maxOutUChars
=u16Length
;
371 ratio
=(u16Length
+(m
->bLen
-1))/m
->bLen
;
372 if(ratio
>extData
->maxUCharsPerByte
) {
373 extData
->maxUCharsPerByte
=ratio
;
380 * Recursive toUTable generator core function.
382 * - start<limit (There is at least one mapping.)
383 * - The mappings are sorted lexically. (Access is through the reverseMap.)
384 * - All mappings between start and limit have input sequences that share
385 * the same prefix of unitIndex length, and therefore all of these sequences
386 * are at least unitIndex+1 long.
387 * - There are only relevant mappings available through the reverseMap,
388 * see reduceToUMappings().
390 * One function invocation generates one section table.
393 * 1. Count the number of unique unit values and get the low/high unit values
394 * that occur at unitIndex.
395 * 2. Allocate the section table with possible optimization for linear access.
396 * 3. Write temporary version of the section table with start indexes of
397 * subsections, each corresponding to one unit value at unitIndex.
398 * 4. Iterate through the table once more, and depending on the subsection length:
399 * 0: write 0 as a result value (unused byte in linear-access section table)
400 * >0: if there is one mapping with an input unit sequence of unitIndex+1
401 * then defaultValue=compute the mapping result for this whole sequence
402 * else defaultValue=0
404 * recurse into the subsection
407 generateToUTable(CnvExtData
*extData
, UCMTable
*table
,
408 int32_t start
, int32_t limit
, int32_t unitIndex
,
409 uint32_t defaultValue
) {
410 UCMapping
*mappings
, *m
;
412 int32_t i
, j
, uniqueCount
, count
, subStart
, subLimit
;
415 int32_t low
, high
, prev
;
419 mappings
=table
->mappings
;
420 map
=table
->reverseMap
;
422 /* step 1: examine the input units; set low, high, uniqueCount */
423 m
=mappings
+map
[start
];
424 bytes
=UCM_GET_BYTES(table
, m
);
425 low
=bytes
[unitIndex
];
429 for(i
=start
+1; i
<limit
; ++i
) {
431 bytes
=UCM_GET_BYTES(table
, m
);
432 high
=bytes
[unitIndex
];
440 /* step 2: allocate the section; set count, section */
442 if(unitIndex
==0 || uniqueCount
>=(3*count
)/4) {
444 * for the root table and for fairly full tables:
445 * allocate for direct, linear array access
446 * by keeping count, to write an entry for each unit value
453 /* allocate the section: 1 entry for the header + count for the items */
454 section
=(uint32_t *)utm_allocN(extData
->toUTable
, 1+count
);
456 /* write the section header */
457 *section
++=((uint32_t)count
<<UCNV_EXT_TO_U_BYTE_SHIFT
)|defaultValue
;
459 /* step 3: write temporary section table with subsection starts */
460 prev
=low
-1; /* just before low to prevent empty subsections before low */
461 j
=0; /* section table index */
462 for(i
=start
; i
<limit
; ++i
) {
464 bytes
=UCM_GET_BYTES(table
, m
);
465 high
=bytes
[unitIndex
];
468 /* start of a new subsection for unit high */
469 if(count
>uniqueCount
) {
470 /* write empty subsections for unused units in a linear table */
472 section
[j
++]=((uint32_t)prev
<<UCNV_EXT_TO_U_BYTE_SHIFT
)|(uint32_t)i
;
478 /* write the entry with the subsection start */
479 section
[j
++]=((uint32_t)high
<<UCNV_EXT_TO_U_BYTE_SHIFT
)|(uint32_t)i
;
482 /* assert(j==count) */
484 /* step 4: recurse and write results */
485 subLimit
=UCNV_EXT_TO_U_GET_VALUE(section
[0]);
486 for(j
=0; j
<count
; ++j
) {
488 subLimit
= (j
+1)<count
? UCNV_EXT_TO_U_GET_VALUE(section
[j
+1]) : limit
;
490 /* remove the subStart temporary value */
491 section
[j
]&=~UCNV_EXT_TO_U_VALUE_MASK
;
493 if(subStart
==subLimit
) {
494 /* leave the value zero: empty subsection for unused unit in a linear table */
498 /* see if there is exactly one input unit sequence of length unitIndex+1 */
500 m
=mappings
+map
[subStart
];
501 if(m
->bLen
==unitIndex
+1) {
502 /* do not include this in generateToUTable() */
505 if(subStart
<subLimit
&& mappings
[map
[subStart
]].bLen
==unitIndex
+1) {
506 /* print error for multiple same-input-sequence mappings */
507 fprintf(stderr
, "error: multiple mappings from same bytes\n");
508 ucm_printMapping(table
, m
, stderr
);
509 ucm_printMapping(table
, mappings
+map
[subStart
], stderr
);
513 defaultValue
=getToUnicodeValue(extData
, table
, m
);
516 if(subStart
==subLimit
) {
517 /* write the result for the input sequence ending here */
518 section
[j
]|=defaultValue
;
520 /* write the index to the subsection table */
521 section
[j
]|=(uint32_t)utm_countItems(extData
->toUTable
);
524 if(!generateToUTable(extData
, table
, subStart
, subLimit
, unitIndex
+1, defaultValue
)) {
533 * Generate the toUTable and toUUChars from the input table.
534 * The input table must be sorted, and all precision flags must be 0..3.
535 * This function will modify the table's reverseMap.
538 makeToUTable(CnvExtData
*extData
, UCMTable
*table
) {
541 toUCount
=reduceToUMappings(table
);
543 extData
->toUTable
=utm_open("cnv extension toUTable", 0x10000, UCNV_EXT_TO_U_MIN_CODE_POINT
, 4);
544 extData
->toUUChars
=utm_open("cnv extension toUUChars", 0x10000, UCNV_EXT_TO_U_INDEX_MASK
+1, 2);
546 return generateToUTable(extData
, table
, 0, toUCount
, 0, 0);
549 /* from Unicode ------------------------------------------------------------- */
553 * rebuild reverseMap with mapping indexes for mappings relevant for from Unicode
554 * change each Unicode string to encode all but the first code point in 16-bit form
557 * for each unique code point
558 * write an entry in the 3-stage trie
559 * check that there is only one single-code point sequence
560 * start recursion for following 16-bit input units
564 * Remove toUnicode fallbacks and non-<subchar1> SUB mappings
565 * which are irrelevant for the fromUnicode extension table.
566 * Overwrite the reverseMap with an index array to the relevant mappings.
567 * Modify the code point sequences to a generator-friendly format where
568 * the first code points remains unchanged but the following are recoded
569 * into 16-bit Unicode string form.
570 * The table must be sorted.
571 * Destroys previous data in the reverseMap.
574 prepareFromUMappings(UCMTable
*table
) {
575 UCMapping
*mappings
, *m
;
580 mappings
=table
->mappings
;
581 map
=table
->reverseMap
;
582 count
=table
->mappingsLength
;
585 * we do not go through the map on input because the mappings are
590 for(i
=j
=0; i
<count
; ++m
, ++i
) {
592 if(flag
==0 || flag
==1 || (flag
==2 && m
->bLen
==1)) {
596 /* recode all but the first code point to 16-bit Unicode */
602 u32
=UCM_GET_CODE_POINTS(table
, m
);
603 u
=(UChar
*)u32
; /* destructive in-place recoding */
604 for(r
=2, q
=1; q
<m
->uLen
; ++q
) {
606 U16_APPEND_UNSAFE(u
, r
, c
);
609 /* counts the first code point always at 2 - the first 16-bit unit is at 16-bit index 2 */
619 getFromUBytesValue(CnvExtData
*extData
, UCMTable
*table
, UCMapping
*m
) {
620 uint8_t *bytes
, *resultBytes
;
622 int32_t u16Length
, ratio
;
626 * no mapping, <subchar1> preferred
628 * no need to count in statistics because the subchars are already
629 * counted for maxOutBytes and maxBytesPerUChar in UConverterStaticData,
630 * and this non-mapping does not count for maxInUChars which are always
631 * trivially at least two if counting unmappable supplementary code points
633 return UCNV_EXT_FROM_U_SUBCHAR1
;
636 bytes
=UCM_GET_BYTES(table
, m
);
639 /* 1..3: store the bytes in the value word */
641 value
=((uint32_t)*bytes
++)<<16;
643 value
|=((uint32_t)*bytes
++)<<8;
648 /* the parser enforces m->bLen<=UCNV_EXT_MAX_BYTES */
649 /* store the bytes in fromUBytes[] and the index in the value word */
650 value
=(uint32_t)utm_countItems(extData
->fromUBytes
);
651 resultBytes
=utm_allocN(extData
->fromUBytes
, m
->bLen
);
652 uprv_memcpy(resultBytes
, bytes
, m
->bLen
);
655 value
|=(uint32_t)m
->bLen
<<UCNV_EXT_FROM_U_LENGTH_SHIFT
;
657 value
|=UCNV_EXT_FROM_U_ROUNDTRIP_FLAG
;
660 /* calculate the real UTF-16 length (see recoding in prepareFromUMappings()) */
662 u16Length
=U16_LENGTH(m
->u
);
664 u16Length
=U16_LENGTH(UCM_GET_CODE_POINTS(table
, m
)[0])+(m
->uLen
-2);
667 /* update statistics */
668 if(u16Length
>extData
->maxInUChars
) {
669 extData
->maxInUChars
=u16Length
;
671 if(m
->bLen
>extData
->maxOutBytes
) {
672 extData
->maxOutBytes
=m
->bLen
;
675 ratio
=(m
->bLen
+(u16Length
-1))/u16Length
;
676 if(ratio
>extData
->maxBytesPerUChar
) {
677 extData
->maxBytesPerUChar
=ratio
;
684 * works like generateToUTable(), except that the
685 * output section consists of two arrays, one for input UChars and one
688 * also, fromUTable sections are always stored in a compact form for
689 * access via binary search
692 generateFromUTable(CnvExtData
*extData
, UCMTable
*table
,
693 int32_t start
, int32_t limit
, int32_t unitIndex
,
694 uint32_t defaultValue
) {
695 UCMapping
*mappings
, *m
;
697 int32_t i
, j
, uniqueCount
, count
, subStart
, subLimit
;
700 UChar32 low
, high
, prev
;
702 UChar
*sectionUChars
;
703 uint32_t *sectionValues
;
705 mappings
=table
->mappings
;
706 map
=table
->reverseMap
;
708 /* step 1: examine the input units; set low, high, uniqueCount */
709 m
=mappings
+map
[start
];
710 uchars
=(UChar
*)UCM_GET_CODE_POINTS(table
, m
);
711 low
=uchars
[unitIndex
];
715 for(i
=start
+1; i
<limit
; ++i
) {
717 uchars
=(UChar
*)UCM_GET_CODE_POINTS(table
, m
);
718 high
=uchars
[unitIndex
];
726 /* step 2: allocate the section; set count, section */
727 /* the fromUTable always stores for access via binary search */
730 /* allocate the section: 1 entry for the header + count for the items */
731 sectionUChars
=(UChar
*)utm_allocN(extData
->fromUTableUChars
, 1+count
);
732 sectionValues
=(uint32_t *)utm_allocN(extData
->fromUTableValues
, 1+count
);
734 /* write the section header */
735 *sectionUChars
++=(UChar
)count
;
736 *sectionValues
++=defaultValue
;
738 /* step 3: write temporary section table with subsection starts */
739 prev
=low
-1; /* just before low to prevent empty subsections before low */
740 j
=0; /* section table index */
741 for(i
=start
; i
<limit
; ++i
) {
743 uchars
=(UChar
*)UCM_GET_CODE_POINTS(table
, m
);
744 high
=uchars
[unitIndex
];
747 /* start of a new subsection for unit high */
750 /* write the entry with the subsection start */
751 sectionUChars
[j
]=(UChar
)high
;
752 sectionValues
[j
]=(uint32_t)i
;
756 /* assert(j==count) */
758 /* step 4: recurse and write results */
759 subLimit
=(int32_t)(sectionValues
[0]);
760 for(j
=0; j
<count
; ++j
) {
762 subLimit
= (j
+1)<count
? (int32_t)(sectionValues
[j
+1]) : limit
;
764 /* see if there is exactly one input unit sequence of length unitIndex+1 */
766 m
=mappings
+map
[subStart
];
767 if(m
->uLen
==unitIndex
+1) {
768 /* do not include this in generateToUTable() */
771 if(subStart
<subLimit
&& mappings
[map
[subStart
]].uLen
==unitIndex
+1) {
772 /* print error for multiple same-input-sequence mappings */
773 fprintf(stderr
, "error: multiple mappings from same Unicode code points\n");
774 ucm_printMapping(table
, m
, stderr
);
775 ucm_printMapping(table
, mappings
+map
[subStart
], stderr
);
779 defaultValue
=getFromUBytesValue(extData
, table
, m
);
782 if(subStart
==subLimit
) {
783 /* write the result for the input sequence ending here */
784 sectionValues
[j
]=defaultValue
;
786 /* write the index to the subsection table */
787 sectionValues
[j
]=(uint32_t)utm_countItems(extData
->fromUTableValues
);
790 if(!generateFromUTable(extData
, table
, subStart
, subLimit
, unitIndex
+1, defaultValue
)) {
799 * add entries to the fromUnicode trie,
800 * assume to be called with code points in ascending order
801 * and use that to build the trie in precompacted form
804 addFromUTrieEntry(CnvExtData
*extData
, UChar32 c
, uint32_t value
) {
805 int32_t i1
, i2
, i3
, i3b
, nextOffset
, min
, newBlock
;
812 * compute the index for each stage,
813 * allocate a stage block if necessary,
814 * and write the stage value
817 if(i1
>=extData
->stage1Top
) {
818 extData
->stage1Top
=i1
+1;
821 nextOffset
=(c
>>4)&0x3f;
823 if(extData
->stage1
[i1
]==0) {
824 /* allocate another block in stage 2; overlap with the previous block */
825 newBlock
=extData
->stage2Top
;
826 min
=newBlock
-nextOffset
; /* minimum block start with overlap */
827 while(min
<newBlock
&& extData
->stage2
[newBlock
-1]==0) {
831 extData
->stage1
[i1
]=(uint16_t)newBlock
;
832 extData
->stage2Top
=newBlock
+MBCS_STAGE_2_BLOCK_SIZE
;
833 if(extData
->stage2Top
>LENGTHOF(extData
->stage2
)) {
834 fprintf(stderr
, "error: too many stage 2 entries at U+%04x\n", (int)c
);
835 exit(U_MEMORY_ALLOCATION_ERROR
);
839 i2
=extData
->stage1
[i1
]+nextOffset
;
842 if(extData
->stage2
[i2
]==0) {
843 /* allocate another block in stage 3; overlap with the previous block */
844 newBlock
=extData
->stage3Top
;
845 min
=newBlock
-nextOffset
; /* minimum block start with overlap */
846 while(min
<newBlock
&& extData
->stage3
[newBlock
-1]==0) {
850 /* round up to a multiple of stage 3 granularity >1 (similar to utrie.c) */
851 newBlock
=(newBlock
+(UCNV_EXT_STAGE_3_GRANULARITY
-1))&~(UCNV_EXT_STAGE_3_GRANULARITY
-1);
852 extData
->stage2
[i2
]=(uint16_t)(newBlock
>>UCNV_EXT_STAGE_2_LEFT_SHIFT
);
854 extData
->stage3Top
=newBlock
+MBCS_STAGE_3_BLOCK_SIZE
;
855 if(extData
->stage3Top
>LENGTHOF(extData
->stage3
)) {
856 fprintf(stderr
, "error: too many stage 3 entries at U+%04x\n", (int)c
);
857 exit(U_MEMORY_ALLOCATION_ERROR
);
861 i3
=((int32_t)extData
->stage2
[i2
]<<UCNV_EXT_STAGE_2_LEFT_SHIFT
)+nextOffset
;
863 * assume extData->stage3[i3]==0 because we get
864 * code points in strictly ascending order
867 if(value
==UCNV_EXT_FROM_U_SUBCHAR1
) {
868 /* <subchar1> SUB mapping, see getFromUBytesValue() and prepareFromUMappings() */
869 extData
->stage3
[i3
]=1;
872 * precompaction is not optimal for <subchar1> |2 mappings because
873 * stage3 values for them are all the same, unlike for other mappings
874 * which all have unique values;
875 * use a simple compaction of reusing a whole block filled with these
879 /* is the entire block filled with <subchar1> |2 mappings? */
880 if(nextOffset
==MBCS_STAGE_3_BLOCK_SIZE
-1) {
881 for(min
=i3
-nextOffset
;
882 min
<i3
&& extData
->stage3
[min
]==1;
886 /* the entire block is filled with these mappings */
887 if(extData
->stage3Sub1Block
!=0) {
888 /* point to the previous such block and remove this block from stage3 */
889 extData
->stage2
[i2
]=extData
->stage3Sub1Block
;
890 extData
->stage3Top
-=MBCS_STAGE_3_BLOCK_SIZE
;
891 uprv_memset(extData
->stage3
+extData
->stage3Top
, 0, MBCS_STAGE_3_BLOCK_SIZE
*2);
893 /* remember this block's stage2 entry */
894 extData
->stage3Sub1Block
=extData
->stage2
[i2
];
899 if((i3b
=extData
->stage3bTop
++)>=LENGTHOF(extData
->stage3b
)) {
900 fprintf(stderr
, "error: too many stage 3b entries at U+%04x\n", (int)c
);
901 exit(U_MEMORY_ALLOCATION_ERROR
);
904 /* roundtrip or fallback mapping */
905 extData
->stage3
[i3
]=(uint16_t)i3b
;
906 extData
->stage3b
[i3b
]=value
;
911 generateFromUTrie(CnvExtData
*extData
, UCMTable
*table
, int32_t mapLength
) {
912 UCMapping
*mappings
, *m
;
915 int32_t subStart
, subLimit
;
924 mappings
=table
->mappings
;
925 map
=table
->reverseMap
;
928 * iterate over same-initial-code point mappings,
929 * enter the initial code point into the trie,
930 * and start a recursion on the corresponding mappings section
931 * with generateFromUTable()
934 codePoints
=UCM_GET_CODE_POINTS(table
, m
);
937 while(subLimit
<mapLength
) {
938 /* get a new subsection of mappings starting with the same code point */
941 while(next
==c
&& ++subLimit
<mapLength
) {
942 m
=mappings
+map
[subLimit
];
943 codePoints
=UCM_GET_CODE_POINTS(table
, m
);
948 * compute the value for this code point;
949 * if there is a mapping for this code point alone, it is at subStart
950 * because the table is sorted lexically
953 m
=mappings
+map
[subStart
];
954 codePoints
=UCM_GET_CODE_POINTS(table
, m
);
956 /* do not include this in generateFromUTable() */
959 if(subStart
<subLimit
&& mappings
[map
[subStart
]].uLen
==1) {
960 /* print error for multiple same-input-sequence mappings */
961 fprintf(stderr
, "error: multiple mappings from same Unicode code points\n");
962 ucm_printMapping(table
, m
, stderr
);
963 ucm_printMapping(table
, mappings
+map
[subStart
], stderr
);
967 value
=getFromUBytesValue(extData
, table
, m
);
970 if(subStart
==subLimit
) {
971 /* write the result for this one code point */
972 addFromUTrieEntry(extData
, c
, value
);
974 /* write the index to the subsection table */
975 addFromUTrieEntry(extData
, c
, (uint32_t)utm_countItems(extData
->fromUTableValues
));
977 /* recurse, starting from 16-bit-unit index 2, the first 16-bit unit after c */
978 if(!generateFromUTable(extData
, table
, subStart
, subLimit
, 2, value
)) {
987 * Generate the fromU data structures from the input table.
988 * The input table must be sorted, and all precision flags must be 0..3.
989 * This function will modify the table's reverseMap.
992 makeFromUTable(CnvExtData
*extData
, UCMTable
*table
) {
994 int32_t i
, stage1Top
, fromUCount
;
996 fromUCount
=prepareFromUMappings(table
);
998 extData
->fromUTableUChars
=utm_open("cnv extension fromUTableUChars", 0x10000, UCNV_EXT_FROM_U_DATA_MASK
+1, 2);
999 extData
->fromUTableValues
=utm_open("cnv extension fromUTableValues", 0x10000, UCNV_EXT_FROM_U_DATA_MASK
+1, 4);
1000 extData
->fromUBytes
=utm_open("cnv extension fromUBytes", 0x10000, UCNV_EXT_FROM_U_DATA_MASK
+1, 1);
1002 /* allocate all-unassigned stage blocks */
1003 extData
->stage2Top
=MBCS_STAGE_2_FIRST_ASSIGNED
;
1004 extData
->stage3Top
=MBCS_STAGE_3_FIRST_ASSIGNED
;
1007 * stage 3b stores only unique values, and in
1008 * index 0: 0 for "no mapping"
1009 * index 1: "no mapping" with preference for <subchar1> rather than <subchar>
1011 extData
->stage3b
[1]=UCNV_EXT_FROM_U_SUBCHAR1
;
1012 extData
->stage3bTop
=2;
1014 /* allocate the first entry in the fromUTable because index 0 means "no result" */
1015 utm_alloc(extData
->fromUTableUChars
);
1016 utm_alloc(extData
->fromUTableValues
);
1018 if(!generateFromUTrie(extData
, table
, fromUCount
)) {
1023 * offset the stage 1 trie entries by stage1Top because they will
1024 * be stored in a single array
1026 stage1
=extData
->stage1
;
1027 stage1Top
=extData
->stage1Top
;
1028 for(i
=0; i
<stage1Top
; ++i
) {
1029 stage1
[i
]=(uint16_t)(stage1
[i
]+stage1Top
);
1035 /* -------------------------------------------------------------------------- */
1038 CnvExtAddTable(NewConverter
*cnvData
, UCMTable
*table
, UConverterStaticData
*staticData
) {
1039 CnvExtData
*extData
;
1041 if(table
->unicodeMask
&UCNV_HAS_SURROGATES
) {
1042 fprintf(stderr
, "error: contains mappings for surrogate code points\n");
1046 staticData
->conversionType
=UCNV_MBCS
;
1048 extData
=(CnvExtData
*)cnvData
;
1051 * assume that the table is sorted
1053 * call the functions in this order because
1054 * makeToUTable() modifies the original reverseMap,
1055 * makeFromUTable() writes a whole new mapping into reverseMap
1058 makeToUTable(extData
, table
) &&
1059 makeFromUTable(extData
, table
);