]> git.saurik.com Git - apple/icu.git/blame - icuSources/tools/makeconv/gencnvex.c
ICU-461.18.tar.gz
[apple/icu.git] / icuSources / tools / makeconv / gencnvex.c
CommitLineData
374ca955
A
1/*
2*******************************************************************************
3*
46f4442e 4* Copyright (C) 2003-2007, International Business Machines
374ca955
A
5* Corporation and others. All Rights Reserved.
6*
7*******************************************************************************
8* file name: gencnvex.c
9* encoding: US-ASCII
10* tab size: 8 (not used)
11* indentation:4
12*
13* created on: 2003oct12
14* created by: Markus W. Scherer
15*/
16
17#include <stdio.h>
18#include "unicode/utypes.h"
19#include "unicode/ustring.h"
20#include "cstring.h"
21#include "cmemory.h"
22#include "ucnv_cnv.h"
23#include "ucnvmbcs.h"
24#include "toolutil.h"
25#include "unewdata.h"
26#include "ucm.h"
27#include "makeconv.h"
28#include "genmbcs.h"
29
30#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
31
73c04bcf 32
374ca955
A
33static void
34CnvExtClose(NewConverter *cnvData);
35
36static UBool
37CnvExtIsValid(NewConverter *cnvData,
38 const uint8_t *bytes, int32_t length);
39
40static UBool
41CnvExtAddTable(NewConverter *cnvData, UCMTable *table, UConverterStaticData *staticData);
42
43static uint32_t
44CnvExtWrite(NewConverter *cnvData, const UConverterStaticData *staticData,
45 UNewDataMemory *pData, int32_t tableType);
46
47typedef struct CnvExtData {
48 NewConverter newConverter;
49
50 UCMFile *ucm;
51
52 /* toUnicode (state table in ucm->states) */
53 UToolMemory *toUTable, *toUUChars;
54
55 /* fromUnicode */
56 UToolMemory *fromUTableUChars, *fromUTableValues, *fromUBytes;
57
58 uint16_t stage1[MBCS_STAGE_1_SIZE];
59 uint16_t stage2[MBCS_STAGE_2_SIZE];
60 uint16_t stage3[0x10000<<UCNV_EXT_STAGE_2_LEFT_SHIFT]; /* 0x10000 because of 16-bit stage 2/3 indexes */
61 uint32_t stage3b[0x10000];
62
63 int32_t stage1Top, stage2Top, stage3Top, stage3bTop;
64
65 /* for stage3 compaction of <subchar1> |2 mappings */
66 uint16_t stage3Sub1Block;
67
68 /* statistics */
69 int32_t
70 maxInBytes, maxOutBytes, maxBytesPerUChar,
71 maxInUChars, maxOutUChars, maxUCharsPerByte;
72} CnvExtData;
73
74NewConverter *
75CnvExtOpen(UCMFile *ucm) {
76 CnvExtData *extData;
77
78 extData=(CnvExtData *)uprv_malloc(sizeof(CnvExtData));
46f4442e
A
79 if(extData==NULL) {
80 printf("out of memory\n");
81 exit(U_MEMORY_ALLOCATION_ERROR);
82 }
83 uprv_memset(extData, 0, sizeof(CnvExtData));
374ca955 84
46f4442e 85 extData->ucm=ucm; /* aliased, not owned */
374ca955 86
46f4442e
A
87 extData->newConverter.close=CnvExtClose;
88 extData->newConverter.isValid=CnvExtIsValid;
89 extData->newConverter.addTable=CnvExtAddTable;
90 extData->newConverter.write=CnvExtWrite;
374ca955
A
91 return &extData->newConverter;
92}
93
94static void
95CnvExtClose(NewConverter *cnvData) {
96 CnvExtData *extData=(CnvExtData *)cnvData;
97 if(extData!=NULL) {
98 utm_close(extData->toUTable);
99 utm_close(extData->toUUChars);
100 utm_close(extData->fromUTableUChars);
101 utm_close(extData->fromUTableValues);
102 utm_close(extData->fromUBytes);
103 }
104}
105
106/* we do not expect this to be called */
107static UBool
108CnvExtIsValid(NewConverter *cnvData,
109 const uint8_t *bytes, int32_t length) {
110 return FALSE;
111}
112
113static uint32_t
114CnvExtWrite(NewConverter *cnvData, const UConverterStaticData *staticData,
115 UNewDataMemory *pData, int32_t tableType) {
116 CnvExtData *extData=(CnvExtData *)cnvData;
117 int32_t length, top, headerSize;
118
119 int32_t indexes[UCNV_EXT_INDEXES_MIN_LENGTH]={ 0 };
120
121 if(tableType&TABLE_BASE) {
122 headerSize=0;
123 } else {
124 _MBCSHeader header={ { 0, 0, 0, 0 }, 0, 0, 0, 0, 0, 0, 0 };
125
126 /* write the header and base table name for an extension-only table */
127 length=(int32_t)uprv_strlen(extData->ucm->baseName)+1;
128 while(length&3) {
129 /* add padding */
130 extData->ucm->baseName[length++]=0;
131 }
132
46f4442e 133 headerSize=MBCS_HEADER_V4_LENGTH*4+length;
374ca955
A
134
135 /* fill the header */
136 header.version[0]=4;
137 header.version[1]=2;
138 header.flags=(uint32_t)((headerSize<<8)|MBCS_OUTPUT_EXT_ONLY);
139
140 /* write the header and the base table name */
46f4442e 141 udata_writeBlock(pData, &header, MBCS_HEADER_V4_LENGTH*4);
374ca955
A
142 udata_writeBlock(pData, extData->ucm->baseName, length);
143 }
144
145 /* fill indexes[] - offsets/indexes are in units of the target array */
146 top=0;
147
148 indexes[UCNV_EXT_INDEXES_LENGTH]=length=UCNV_EXT_INDEXES_MIN_LENGTH;
149 top+=length*4;
150
151 indexes[UCNV_EXT_TO_U_INDEX]=top;
152 indexes[UCNV_EXT_TO_U_LENGTH]=length=utm_countItems(extData->toUTable);
153 top+=length*4;
154
155 indexes[UCNV_EXT_TO_U_UCHARS_INDEX]=top;
156 indexes[UCNV_EXT_TO_U_UCHARS_LENGTH]=length=utm_countItems(extData->toUUChars);
157 top+=length*2;
158
159 indexes[UCNV_EXT_FROM_U_UCHARS_INDEX]=top;
160 length=utm_countItems(extData->fromUTableUChars);
161 top+=length*2;
162
163 if(top&3) {
164 /* add padding */
165 *((UChar *)utm_alloc(extData->fromUTableUChars))=0;
166 *((uint32_t *)utm_alloc(extData->fromUTableValues))=0;
167 ++length;
168 top+=2;
169 }
170 indexes[UCNV_EXT_FROM_U_LENGTH]=length;
171
172 indexes[UCNV_EXT_FROM_U_VALUES_INDEX]=top;
173 top+=length*4;
174
175 indexes[UCNV_EXT_FROM_U_BYTES_INDEX]=top;
176 length=utm_countItems(extData->fromUBytes);
177 top+=length;
178
179 if(top&1) {
180 /* add padding */
181 *((uint8_t *)utm_alloc(extData->fromUBytes))=0;
182 ++length;
183 ++top;
184 }
185 indexes[UCNV_EXT_FROM_U_BYTES_LENGTH]=length;
186
187 indexes[UCNV_EXT_FROM_U_STAGE_12_INDEX]=top;
188 indexes[UCNV_EXT_FROM_U_STAGE_1_LENGTH]=length=extData->stage1Top;
189 indexes[UCNV_EXT_FROM_U_STAGE_12_LENGTH]=length+=extData->stage2Top;
190 top+=length*2;
191
192 indexes[UCNV_EXT_FROM_U_STAGE_3_INDEX]=top;
193 length=extData->stage3Top;
194 top+=length*2;
195
196 if(top&3) {
197 /* add padding */
198 extData->stage3[extData->stage3Top++]=0;
199 ++length;
200 top+=2;
201 }
202 indexes[UCNV_EXT_FROM_U_STAGE_3_LENGTH]=length;
203
204 indexes[UCNV_EXT_FROM_U_STAGE_3B_INDEX]=top;
205 indexes[UCNV_EXT_FROM_U_STAGE_3B_LENGTH]=length=extData->stage3bTop;
206 top+=length*4;
207
208 indexes[UCNV_EXT_SIZE]=top;
209
210 /* statistics */
211 indexes[UCNV_EXT_COUNT_BYTES]=
212 (extData->maxInBytes<<16)|
213 (extData->maxOutBytes<<8)|
214 extData->maxBytesPerUChar;
215 indexes[UCNV_EXT_COUNT_UCHARS]=
216 (extData->maxInUChars<<16)|
217 (extData->maxOutUChars<<8)|
218 extData->maxUCharsPerByte;
219
220 indexes[UCNV_EXT_FLAGS]=extData->ucm->ext->unicodeMask;
221
222 /* write the extension data */
223 udata_writeBlock(pData, indexes, sizeof(indexes));
224 udata_writeBlock(pData, utm_getStart(extData->toUTable), indexes[UCNV_EXT_TO_U_LENGTH]*4);
225 udata_writeBlock(pData, utm_getStart(extData->toUUChars), indexes[UCNV_EXT_TO_U_UCHARS_LENGTH]*2);
226
227 udata_writeBlock(pData, utm_getStart(extData->fromUTableUChars), indexes[UCNV_EXT_FROM_U_LENGTH]*2);
228 udata_writeBlock(pData, utm_getStart(extData->fromUTableValues), indexes[UCNV_EXT_FROM_U_LENGTH]*4);
229 udata_writeBlock(pData, utm_getStart(extData->fromUBytes), indexes[UCNV_EXT_FROM_U_BYTES_LENGTH]);
230
231 udata_writeBlock(pData, extData->stage1, extData->stage1Top*2);
232 udata_writeBlock(pData, extData->stage2, extData->stage2Top*2);
233 udata_writeBlock(pData, extData->stage3, extData->stage3Top*2);
234 udata_writeBlock(pData, extData->stage3b, extData->stage3bTop*4);
235
236#if 0
237 {
238 int32_t i, j;
239
240 length=extData->stage1Top;
241 printf("\nstage1[%x]:\n", length);
242
243 for(i=0; i<length; ++i) {
244 if(extData->stage1[i]!=length) {
245 printf("stage1[%04x]=%04x\n", i, extData->stage1[i]);
246 }
247 }
248
249 j=length;
250 length=extData->stage2Top;
251 printf("\nstage2[%x]:\n", length);
252
253 for(i=0; i<length; ++j, ++i) {
254 if(extData->stage2[i]!=0) {
255 printf("stage12[%04x]=%04x\n", j, extData->stage2[i]);
256 }
257 }
258
259 length=extData->stage3Top;
260 printf("\nstage3[%x]:\n", length);
261
262 for(i=0; i<length; ++i) {
263 if(extData->stage3[i]!=0) {
264 printf("stage3[%04x]=%04x\n", i, extData->stage3[i]);
265 }
266 }
267
268 length=extData->stage3bTop;
269 printf("\nstage3b[%x]:\n", length);
270
271 for(i=0; i<length; ++i) {
272 if(extData->stage3b[i]!=0) {
273 printf("stage3b[%04x]=%08x\n", i, extData->stage3b[i]);
274 }
275 }
276 }
277#endif
278
279 if(VERBOSE) {
280 printf("size of extension data: %ld\n", (long)top);
281 }
282
283 /* return the number of bytes that should have been written */
284 return (uint32_t)(headerSize+top);
285}
286
287/* to Unicode --------------------------------------------------------------- */
288
289/*
290 * Remove fromUnicode fallbacks and SUB mappings which are irrelevant for
291 * the toUnicode table.
46f4442e
A
292 * This includes mappings with MBCS_FROM_U_EXT_FLAG which were suitable
293 * for the base toUnicode table but not for the base fromUnicode table.
374ca955 294 * The table must be sorted.
46f4442e 295 * Modifies previous data in the reverseMap.
374ca955
A
296 */
297static int32_t
298reduceToUMappings(UCMTable *table) {
299 UCMapping *mappings;
300 int32_t *map;
301 int32_t i, j, count;
302 int8_t flag;
303
304 mappings=table->mappings;
305 map=table->reverseMap;
306 count=table->mappingsLength;
307
308 /* leave the map alone for the initial mappings with desired flags */
309 for(i=j=0; i<count; ++i) {
310 flag=mappings[map[i]].f;
311 if(flag!=0 && flag!=3) {
312 break;
313 }
314 }
315
316 /* reduce from here to the rest */
317 for(j=i; i<count; ++i) {
318 flag=mappings[map[i]].f;
319 if(flag==0 || flag==3) {
320 map[j++]=map[i];
321 }
322 }
323
324 return j;
325}
326
327static uint32_t
328getToUnicodeValue(CnvExtData *extData, UCMTable *table, UCMapping *m) {
329 UChar32 *u32;
330 UChar *u;
331 uint32_t value;
332 int32_t u16Length, ratio;
333 UErrorCode errorCode;
334
335 /* write the Unicode result code point or string index */
336 if(m->uLen==1) {
337 u16Length=U16_LENGTH(m->u);
338 value=(uint32_t)(UCNV_EXT_TO_U_MIN_CODE_POINT+m->u);
339 } else {
340 /* the parser enforces m->uLen<=UCNV_EXT_MAX_UCHARS */
341
342 /* get the result code point string and its 16-bit string length */
343 u32=UCM_GET_CODE_POINTS(table, m);
344 errorCode=U_ZERO_ERROR;
345 u_strFromUTF32(NULL, 0, &u16Length, u32, m->uLen, &errorCode);
346 if(U_FAILURE(errorCode) && errorCode!=U_BUFFER_OVERFLOW_ERROR) {
347 exit(errorCode);
348 }
349
350 /* allocate it and put its length and index into the value */
351 value=
352 (((uint32_t)m->uLen+UCNV_EXT_TO_U_LENGTH_OFFSET)<<UCNV_EXT_TO_U_LENGTH_SHIFT)|
353 ((uint32_t)utm_countItems(extData->toUUChars));
354 u=utm_allocN(extData->toUUChars, u16Length);
355
356 /* write the result 16-bit string */
357 errorCode=U_ZERO_ERROR;
358 u_strFromUTF32(u, u16Length, NULL, u32, m->uLen, &errorCode);
359 if(U_FAILURE(errorCode) && errorCode!=U_BUFFER_OVERFLOW_ERROR) {
360 exit(errorCode);
361 }
362 }
363 if(m->f==0) {
364 value|=UCNV_EXT_TO_U_ROUNDTRIP_FLAG;
365 }
366
367 /* update statistics */
368 if(m->bLen>extData->maxInBytes) {
369 extData->maxInBytes=m->bLen;
370 }
371 if(u16Length>extData->maxOutUChars) {
372 extData->maxOutUChars=u16Length;
373 }
374
375 ratio=(u16Length+(m->bLen-1))/m->bLen;
376 if(ratio>extData->maxUCharsPerByte) {
377 extData->maxUCharsPerByte=ratio;
378 }
379
380 return value;
381}
382
383/*
384 * Recursive toUTable generator core function.
385 * Preconditions:
386 * - start<limit (There is at least one mapping.)
387 * - The mappings are sorted lexically. (Access is through the reverseMap.)
388 * - All mappings between start and limit have input sequences that share
389 * the same prefix of unitIndex length, and therefore all of these sequences
390 * are at least unitIndex+1 long.
391 * - There are only relevant mappings available through the reverseMap,
392 * see reduceToUMappings().
393 *
394 * One function invocation generates one section table.
395 *
396 * Steps:
397 * 1. Count the number of unique unit values and get the low/high unit values
398 * that occur at unitIndex.
399 * 2. Allocate the section table with possible optimization for linear access.
400 * 3. Write temporary version of the section table with start indexes of
401 * subsections, each corresponding to one unit value at unitIndex.
402 * 4. Iterate through the table once more, and depending on the subsection length:
403 * 0: write 0 as a result value (unused byte in linear-access section table)
404 * >0: if there is one mapping with an input unit sequence of unitIndex+1
405 * then defaultValue=compute the mapping result for this whole sequence
406 * else defaultValue=0
407 *
408 * recurse into the subsection
409 */
410static UBool
411generateToUTable(CnvExtData *extData, UCMTable *table,
412 int32_t start, int32_t limit, int32_t unitIndex,
413 uint32_t defaultValue) {
414 UCMapping *mappings, *m;
415 int32_t *map;
416 int32_t i, j, uniqueCount, count, subStart, subLimit;
417
418 uint8_t *bytes;
419 int32_t low, high, prev;
420
421 uint32_t *section;
422
423 mappings=table->mappings;
424 map=table->reverseMap;
425
426 /* step 1: examine the input units; set low, high, uniqueCount */
427 m=mappings+map[start];
428 bytes=UCM_GET_BYTES(table, m);
429 low=bytes[unitIndex];
430 uniqueCount=1;
431
432 prev=high=low;
433 for(i=start+1; i<limit; ++i) {
434 m=mappings+map[i];
435 bytes=UCM_GET_BYTES(table, m);
436 high=bytes[unitIndex];
437
438 if(high!=prev) {
439 prev=high;
440 ++uniqueCount;
441 }
442 }
443
444 /* step 2: allocate the section; set count, section */
445 count=(high-low)+1;
46f4442e 446 if(count<0x100 && (unitIndex==0 || uniqueCount>=(3*count)/4)) {
374ca955
A
447 /*
448 * for the root table and for fairly full tables:
449 * allocate for direct, linear array access
450 * by keeping count, to write an entry for each unit value
451 * from low to high
46f4442e
A
452 * exception: use a compact table if count==0x100 because
453 * that cannot be encoded in the length byte
374ca955
A
454 */
455 } else {
456 count=uniqueCount;
457 }
458
46f4442e
A
459 if(count>=0x100) {
460 fprintf(stderr, "error: toUnicode extension table section overflow: %ld section entries\n", (long)count);
461 return FALSE;
462 }
463
374ca955
A
464 /* allocate the section: 1 entry for the header + count for the items */
465 section=(uint32_t *)utm_allocN(extData->toUTable, 1+count);
466
467 /* write the section header */
468 *section++=((uint32_t)count<<UCNV_EXT_TO_U_BYTE_SHIFT)|defaultValue;
469
470 /* step 3: write temporary section table with subsection starts */
471 prev=low-1; /* just before low to prevent empty subsections before low */
472 j=0; /* section table index */
473 for(i=start; i<limit; ++i) {
474 m=mappings+map[i];
475 bytes=UCM_GET_BYTES(table, m);
476 high=bytes[unitIndex];
477
478 if(high!=prev) {
479 /* start of a new subsection for unit high */
480 if(count>uniqueCount) {
481 /* write empty subsections for unused units in a linear table */
482 while(++prev<high) {
483 section[j++]=((uint32_t)prev<<UCNV_EXT_TO_U_BYTE_SHIFT)|(uint32_t)i;
484 }
485 } else {
486 prev=high;
487 }
488
489 /* write the entry with the subsection start */
490 section[j++]=((uint32_t)high<<UCNV_EXT_TO_U_BYTE_SHIFT)|(uint32_t)i;
491 }
492 }
493 /* assert(j==count) */
494
495 /* step 4: recurse and write results */
496 subLimit=UCNV_EXT_TO_U_GET_VALUE(section[0]);
497 for(j=0; j<count; ++j) {
498 subStart=subLimit;
499 subLimit= (j+1)<count ? UCNV_EXT_TO_U_GET_VALUE(section[j+1]) : limit;
500
501 /* remove the subStart temporary value */
502 section[j]&=~UCNV_EXT_TO_U_VALUE_MASK;
503
504 if(subStart==subLimit) {
505 /* leave the value zero: empty subsection for unused unit in a linear table */
506 continue;
507 }
508
509 /* see if there is exactly one input unit sequence of length unitIndex+1 */
510 defaultValue=0;
511 m=mappings+map[subStart];
512 if(m->bLen==unitIndex+1) {
513 /* do not include this in generateToUTable() */
514 ++subStart;
515
516 if(subStart<subLimit && mappings[map[subStart]].bLen==unitIndex+1) {
517 /* print error for multiple same-input-sequence mappings */
518 fprintf(stderr, "error: multiple mappings from same bytes\n");
519 ucm_printMapping(table, m, stderr);
520 ucm_printMapping(table, mappings+map[subStart], stderr);
521 return FALSE;
522 }
523
524 defaultValue=getToUnicodeValue(extData, table, m);
525 }
526
527 if(subStart==subLimit) {
528 /* write the result for the input sequence ending here */
529 section[j]|=defaultValue;
530 } else {
531 /* write the index to the subsection table */
532 section[j]|=(uint32_t)utm_countItems(extData->toUTable);
533
534 /* recurse */
535 if(!generateToUTable(extData, table, subStart, subLimit, unitIndex+1, defaultValue)) {
536 return FALSE;
537 }
538 }
539 }
540 return TRUE;
541}
542
543/*
544 * Generate the toUTable and toUUChars from the input table.
545 * The input table must be sorted, and all precision flags must be 0..3.
546 * This function will modify the table's reverseMap.
547 */
548static UBool
549makeToUTable(CnvExtData *extData, UCMTable *table) {
550 int32_t toUCount;
551
552 toUCount=reduceToUMappings(table);
553
554 extData->toUTable=utm_open("cnv extension toUTable", 0x10000, UCNV_EXT_TO_U_MIN_CODE_POINT, 4);
555 extData->toUUChars=utm_open("cnv extension toUUChars", 0x10000, UCNV_EXT_TO_U_INDEX_MASK+1, 2);
556
557 return generateToUTable(extData, table, 0, toUCount, 0, 0);
558}
559
560/* from Unicode ------------------------------------------------------------- */
561
562/*
563 * preprocessing:
564 * rebuild reverseMap with mapping indexes for mappings relevant for from Unicode
565 * change each Unicode string to encode all but the first code point in 16-bit form
566 *
567 * generation:
568 * for each unique code point
569 * write an entry in the 3-stage trie
570 * check that there is only one single-code point sequence
571 * start recursion for following 16-bit input units
572 */
573
574/*
575 * Remove toUnicode fallbacks and non-<subchar1> SUB mappings
576 * which are irrelevant for the fromUnicode extension table.
46f4442e 577 * Remove MBCS_FROM_U_EXT_FLAG bits.
374ca955
A
578 * Overwrite the reverseMap with an index array to the relevant mappings.
579 * Modify the code point sequences to a generator-friendly format where
580 * the first code points remains unchanged but the following are recoded
581 * into 16-bit Unicode string form.
582 * The table must be sorted.
583 * Destroys previous data in the reverseMap.
584 */
585static int32_t
586prepareFromUMappings(UCMTable *table) {
587 UCMapping *mappings, *m;
588 int32_t *map;
589 int32_t i, j, count;
590 int8_t flag;
591
592 mappings=table->mappings;
593 map=table->reverseMap;
594 count=table->mappingsLength;
595
596 /*
597 * we do not go through the map on input because the mappings are
598 * sorted lexically
599 */
600 m=mappings;
601
602 for(i=j=0; i<count; ++m, ++i) {
603 flag=m->f;
46f4442e
A
604 if(flag>=0) {
605 flag&=MBCS_FROM_U_EXT_MASK;
606 m->f=flag;
607 }
374ca955
A
608 if(flag==0 || flag==1 || (flag==2 && m->bLen==1)) {
609 map[j++]=i;
610
611 if(m->uLen>1) {
612 /* recode all but the first code point to 16-bit Unicode */
613 UChar32 *u32;
614 UChar *u;
615 UChar32 c;
616 int32_t q, r;
617
618 u32=UCM_GET_CODE_POINTS(table, m);
619 u=(UChar *)u32; /* destructive in-place recoding */
620 for(r=2, q=1; q<m->uLen; ++q) {
621 c=u32[q];
622 U16_APPEND_UNSAFE(u, r, c);
623 }
624
625 /* counts the first code point always at 2 - the first 16-bit unit is at 16-bit index 2 */
626 m->uLen=(int8_t)r;
627 }
628 }
629 }
630
631 return j;
632}
633
634static uint32_t
635getFromUBytesValue(CnvExtData *extData, UCMTable *table, UCMapping *m) {
636 uint8_t *bytes, *resultBytes;
637 uint32_t value;
638 int32_t u16Length, ratio;
639
640 if(m->f==2) {
641 /*
642 * no mapping, <subchar1> preferred
643 *
644 * no need to count in statistics because the subchars are already
645 * counted for maxOutBytes and maxBytesPerUChar in UConverterStaticData,
646 * and this non-mapping does not count for maxInUChars which are always
647 * trivially at least two if counting unmappable supplementary code points
648 */
649 return UCNV_EXT_FROM_U_SUBCHAR1;
650 }
651
652 bytes=UCM_GET_BYTES(table, m);
653 value=0;
654 switch(m->bLen) {
655 /* 1..3: store the bytes in the value word */
656 case 3:
657 value=((uint32_t)*bytes++)<<16;
658 case 2:
659 value|=((uint32_t)*bytes++)<<8;
660 case 1:
661 value|=*bytes;
662 break;
663 default:
664 /* the parser enforces m->bLen<=UCNV_EXT_MAX_BYTES */
665 /* store the bytes in fromUBytes[] and the index in the value word */
666 value=(uint32_t)utm_countItems(extData->fromUBytes);
667 resultBytes=utm_allocN(extData->fromUBytes, m->bLen);
668 uprv_memcpy(resultBytes, bytes, m->bLen);
669 break;
670 }
671 value|=(uint32_t)m->bLen<<UCNV_EXT_FROM_U_LENGTH_SHIFT;
672 if(m->f==0) {
673 value|=UCNV_EXT_FROM_U_ROUNDTRIP_FLAG;
674 }
675
676 /* calculate the real UTF-16 length (see recoding in prepareFromUMappings()) */
677 if(m->uLen==1) {
678 u16Length=U16_LENGTH(m->u);
679 } else {
680 u16Length=U16_LENGTH(UCM_GET_CODE_POINTS(table, m)[0])+(m->uLen-2);
681 }
682
683 /* update statistics */
684 if(u16Length>extData->maxInUChars) {
685 extData->maxInUChars=u16Length;
686 }
687 if(m->bLen>extData->maxOutBytes) {
688 extData->maxOutBytes=m->bLen;
689 }
690
691 ratio=(m->bLen+(u16Length-1))/u16Length;
692 if(ratio>extData->maxBytesPerUChar) {
693 extData->maxBytesPerUChar=ratio;
694 }
695
696 return value;
697}
698
699/*
700 * works like generateToUTable(), except that the
701 * output section consists of two arrays, one for input UChars and one
702 * for result values
703 *
704 * also, fromUTable sections are always stored in a compact form for
705 * access via binary search
706 */
707static UBool
708generateFromUTable(CnvExtData *extData, UCMTable *table,
709 int32_t start, int32_t limit, int32_t unitIndex,
710 uint32_t defaultValue) {
711 UCMapping *mappings, *m;
712 int32_t *map;
713 int32_t i, j, uniqueCount, count, subStart, subLimit;
714
715 UChar *uchars;
716 UChar32 low, high, prev;
717
718 UChar *sectionUChars;
719 uint32_t *sectionValues;
720
721 mappings=table->mappings;
722 map=table->reverseMap;
723
724 /* step 1: examine the input units; set low, high, uniqueCount */
725 m=mappings+map[start];
726 uchars=(UChar *)UCM_GET_CODE_POINTS(table, m);
727 low=uchars[unitIndex];
728 uniqueCount=1;
729
730 prev=high=low;
731 for(i=start+1; i<limit; ++i) {
732 m=mappings+map[i];
733 uchars=(UChar *)UCM_GET_CODE_POINTS(table, m);
734 high=uchars[unitIndex];
735
736 if(high!=prev) {
737 prev=high;
738 ++uniqueCount;
739 }
740 }
741
742 /* step 2: allocate the section; set count, section */
743 /* the fromUTable always stores for access via binary search */
744 count=uniqueCount;
745
746 /* allocate the section: 1 entry for the header + count for the items */
747 sectionUChars=(UChar *)utm_allocN(extData->fromUTableUChars, 1+count);
748 sectionValues=(uint32_t *)utm_allocN(extData->fromUTableValues, 1+count);
749
750 /* write the section header */
751 *sectionUChars++=(UChar)count;
752 *sectionValues++=defaultValue;
753
754 /* step 3: write temporary section table with subsection starts */
755 prev=low-1; /* just before low to prevent empty subsections before low */
756 j=0; /* section table index */
757 for(i=start; i<limit; ++i) {
758 m=mappings+map[i];
759 uchars=(UChar *)UCM_GET_CODE_POINTS(table, m);
760 high=uchars[unitIndex];
761
762 if(high!=prev) {
763 /* start of a new subsection for unit high */
764 prev=high;
765
766 /* write the entry with the subsection start */
767 sectionUChars[j]=(UChar)high;
768 sectionValues[j]=(uint32_t)i;
769 ++j;
770 }
771 }
772 /* assert(j==count) */
773
774 /* step 4: recurse and write results */
775 subLimit=(int32_t)(sectionValues[0]);
776 for(j=0; j<count; ++j) {
777 subStart=subLimit;
778 subLimit= (j+1)<count ? (int32_t)(sectionValues[j+1]) : limit;
779
780 /* see if there is exactly one input unit sequence of length unitIndex+1 */
781 defaultValue=0;
782 m=mappings+map[subStart];
783 if(m->uLen==unitIndex+1) {
784 /* do not include this in generateToUTable() */
785 ++subStart;
786
787 if(subStart<subLimit && mappings[map[subStart]].uLen==unitIndex+1) {
788 /* print error for multiple same-input-sequence mappings */
789 fprintf(stderr, "error: multiple mappings from same Unicode code points\n");
790 ucm_printMapping(table, m, stderr);
791 ucm_printMapping(table, mappings+map[subStart], stderr);
792 return FALSE;
793 }
794
795 defaultValue=getFromUBytesValue(extData, table, m);
796 }
797
798 if(subStart==subLimit) {
799 /* write the result for the input sequence ending here */
800 sectionValues[j]=defaultValue;
801 } else {
802 /* write the index to the subsection table */
803 sectionValues[j]=(uint32_t)utm_countItems(extData->fromUTableValues);
804
805 /* recurse */
806 if(!generateFromUTable(extData, table, subStart, subLimit, unitIndex+1, defaultValue)) {
807 return FALSE;
808 }
809 }
810 }
811 return TRUE;
812}
813
814/*
815 * add entries to the fromUnicode trie,
816 * assume to be called with code points in ascending order
817 * and use that to build the trie in precompacted form
818 */
819static void
820addFromUTrieEntry(CnvExtData *extData, UChar32 c, uint32_t value) {
821 int32_t i1, i2, i3, i3b, nextOffset, min, newBlock;
822
823 if(value==0) {
824 return;
825 }
826
827 /*
828 * compute the index for each stage,
829 * allocate a stage block if necessary,
830 * and write the stage value
831 */
832 i1=c>>10;
833 if(i1>=extData->stage1Top) {
834 extData->stage1Top=i1+1;
835 }
836
837 nextOffset=(c>>4)&0x3f;
838
839 if(extData->stage1[i1]==0) {
840 /* allocate another block in stage 2; overlap with the previous block */
841 newBlock=extData->stage2Top;
842 min=newBlock-nextOffset; /* minimum block start with overlap */
843 while(min<newBlock && extData->stage2[newBlock-1]==0) {
844 --newBlock;
845 }
846
847 extData->stage1[i1]=(uint16_t)newBlock;
848 extData->stage2Top=newBlock+MBCS_STAGE_2_BLOCK_SIZE;
849 if(extData->stage2Top>LENGTHOF(extData->stage2)) {
850 fprintf(stderr, "error: too many stage 2 entries at U+%04x\n", (int)c);
851 exit(U_MEMORY_ALLOCATION_ERROR);
852 }
853 }
854
855 i2=extData->stage1[i1]+nextOffset;
856 nextOffset=c&0xf;
857
858 if(extData->stage2[i2]==0) {
859 /* allocate another block in stage 3; overlap with the previous block */
860 newBlock=extData->stage3Top;
861 min=newBlock-nextOffset; /* minimum block start with overlap */
862 while(min<newBlock && extData->stage3[newBlock-1]==0) {
863 --newBlock;
864 }
865
866 /* round up to a multiple of stage 3 granularity >1 (similar to utrie.c) */
867 newBlock=(newBlock+(UCNV_EXT_STAGE_3_GRANULARITY-1))&~(UCNV_EXT_STAGE_3_GRANULARITY-1);
868 extData->stage2[i2]=(uint16_t)(newBlock>>UCNV_EXT_STAGE_2_LEFT_SHIFT);
869
870 extData->stage3Top=newBlock+MBCS_STAGE_3_BLOCK_SIZE;
871 if(extData->stage3Top>LENGTHOF(extData->stage3)) {
872 fprintf(stderr, "error: too many stage 3 entries at U+%04x\n", (int)c);
873 exit(U_MEMORY_ALLOCATION_ERROR);
874 }
875 }
876
877 i3=((int32_t)extData->stage2[i2]<<UCNV_EXT_STAGE_2_LEFT_SHIFT)+nextOffset;
878 /*
879 * assume extData->stage3[i3]==0 because we get
880 * code points in strictly ascending order
881 */
882
883 if(value==UCNV_EXT_FROM_U_SUBCHAR1) {
884 /* <subchar1> SUB mapping, see getFromUBytesValue() and prepareFromUMappings() */
885 extData->stage3[i3]=1;
886
887 /*
888 * precompaction is not optimal for <subchar1> |2 mappings because
889 * stage3 values for them are all the same, unlike for other mappings
890 * which all have unique values;
891 * use a simple compaction of reusing a whole block filled with these
892 * mappings
893 */
894
895 /* is the entire block filled with <subchar1> |2 mappings? */
896 if(nextOffset==MBCS_STAGE_3_BLOCK_SIZE-1) {
897 for(min=i3-nextOffset;
898 min<i3 && extData->stage3[min]==1;
899 ++min) {}
900
901 if(min==i3) {
902 /* the entire block is filled with these mappings */
903 if(extData->stage3Sub1Block!=0) {
904 /* point to the previous such block and remove this block from stage3 */
905 extData->stage2[i2]=extData->stage3Sub1Block;
906 extData->stage3Top-=MBCS_STAGE_3_BLOCK_SIZE;
907 uprv_memset(extData->stage3+extData->stage3Top, 0, MBCS_STAGE_3_BLOCK_SIZE*2);
908 } else {
909 /* remember this block's stage2 entry */
910 extData->stage3Sub1Block=extData->stage2[i2];
911 }
912 }
913 }
914 } else {
915 if((i3b=extData->stage3bTop++)>=LENGTHOF(extData->stage3b)) {
916 fprintf(stderr, "error: too many stage 3b entries at U+%04x\n", (int)c);
917 exit(U_MEMORY_ALLOCATION_ERROR);
918 }
919
920 /* roundtrip or fallback mapping */
921 extData->stage3[i3]=(uint16_t)i3b;
922 extData->stage3b[i3b]=value;
923 }
924}
925
926static UBool
927generateFromUTrie(CnvExtData *extData, UCMTable *table, int32_t mapLength) {
928 UCMapping *mappings, *m;
929 int32_t *map;
930 uint32_t value;
931 int32_t subStart, subLimit;
932
933 UChar32 *codePoints;
934 UChar32 c, next;
935
936 if(mapLength==0) {
937 return TRUE;
938 }
939
940 mappings=table->mappings;
941 map=table->reverseMap;
942
943 /*
944 * iterate over same-initial-code point mappings,
945 * enter the initial code point into the trie,
946 * and start a recursion on the corresponding mappings section
947 * with generateFromUTable()
948 */
949 m=mappings+map[0];
950 codePoints=UCM_GET_CODE_POINTS(table, m);
951 next=codePoints[0];
952 subLimit=0;
953 while(subLimit<mapLength) {
954 /* get a new subsection of mappings starting with the same code point */
955 subStart=subLimit;
956 c=next;
957 while(next==c && ++subLimit<mapLength) {
958 m=mappings+map[subLimit];
959 codePoints=UCM_GET_CODE_POINTS(table, m);
960 next=codePoints[0];
961 }
962
963 /*
964 * compute the value for this code point;
965 * if there is a mapping for this code point alone, it is at subStart
966 * because the table is sorted lexically
967 */
968 value=0;
969 m=mappings+map[subStart];
970 codePoints=UCM_GET_CODE_POINTS(table, m);
971 if(m->uLen==1) {
972 /* do not include this in generateFromUTable() */
973 ++subStart;
974
975 if(subStart<subLimit && mappings[map[subStart]].uLen==1) {
976 /* print error for multiple same-input-sequence mappings */
977 fprintf(stderr, "error: multiple mappings from same Unicode code points\n");
978 ucm_printMapping(table, m, stderr);
979 ucm_printMapping(table, mappings+map[subStart], stderr);
980 return FALSE;
981 }
982
983 value=getFromUBytesValue(extData, table, m);
984 }
985
986 if(subStart==subLimit) {
987 /* write the result for this one code point */
988 addFromUTrieEntry(extData, c, value);
989 } else {
990 /* write the index to the subsection table */
991 addFromUTrieEntry(extData, c, (uint32_t)utm_countItems(extData->fromUTableValues));
992
993 /* recurse, starting from 16-bit-unit index 2, the first 16-bit unit after c */
994 if(!generateFromUTable(extData, table, subStart, subLimit, 2, value)) {
995 return FALSE;
996 }
997 }
998 }
999 return TRUE;
1000}
1001
1002/*
1003 * Generate the fromU data structures from the input table.
1004 * The input table must be sorted, and all precision flags must be 0..3.
1005 * This function will modify the table's reverseMap.
1006 */
1007static UBool
1008makeFromUTable(CnvExtData *extData, UCMTable *table) {
1009 uint16_t *stage1;
1010 int32_t i, stage1Top, fromUCount;
1011
1012 fromUCount=prepareFromUMappings(table);
1013
1014 extData->fromUTableUChars=utm_open("cnv extension fromUTableUChars", 0x10000, UCNV_EXT_FROM_U_DATA_MASK+1, 2);
1015 extData->fromUTableValues=utm_open("cnv extension fromUTableValues", 0x10000, UCNV_EXT_FROM_U_DATA_MASK+1, 4);
1016 extData->fromUBytes=utm_open("cnv extension fromUBytes", 0x10000, UCNV_EXT_FROM_U_DATA_MASK+1, 1);
1017
1018 /* allocate all-unassigned stage blocks */
1019 extData->stage2Top=MBCS_STAGE_2_FIRST_ASSIGNED;
1020 extData->stage3Top=MBCS_STAGE_3_FIRST_ASSIGNED;
1021
1022 /*
1023 * stage 3b stores only unique values, and in
1024 * index 0: 0 for "no mapping"
1025 * index 1: "no mapping" with preference for <subchar1> rather than <subchar>
1026 */
1027 extData->stage3b[1]=UCNV_EXT_FROM_U_SUBCHAR1;
1028 extData->stage3bTop=2;
1029
1030 /* allocate the first entry in the fromUTable because index 0 means "no result" */
1031 utm_alloc(extData->fromUTableUChars);
1032 utm_alloc(extData->fromUTableValues);
1033
1034 if(!generateFromUTrie(extData, table, fromUCount)) {
1035 return FALSE;
1036 }
1037
1038 /*
1039 * offset the stage 1 trie entries by stage1Top because they will
1040 * be stored in a single array
1041 */
1042 stage1=extData->stage1;
1043 stage1Top=extData->stage1Top;
1044 for(i=0; i<stage1Top; ++i) {
1045 stage1[i]=(uint16_t)(stage1[i]+stage1Top);
1046 }
1047
1048 return TRUE;
1049}
1050
1051/* -------------------------------------------------------------------------- */
1052
1053static UBool
1054CnvExtAddTable(NewConverter *cnvData, UCMTable *table, UConverterStaticData *staticData) {
1055 CnvExtData *extData;
1056
1057 if(table->unicodeMask&UCNV_HAS_SURROGATES) {
1058 fprintf(stderr, "error: contains mappings for surrogate code points\n");
1059 return FALSE;
1060 }
1061
1062 staticData->conversionType=UCNV_MBCS;
1063
1064 extData=(CnvExtData *)cnvData;
1065
1066 /*
1067 * assume that the table is sorted
1068 *
1069 * call the functions in this order because
1070 * makeToUTable() modifies the original reverseMap,
1071 * makeFromUTable() writes a whole new mapping into reverseMap
1072 */
1073 return
1074 makeToUTable(extData, table) &&
1075 makeFromUTable(extData, table);
1076}