ICU-57166.0.1.tar.gz

[apple/icu.git] / icuSources / tools / makeconv / gencnvex.c

diff --git a/icuSources/tools/makeconv/gencnvex.c b/icuSources/tools/makeconv/gencnvex.c
index c5fc7d464c3d479d3b70ce2089af4d19814d1013..6ea944186df2de86423cf8a8583d4ee0029c1975 100644 (file)
--- a/icuSources/tools/makeconv/gencnvex.c
+++ b/icuSources/tools/makeconv/gencnvex.c
@@ -1,7 +1,7 @@
 /*
 *******************************************************************************
 *
-*   Copyright (C) 2003-2006, International Business Machines
+*   Copyright (C) 2003-2014, International Business Machines
 *   Corporation and others.  All Rights Reserved.
 *
 *******************************************************************************
@@ -27,9 +27,6 @@
 #include "makeconv.h"
 #include "genmbcs.h"
 
-#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
-
-
 static void
 CnvExtClose(NewConverter *cnvData);
 
@@ -76,16 +73,18 @@ CnvExtOpen(UCMFile *ucm) {
     CnvExtData *extData;
     
     extData=(CnvExtData *)uprv_malloc(sizeof(CnvExtData));
-    if(extData!=NULL) {
-        uprv_memset(extData, 0, sizeof(CnvExtData));
+    if(extData==NULL) {
+        printf("out of memory\n");
+        exit(U_MEMORY_ALLOCATION_ERROR);
+    }
+    uprv_memset(extData, 0, sizeof(CnvExtData));
 
-        extData->ucm=ucm; /* aliased, not owned */
+    extData->ucm=ucm; /* aliased, not owned */
 
-        extData->newConverter.close=CnvExtClose;
-        extData->newConverter.isValid=CnvExtIsValid;
-        extData->newConverter.addTable=CnvExtAddTable;
-        extData->newConverter.write=CnvExtWrite;
-    }
+    extData->newConverter.close=CnvExtClose;
+    extData->newConverter.isValid=CnvExtIsValid;
+    extData->newConverter.addTable=CnvExtAddTable;
+    extData->newConverter.write=CnvExtWrite;
     return &extData->newConverter;
 }
 
@@ -98,6 +97,7 @@ CnvExtClose(NewConverter *cnvData) {
         utm_close(extData->fromUTableUChars);
         utm_close(extData->fromUTableValues);
         utm_close(extData->fromUBytes);
+        uprv_free(extData);
     }
 }
 
@@ -128,7 +128,7 @@ CnvExtWrite(NewConverter *cnvData, const UConverterStaticData *staticData,
             extData->ucm->baseName[length++]=0;
         }
 
-        headerSize=sizeof(header)+length;
+        headerSize=MBCS_HEADER_V4_LENGTH*4+length;
 
         /* fill the header */
         header.version[0]=4;
@@ -136,7 +136,7 @@ CnvExtWrite(NewConverter *cnvData, const UConverterStaticData *staticData,
         header.flags=(uint32_t)((headerSize<<8)|MBCS_OUTPUT_EXT_ONLY);
 
         /* write the header and the base table name */
-        udata_writeBlock(pData, &header, sizeof(header));
+        udata_writeBlock(pData, &header, MBCS_HEADER_V4_LENGTH*4);
         udata_writeBlock(pData, extData->ucm->baseName, length);
     }
 
@@ -287,8 +287,10 @@ CnvExtWrite(NewConverter *cnvData, const UConverterStaticData *staticData,
 /*
  * Remove fromUnicode fallbacks and SUB mappings which are irrelevant for
  * the toUnicode table.
+ * This includes mappings with MBCS_FROM_U_EXT_FLAG which were suitable
+ * for the base toUnicode table but not for the base fromUnicode table.
  * The table must be sorted.
- * Destroys previous data in the reverseMap.
+ * Modifies previous data in the reverseMap.
  */
 static int32_t
 reduceToUMappings(UCMTable *table) {
@@ -345,7 +347,7 @@ getToUnicodeValue(CnvExtData *extData, UCMTable *table, UCMapping *m) {
 
         /* allocate it and put its length and index into the value */
         value=
-            (((uint32_t)m->uLen+UCNV_EXT_TO_U_LENGTH_OFFSET)<<UCNV_EXT_TO_U_LENGTH_SHIFT)|
+            (((uint32_t)u16Length+UCNV_EXT_TO_U_LENGTH_OFFSET)<<UCNV_EXT_TO_U_LENGTH_SHIFT)|
             ((uint32_t)utm_countItems(extData->toUUChars));
         u=utm_allocN(extData->toUUChars, u16Length);
 
@@ -439,17 +441,24 @@ generateToUTable(CnvExtData *extData, UCMTable *table,
 
     /* step 2: allocate the section; set count, section */
     count=(high-low)+1;
-    if(unitIndex==0 || uniqueCount>=(3*count)/4) {
+    if(count<0x100 && (unitIndex==0 || uniqueCount>=(3*count)/4)) {
         /*
          * for the root table and for fairly full tables:
          * allocate for direct, linear array access
          * by keeping count, to write an entry for each unit value
          * from low to high
+         * exception: use a compact table if count==0x100 because
+         * that cannot be encoded in the length byte
          */
     } else {
         count=uniqueCount;
     }
 
+    if(count>=0x100) {
+        fprintf(stderr, "error: toUnicode extension table section overflow: %ld section entries\n", (long)count);
+        return FALSE;
+    }
+
     /* allocate the section: 1 entry for the header + count for the items */
     section=(uint32_t *)utm_allocN(extData->toUTable, 1+count);
 
@@ -563,6 +572,7 @@ makeToUTable(CnvExtData *extData, UCMTable *table) {
 /*
  * Remove toUnicode fallbacks and non-<subchar1> SUB mappings
  * which are irrelevant for the fromUnicode extension table.
+ * Remove MBCS_FROM_U_EXT_FLAG bits.
  * Overwrite the reverseMap with an index array to the relevant mappings.
  * Modify the code point sequences to a generator-friendly format where
  * the first code points remains unchanged but the following are recoded
@@ -589,7 +599,11 @@ prepareFromUMappings(UCMTable *table) {
 
     for(i=j=0; i<count; ++m, ++i) {
         flag=m->f;
-        if(flag==0 || flag==1 || (flag==2 && m->bLen==1)) {
+        if(flag>=0) {
+            flag&=MBCS_FROM_U_EXT_MASK;
+            m->f=flag;
+        }
+        if(flag==0 || flag==1 || (flag==2 && m->bLen==1) || flag==4) {
             map[j++]=i;
 
             if(m->uLen>1) {
@@ -655,6 +669,8 @@ getFromUBytesValue(CnvExtData *extData, UCMTable *table, UCMapping *m) {
     value|=(uint32_t)m->bLen<<UCNV_EXT_FROM_U_LENGTH_SHIFT;
     if(m->f==0) {
         value|=UCNV_EXT_FROM_U_ROUNDTRIP_FLAG;
+    } else if(m->f==4) {
+        value|=UCNV_EXT_FROM_U_GOOD_ONE_WAY_FLAG;
     }
 
     /* calculate the real UTF-16 length (see recoding in prepareFromUMappings()) */
@@ -830,7 +846,7 @@ addFromUTrieEntry(CnvExtData *extData, UChar32 c, uint32_t value) {
 
         extData->stage1[i1]=(uint16_t)newBlock;
         extData->stage2Top=newBlock+MBCS_STAGE_2_BLOCK_SIZE;
-        if(extData->stage2Top>LENGTHOF(extData->stage2)) {
+        if(extData->stage2Top>UPRV_LENGTHOF(extData->stage2)) {
             fprintf(stderr, "error: too many stage 2 entries at U+%04x\n", (int)c);
             exit(U_MEMORY_ALLOCATION_ERROR);
         }
@@ -852,7 +868,7 @@ addFromUTrieEntry(CnvExtData *extData, UChar32 c, uint32_t value) {
         extData->stage2[i2]=(uint16_t)(newBlock>>UCNV_EXT_STAGE_2_LEFT_SHIFT);
 
         extData->stage3Top=newBlock+MBCS_STAGE_3_BLOCK_SIZE;
-        if(extData->stage3Top>LENGTHOF(extData->stage3)) {
+        if(extData->stage3Top>UPRV_LENGTHOF(extData->stage3)) {
             fprintf(stderr, "error: too many stage 3 entries at U+%04x\n", (int)c);
             exit(U_MEMORY_ALLOCATION_ERROR);
         }
@@ -896,7 +912,7 @@ addFromUTrieEntry(CnvExtData *extData, UChar32 c, uint32_t value) {
             }
         }
     } else {
-        if((i3b=extData->stage3bTop++)>=LENGTHOF(extData->stage3b)) {
+        if((i3b=extData->stage3bTop++)>=UPRV_LENGTHOF(extData->stage3b)) {
             fprintf(stderr, "error: too many stage 3b entries at U+%04x\n", (int)c);
             exit(U_MEMORY_ALLOCATION_ERROR);
         }
@@ -1058,4 +1074,3 @@ CnvExtAddTable(NewConverter *cnvData, UCMTable *table, UConverterStaticData *sta
         makeToUTable(extData, table) &&
         makeFromUTable(extData, table);
 }
-