icuSources/tools/makeconv/genmbcs.c

   1 /*
   2 *******************************************************************************
   3 *
   4 *   Copyright (C) 2000-2006, International Business Machines
   5 *   Corporation and others.  All Rights Reserved.
   6 *
   7 *******************************************************************************
   8 *   file name:  genmbcs.c
   9 *   encoding:   US-ASCII
  10 *   tab size:   8 (not used)
  11 *   indentation:4
  12 *
  13 *   created on: 2000jul06
  14 *   created by: Markus W. Scherer
  15 */
  16
  17 #include <stdio.h>
  18 #include "unicode/utypes.h"
  19 #include "cstring.h"
  20 #include "cmemory.h"
  21 #include "unewdata.h"
  22 #include "ucnv_cnv.h"
  23 #include "ucnvmbcs.h"
  24 #include "ucm.h"
  25 #include "makeconv.h"
  26 #include "genmbcs.h"
  27
  28
  29 typedef struct MBCSData {
  30     NewConverter newConverter;
  31
  32     UCMFile *ucm;
  33
  34     /* toUnicode (state table in ucm->states) */
  35     _MBCSToUFallback toUFallbacks[MBCS_MAX_FALLBACK_COUNT];
  36     int32_t countToUFallbacks;
  37     uint16_t *unicodeCodeUnits;
  38
  39     /* fromUnicode */
  40     uint16_t stage1[MBCS_STAGE_1_SIZE];
  41     uint16_t stage2Single[MBCS_STAGE_2_SIZE]; /* stage 2 for single-byte codepages */
  42     uint32_t stage2[MBCS_STAGE_2_SIZE]; /* stage 2 for MBCS */
  43     uint8_t *fromUBytes;
  44     uint32_t stage2Top, stage3Top;
  45 } MBCSData;
  46
  47 /* prototypes */
  48 static void
  49 MBCSClose(NewConverter *cnvData);
  50
  51 static UBool
  52 MBCSStartMappings(MBCSData *mbcsData);
  53
  54 static UBool
  55 MBCSAddToUnicode(MBCSData *mbcsData,
  56                  const uint8_t *bytes, int32_t length,
  57                  UChar32 c,
  58                  int8_t flag);
  59
  60 static UBool
  61 MBCSIsValid(NewConverter *cnvData,
  62             const uint8_t *bytes, int32_t length);
  63
  64 static UBool
  65 MBCSSingleAddFromUnicode(MBCSData *mbcsData,
  66                          const uint8_t *bytes, int32_t length,
  67                          UChar32 c,
  68                          int8_t flag);
  69
  70 static UBool
  71 MBCSAddFromUnicode(MBCSData *mbcsData,
  72                    const uint8_t *bytes, int32_t length,
  73                    UChar32 c,
  74                    int8_t flag);
  75
  76 static void
  77 MBCSPostprocess(MBCSData *mbcsData, const UConverterStaticData *staticData);
  78
  79 static UBool
  80 MBCSAddTable(NewConverter *cnvData, UCMTable *table, UConverterStaticData *staticData);
  81
  82 static uint32_t
  83 MBCSWrite(NewConverter *cnvData, const UConverterStaticData *staticData,
  84           UNewDataMemory *pData, int32_t tableType);
  85
  86 /* helper ------------------------------------------------------------------- */
  87
  88 static U_INLINE char
  89 hexDigit(uint8_t digit) {
  90     return digit<=9 ? (char)('0'+digit) : (char)('a'-10+digit);
  91 }
  92
  93 static U_INLINE char *
  94 printBytes(char *buffer, const uint8_t *bytes, int32_t length) {
  95     char *s=buffer;
  96     while(length>0) {
  97         *s++=hexDigit((uint8_t)(*bytes>>4));
  98         *s++=hexDigit((uint8_t)(*bytes&0xf));
  99         ++bytes;
 100         --length;
 101     }
 102
 103     *s=0;
 104     return buffer;
 105 }
 106
 107 /* implementation ----------------------------------------------------------- */
 108
 109 static void
 110 MBCSInit(MBCSData *mbcsData, UCMFile *ucm) {
 111     int32_t i, maxCharLength;
 112
 113     uprv_memset(mbcsData, 0, sizeof(MBCSData));
 114
 115     maxCharLength=ucm->states.maxCharLength;
 116
 117     mbcsData->ucm=ucm; /* aliased, not owned */
 118
 119     mbcsData->newConverter.close=MBCSClose;
 120     mbcsData->newConverter.isValid=MBCSIsValid;
 121     mbcsData->newConverter.addTable=MBCSAddTable;
 122     mbcsData->newConverter.write=MBCSWrite;
 123
 124     mbcsData->stage2Top=MBCS_STAGE_2_FIRST_ASSIGNED; /* after stage 1 and one all-unassigned stage 2 block */
 125     mbcsData->stage3Top=16*maxCharLength; /* after one all-unassigned stage 3 block */
 126
 127     /* point all entries in stage 1 to the "all-unassigned" first block in stage 2 */
 128     for(i=0; i<MBCS_STAGE_1_SIZE; ++i) {
 129         mbcsData->stage1[i]=MBCS_STAGE_2_ALL_UNASSIGNED_INDEX;
 130     }
 131 }
 132
 133 NewConverter *
 134 MBCSOpen(UCMFile *ucm) {
 135     MBCSData *mbcsData=(MBCSData *)uprv_malloc(sizeof(MBCSData));
 136     if(mbcsData!=NULL) {
 137         MBCSInit(mbcsData, ucm);
 138     }
 139     return &mbcsData->newConverter;
 140 }
 141
 142 static void
 143 MBCSClose(NewConverter *cnvData) {
 144     MBCSData *mbcsData=(MBCSData *)cnvData;
 145     if(mbcsData!=NULL) {
 146         uprv_free(mbcsData->unicodeCodeUnits);
 147         uprv_free(mbcsData->fromUBytes);
 148         uprv_free(mbcsData);
 149     }
 150 }
 151
 152 static UBool
 153 MBCSStartMappings(MBCSData *mbcsData) {
 154     int32_t i, sum;
 155
 156     /* allocate the code unit array and prefill it with "unassigned" values */
 157     sum=mbcsData->ucm->states.countToUCodeUnits;
 158     if(VERBOSE) {
 159         printf("the total number of offsets is 0x%lx=%ld\n", (long)sum, (long)sum);
 160     }
 161
 162     if(sum>0) {
 163         mbcsData->unicodeCodeUnits=(uint16_t *)uprv_malloc(sum*sizeof(uint16_t));
 164         if(mbcsData->unicodeCodeUnits==NULL) {
 165             fprintf(stderr, "error: out of memory allocating %ld 16-bit code units\n",
 166                 (long)sum);
 167             return FALSE;
 168         }
 169         for(i=0; i<sum; ++i) {
 170             mbcsData->unicodeCodeUnits[i]=0xfffe;
 171         }
 172     }
 173
 174     /* allocate the codepage mappings and preset the first 16 characters to 0 */
 175     if(mbcsData->ucm->states.maxCharLength==1) {
 176         /* allocate 64k 16-bit results for single-byte codepages */
 177         sum=0x20000;
 178     } else {
 179         /* allocate 1M * maxCharLength bytes for at most 1M mappings */
 180         sum=0x100000*mbcsData->ucm->states.maxCharLength;
 181     }
 182     mbcsData->fromUBytes=(uint8_t *)uprv_malloc(sum);
 183     if(mbcsData->fromUBytes==NULL) {
 184         fprintf(stderr, "error: out of memory allocating %ld B for target mappings\n", (long)sum);
 185         return FALSE;
 186     }
 187     /* initialize the all-unassigned first stage 3 block */
 188     uprv_memset(mbcsData->fromUBytes, 0, 64);
 189
 190     return TRUE;
 191 }
 192
 193 /* return TRUE for success */
 194 static UBool
 195 setFallback(MBCSData *mbcsData, uint32_t offset, UChar32 c) {
 196     int32_t i=ucm_findFallback(mbcsData->toUFallbacks, mbcsData->countToUFallbacks, offset);
 197     if(i>=0) {
 198         /* if there is already a fallback for this offset, then overwrite it */
 199         mbcsData->toUFallbacks[i].codePoint=c;
 200         return TRUE;
 201     } else {
 202         /* if there is no fallback for this offset, then add one */
 203         i=mbcsData->countToUFallbacks;
 204         if(i>=MBCS_MAX_FALLBACK_COUNT) {
 205             fprintf(stderr, "error: too many toUnicode fallbacks, currently at: U+%x\n", (int)c);
 206             return FALSE;
 207         } else {
 208             mbcsData->toUFallbacks[i].offset=offset;
 209             mbcsData->toUFallbacks[i].codePoint=c;
 210             mbcsData->countToUFallbacks=i+1;
 211             return TRUE;
 212         }
 213     }
 214 }
 215
 216 /* remove fallback if there is one with this offset; return the code point if there was such a fallback, otherwise -1 */
 217 static int32_t
 218 removeFallback(MBCSData *mbcsData, uint32_t offset) {
 219     int32_t i=ucm_findFallback(mbcsData->toUFallbacks, mbcsData->countToUFallbacks, offset);
 220     if(i>=0) {
 221         _MBCSToUFallback *toUFallbacks;
 222         int32_t limit, old;
 223
 224         toUFallbacks=mbcsData->toUFallbacks;
 225         limit=mbcsData->countToUFallbacks;
 226         old=(int32_t)toUFallbacks[i].codePoint;
 227
 228         /* copy the last fallback entry here to keep the list contiguous */
 229         toUFallbacks[i].offset=toUFallbacks[limit-1].offset;
 230         toUFallbacks[i].codePoint=toUFallbacks[limit-1].codePoint;
 231         mbcsData->countToUFallbacks=limit-1;
 232         return old;
 233     } else {
 234         return -1;
 235     }
 236 }
 237
 238 /*
 239  * isFallback is almost a boolean:
 240  * 1 (TRUE)  this is a fallback mapping
 241  * 0 (FALSE) this is a precise mapping
 242  * -1        the precision of this mapping is not specified
 243  */
 244 static UBool
 245 MBCSAddToUnicode(MBCSData *mbcsData,
 246                  const uint8_t *bytes, int32_t length,
 247                  UChar32 c,
 248                  int8_t flag) {
 249     char buffer[10];
 250     uint32_t offset=0;
 251     int32_t i=0, entry, old;
 252     uint8_t state=0;
 253
 254     if(mbcsData->ucm->states.countStates==0) {
 255         fprintf(stderr, "error: there is no state information!\n");
 256         return FALSE;
 257     }
 258
 259     /* for SI/SO (like EBCDIC-stateful), double-byte sequences start in state 1 */
 260     if(length==2 && mbcsData->ucm->states.outputType==MBCS_OUTPUT_2_SISO) {
 261         state=1;
 262     }
 263
 264     /*
 265      * Walk down the state table like in conversion,
 266      * much like getNextUChar().
 267      * We assume that c<=0x10ffff.
 268      */
 269     for(i=0;;) {
 270         entry=mbcsData->ucm->states.stateTable[state][bytes[i++]];
 271         if(MBCS_ENTRY_IS_TRANSITION(entry)) {
 272             if(i==length) {
 273                 fprintf(stderr, "error: byte sequence too short, ends in non-final state %hu: 0x%s (U+%x)\n",
 274                     (short)state, printBytes(buffer, bytes, length), (int)c);
 275                 return FALSE;
 276             }
 277             state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry);
 278             offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry);
 279         } else {
 280             if(i<length) {
 281                 fprintf(stderr, "error: byte sequence too long by %d bytes, final state %hu: 0x%s (U+%x)\n",
 282                     (int)(length-i), state, printBytes(buffer, bytes, length), (int)c);
 283                 return FALSE;
 284             }
 285             switch(MBCS_ENTRY_FINAL_ACTION(entry)) {
 286             case MBCS_STATE_ILLEGAL:
 287                 fprintf(stderr, "error: byte sequence ends in illegal state at U+%04x<->0x%s\n",
 288                     (int)c, printBytes(buffer, bytes, length));
 289                 return FALSE;
 290             case MBCS_STATE_CHANGE_ONLY:
 291                 fprintf(stderr, "error: byte sequence ends in state-change-only at U+%04x<->0x%s\n",
 292                     (int)c, printBytes(buffer, bytes, length));
 293                 return FALSE;
 294             case MBCS_STATE_UNASSIGNED:
 295                 fprintf(stderr, "error: byte sequence ends in unassigned state at U+%04x<->0x%s\n",
 296                     (int)c, printBytes(buffer, bytes, length));
 297                 return FALSE;
 298             case MBCS_STATE_FALLBACK_DIRECT_16:
 299             case MBCS_STATE_VALID_DIRECT_16:
 300             case MBCS_STATE_FALLBACK_DIRECT_20:
 301             case MBCS_STATE_VALID_DIRECT_20:
 302                 if(MBCS_ENTRY_SET_STATE(entry, 0)!=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, 0xfffe)) {
 303                     /* the "direct" action's value is not "valid-direct-16-unassigned" any more */
 304                     if(MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_DIRECT_16 || MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_FALLBACK_DIRECT_16) {
 305                         old=MBCS_ENTRY_FINAL_VALUE(entry);
 306                     } else {
 307                         old=0x10000+MBCS_ENTRY_FINAL_VALUE(entry);
 308                     }
 309                     if(flag>=0) {
 310                         fprintf(stderr, "error: duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n",
 311                             (int)c, printBytes(buffer, bytes, length), (int)old);
 312                         return FALSE;
 313                     } else if(VERBOSE) {
 314                         fprintf(stderr, "duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n",
 315                             (int)c, printBytes(buffer, bytes, length), (int)old);
 316                     }
 317                     /*
 318                      * Continue after the above warning
 319                      * if the precision of the mapping is unspecified.
 320                      */
 321                 }
 322                 /* reassign the correct action code */
 323                 entry=MBCS_ENTRY_FINAL_SET_ACTION(entry, (MBCS_STATE_VALID_DIRECT_16+(flag==3 ? 2 : 0)+(c>=0x10000 ? 1 : 0)));
 324
 325                 /* put the code point into bits 22..7 for BMP, c-0x10000 into 26..7 for others */
 326                 if(c<=0xffff) {
 327                     entry=MBCS_ENTRY_FINAL_SET_VALUE(entry, c);
 328                 } else {
 329                     entry=MBCS_ENTRY_FINAL_SET_VALUE(entry, c-0x10000);
 330                 }
 331                 mbcsData->ucm->states.stateTable[state][bytes[i-1]]=entry;
 332                 break;
 333             case MBCS_STATE_VALID_16:
 334                 /* bits 26..16 are not used, 0 */
 335                 /* bits 15..7 contain the final offset delta to one 16-bit code unit */
 336                 offset+=MBCS_ENTRY_FINAL_VALUE_16(entry);
 337                 /* check that this byte sequence is still unassigned */
 338                 if((old=mbcsData->unicodeCodeUnits[offset])!=0xfffe || (old=removeFallback(mbcsData, offset))!=-1) {
 339                     if(flag>=0) {
 340                         fprintf(stderr, "error: duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n",
 341                             (int)c, printBytes(buffer, bytes, length), (int)old);
 342                         return FALSE;
 343                     } else if(VERBOSE) {
 344                         fprintf(stderr, "duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n",
 345                             (int)c, printBytes(buffer, bytes, length), (int)old);
 346                     }
 347                 }
 348                 if(c>=0x10000) {
 349                     fprintf(stderr, "error: code point does not fit into valid-16-bit state at U+%04x<->0x%s\n",
 350                         (int)c, printBytes(buffer, bytes, length));
 351                     return FALSE;
 352                 }
 353                 if(flag>0) {
 354                     /* assign only if there is no precise mapping */
 355                     if(mbcsData->unicodeCodeUnits[offset]==0xfffe) {
 356                         return setFallback(mbcsData, offset, c);
 357                     }
 358                 } else {
 359                     mbcsData->unicodeCodeUnits[offset]=(uint16_t)c;
 360                 }
 361                 break;
 362             case MBCS_STATE_VALID_16_PAIR:
 363                 /* bits 26..16 are not used, 0 */
 364                 /* bits 15..7 contain the final offset delta to two 16-bit code units */
 365                 offset+=MBCS_ENTRY_FINAL_VALUE_16(entry);
 366                 /* check that this byte sequence is still unassigned */
 367                 old=mbcsData->unicodeCodeUnits[offset];
 368                 if(old<0xfffe) {
 369                     int32_t real;
 370                     if(old<0xd800) {
 371                         real=old;
 372                     } else if(old<=0xdfff) {
 373                         real=0x10000+((old&0x3ff)<<10)+((mbcsData->unicodeCodeUnits[offset+1])&0x3ff);
 374                     } else /* old<=0xe001 */ {
 375                         real=mbcsData->unicodeCodeUnits[offset+1];
 376                     }
 377                     if(flag>=0) {
 378                         fprintf(stderr, "error: duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n",
 379                             (int)c, printBytes(buffer, bytes, length), (int)real);
 380                         return FALSE;
 381                     } else if(VERBOSE) {
 382                         fprintf(stderr, "duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n",
 383                             (int)c, printBytes(buffer, bytes, length), (int)real);
 384                     }
 385                 }
 386                 if(flag>0) {
 387                     /* assign only if there is no precise mapping */
 388                     if(old<=0xdbff || old==0xe000) {
 389                         /* do nothing */
 390                     } else if(c<=0xffff) {
 391                         /* set a BMP fallback code point as a pair with 0xe001 */
 392                         mbcsData->unicodeCodeUnits[offset++]=0xe001;
 393                         mbcsData->unicodeCodeUnits[offset]=(uint16_t)c;
 394                     } else {
 395                         /* set a fallback surrogate pair with two second surrogates */
 396                         mbcsData->unicodeCodeUnits[offset++]=(uint16_t)(0xdbc0+(c>>10));
 397                         mbcsData->unicodeCodeUnits[offset]=(uint16_t)(0xdc00+(c&0x3ff));
 398                     }
 399                 } else {
 400                     if(c<0xd800) {
 401                         /* set a BMP code point */
 402                         mbcsData->unicodeCodeUnits[offset]=(uint16_t)c;
 403                     } else if(c<=0xffff) {
 404                         /* set a BMP code point above 0xd800 as a pair with 0xe000 */
 405                         mbcsData->unicodeCodeUnits[offset++]=0xe000;
 406                         mbcsData->unicodeCodeUnits[offset]=(uint16_t)c;
 407                     } else {
 408                         /* set a surrogate pair */
 409                         mbcsData->unicodeCodeUnits[offset++]=(uint16_t)(0xd7c0+(c>>10));
 410                         mbcsData->unicodeCodeUnits[offset]=(uint16_t)(0xdc00+(c&0x3ff));
 411                     }
 412                 }
 413                 break;
 414             default:
 415                 /* reserved, must never occur */
 416                 fprintf(stderr, "internal error: byte sequence reached reserved action code, entry 0x%02x: 0x%s (U+%x)\n",
 417                     (int)entry, printBytes(buffer, bytes, length), (int)c);
 418                 return FALSE;
 419             }
 420
 421             return TRUE;
 422         }
 423     }
 424 }
 425
 426 /* is this byte sequence valid? (this is almost the same as MBCSAddToUnicode()) */
 427 static UBool
 428 MBCSIsValid(NewConverter *cnvData,
 429             const uint8_t *bytes, int32_t length) {
 430     MBCSData *mbcsData=(MBCSData *)cnvData;
 431
 432     return (UBool)(1==ucm_countChars(&mbcsData->ucm->states, bytes, length));
 433 }
 434
 435 static UBool
 436 MBCSSingleAddFromUnicode(MBCSData *mbcsData,
 437                          const uint8_t *bytes, int32_t length,
 438                          UChar32 c,
 439                          int8_t flag) {
 440     uint16_t *p;
 441     uint32_t index;
 442     uint16_t old;
 443     uint8_t b;
 444
 445     /* ignore |2 SUB mappings */
 446     if(flag==2) {
 447         return TRUE;
 448     }
 449
 450     /*
 451      * Walk down the triple-stage compact array ("trie") and
 452      * allocate parts as necessary.
 453      * Note that the first stage 2 and 3 blocks are reserved for all-unassigned mappings.
 454      * We assume that length<=maxCharLength and that c<=0x10ffff.
 455      */
 456     b=*bytes;
 457
 458     /* inspect stage 1 */
 459     index=c>>10;
 460     if(mbcsData->stage1[index]==MBCS_STAGE_2_ALL_UNASSIGNED_INDEX) {
 461         /* allocate another block in stage 2 */
 462         if(mbcsData->stage2Top>=MBCS_MAX_STAGE_2_TOP) {
 463             fprintf(stderr, "error: too many stage 2 entries at U+%04x<->0x%02x\n", (int)c, b);
 464             return FALSE;
 465         }
 466
 467         /*
 468          * each stage 2 block contains 64 16-bit words:
 469          * 6 code point bits 9..4 with 1 stage 3 index
 470          */
 471         mbcsData->stage1[index]=(uint16_t)mbcsData->stage2Top;
 472         mbcsData->stage2Top+=MBCS_STAGE_2_BLOCK_SIZE;
 473     }
 474
 475     /* inspect stage 2 */
 476     index=(uint32_t)mbcsData->stage1[index]+((c>>4)&0x3f);
 477     if(mbcsData->stage2Single[index]==0) {
 478         /* allocate another block in stage 3 */
 479         if(mbcsData->stage3Top>=0x10000) {
 480             fprintf(stderr, "error: too many code points at U+%04x<->0x%02x\n", (int)c, b);
 481             return FALSE;
 482         }
 483         /* each block has 16 uint16_t entries */
 484         mbcsData->stage2Single[index]=(uint16_t)mbcsData->stage3Top;
 485         uprv_memset(mbcsData->fromUBytes+2*mbcsData->stage3Top, 0, 32);
 486         mbcsData->stage3Top+=16;
 487     }
 488
 489     /* write the codepage entry into stage 3 and get the previous entry */
 490     p=(uint16_t *)mbcsData->fromUBytes+mbcsData->stage2Single[index]+(c&0xf);
 491     old=*p;
 492     if(flag<=0) {
 493         *p=(uint16_t)(0xf00|b);
 494     } else if(IS_PRIVATE_USE(c)) {
 495         *p=(uint16_t)(0xc00|b);
 496     } else {
 497         *p=(uint16_t)(0x800|b);
 498     }
 499
 500     /* check that this Unicode code point was still unassigned */
 501     if(old>=0x100) {
 502         if(flag>=0) {
 503             fprintf(stderr, "error: duplicate Unicode code point at U+%04x<->0x%02x see 0x%02x\n",
 504                 (int)c, b, old&0xff);
 505             return FALSE;
 506         } else if(VERBOSE) {
 507             fprintf(stderr, "duplicate Unicode code point at U+%04x<->0x%02x see 0x%02x\n",
 508                 (int)c, b, old&0xff);
 509         }
 510         /* continue after the above warning if the precision of the mapping is unspecified */
 511     }
 512
 513     return TRUE;
 514 }
 515
 516 static UBool
 517 MBCSAddFromUnicode(MBCSData *mbcsData,
 518                    const uint8_t *bytes, int32_t length,
 519                    UChar32 c,
 520                    int8_t flag) {
 521     char buffer[10];
 522     const uint8_t *pb;
 523     uint8_t *p;
 524     uint32_t index, b, old;
 525     int32_t maxCharLength;
 526
 527     /* ignore |2 SUB mappings */
 528     if(flag==2) {
 529         return TRUE;
 530     }
 531
 532     maxCharLength=mbcsData->ucm->states.maxCharLength;
 533
 534     if(maxCharLength==1) {
 535         return MBCSSingleAddFromUnicode(mbcsData, bytes, length, c, flag);
 536     }
 537
 538     if( mbcsData->ucm->states.outputType==MBCS_OUTPUT_2_SISO &&
 539         (*bytes==0xe || *bytes==0xf)
 540     ) {
 541         fprintf(stderr, "error: illegal mapping to SI or SO for SI/SO codepage: U+%04x<->0x%s\n",
 542             (int)c, printBytes(buffer, bytes, length));
 543         return FALSE;
 544     }
 545
 546     if(flag==1 && length==1 && *bytes==0) {
 547         fprintf(stderr, "error: unable to encode a |1 fallback from U+%04x to 0x%02x\n",
 548             (int)c, *bytes);
 549         return FALSE;
 550     }
 551
 552     /*
 553      * Walk down the triple-stage compact array ("trie") and
 554      * allocate parts as necessary.
 555      * Note that the first stage 2 and 3 blocks are reserved for
 556      * all-unassigned mappings.
 557      * We assume that length<=maxCharLength and that c<=0x10ffff.
 558      */
 559
 560     /* inspect stage 1 */
 561     index=c>>10;
 562     if(mbcsData->stage1[index]==MBCS_STAGE_2_ALL_UNASSIGNED_INDEX) {
 563         /* allocate another block in stage 2 */
 564         if(mbcsData->stage2Top>=MBCS_MAX_STAGE_2_TOP) {
 565             fprintf(stderr, "error: too many stage 2 entries at U+%04x<->0x%s\n",
 566                 (int)c, printBytes(buffer, bytes, length));
 567             return FALSE;
 568         }
 569
 570         /*
 571          * each stage 2 block contains 64 32-bit words:
 572          * 6 code point bits 9..4 with value with bits 31..16 "assigned" flags and bits 15..0 stage 3 index
 573          */
 574         mbcsData->stage1[index]=(uint16_t)mbcsData->stage2Top;
 575         mbcsData->stage2Top+=MBCS_STAGE_2_BLOCK_SIZE;
 576     }
 577
 578     /* inspect stage 2 */
 579     index=mbcsData->stage1[index]+((c>>4)&0x3f);
 580     if(mbcsData->stage2[index]==0) {
 581         /* allocate another block in stage 3 */
 582         if(mbcsData->stage3Top>=0x100000*(uint32_t)maxCharLength) {
 583             fprintf(stderr, "error: too many code points at U+%04x<->0x%s\n",
 584                 (int)c, printBytes(buffer, bytes, length));
 585             return FALSE;
 586         }
 587         /* each block has 16*maxCharLength bytes */
 588         mbcsData->stage2[index]=(mbcsData->stage3Top/16)/maxCharLength;
 589         uprv_memset(mbcsData->fromUBytes+mbcsData->stage3Top, 0, 16*maxCharLength);
 590         mbcsData->stage3Top+=16*maxCharLength;
 591     }
 592
 593     /* write the codepage bytes into stage 3 and get the previous bytes */
 594
 595     /* assemble the bytes into a single integer */
 596     pb=bytes;
 597     b=0;
 598     switch(length) {
 599     case 4:
 600         b=*pb++;
 601     case 3:
 602         b=(b<<8)|*pb++;
 603     case 2:
 604         b=(b<<8)|*pb++;
 605     case 1:
 606     default:
 607         b=(b<<8)|*pb++;
 608         break;
 609     }
 610
 611     old=0;
 612     p=mbcsData->fromUBytes+(16*(uint32_t)(uint16_t)mbcsData->stage2[index]+(c&0xf))*maxCharLength;
 613     switch(maxCharLength) {
 614     case 2:
 615         old=*(uint16_t *)p;
 616         *(uint16_t *)p=(uint16_t)b;
 617         break;
 618     case 3:
 619         old=(uint32_t)*p<<16;
 620         *p++=(uint8_t)(b>>16);
 621         old|=(uint32_t)*p<<8;
 622         *p++=(uint8_t)(b>>8);
 623         old|=*p;
 624         *p=(uint8_t)b;
 625         break;
 626     case 4:
 627         old=*(uint32_t *)p;
 628         *(uint32_t *)p=b;
 629         break;
 630     default:
 631         /* will never occur */
 632         break;
 633     }
 634
 635     /* check that this Unicode code point was still unassigned */
 636     if((mbcsData->stage2[index]&(1UL<<(16+(c&0xf))))!=0 || old!=0) {
 637         if(flag>=0) {
 638             fprintf(stderr, "error: duplicate Unicode code point at U+%04x<->0x%s see 0x%02x\n",
 639                 (int)c, printBytes(buffer, bytes, length), (int)old);
 640             return FALSE;
 641         } else if(VERBOSE) {
 642             fprintf(stderr, "duplicate Unicode code point at U+%04x<->0x%s see 0x%02x\n",
 643                 (int)c, printBytes(buffer, bytes, length), (int)old);
 644         }
 645         /* continue after the above warning if the precision of the mapping is
 646            unspecified */
 647     }
 648     if(flag<=0) {
 649         /* set the roundtrip flag */
 650         mbcsData->stage2[index]|=(1UL<<(16+(c&0xf)));
 651     }
 652
 653     return TRUE;
 654 }
 655
 656 /* we can assume that the table only contains 1:1 mappings with <=4 bytes each */
 657 static UBool
 658 MBCSAddTable(NewConverter *cnvData, UCMTable *table, UConverterStaticData *staticData) {
 659     MBCSData *mbcsData;
 660     UCMapping *m;
 661     UChar32 c;
 662     int32_t i;
 663     UBool isOK;
 664
 665     staticData->unicodeMask=table->unicodeMask;
 666     if(staticData->unicodeMask==3) {
 667         fprintf(stderr, "error: contains mappings for both supplementary and surrogate code points\n");
 668         return FALSE;
 669     }
 670
 671     staticData->conversionType=UCNV_MBCS;
 672
 673     mbcsData=(MBCSData *)cnvData;
 674
 675     if(!MBCSStartMappings(mbcsData)) {
 676         return FALSE;
 677     }
 678
 679     isOK=TRUE;
 680
 681     m=table->mappings;
 682     for(i=0; i<table->mappingsLength; ++m, ++i) {
 683         c=m->u;
 684
 685         switch(m->f) {
 686         case -1:
 687             /* there was no precision/fallback indicator */
 688             /* fall through to set the mappings */
 689         case 0:
 690             /* set roundtrip mappings */
 691             isOK&=MBCSAddToUnicode(mbcsData, m->b.bytes, m->bLen, c, m->f) &&
 692                   MBCSAddFromUnicode(mbcsData, m->b.bytes, m->bLen, c, m->f);
 693             break;
 694         case 1:
 695             /* set only a fallback mapping from Unicode to codepage */
 696             staticData->hasFromUnicodeFallback=TRUE;
 697             isOK&=MBCSAddFromUnicode(mbcsData, m->b.bytes, m->bLen, c, m->f);
 698             break;
 699         case 2:
 700             /* ignore |2 SUB mappings */
 701             break;
 702         case 3:
 703             /* set only a fallback mapping from codepage to Unicode */
 704             staticData->hasToUnicodeFallback=TRUE;
 705             isOK&=MBCSAddToUnicode(mbcsData, m->b.bytes, m->bLen, c, m->f);
 706             break;
 707         default:
 708             /* will not occur because the parser checked it already */
 709             fprintf(stderr, "error: illegal fallback indicator %d\n", m->f);
 710             return FALSE;
 711         }
 712     }
 713
 714     MBCSPostprocess(mbcsData, staticData);
 715
 716     return isOK;
 717 }
 718
 719 static UBool
 720 transformEUC(MBCSData *mbcsData) {
 721     uint8_t *p8;
 722     uint32_t i, value, oldLength, old3Top, new3Top;
 723     uint8_t b;
 724
 725     oldLength=mbcsData->ucm->states.maxCharLength;
 726     if(oldLength<3) {
 727         return FALSE;
 728     }
 729
 730     old3Top=mbcsData->stage3Top;
 731
 732     /* careful: 2-byte and 4-byte codes are stored in platform endianness! */
 733
 734     /* test if all first bytes are in {0, 0x8e, 0x8f} */
 735     p8=mbcsData->fromUBytes;
 736
 737 #if !U_IS_BIG_ENDIAN
 738     if(oldLength==4) {
 739         p8+=3;
 740     }
 741 #endif
 742
 743     for(i=0; i<old3Top; i+=oldLength) {
 744         b=p8[i];
 745         if(b!=0 && b!=0x8e && b!=0x8f) {
 746             /* some first byte does not fit the EUC pattern, nothing to be done */
 747             return FALSE;
 748         }
 749     }
 750     /* restore p if it was modified above */
 751     p8=mbcsData->fromUBytes;
 752
 753     /* modify outputType and adjust stage3Top */
 754     mbcsData->ucm->states.outputType=(int8_t)(MBCS_OUTPUT_3_EUC+oldLength-3);
 755     mbcsData->stage3Top=new3Top=(old3Top*(oldLength-1))/oldLength;
 756
 757     /*
 758      * EUC-encode all byte sequences;
 759      * see "CJKV Information Processing" (1st ed. 1999) from Ken Lunde, O'Reilly,
 760      * p. 161 in chapter 4 "Encoding Methods"
 761      *
 762      * This also must reverse the byte order if the platform is little-endian!
 763      */
 764     if(oldLength==3) {
 765         uint16_t *q=(uint16_t *)p8;
 766         for(i=0; i<old3Top; i+=oldLength) {
 767             b=*p8;
 768             if(b==0) {
 769                 /* short sequences are stored directly */
 770                 /* code set 0 or 1 */
 771                 (*q++)=(uint16_t)((p8[1]<<8)|p8[2]);
 772             } else if(b==0x8e) {
 773                 /* code set 2 */
 774                 (*q++)=(uint16_t)(((p8[1]&0x7f)<<8)|p8[2]);
 775             } else /* b==0x8f */ {
 776                 /* code set 3 */
 777                 (*q++)=(uint16_t)((p8[1]<<8)|(p8[2]&0x7f));
 778             }
 779             p8+=3;
 780         }
 781     } else /* oldLength==4 */ {
 782         uint8_t *q=p8;
 783         uint32_t *p32=(uint32_t *)p8;
 784         for(i=0; i<old3Top; i+=4) {
 785             value=(*p32++);
 786             if(value<=0xffffff) {
 787                 /* short sequences are stored directly */
 788                 /* code set 0 or 1 */
 789                 (*q++)=(uint8_t)(value>>16);
 790                 (*q++)=(uint8_t)(value>>8);
 791                 (*q++)=(uint8_t)value;
 792             } else if(value<=0x8effffff) {
 793                 /* code set 2 */
 794                 (*q++)=(uint8_t)((value>>16)&0x7f);
 795                 (*q++)=(uint8_t)(value>>8);
 796                 (*q++)=(uint8_t)value;
 797             } else /* first byte is 0x8f */ {
 798                 /* code set 3 */
 799                 (*q++)=(uint8_t)(value>>16);
 800                 (*q++)=(uint8_t)((value>>8)&0x7f);
 801                 (*q++)=(uint8_t)value;
 802             }
 803         }
 804     }
 805
 806     return TRUE;
 807 }
 808
 809 /*
 810  * Compact stage 2 for SBCS by overlapping adjacent stage 2 blocks as far
 811  * as possible. Overlapping is done on unassigned head and tail
 812  * parts of blocks in steps of MBCS_STAGE_2_MULTIPLIER.
 813  * Stage 1 indexes need to be adjusted accordingly.
 814  * This function is very similar to genprops/store.c/compactStage().
 815  */
 816 static void
 817 singleCompactStage2(MBCSData *mbcsData) {
 818     /* this array maps the ordinal number of a stage 2 block to its new stage 1 index */
 819     uint16_t map[MBCS_STAGE_2_MAX_BLOCKS];
 820     uint16_t i, start, prevEnd, newStart;
 821
 822     /* enter the all-unassigned first stage 2 block into the map */
 823     map[0]=MBCS_STAGE_2_ALL_UNASSIGNED_INDEX;
 824
 825     /* begin with the first block after the all-unassigned one */
 826     start=newStart=MBCS_STAGE_2_FIRST_ASSIGNED;
 827     while(start<mbcsData->stage2Top) {
 828         prevEnd=(uint16_t)(newStart-1);
 829
 830         /* find the size of the overlap */
 831         for(i=0; i<MBCS_STAGE_2_BLOCK_SIZE && mbcsData->stage2Single[start+i]==0 && mbcsData->stage2Single[prevEnd-i]==0; ++i) {}
 832
 833         if(i>0) {
 834             map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=(uint16_t)(newStart-i);
 835
 836             /* move the non-overlapping indexes to their new positions */
 837             start+=i;
 838             for(i=(uint16_t)(MBCS_STAGE_2_BLOCK_SIZE-i); i>0; --i) {
 839                 mbcsData->stage2Single[newStart++]=mbcsData->stage2Single[start++];
 840             }
 841         } else if(newStart<start) {
 842             /* move the indexes to their new positions */
 843             map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=newStart;
 844             for(i=MBCS_STAGE_2_BLOCK_SIZE; i>0; --i) {
 845                 mbcsData->stage2Single[newStart++]=mbcsData->stage2Single[start++];
 846             }
 847         } else /* no overlap && newStart==start */ {
 848             map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=start;
 849             start=newStart+=MBCS_STAGE_2_BLOCK_SIZE;
 850         }
 851     }
 852
 853     /* adjust stage2Top */
 854     if(VERBOSE && newStart<mbcsData->stage2Top) {
 855         printf("compacting stage 2 from stage2Top=0x%lx to 0x%lx, saving %ld bytes\n",
 856                 (unsigned long)mbcsData->stage2Top, (unsigned long)newStart,
 857                 (long)(mbcsData->stage2Top-newStart)*2);
 858     }
 859     mbcsData->stage2Top=newStart;
 860
 861     /* now adjust stage 1 */
 862     for(i=0; i<MBCS_STAGE_1_SIZE; ++i) {
 863         mbcsData->stage1[i]=map[mbcsData->stage1[i]>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT];
 864     }
 865 }
 866
 867 /* Compact stage 3 for SBCS - same algorithm as above. */
 868 static void
 869 singleCompactStage3(MBCSData *mbcsData) {
 870     uint16_t *stage3=(uint16_t *)mbcsData->fromUBytes;
 871
 872     /* this array maps the ordinal number of a stage 3 block to its new stage 2 index */
 873     uint16_t map[0x1000];
 874     uint16_t i, start, prevEnd, newStart;
 875
 876     /* enter the all-unassigned first stage 3 block into the map */
 877     map[0]=0;
 878
 879     /* begin with the first block after the all-unassigned one */
 880     start=newStart=16;
 881     while(start<mbcsData->stage3Top) {
 882         prevEnd=(uint16_t)(newStart-1);
 883
 884         /* find the size of the overlap */
 885         for(i=0; i<16 && stage3[start+i]==0 && stage3[prevEnd-i]==0; ++i) {}
 886
 887         if(i>0) {
 888             map[start>>4]=(uint16_t)(newStart-i);
 889
 890             /* move the non-overlapping indexes to their new positions */
 891             start+=i;
 892             for(i=(uint16_t)(16-i); i>0; --i) {
 893                 stage3[newStart++]=stage3[start++];
 894             }
 895         } else if(newStart<start) {
 896             /* move the indexes to their new positions */
 897             map[start>>4]=newStart;
 898             for(i=16; i>0; --i) {
 899                 stage3[newStart++]=stage3[start++];
 900             }
 901         } else /* no overlap && newStart==start */ {
 902             map[start>>4]=start;
 903             start=newStart+=16;
 904         }
 905     }
 906
 907     /* adjust stage3Top */
 908     if(VERBOSE && newStart<mbcsData->stage3Top) {
 909         printf("compacting stage 3 from stage3Top=0x%lx to 0x%lx, saving %ld bytes\n",
 910                 (unsigned long)mbcsData->stage3Top, (unsigned long)newStart,
 911                 (long)(mbcsData->stage3Top-newStart)*2);
 912     }
 913     mbcsData->stage3Top=newStart;
 914
 915     /* now adjust stage 2 */
 916     for(i=0; i<mbcsData->stage2Top; ++i) {
 917         mbcsData->stage2Single[i]=map[mbcsData->stage2Single[i]>>4];
 918     }
 919 }
 920
 921 /*
 922  * Compact stage 2 by overlapping adjacent stage 2 blocks as far
 923  * as possible. Overlapping is done on unassigned head and tail
 924  * parts of blocks in steps of MBCS_STAGE_2_MULTIPLIER.
 925  * Stage 1 indexes need to be adjusted accordingly.
 926  * This function is very similar to genprops/store.c/compactStage().
 927  */
 928 static void
 929 compactStage2(MBCSData *mbcsData) {
 930     /* this array maps the ordinal number of a stage 2 block to its new stage 1 index */
 931     uint16_t map[MBCS_STAGE_2_MAX_BLOCKS];
 932     uint16_t i, start, prevEnd, newStart;
 933
 934     /* enter the all-unassigned first stage 2 block into the map */
 935     map[0]=MBCS_STAGE_2_ALL_UNASSIGNED_INDEX;
 936
 937     /* begin with the first block after the all-unassigned one */
 938     start=newStart=MBCS_STAGE_2_FIRST_ASSIGNED;
 939     while(start<mbcsData->stage2Top) {
 940         prevEnd=(uint16_t)(newStart-1);
 941
 942         /* find the size of the overlap */
 943         for(i=0; i<MBCS_STAGE_2_BLOCK_SIZE && mbcsData->stage2[start+i]==0 && mbcsData->stage2[prevEnd-i]==0; ++i) {}
 944
 945         if(i>0) {
 946             map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=(uint16_t)(newStart-i);
 947
 948             /* move the non-overlapping indexes to their new positions */
 949             start+=i;
 950             for(i=(uint16_t)(MBCS_STAGE_2_BLOCK_SIZE-i); i>0; --i) {
 951                 mbcsData->stage2[newStart++]=mbcsData->stage2[start++];
 952             }
 953         } else if(newStart<start) {
 954             /* move the indexes to their new positions */
 955             map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=newStart;
 956             for(i=MBCS_STAGE_2_BLOCK_SIZE; i>0; --i) {
 957                 mbcsData->stage2[newStart++]=mbcsData->stage2[start++];
 958             }
 959         } else /* no overlap && newStart==start */ {
 960             map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=start;
 961             start=newStart+=MBCS_STAGE_2_BLOCK_SIZE;
 962         }
 963     }
 964
 965     /* adjust stage2Top */
 966     if(VERBOSE && newStart<mbcsData->stage2Top) {
 967         printf("compacting stage 2 from stage2Top=0x%lx to 0x%lx, saving %ld bytes\n",
 968                 (unsigned long)mbcsData->stage2Top, (unsigned long)newStart,
 969                 (long)(mbcsData->stage2Top-newStart)*4);
 970     }
 971     mbcsData->stage2Top=newStart;
 972
 973     /* now adjust stage 1 */
 974     for(i=0; i<MBCS_STAGE_1_SIZE; ++i) {
 975         mbcsData->stage1[i]=map[mbcsData->stage1[i]>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT];
 976     }
 977 }
 978
 979 static void
 980 MBCSPostprocess(MBCSData *mbcsData, const UConverterStaticData *staticData) {
 981     UCMStates *states;
 982     int32_t maxCharLength;
 983
 984     states=&mbcsData->ucm->states;
 985     maxCharLength=states->maxCharLength;
 986
 987     /* this needs to be printed before the EUC transformation because later maxCharLength might not be correct */
 988     if(VERBOSE) {
 989         printf("number of codepage characters in 16-blocks: 0x%lx=%lu\n",
 990                (unsigned long)mbcsData->stage3Top/maxCharLength,
 991                (unsigned long)mbcsData->stage3Top/maxCharLength);
 992     }
 993
 994     ucm_optimizeStates(states,
 995                        &mbcsData->unicodeCodeUnits,
 996                        mbcsData->toUFallbacks, mbcsData->countToUFallbacks,
 997                        VERBOSE);
 998
 999     /* try to compact the fromUnicode tables */
1000     transformEUC(mbcsData);
1001     if(maxCharLength==1) {
1002         singleCompactStage3(mbcsData);
1003         singleCompactStage2(mbcsData);
1004     } else {
1005         compactStage2(mbcsData);
1006     }
1007 }
1008
1009 static uint32_t
1010 MBCSWrite(NewConverter *cnvData, const UConverterStaticData *staticData,
1011           UNewDataMemory *pData, int32_t tableType) {
1012     MBCSData *mbcsData=(MBCSData *)cnvData;
1013     uint32_t top;
1014     int32_t i, stage1Top;
1015
1016     _MBCSHeader header={ { 0, 0, 0, 0 }, 0, 0, 0, 0, 0, 0, 0 };
1017
1018     /* adjust stage 1 entries to include the size of stage 1 in the offsets to stage 2 */
1019     if(mbcsData->ucm->states.maxCharLength==1) {
1020         if(staticData->unicodeMask&UCNV_HAS_SUPPLEMENTARY) {
1021             stage1Top=MBCS_STAGE_1_SIZE; /* 0x440==1088 */
1022         } else {
1023             stage1Top=0x40; /* 0x40==64 */
1024         }
1025         for(i=0; i<stage1Top; ++i) {
1026             mbcsData->stage1[i]+=(uint16_t)stage1Top;
1027         }
1028
1029         /* stage2Top has counted 16-bit results, now we need to count bytes */
1030         mbcsData->stage2Top*=2;
1031
1032         /* stage3Top has counted 16-bit results, now we need to count bytes */
1033         mbcsData->stage3Top*=2;
1034     } else {
1035         if(staticData->unicodeMask&UCNV_HAS_SUPPLEMENTARY) {
1036             stage1Top=MBCS_STAGE_1_SIZE; /* 0x440==1088 */
1037         } else {
1038             stage1Top=0x40; /* 0x40==64 */
1039         }
1040         for(i=0; i<stage1Top; ++i) {
1041             mbcsData->stage1[i]+=(uint16_t)stage1Top/2; /* stage 2 contains 32-bit entries, stage 1 16-bit entries */
1042         }
1043
1044         /* stage2Top has counted 32-bit results, now we need to count bytes */
1045         mbcsData->stage2Top*=4;
1046
1047         /* stage3Top has already counted bytes */
1048     }
1049
1050     /* round up stage2Top and stage3Top so that the sizes of all data blocks are multiples of 4 */
1051     mbcsData->stage2Top=(mbcsData->stage2Top+3)&~3;
1052     mbcsData->stage3Top=(mbcsData->stage3Top+3)&~3;
1053
1054     /* fill the header */
1055     header.version[0]=4;
1056     header.version[1]=2;
1057     header.countStates=mbcsData->ucm->states.countStates;
1058     header.countToUFallbacks=mbcsData->countToUFallbacks;
1059
1060     header.offsetToUCodeUnits=
1061         sizeof(_MBCSHeader)+
1062         mbcsData->ucm->states.countStates*1024+
1063         mbcsData->countToUFallbacks*sizeof(_MBCSToUFallback);
1064     header.offsetFromUTable=
1065         header.offsetToUCodeUnits+
1066         mbcsData->ucm->states.countToUCodeUnits*2;
1067     header.offsetFromUBytes=
1068         header.offsetFromUTable+
1069         stage1Top*2+
1070         mbcsData->stage2Top;
1071     header.fromUBytesLength=mbcsData->stage3Top;
1072
1073     top=header.offsetFromUBytes+header.fromUBytesLength;
1074
1075     header.flags=(uint8_t)(mbcsData->ucm->states.outputType);
1076
1077     if(tableType&TABLE_EXT) {
1078         if(top>0xffffff) {
1079             fprintf(stderr, "error: offset 0x%lx to extension table exceeds 0xffffff\n", (long)top);
1080             return 0;
1081         }
1082
1083         header.flags|=top<<8;
1084     }
1085
1086     /* write the MBCS data */
1087     udata_writeBlock(pData, &header, sizeof(_MBCSHeader));
1088     udata_writeBlock(pData, mbcsData->ucm->states.stateTable, header.countStates*1024);
1089     udata_writeBlock(pData, mbcsData->toUFallbacks, mbcsData->countToUFallbacks*sizeof(_MBCSToUFallback));
1090     udata_writeBlock(pData, mbcsData->unicodeCodeUnits, mbcsData->ucm->states.countToUCodeUnits*2);
1091     udata_writeBlock(pData, mbcsData->stage1, stage1Top*2);
1092     if(mbcsData->ucm->states.maxCharLength==1) {
1093         udata_writeBlock(pData, mbcsData->stage2Single, mbcsData->stage2Top);
1094     } else {
1095         udata_writeBlock(pData, mbcsData->stage2, mbcsData->stage2Top);
1096     }
1097     udata_writeBlock(pData, mbcsData->fromUBytes, mbcsData->stage3Top);
1098
1099     /* return the number of bytes that should have been written */
1100     return header.offsetFromUBytes+header.fromUBytesLength;
1101 }
1102