icuSources/tools/toolutil/ucm.cpp

   1 // © 2016 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3 /*
   4 *******************************************************************************
   5 *
   6 *   Copyright (C) 2003-2013, International Business Machines
   7 *   Corporation and others.  All Rights Reserved.
   8 *
   9 *******************************************************************************
  10 *   file name:  ucm.c
  11 *   encoding:   UTF-8
  12 *   tab size:   8 (not used)
  13 *   indentation:4
  14 *
  15 *   created on: 2003jun20
  16 *   created by: Markus W. Scherer
  17 *
  18 *   This file reads a .ucm file, stores its mappings and sorts them.
  19 *   It implements handling of Unicode conversion mappings from .ucm files
  20 *   for makeconv, canonucm, rptp2ucm, etc.
  21 *
  22 *   Unicode code point sequences with a length of more than 1,
  23 *   as well as byte sequences with more than 4 bytes or more than one complete
  24 *   character sequence are handled to support m:n mappings.
  25 */
  26
  27 #include "unicode/utypes.h"
  28 #include "unicode/ustring.h"
  29 #include "cstring.h"
  30 #include "cmemory.h"
  31 #include "filestrm.h"
  32 #include "uarrsort.h"
  33 #include "ucnvmbcs.h"
  34 #include "ucnv_bld.h"
  35 #include "ucnv_ext.h"
  36 #include "uparse.h"
  37 #include "ucm.h"
  38 #include <stdio.h>
  39
  40 #if !UCONFIG_NO_CONVERSION
  41
  42 /* -------------------------------------------------------------------------- */
  43
  44 static void
  45 printMapping(UCMapping *m, UChar32 *codePoints, uint8_t *bytes, FILE *f) {
  46     int32_t j;
  47
  48     for(j=0; j<m->uLen; ++j) {
  49         fprintf(f, "<U%04lX>", (long)codePoints[j]);
  50     }
  51
  52     fputc(' ', f);
  53
  54     for(j=0; j<m->bLen; ++j) {
  55         fprintf(f, "\\x%02X", bytes[j]);
  56     }
  57
  58     if(m->f>=0) {
  59         fprintf(f, " |%u\n", m->f);
  60     } else {
  61         fputs("\n", f);
  62     }
  63 }
  64
  65 U_CAPI void U_EXPORT2
  66 ucm_printMapping(UCMTable *table, UCMapping *m, FILE *f) {
  67     printMapping(m, UCM_GET_CODE_POINTS(table, m), UCM_GET_BYTES(table, m), f);
  68 }
  69
  70 U_CAPI void U_EXPORT2
  71 ucm_printTable(UCMTable *table, FILE *f, UBool byUnicode) {
  72     UCMapping *m;
  73     int32_t i, length;
  74
  75     m=table->mappings;
  76     length=table->mappingsLength;
  77     if(byUnicode) {
  78         for(i=0; i<length; ++m, ++i) {
  79             ucm_printMapping(table, m, f);
  80         }
  81     } else {
  82         const int32_t *map=table->reverseMap;
  83         for(i=0; i<length; ++i) {
  84             ucm_printMapping(table, m+map[i], f);
  85         }
  86     }
  87 }
  88
  89 /* mapping comparisons ------------------------------------------------------ */
  90
  91 static int32_t
  92 compareUnicode(UCMTable *lTable, const UCMapping *l,
  93                UCMTable *rTable, const UCMapping *r) {
  94     const UChar32 *lu, *ru;
  95     int32_t result, i, length;
  96
  97     if(l->uLen==1 && r->uLen==1) {
  98         /* compare two single code points */
  99         return l->u-r->u;
 100     }
 101
 102     /* get pointers to the code point sequences */
 103     lu=UCM_GET_CODE_POINTS(lTable, l);
 104     ru=UCM_GET_CODE_POINTS(rTable, r);
 105
 106     /* get the minimum length */
 107     if(l->uLen<=r->uLen) {
 108         length=l->uLen;
 109     } else {
 110         length=r->uLen;
 111     }
 112
 113     /* compare the code points */
 114     for(i=0; i<length; ++i) {
 115         result=lu[i]-ru[i];
 116         if(result!=0) {
 117             return result;
 118         }
 119     }
 120
 121     /* compare the lengths */
 122     return l->uLen-r->uLen;
 123 }
 124
 125 static int32_t
 126 compareBytes(UCMTable *lTable, const UCMapping *l,
 127              UCMTable *rTable, const UCMapping *r,
 128              UBool lexical) {
 129     const uint8_t *lb, *rb;
 130     int32_t result, i, length;
 131
 132     /*
 133      * A lexical comparison is used for sorting in the builder, to allow
 134      * an efficient search for a byte sequence that could be a prefix
 135      * of a previously entered byte sequence.
 136      *
 137      * Comparing by lengths first is for compatibility with old .ucm tools
 138      * like canonucm and rptp2ucm.
 139      */
 140     if(lexical) {
 141         /* get the minimum length and continue */
 142         if(l->bLen<=r->bLen) {
 143             length=l->bLen;
 144         } else {
 145             length=r->bLen;
 146         }
 147     } else {
 148         /* compare lengths first */
 149         result=l->bLen-r->bLen;
 150         if(result!=0) {
 151             return result;
 152         } else {
 153             length=l->bLen;
 154         }
 155     }
 156
 157     /* get pointers to the byte sequences */
 158     lb=UCM_GET_BYTES(lTable, l);
 159     rb=UCM_GET_BYTES(rTable, r);
 160
 161     /* compare the bytes */
 162     for(i=0; i<length; ++i) {
 163         result=lb[i]-rb[i];
 164         if(result!=0) {
 165             return result;
 166         }
 167     }
 168
 169     /* compare the lengths */
 170     return l->bLen-r->bLen;
 171 }
 172
 173 /* compare UCMappings for sorting */
 174 static int32_t
 175 compareMappings(UCMTable *lTable, const UCMapping *l,
 176                 UCMTable *rTable, const UCMapping *r,
 177                 UBool uFirst) {
 178     int32_t result;
 179
 180     /* choose which side to compare first */
 181     if(uFirst) {
 182         /* Unicode then bytes */
 183         result=compareUnicode(lTable, l, rTable, r);
 184         if(result==0) {
 185             result=compareBytes(lTable, l, rTable, r, FALSE); /* not lexically, like canonucm */
 186         }
 187     } else {
 188         /* bytes then Unicode */
 189         result=compareBytes(lTable, l, rTable, r, TRUE); /* lexically, for builder */
 190         if(result==0) {
 191             result=compareUnicode(lTable, l, rTable, r);
 192         }
 193     }
 194
 195     if(result!=0) {
 196         return result;
 197     }
 198
 199     /* compare the flags */
 200     return l->f-r->f;
 201 }
 202 U_CDECL_BEGIN
 203 /* sorting by Unicode first sorts mappings directly */
 204 static int32_t  U_CALLCONV
 205 compareMappingsUnicodeFirst(const void *context, const void *left, const void *right) {
 206     return compareMappings(
 207         (UCMTable *)context, (const UCMapping *)left,
 208         (UCMTable *)context, (const UCMapping *)right, TRUE);
 209 }
 210
 211 /* sorting by bytes first sorts the reverseMap; use indirection to mappings */
 212 static int32_t U_CALLCONV
 213 compareMappingsBytesFirst(const void *context, const void *left, const void *right) {
 214     UCMTable *table=(UCMTable *)context;
 215     int32_t l=*(const int32_t *)left, r=*(const int32_t *)right;
 216     return compareMappings(
 217         table, table->mappings+l,
 218         table, table->mappings+r, FALSE);
 219 }
 220 U_CDECL_END
 221
 222 U_CAPI void U_EXPORT2
 223 ucm_sortTable(UCMTable *t) {
 224     UErrorCode errorCode;
 225     int32_t i;
 226
 227     if(t->isSorted) {
 228         return;
 229     }
 230
 231     errorCode=U_ZERO_ERROR;
 232
 233     /* 1. sort by Unicode first */
 234     uprv_sortArray(t->mappings, t->mappingsLength, sizeof(UCMapping),
 235                    compareMappingsUnicodeFirst, t,
 236                    FALSE, &errorCode);
 237
 238     /* build the reverseMap */
 239     if(t->reverseMap==NULL) {
 240         /*
 241          * allocate mappingsCapacity instead of mappingsLength so that
 242          * if mappings are added, the reverseMap need not be
 243          * reallocated each time
 244          * (see ucm_moveMappings() and ucm_addMapping())
 245          */
 246         t->reverseMap=(int32_t *)uprv_malloc(t->mappingsCapacity*sizeof(int32_t));
 247         if(t->reverseMap==NULL) {
 248             fprintf(stderr, "ucm error: unable to allocate reverseMap\n");
 249             exit(U_MEMORY_ALLOCATION_ERROR);
 250         }
 251     }
 252     for(i=0; i<t->mappingsLength; ++i) {
 253         t->reverseMap[i]=i;
 254     }
 255
 256     /* 2. sort reverseMap by mappings bytes first */
 257     uprv_sortArray(t->reverseMap, t->mappingsLength, sizeof(int32_t),
 258                    compareMappingsBytesFirst, t,
 259                    FALSE, &errorCode);
 260
 261     if(U_FAILURE(errorCode)) {
 262         fprintf(stderr, "ucm error: sortTable()/uprv_sortArray() fails - %s\n",
 263                 u_errorName(errorCode));
 264         exit(errorCode);
 265     }
 266
 267     t->isSorted=TRUE;
 268 }
 269
 270 /*
 271  * remove mappings with their move flag set from the base table
 272  * and move some of them (with UCM_MOVE_TO_EXT) to the extension table
 273  */
 274 U_CAPI void U_EXPORT2
 275 ucm_moveMappings(UCMTable *base, UCMTable *ext) {
 276     UCMapping *mb, *mbLimit;
 277     int8_t flag;
 278
 279     mb=base->mappings;
 280     mbLimit=mb+base->mappingsLength;
 281
 282     while(mb<mbLimit) {
 283         flag=mb->moveFlag;
 284         if(flag!=0) {
 285             /* reset the move flag */
 286             mb->moveFlag=0;
 287
 288             if(ext!=NULL && (flag&UCM_MOVE_TO_EXT)) {
 289                 /* add the mapping to the extension table */
 290                 ucm_addMapping(ext, mb, UCM_GET_CODE_POINTS(base, mb), UCM_GET_BYTES(base, mb));
 291             }
 292
 293             /* remove this mapping: move the last base mapping down and overwrite the current one */
 294             if(mb<(mbLimit-1)) {
 295                 uprv_memcpy(mb, mbLimit-1, sizeof(UCMapping));
 296             }
 297             --mbLimit;
 298             --base->mappingsLength;
 299             base->isSorted=FALSE;
 300         } else {
 301             ++mb;
 302         }
 303     }
 304 }
 305
 306 enum {
 307     NEEDS_MOVE=1,
 308     HAS_ERRORS=2
 309 };
 310
 311 static uint8_t
 312 checkBaseExtUnicode(UCMStates *baseStates, UCMTable *base, UCMTable *ext,
 313                     UBool moveToExt, UBool intersectBase) {
 314     (void)baseStates;
 315
 316     UCMapping *mb, *me, *mbLimit, *meLimit;
 317     int32_t cmp;
 318     uint8_t result;
 319
 320     mb=base->mappings;
 321     mbLimit=mb+base->mappingsLength;
 322
 323     me=ext->mappings;
 324     meLimit=me+ext->mappingsLength;
 325
 326     result=0;
 327
 328     for(;;) {
 329         /* skip irrelevant mappings on both sides */
 330         for(;;) {
 331             if(mb==mbLimit) {
 332                 return result;
 333             }
 334
 335             if((0<=mb->f && mb->f<=2) || mb->f==4) {
 336                 break;
 337             }
 338
 339             ++mb;
 340         }
 341
 342         for(;;) {
 343             if(me==meLimit) {
 344                 return result;
 345             }
 346
 347             if((0<=me->f && me->f<=2) || me->f==4) {
 348                 break;
 349             }
 350
 351             ++me;
 352         }
 353
 354         /* compare the base and extension mappings */
 355         cmp=compareUnicode(base, mb, ext, me);
 356         if(cmp<0) {
 357             if(intersectBase && (intersectBase!=2 || mb->bLen>1)) {
 358                 /*
 359                  * mapping in base but not in ext, move it
 360                  *
 361                  * if ext is DBCS, move DBCS mappings here
 362                  * and check SBCS ones for Unicode prefix below
 363                  */
 364                 mb->moveFlag|=UCM_MOVE_TO_EXT;
 365                 result|=NEEDS_MOVE;
 366
 367             /* does mb map from an input sequence that is a prefix of me's? */
 368             } else if( mb->uLen<me->uLen &&
 369                 0==uprv_memcmp(UCM_GET_CODE_POINTS(base, mb), UCM_GET_CODE_POINTS(ext, me), 4*mb->uLen)
 370             ) {
 371                 if(moveToExt) {
 372                     /* mark this mapping to be moved to the extension table */
 373                     mb->moveFlag|=UCM_MOVE_TO_EXT;
 374                     result|=NEEDS_MOVE;
 375                 } else {
 376                     fprintf(stderr,
 377                             "ucm error: the base table contains a mapping whose input sequence\n"
 378                             "           is a prefix of the input sequence of an extension mapping\n");
 379                     ucm_printMapping(base, mb, stderr);
 380                     ucm_printMapping(ext, me, stderr);
 381                     result|=HAS_ERRORS;
 382                 }
 383             }
 384
 385             ++mb;
 386         } else if(cmp==0) {
 387             /*
 388              * same output: remove the extension mapping,
 389              * otherwise treat as an error
 390              */
 391             if( mb->f==me->f && mb->bLen==me->bLen &&
 392                 0==uprv_memcmp(UCM_GET_BYTES(base, mb), UCM_GET_BYTES(ext, me), mb->bLen)
 393             ) {
 394                 me->moveFlag|=UCM_REMOVE_MAPPING;
 395                 result|=NEEDS_MOVE;
 396             } else if(intersectBase) {
 397                 /* mapping in base but not in ext, move it */
 398                 mb->moveFlag|=UCM_MOVE_TO_EXT;
 399                 result|=NEEDS_MOVE;
 400             } else {
 401                 fprintf(stderr,
 402                         "ucm error: the base table contains a mapping whose input sequence\n"
 403                         "           is the same as the input sequence of an extension mapping\n"
 404                         "           but it maps differently\n");
 405                 ucm_printMapping(base, mb, stderr);
 406                 ucm_printMapping(ext, me, stderr);
 407                 result|=HAS_ERRORS;
 408             }
 409
 410             ++mb;
 411         } else /* cmp>0 */ {
 412             ++me;
 413         }
 414     }
 415 }
 416
 417 static uint8_t
 418 checkBaseExtBytes(UCMStates *baseStates, UCMTable *base, UCMTable *ext,
 419                   UBool moveToExt, UBool intersectBase) {
 420     UCMapping *mb, *me;
 421     int32_t *baseMap, *extMap;
 422     int32_t b, e, bLimit, eLimit, cmp;
 423     uint8_t result;
 424     UBool isSISO;
 425
 426     baseMap=base->reverseMap;
 427     extMap=ext->reverseMap;
 428
 429     b=e=0;
 430     bLimit=base->mappingsLength;
 431     eLimit=ext->mappingsLength;
 432
 433     result=0;
 434
 435     isSISO=(UBool)(baseStates->outputType==MBCS_OUTPUT_2_SISO);
 436
 437     for(;;) {
 438         /* skip irrelevant mappings on both sides */
 439         for(;; ++b) {
 440             if(b==bLimit) {
 441                 return result;
 442             }
 443             mb=base->mappings+baseMap[b];
 444
 445             if(intersectBase==2 && mb->bLen==1) {
 446                 /*
 447                  * comparing a base against a DBCS extension:
 448                  * leave SBCS base mappings alone
 449                  */
 450                 continue;
 451             }
 452
 453             if(mb->f==0 || mb->f==3) {
 454                 break;
 455             }
 456         }
 457
 458         for(;;) {
 459             if(e==eLimit) {
 460                 return result;
 461             }
 462             me=ext->mappings+extMap[e];
 463
 464             if(me->f==0 || me->f==3) {
 465                 break;
 466             }
 467
 468             ++e;
 469         }
 470
 471         /* compare the base and extension mappings */
 472         cmp=compareBytes(base, mb, ext, me, TRUE);
 473         if(cmp<0) {
 474             if(intersectBase) {
 475                 /* mapping in base but not in ext, move it */
 476                 mb->moveFlag|=UCM_MOVE_TO_EXT;
 477                 result|=NEEDS_MOVE;
 478
 479             /*
 480              * does mb map from an input sequence that is a prefix of me's?
 481              * for SI/SO tables, a single byte is never a prefix because it
 482              * occurs in a separate single-byte state
 483              */
 484             } else if( mb->bLen<me->bLen &&
 485                 (!isSISO || mb->bLen>1) &&
 486                 0==uprv_memcmp(UCM_GET_BYTES(base, mb), UCM_GET_BYTES(ext, me), mb->bLen)
 487             ) {
 488                 if(moveToExt) {
 489                     /* mark this mapping to be moved to the extension table */
 490                     mb->moveFlag|=UCM_MOVE_TO_EXT;
 491                     result|=NEEDS_MOVE;
 492                 } else {
 493                     fprintf(stderr,
 494                             "ucm error: the base table contains a mapping whose input sequence\n"
 495                             "           is a prefix of the input sequence of an extension mapping\n");
 496                     ucm_printMapping(base, mb, stderr);
 497                     ucm_printMapping(ext, me, stderr);
 498                     result|=HAS_ERRORS;
 499                 }
 500             }
 501
 502             ++b;
 503         } else if(cmp==0) {
 504             /*
 505              * same output: remove the extension mapping,
 506              * otherwise treat as an error
 507              */
 508             if( mb->f==me->f && mb->uLen==me->uLen &&
 509                 0==uprv_memcmp(UCM_GET_CODE_POINTS(base, mb), UCM_GET_CODE_POINTS(ext, me), 4*mb->uLen)
 510             ) {
 511                 me->moveFlag|=UCM_REMOVE_MAPPING;
 512                 result|=NEEDS_MOVE;
 513             } else if(intersectBase) {
 514                 /* mapping in base but not in ext, move it */
 515                 mb->moveFlag|=UCM_MOVE_TO_EXT;
 516                 result|=NEEDS_MOVE;
 517             } else {
 518                 fprintf(stderr,
 519                         "ucm error: the base table contains a mapping whose input sequence\n"
 520                         "           is the same as the input sequence of an extension mapping\n"
 521                         "           but it maps differently\n");
 522                 ucm_printMapping(base, mb, stderr);
 523                 ucm_printMapping(ext, me, stderr);
 524                 result|=HAS_ERRORS;
 525             }
 526
 527             ++b;
 528         } else /* cmp>0 */ {
 529             ++e;
 530         }
 531     }
 532 }
 533
 534 U_CAPI UBool U_EXPORT2
 535 ucm_checkValidity(UCMTable *table, UCMStates *baseStates) {
 536     UCMapping *m, *mLimit;
 537     int32_t count;
 538     UBool isOK;
 539
 540     m=table->mappings;
 541     mLimit=m+table->mappingsLength;
 542     isOK=TRUE;
 543
 544     while(m<mLimit) {
 545         count=ucm_countChars(baseStates, UCM_GET_BYTES(table, m), m->bLen);
 546         if(count<1) {
 547             ucm_printMapping(table, m, stderr);
 548             isOK=FALSE;
 549         }
 550         ++m;
 551     }
 552
 553     return isOK;
 554 }
 555
 556 U_CAPI UBool U_EXPORT2
 557 ucm_checkBaseExt(UCMStates *baseStates,
 558                  UCMTable *base, UCMTable *ext, UCMTable *moveTarget,
 559                  UBool intersectBase) {
 560     uint8_t result;
 561
 562     /* if we have an extension table, we must always use precision flags */
 563     if(base->flagsType&UCM_FLAGS_IMPLICIT) {
 564         fprintf(stderr, "ucm error: the base table contains mappings without precision flags\n");
 565         return FALSE;
 566     }
 567     if(ext->flagsType&UCM_FLAGS_IMPLICIT) {
 568         fprintf(stderr, "ucm error: extension table contains mappings without precision flags\n");
 569         return FALSE;
 570     }
 571
 572     /* checking requires both tables to be sorted */
 573     ucm_sortTable(base);
 574     ucm_sortTable(ext);
 575
 576     /* check */
 577     result=
 578         checkBaseExtUnicode(baseStates, base, ext, (UBool)(moveTarget!=NULL), intersectBase)|
 579         checkBaseExtBytes(baseStates, base, ext, (UBool)(moveTarget!=NULL), intersectBase);
 580
 581     if(result&HAS_ERRORS) {
 582         return FALSE;
 583     }
 584
 585     if(result&NEEDS_MOVE) {
 586         ucm_moveMappings(ext, NULL);
 587         ucm_moveMappings(base, moveTarget);
 588         ucm_sortTable(base);
 589         ucm_sortTable(ext);
 590         if(moveTarget!=NULL) {
 591             ucm_sortTable(moveTarget);
 592         }
 593     }
 594
 595     return TRUE;
 596 }
 597
 598 /* merge tables for rptp2ucm ------------------------------------------------ */
 599
 600 U_CAPI void U_EXPORT2
 601 ucm_mergeTables(UCMTable *fromUTable, UCMTable *toUTable,
 602                 const uint8_t *subchar, int32_t subcharLength,
 603                 uint8_t subchar1) {
 604     UCMapping *fromUMapping, *toUMapping;
 605     int32_t fromUIndex, toUIndex, fromUTop, toUTop, cmp;
 606
 607     ucm_sortTable(fromUTable);
 608     ucm_sortTable(toUTable);
 609
 610     fromUMapping=fromUTable->mappings;
 611     toUMapping=toUTable->mappings;
 612
 613     fromUTop=fromUTable->mappingsLength;
 614     toUTop=toUTable->mappingsLength;
 615
 616     fromUIndex=toUIndex=0;
 617
 618     while(fromUIndex<fromUTop && toUIndex<toUTop) {
 619         cmp=compareMappings(fromUTable, fromUMapping, toUTable, toUMapping, TRUE);
 620         if(cmp==0) {
 621             /* equal: roundtrip, nothing to do (flags are initially 0) */
 622             ++fromUMapping;
 623             ++toUMapping;
 624
 625             ++fromUIndex;
 626             ++toUIndex;
 627         } else if(cmp<0) {
 628             /*
 629              * the fromU mapping does not have a toU counterpart:
 630              * fallback Unicode->codepage
 631              */
 632             if( (fromUMapping->bLen==subcharLength &&
 633                  0==uprv_memcmp(UCM_GET_BYTES(fromUTable, fromUMapping), subchar, subcharLength)) ||
 634                 (subchar1!=0 && fromUMapping->bLen==1 && fromUMapping->b.bytes[0]==subchar1)
 635             ) {
 636                 fromUMapping->f=2; /* SUB mapping */
 637             } else {
 638                 fromUMapping->f=1; /* normal fallback */
 639             }
 640
 641             ++fromUMapping;
 642             ++fromUIndex;
 643         } else {
 644             /*
 645              * the toU mapping does not have a fromU counterpart:
 646              * (reverse) fallback codepage->Unicode, copy it to the fromU table
 647              */
 648
 649             /* ignore reverse fallbacks to Unicode SUB */
 650             if(!(toUMapping->uLen==1 && (toUMapping->u==0xfffd || toUMapping->u==0x1a))) {
 651                 toUMapping->f=3; /* reverse fallback */
 652                 ucm_addMapping(fromUTable, toUMapping, UCM_GET_CODE_POINTS(toUTable, toUMapping), UCM_GET_BYTES(toUTable, toUMapping));
 653
 654                 /* the table may have been reallocated */
 655                 fromUMapping=fromUTable->mappings+fromUIndex;
 656             }
 657
 658             ++toUMapping;
 659             ++toUIndex;
 660         }
 661     }
 662
 663     /* either one or both tables are exhausted */
 664     while(fromUIndex<fromUTop) {
 665         /* leftover fromU mappings are fallbacks */
 666         if( (fromUMapping->bLen==subcharLength &&
 667              0==uprv_memcmp(UCM_GET_BYTES(fromUTable, fromUMapping), subchar, subcharLength)) ||
 668             (subchar1!=0 && fromUMapping->bLen==1 && fromUMapping->b.bytes[0]==subchar1)
 669         ) {
 670             fromUMapping->f=2; /* SUB mapping */
 671         } else {
 672             fromUMapping->f=1; /* normal fallback */
 673         }
 674
 675         ++fromUMapping;
 676         ++fromUIndex;
 677     }
 678
 679     while(toUIndex<toUTop) {
 680         /* leftover toU mappings are reverse fallbacks */
 681
 682         /* ignore reverse fallbacks to Unicode SUB */
 683         if(!(toUMapping->uLen==1 && (toUMapping->u==0xfffd || toUMapping->u==0x1a))) {
 684             toUMapping->f=3; /* reverse fallback */
 685             ucm_addMapping(fromUTable, toUMapping, UCM_GET_CODE_POINTS(toUTable, toUMapping), UCM_GET_BYTES(toUTable, toUMapping));
 686         }
 687
 688         ++toUMapping;
 689         ++toUIndex;
 690     }
 691
 692     fromUTable->isSorted=FALSE;
 693 }
 694
 695 /* separate extension mappings out of base table for rptp2ucm --------------- */
 696
 697 U_CAPI UBool U_EXPORT2
 698 ucm_separateMappings(UCMFile *ucm, UBool isSISO) {
 699     UCMTable *table;
 700     UCMapping *m, *mLimit;
 701     int32_t type;
 702     UBool needsMove, isOK;
 703
 704     table=ucm->base;
 705     m=table->mappings;
 706     mLimit=m+table->mappingsLength;
 707
 708     needsMove=FALSE;
 709     isOK=TRUE;
 710
 711     for(; m<mLimit; ++m) {
 712         if(isSISO && m->bLen==1 && (m->b.bytes[0]==0xe || m->b.bytes[0]==0xf)) {
 713             fprintf(stderr, "warning: removing illegal mapping from an SI/SO-stateful table\n");
 714             ucm_printMapping(table, m, stderr);
 715             m->moveFlag|=UCM_REMOVE_MAPPING;
 716             needsMove=TRUE;
 717             continue;
 718         }
 719
 720         type=ucm_mappingType(
 721                 &ucm->states, m,
 722                 UCM_GET_CODE_POINTS(table, m), UCM_GET_BYTES(table, m));
 723         if(type<0) {
 724             /* illegal byte sequence */
 725             printMapping(m, UCM_GET_CODE_POINTS(table, m), UCM_GET_BYTES(table, m), stderr);
 726             isOK=FALSE;
 727         } else if(type>0) {
 728             m->moveFlag|=UCM_MOVE_TO_EXT;
 729             needsMove=TRUE;
 730         }
 731     }
 732
 733     if(!isOK) {
 734         return FALSE;
 735     }
 736     if(needsMove) {
 737         ucm_moveMappings(ucm->base, ucm->ext);
 738         return ucm_checkBaseExt(&ucm->states, ucm->base, ucm->ext, ucm->ext, FALSE);
 739     } else {
 740         ucm_sortTable(ucm->base);
 741         return TRUE;
 742     }
 743 }
 744
 745 /* ucm parser --------------------------------------------------------------- */
 746
 747 U_CAPI int8_t U_EXPORT2
 748 ucm_parseBytes(uint8_t bytes[UCNV_EXT_MAX_BYTES], const char *line, const char **ps) {
 749     const char *s=*ps;
 750     char *end;
 751     uint8_t byte;
 752     int8_t bLen;
 753
 754     bLen=0;
 755     for(;;) {
 756         /* skip an optional plus sign */
 757         if(bLen>0 && *s=='+') {
 758             ++s;
 759         }
 760         if(*s!='\\') {
 761             break;
 762         }
 763
 764         if( s[1]!='x' ||
 765             (byte=(uint8_t)uprv_strtoul(s+2, &end, 16), end)!=s+4
 766         ) {
 767             fprintf(stderr, "ucm error: byte must be formatted as \\xXX (2 hex digits) - \"%s\"\n", line);
 768             return -1;
 769         }
 770
 771         if(bLen==UCNV_EXT_MAX_BYTES) {
 772             fprintf(stderr, "ucm error: too many bytes on \"%s\"\n", line);
 773             return -1;
 774         }
 775         bytes[bLen++]=byte;
 776         s=end;
 777     }
 778
 779     *ps=s;
 780     return bLen;
 781 }
 782
 783 /* parse a mapping line; must not be empty */
 784 U_CAPI UBool U_EXPORT2
 785 ucm_parseMappingLine(UCMapping *m,
 786                      UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
 787                      uint8_t bytes[UCNV_EXT_MAX_BYTES],
 788                      const char *line) {
 789     const char *s;
 790     char *end;
 791     UChar32 cp;
 792     int32_t u16Length;
 793     int8_t uLen, bLen, f;
 794
 795     s=line;
 796     uLen=bLen=0;
 797
 798     /* parse code points */
 799     for(;;) {
 800         /* skip an optional plus sign */
 801         if(uLen>0 && *s=='+') {
 802             ++s;
 803         }
 804         if(*s!='<') {
 805             break;
 806         }
 807
 808         if( s[1]!='U' ||
 809             (cp=(UChar32)uprv_strtoul(s+2, &end, 16), end)==s+2 ||
 810             *end!='>'
 811         ) {
 812             fprintf(stderr, "ucm error: Unicode code point must be formatted as <UXXXX> (1..6 hex digits) - \"%s\"\n", line);
 813             return FALSE;
 814         }
 815         if((uint32_t)cp>0x10ffff || U_IS_SURROGATE(cp)) {
 816             fprintf(stderr, "ucm error: Unicode code point must be 0..d7ff or e000..10ffff - \"%s\"\n", line);
 817             return FALSE;
 818         }
 819
 820         if(uLen==UCNV_EXT_MAX_UCHARS) {
 821             fprintf(stderr, "ucm error: too many code points on \"%s\"\n", line);
 822             return FALSE;
 823         }
 824         codePoints[uLen++]=cp;
 825         s=end+1;
 826     }
 827
 828     if(uLen==0) {
 829         fprintf(stderr, "ucm error: no Unicode code points on \"%s\"\n", line);
 830         return FALSE;
 831     } else if(uLen==1) {
 832         m->u=codePoints[0];
 833     } else {
 834         UErrorCode errorCode=U_ZERO_ERROR;
 835         u_strFromUTF32(NULL, 0, &u16Length, codePoints, uLen, &errorCode);
 836         if( (U_FAILURE(errorCode) && errorCode!=U_BUFFER_OVERFLOW_ERROR) ||
 837             u16Length>UCNV_EXT_MAX_UCHARS
 838         ) {
 839             fprintf(stderr, "ucm error: too many UChars on \"%s\"\n", line);
 840             return FALSE;
 841         }
 842     }
 843
 844     s=u_skipWhitespace(s);
 845
 846     /* parse bytes */
 847     bLen=ucm_parseBytes(bytes, line, &s);
 848
 849     if(bLen<0) {
 850         return FALSE;
 851     } else if(bLen==0) {
 852         fprintf(stderr, "ucm error: no bytes on \"%s\"\n", line);
 853         return FALSE;
 854     } else if(bLen<=4) {
 855         uprv_memcpy(m->b.bytes, bytes, bLen);
 856     }
 857
 858     /* skip everything until the fallback indicator, even the start of a comment */
 859     for(;;) {
 860         if(*s==0) {
 861             f=-1; /* no fallback indicator */
 862             break;
 863         } else if(*s=='|') {
 864             f=(int8_t)(s[1]-'0');
 865             if((uint8_t)f>4) {
 866                 fprintf(stderr, "ucm error: fallback indicator must be |0..|4 - \"%s\"\n", line);
 867                 return FALSE;
 868             }
 869             break;
 870         }
 871         ++s;
 872     }
 873
 874     m->uLen=uLen;
 875     m->bLen=bLen;
 876     m->f=f;
 877     return TRUE;
 878 }
 879
 880 /* general APIs ------------------------------------------------------------- */
 881
 882 U_CAPI UCMTable * U_EXPORT2
 883 ucm_openTable() {
 884     UCMTable *table=(UCMTable *)uprv_malloc(sizeof(UCMTable));
 885     if(table==NULL) {
 886         fprintf(stderr, "ucm error: unable to allocate a UCMTable\n");
 887         exit(U_MEMORY_ALLOCATION_ERROR);
 888     }
 889
 890     memset(table, 0, sizeof(UCMTable));
 891     return table;
 892 }
 893
 894 U_CAPI void U_EXPORT2
 895 ucm_closeTable(UCMTable *table) {
 896     if(table!=NULL) {
 897         uprv_free(table->mappings);
 898         uprv_free(table->codePoints);
 899         uprv_free(table->bytes);
 900         uprv_free(table->reverseMap);
 901         uprv_free(table);
 902     }
 903 }
 904
 905 U_CAPI void U_EXPORT2
 906 ucm_resetTable(UCMTable *table) {
 907     if(table!=NULL) {
 908         table->mappingsLength=0;
 909         table->flagsType=0;
 910         table->unicodeMask=0;
 911         table->bytesLength=table->codePointsLength=0;
 912         table->isSorted=FALSE;
 913     }
 914 }
 915
 916 U_CAPI void U_EXPORT2
 917 ucm_addMapping(UCMTable *table,
 918                UCMapping *m,
 919                UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
 920                uint8_t bytes[UCNV_EXT_MAX_BYTES]) {
 921     UCMapping *tm;
 922     UChar32 c;
 923     int32_t idx;
 924
 925     if(table->mappingsLength>=table->mappingsCapacity) {
 926         /* make the mappings array larger */
 927         if(table->mappingsCapacity==0) {
 928             table->mappingsCapacity=1000;
 929         } else {
 930             table->mappingsCapacity*=10;
 931         }
 932         table->mappings=(UCMapping *)uprv_realloc(table->mappings,
 933                                              table->mappingsCapacity*sizeof(UCMapping));
 934         if(table->mappings==NULL) {
 935             fprintf(stderr, "ucm error: unable to allocate %d UCMappings\n",
 936                             (int)table->mappingsCapacity);
 937             exit(U_MEMORY_ALLOCATION_ERROR);
 938         }
 939
 940         if(table->reverseMap!=NULL) {
 941             /* the reverseMap must be reallocated in a new sort */
 942             uprv_free(table->reverseMap);
 943             table->reverseMap=NULL;
 944         }
 945     }
 946
 947     if(m->uLen>1 && table->codePointsCapacity==0) {
 948         table->codePointsCapacity=10000;
 949         table->codePoints=(UChar32 *)uprv_malloc(table->codePointsCapacity*4);
 950         if(table->codePoints==NULL) {
 951             fprintf(stderr, "ucm error: unable to allocate %d UChar32s\n",
 952                             (int)table->codePointsCapacity);
 953             exit(U_MEMORY_ALLOCATION_ERROR);
 954         }
 955     }
 956
 957     if(m->bLen>4 && table->bytesCapacity==0) {
 958         table->bytesCapacity=10000;
 959         table->bytes=(uint8_t *)uprv_malloc(table->bytesCapacity);
 960         if(table->bytes==NULL) {
 961             fprintf(stderr, "ucm error: unable to allocate %d bytes\n",
 962                             (int)table->bytesCapacity);
 963             exit(U_MEMORY_ALLOCATION_ERROR);
 964         }
 965     }
 966
 967     if(m->uLen>1) {
 968         idx=table->codePointsLength;
 969         table->codePointsLength+=m->uLen;
 970         if(table->codePointsLength>table->codePointsCapacity) {
 971             fprintf(stderr, "ucm error: too many code points in multiple-code point mappings\n");
 972             exit(U_MEMORY_ALLOCATION_ERROR);
 973         }
 974
 975         uprv_memcpy(table->codePoints+idx, codePoints, (size_t)m->uLen*4);
 976         m->u=idx;
 977     }
 978
 979     if(m->bLen>4) {
 980         idx=table->bytesLength;
 981         table->bytesLength+=m->bLen;
 982         if(table->bytesLength>table->bytesCapacity) {
 983             fprintf(stderr, "ucm error: too many bytes in mappings with >4 charset bytes\n");
 984             exit(U_MEMORY_ALLOCATION_ERROR);
 985         }
 986
 987         uprv_memcpy(table->bytes+idx, bytes, m->bLen);
 988         m->b.idx=idx;
 989     }
 990
 991     /* set unicodeMask */
 992     for(idx=0; idx<m->uLen; ++idx) {
 993         c=codePoints[idx];
 994         if(c>=0x10000) {
 995             table->unicodeMask|=UCNV_HAS_SUPPLEMENTARY; /* there are supplementary code points */
 996         } else if(U_IS_SURROGATE(c)) {
 997             table->unicodeMask|=UCNV_HAS_SURROGATES;    /* there are surrogate code points */
 998         }
 999     }
1000
1001     /* set flagsType */
1002     if(m->f<0) {
1003         table->flagsType|=UCM_FLAGS_IMPLICIT;
1004     } else {
1005         table->flagsType|=UCM_FLAGS_EXPLICIT;
1006     }
1007
1008     tm=table->mappings+table->mappingsLength++;
1009     uprv_memcpy(tm, m, sizeof(UCMapping));
1010
1011     table->isSorted=FALSE;
1012 }
1013
1014 U_CAPI UCMFile * U_EXPORT2
1015 ucm_open() {
1016     UCMFile *ucm=(UCMFile *)uprv_malloc(sizeof(UCMFile));
1017     if(ucm==NULL) {
1018         fprintf(stderr, "ucm error: unable to allocate a UCMFile\n");
1019         exit(U_MEMORY_ALLOCATION_ERROR);
1020     }
1021
1022     memset(ucm, 0, sizeof(UCMFile));
1023
1024     ucm->base=ucm_openTable();
1025     ucm->ext=ucm_openTable();
1026
1027     ucm->states.stateFlags[0]=MBCS_STATE_FLAG_DIRECT;
1028     ucm->states.conversionType=UCNV_UNSUPPORTED_CONVERTER;
1029     ucm->states.outputType=-1;
1030     ucm->states.minCharLength=ucm->states.maxCharLength=1;
1031
1032     return ucm;
1033 }
1034
1035 U_CAPI void U_EXPORT2
1036 ucm_close(UCMFile *ucm) {
1037     if(ucm!=NULL) {
1038         ucm_closeTable(ucm->base);
1039         ucm_closeTable(ucm->ext);
1040         uprv_free(ucm);
1041     }
1042 }
1043
1044 U_CAPI int32_t U_EXPORT2
1045 ucm_mappingType(UCMStates *baseStates,
1046                 UCMapping *m,
1047                 UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
1048                 uint8_t bytes[UCNV_EXT_MAX_BYTES]) {
1049     (void)codePoints;
1050     /* check validity of the bytes and count the characters in them */
1051     int32_t count=ucm_countChars(baseStates, bytes, m->bLen);
1052     if(count<1) {
1053         /* illegal byte sequence */
1054         return -1;
1055     }
1056
1057     /*
1058      * Suitable for an ICU conversion base table means:
1059      * - a 1:1 mapping (1 Unicode code point : 1 byte sequence)
1060      * - precision flag 0..3
1061      * - SBCS: any 1:1 mapping
1062      *         (the table stores additional bits to distinguish mapping types)
1063      * - MBCS: not a |2 SUB mapping for <subchar1>
1064      * - MBCS: not a |1 fallback to 0x00
1065      * - MBCS: not a multi-byte mapping with leading 0x00 bytes
1066      *
1067      * Further restrictions for fromUnicode tables
1068      * are enforced in makeconv (MBCSOkForBaseFromUnicode()).
1069      *
1070      * All of the MBCS fromUnicode specific tests could be removed from here,
1071      * but the ones above are for unusual mappings, and removing the tests
1072      * from here would change canonucm output which seems gratuitous.
1073      * (Markus Scherer 2006-nov-28)
1074      *
1075      * Exception: All implicit mappings (f<0) that need to be moved
1076      * because of fromUnicode restrictions _must_ be moved here because
1077      * makeconv uses a hack for moving mappings only for the fromUnicode table
1078      * that only works with non-negative values of f.
1079      */
1080     if( m->uLen==1 && count==1 && m->f<=3 &&
1081         (baseStates->maxCharLength==1 ||
1082             !((m->f==2 && m->bLen==1) ||
1083               (m->f==1 && bytes[0]==0) ||
1084               (m->f<=1 && m->bLen>1 && bytes[0]==0)))
1085     ) {
1086         return 0; /* suitable for a base table */
1087     } else {
1088         return 1; /* needs to go into an extension table */
1089     }
1090 }
1091
1092 U_CAPI UBool U_EXPORT2
1093 ucm_addMappingAuto(UCMFile *ucm, UBool forBase, UCMStates *baseStates,
1094                    UCMapping *m,
1095                    UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
1096                    uint8_t bytes[UCNV_EXT_MAX_BYTES]) {
1097     int32_t type;
1098
1099     if(m->f==2 && m->uLen>1) {
1100         fprintf(stderr, "ucm error: illegal <subchar1> |2 mapping from multiple code points\n");
1101         printMapping(m, codePoints, bytes, stderr);
1102         return FALSE;
1103     }
1104
1105     if(baseStates!=NULL) {
1106         /* check validity of the bytes and count the characters in them */
1107         type=ucm_mappingType(baseStates, m, codePoints, bytes);
1108         if(type<0) {
1109             /* illegal byte sequence */
1110             printMapping(m, codePoints, bytes, stderr);
1111             return FALSE;
1112         }
1113     } else {
1114         /* not used - adding a mapping for an extension-only table before its base table is read */
1115         type=1;
1116     }
1117
1118     /*
1119      * Add the mapping to the base table if this is requested and suitable.
1120      * Otherwise, add it to the extension table.
1121      */
1122     if(forBase && type==0) {
1123         ucm_addMapping(ucm->base, m, codePoints, bytes);
1124     } else {
1125         ucm_addMapping(ucm->ext, m, codePoints, bytes);
1126     }
1127
1128     return TRUE;
1129 }
1130
1131 U_CAPI UBool U_EXPORT2
1132 ucm_addMappingFromLine(UCMFile *ucm, const char *line, UBool forBase, UCMStates *baseStates) {
1133   UCMapping m={ 0, {0}, 0, 0, 0, 0 };
1134     UChar32 codePoints[UCNV_EXT_MAX_UCHARS];
1135     uint8_t bytes[UCNV_EXT_MAX_BYTES];
1136
1137     const char *s;
1138
1139     /* ignore empty and comment lines */
1140     if(line[0]=='#' || *(s=u_skipWhitespace(line))==0 || *s=='\n' || *s=='\r') {
1141         return TRUE;
1142     }
1143
1144     return
1145         ucm_parseMappingLine(&m, codePoints, bytes, line) &&
1146         ucm_addMappingAuto(ucm, forBase, baseStates, &m, codePoints, bytes);
1147 }
1148
1149 U_CAPI void U_EXPORT2
1150 ucm_readTable(UCMFile *ucm, FileStream* convFile,
1151               UBool forBase, UCMStates *baseStates,
1152               UErrorCode *pErrorCode) {
1153     char line[500];
1154     char *end;
1155     UBool isOK;
1156
1157     if(U_FAILURE(*pErrorCode)) {
1158         return;
1159     }
1160
1161     isOK=TRUE;
1162
1163     for(;;) {
1164         /* read the next line */
1165         if(!T_FileStream_readLine(convFile, line, sizeof(line))) {
1166             fprintf(stderr, "incomplete charmap section\n");
1167             isOK=FALSE;
1168             break;
1169         }
1170
1171         /* remove CR LF */
1172         end=uprv_strchr(line, 0);
1173         while(line<end && (*(end-1)=='\r' || *(end-1)=='\n')) {
1174             --end;
1175         }
1176         *end=0;
1177
1178         /* ignore empty and comment lines */
1179         if(line[0]==0 || line[0]=='#') {
1180             continue;
1181         }
1182
1183         /* stop at the end of the mapping table */
1184         if(0==uprv_strcmp(line, "END CHARMAP")) {
1185             break;
1186         }
1187
1188         isOK&=ucm_addMappingFromLine(ucm, line, forBase, baseStates);
1189     }
1190
1191     if(!isOK) {
1192         *pErrorCode=U_INVALID_TABLE_FORMAT;
1193     }
1194 }
1195 #endif