icuSources/test/cintltst/utransts.c

   1 /*
   2  *******************************************************************************
   3  *   Copyright (C) 1997-2016 International Business Machines
   4  *   Corporation and others.  All Rights Reserved.
   5  *******************************************************************************
   6  *   Date        Name        Description
   7  *   06/23/00    aliu        Creation.
   8  *******************************************************************************
   9  */
  10
  11 #include "unicode/utypes.h"
  12
  13 #if !UCONFIG_NO_TRANSLITERATION
  14
  15 #include <stdlib.h>
  16 #include <string.h>
  17 #include "unicode/utrans.h"
  18 #include "unicode/ustring.h"
  19 #include "unicode/uset.h"
  20 #include "cintltst.h"
  21 #include "cmemory.h"
  22
  23 #define TEST(x) addTest(root, &x, "utrans/" # x)
  24
  25 static void TestAPI(void);
  26 static void TestSimpleRules(void);
  27 static void TestFilter(void);
  28 static void TestOpenInverse(void);
  29 static void TestClone(void);
  30 static void TestRegisterUnregister(void);
  31 static void TestExtractBetween(void);
  32 static void TestUnicodeIDs(void);
  33 static void TestGetRulesAndSourceSet(void);
  34 static void TestDataVariantsCompounds(void);
  35
  36 static void _expectRules(const char*, const char*, const char*);
  37 static void _expect(const UTransliterator* trans, const char* cfrom, const char* cto);
  38
  39 void addUTransTest(TestNode** root);
  40
  41
  42 void
  43 addUTransTest(TestNode** root) {
  44     TEST(TestAPI);
  45     TEST(TestSimpleRules);
  46     TEST(TestFilter);
  47     TEST(TestOpenInverse);
  48     TEST(TestClone);
  49     TEST(TestRegisterUnregister);
  50     TEST(TestExtractBetween);
  51     TEST(TestUnicodeIDs);
  52     TEST(TestGetRulesAndSourceSet);
  53     TEST(TestDataVariantsCompounds);
  54 }
  55
  56 /*------------------------------------------------------------------
  57  * Replaceable glue
  58  *
  59  * To test the Replaceable glue we have to dummy up a C-based
  60  * Replaceable callback.  This code is for testing purposes only.
  61  *------------------------------------------------------------------*/
  62
  63 typedef struct XReplaceable {
  64     UChar* text;    /* MUST BE null-terminated */
  65 } XReplaceable;
  66
  67 static void InitXReplaceable(XReplaceable* rep, const char* cstring) {
  68     rep->text = malloc(sizeof(UChar) * (strlen(cstring)+1));
  69     u_uastrcpy(rep->text, cstring);
  70 }
  71
  72 static void FreeXReplaceable(XReplaceable* rep) {
  73     if (rep->text != NULL) {
  74         free(rep->text);
  75         rep->text = NULL;
  76     }
  77 }
  78
  79 /* UReplaceableCallbacks callback */
  80 static int32_t Xlength(const UReplaceable* rep) {
  81     const XReplaceable* x = (const XReplaceable*)rep;
  82     return u_strlen(x->text);
  83 }
  84
  85 /* UReplaceableCallbacks callback */
  86 static UChar XcharAt(const UReplaceable* rep, int32_t offset) {
  87     const XReplaceable* x = (const XReplaceable*)rep;
  88     return x->text[offset];
  89 }
  90
  91 /* UReplaceableCallbacks callback */
  92 static UChar32 Xchar32At(const UReplaceable* rep, int32_t offset) {
  93     const XReplaceable* x = (const XReplaceable*)rep;
  94     return x->text[offset];
  95 }
  96
  97 /* UReplaceableCallbacks callback */
  98 static void Xreplace(UReplaceable* rep, int32_t start, int32_t limit,
  99               const UChar* text, int32_t textLength) {
 100     XReplaceable* x = (XReplaceable*)rep;
 101     int32_t newLen = Xlength(rep) + limit - start + textLength;
 102     UChar* newText = (UChar*) malloc(sizeof(UChar) * (newLen+1));
 103     u_strncpy(newText, x->text, start);
 104     u_strncpy(newText + start, text, textLength);
 105     u_strcpy(newText + start + textLength, x->text + limit);
 106     free(x->text);
 107     x->text = newText;
 108 }
 109
 110 /* UReplaceableCallbacks callback */
 111 static void Xcopy(UReplaceable* rep, int32_t start, int32_t limit, int32_t dest) {
 112     XReplaceable* x = (XReplaceable*)rep;
 113     int32_t newLen = Xlength(rep) + limit - start;
 114     UChar* newText = (UChar*) malloc(sizeof(UChar) * (newLen+1));
 115     u_strncpy(newText, x->text, dest);
 116     u_strncpy(newText + dest, x->text + start, limit - start);
 117     u_strcpy(newText + dest + limit - start, x->text + dest);
 118     free(x->text);
 119     x->text = newText;
 120 }
 121
 122 /* UReplaceableCallbacks callback */
 123 static void Xextract(UReplaceable* rep, int32_t start, int32_t limit, UChar* dst) {
 124     XReplaceable* x = (XReplaceable*)rep;
 125     int32_t len = limit - start;
 126     u_strncpy(dst, x->text, len);
 127 }
 128
 129 static void InitXReplaceableCallbacks(UReplaceableCallbacks* callbacks) {
 130     callbacks->length = Xlength;
 131     callbacks->charAt = XcharAt;
 132     callbacks->char32At = Xchar32At;
 133     callbacks->replace = Xreplace;
 134     callbacks->extract = Xextract;
 135     callbacks->copy = Xcopy;
 136 }
 137
 138 /*------------------------------------------------------------------
 139  * Tests
 140  *------------------------------------------------------------------*/
 141
 142 static void TestAPI() {
 143     enum { BUF_CAP = 128 };
 144     char buf[BUF_CAP], buf2[BUF_CAP];
 145     UErrorCode status = U_ZERO_ERROR;
 146     UTransliterator* trans = NULL;
 147     int32_t i, n;
 148
 149     /* Test getAvailableIDs */
 150     n = utrans_countAvailableIDs();
 151     if (n < 1) {
 152         log_err("FAIL: utrans_countAvailableIDs() returned %d\n", n);
 153     } else {
 154         log_verbose("System ID count: %d\n", n);
 155     }
 156     for (i=0; i<n; ++i) {
 157         utrans_getAvailableID(i, buf, BUF_CAP);
 158         if (*buf == 0) {
 159             log_err("FAIL: System transliterator %d: \"\"\n", i);
 160         } else {
 161             log_verbose("System transliterator %d: \"%s\"\n", i, buf);
 162         }
 163     }
 164
 165     /* Test open */
 166     utrans_getAvailableID(0, buf, BUF_CAP);
 167     trans = utrans_open(buf, UTRANS_FORWARD,NULL,0,NULL, &status);
 168     if (U_FAILURE(status)) {
 169         log_err("FAIL: utrans_open(%s) failed, error=%s\n",
 170                 buf, u_errorName(status));
 171     }
 172
 173     else {
 174         /* Test getID */
 175         utrans_getID(trans, buf2, BUF_CAP);
 176         if (0 != strcmp(buf, buf2)) {
 177             log_err("FAIL: utrans_getID(%s) returned %s\n",
 178                     buf, buf2);
 179         }
 180         utrans_close(trans);
 181     }
 182 }
 183
 184 static void TestUnicodeIDs() {
 185     UEnumeration *uenum;
 186     UTransliterator *utrans;
 187     const UChar *id, *id2;
 188     int32_t idLength, id2Length, count, count2;
 189
 190     UErrorCode errorCode;
 191
 192     errorCode=U_ZERO_ERROR;
 193     uenum=utrans_openIDs(&errorCode);
 194     if(U_FAILURE(errorCode)) {
 195         log_err("utrans_openIDs() failed - %s\n", u_errorName(errorCode));
 196         return;
 197     }
 198
 199     count=uenum_count(uenum, &errorCode);
 200     if(U_FAILURE(errorCode) || count<1) {
 201         log_err("uenum_count(transliterator IDs)=%d - %s\n", count, u_errorName(errorCode));
 202     }
 203
 204     count=0;
 205     for(;;) {
 206         id=uenum_unext(uenum, &idLength, &errorCode);
 207         if(U_FAILURE(errorCode)) {
 208             log_err("uenum_unext(transliterator ID %d) failed - %s\n", count, u_errorName(errorCode));
 209             break;
 210         }
 211         if(id==NULL) {
 212             break;
 213         }
 214
 215         if(++count>10) {
 216             /* try to actually open only a few transliterators */
 217             continue;
 218         }
 219
 220         utrans=utrans_openU(id, idLength, UTRANS_FORWARD, NULL, 0, NULL, &errorCode);
 221         if(U_FAILURE(errorCode)) {
 222             log_err("utrans_openU(%s) failed - %s\n", aescstrdup(id, idLength), u_errorName(errorCode));
 223             continue;
 224         }
 225
 226         id2=utrans_getUnicodeID(utrans, &id2Length);
 227         if(idLength!=id2Length || 0!=u_memcmp(id, id2, idLength)) {
 228             log_err("utrans_getUnicodeID(%s) does not match the original ID\n", aescstrdup(id, idLength));
 229         }
 230
 231         utrans_close(utrans);
 232     }
 233
 234     uenum_reset(uenum, &errorCode);
 235     if(U_FAILURE(errorCode) || count<1) {
 236         log_err("uenum_reset(transliterator IDs) failed - %s\n", u_errorName(errorCode));
 237     } else {
 238         count2=uenum_count(uenum, &errorCode);
 239         if(U_FAILURE(errorCode) || count<1) {
 240             log_err("2nd uenum_count(transliterator IDs)=%d - %s\n", count2, u_errorName(errorCode));
 241         } else if(count!=count2) {
 242             log_err("uenum_unext(transliterator IDs) returned %d IDs but uenum_count() after uenum_reset() claims there are %d\n", count, count2);
 243         }
 244     }
 245
 246     uenum_close(uenum);
 247 }
 248
 249 static void TestOpenInverse(){
 250     UErrorCode status=U_ZERO_ERROR;
 251     UTransliterator* t1=NULL;
 252     UTransliterator* inverse1=NULL;
 253     enum { BUF_CAP = 128 };
 254     char buf1[BUF_CAP];
 255     int32_t i=0;
 256
 257     const char TransID[][25]={
 258            "Halfwidth-Fullwidth",
 259            "Fullwidth-Halfwidth",
 260            "Greek-Latin" ,
 261            "Latin-Greek",
 262            /*"Arabic-Latin", // Removed in 2.0*/
 263            /*"Latin-Arabic", // Removed in 2.0*/
 264            "Katakana-Latin",
 265            "Latin-Katakana",
 266            /*"Hebrew-Latin", // Removed in 2.0*/
 267            /*"Latin-Hebrew", // Removed in 2.0*/
 268            "Cyrillic-Latin",
 269            "Latin-Cyrillic",
 270            "Devanagari-Latin",
 271            "Latin-Devanagari",
 272            "Any-Hex",
 273            "Hex-Any"
 274          };
 275
 276     for(i=0; i<UPRV_LENGTHOF(TransID); i=i+2){
 277         status = U_ZERO_ERROR;
 278         t1=utrans_open(TransID[i], UTRANS_FORWARD,NULL,0,NULL, &status);
 279         if(t1 == NULL || U_FAILURE(status)){
 280             log_data_err("FAIL: in instantiation for id=%s -> %s (Are you missing data?)\n", TransID[i], u_errorName(status));
 281             continue;
 282         }
 283         inverse1=utrans_openInverse(t1, &status);
 284         if(U_FAILURE(status)){
 285             log_err("FAIL: utrans_openInverse() failed for id=%s. Error=%s\n", TransID[i], myErrorName(status));
 286             continue;
 287         }
 288         utrans_getID(inverse1, buf1, BUF_CAP);
 289         if(strcmp(buf1, TransID[i+1]) != 0){
 290             log_err("FAIL :openInverse() for %s returned %s instead of %s\n", TransID[i], buf1, TransID[i+1]);
 291         }
 292         utrans_close(t1);
 293         utrans_close(inverse1);
 294    }
 295 }
 296
 297 static void TestClone(){
 298     UErrorCode status=U_ZERO_ERROR;
 299     UTransliterator* t1=NULL;
 300     UTransliterator* t2=NULL;
 301     UTransliterator* t3=NULL;
 302     UTransliterator* t4=NULL;
 303     enum { BUF_CAP = 128 };
 304     char buf1[BUF_CAP], buf2[BUF_CAP], buf3[BUF_CAP];
 305
 306     t1=utrans_open("Latin-Devanagari", UTRANS_FORWARD, NULL,0,NULL,&status);
 307     if(U_FAILURE(status)){
 308         log_data_err("FAIL: construction -> %s (Are you missing data?)\n", u_errorName(status));
 309         return;
 310     }
 311     t2=utrans_open("Latin-Greek", UTRANS_FORWARD, NULL,0,NULL,&status);
 312     if(U_FAILURE(status)){
 313         log_err("FAIL: construction\n");
 314         utrans_close(t1);
 315         return;
 316     }
 317
 318     t3=utrans_clone(t1, &status);
 319     t4=utrans_clone(t2, &status);
 320
 321     utrans_getID(t1, buf1, BUF_CAP);
 322     utrans_getID(t2, buf2, BUF_CAP);
 323     utrans_getID(t3, buf3, BUF_CAP);
 324
 325     if(strcmp(buf1, buf3) != 0 ||
 326         strcmp(buf1, buf2) == 0) {
 327         log_err("FAIL: utrans_clone() failed\n");
 328     }
 329
 330     utrans_getID(t4, buf3, BUF_CAP);
 331
 332     if(strcmp(buf2, buf3) != 0 ||
 333         strcmp(buf1, buf3) == 0) {
 334         log_err("FAIL: utrans_clone() failed\n");
 335     }
 336
 337     utrans_close(t1);
 338     utrans_close(t2);
 339     utrans_close(t3);
 340     utrans_close(t4);
 341
 342 }
 343
 344 static void TestRegisterUnregister(){
 345     UErrorCode status=U_ZERO_ERROR;
 346     UTransliterator* t1=NULL;
 347     UTransliterator* rules=NULL, *rules2;
 348     UTransliterator* inverse1=NULL;
 349     UChar rule[]={ 0x0061, 0x003c, 0x003e, 0x0063}; /*a<>b*/
 350
 351     U_STRING_DECL(ID, "TestA-TestB", 11);
 352     U_STRING_INIT(ID, "TestA-TestB", 11);
 353
 354     /* Make sure it doesn't exist */
 355     t1=utrans_open("TestA-TestB", UTRANS_FORWARD,NULL,0,NULL, &status);
 356     if(t1 != NULL || U_SUCCESS(status)) {
 357         log_err("FAIL: TestA-TestB already registered\n");
 358         return;
 359     }
 360     status=U_ZERO_ERROR;
 361     /* Check inverse too */
 362     inverse1=utrans_open("TestA-TestB", UTRANS_REVERSE, NULL,0,NULL,&status);
 363     if(inverse1 != NULL || U_SUCCESS(status)) {
 364         log_err("FAIL: TestA-TestB already registered\n");
 365         return;
 366     }
 367     status=U_ZERO_ERROR;
 368     /* Create it */
 369     rules=utrans_open("TestA-TestB",UTRANS_FORWARD, rule, 4, NULL, &status);
 370     if(U_FAILURE(status)){
 371         log_err("FAIL: utrans_openRules(a<>B) failed with error=%s\n", myErrorName(status));
 372         return;
 373     }
 374
 375     /* clone it so we can register it a second time */
 376     rules2=utrans_clone(rules, &status);
 377     if(U_FAILURE(status)) {
 378         log_err("FAIL: utrans_clone(a<>B) failed with error=%s\n", myErrorName(status));
 379         return;
 380     }
 381
 382     status=U_ZERO_ERROR;
 383     /* Register it */
 384     utrans_register(rules, &status);
 385     if(U_FAILURE(status)){
 386         log_err("FAIL: utrans_register failed with error=%s\n", myErrorName(status));
 387         return;
 388     }
 389     status=U_ZERO_ERROR;
 390     /* Now check again -- should exist now*/
 391     t1= utrans_open("TestA-TestB", UTRANS_FORWARD, NULL,0,NULL,&status);
 392     if(U_FAILURE(status) || t1 == NULL){
 393         log_err("FAIL: TestA-TestB not registered\n");
 394         return;
 395     }
 396     utrans_close(t1);
 397
 398     /*unregister the instance*/
 399     status=U_ZERO_ERROR;
 400     utrans_unregister("TestA-TestB");
 401     /* now Make sure it doesn't exist */
 402     t1=utrans_open("TestA-TestB", UTRANS_FORWARD,NULL,0,NULL, &status);
 403     if(U_SUCCESS(status) || t1 != NULL) {
 404         log_err("FAIL: TestA-TestB isn't unregistered\n");
 405         return;
 406     }
 407     utrans_close(t1);
 408
 409     /* now with utrans_unregisterID(const UChar *) */
 410     status=U_ZERO_ERROR;
 411     utrans_register(rules2, &status);
 412     if(U_FAILURE(status)){
 413         log_err("FAIL: 2nd utrans_register failed with error=%s\n", myErrorName(status));
 414         return;
 415     }
 416     status=U_ZERO_ERROR;
 417     /* Now check again -- should exist now*/
 418     t1= utrans_open("TestA-TestB", UTRANS_FORWARD, NULL,0,NULL,&status);
 419     if(U_FAILURE(status) || t1 == NULL){
 420         log_err("FAIL: 2nd TestA-TestB not registered\n");
 421         return;
 422     }
 423     utrans_close(t1);
 424
 425     /*unregister the instance*/
 426     status=U_ZERO_ERROR;
 427     utrans_unregisterID(ID, -1);
 428     /* now Make sure it doesn't exist */
 429     t1=utrans_openU(ID, -1, UTRANS_FORWARD,NULL,0,NULL, &status);
 430     if(U_SUCCESS(status) || t1 != NULL) {
 431         log_err("FAIL: 2nd TestA-TestB isn't unregistered\n");
 432         return;
 433     }
 434
 435     utrans_close(t1);
 436     utrans_close(inverse1);
 437 }
 438
 439 static void TestSimpleRules() {
 440     /* Test rules */
 441     /* Example: rules 1. ab>x|y
 442      *                2. yc>z
 443      *
 444      * []|eabcd  start - no match, copy e to tranlated buffer
 445      * [e]|abcd  match rule 1 - copy output & adjust cursor
 446      * [ex|y]cd  match rule 2 - copy output & adjust cursor
 447      * [exz]|d   no match, copy d to transliterated buffer
 448      * [exzd]|   done
 449      */
 450     _expectRules("ab>x|y;"
 451                  "yc>z",
 452                  "eabcd", "exzd");
 453
 454     /* Another set of rules:
 455      *    1. ab>x|yzacw
 456      *    2. za>q
 457      *    3. qc>r
 458      *    4. cw>n
 459      *
 460      * []|ab       Rule 1
 461      * [x|yzacw]   No match
 462      * [xy|zacw]   Rule 2
 463      * [xyq|cw]    Rule 4
 464      * [xyqn]|     Done
 465      */
 466     _expectRules("ab>x|yzacw;"
 467                  "za>q;"
 468                  "qc>r;"
 469                  "cw>n",
 470                  "ab", "xyqn");
 471
 472     /* Test categories
 473      */
 474     _expectRules("$dummy=" "\\uE100" ";" /* careful here with E100 */
 475                  "$vowel=[aeiouAEIOU];"
 476                  "$lu=[:Lu:];"
 477                  "$vowel } $lu > '!';"
 478                  "$vowel > '&';"
 479                  "'!' { $lu > '^';"
 480                  "$lu > '*';"
 481                  "a > ERROR",
 482                  "abcdefgABCDEFGU", "&bcd&fg!^**!^*&");
 483
 484     /* Test multiple passes
 485     */
 486     _expectRules("abc > xy;"
 487                  "::Null;"
 488                  "aba > z;",
 489                  "abc ababc aba", "xy abxy z");
 490 }
 491
 492 static void TestFilter() {
 493     UErrorCode status = U_ZERO_ERROR;
 494     UChar filt[128];
 495     UChar buf[128];
 496     UChar exp[128];
 497     char *cbuf;
 498     int32_t limit;
 499     const char* DATA[] = {
 500         "[^c]", /* Filter out 'c' */
 501         "abcde",
 502         "\\u0061\\u0062c\\u0064\\u0065",
 503
 504         "", /* No filter */
 505         "abcde",
 506         "\\u0061\\u0062\\u0063\\u0064\\u0065"
 507     };
 508     int32_t DATA_length = UPRV_LENGTHOF(DATA);
 509     int32_t i;
 510
 511     UTransliterator* hex = utrans_open("Any-Hex", UTRANS_FORWARD, NULL,0,NULL,&status);
 512
 513     if (hex == 0 || U_FAILURE(status)) {
 514         log_err("FAIL: utrans_open(Unicode-Hex) failed, error=%s\n",
 515                 u_errorName(status));
 516         goto exit;
 517     }
 518
 519     for (i=0; i<DATA_length; i+=3) {
 520         /*u_uastrcpy(filt, DATA[i]);*/
 521         u_charsToUChars(DATA[i], filt, (int32_t)strlen(DATA[i])+1);
 522         utrans_setFilter(hex, filt, -1, &status);
 523
 524         if (U_FAILURE(status)) {
 525             log_err("FAIL: utrans_setFilter() failed, error=%s\n",
 526                     u_errorName(status));
 527             goto exit;
 528         }
 529
 530         /*u_uastrcpy(buf, DATA[i+1]);*/
 531         u_charsToUChars(DATA[i+1], buf, (int32_t)strlen(DATA[i+1])+1);
 532         limit = 5;
 533         utrans_transUChars(hex, buf, NULL, 128, 0, &limit, &status);
 534
 535         if (U_FAILURE(status)) {
 536             log_err("FAIL: utrans_transUChars() failed, error=%s\n",
 537                     u_errorName(status));
 538             goto exit;
 539         }
 540
 541         cbuf=aescstrdup(buf, -1);
 542         u_charsToUChars(DATA[i+2], exp, (int32_t)strlen(DATA[i+2])+1);
 543         if (0 == u_strcmp(buf, exp)) {
 544             log_verbose("Ok: %s | %s -> %s\n", DATA[i+1], DATA[i], cbuf);
 545         } else {
 546             log_err("FAIL: %s | %s -> %s, expected %s\n", DATA[i+1], DATA[i], cbuf, DATA[i+2]);
 547         }
 548     }
 549
 550  exit:
 551     utrans_close(hex);
 552 }
 553
 554 /**
 555  * Test the UReplaceableCallback extractBetween support.  We use a
 556  * transliterator known to rely on this call.
 557  */
 558 static void TestExtractBetween() {
 559
 560     UTransliterator *trans;
 561     UErrorCode status = U_ZERO_ERROR;
 562     UParseError parseErr;
 563
 564     trans = utrans_open("Lower", UTRANS_FORWARD, NULL, -1,
 565                         &parseErr, &status);
 566
 567     if (U_FAILURE(status)) {
 568         log_err("FAIL: utrans_open(Lower) failed, error=%s\n",
 569                 u_errorName(status));
 570     } else {
 571         _expect(trans, "ABC", "abc");
 572
 573         utrans_close(trans);
 574     }
 575 }
 576
 577 /**
 578  * Test utrans_toRules, utrans_getSourceSet
 579  */
 580
 581 /* A simple transform with a small filter & source set: rules 50-100 chars unescaped, 100-200 chars escaped,
 582    filter & source set 4-20 chars */
 583 static const UChar transSimpleID[] = { 0x79,0x6F,0x2D,0x79,0x6F,0x5F,0x42,0x4A,0 }; /* "yo-yo_BJ" */
 584 static const char* transSimpleCName = "yo-yo_BJ";
 585
 586 enum { kUBufMax = 256 };
 587 static void TestGetRulesAndSourceSet() {
 588     UErrorCode status = U_ZERO_ERROR;
 589     UTransliterator *utrans = utrans_openU(transSimpleID, -1, UTRANS_FORWARD, NULL, 0, NULL, &status);
 590     if ( U_SUCCESS(status) ) {
 591         USet* uset;
 592         UChar ubuf[kUBufMax];
 593         int32_t ulen;
 594
 595         status = U_ZERO_ERROR;
 596         ulen = utrans_toRules(utrans, FALSE, ubuf, kUBufMax, &status);
 597         if ( U_FAILURE(status) || ulen <= 50 || ulen >= 100) {
 598             log_err("FAIL: utrans_toRules unescaped, expected noErr and len 50-100, got error=%s and len=%d\n",
 599                     u_errorName(status), ulen);
 600         }
 601
 602         status = U_ZERO_ERROR;
 603         ulen = utrans_toRules(utrans, FALSE, NULL, 0, &status);
 604         if ( status != U_BUFFER_OVERFLOW_ERROR || ulen <= 50 || ulen >= 100) {
 605             log_err("FAIL: utrans_toRules unescaped, expected U_BUFFER_OVERFLOW_ERROR and len 50-100, got error=%s and len=%d\n",
 606                     u_errorName(status), ulen);
 607         }
 608
 609         status = U_ZERO_ERROR;
 610         ulen = utrans_toRules(utrans, TRUE, ubuf, kUBufMax, &status);
 611         if ( U_FAILURE(status) || ulen <= 100 || ulen >= 200) {
 612             log_err("FAIL: utrans_toRules escaped, expected noErr and len 100-200, got error=%s and len=%d\n",
 613                     u_errorName(status), ulen);
 614         }
 615
 616         status = U_ZERO_ERROR;
 617         uset = utrans_getSourceSet(utrans, FALSE, NULL, &status);
 618         ulen = uset_toPattern(uset, ubuf, kUBufMax, FALSE, &status);
 619         uset_close(uset);
 620         if ( U_FAILURE(status) || ulen <= 4 || ulen >= 20) {
 621             log_err("FAIL: utrans_getSourceSet useFilter, expected noErr and len 4-20, got error=%s and len=%d\n",
 622                     u_errorName(status), ulen);
 623         }
 624
 625         status = U_ZERO_ERROR;
 626         uset = utrans_getSourceSet(utrans, TRUE, NULL, &status);
 627         ulen = uset_toPattern(uset, ubuf, kUBufMax, FALSE, &status);
 628         uset_close(uset);
 629         if ( U_FAILURE(status) || ulen <= 4 || ulen >= 20) {
 630             log_err("FAIL: utrans_getSourceSet ignoreFilter, expected noErr and len 4-20, got error=%s and len=%d\n",
 631                     u_errorName(status), ulen);
 632         }
 633
 634         utrans_close(utrans);
 635     } else {
 636         log_data_err("FAIL: utrans_openRules(%s) failed, error=%s (Are you missing data?)\n",
 637                 transSimpleCName, u_errorName(status));
 638     }
 639 }
 640
 641 typedef struct {
 642     const char * transID;
 643     const char * sourceText;
 644     const char * targetText;
 645 } TransIDSourceTarg;
 646
 647 static const TransIDSourceTarg dataVarCompItems[] = {
 648     { "Simplified-Traditional",
 649        "\\u4E0B\\u9762\\u662F\\u4E00\\u4E9B\\u4ECE\\u7B80\\u4F53\\u8F6C\\u6362\\u4E3A\\u7E41\\u4F53\\u5B57\\u793A\\u4F8B\\u6587\\u672C\\u3002",
 650        "\\u4E0B\\u9762\\u662F\\u4E00\\u4E9B\\u5F9E\\u7C21\\u9AD4\\u8F49\\u63DB\\u70BA\\u7E41\\u9AD4\\u5B57\\u793A\\u4F8B\\u6587\\u672C\\u3002" },
 651     { "Halfwidth-Fullwidth",
 652       "Sample text, \\uFF7B\\uFF9D\\uFF8C\\uFF9F\\uFF99\\uFF83\\uFF77\\uFF7D\\uFF84.",
 653       "\\uFF33\\uFF41\\uFF4D\\uFF50\\uFF4C\\uFF45\\u3000\\uFF54\\uFF45\\uFF58\\uFF54\\uFF0C\\u3000\\u30B5\\u30F3\\u30D7\\u30EB\\u30C6\\u30AD\\u30B9\\u30C8\\uFF0E" },
 654     { "Han-Latin/Names; Latin-Bopomofo",
 655        "\\u4E07\\u4FDF\\u919C\\u5974\\u3001\\u533A\\u695A\\u826F\\u3001\\u4EFB\\u70E8\\u3001\\u5CB3\\u98DB",
 656        "\\u3107\\u311B\\u02CB \\u3111\\u3127\\u02CA \\u3114\\u3121\\u02C7 \\u310B\\u3128\\u02CA\\u3001 \\u3121 \\u3114\\u3128\\u02C7 \\u310C\\u3127\\u3124\\u02CA\\u3001 \\u3116\\u3123\\u02CA \\u3127\\u311D\\u02CB\\u3001 \\u3129\\u311D\\u02CB \\u3108\\u311F" },
 657     { "Greek-Latin",
 658       "\\u1F08 \\u1FBC \\u1F89 \\u1FEC",
 659       "A \\u0100I H\\u0100I RH" },
 660     { "Greek-Latin/BGN",
 661       "\\u1F08 \\u1FBC \\u1F89 \\u1FEC",
 662       "A\\u0313 A\\u0345 A\\u0314\\u0345 \\u1FEC" },
 663     { "Greek-Latin/UNGEGN",
 664       "\\u1F08 \\u1FBC \\u1F89 \\u1FEC",
 665       "A A A R" },
 666     { NULL, NULL, NULL }
 667 };
 668
 669 enum { kBBufMax = 384 };
 670 static void TestDataVariantsCompounds() {
 671     const TransIDSourceTarg* itemsPtr;
 672     for (itemsPtr = dataVarCompItems; itemsPtr->transID != NULL; itemsPtr++) {
 673         UErrorCode status = U_ZERO_ERROR;
 674         UChar utrid[kUBufMax];
 675         int32_t utridlen = u_unescape(itemsPtr->transID, utrid, kUBufMax);
 676         UTransliterator* utrans = utrans_openU(utrid, utridlen, UTRANS_FORWARD, NULL, 0, NULL, &status);
 677         if (U_FAILURE(status)) {
 678             log_data_err("FAIL: utrans_openRules(%s) failed, error=%s (Are you missing data?)\n", itemsPtr->transID, u_errorName(status));
 679             continue;
 680         }
 681         UChar text[kUBufMax];
 682         int32_t textLen =  u_unescape(itemsPtr->sourceText, text, kUBufMax);
 683         int32_t textLim = textLen;
 684         utrans_transUChars(utrans, text, &textLen, kUBufMax, 0, &textLim, &status);
 685         if (U_FAILURE(status)) {
 686             log_err("FAIL: utrans_transUChars(%s) failed, error=%s\n", itemsPtr->transID, u_errorName(status));
 687         } else {
 688             UChar expect[kUBufMax];
 689             int32_t expectLen =  u_unescape(itemsPtr->targetText, expect, kUBufMax);
 690             if (textLen != expectLen || u_strncmp(text, expect, textLen) != 0) {
 691                 char btext[kBBufMax], bexpect[kBBufMax];
 692                 u_austrncpy(btext, text, textLen);
 693                 u_austrncpy(bexpect, expect, expectLen);
 694                 log_err("FAIL: utrans_transUChars(%s),\n       expect %s\n       get    %s\n", itemsPtr->transID, bexpect, btext);
 695             }
 696         }
 697         utrans_close(utrans);
 698     }
 699 }
 700
 701 static void _expectRules(const char* crules,
 702                   const char* cfrom,
 703                   const char* cto) {
 704     /* u_uastrcpy has no capacity param for the buffer -- so just
 705      * make all buffers way too big */
 706     enum { CAP = 256 };
 707     UChar rules[CAP];
 708     UTransliterator *trans;
 709     UErrorCode status = U_ZERO_ERROR;
 710     UParseError parseErr;
 711
 712     u_uastrcpy(rules, crules);
 713
 714     trans = utrans_open(crules /*use rules as ID*/, UTRANS_FORWARD, rules, -1,
 715                              &parseErr, &status);
 716     if (U_FAILURE(status)) {
 717         utrans_close(trans);
 718         log_data_err("FAIL: utrans_openRules(%s) failed, error=%s (Are you missing data?)\n",
 719                 crules, u_errorName(status));
 720         return;
 721     }
 722
 723     _expect(trans, cfrom, cto);
 724
 725     utrans_close(trans);
 726 }
 727
 728 static void _expect(const UTransliterator* trans,
 729              const char* cfrom,
 730              const char* cto) {
 731     /* u_uastrcpy has no capacity param for the buffer -- so just
 732      * make all buffers way too big */
 733     enum { CAP = 256 };
 734     UChar from[CAP];
 735     UChar to[CAP];
 736     UChar buf[CAP];
 737     const UChar *ID;
 738     int32_t IDLength;
 739     const char *id;
 740
 741     UErrorCode status = U_ZERO_ERROR;
 742     int32_t limit;
 743     UTransPosition pos;
 744     XReplaceable xrep;
 745     XReplaceable *xrepPtr = &xrep;
 746     UReplaceableCallbacks xrepVtable;
 747
 748     u_uastrcpy(from, cfrom);
 749     u_uastrcpy(to, cto);
 750
 751     ID = utrans_getUnicodeID(trans, &IDLength);
 752     id = aescstrdup(ID, IDLength);
 753
 754     /* utrans_transUChars() */
 755     u_strcpy(buf, from);
 756     limit = u_strlen(buf);
 757     utrans_transUChars(trans, buf, NULL, CAP, 0, &limit, &status);
 758     if (U_FAILURE(status)) {
 759         log_err("FAIL: utrans_transUChars() failed, error=%s\n",
 760                 u_errorName(status));
 761         return;
 762     }
 763
 764     if (0 == u_strcmp(buf, to)) {
 765         log_verbose("Ok: utrans_transUChars(%s) x %s -> %s\n",
 766                     id, cfrom, cto);
 767     } else {
 768         char actual[CAP];
 769         u_austrcpy(actual, buf);
 770         log_err("FAIL: utrans_transUChars(%s) x %s -> %s, expected %s\n",
 771                 id, cfrom, actual, cto);
 772     }
 773
 774     /* utrans_transIncrementalUChars() */
 775     u_strcpy(buf, from);
 776     pos.start = pos.contextStart = 0;
 777     pos.limit = pos.contextLimit = u_strlen(buf);
 778     utrans_transIncrementalUChars(trans, buf, NULL, CAP, &pos, &status);
 779     utrans_transUChars(trans, buf, NULL, CAP, pos.start, &pos.limit, &status);
 780     if (U_FAILURE(status)) {
 781         log_err("FAIL: utrans_transIncrementalUChars() failed, error=%s\n",
 782                 u_errorName(status));
 783         return;
 784     }
 785
 786     if (0 == u_strcmp(buf, to)) {
 787         log_verbose("Ok: utrans_transIncrementalUChars(%s) x %s -> %s\n",
 788                     id, cfrom, cto);
 789     } else {
 790         char actual[CAP];
 791         u_austrcpy(actual, buf);
 792         log_err("FAIL: utrans_transIncrementalUChars(%s) x %s -> %s, expected %s\n",
 793                 id, cfrom, actual, cto);
 794     }
 795
 796     /* utrans_trans() */
 797     InitXReplaceableCallbacks(&xrepVtable);
 798     InitXReplaceable(&xrep, cfrom);
 799     limit = u_strlen(from);
 800     utrans_trans(trans, (UReplaceable*)xrepPtr, &xrepVtable, 0, &limit, &status);
 801     if (U_FAILURE(status)) {
 802         log_err("FAIL: utrans_trans() failed, error=%s\n",
 803                 u_errorName(status));
 804         FreeXReplaceable(&xrep);
 805         return;
 806     }
 807
 808     if (0 == u_strcmp(xrep.text, to)) {
 809         log_verbose("Ok: utrans_trans(%s) x %s -> %s\n",
 810                     id, cfrom, cto);
 811     } else {
 812         char actual[CAP];
 813         u_austrcpy(actual, xrep.text);
 814         log_err("FAIL: utrans_trans(%s) x %s -> %s, expected %s\n",
 815                 id, cfrom, actual, cto);
 816     }
 817     FreeXReplaceable(&xrep);
 818
 819     /* utrans_transIncremental() */
 820     InitXReplaceable(&xrep, cfrom);
 821     pos.start = pos.contextStart = 0;
 822     pos.limit = pos.contextLimit = u_strlen(from);
 823     utrans_transIncremental(trans, (UReplaceable*)xrepPtr, &xrepVtable, &pos, &status);
 824     utrans_trans(trans, (UReplaceable*)xrepPtr, &xrepVtable, pos.start, &pos.limit, &status);
 825     if (U_FAILURE(status)) {
 826         log_err("FAIL: utrans_transIncremental() failed, error=%s\n",
 827                 u_errorName(status));
 828         FreeXReplaceable(&xrep);
 829         return;
 830     }
 831
 832     if (0 == u_strcmp(xrep.text, to)) {
 833         log_verbose("Ok: utrans_transIncremental(%s) x %s -> %s\n",
 834                     id, cfrom, cto);
 835     } else {
 836         char actual[CAP];
 837         u_austrcpy(actual, xrep.text);
 838         log_err("FAIL: utrans_transIncremental(%s) x %s -> %s, expected %s\n",
 839                 id, cfrom, actual, cto);
 840     }
 841     FreeXReplaceable(&xrep);
 842 }
 843
 844 #endif /* #if !UCONFIG_NO_TRANSLITERATION */