icuSources/test/intltest/ucdtest.cpp

   1 /********************************************************************
   2  * COPYRIGHT:
   3  * Copyright (c) 1997-2013, International Business Machines Corporation and
   4  * others. All Rights Reserved.
   5  ********************************************************************/
   6
   7 #include "unicode/ustring.h"
   8 #include "unicode/uchar.h"
   9 #include "unicode/uniset.h"
  10 #include "unicode/putil.h"
  11 #include "unicode/uscript.h"
  12 #include "cstring.h"
  13 #include "hash.h"
  14 #include "patternprops.h"
  15 #include "normalizer2impl.h"
  16 #include "uparse.h"
  17 #include "ucdtest.h"
  18
  19 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof(array[0]))
  20
  21 static const char *ignorePropNames[]={
  22     "FC_NFKC",
  23     "NFD_QC",
  24     "NFC_QC",
  25     "NFKD_QC",
  26     "NFKC_QC",
  27     "Expands_On_NFD",
  28     "Expands_On_NFC",
  29     "Expands_On_NFKD",
  30     "Expands_On_NFKC",
  31     "NFKC_CF"
  32 };
  33
  34 UnicodeTest::UnicodeTest()
  35 {
  36     UErrorCode errorCode=U_ZERO_ERROR;
  37     unknownPropertyNames=new U_NAMESPACE_QUALIFIER Hashtable(errorCode);
  38     if(U_FAILURE(errorCode)) {
  39         delete unknownPropertyNames;
  40         unknownPropertyNames=NULL;
  41     }
  42     // Ignore some property names altogether.
  43     for(int32_t i=0; i<LENGTHOF(ignorePropNames); ++i) {
  44         unknownPropertyNames->puti(UnicodeString(ignorePropNames[i], -1, US_INV), 1, errorCode);
  45     }
  46 }
  47
  48 UnicodeTest::~UnicodeTest()
  49 {
  50     delete unknownPropertyNames;
  51 }
  52
  53 void UnicodeTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ )
  54 {
  55     if(exec) {
  56         logln("TestSuite UnicodeTest: ");
  57     }
  58     TESTCASE_AUTO_BEGIN;
  59     TESTCASE_AUTO(TestAdditionalProperties);
  60     TESTCASE_AUTO(TestBinaryValues);
  61     TESTCASE_AUTO(TestConsistency);
  62     TESTCASE_AUTO(TestPatternProperties);
  63     TESTCASE_AUTO(TestScriptMetadata);
  64     TESTCASE_AUTO_END;
  65 }
  66
  67 //====================================================
  68 // private data used by the tests
  69 //====================================================
  70
  71 // test DerivedCoreProperties.txt -------------------------------------------
  72
  73 // copied from genprops.c
  74 static int32_t
  75 getTokenIndex(const char *const tokens[], int32_t countTokens, const char *s) {
  76     const char *t, *z;
  77     int32_t i, j;
  78
  79     s=u_skipWhitespace(s);
  80     for(i=0; i<countTokens; ++i) {
  81         t=tokens[i];
  82         if(t!=NULL) {
  83             for(j=0;; ++j) {
  84                 if(t[j]!=0) {
  85                     if(s[j]!=t[j]) {
  86                         break;
  87                     }
  88                 } else {
  89                     z=u_skipWhitespace(s+j);
  90                     if(*z==';' || *z==0) {
  91                         return i;
  92                     } else {
  93                         break;
  94                     }
  95                 }
  96             }
  97         }
  98     }
  99     return -1;
 100 }
 101
 102 static const char *const
 103 derivedPropsNames[]={
 104     "Math",
 105     "Alphabetic",
 106     "Lowercase",
 107     "Uppercase",
 108     "ID_Start",
 109     "ID_Continue",
 110     "XID_Start",
 111     "XID_Continue",
 112     "Default_Ignorable_Code_Point",
 113     "Full_Composition_Exclusion",
 114     "Grapheme_Extend",
 115     "Grapheme_Link", /* Unicode 5 moves this property here from PropList.txt */
 116     "Grapheme_Base",
 117     "Cased",
 118     "Case_Ignorable",
 119     "Changes_When_Lowercased",
 120     "Changes_When_Uppercased",
 121     "Changes_When_Titlecased",
 122     "Changes_When_Casefolded",
 123     "Changes_When_Casemapped",
 124     "Changes_When_NFKC_Casefolded"
 125 };
 126
 127 static const UProperty
 128 derivedPropsIndex[]={
 129     UCHAR_MATH,
 130     UCHAR_ALPHABETIC,
 131     UCHAR_LOWERCASE,
 132     UCHAR_UPPERCASE,
 133     UCHAR_ID_START,
 134     UCHAR_ID_CONTINUE,
 135     UCHAR_XID_START,
 136     UCHAR_XID_CONTINUE,
 137     UCHAR_DEFAULT_IGNORABLE_CODE_POINT,
 138     UCHAR_FULL_COMPOSITION_EXCLUSION,
 139     UCHAR_GRAPHEME_EXTEND,
 140     UCHAR_GRAPHEME_LINK,
 141     UCHAR_GRAPHEME_BASE,
 142     UCHAR_CASED,
 143     UCHAR_CASE_IGNORABLE,
 144     UCHAR_CHANGES_WHEN_LOWERCASED,
 145     UCHAR_CHANGES_WHEN_UPPERCASED,
 146     UCHAR_CHANGES_WHEN_TITLECASED,
 147     UCHAR_CHANGES_WHEN_CASEFOLDED,
 148     UCHAR_CHANGES_WHEN_CASEMAPPED,
 149     UCHAR_CHANGES_WHEN_NFKC_CASEFOLDED
 150 };
 151
 152 static int32_t numErrors[LENGTHOF(derivedPropsIndex)]={ 0 };
 153
 154 enum { MAX_ERRORS=50 };
 155
 156 U_CFUNC void U_CALLCONV
 157 derivedPropsLineFn(void *context,
 158                    char *fields[][2], int32_t /* fieldCount */,
 159                    UErrorCode *pErrorCode)
 160 {
 161     UnicodeTest *me=(UnicodeTest *)context;
 162     uint32_t start, end;
 163     int32_t i;
 164
 165     u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode);
 166     if(U_FAILURE(*pErrorCode)) {
 167         me->errln("UnicodeTest: syntax error in DerivedCoreProperties.txt or DerivedNormalizationProps.txt field 0 at %s\n", fields[0][0]);
 168         return;
 169     }
 170
 171     /* parse derived binary property name, ignore unknown names */
 172     i=getTokenIndex(derivedPropsNames, LENGTHOF(derivedPropsNames), fields[1][0]);
 173     if(i<0) {
 174         UnicodeString propName(fields[1][0], (int32_t)(fields[1][1]-fields[1][0]));
 175         propName.trim();
 176         if(me->unknownPropertyNames->find(propName)==NULL) {
 177             UErrorCode errorCode=U_ZERO_ERROR;
 178             me->unknownPropertyNames->puti(propName, 1, errorCode);
 179             me->errln("UnicodeTest warning: unknown property name '%s' in DerivedCoreProperties.txt or DerivedNormalizationProps.txt\n", fields[1][0]);
 180         }
 181         return;
 182     }
 183
 184     me->derivedProps[i].add(start, end);
 185 }
 186
 187 void UnicodeTest::TestAdditionalProperties() {
 188 #if !UCONFIG_NO_NORMALIZATION
 189     // test DerivedCoreProperties.txt and DerivedNormalizationProps.txt
 190     if(LENGTHOF(derivedProps)<LENGTHOF(derivedPropsNames)) {
 191         errln("error: UnicodeTest::derivedProps[] too short, need at least %d UnicodeSets\n",
 192               LENGTHOF(derivedPropsNames));
 193         return;
 194     }
 195     if(LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)) {
 196         errln("error in ucdtest.cpp: LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)\n");
 197         return;
 198     }
 199
 200     char newPath[256];
 201     char backupPath[256];
 202     char *fields[2][2];
 203     UErrorCode errorCode=U_ZERO_ERROR;
 204
 205     /* Look inside ICU_DATA first */
 206     strcpy(newPath, pathToDataDirectory());
 207     strcat(newPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt");
 208
 209     // As a fallback, try to guess where the source data was located
 210     // at the time ICU was built, and look there.
 211 #   ifdef U_TOPSRCDIR
 212         strcpy(backupPath, U_TOPSRCDIR  U_FILE_SEP_STRING "data");
 213 #   else
 214         strcpy(backupPath, loadTestData(errorCode));
 215         strcat(backupPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING "data");
 216 #   endif
 217     strcat(backupPath, U_FILE_SEP_STRING);
 218     strcat(backupPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt");
 219
 220     char *path=newPath;
 221     u_parseDelimitedFile(newPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
 222
 223     if(errorCode==U_FILE_ACCESS_ERROR) {
 224         errorCode=U_ZERO_ERROR;
 225         path=backupPath;
 226         u_parseDelimitedFile(backupPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
 227     }
 228     if(U_FAILURE(errorCode)) {
 229         errln("error parsing DerivedCoreProperties.txt: %s\n", u_errorName(errorCode));
 230         return;
 231     }
 232     char *basename=path+strlen(path)-strlen("DerivedCoreProperties.txt");
 233     strcpy(basename, "DerivedNormalizationProps.txt");
 234     u_parseDelimitedFile(path, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
 235     if(U_FAILURE(errorCode)) {
 236         errln("error parsing DerivedNormalizationProps.txt: %s\n", u_errorName(errorCode));
 237         return;
 238     }
 239
 240     // now we have all derived core properties in the UnicodeSets
 241     // run them all through the API
 242     int32_t rangeCount, range;
 243     uint32_t i;
 244     UChar32 start, end;
 245
 246     // test all TRUE properties
 247     for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
 248         rangeCount=derivedProps[i].getRangeCount();
 249         for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
 250             start=derivedProps[i].getRangeStart(range);
 251             end=derivedProps[i].getRangeEnd(range);
 252             for(; start<=end; ++start) {
 253                 if(!u_hasBinaryProperty(start, derivedPropsIndex[i])) {
 254                     dataerrln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==FALSE is wrong", start, derivedPropsNames[i]);
 255                     if(++numErrors[i]>=MAX_ERRORS) {
 256                       dataerrln("Too many errors, moving to the next test");
 257                       break;
 258                     }
 259                 }
 260             }
 261         }
 262     }
 263
 264     // invert all properties
 265     for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
 266         derivedProps[i].complement();
 267     }
 268
 269     // test all FALSE properties
 270     for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
 271         rangeCount=derivedProps[i].getRangeCount();
 272         for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
 273             start=derivedProps[i].getRangeStart(range);
 274             end=derivedProps[i].getRangeEnd(range);
 275             for(; start<=end; ++start) {
 276                 if(u_hasBinaryProperty(start, derivedPropsIndex[i])) {
 277                     errln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==TRUE is wrong\n", start, derivedPropsNames[i]);
 278                     if(++numErrors[i]>=MAX_ERRORS) {
 279                       errln("Too many errors, moving to the next test");
 280                       break;
 281                     }
 282                 }
 283             }
 284         }
 285     }
 286 #endif /* !UCONFIG_NO_NORMALIZATION */
 287 }
 288
 289 void UnicodeTest::TestBinaryValues() {
 290     /*
 291      * Unicode 5.1 explicitly defines binary property value aliases.
 292      * Verify that they are all recognized.
 293      */
 294     UErrorCode errorCode=U_ZERO_ERROR;
 295     UnicodeSet alpha(UNICODE_STRING_SIMPLE("[:Alphabetic:]"), errorCode);
 296     if(U_FAILURE(errorCode)) {
 297         dataerrln("UnicodeSet([:Alphabetic:]) failed - %s", u_errorName(errorCode));
 298         return;
 299     }
 300
 301     static const char *const falseValues[]={ "N", "No", "F", "False" };
 302     static const char *const trueValues[]={ "Y", "Yes", "T", "True" };
 303     int32_t i;
 304     for(i=0; i<LENGTHOF(falseValues); ++i) {
 305         UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
 306         pattern.insert(pattern.length()-2, UnicodeString(falseValues[i], -1, US_INV));
 307         errorCode=U_ZERO_ERROR;
 308         UnicodeSet set(pattern, errorCode);
 309         if(U_FAILURE(errorCode)) {
 310             errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", falseValues[i], u_errorName(errorCode));
 311             continue;
 312         }
 313         set.complement();
 314         if(set!=alpha) {
 315             errln("UnicodeSet([:Alphabetic=%s:]).complement()!=UnicodeSet([:Alphabetic:])\n", falseValues[i]);
 316         }
 317     }
 318     for(i=0; i<LENGTHOF(trueValues); ++i) {
 319         UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
 320         pattern.insert(pattern.length()-2, UnicodeString(trueValues[i], -1, US_INV));
 321         errorCode=U_ZERO_ERROR;
 322         UnicodeSet set(pattern, errorCode);
 323         if(U_FAILURE(errorCode)) {
 324             errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", trueValues[i], u_errorName(errorCode));
 325             continue;
 326         }
 327         if(set!=alpha) {
 328             errln("UnicodeSet([:Alphabetic=%s:])!=UnicodeSet([:Alphabetic:])\n", trueValues[i]);
 329         }
 330     }
 331 }
 332
 333 void UnicodeTest::TestConsistency() {
 334 #if !UCONFIG_NO_NORMALIZATION
 335     /*
 336      * Test for an example that getCanonStartSet() delivers
 337      * all characters that compose from the input one,
 338      * even in multiple steps.
 339      * For example, the set for "I" (0049) should contain both
 340      * I-diaeresis (00CF) and I-diaeresis-acute (1E2E).
 341      * In general, the set for the middle such character should be a subset
 342      * of the set for the first.
 343      */
 344     IcuTestErrorCode errorCode(*this, "TestConsistency");
 345     const Normalizer2 *nfd=Normalizer2::getNFDInstance(errorCode);
 346     const Normalizer2Impl *nfcImpl=Normalizer2Factory::getNFCImpl(errorCode);
 347     if(!nfcImpl->ensureCanonIterData(errorCode) || errorCode.isFailure()) {
 348         dataerrln("Normalizer2::getInstance(NFD) or Normalizer2Factory::getNFCImpl() failed - %s\n",
 349                   errorCode.errorName());
 350         errorCode.reset();
 351         return;
 352     }
 353
 354     UnicodeSet set1, set2;
 355     if (nfcImpl->getCanonStartSet(0x49, set1)) {
 356         /* enumerate all characters that are plausible to be latin letters */
 357         for(UChar start=0xa0; start<0x2000; ++start) {
 358             UnicodeString decomp=nfd->normalize(UnicodeString(start), errorCode);
 359             if(decomp.length()>1 && decomp[0]==0x49) {
 360                 set2.add(start);
 361             }
 362         }
 363
 364         if (set1!=set2) {
 365             errln("[canon start set of 0049] != [all c with canon decomp with 0049]");
 366         }
 367         // This was available in cucdtst.c but the test had to move to intltest
 368         // because the new internal normalization functions are in C++.
 369         //compareUSets(set1, set2,
 370         //             "[canon start set of 0049]", "[all c with canon decomp with 0049]",
 371         //             TRUE);
 372     } else {
 373         errln("NFC.getCanonStartSet() returned FALSE");
 374     }
 375 #endif
 376 }
 377
 378 /**
 379  * Test various implementations of Pattern_Syntax & Pattern_White_Space.
 380  */
 381 void UnicodeTest::TestPatternProperties() {
 382     IcuTestErrorCode errorCode(*this, "TestPatternProperties()");
 383     UnicodeSet syn_pp;
 384     UnicodeSet syn_prop(UNICODE_STRING_SIMPLE("[:Pattern_Syntax:]"), errorCode);
 385     UnicodeSet syn_list(
 386         "[!-/\\:-@\\[-\\^`\\{-~"
 387         "\\u00A1-\\u00A7\\u00A9\\u00AB\\u00AC\\u00AE\\u00B0\\u00B1\\u00B6\\u00BB\\u00BF\\u00D7\\u00F7"
 388         "\\u2010-\\u2027\\u2030-\\u203E\\u2041-\\u2053\\u2055-\\u205E\\u2190-\\u245F\\u2500-\\u2775"
 389         "\\u2794-\\u2BFF\\u2E00-\\u2E7F\\u3001-\\u3003\\u3008-\\u3020\\u3030\\uFD3E\\uFD3F\\uFE45\\uFE46]", errorCode);
 390     UnicodeSet ws_pp;
 391     UnicodeSet ws_prop(UNICODE_STRING_SIMPLE("[:Pattern_White_Space:]"), errorCode);
 392     UnicodeSet ws_list(UNICODE_STRING_SIMPLE("[\\u0009-\\u000D\\ \\u0085\\u200E\\u200F\\u2028\\u2029]"), errorCode);
 393     UnicodeSet syn_ws_pp;
 394     UnicodeSet syn_ws_prop(syn_prop);
 395     syn_ws_prop.addAll(ws_prop);
 396     for(UChar32 c=0; c<=0xffff; ++c) {
 397         if(PatternProps::isSyntax(c)) {
 398             syn_pp.add(c);
 399         }
 400         if(PatternProps::isWhiteSpace(c)) {
 401             ws_pp.add(c);
 402         }
 403         if(PatternProps::isSyntaxOrWhiteSpace(c)) {
 404             syn_ws_pp.add(c);
 405         }
 406     }
 407     compareUSets(syn_pp, syn_prop,
 408                  "PatternProps.isSyntax()", "[:Pattern_Syntax:]", TRUE);
 409     compareUSets(syn_pp, syn_list,
 410                  "PatternProps.isSyntax()", "[Pattern_Syntax ranges]", TRUE);
 411     compareUSets(ws_pp, ws_prop,
 412                  "PatternProps.isWhiteSpace()", "[:Pattern_White_Space:]", TRUE);
 413     compareUSets(ws_pp, ws_list,
 414                  "PatternProps.isWhiteSpace()", "[Pattern_White_Space ranges]", TRUE);
 415     compareUSets(syn_ws_pp, syn_ws_prop,
 416                  "PatternProps.isSyntaxOrWhiteSpace()",
 417                  "[[:Pattern_Syntax:][:Pattern_White_Space:]]", TRUE);
 418 }
 419
 420 // So far only minimal port of Java & cucdtst.c compareUSets().
 421 UBool
 422 UnicodeTest::compareUSets(const UnicodeSet &a, const UnicodeSet &b,
 423                           const char *a_name, const char *b_name,
 424                           UBool diffIsError) {
 425     UBool same= a==b;
 426     if(!same && diffIsError) {
 427         errln("Sets are different: %s vs. %s\n", a_name, b_name);
 428     }
 429     return same;
 430 }
 431
 432 namespace {
 433
 434 /**
 435  * Maps a special script code to the most common script of its encoded characters.
 436  */
 437 UScriptCode getCharScript(UScriptCode script) {
 438     switch(script) {
 439     case USCRIPT_SIMPLIFIED_HAN:
 440     case USCRIPT_TRADITIONAL_HAN:
 441         return USCRIPT_HAN;
 442     case USCRIPT_JAPANESE:
 443         return USCRIPT_HIRAGANA;
 444     case USCRIPT_KOREAN:
 445         return USCRIPT_HANGUL;
 446     default:
 447         return script;
 448     }
 449 }
 450
 451 }  // namespace
 452
 453 void UnicodeTest::TestScriptMetadata() {
 454     IcuTestErrorCode errorCode(*this, "TestScriptMetadata()");
 455     UnicodeSet rtl("[[:bc=R:][:bc=AL:]-[:Cn:]-[:sc=Common:]]", errorCode);
 456     // So far, sample characters are uppercase.
 457     // Georgian is special.
 458     UnicodeSet cased("[[:Lu:]-[:sc=Common:]-[:sc=Geor:]]", errorCode);
 459     for(int32_t sci = 0; sci < USCRIPT_CODE_LIMIT; ++sci) {
 460         UScriptCode sc = (UScriptCode)sci;
 461         // Run the test with -v to see which script has failures:
 462         // .../intltest$ make && ./intltest utility/UnicodeTest/TestScriptMetadata -v | grep -C 3 FAIL
 463         logln(uscript_getShortName(sc));
 464         UScriptUsage usage = uscript_getUsage(sc);
 465         UnicodeString sample = uscript_getSampleUnicodeString(sc);
 466         UnicodeSet scriptSet;
 467         scriptSet.applyIntPropertyValue(UCHAR_SCRIPT, sc, errorCode);
 468         if(usage == USCRIPT_USAGE_NOT_ENCODED) {
 469             assertTrue("not encoded, no sample", sample.isEmpty());
 470             assertFalse("not encoded, not RTL", uscript_isRightToLeft(sc));
 471             assertFalse("not encoded, not LB letters", uscript_breaksBetweenLetters(sc));
 472             assertFalse("not encoded, not cased", uscript_isCased(sc));
 473             assertTrue("not encoded, no characters", scriptSet.isEmpty());
 474         } else {
 475             assertFalse("encoded, has a sample character", sample.isEmpty());
 476             UChar32 firstChar = sample.char32At(0);
 477             UScriptCode charScript = getCharScript(sc);
 478             assertEquals("script(sample(script))",
 479                          charScript, uscript_getScript(firstChar, errorCode));
 480             assertEquals("RTL vs. set", rtl.contains(firstChar), uscript_isRightToLeft(sc));
 481             assertEquals("cased vs. set", cased.contains(firstChar), uscript_isCased(sc));
 482             assertEquals("encoded, has characters", sc == charScript, !scriptSet.isEmpty());
 483             if(uscript_isRightToLeft(sc)) {
 484                 rtl.removeAll(scriptSet);
 485             }
 486             if(uscript_isCased(sc)) {
 487                 cased.removeAll(scriptSet);
 488             }
 489         }
 490     }
 491     UnicodeString pattern;
 492     assertEquals("no remaining RTL characters",
 493                  UnicodeString("[]"), rtl.toPattern(pattern));
 494     assertEquals("no remaining cased characters",
 495                  UnicodeString("[]"), cased.toPattern(pattern));
 496
 497     assertTrue("Hani breaks between letters", uscript_breaksBetweenLetters(USCRIPT_HAN));
 498     assertTrue("Thai breaks between letters", uscript_breaksBetweenLetters(USCRIPT_THAI));
 499     assertFalse("Latn does not break between letters", uscript_breaksBetweenLetters(USCRIPT_LATIN));
 500 }