icuSources/test/intltest/ucdtest.cpp

   1 // © 2016 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3 /********************************************************************
   4  * Copyright (c) 1997-2016, International Business Machines Corporation and
   5  * others. All Rights Reserved.
   6  ********************************************************************/
   7
   8 #include "unicode/ustring.h"
   9 #include "unicode/uchar.h"
  10 #include "unicode/ucpmap.h"
  11 #include "unicode/uniset.h"
  12 #include "unicode/putil.h"
  13 #include "unicode/uscript.h"
  14 #include "unicode/uset.h"
  15 #include "cstring.h"
  16 #include "hash.h"
  17 #include "patternprops.h"
  18 #include "normalizer2impl.h"
  19 #include "testutil.h"
  20 #include "uparse.h"
  21 #include "ucdtest.h"
  22
  23 static const char *ignorePropNames[]={
  24     "FC_NFKC",
  25     "NFD_QC",
  26     "NFC_QC",
  27     "NFKD_QC",
  28     "NFKC_QC",
  29     "Expands_On_NFD",
  30     "Expands_On_NFC",
  31     "Expands_On_NFKD",
  32     "Expands_On_NFKC",
  33     "NFKC_CF"
  34 };
  35
  36 UnicodeTest::UnicodeTest()
  37 {
  38     UErrorCode errorCode=U_ZERO_ERROR;
  39     unknownPropertyNames=new U_NAMESPACE_QUALIFIER Hashtable(errorCode);
  40     if(U_FAILURE(errorCode)) {
  41         delete unknownPropertyNames;
  42         unknownPropertyNames=NULL;
  43     }
  44     // Ignore some property names altogether.
  45     for(int32_t i=0; i<UPRV_LENGTHOF(ignorePropNames); ++i) {
  46         unknownPropertyNames->puti(UnicodeString(ignorePropNames[i], -1, US_INV), 1, errorCode);
  47     }
  48 }
  49
  50 UnicodeTest::~UnicodeTest()
  51 {
  52     delete unknownPropertyNames;
  53 }
  54
  55 void UnicodeTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ )
  56 {
  57     if(exec) {
  58         logln("TestSuite UnicodeTest: ");
  59     }
  60     TESTCASE_AUTO_BEGIN;
  61     TESTCASE_AUTO(TestAdditionalProperties);
  62     TESTCASE_AUTO(TestBinaryValues);
  63     TESTCASE_AUTO(TestConsistency);
  64     TESTCASE_AUTO(TestPatternProperties);
  65     TESTCASE_AUTO(TestScriptMetadata);
  66     TESTCASE_AUTO(TestBidiPairedBracketType);
  67     TESTCASE_AUTO(TestEmojiProperties);
  68     TESTCASE_AUTO(TestIndicPositionalCategory);
  69     TESTCASE_AUTO(TestIndicSyllabicCategory);
  70     TESTCASE_AUTO(TestVerticalOrientation);
  71     TESTCASE_AUTO(TestDefaultScriptExtensions);
  72     TESTCASE_AUTO(TestInvalidCodePointFolding);
  73 #if !UCONFIG_NO_NORMALIZATION
  74     TESTCASE_AUTO(TestBinaryCharacterProperties);
  75     TESTCASE_AUTO(TestIntCharacterProperties);
  76 #endif
  77     TESTCASE_AUTO_END;
  78 }
  79
  80 //====================================================
  81 // private data used by the tests
  82 //====================================================
  83
  84 // test DerivedCoreProperties.txt -------------------------------------------
  85
  86 // copied from genprops.c
  87 static int32_t
  88 getTokenIndex(const char *const tokens[], int32_t countTokens, const char *s) {
  89     const char *t, *z;
  90     int32_t i, j;
  91
  92     s=u_skipWhitespace(s);
  93     for(i=0; i<countTokens; ++i) {
  94         t=tokens[i];
  95         if(t!=NULL) {
  96             for(j=0;; ++j) {
  97                 if(t[j]!=0) {
  98                     if(s[j]!=t[j]) {
  99                         break;
 100                     }
 101                 } else {
 102                     z=u_skipWhitespace(s+j);
 103                     if(*z==';' || *z==0) {
 104                         return i;
 105                     } else {
 106                         break;
 107                     }
 108                 }
 109             }
 110         }
 111     }
 112     return -1;
 113 }
 114
 115 static const char *const
 116 derivedPropsNames[]={
 117     "Math",
 118     "Alphabetic",
 119     "Lowercase",
 120     "Uppercase",
 121     "ID_Start",
 122     "ID_Continue",
 123     "XID_Start",
 124     "XID_Continue",
 125     "Default_Ignorable_Code_Point",
 126     "Full_Composition_Exclusion",
 127     "Grapheme_Extend",
 128     "Grapheme_Link", /* Unicode 5 moves this property here from PropList.txt */
 129     "Grapheme_Base",
 130     "Cased",
 131     "Case_Ignorable",
 132     "Changes_When_Lowercased",
 133     "Changes_When_Uppercased",
 134     "Changes_When_Titlecased",
 135     "Changes_When_Casefolded",
 136     "Changes_When_Casemapped",
 137     "Changes_When_NFKC_Casefolded"
 138 };
 139
 140 static const UProperty
 141 derivedPropsIndex[]={
 142     UCHAR_MATH,
 143     UCHAR_ALPHABETIC,
 144     UCHAR_LOWERCASE,
 145     UCHAR_UPPERCASE,
 146     UCHAR_ID_START,
 147     UCHAR_ID_CONTINUE,
 148     UCHAR_XID_START,
 149     UCHAR_XID_CONTINUE,
 150     UCHAR_DEFAULT_IGNORABLE_CODE_POINT,
 151     UCHAR_FULL_COMPOSITION_EXCLUSION,
 152     UCHAR_GRAPHEME_EXTEND,
 153     UCHAR_GRAPHEME_LINK,
 154     UCHAR_GRAPHEME_BASE,
 155     UCHAR_CASED,
 156     UCHAR_CASE_IGNORABLE,
 157     UCHAR_CHANGES_WHEN_LOWERCASED,
 158     UCHAR_CHANGES_WHEN_UPPERCASED,
 159     UCHAR_CHANGES_WHEN_TITLECASED,
 160     UCHAR_CHANGES_WHEN_CASEFOLDED,
 161     UCHAR_CHANGES_WHEN_CASEMAPPED,
 162     UCHAR_CHANGES_WHEN_NFKC_CASEFOLDED
 163 };
 164
 165 static int32_t numErrors[UPRV_LENGTHOF(derivedPropsIndex)]={ 0 };
 166
 167 enum { MAX_ERRORS=50 };
 168
 169 U_CFUNC void U_CALLCONV
 170 derivedPropsLineFn(void *context,
 171                    char *fields[][2], int32_t /* fieldCount */,
 172                    UErrorCode *pErrorCode)
 173 {
 174     UnicodeTest *me=(UnicodeTest *)context;
 175     uint32_t start, end;
 176     int32_t i;
 177
 178     u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode);
 179     if(U_FAILURE(*pErrorCode)) {
 180         me->errln("UnicodeTest: syntax error in DerivedCoreProperties.txt or DerivedNormalizationProps.txt field 0 at %s\n", fields[0][0]);
 181         return;
 182     }
 183
 184     /* parse derived binary property name, ignore unknown names */
 185     i=getTokenIndex(derivedPropsNames, UPRV_LENGTHOF(derivedPropsNames), fields[1][0]);
 186     if(i<0) {
 187         UnicodeString propName(fields[1][0], (int32_t)(fields[1][1]-fields[1][0]));
 188         propName.trim();
 189         if(me->unknownPropertyNames->find(propName)==NULL) {
 190             UErrorCode errorCode=U_ZERO_ERROR;
 191             me->unknownPropertyNames->puti(propName, 1, errorCode);
 192             me->errln("UnicodeTest warning: unknown property name '%s' in DerivedCoreProperties.txt or DerivedNormalizationProps.txt\n", fields[1][0]);
 193         }
 194         return;
 195     }
 196
 197     me->derivedProps[i].add(start, end);
 198 }
 199
 200 void UnicodeTest::TestAdditionalProperties() {
 201 #if !UCONFIG_NO_NORMALIZATION
 202     // test DerivedCoreProperties.txt and DerivedNormalizationProps.txt
 203     if(UPRV_LENGTHOF(derivedProps)<UPRV_LENGTHOF(derivedPropsNames)) {
 204         errln("error: UnicodeTest::derivedProps[] too short, need at least %d UnicodeSets\n",
 205               UPRV_LENGTHOF(derivedPropsNames));
 206         return;
 207     }
 208     if(UPRV_LENGTHOF(derivedPropsIndex)!=UPRV_LENGTHOF(derivedPropsNames)) {
 209         errln("error in ucdtest.cpp: UPRV_LENGTHOF(derivedPropsIndex)!=UPRV_LENGTHOF(derivedPropsNames)\n");
 210         return;
 211     }
 212
 213     char path[500];
 214     if(getUnidataPath(path) == NULL) {
 215         errln("unable to find path to source/data/unidata/");
 216         return;
 217     }
 218     char *basename=strchr(path, 0);
 219     strcpy(basename, "DerivedCoreProperties.txt");
 220
 221     char *fields[2][2];
 222     UErrorCode errorCode=U_ZERO_ERROR;
 223     u_parseDelimitedFile(path, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
 224     if(U_FAILURE(errorCode)) {
 225         errln("error parsing DerivedCoreProperties.txt: %s\n", u_errorName(errorCode));
 226         return;
 227     }
 228
 229     strcpy(basename, "DerivedNormalizationProps.txt");
 230     u_parseDelimitedFile(path, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
 231     if(U_FAILURE(errorCode)) {
 232         errln("error parsing DerivedNormalizationProps.txt: %s\n", u_errorName(errorCode));
 233         return;
 234     }
 235
 236     // now we have all derived core properties in the UnicodeSets
 237     // run them all through the API
 238     int32_t rangeCount, range;
 239     uint32_t i;
 240     UChar32 start, end;
 241
 242     // test all TRUE properties
 243     for(i=0; i<UPRV_LENGTHOF(derivedPropsNames); ++i) {
 244         rangeCount=derivedProps[i].getRangeCount();
 245         for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
 246             start=derivedProps[i].getRangeStart(range);
 247             end=derivedProps[i].getRangeEnd(range);
 248             for(; start<=end; ++start) {
 249                 if(!u_hasBinaryProperty(start, derivedPropsIndex[i])) {
 250                     dataerrln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==FALSE is wrong", start, derivedPropsNames[i]);
 251                     if(++numErrors[i]>=MAX_ERRORS) {
 252                       dataerrln("Too many errors, moving to the next test");
 253                       break;
 254                     }
 255                 }
 256             }
 257         }
 258     }
 259
 260     // invert all properties
 261     for(i=0; i<UPRV_LENGTHOF(derivedPropsNames); ++i) {
 262         derivedProps[i].complement();
 263     }
 264
 265     // test all FALSE properties
 266     for(i=0; i<UPRV_LENGTHOF(derivedPropsNames); ++i) {
 267         rangeCount=derivedProps[i].getRangeCount();
 268         for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
 269             start=derivedProps[i].getRangeStart(range);
 270             end=derivedProps[i].getRangeEnd(range);
 271             for(; start<=end; ++start) {
 272                 if(u_hasBinaryProperty(start, derivedPropsIndex[i])) {
 273                     errln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==TRUE is wrong\n", start, derivedPropsNames[i]);
 274                     if(++numErrors[i]>=MAX_ERRORS) {
 275                       errln("Too many errors, moving to the next test");
 276                       break;
 277                     }
 278                 }
 279             }
 280         }
 281     }
 282 #endif /* !UCONFIG_NO_NORMALIZATION */
 283 }
 284
 285 void UnicodeTest::TestBinaryValues() {
 286     /*
 287      * Unicode 5.1 explicitly defines binary property value aliases.
 288      * Verify that they are all recognized.
 289      */
 290     UErrorCode errorCode=U_ZERO_ERROR;
 291     UnicodeSet alpha(UNICODE_STRING_SIMPLE("[:Alphabetic:]"), errorCode);
 292     if(U_FAILURE(errorCode)) {
 293         dataerrln("UnicodeSet([:Alphabetic:]) failed - %s", u_errorName(errorCode));
 294         return;
 295     }
 296
 297     static const char *const falseValues[]={ "N", "No", "F", "False" };
 298     static const char *const trueValues[]={ "Y", "Yes", "T", "True" };
 299     int32_t i;
 300     for(i=0; i<UPRV_LENGTHOF(falseValues); ++i) {
 301         UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
 302         pattern.insert(pattern.length()-2, UnicodeString(falseValues[i], -1, US_INV));
 303         errorCode=U_ZERO_ERROR;
 304         UnicodeSet set(pattern, errorCode);
 305         if(U_FAILURE(errorCode)) {
 306             errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", falseValues[i], u_errorName(errorCode));
 307             continue;
 308         }
 309         set.complement();
 310         if(set!=alpha) {
 311             errln("UnicodeSet([:Alphabetic=%s:]).complement()!=UnicodeSet([:Alphabetic:])\n", falseValues[i]);
 312         }
 313     }
 314     for(i=0; i<UPRV_LENGTHOF(trueValues); ++i) {
 315         UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
 316         pattern.insert(pattern.length()-2, UnicodeString(trueValues[i], -1, US_INV));
 317         errorCode=U_ZERO_ERROR;
 318         UnicodeSet set(pattern, errorCode);
 319         if(U_FAILURE(errorCode)) {
 320             errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", trueValues[i], u_errorName(errorCode));
 321             continue;
 322         }
 323         if(set!=alpha) {
 324             errln("UnicodeSet([:Alphabetic=%s:])!=UnicodeSet([:Alphabetic:])\n", trueValues[i]);
 325         }
 326     }
 327 }
 328
 329 void UnicodeTest::TestConsistency() {
 330 #if !UCONFIG_NO_NORMALIZATION
 331     /*
 332      * Test for an example that getCanonStartSet() delivers
 333      * all characters that compose from the input one,
 334      * even in multiple steps.
 335      * For example, the set for "I" (0049) should contain both
 336      * I-diaeresis (00CF) and I-diaeresis-acute (1E2E).
 337      * In general, the set for the middle such character should be a subset
 338      * of the set for the first.
 339      */
 340     IcuTestErrorCode errorCode(*this, "TestConsistency");
 341     const Normalizer2 *nfd=Normalizer2::getNFDInstance(errorCode);
 342     const Normalizer2Impl *nfcImpl=Normalizer2Factory::getNFCImpl(errorCode);
 343     if(!nfcImpl->ensureCanonIterData(errorCode) || errorCode.isFailure()) {
 344         dataerrln("Normalizer2::getInstance(NFD) or Normalizer2Factory::getNFCImpl() failed - %s\n",
 345                   errorCode.errorName());
 346         errorCode.reset();
 347         return;
 348     }
 349
 350     UnicodeSet set1, set2;
 351     if (nfcImpl->getCanonStartSet(0x49, set1)) {
 352         /* enumerate all characters that are plausible to be latin letters */
 353         for(UChar start=0xa0; start<0x2000; ++start) {
 354             UnicodeString decomp=nfd->normalize(UnicodeString(start), errorCode);
 355             if(decomp.length()>1 && decomp[0]==0x49) {
 356                 set2.add(start);
 357             }
 358         }
 359
 360         if (set1!=set2) {
 361             errln("[canon start set of 0049] != [all c with canon decomp with 0049]");
 362         }
 363         // This was available in cucdtst.c but the test had to move to intltest
 364         // because the new internal normalization functions are in C++.
 365         //compareUSets(set1, set2,
 366         //             "[canon start set of 0049]", "[all c with canon decomp with 0049]",
 367         //             TRUE);
 368     } else {
 369         errln("NFC.getCanonStartSet() returned FALSE");
 370     }
 371 #endif
 372 }
 373
 374 /**
 375  * Test various implementations of Pattern_Syntax & Pattern_White_Space.
 376  */
 377 void UnicodeTest::TestPatternProperties() {
 378     IcuTestErrorCode errorCode(*this, "TestPatternProperties()");
 379     UnicodeSet syn_pp;
 380     UnicodeSet syn_prop(UNICODE_STRING_SIMPLE("[:Pattern_Syntax:]"), errorCode);
 381     UnicodeSet syn_list(
 382         "[!-/\\:-@\\[-\\^`\\{-~"
 383         "\\u00A1-\\u00A7\\u00A9\\u00AB\\u00AC\\u00AE\\u00B0\\u00B1\\u00B6\\u00BB\\u00BF\\u00D7\\u00F7"
 384         "\\u2010-\\u2027\\u2030-\\u203E\\u2041-\\u2053\\u2055-\\u205E\\u2190-\\u245F\\u2500-\\u2775"
 385         "\\u2794-\\u2BFF\\u2E00-\\u2E7F\\u3001-\\u3003\\u3008-\\u3020\\u3030\\uFD3E\\uFD3F\\uFE45\\uFE46]", errorCode);
 386     UnicodeSet ws_pp;
 387     UnicodeSet ws_prop(UNICODE_STRING_SIMPLE("[:Pattern_White_Space:]"), errorCode);
 388     UnicodeSet ws_list(UNICODE_STRING_SIMPLE("[\\u0009-\\u000D\\ \\u0085\\u200E\\u200F\\u2028\\u2029]"), errorCode);
 389     UnicodeSet syn_ws_pp;
 390     UnicodeSet syn_ws_prop(syn_prop);
 391     syn_ws_prop.addAll(ws_prop);
 392     for(UChar32 c=0; c<=0xffff; ++c) {
 393         if(PatternProps::isSyntax(c)) {
 394             syn_pp.add(c);
 395         }
 396         if(PatternProps::isWhiteSpace(c)) {
 397             ws_pp.add(c);
 398         }
 399         if(PatternProps::isSyntaxOrWhiteSpace(c)) {
 400             syn_ws_pp.add(c);
 401         }
 402     }
 403     compareUSets(syn_pp, syn_prop,
 404                  "PatternProps.isSyntax()", "[:Pattern_Syntax:]", TRUE);
 405     compareUSets(syn_pp, syn_list,
 406                  "PatternProps.isSyntax()", "[Pattern_Syntax ranges]", TRUE);
 407     compareUSets(ws_pp, ws_prop,
 408                  "PatternProps.isWhiteSpace()", "[:Pattern_White_Space:]", TRUE);
 409     compareUSets(ws_pp, ws_list,
 410                  "PatternProps.isWhiteSpace()", "[Pattern_White_Space ranges]", TRUE);
 411     compareUSets(syn_ws_pp, syn_ws_prop,
 412                  "PatternProps.isSyntaxOrWhiteSpace()",
 413                  "[[:Pattern_Syntax:][:Pattern_White_Space:]]", TRUE);
 414 }
 415
 416 // So far only minimal port of Java & cucdtst.c compareUSets().
 417 UBool
 418 UnicodeTest::compareUSets(const UnicodeSet &a, const UnicodeSet &b,
 419                           const char *a_name, const char *b_name,
 420                           UBool diffIsError) {
 421     UBool same= a==b;
 422     if(!same && diffIsError) {
 423         errln("Sets are different: %s vs. %s\n", a_name, b_name);
 424     }
 425     return same;
 426 }
 427
 428 namespace {
 429
 430 /**
 431  * Maps a special script code to the most common script of its encoded characters.
 432  */
 433 UScriptCode getCharScript(UScriptCode script) {
 434     switch(script) {
 435     case USCRIPT_HAN_WITH_BOPOMOFO:
 436     case USCRIPT_SIMPLIFIED_HAN:
 437     case USCRIPT_TRADITIONAL_HAN:
 438         return USCRIPT_HAN;
 439     case USCRIPT_JAPANESE:
 440         return USCRIPT_HIRAGANA;
 441     case USCRIPT_JAMO:
 442     case USCRIPT_KOREAN:
 443         return USCRIPT_HANGUL;
 444     case USCRIPT_SYMBOLS_EMOJI:
 445         return USCRIPT_SYMBOLS;
 446     default:
 447         return script;
 448     }
 449 }
 450
 451 }  // namespace
 452
 453 void UnicodeTest::TestScriptMetadata() {
 454     IcuTestErrorCode errorCode(*this, "TestScriptMetadata()");
 455     UnicodeSet rtl("[[:bc=R:][:bc=AL:]-[:Cn:]-[:sc=Common:]]", errorCode);
 456     // So far, sample characters are uppercase.
 457     // Georgian is special.
 458     UnicodeSet cased("[[:Lu:]-[:sc=Common:]-[:sc=Geor:]]", errorCode);
 459     for(int32_t sci = 0; sci < USCRIPT_CODE_LIMIT; ++sci) {
 460         UScriptCode sc = (UScriptCode)sci;
 461         // Run the test with -v to see which script has failures:
 462         // .../intltest$ make && ./intltest utility/UnicodeTest/TestScriptMetadata -v | grep -C 6 FAIL
 463         logln(uscript_getShortName(sc));
 464         UScriptUsage usage = uscript_getUsage(sc);
 465         UnicodeString sample = uscript_getSampleUnicodeString(sc);
 466         UnicodeSet scriptSet;
 467         scriptSet.applyIntPropertyValue(UCHAR_SCRIPT, sc, errorCode);
 468         if(usage == USCRIPT_USAGE_NOT_ENCODED) {
 469             assertTrue("not encoded, no sample", sample.isEmpty());
 470             assertFalse("not encoded, not RTL", uscript_isRightToLeft(sc));
 471             assertFalse("not encoded, not LB letters", uscript_breaksBetweenLetters(sc));
 472             assertFalse("not encoded, not cased", uscript_isCased(sc));
 473             assertTrue("not encoded, no characters", scriptSet.isEmpty());
 474         } else {
 475             assertFalse("encoded, has a sample character", sample.isEmpty());
 476             UChar32 firstChar = sample.char32At(0);
 477             UScriptCode charScript = getCharScript(sc);
 478             assertEquals("script(sample(script))",
 479                          (int32_t)charScript, (int32_t)uscript_getScript(firstChar, errorCode));
 480             assertEquals("RTL vs. set", (UBool)rtl.contains(firstChar), (UBool)uscript_isRightToLeft(sc));
 481             assertEquals("cased vs. set", (UBool)cased.contains(firstChar), (UBool)uscript_isCased(sc));
 482             assertEquals("encoded, has characters", (UBool)(sc == charScript), (UBool)(!scriptSet.isEmpty()));
 483             if(uscript_isRightToLeft(sc)) {
 484                 rtl.removeAll(scriptSet);
 485             }
 486             if(uscript_isCased(sc)) {
 487                 cased.removeAll(scriptSet);
 488             }
 489         }
 490     }
 491     UnicodeString pattern;
 492     assertEquals("no remaining RTL characters",
 493                  UnicodeString("[]"), rtl.toPattern(pattern));
 494     assertEquals("no remaining cased characters",
 495                  UnicodeString("[]"), cased.toPattern(pattern));
 496
 497     assertTrue("Hani breaks between letters", uscript_breaksBetweenLetters(USCRIPT_HAN));
 498     assertTrue("Thai breaks between letters", uscript_breaksBetweenLetters(USCRIPT_THAI));
 499     assertFalse("Latn does not break between letters", uscript_breaksBetweenLetters(USCRIPT_LATIN));
 500 }
 501
 502 void UnicodeTest::TestBidiPairedBracketType() {
 503     // BidiBrackets-6.3.0.txt says:
 504     //
 505     // The set of code points listed in this file was originally derived
 506     // using the character properties General_Category (gc), Bidi_Class (bc),
 507     // Bidi_Mirrored (Bidi_M), and Bidi_Mirroring_Glyph (bmg), as follows:
 508     // two characters, A and B, form a pair if A has gc=Ps and B has gc=Pe,
 509     // both have bc=ON and Bidi_M=Y, and bmg of A is B. Bidi_Paired_Bracket
 510     // maps A to B and vice versa, and their Bidi_Paired_Bracket_Type
 511     // property values are Open and Close, respectively.
 512     IcuTestErrorCode errorCode(*this, "TestBidiPairedBracketType()");
 513     UnicodeSet bpt("[:^bpt=n:]", errorCode);
 514     assertTrue("bpt!=None is not empty", !bpt.isEmpty());
 515     // The following should always be true.
 516     UnicodeSet mirrored("[:Bidi_M:]", errorCode);
 517     UnicodeSet other_neutral("[:bc=ON:]", errorCode);
 518     assertTrue("bpt!=None is a subset of Bidi_M", mirrored.containsAll(bpt));
 519     assertTrue("bpt!=None is a subset of bc=ON", other_neutral.containsAll(bpt));
 520     // The following are true at least initially in Unicode 6.3.
 521     UnicodeSet bpt_open("[:bpt=o:]", errorCode);
 522     UnicodeSet bpt_close("[:bpt=c:]", errorCode);
 523     UnicodeSet ps("[:Ps:]", errorCode);
 524     UnicodeSet pe("[:Pe:]", errorCode);
 525     assertTrue("bpt=Open is a subset of Ps", ps.containsAll(bpt_open));
 526     assertTrue("bpt=Close is a subset of Pe", pe.containsAll(bpt_close));
 527 }
 528
 529 void UnicodeTest::TestEmojiProperties() {
 530     assertFalse("space is not Emoji", u_hasBinaryProperty(0x20, UCHAR_EMOJI));
 531     assertTrue("shooting star is Emoji", u_hasBinaryProperty(0x1F320, UCHAR_EMOJI));
 532     IcuTestErrorCode errorCode(*this, "TestEmojiProperties()");
 533     UnicodeSet emoji("[:Emoji:]", errorCode);
 534     assertTrue("lots of Emoji", emoji.size() > 700);
 535
 536     assertTrue("shooting star is Emoji_Presentation",
 537                u_hasBinaryProperty(0x1F320, UCHAR_EMOJI_PRESENTATION));
 538     assertTrue("Fitzpatrick 6 is Emoji_Modifier",
 539                u_hasBinaryProperty(0x1F3FF, UCHAR_EMOJI_MODIFIER));
 540     assertTrue("happy person is Emoji_Modifier_Base",
 541                u_hasBinaryProperty(0x1F64B, UCHAR_EMOJI_MODIFIER_BASE));
 542     assertTrue("asterisk is Emoji_Component",
 543                u_hasBinaryProperty(0x2A, UCHAR_EMOJI_COMPONENT));
 544     assertTrue("copyright is Extended_Pictographic",
 545                u_hasBinaryProperty(0xA9, UCHAR_EXTENDED_PICTOGRAPHIC));
 546 #if U_PLATFORM_IS_DARWIN_BASED
 547     assertTrue("TAG char is Emoji_Component",
 548                u_hasBinaryProperty(0xE0061, UCHAR_EMOJI_COMPONENT)); // Apple addition
 549     assertTrue("ZWJ char is Emoji_Component",
 550                u_hasBinaryProperty(0x200D, UCHAR_EMOJI_COMPONENT)); // Apple addition
 551 #endif
 552 }
 553
 554 void UnicodeTest::TestIndicPositionalCategory() {
 555     IcuTestErrorCode errorCode(*this, "TestIndicPositionalCategory()");
 556     UnicodeSet na(u"[:InPC=NA:]", errorCode);
 557     assertTrue("mostly NA", 1000000 <= na.size() && na.size() <= UCHAR_MAX_VALUE - 500);
 558     UnicodeSet vol(u"[:InPC=Visual_Order_Left:]", errorCode);
 559     assertTrue("some Visual_Order_Left", 19 <= vol.size() && vol.size() <= 100);
 560     assertEquals("U+08FF: NA", U_INPC_NA,
 561                  u_getIntPropertyValue(0x08FF, UCHAR_INDIC_POSITIONAL_CATEGORY));
 562     assertEquals("U+0900: Top", U_INPC_TOP,
 563                  u_getIntPropertyValue(0x0900, UCHAR_INDIC_POSITIONAL_CATEGORY));
 564     assertEquals("U+10A06: Overstruck", U_INPC_OVERSTRUCK,
 565                  u_getIntPropertyValue(0x10A06, UCHAR_INDIC_POSITIONAL_CATEGORY));
 566 }
 567
 568 void UnicodeTest::TestIndicSyllabicCategory() {
 569     IcuTestErrorCode errorCode(*this, "TestIndicSyllabicCategory()");
 570     UnicodeSet other(u"[:InSC=Other:]", errorCode);
 571     assertTrue("mostly Other", 1000000 <= other.size() && other.size() <= UCHAR_MAX_VALUE - 500);
 572     UnicodeSet ava(u"[:InSC=Avagraha:]", errorCode);
 573     assertTrue("some Avagraha", 16 <= ava.size() && ava.size() <= 100);
 574     assertEquals("U+08FF: Other", U_INSC_OTHER,
 575                  u_getIntPropertyValue(0x08FF, UCHAR_INDIC_SYLLABIC_CATEGORY));
 576     assertEquals("U+0900: Bindu", U_INSC_BINDU,
 577                  u_getIntPropertyValue(0x0900, UCHAR_INDIC_SYLLABIC_CATEGORY));
 578     assertEquals("U+11065: Brahmi_Joining_Number", U_INSC_BRAHMI_JOINING_NUMBER,
 579                  u_getIntPropertyValue(0x11065, UCHAR_INDIC_SYLLABIC_CATEGORY));
 580 }
 581
 582 void UnicodeTest::TestVerticalOrientation() {
 583     IcuTestErrorCode errorCode(*this, "TestVerticalOrientation()");
 584     UnicodeSet r(u"[:vo=R:]", errorCode);
 585     assertTrue("mostly R", 0xc0000 <= r.size() && r.size() <= 0xd0000);
 586     UnicodeSet u(u"[:vo=U:]", errorCode);
 587     assertTrue("much U", 0x40000 <= u.size() && u.size() <= 0x50000);
 588     UnicodeSet tu(u"[:vo=Tu:]", errorCode);
 589     assertTrue("some Tu", 147 <= tu.size() && tu.size() <= 300);
 590     assertEquals("U+0E01: Rotated", U_VO_ROTATED,
 591                  u_getIntPropertyValue(0x0E01, UCHAR_VERTICAL_ORIENTATION));
 592     assertEquals("U+3008: Transformed_Rotated", U_VO_TRANSFORMED_ROTATED,
 593                  u_getIntPropertyValue(0x3008, UCHAR_VERTICAL_ORIENTATION));
 594     assertEquals("U+33333: Upright", U_VO_UPRIGHT,
 595                  u_getIntPropertyValue(0x33333, UCHAR_VERTICAL_ORIENTATION));
 596 }
 597
 598 void UnicodeTest::TestDefaultScriptExtensions() {
 599     // Block 3000..303F CJK Symbols and Punctuation defaults to scx=Bopo Hang Hani Hira Kana Yiii
 600     // but some of its characters revert to scx=<script> which is usually Common.
 601     IcuTestErrorCode errorCode(*this, "TestDefaultScriptExtensions()");
 602     UScriptCode scx[20];
 603     scx[0] = USCRIPT_INVALID_CODE;
 604     assertEquals("U+3000 num scx", 1,  // IDEOGRAPHIC SPACE
 605                  uscript_getScriptExtensions(0x3000, scx, UPRV_LENGTHOF(scx), errorCode));
 606     assertEquals("U+3000 num scx[0]", USCRIPT_COMMON, scx[0]);
 607     scx[0] = USCRIPT_INVALID_CODE;
 608     assertEquals("U+3012 num scx", 1,  // POSTAL MARK
 609                  uscript_getScriptExtensions(0x3012, scx, UPRV_LENGTHOF(scx), errorCode));
 610     assertEquals("U+3012 num scx[0]", USCRIPT_COMMON, scx[0]);
 611 }
 612
 613 void UnicodeTest::TestInvalidCodePointFolding(void) {
 614     // Test behavior when an invalid code point is passed to u_foldCase
 615     static const UChar32 invalidCodePoints[] = {
 616             0xD800, // lead surrogate
 617             0xDFFF, // trail surrogate
 618             0xFDD0, // noncharacter
 619             0xFFFF, // noncharacter
 620             0x110000, // out of range
 621             -1 // negative
 622     };
 623     for (int32_t i=0; i<UPRV_LENGTHOF(invalidCodePoints); ++i) {
 624         UChar32 cp = invalidCodePoints[i];
 625         assertEquals("Invalid code points should be echoed back",
 626                 cp, u_foldCase(cp, U_FOLD_CASE_DEFAULT));
 627         assertEquals("Invalid code points should be echoed back",
 628                 cp, u_foldCase(cp, U_FOLD_CASE_EXCLUDE_SPECIAL_I));
 629     }
 630 }
 631
 632 void UnicodeTest::TestBinaryCharacterProperties() {
 633 #if !UCONFIG_NO_NORMALIZATION
 634     IcuTestErrorCode errorCode(*this, "TestBinaryCharacterProperties()");
 635     // Spot-check getBinaryPropertySet() vs. hasBinaryProperty().
 636     for (int32_t prop = 0; prop < UCHAR_BINARY_LIMIT; ++prop) {
 637         const USet *uset = u_getBinaryPropertySet((UProperty)prop, errorCode);
 638         if (errorCode.errIfFailureAndReset("u_getBinaryPropertySet(%d)", (int)prop)) {
 639             continue;
 640         }
 641         const UnicodeSet &set = *UnicodeSet::fromUSet(uset);
 642         int32_t size = set.size();
 643         if (size == 0) {
 644             assertFalse(UnicodeString("!hasBinaryProperty(U+0020, ") + prop + u")",
 645                 u_hasBinaryProperty(0x20, (UProperty)prop));
 646             assertFalse(UnicodeString("!hasBinaryProperty(U+0061, ") + prop + u")",
 647                 u_hasBinaryProperty(0x61, (UProperty)prop));
 648             assertFalse(UnicodeString("!hasBinaryProperty(U+4E00, ") + prop + u")",
 649                 u_hasBinaryProperty(0x4e00, (UProperty)prop));
 650         } else {
 651             UChar32 c = set.charAt(0);
 652             if (c > 0) {
 653                 assertFalse(
 654                     UnicodeString("!hasBinaryProperty(") + TestUtility::hex(c - 1) +
 655                         u", " + prop + u")",
 656                     u_hasBinaryProperty(c - 1, (UProperty)prop));
 657             }
 658             assertTrue(
 659                 UnicodeString("hasBinaryProperty(") + TestUtility::hex(c) +
 660                     u", " + prop + u")",
 661                 u_hasBinaryProperty(c, (UProperty)prop));
 662             c = set.charAt(size - 1);
 663             assertTrue(
 664                 UnicodeString("hasBinaryProperty(") + TestUtility::hex(c) +
 665                     u", " + prop + u")",
 666                 u_hasBinaryProperty(c, (UProperty)prop));
 667             if (c < 0x10ffff) {
 668                 assertFalse(
 669                     UnicodeString("!hasBinaryProperty(") + TestUtility::hex(c + 1) +
 670                         u", " + prop + u")",
 671                     u_hasBinaryProperty(c + 1, (UProperty)prop));
 672             }
 673         }
 674     }
 675 #endif
 676 }
 677
 678 void UnicodeTest::TestIntCharacterProperties() {
 679 #if !UCONFIG_NO_NORMALIZATION
 680     IcuTestErrorCode errorCode(*this, "TestIntCharacterProperties()");
 681     // Spot-check getIntPropertyMap() vs. getIntPropertyValue().
 682     for (int32_t prop = UCHAR_INT_START; prop < UCHAR_INT_LIMIT; ++prop) {
 683         const UCPMap *map = u_getIntPropertyMap((UProperty)prop, errorCode);
 684         if (errorCode.errIfFailureAndReset("u_getIntPropertyMap(%d)", (int)prop)) {
 685             continue;
 686         }
 687         uint32_t value;
 688         UChar32 end = ucpmap_getRange(map, 0, UCPMAP_RANGE_NORMAL, 0, nullptr, nullptr, &value);
 689         assertTrue("int property first range", end >= 0);
 690         UChar32 c = end / 2;
 691         assertEquals(UnicodeString("int property first range value at ") + TestUtility::hex(c),
 692             u_getIntPropertyValue(c, (UProperty)prop), value);
 693         end = ucpmap_getRange(map, 0x5000, UCPMAP_RANGE_NORMAL, 0, nullptr, nullptr, &value);
 694         assertTrue("int property later range", end >= 0);
 695         assertEquals(UnicodeString("int property later range value at ") + TestUtility::hex(end),
 696             u_getIntPropertyValue(end, (UProperty)prop), value);
 697         // ucpmap_get() API coverage
 698         // TODO: move to cucdtst.c
 699         assertEquals(
 700             "int property upcmap_get(U+0061)",
 701             u_getIntPropertyValue(0x61, (UProperty)prop), ucpmap_get(map, 0x61));
 702     }
 703 #endif
 704 }