icuSources/test/intltest/ucdtest.cpp

   1 /********************************************************************
   2  * COPYRIGHT:
   3  * Copyright (c) 1997-2011, International Business Machines Corporation and
   4  * others. All Rights Reserved.
   5  ********************************************************************/
   6
   7 #include "unicode/ustring.h"
   8 #include "unicode/uchar.h"
   9 #include "unicode/uniset.h"
  10 #include "unicode/putil.h"
  11 #include "cstring.h"
  12 #include "hash.h"
  13 #include "patternprops.h"
  14 #include "normalizer2impl.h"
  15 #include "uparse.h"
  16 #include "ucdtest.h"
  17
  18 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof(array[0]))
  19
  20 static const char *ignorePropNames[]={
  21     "FC_NFKC",
  22     "NFD_QC",
  23     "NFC_QC",
  24     "NFKD_QC",
  25     "NFKC_QC",
  26     "Expands_On_NFD",
  27     "Expands_On_NFC",
  28     "Expands_On_NFKD",
  29     "Expands_On_NFKC",
  30     "NFKC_CF"
  31 };
  32
  33 UnicodeTest::UnicodeTest()
  34 {
  35     UErrorCode errorCode=U_ZERO_ERROR;
  36     unknownPropertyNames=new U_NAMESPACE_QUALIFIER Hashtable(errorCode);
  37     if(U_FAILURE(errorCode)) {
  38         delete unknownPropertyNames;
  39         unknownPropertyNames=NULL;
  40     }
  41     // Ignore some property names altogether.
  42     for(int32_t i=0; i<LENGTHOF(ignorePropNames); ++i) {
  43         unknownPropertyNames->puti(UnicodeString(ignorePropNames[i], -1, US_INV), 1, errorCode);
  44     }
  45 }
  46
  47 UnicodeTest::~UnicodeTest()
  48 {
  49     delete unknownPropertyNames;
  50 }
  51
  52 void UnicodeTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ )
  53 {
  54     if(exec) {
  55         logln("TestSuite UnicodeTest: ");
  56     }
  57     TESTCASE_AUTO_BEGIN;
  58     TESTCASE_AUTO(TestAdditionalProperties);
  59     TESTCASE_AUTO(TestBinaryValues);
  60     TESTCASE_AUTO(TestConsistency);
  61     TESTCASE_AUTO(TestPatternProperties);
  62     TESTCASE_AUTO_END;
  63 }
  64
  65 //====================================================
  66 // private data used by the tests
  67 //====================================================
  68
  69 // test DerivedCoreProperties.txt -------------------------------------------
  70
  71 // copied from genprops.c
  72 static int32_t
  73 getTokenIndex(const char *const tokens[], int32_t countTokens, const char *s) {
  74     const char *t, *z;
  75     int32_t i, j;
  76
  77     s=u_skipWhitespace(s);
  78     for(i=0; i<countTokens; ++i) {
  79         t=tokens[i];
  80         if(t!=NULL) {
  81             for(j=0;; ++j) {
  82                 if(t[j]!=0) {
  83                     if(s[j]!=t[j]) {
  84                         break;
  85                     }
  86                 } else {
  87                     z=u_skipWhitespace(s+j);
  88                     if(*z==';' || *z==0) {
  89                         return i;
  90                     } else {
  91                         break;
  92                     }
  93                 }
  94             }
  95         }
  96     }
  97     return -1;
  98 }
  99
 100 static const char *const
 101 derivedPropsNames[]={
 102     "Math",
 103     "Alphabetic",
 104     "Lowercase",
 105     "Uppercase",
 106     "ID_Start",
 107     "ID_Continue",
 108     "XID_Start",
 109     "XID_Continue",
 110     "Default_Ignorable_Code_Point",
 111     "Full_Composition_Exclusion",
 112     "Grapheme_Extend",
 113     "Grapheme_Link", /* Unicode 5 moves this property here from PropList.txt */
 114     "Grapheme_Base",
 115     "Cased",
 116     "Case_Ignorable",
 117     "Changes_When_Lowercased",
 118     "Changes_When_Uppercased",
 119     "Changes_When_Titlecased",
 120     "Changes_When_Casefolded",
 121     "Changes_When_Casemapped",
 122     "Changes_When_NFKC_Casefolded"
 123 };
 124
 125 static const UProperty
 126 derivedPropsIndex[]={
 127     UCHAR_MATH,
 128     UCHAR_ALPHABETIC,
 129     UCHAR_LOWERCASE,
 130     UCHAR_UPPERCASE,
 131     UCHAR_ID_START,
 132     UCHAR_ID_CONTINUE,
 133     UCHAR_XID_START,
 134     UCHAR_XID_CONTINUE,
 135     UCHAR_DEFAULT_IGNORABLE_CODE_POINT,
 136     UCHAR_FULL_COMPOSITION_EXCLUSION,
 137     UCHAR_GRAPHEME_EXTEND,
 138     UCHAR_GRAPHEME_LINK,
 139     UCHAR_GRAPHEME_BASE,
 140     UCHAR_CASED,
 141     UCHAR_CASE_IGNORABLE,
 142     UCHAR_CHANGES_WHEN_LOWERCASED,
 143     UCHAR_CHANGES_WHEN_UPPERCASED,
 144     UCHAR_CHANGES_WHEN_TITLECASED,
 145     UCHAR_CHANGES_WHEN_CASEFOLDED,
 146     UCHAR_CHANGES_WHEN_CASEMAPPED,
 147     UCHAR_CHANGES_WHEN_NFKC_CASEFOLDED
 148 };
 149
 150 static int32_t numErrors[LENGTHOF(derivedPropsIndex)]={ 0 };
 151
 152 enum { MAX_ERRORS=50 };
 153
 154 U_CFUNC void U_CALLCONV
 155 derivedPropsLineFn(void *context,
 156                    char *fields[][2], int32_t /* fieldCount */,
 157                    UErrorCode *pErrorCode)
 158 {
 159     UnicodeTest *me=(UnicodeTest *)context;
 160     uint32_t start, end;
 161     int32_t i;
 162
 163     u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode);
 164     if(U_FAILURE(*pErrorCode)) {
 165         me->errln("UnicodeTest: syntax error in DerivedCoreProperties.txt or DerivedNormalizationProps.txt field 0 at %s\n", fields[0][0]);
 166         return;
 167     }
 168
 169     /* parse derived binary property name, ignore unknown names */
 170     i=getTokenIndex(derivedPropsNames, LENGTHOF(derivedPropsNames), fields[1][0]);
 171     if(i<0) {
 172         UnicodeString propName(fields[1][0], (int32_t)(fields[1][1]-fields[1][0]));
 173         propName.trim();
 174         if(me->unknownPropertyNames->find(propName)==NULL) {
 175             UErrorCode errorCode=U_ZERO_ERROR;
 176             me->unknownPropertyNames->puti(propName, 1, errorCode);
 177             me->errln("UnicodeTest warning: unknown property name '%s' in DerivedCoreProperties.txt or DerivedNormalizationProps.txt\n", fields[1][0]);
 178         }
 179         return;
 180     }
 181
 182     me->derivedProps[i].add(start, end);
 183 }
 184
 185 void UnicodeTest::TestAdditionalProperties() {
 186 #if !UCONFIG_NO_NORMALIZATION
 187     // test DerivedCoreProperties.txt and DerivedNormalizationProps.txt
 188     if(LENGTHOF(derivedProps)<LENGTHOF(derivedPropsNames)) {
 189         errln("error: UnicodeTest::derivedProps[] too short, need at least %d UnicodeSets\n",
 190               LENGTHOF(derivedPropsNames));
 191         return;
 192     }
 193     if(LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)) {
 194         errln("error in ucdtest.cpp: LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)\n");
 195         return;
 196     }
 197
 198     char newPath[256];
 199     char backupPath[256];
 200     char *fields[2][2];
 201     UErrorCode errorCode=U_ZERO_ERROR;
 202
 203     /* Look inside ICU_DATA first */
 204     strcpy(newPath, pathToDataDirectory());
 205     strcat(newPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt");
 206
 207     // As a fallback, try to guess where the source data was located
 208     // at the time ICU was built, and look there.
 209 #   ifdef U_TOPSRCDIR
 210         strcpy(backupPath, U_TOPSRCDIR  U_FILE_SEP_STRING "data");
 211 #   else
 212         strcpy(backupPath, loadTestData(errorCode));
 213         strcat(backupPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING "data");
 214 #   endif
 215     strcat(backupPath, U_FILE_SEP_STRING);
 216     strcat(backupPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt");
 217
 218     char *path=newPath;
 219     u_parseDelimitedFile(newPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
 220
 221     if(errorCode==U_FILE_ACCESS_ERROR) {
 222         errorCode=U_ZERO_ERROR;
 223         path=backupPath;
 224         u_parseDelimitedFile(backupPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
 225     }
 226     if(U_FAILURE(errorCode)) {
 227         errln("error parsing DerivedCoreProperties.txt: %s\n", u_errorName(errorCode));
 228         return;
 229     }
 230     char *basename=path+strlen(path)-strlen("DerivedCoreProperties.txt");
 231     strcpy(basename, "DerivedNormalizationProps.txt");
 232     u_parseDelimitedFile(path, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
 233     if(U_FAILURE(errorCode)) {
 234         errln("error parsing DerivedNormalizationProps.txt: %s\n", u_errorName(errorCode));
 235         return;
 236     }
 237
 238     // now we have all derived core properties in the UnicodeSets
 239     // run them all through the API
 240     int32_t rangeCount, range;
 241     uint32_t i;
 242     UChar32 start, end;
 243
 244     // test all TRUE properties
 245     for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
 246         rangeCount=derivedProps[i].getRangeCount();
 247         for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
 248             start=derivedProps[i].getRangeStart(range);
 249             end=derivedProps[i].getRangeEnd(range);
 250             for(; start<=end; ++start) {
 251                 if(!u_hasBinaryProperty(start, derivedPropsIndex[i])) {
 252                     dataerrln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==FALSE is wrong", start, derivedPropsNames[i]);
 253                     if(++numErrors[i]>=MAX_ERRORS) {
 254                       dataerrln("Too many errors, moving to the next test");
 255                       break;
 256                     }
 257                 }
 258             }
 259         }
 260     }
 261
 262     // invert all properties
 263     for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
 264         derivedProps[i].complement();
 265     }
 266
 267     // test all FALSE properties
 268     for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
 269         rangeCount=derivedProps[i].getRangeCount();
 270         for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
 271             start=derivedProps[i].getRangeStart(range);
 272             end=derivedProps[i].getRangeEnd(range);
 273             for(; start<=end; ++start) {
 274                 if(u_hasBinaryProperty(start, derivedPropsIndex[i])) {
 275                     errln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==TRUE is wrong\n", start, derivedPropsNames[i]);
 276                     if(++numErrors[i]>=MAX_ERRORS) {
 277                       errln("Too many errors, moving to the next test");
 278                       break;
 279                     }
 280                 }
 281             }
 282         }
 283     }
 284 #endif /* !UCONFIG_NO_NORMALIZATION */
 285 }
 286
 287 void UnicodeTest::TestBinaryValues() {
 288     /*
 289      * Unicode 5.1 explicitly defines binary property value aliases.
 290      * Verify that they are all recognized.
 291      */
 292     UErrorCode errorCode=U_ZERO_ERROR;
 293     UnicodeSet alpha(UNICODE_STRING_SIMPLE("[:Alphabetic:]"), errorCode);
 294     if(U_FAILURE(errorCode)) {
 295         dataerrln("UnicodeSet([:Alphabetic:]) failed - %s", u_errorName(errorCode));
 296         return;
 297     }
 298
 299     static const char *const falseValues[]={ "N", "No", "F", "False" };
 300     static const char *const trueValues[]={ "Y", "Yes", "T", "True" };
 301     int32_t i;
 302     for(i=0; i<LENGTHOF(falseValues); ++i) {
 303         UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
 304         pattern.insert(pattern.length()-2, UnicodeString(falseValues[i], -1, US_INV));
 305         errorCode=U_ZERO_ERROR;
 306         UnicodeSet set(pattern, errorCode);
 307         if(U_FAILURE(errorCode)) {
 308             errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", falseValues[i], u_errorName(errorCode));
 309             continue;
 310         }
 311         set.complement();
 312         if(set!=alpha) {
 313             errln("UnicodeSet([:Alphabetic=%s:]).complement()!=UnicodeSet([:Alphabetic:])\n", falseValues[i]);
 314         }
 315     }
 316     for(i=0; i<LENGTHOF(trueValues); ++i) {
 317         UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
 318         pattern.insert(pattern.length()-2, UnicodeString(trueValues[i], -1, US_INV));
 319         errorCode=U_ZERO_ERROR;
 320         UnicodeSet set(pattern, errorCode);
 321         if(U_FAILURE(errorCode)) {
 322             errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", trueValues[i], u_errorName(errorCode));
 323             continue;
 324         }
 325         if(set!=alpha) {
 326             errln("UnicodeSet([:Alphabetic=%s:])!=UnicodeSet([:Alphabetic:])\n", trueValues[i]);
 327         }
 328     }
 329 }
 330
 331 void UnicodeTest::TestConsistency() {
 332 #if !UCONFIG_NO_NORMALIZATION
 333     /*
 334      * Test for an example that getCanonStartSet() delivers
 335      * all characters that compose from the input one,
 336      * even in multiple steps.
 337      * For example, the set for "I" (0049) should contain both
 338      * I-diaeresis (00CF) and I-diaeresis-acute (1E2E).
 339      * In general, the set for the middle such character should be a subset
 340      * of the set for the first.
 341      */
 342     IcuTestErrorCode errorCode(*this, "TestConsistency");
 343     const Normalizer2 *nfd=Normalizer2::getNFDInstance(errorCode);
 344     const Normalizer2Impl *nfcImpl=Normalizer2Factory::getNFCImpl(errorCode);
 345     if(!nfcImpl->ensureCanonIterData(errorCode) || errorCode.isFailure()) {
 346         dataerrln("Normalizer2::getInstance(NFD) or Normalizer2Factory::getNFCImpl() failed - %s\n",
 347                   errorCode.errorName());
 348         errorCode.reset();
 349         return;
 350     }
 351
 352     UnicodeSet set1, set2;
 353     if (nfcImpl->getCanonStartSet(0x49, set1)) {
 354         /* enumerate all characters that are plausible to be latin letters */
 355         for(UChar start=0xa0; start<0x2000; ++start) {
 356             UnicodeString decomp=nfd->normalize(UnicodeString(start), errorCode);
 357             if(decomp.length()>1 && decomp[0]==0x49) {
 358                 set2.add(start);
 359             }
 360         }
 361
 362         if (set1!=set2) {
 363             errln("[canon start set of 0049] != [all c with canon decomp with 0049]");
 364         }
 365         // This was available in cucdtst.c but the test had to move to intltest
 366         // because the new internal normalization functions are in C++.
 367         //compareUSets(set1, set2,
 368         //             "[canon start set of 0049]", "[all c with canon decomp with 0049]",
 369         //             TRUE);
 370     } else {
 371         errln("NFC.getCanonStartSet() returned FALSE");
 372     }
 373 #endif
 374 }
 375
 376 /**
 377  * Test various implementations of Pattern_Syntax & Pattern_White_Space.
 378  */
 379 void UnicodeTest::TestPatternProperties() {
 380     IcuTestErrorCode errorCode(*this, "TestPatternProperties()");
 381     UnicodeSet syn_pp;
 382     UnicodeSet syn_prop(UNICODE_STRING_SIMPLE("[:Pattern_Syntax:]"), errorCode);
 383     UnicodeSet syn_list(
 384         "[!-/\\:-@\\[-\\^`\\{-~"
 385         "\\u00A1-\\u00A7\\u00A9\\u00AB\\u00AC\\u00AE\\u00B0\\u00B1\\u00B6\\u00BB\\u00BF\\u00D7\\u00F7"
 386         "\\u2010-\\u2027\\u2030-\\u203E\\u2041-\\u2053\\u2055-\\u205E\\u2190-\\u245F\\u2500-\\u2775"
 387         "\\u2794-\\u2BFF\\u2E00-\\u2E7F\\u3001-\\u3003\\u3008-\\u3020\\u3030\\uFD3E\\uFD3F\\uFE45\\uFE46]", errorCode);
 388     UnicodeSet ws_pp;
 389     UnicodeSet ws_prop(UNICODE_STRING_SIMPLE("[:Pattern_White_Space:]"), errorCode);
 390     UnicodeSet ws_list(UNICODE_STRING_SIMPLE("[\\u0009-\\u000D\\ \\u0085\\u200E\\u200F\\u2028\\u2029]"), errorCode);
 391     UnicodeSet syn_ws_pp;
 392     UnicodeSet syn_ws_prop(syn_prop);
 393     syn_ws_prop.addAll(ws_prop);
 394     for(UChar32 c=0; c<=0xffff; ++c) {
 395         if(PatternProps::isSyntax(c)) {
 396             syn_pp.add(c);
 397         }
 398         if(PatternProps::isWhiteSpace(c)) {
 399             ws_pp.add(c);
 400         }
 401         if(PatternProps::isSyntaxOrWhiteSpace(c)) {
 402             syn_ws_pp.add(c);
 403         }
 404     }
 405     compareUSets(syn_pp, syn_prop,
 406                  "PatternProps.isSyntax()", "[:Pattern_Syntax:]", TRUE);
 407     compareUSets(syn_pp, syn_list,
 408                  "PatternProps.isSyntax()", "[Pattern_Syntax ranges]", TRUE);
 409     compareUSets(ws_pp, ws_prop,
 410                  "PatternProps.isWhiteSpace()", "[:Pattern_White_Space:]", TRUE);
 411     compareUSets(ws_pp, ws_list,
 412                  "PatternProps.isWhiteSpace()", "[Pattern_White_Space ranges]", TRUE);
 413     compareUSets(syn_ws_pp, syn_ws_prop,
 414                  "PatternProps.isSyntaxOrWhiteSpace()",
 415                  "[[:Pattern_Syntax:][:Pattern_White_Space:]]", TRUE);
 416 }
 417
 418 // So far only minimal port of Java & cucdtst.c compareUSets().
 419 UBool
 420 UnicodeTest::compareUSets(const UnicodeSet &a, const UnicodeSet &b,
 421                           const char *a_name, const char *b_name,
 422                           UBool diffIsError) {
 423     UBool same= a==b;
 424     if(!same && diffIsError) {
 425         errln("Sets are different: %s vs. %s\n", a_name, b_name);
 426     }
 427     return same;
 428 }