1 /********************************************************************
3 * Copyright (c) 1997-2013, International Business Machines Corporation and
4 * others. All Rights Reserved.
5 ********************************************************************/
7 #include "unicode/ustring.h"
8 #include "unicode/uchar.h"
9 #include "unicode/uniset.h"
10 #include "unicode/putil.h"
11 #include "unicode/uscript.h"
14 #include "patternprops.h"
15 #include "normalizer2impl.h"
19 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof(array[0]))
21 static const char *ignorePropNames
[]={
34 UnicodeTest::UnicodeTest()
36 UErrorCode errorCode
=U_ZERO_ERROR
;
37 unknownPropertyNames
=new U_NAMESPACE_QUALIFIER
Hashtable(errorCode
);
38 if(U_FAILURE(errorCode
)) {
39 delete unknownPropertyNames
;
40 unknownPropertyNames
=NULL
;
42 // Ignore some property names altogether.
43 for(int32_t i
=0; i
<LENGTHOF(ignorePropNames
); ++i
) {
44 unknownPropertyNames
->puti(UnicodeString(ignorePropNames
[i
], -1, US_INV
), 1, errorCode
);
48 UnicodeTest::~UnicodeTest()
50 delete unknownPropertyNames
;
53 void UnicodeTest::runIndexedTest( int32_t index
, UBool exec
, const char* &name
, char* /*par*/ )
56 logln("TestSuite UnicodeTest: ");
59 TESTCASE_AUTO(TestAdditionalProperties
);
60 TESTCASE_AUTO(TestBinaryValues
);
61 TESTCASE_AUTO(TestConsistency
);
62 TESTCASE_AUTO(TestPatternProperties
);
63 TESTCASE_AUTO(TestScriptMetadata
);
64 TESTCASE_AUTO(TestBidiPairedBracketType
);
68 //====================================================
69 // private data used by the tests
70 //====================================================
72 // test DerivedCoreProperties.txt -------------------------------------------
74 // copied from genprops.c
76 getTokenIndex(const char *const tokens
[], int32_t countTokens
, const char *s
) {
80 s
=u_skipWhitespace(s
);
81 for(i
=0; i
<countTokens
; ++i
) {
90 z
=u_skipWhitespace(s
+j
);
91 if(*z
==';' || *z
==0) {
103 static const char *const
104 derivedPropsNames
[]={
113 "Default_Ignorable_Code_Point",
114 "Full_Composition_Exclusion",
116 "Grapheme_Link", /* Unicode 5 moves this property here from PropList.txt */
120 "Changes_When_Lowercased",
121 "Changes_When_Uppercased",
122 "Changes_When_Titlecased",
123 "Changes_When_Casefolded",
124 "Changes_When_Casemapped",
125 "Changes_When_NFKC_Casefolded"
128 static const UProperty
129 derivedPropsIndex
[]={
138 UCHAR_DEFAULT_IGNORABLE_CODE_POINT
,
139 UCHAR_FULL_COMPOSITION_EXCLUSION
,
140 UCHAR_GRAPHEME_EXTEND
,
144 UCHAR_CASE_IGNORABLE
,
145 UCHAR_CHANGES_WHEN_LOWERCASED
,
146 UCHAR_CHANGES_WHEN_UPPERCASED
,
147 UCHAR_CHANGES_WHEN_TITLECASED
,
148 UCHAR_CHANGES_WHEN_CASEFOLDED
,
149 UCHAR_CHANGES_WHEN_CASEMAPPED
,
150 UCHAR_CHANGES_WHEN_NFKC_CASEFOLDED
153 static int32_t numErrors
[LENGTHOF(derivedPropsIndex
)]={ 0 };
155 enum { MAX_ERRORS
=50 };
157 U_CFUNC
void U_CALLCONV
158 derivedPropsLineFn(void *context
,
159 char *fields
[][2], int32_t /* fieldCount */,
160 UErrorCode
*pErrorCode
)
162 UnicodeTest
*me
=(UnicodeTest
*)context
;
166 u_parseCodePointRange(fields
[0][0], &start
, &end
, pErrorCode
);
167 if(U_FAILURE(*pErrorCode
)) {
168 me
->errln("UnicodeTest: syntax error in DerivedCoreProperties.txt or DerivedNormalizationProps.txt field 0 at %s\n", fields
[0][0]);
172 /* parse derived binary property name, ignore unknown names */
173 i
=getTokenIndex(derivedPropsNames
, LENGTHOF(derivedPropsNames
), fields
[1][0]);
175 UnicodeString
propName(fields
[1][0], (int32_t)(fields
[1][1]-fields
[1][0]));
177 if(me
->unknownPropertyNames
->find(propName
)==NULL
) {
178 UErrorCode errorCode
=U_ZERO_ERROR
;
179 me
->unknownPropertyNames
->puti(propName
, 1, errorCode
);
180 me
->errln("UnicodeTest warning: unknown property name '%s' in DerivedCoreProperties.txt or DerivedNormalizationProps.txt\n", fields
[1][0]);
185 me
->derivedProps
[i
].add(start
, end
);
188 void UnicodeTest::TestAdditionalProperties() {
189 #if !UCONFIG_NO_NORMALIZATION
190 // test DerivedCoreProperties.txt and DerivedNormalizationProps.txt
191 if(LENGTHOF(derivedProps
)<LENGTHOF(derivedPropsNames
)) {
192 errln("error: UnicodeTest::derivedProps[] too short, need at least %d UnicodeSets\n",
193 LENGTHOF(derivedPropsNames
));
196 if(LENGTHOF(derivedPropsIndex
)!=LENGTHOF(derivedPropsNames
)) {
197 errln("error in ucdtest.cpp: LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)\n");
202 char backupPath
[256];
204 UErrorCode errorCode
=U_ZERO_ERROR
;
206 /* Look inside ICU_DATA first */
207 strcpy(newPath
, pathToDataDirectory());
208 strcat(newPath
, "unidata" U_FILE_SEP_STRING
"DerivedCoreProperties.txt");
210 // As a fallback, try to guess where the source data was located
211 // at the time ICU was built, and look there.
213 strcpy(backupPath
, U_TOPSRCDIR U_FILE_SEP_STRING
"data");
215 strcpy(backupPath
, loadTestData(errorCode
));
216 strcat(backupPath
, U_FILE_SEP_STRING
".." U_FILE_SEP_STRING
".." U_FILE_SEP_STRING
".." U_FILE_SEP_STRING
".." U_FILE_SEP_STRING
"data");
218 strcat(backupPath
, U_FILE_SEP_STRING
);
219 strcat(backupPath
, "unidata" U_FILE_SEP_STRING
"DerivedCoreProperties.txt");
222 u_parseDelimitedFile(newPath
, ';', fields
, 2, derivedPropsLineFn
, this, &errorCode
);
224 if(errorCode
==U_FILE_ACCESS_ERROR
) {
225 errorCode
=U_ZERO_ERROR
;
227 u_parseDelimitedFile(backupPath
, ';', fields
, 2, derivedPropsLineFn
, this, &errorCode
);
229 if(U_FAILURE(errorCode
)) {
230 errln("error parsing DerivedCoreProperties.txt: %s\n", u_errorName(errorCode
));
233 char *basename
=path
+strlen(path
)-strlen("DerivedCoreProperties.txt");
234 strcpy(basename
, "DerivedNormalizationProps.txt");
235 u_parseDelimitedFile(path
, ';', fields
, 2, derivedPropsLineFn
, this, &errorCode
);
236 if(U_FAILURE(errorCode
)) {
237 errln("error parsing DerivedNormalizationProps.txt: %s\n", u_errorName(errorCode
));
241 // now we have all derived core properties in the UnicodeSets
242 // run them all through the API
243 int32_t rangeCount
, range
;
247 // test all TRUE properties
248 for(i
=0; i
<LENGTHOF(derivedPropsNames
); ++i
) {
249 rangeCount
=derivedProps
[i
].getRangeCount();
250 for(range
=0; range
<rangeCount
&& numErrors
[i
]<MAX_ERRORS
; ++range
) {
251 start
=derivedProps
[i
].getRangeStart(range
);
252 end
=derivedProps
[i
].getRangeEnd(range
);
253 for(; start
<=end
; ++start
) {
254 if(!u_hasBinaryProperty(start
, derivedPropsIndex
[i
])) {
255 dataerrln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==FALSE is wrong", start
, derivedPropsNames
[i
]);
256 if(++numErrors
[i
]>=MAX_ERRORS
) {
257 dataerrln("Too many errors, moving to the next test");
265 // invert all properties
266 for(i
=0; i
<LENGTHOF(derivedPropsNames
); ++i
) {
267 derivedProps
[i
].complement();
270 // test all FALSE properties
271 for(i
=0; i
<LENGTHOF(derivedPropsNames
); ++i
) {
272 rangeCount
=derivedProps
[i
].getRangeCount();
273 for(range
=0; range
<rangeCount
&& numErrors
[i
]<MAX_ERRORS
; ++range
) {
274 start
=derivedProps
[i
].getRangeStart(range
);
275 end
=derivedProps
[i
].getRangeEnd(range
);
276 for(; start
<=end
; ++start
) {
277 if(u_hasBinaryProperty(start
, derivedPropsIndex
[i
])) {
278 errln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==TRUE is wrong\n", start
, derivedPropsNames
[i
]);
279 if(++numErrors
[i
]>=MAX_ERRORS
) {
280 errln("Too many errors, moving to the next test");
287 #endif /* !UCONFIG_NO_NORMALIZATION */
290 void UnicodeTest::TestBinaryValues() {
292 * Unicode 5.1 explicitly defines binary property value aliases.
293 * Verify that they are all recognized.
295 UErrorCode errorCode
=U_ZERO_ERROR
;
296 UnicodeSet
alpha(UNICODE_STRING_SIMPLE("[:Alphabetic:]"), errorCode
);
297 if(U_FAILURE(errorCode
)) {
298 dataerrln("UnicodeSet([:Alphabetic:]) failed - %s", u_errorName(errorCode
));
302 static const char *const falseValues
[]={ "N", "No", "F", "False" };
303 static const char *const trueValues
[]={ "Y", "Yes", "T", "True" };
305 for(i
=0; i
<LENGTHOF(falseValues
); ++i
) {
306 UnicodeString pattern
=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
307 pattern
.insert(pattern
.length()-2, UnicodeString(falseValues
[i
], -1, US_INV
));
308 errorCode
=U_ZERO_ERROR
;
309 UnicodeSet
set(pattern
, errorCode
);
310 if(U_FAILURE(errorCode
)) {
311 errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", falseValues
[i
], u_errorName(errorCode
));
316 errln("UnicodeSet([:Alphabetic=%s:]).complement()!=UnicodeSet([:Alphabetic:])\n", falseValues
[i
]);
319 for(i
=0; i
<LENGTHOF(trueValues
); ++i
) {
320 UnicodeString pattern
=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
321 pattern
.insert(pattern
.length()-2, UnicodeString(trueValues
[i
], -1, US_INV
));
322 errorCode
=U_ZERO_ERROR
;
323 UnicodeSet
set(pattern
, errorCode
);
324 if(U_FAILURE(errorCode
)) {
325 errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", trueValues
[i
], u_errorName(errorCode
));
329 errln("UnicodeSet([:Alphabetic=%s:])!=UnicodeSet([:Alphabetic:])\n", trueValues
[i
]);
334 void UnicodeTest::TestConsistency() {
335 #if !UCONFIG_NO_NORMALIZATION
337 * Test for an example that getCanonStartSet() delivers
338 * all characters that compose from the input one,
339 * even in multiple steps.
340 * For example, the set for "I" (0049) should contain both
341 * I-diaeresis (00CF) and I-diaeresis-acute (1E2E).
342 * In general, the set for the middle such character should be a subset
343 * of the set for the first.
345 IcuTestErrorCode
errorCode(*this, "TestConsistency");
346 const Normalizer2
*nfd
=Normalizer2::getNFDInstance(errorCode
);
347 const Normalizer2Impl
*nfcImpl
=Normalizer2Factory::getNFCImpl(errorCode
);
348 if(!nfcImpl
->ensureCanonIterData(errorCode
) || errorCode
.isFailure()) {
349 dataerrln("Normalizer2::getInstance(NFD) or Normalizer2Factory::getNFCImpl() failed - %s\n",
350 errorCode
.errorName());
355 UnicodeSet set1
, set2
;
356 if (nfcImpl
->getCanonStartSet(0x49, set1
)) {
357 /* enumerate all characters that are plausible to be latin letters */
358 for(UChar start
=0xa0; start
<0x2000; ++start
) {
359 UnicodeString decomp
=nfd
->normalize(UnicodeString(start
), errorCode
);
360 if(decomp
.length()>1 && decomp
[0]==0x49) {
366 errln("[canon start set of 0049] != [all c with canon decomp with 0049]");
368 // This was available in cucdtst.c but the test had to move to intltest
369 // because the new internal normalization functions are in C++.
370 //compareUSets(set1, set2,
371 // "[canon start set of 0049]", "[all c with canon decomp with 0049]",
374 errln("NFC.getCanonStartSet() returned FALSE");
380 * Test various implementations of Pattern_Syntax & Pattern_White_Space.
382 void UnicodeTest::TestPatternProperties() {
383 IcuTestErrorCode
errorCode(*this, "TestPatternProperties()");
385 UnicodeSet
syn_prop(UNICODE_STRING_SIMPLE("[:Pattern_Syntax:]"), errorCode
);
387 "[!-/\\:-@\\[-\\^`\\{-~"
388 "\\u00A1-\\u00A7\\u00A9\\u00AB\\u00AC\\u00AE\\u00B0\\u00B1\\u00B6\\u00BB\\u00BF\\u00D7\\u00F7"
389 "\\u2010-\\u2027\\u2030-\\u203E\\u2041-\\u2053\\u2055-\\u205E\\u2190-\\u245F\\u2500-\\u2775"
390 "\\u2794-\\u2BFF\\u2E00-\\u2E7F\\u3001-\\u3003\\u3008-\\u3020\\u3030\\uFD3E\\uFD3F\\uFE45\\uFE46]", errorCode
);
392 UnicodeSet
ws_prop(UNICODE_STRING_SIMPLE("[:Pattern_White_Space:]"), errorCode
);
393 UnicodeSet
ws_list(UNICODE_STRING_SIMPLE("[\\u0009-\\u000D\\ \\u0085\\u200E\\u200F\\u2028\\u2029]"), errorCode
);
394 UnicodeSet syn_ws_pp
;
395 UnicodeSet
syn_ws_prop(syn_prop
);
396 syn_ws_prop
.addAll(ws_prop
);
397 for(UChar32 c
=0; c
<=0xffff; ++c
) {
398 if(PatternProps::isSyntax(c
)) {
401 if(PatternProps::isWhiteSpace(c
)) {
404 if(PatternProps::isSyntaxOrWhiteSpace(c
)) {
408 compareUSets(syn_pp
, syn_prop
,
409 "PatternProps.isSyntax()", "[:Pattern_Syntax:]", TRUE
);
410 compareUSets(syn_pp
, syn_list
,
411 "PatternProps.isSyntax()", "[Pattern_Syntax ranges]", TRUE
);
412 compareUSets(ws_pp
, ws_prop
,
413 "PatternProps.isWhiteSpace()", "[:Pattern_White_Space:]", TRUE
);
414 compareUSets(ws_pp
, ws_list
,
415 "PatternProps.isWhiteSpace()", "[Pattern_White_Space ranges]", TRUE
);
416 compareUSets(syn_ws_pp
, syn_ws_prop
,
417 "PatternProps.isSyntaxOrWhiteSpace()",
418 "[[:Pattern_Syntax:][:Pattern_White_Space:]]", TRUE
);
421 // So far only minimal port of Java & cucdtst.c compareUSets().
423 UnicodeTest::compareUSets(const UnicodeSet
&a
, const UnicodeSet
&b
,
424 const char *a_name
, const char *b_name
,
427 if(!same
&& diffIsError
) {
428 errln("Sets are different: %s vs. %s\n", a_name
, b_name
);
436 * Maps a special script code to the most common script of its encoded characters.
438 UScriptCode
getCharScript(UScriptCode script
) {
440 case USCRIPT_SIMPLIFIED_HAN
:
441 case USCRIPT_TRADITIONAL_HAN
:
443 case USCRIPT_JAPANESE
:
444 return USCRIPT_HIRAGANA
;
446 return USCRIPT_HANGUL
;
454 void UnicodeTest::TestScriptMetadata() {
455 IcuTestErrorCode
errorCode(*this, "TestScriptMetadata()");
456 UnicodeSet
rtl("[[:bc=R:][:bc=AL:]-[:Cn:]-[:sc=Common:]]", errorCode
);
457 // So far, sample characters are uppercase.
458 // Georgian is special.
459 UnicodeSet
cased("[[:Lu:]-[:sc=Common:]-[:sc=Geor:]]", errorCode
);
460 for(int32_t sci
= 0; sci
< USCRIPT_CODE_LIMIT
; ++sci
) {
461 UScriptCode sc
= (UScriptCode
)sci
;
462 // Run the test with -v to see which script has failures:
463 // .../intltest$ make && ./intltest utility/UnicodeTest/TestScriptMetadata -v | grep -C 3 FAIL
464 logln(uscript_getShortName(sc
));
465 UScriptUsage usage
= uscript_getUsage(sc
);
466 UnicodeString sample
= uscript_getSampleUnicodeString(sc
);
467 UnicodeSet scriptSet
;
468 scriptSet
.applyIntPropertyValue(UCHAR_SCRIPT
, sc
, errorCode
);
469 if(usage
== USCRIPT_USAGE_NOT_ENCODED
) {
470 assertTrue("not encoded, no sample", sample
.isEmpty());
471 assertFalse("not encoded, not RTL", uscript_isRightToLeft(sc
));
472 assertFalse("not encoded, not LB letters", uscript_breaksBetweenLetters(sc
));
473 assertFalse("not encoded, not cased", uscript_isCased(sc
));
474 assertTrue("not encoded, no characters", scriptSet
.isEmpty());
476 assertFalse("encoded, has a sample character", sample
.isEmpty());
477 UChar32 firstChar
= sample
.char32At(0);
478 UScriptCode charScript
= getCharScript(sc
);
479 assertEquals("script(sample(script))",
480 (int32_t)charScript
, (int32_t)uscript_getScript(firstChar
, errorCode
));
481 assertEquals("RTL vs. set", (UBool
)rtl
.contains(firstChar
), (UBool
)uscript_isRightToLeft(sc
));
482 assertEquals("cased vs. set", (UBool
)cased
.contains(firstChar
), (UBool
)uscript_isCased(sc
));
483 assertEquals("encoded, has characters", (UBool
)(sc
== charScript
), (UBool
)(!scriptSet
.isEmpty()));
484 if(uscript_isRightToLeft(sc
)) {
485 rtl
.removeAll(scriptSet
);
487 if(uscript_isCased(sc
)) {
488 cased
.removeAll(scriptSet
);
492 UnicodeString pattern
;
493 assertEquals("no remaining RTL characters",
494 UnicodeString("[]"), rtl
.toPattern(pattern
));
495 assertEquals("no remaining cased characters",
496 UnicodeString("[]"), cased
.toPattern(pattern
));
498 assertTrue("Hani breaks between letters", uscript_breaksBetweenLetters(USCRIPT_HAN
));
499 assertTrue("Thai breaks between letters", uscript_breaksBetweenLetters(USCRIPT_THAI
));
500 assertFalse("Latn does not break between letters", uscript_breaksBetweenLetters(USCRIPT_LATIN
));
503 void UnicodeTest::TestBidiPairedBracketType() {
504 // BidiBrackets-6.3.0.txt says:
506 // The set of code points listed in this file was originally derived
507 // using the character properties General_Category (gc), Bidi_Class (bc),
508 // Bidi_Mirrored (Bidi_M), and Bidi_Mirroring_Glyph (bmg), as follows:
509 // two characters, A and B, form a pair if A has gc=Ps and B has gc=Pe,
510 // both have bc=ON and Bidi_M=Y, and bmg of A is B. Bidi_Paired_Bracket
511 // maps A to B and vice versa, and their Bidi_Paired_Bracket_Type
512 // property values are Open and Close, respectively.
513 IcuTestErrorCode
errorCode(*this, "TestBidiPairedBracketType()");
514 UnicodeSet
bpt("[:^bpt=n:]", errorCode
);
515 assertTrue("bpt!=None is not empty", !bpt
.isEmpty());
516 // The following should always be true.
517 UnicodeSet
mirrored("[:Bidi_M:]", errorCode
);
518 UnicodeSet
other_neutral("[:bc=ON:]", errorCode
);
519 assertTrue("bpt!=None is a subset of Bidi_M", mirrored
.containsAll(bpt
));
520 assertTrue("bpt!=None is a subset of bc=ON", other_neutral
.containsAll(bpt
));
521 // The following are true at least initially in Unicode 6.3.
522 UnicodeSet
bpt_open("[:bpt=o:]", errorCode
);
523 UnicodeSet
bpt_close("[:bpt=c:]", errorCode
);
524 UnicodeSet
ps("[:Ps:]", errorCode
);
525 UnicodeSet
pe("[:Pe:]", errorCode
);
526 assertTrue("bpt=Open is a subset of Ps", ps
.containsAll(bpt_open
));
527 assertTrue("bpt=Close is a subset of Pe", pe
.containsAll(bpt_close
));