1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /********************************************************************
4 * Copyright (c) 1997-2016, International Business Machines Corporation and
5 * others. All Rights Reserved.
6 ********************************************************************/
8 #include "unicode/ustring.h"
9 #include "unicode/uchar.h"
10 #include "unicode/uniset.h"
11 #include "unicode/putil.h"
12 #include "unicode/uscript.h"
15 #include "patternprops.h"
16 #include "normalizer2impl.h"
20 static const char *ignorePropNames
[]={
33 UnicodeTest::UnicodeTest()
35 UErrorCode errorCode
=U_ZERO_ERROR
;
36 unknownPropertyNames
=new U_NAMESPACE_QUALIFIER
Hashtable(errorCode
);
37 if(U_FAILURE(errorCode
)) {
38 delete unknownPropertyNames
;
39 unknownPropertyNames
=NULL
;
41 // Ignore some property names altogether.
42 for(int32_t i
=0; i
<UPRV_LENGTHOF(ignorePropNames
); ++i
) {
43 unknownPropertyNames
->puti(UnicodeString(ignorePropNames
[i
], -1, US_INV
), 1, errorCode
);
47 UnicodeTest::~UnicodeTest()
49 delete unknownPropertyNames
;
52 void UnicodeTest::runIndexedTest( int32_t index
, UBool exec
, const char* &name
, char* /*par*/ )
55 logln("TestSuite UnicodeTest: ");
58 TESTCASE_AUTO(TestAdditionalProperties
);
59 TESTCASE_AUTO(TestBinaryValues
);
60 TESTCASE_AUTO(TestConsistency
);
61 TESTCASE_AUTO(TestPatternProperties
);
62 TESTCASE_AUTO(TestScriptMetadata
);
63 TESTCASE_AUTO(TestBidiPairedBracketType
);
64 TESTCASE_AUTO(TestEmojiProperties
);
68 //====================================================
69 // private data used by the tests
70 //====================================================
72 // test DerivedCoreProperties.txt -------------------------------------------
74 // copied from genprops.c
76 getTokenIndex(const char *const tokens
[], int32_t countTokens
, const char *s
) {
80 s
=u_skipWhitespace(s
);
81 for(i
=0; i
<countTokens
; ++i
) {
90 z
=u_skipWhitespace(s
+j
);
91 if(*z
==';' || *z
==0) {
103 static const char *const
104 derivedPropsNames
[]={
113 "Default_Ignorable_Code_Point",
114 "Full_Composition_Exclusion",
116 "Grapheme_Link", /* Unicode 5 moves this property here from PropList.txt */
120 "Changes_When_Lowercased",
121 "Changes_When_Uppercased",
122 "Changes_When_Titlecased",
123 "Changes_When_Casefolded",
124 "Changes_When_Casemapped",
125 "Changes_When_NFKC_Casefolded"
128 static const UProperty
129 derivedPropsIndex
[]={
138 UCHAR_DEFAULT_IGNORABLE_CODE_POINT
,
139 UCHAR_FULL_COMPOSITION_EXCLUSION
,
140 UCHAR_GRAPHEME_EXTEND
,
144 UCHAR_CASE_IGNORABLE
,
145 UCHAR_CHANGES_WHEN_LOWERCASED
,
146 UCHAR_CHANGES_WHEN_UPPERCASED
,
147 UCHAR_CHANGES_WHEN_TITLECASED
,
148 UCHAR_CHANGES_WHEN_CASEFOLDED
,
149 UCHAR_CHANGES_WHEN_CASEMAPPED
,
150 UCHAR_CHANGES_WHEN_NFKC_CASEFOLDED
153 static int32_t numErrors
[UPRV_LENGTHOF(derivedPropsIndex
)]={ 0 };
155 enum { MAX_ERRORS
=50 };
157 U_CFUNC
void U_CALLCONV
158 derivedPropsLineFn(void *context
,
159 char *fields
[][2], int32_t /* fieldCount */,
160 UErrorCode
*pErrorCode
)
162 UnicodeTest
*me
=(UnicodeTest
*)context
;
166 u_parseCodePointRange(fields
[0][0], &start
, &end
, pErrorCode
);
167 if(U_FAILURE(*pErrorCode
)) {
168 me
->errln("UnicodeTest: syntax error in DerivedCoreProperties.txt or DerivedNormalizationProps.txt field 0 at %s\n", fields
[0][0]);
172 /* parse derived binary property name, ignore unknown names */
173 i
=getTokenIndex(derivedPropsNames
, UPRV_LENGTHOF(derivedPropsNames
), fields
[1][0]);
175 UnicodeString
propName(fields
[1][0], (int32_t)(fields
[1][1]-fields
[1][0]));
177 if(me
->unknownPropertyNames
->find(propName
)==NULL
) {
178 UErrorCode errorCode
=U_ZERO_ERROR
;
179 me
->unknownPropertyNames
->puti(propName
, 1, errorCode
);
180 me
->errln("UnicodeTest warning: unknown property name '%s' in DerivedCoreProperties.txt or DerivedNormalizationProps.txt\n", fields
[1][0]);
185 me
->derivedProps
[i
].add(start
, end
);
188 void UnicodeTest::TestAdditionalProperties() {
189 #if !UCONFIG_NO_NORMALIZATION
190 // test DerivedCoreProperties.txt and DerivedNormalizationProps.txt
191 if(UPRV_LENGTHOF(derivedProps
)<UPRV_LENGTHOF(derivedPropsNames
)) {
192 errln("error: UnicodeTest::derivedProps[] too short, need at least %d UnicodeSets\n",
193 UPRV_LENGTHOF(derivedPropsNames
));
196 if(UPRV_LENGTHOF(derivedPropsIndex
)!=UPRV_LENGTHOF(derivedPropsNames
)) {
197 errln("error in ucdtest.cpp: UPRV_LENGTHOF(derivedPropsIndex)!=UPRV_LENGTHOF(derivedPropsNames)\n");
202 if(getUnidataPath(path
) == NULL
) {
203 errln("unable to find path to source/data/unidata/");
206 char *basename
=strchr(path
, 0);
207 strcpy(basename
, "DerivedCoreProperties.txt");
210 UErrorCode errorCode
=U_ZERO_ERROR
;
211 u_parseDelimitedFile(path
, ';', fields
, 2, derivedPropsLineFn
, this, &errorCode
);
212 if(U_FAILURE(errorCode
)) {
213 errln("error parsing DerivedCoreProperties.txt: %s\n", u_errorName(errorCode
));
217 strcpy(basename
, "DerivedNormalizationProps.txt");
218 u_parseDelimitedFile(path
, ';', fields
, 2, derivedPropsLineFn
, this, &errorCode
);
219 if(U_FAILURE(errorCode
)) {
220 errln("error parsing DerivedNormalizationProps.txt: %s\n", u_errorName(errorCode
));
224 // now we have all derived core properties in the UnicodeSets
225 // run them all through the API
226 int32_t rangeCount
, range
;
230 // test all TRUE properties
231 for(i
=0; i
<UPRV_LENGTHOF(derivedPropsNames
); ++i
) {
232 rangeCount
=derivedProps
[i
].getRangeCount();
233 for(range
=0; range
<rangeCount
&& numErrors
[i
]<MAX_ERRORS
; ++range
) {
234 start
=derivedProps
[i
].getRangeStart(range
);
235 end
=derivedProps
[i
].getRangeEnd(range
);
236 for(; start
<=end
; ++start
) {
237 if(!u_hasBinaryProperty(start
, derivedPropsIndex
[i
])) {
238 dataerrln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==FALSE is wrong", start
, derivedPropsNames
[i
]);
239 if(++numErrors
[i
]>=MAX_ERRORS
) {
240 dataerrln("Too many errors, moving to the next test");
248 // invert all properties
249 for(i
=0; i
<UPRV_LENGTHOF(derivedPropsNames
); ++i
) {
250 derivedProps
[i
].complement();
253 // test all FALSE properties
254 for(i
=0; i
<UPRV_LENGTHOF(derivedPropsNames
); ++i
) {
255 rangeCount
=derivedProps
[i
].getRangeCount();
256 for(range
=0; range
<rangeCount
&& numErrors
[i
]<MAX_ERRORS
; ++range
) {
257 start
=derivedProps
[i
].getRangeStart(range
);
258 end
=derivedProps
[i
].getRangeEnd(range
);
259 for(; start
<=end
; ++start
) {
260 if(u_hasBinaryProperty(start
, derivedPropsIndex
[i
])) {
261 errln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==TRUE is wrong\n", start
, derivedPropsNames
[i
]);
262 if(++numErrors
[i
]>=MAX_ERRORS
) {
263 errln("Too many errors, moving to the next test");
270 #endif /* !UCONFIG_NO_NORMALIZATION */
273 void UnicodeTest::TestBinaryValues() {
275 * Unicode 5.1 explicitly defines binary property value aliases.
276 * Verify that they are all recognized.
278 UErrorCode errorCode
=U_ZERO_ERROR
;
279 UnicodeSet
alpha(UNICODE_STRING_SIMPLE("[:Alphabetic:]"), errorCode
);
280 if(U_FAILURE(errorCode
)) {
281 dataerrln("UnicodeSet([:Alphabetic:]) failed - %s", u_errorName(errorCode
));
285 static const char *const falseValues
[]={ "N", "No", "F", "False" };
286 static const char *const trueValues
[]={ "Y", "Yes", "T", "True" };
288 for(i
=0; i
<UPRV_LENGTHOF(falseValues
); ++i
) {
289 UnicodeString pattern
=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
290 pattern
.insert(pattern
.length()-2, UnicodeString(falseValues
[i
], -1, US_INV
));
291 errorCode
=U_ZERO_ERROR
;
292 UnicodeSet
set(pattern
, errorCode
);
293 if(U_FAILURE(errorCode
)) {
294 errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", falseValues
[i
], u_errorName(errorCode
));
299 errln("UnicodeSet([:Alphabetic=%s:]).complement()!=UnicodeSet([:Alphabetic:])\n", falseValues
[i
]);
302 for(i
=0; i
<UPRV_LENGTHOF(trueValues
); ++i
) {
303 UnicodeString pattern
=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
304 pattern
.insert(pattern
.length()-2, UnicodeString(trueValues
[i
], -1, US_INV
));
305 errorCode
=U_ZERO_ERROR
;
306 UnicodeSet
set(pattern
, errorCode
);
307 if(U_FAILURE(errorCode
)) {
308 errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", trueValues
[i
], u_errorName(errorCode
));
312 errln("UnicodeSet([:Alphabetic=%s:])!=UnicodeSet([:Alphabetic:])\n", trueValues
[i
]);
317 void UnicodeTest::TestConsistency() {
318 #if !UCONFIG_NO_NORMALIZATION
320 * Test for an example that getCanonStartSet() delivers
321 * all characters that compose from the input one,
322 * even in multiple steps.
323 * For example, the set for "I" (0049) should contain both
324 * I-diaeresis (00CF) and I-diaeresis-acute (1E2E).
325 * In general, the set for the middle such character should be a subset
326 * of the set for the first.
328 IcuTestErrorCode
errorCode(*this, "TestConsistency");
329 const Normalizer2
*nfd
=Normalizer2::getNFDInstance(errorCode
);
330 const Normalizer2Impl
*nfcImpl
=Normalizer2Factory::getNFCImpl(errorCode
);
331 if(!nfcImpl
->ensureCanonIterData(errorCode
) || errorCode
.isFailure()) {
332 dataerrln("Normalizer2::getInstance(NFD) or Normalizer2Factory::getNFCImpl() failed - %s\n",
333 errorCode
.errorName());
338 UnicodeSet set1
, set2
;
339 if (nfcImpl
->getCanonStartSet(0x49, set1
)) {
340 /* enumerate all characters that are plausible to be latin letters */
341 for(UChar start
=0xa0; start
<0x2000; ++start
) {
342 UnicodeString decomp
=nfd
->normalize(UnicodeString(start
), errorCode
);
343 if(decomp
.length()>1 && decomp
[0]==0x49) {
349 errln("[canon start set of 0049] != [all c with canon decomp with 0049]");
351 // This was available in cucdtst.c but the test had to move to intltest
352 // because the new internal normalization functions are in C++.
353 //compareUSets(set1, set2,
354 // "[canon start set of 0049]", "[all c with canon decomp with 0049]",
357 errln("NFC.getCanonStartSet() returned FALSE");
363 * Test various implementations of Pattern_Syntax & Pattern_White_Space.
365 void UnicodeTest::TestPatternProperties() {
366 IcuTestErrorCode
errorCode(*this, "TestPatternProperties()");
368 UnicodeSet
syn_prop(UNICODE_STRING_SIMPLE("[:Pattern_Syntax:]"), errorCode
);
370 "[!-/\\:-@\\[-\\^`\\{-~"
371 "\\u00A1-\\u00A7\\u00A9\\u00AB\\u00AC\\u00AE\\u00B0\\u00B1\\u00B6\\u00BB\\u00BF\\u00D7\\u00F7"
372 "\\u2010-\\u2027\\u2030-\\u203E\\u2041-\\u2053\\u2055-\\u205E\\u2190-\\u245F\\u2500-\\u2775"
373 "\\u2794-\\u2BFF\\u2E00-\\u2E7F\\u3001-\\u3003\\u3008-\\u3020\\u3030\\uFD3E\\uFD3F\\uFE45\\uFE46]", errorCode
);
375 UnicodeSet
ws_prop(UNICODE_STRING_SIMPLE("[:Pattern_White_Space:]"), errorCode
);
376 UnicodeSet
ws_list(UNICODE_STRING_SIMPLE("[\\u0009-\\u000D\\ \\u0085\\u200E\\u200F\\u2028\\u2029]"), errorCode
);
377 UnicodeSet syn_ws_pp
;
378 UnicodeSet
syn_ws_prop(syn_prop
);
379 syn_ws_prop
.addAll(ws_prop
);
380 for(UChar32 c
=0; c
<=0xffff; ++c
) {
381 if(PatternProps::isSyntax(c
)) {
384 if(PatternProps::isWhiteSpace(c
)) {
387 if(PatternProps::isSyntaxOrWhiteSpace(c
)) {
391 compareUSets(syn_pp
, syn_prop
,
392 "PatternProps.isSyntax()", "[:Pattern_Syntax:]", TRUE
);
393 compareUSets(syn_pp
, syn_list
,
394 "PatternProps.isSyntax()", "[Pattern_Syntax ranges]", TRUE
);
395 compareUSets(ws_pp
, ws_prop
,
396 "PatternProps.isWhiteSpace()", "[:Pattern_White_Space:]", TRUE
);
397 compareUSets(ws_pp
, ws_list
,
398 "PatternProps.isWhiteSpace()", "[Pattern_White_Space ranges]", TRUE
);
399 compareUSets(syn_ws_pp
, syn_ws_prop
,
400 "PatternProps.isSyntaxOrWhiteSpace()",
401 "[[:Pattern_Syntax:][:Pattern_White_Space:]]", TRUE
);
404 // So far only minimal port of Java & cucdtst.c compareUSets().
406 UnicodeTest::compareUSets(const UnicodeSet
&a
, const UnicodeSet
&b
,
407 const char *a_name
, const char *b_name
,
410 if(!same
&& diffIsError
) {
411 errln("Sets are different: %s vs. %s\n", a_name
, b_name
);
419 * Maps a special script code to the most common script of its encoded characters.
421 UScriptCode
getCharScript(UScriptCode script
) {
423 case USCRIPT_HAN_WITH_BOPOMOFO
:
424 case USCRIPT_SIMPLIFIED_HAN
:
425 case USCRIPT_TRADITIONAL_HAN
:
427 case USCRIPT_JAPANESE
:
428 return USCRIPT_HIRAGANA
;
431 return USCRIPT_HANGUL
;
432 case USCRIPT_SYMBOLS_EMOJI
:
433 return USCRIPT_SYMBOLS
;
441 void UnicodeTest::TestScriptMetadata() {
442 IcuTestErrorCode
errorCode(*this, "TestScriptMetadata()");
443 UnicodeSet
rtl("[[:bc=R:][:bc=AL:]-[:Cn:]-[:sc=Common:]]", errorCode
);
444 // So far, sample characters are uppercase.
445 // Georgian is special.
446 UnicodeSet
cased("[[:Lu:]-[:sc=Common:]-[:sc=Geor:]]", errorCode
);
447 for(int32_t sci
= 0; sci
< USCRIPT_CODE_LIMIT
; ++sci
) {
448 UScriptCode sc
= (UScriptCode
)sci
;
449 // Run the test with -v to see which script has failures:
450 // .../intltest$ make && ./intltest utility/UnicodeTest/TestScriptMetadata -v | grep -C 6 FAIL
451 logln(uscript_getShortName(sc
));
452 UScriptUsage usage
= uscript_getUsage(sc
);
453 UnicodeString sample
= uscript_getSampleUnicodeString(sc
);
454 UnicodeSet scriptSet
;
455 scriptSet
.applyIntPropertyValue(UCHAR_SCRIPT
, sc
, errorCode
);
456 if(usage
== USCRIPT_USAGE_NOT_ENCODED
) {
457 assertTrue("not encoded, no sample", sample
.isEmpty());
458 assertFalse("not encoded, not RTL", uscript_isRightToLeft(sc
));
459 assertFalse("not encoded, not LB letters", uscript_breaksBetweenLetters(sc
));
460 assertFalse("not encoded, not cased", uscript_isCased(sc
));
461 assertTrue("not encoded, no characters", scriptSet
.isEmpty());
463 assertFalse("encoded, has a sample character", sample
.isEmpty());
464 UChar32 firstChar
= sample
.char32At(0);
465 UScriptCode charScript
= getCharScript(sc
);
466 assertEquals("script(sample(script))",
467 (int32_t)charScript
, (int32_t)uscript_getScript(firstChar
, errorCode
));
468 assertEquals("RTL vs. set", (UBool
)rtl
.contains(firstChar
), (UBool
)uscript_isRightToLeft(sc
));
469 assertEquals("cased vs. set", (UBool
)cased
.contains(firstChar
), (UBool
)uscript_isCased(sc
));
470 assertEquals("encoded, has characters", (UBool
)(sc
== charScript
), (UBool
)(!scriptSet
.isEmpty()));
471 if(uscript_isRightToLeft(sc
)) {
472 rtl
.removeAll(scriptSet
);
474 if(uscript_isCased(sc
)) {
475 cased
.removeAll(scriptSet
);
479 UnicodeString pattern
;
480 assertEquals("no remaining RTL characters",
481 UnicodeString("[]"), rtl
.toPattern(pattern
));
482 assertEquals("no remaining cased characters",
483 UnicodeString("[]"), cased
.toPattern(pattern
));
485 assertTrue("Hani breaks between letters", uscript_breaksBetweenLetters(USCRIPT_HAN
));
486 assertTrue("Thai breaks between letters", uscript_breaksBetweenLetters(USCRIPT_THAI
));
487 assertFalse("Latn does not break between letters", uscript_breaksBetweenLetters(USCRIPT_LATIN
));
490 void UnicodeTest::TestBidiPairedBracketType() {
491 // BidiBrackets-6.3.0.txt says:
493 // The set of code points listed in this file was originally derived
494 // using the character properties General_Category (gc), Bidi_Class (bc),
495 // Bidi_Mirrored (Bidi_M), and Bidi_Mirroring_Glyph (bmg), as follows:
496 // two characters, A and B, form a pair if A has gc=Ps and B has gc=Pe,
497 // both have bc=ON and Bidi_M=Y, and bmg of A is B. Bidi_Paired_Bracket
498 // maps A to B and vice versa, and their Bidi_Paired_Bracket_Type
499 // property values are Open and Close, respectively.
500 IcuTestErrorCode
errorCode(*this, "TestBidiPairedBracketType()");
501 UnicodeSet
bpt("[:^bpt=n:]", errorCode
);
502 assertTrue("bpt!=None is not empty", !bpt
.isEmpty());
503 // The following should always be true.
504 UnicodeSet
mirrored("[:Bidi_M:]", errorCode
);
505 UnicodeSet
other_neutral("[:bc=ON:]", errorCode
);
506 assertTrue("bpt!=None is a subset of Bidi_M", mirrored
.containsAll(bpt
));
507 assertTrue("bpt!=None is a subset of bc=ON", other_neutral
.containsAll(bpt
));
508 // The following are true at least initially in Unicode 6.3.
509 UnicodeSet
bpt_open("[:bpt=o:]", errorCode
);
510 UnicodeSet
bpt_close("[:bpt=c:]", errorCode
);
511 UnicodeSet
ps("[:Ps:]", errorCode
);
512 UnicodeSet
pe("[:Pe:]", errorCode
);
513 assertTrue("bpt=Open is a subset of Ps", ps
.containsAll(bpt_open
));
514 assertTrue("bpt=Close is a subset of Pe", pe
.containsAll(bpt_close
));
517 void UnicodeTest::TestEmojiProperties() {
518 assertFalse("space is not Emoji", u_hasBinaryProperty(0x20, UCHAR_EMOJI
));
519 assertTrue("shooting star is Emoji", u_hasBinaryProperty(0x1F320, UCHAR_EMOJI
));
520 IcuTestErrorCode
errorCode(*this, "TestEmojiProperties()");
521 UnicodeSet
emoji("[:Emoji:]", errorCode
);
522 assertTrue("lots of Emoji", emoji
.size() > 700);
524 assertTrue("shooting star is Emoji_Presentation",
525 u_hasBinaryProperty(0x1F320, UCHAR_EMOJI_PRESENTATION
));
526 assertTrue("Fitzpatrick 6 is Emoji_Modifier",
527 u_hasBinaryProperty(0x1F3FF, UCHAR_EMOJI_MODIFIER
));
528 assertTrue("happy person is Emoji_Modifier_Base",
529 u_hasBinaryProperty(0x1F64B, UCHAR_EMOJI_MODIFIER_BASE
));