1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /********************************************************************
5 * Copyright (c) 1997-2016, International Business Machines Corporation and
6 * others. All Rights Reserved.
7 ********************************************************************/
9 #include "unicode/utypes.h"
11 #if !UCONFIG_NO_NORMALIZATION
13 #include "unicode/uchar.h"
14 #include "unicode/errorcode.h"
15 #include "unicode/normlzr.h"
16 #include "unicode/uniset.h"
17 #include "unicode/usetiter.h"
18 #include "unicode/schriter.h"
19 #include "unicode/utf16.h"
22 #include "normalizer2impl.h"
25 #define ARRAY_LENGTH(array) UPRV_LENGTHOF(array)
27 #define CASE(id,test) case id: \
31 logln((UnicodeString)""); \
36 static UErrorCode status
= U_ZERO_ERROR
;
38 void BasicNormalizerTest::runIndexedTest(int32_t index
, UBool exec
,
39 const char* &name
, char* /*par*/) {
42 CASE(1,TestCompatDecomp
);
43 CASE(2,TestCanonCompose
);
44 CASE(3,TestCompatCompose
);
46 CASE(5,TestHangulDecomp
);
47 CASE(6,TestHangulCompose
);
49 CASE(8,TestCompositionExclusion
);
50 CASE(9,TestZeroIndex
);
51 CASE(10,TestVerisign
);
52 CASE(11,TestPreviousNext
);
53 CASE(12,TestNormalizerAPI
);
54 CASE(13,TestConcatenate
);
55 CASE(14,FindFoldFCDExceptions
);
57 CASE(16,TestSkippable
);
58 #if !UCONFIG_NO_FILE_IO && !UCONFIG_NO_LEGACY_CONVERSION
59 CASE(17,TestCustomComp
);
60 CASE(18,TestCustomFCC
);
62 CASE(19,TestFilteredNormalizer2Coverage
);
63 default: name
= ""; break;
68 * Convert Java-style strings with \u Unicode escapes into UnicodeString objects
70 static UnicodeString
str(const char *input
)
72 UnicodeString
str(input
, ""); // Invariant conversion
73 return str
.unescape();
77 BasicNormalizerTest::BasicNormalizerTest()
80 // Input Decomposed Composed
82 canonTests
[0][0] = str("cat"); canonTests
[0][1] = str("cat"); canonTests
[0][2] = str("cat");
84 canonTests
[1][0] = str("\\u00e0ardvark"); canonTests
[1][1] = str("a\\u0300ardvark"); canonTests
[1][2] = str("\\u00e0ardvark");
86 canonTests
[2][0] = str("\\u1e0a"); canonTests
[2][1] = str("D\\u0307"); canonTests
[2][2] = str("\\u1e0a"); // D-dot_above
88 canonTests
[3][0] = str("D\\u0307"); canonTests
[3][1] = str("D\\u0307"); canonTests
[3][2] = str("\\u1e0a"); // D dot_above
90 canonTests
[4][0] = str("\\u1e0c\\u0307"); canonTests
[4][1] = str("D\\u0323\\u0307"); canonTests
[4][2] = str("\\u1e0c\\u0307"); // D-dot_below dot_above
92 canonTests
[5][0] = str("\\u1e0a\\u0323"); canonTests
[5][1] = str("D\\u0323\\u0307"); canonTests
[5][2] = str("\\u1e0c\\u0307"); // D-dot_above dot_below
94 canonTests
[6][0] = str("D\\u0307\\u0323"); canonTests
[6][1] = str("D\\u0323\\u0307"); canonTests
[6][2] = str("\\u1e0c\\u0307"); // D dot_below dot_above
96 canonTests
[7][0] = str("\\u1e10\\u0307\\u0323"); canonTests
[7][1] = str("D\\u0327\\u0323\\u0307"); canonTests
[7][2] = str("\\u1e10\\u0323\\u0307"); // D dot_below cedilla dot_above
98 canonTests
[8][0] = str("D\\u0307\\u0328\\u0323"); canonTests
[8][1] = str("D\\u0328\\u0323\\u0307"); canonTests
[8][2] = str("\\u1e0c\\u0328\\u0307"); // D dot_above ogonek dot_below
100 canonTests
[9][0] = str("\\u1E14"); canonTests
[9][1] = str("E\\u0304\\u0300"); canonTests
[9][2] = str("\\u1E14"); // E-macron-grave
102 canonTests
[10][0] = str("\\u0112\\u0300"); canonTests
[10][1] = str("E\\u0304\\u0300"); canonTests
[10][2] = str("\\u1E14"); // E-macron + grave
104 canonTests
[11][0] = str("\\u00c8\\u0304"); canonTests
[11][1] = str("E\\u0300\\u0304"); canonTests
[11][2] = str("\\u00c8\\u0304"); // E-grave + macron
106 canonTests
[12][0] = str("\\u212b"); canonTests
[12][1] = str("A\\u030a"); canonTests
[12][2] = str("\\u00c5"); // angstrom_sign
108 canonTests
[13][0] = str("\\u00c5"); canonTests
[13][1] = str("A\\u030a"); canonTests
[13][2] = str("\\u00c5"); // A-ring
110 canonTests
[14][0] = str("\\u00C4ffin"); canonTests
[14][1] = str("A\\u0308ffin"); canonTests
[14][2] = str("\\u00C4ffin");
112 canonTests
[15][0] = str("\\u00C4\\uFB03n"); canonTests
[15][1] = str("A\\u0308\\uFB03n"); canonTests
[15][2] = str("\\u00C4\\uFB03n");
114 canonTests
[16][0] = str("Henry IV"); canonTests
[16][1] = str("Henry IV"); canonTests
[16][2] = str("Henry IV");
116 canonTests
[17][0] = str("Henry \\u2163"); canonTests
[17][1] = str("Henry \\u2163"); canonTests
[17][2] = str("Henry \\u2163");
118 canonTests
[18][0] = str("\\u30AC"); canonTests
[18][1] = str("\\u30AB\\u3099"); canonTests
[18][2] = str("\\u30AC"); // ga (Katakana)
120 canonTests
[19][0] = str("\\u30AB\\u3099"); canonTests
[19][1] = str("\\u30AB\\u3099"); canonTests
[19][2] = str("\\u30AC"); // ka + ten
122 canonTests
[20][0] = str("\\uFF76\\uFF9E"); canonTests
[20][1] = str("\\uFF76\\uFF9E"); canonTests
[20][2] = str("\\uFF76\\uFF9E"); // hw_ka + hw_ten
124 canonTests
[21][0] = str("\\u30AB\\uFF9E"); canonTests
[21][1] = str("\\u30AB\\uFF9E"); canonTests
[21][2] = str("\\u30AB\\uFF9E"); // ka + hw_ten
126 canonTests
[22][0] = str("\\uFF76\\u3099"); canonTests
[22][1] = str("\\uFF76\\u3099"); canonTests
[22][2] = str("\\uFF76\\u3099"); // hw_ka + ten
128 canonTests
[23][0] = str("A\\u0300\\u0316"); canonTests
[23][1] = str("A\\u0316\\u0300"); canonTests
[23][2] = str("\\u00C0\\u0316");
131 // Input Decomposed Composed
132 compatTests
[0][0] = str("cat"); compatTests
[0][1] = str("cat"); compatTests
[0][2] = str("cat") ;
134 compatTests
[1][0] = str("\\uFB4f"); compatTests
[1][1] = str("\\u05D0\\u05DC"); compatTests
[1][2] = str("\\u05D0\\u05DC"); // Alef-Lamed vs. Alef, Lamed
136 compatTests
[2][0] = str("\\u00C4ffin"); compatTests
[2][1] = str("A\\u0308ffin"); compatTests
[2][2] = str("\\u00C4ffin") ;
138 compatTests
[3][0] = str("\\u00C4\\uFB03n"); compatTests
[3][1] = str("A\\u0308ffin"); compatTests
[3][2] = str("\\u00C4ffin") ; // ffi ligature -> f + f + i
140 compatTests
[4][0] = str("Henry IV"); compatTests
[4][1] = str("Henry IV"); compatTests
[4][2] = str("Henry IV") ;
142 compatTests
[5][0] = str("Henry \\u2163"); compatTests
[5][1] = str("Henry IV"); compatTests
[5][2] = str("Henry IV") ;
144 compatTests
[6][0] = str("\\u30AC"); compatTests
[6][1] = str("\\u30AB\\u3099"); compatTests
[6][2] = str("\\u30AC") ; // ga (Katakana)
146 compatTests
[7][0] = str("\\u30AB\\u3099"); compatTests
[7][1] = str("\\u30AB\\u3099"); compatTests
[7][2] = str("\\u30AC") ; // ka + ten
148 compatTests
[8][0] = str("\\uFF76\\u3099"); compatTests
[8][1] = str("\\u30AB\\u3099"); compatTests
[8][2] = str("\\u30AC") ; // hw_ka + ten
150 /* These two are broken in Unicode 2.1.2 but fixed in 2.1.5 and later */
151 compatTests
[9][0] = str("\\uFF76\\uFF9E"); compatTests
[9][1] = str("\\u30AB\\u3099"); compatTests
[9][2] = str("\\u30AC") ; // hw_ka + hw_ten
153 compatTests
[10][0] = str("\\u30AB\\uFF9E"); compatTests
[10][1] = str("\\u30AB\\u3099"); compatTests
[10][2] = str("\\u30AC") ; // ka + hw_ten
155 /* Hangul Canonical */
156 // Input Decomposed Composed
157 hangulCanon
[0][0] = str("\\ud4db"); hangulCanon
[0][1] = str("\\u1111\\u1171\\u11b6"); hangulCanon
[0][2] = str("\\ud4db") ;
159 hangulCanon
[1][0] = str("\\u1111\\u1171\\u11b6"), hangulCanon
[1][1] = str("\\u1111\\u1171\\u11b6"), hangulCanon
[1][2] = str("\\ud4db");
162 BasicNormalizerTest::~BasicNormalizerTest()
166 void BasicNormalizerTest::TestPrevious()
168 Normalizer
* norm
= new Normalizer("", UNORM_NFD
);
170 logln("testing decomp...");
172 for (i
= 0; i
< ARRAY_LENGTH(canonTests
); i
++) {
173 backAndForth(norm
, canonTests
[i
][0]);
176 logln("testing compose...");
177 norm
->setMode(UNORM_NFC
);
178 for (i
= 0; i
< ARRAY_LENGTH(canonTests
); i
++) {
179 backAndForth(norm
, canonTests
[i
][0]);
185 void BasicNormalizerTest::TestDecomp()
187 Normalizer
* norm
= new Normalizer("", UNORM_NFD
);
188 iterateTest(norm
, canonTests
, ARRAY_LENGTH(canonTests
), 1);
189 staticTest(UNORM_NFD
, 0, canonTests
, ARRAY_LENGTH(canonTests
), 1);
193 void BasicNormalizerTest::TestCompatDecomp()
195 Normalizer
* norm
= new Normalizer("", UNORM_NFKD
);
196 iterateTest(norm
, compatTests
, ARRAY_LENGTH(compatTests
), 1);
198 staticTest(UNORM_NFKD
, 0,
199 compatTests
, ARRAY_LENGTH(compatTests
), 1);
203 void BasicNormalizerTest::TestCanonCompose()
205 Normalizer
* norm
= new Normalizer("", UNORM_NFC
);
206 iterateTest(norm
, canonTests
, ARRAY_LENGTH(canonTests
), 2);
208 staticTest(UNORM_NFC
, 0, canonTests
,
209 ARRAY_LENGTH(canonTests
), 2);
213 void BasicNormalizerTest::TestCompatCompose()
215 Normalizer
* norm
= new Normalizer("", UNORM_NFKC
);
216 iterateTest(norm
, compatTests
, ARRAY_LENGTH(compatTests
), 2);
218 staticTest(UNORM_NFKC
, 0,
219 compatTests
, ARRAY_LENGTH(compatTests
), 2);
224 //-------------------------------------------------------------------------------
226 void BasicNormalizerTest::TestHangulCompose()
228 // Make sure that the static composition methods work
229 logln("Canonical composition...");
230 staticTest(UNORM_NFC
, 0, hangulCanon
, ARRAY_LENGTH(hangulCanon
), 2);
231 logln("Compatibility composition...");
233 // Now try iterative composition....
234 logln("Static composition...");
235 Normalizer
* norm
= new Normalizer("", UNORM_NFC
);
236 iterateTest(norm
, hangulCanon
, ARRAY_LENGTH(hangulCanon
), 2);
237 norm
->setMode(UNORM_NFKC
);
239 // And finally, make sure you can do it in reverse too
240 logln("Reverse iteration...");
241 norm
->setMode(UNORM_NFC
);
242 for (uint32_t i
= 0; i
< ARRAY_LENGTH(hangulCanon
); i
++) {
243 backAndForth(norm
, hangulCanon
[i
][0]);
248 void BasicNormalizerTest::TestHangulDecomp()
250 // Make sure that the static decomposition methods work
251 logln("Canonical decomposition...");
252 staticTest(UNORM_NFD
, 0, hangulCanon
, ARRAY_LENGTH(hangulCanon
), 1);
253 logln("Compatibility decomposition...");
255 // Now the iterative decomposition methods...
256 logln("Iterative decomposition...");
257 Normalizer
* norm
= new Normalizer("", UNORM_NFD
);
258 iterateTest(norm
, hangulCanon
, ARRAY_LENGTH(hangulCanon
), 1);
259 norm
->setMode(UNORM_NFKD
);
261 // And finally, make sure you can do it in reverse too
262 logln("Reverse iteration...");
263 norm
->setMode(UNORM_NFD
);
264 for (uint32_t i
= 0; i
< ARRAY_LENGTH(hangulCanon
); i
++) {
265 backAndForth(norm
, hangulCanon
[i
][0]);
271 * The Tibetan vowel sign AA, 0f71, was messed up prior to Unicode version 2.1.9.
273 void BasicNormalizerTest::TestTibetan(void) {
274 UnicodeString decomp
[1][3];
275 decomp
[0][0] = str("\\u0f77");
276 decomp
[0][1] = str("\\u0f77");
277 decomp
[0][2] = str("\\u0fb2\\u0f71\\u0f80");
279 UnicodeString compose
[1][3];
280 compose
[0][0] = str("\\u0fb2\\u0f71\\u0f80");
281 compose
[0][1] = str("\\u0fb2\\u0f71\\u0f80");
282 compose
[0][2] = str("\\u0fb2\\u0f71\\u0f80");
284 staticTest(UNORM_NFD
, 0, decomp
, ARRAY_LENGTH(decomp
), 1);
285 staticTest(UNORM_NFKD
, 0, decomp
, ARRAY_LENGTH(decomp
), 2);
286 staticTest(UNORM_NFC
, 0, compose
, ARRAY_LENGTH(compose
), 1);
287 staticTest(UNORM_NFKC
, 0, compose
, ARRAY_LENGTH(compose
), 2);
291 * Make sure characters in the CompositionExclusion.txt list do not get
294 void BasicNormalizerTest::TestCompositionExclusion(void) {
295 // This list is generated from CompositionExclusion.txt.
296 // Update whenever the normalizer tables are updated. Note
297 // that we test all characters listed, even those that can be
298 // derived from the Unicode DB and are therefore commented
300 // ### TODO read composition exclusion from source/data/unidata file
301 // and test against that
302 UnicodeString EXCLUDED
= str(
303 "\\u0340\\u0341\\u0343\\u0344\\u0374\\u037E\\u0387\\u0958"
304 "\\u0959\\u095A\\u095B\\u095C\\u095D\\u095E\\u095F\\u09DC"
305 "\\u09DD\\u09DF\\u0A33\\u0A36\\u0A59\\u0A5A\\u0A5B\\u0A5E"
306 "\\u0B5C\\u0B5D\\u0F43\\u0F4D\\u0F52\\u0F57\\u0F5C\\u0F69"
307 "\\u0F73\\u0F75\\u0F76\\u0F78\\u0F81\\u0F93\\u0F9D\\u0FA2"
308 "\\u0FA7\\u0FAC\\u0FB9\\u1F71\\u1F73\\u1F75\\u1F77\\u1F79"
309 "\\u1F7B\\u1F7D\\u1FBB\\u1FBE\\u1FC9\\u1FCB\\u1FD3\\u1FDB"
310 "\\u1FE3\\u1FEB\\u1FEE\\u1FEF\\u1FF9\\u1FFB\\u1FFD\\u2000"
311 "\\u2001\\u2126\\u212A\\u212B\\u2329\\u232A\\uF900\\uFA10"
312 "\\uFA12\\uFA15\\uFA20\\uFA22\\uFA25\\uFA26\\uFA2A\\uFB1F"
313 "\\uFB2A\\uFB2B\\uFB2C\\uFB2D\\uFB2E\\uFB2F\\uFB30\\uFB31"
314 "\\uFB32\\uFB33\\uFB34\\uFB35\\uFB36\\uFB38\\uFB39\\uFB3A"
315 "\\uFB3B\\uFB3C\\uFB3E\\uFB40\\uFB41\\uFB43\\uFB44\\uFB46"
316 "\\uFB47\\uFB48\\uFB49\\uFB4A\\uFB4B\\uFB4C\\uFB4D\\uFB4E"
318 for (int32_t i
=0; i
<EXCLUDED
.length(); ++i
) {
319 UnicodeString
a(EXCLUDED
.charAt(i
));
322 Normalizer::normalize(a
, UNORM_NFKD
, 0, b
, status
);
323 Normalizer::normalize(b
, UNORM_NFC
, 0, c
, status
);
325 errln("FAIL: " + hex(a
) + " x DECOMP_COMPAT => " +
326 hex(b
) + " x COMPOSE => " +
328 } else if (verbose
) {
329 logln("Ok: " + hex(a
) + " x DECOMP_COMPAT => " +
330 hex(b
) + " x COMPOSE => " +
337 * Test for a problem that showed up just before ICU 1.6 release
338 * having to do with combining characters with an index of zero.
339 * Such characters do not participate in any canonical
340 * decompositions. However, having an index of zero means that
341 * they all share one typeMask[] entry, that is, they all have to
342 * map to the same canonical class, which is not the case, in
345 void BasicNormalizerTest::TestZeroIndex(void) {
346 const char* DATA
[] = {
347 // Expect col1 x COMPOSE_COMPAT => col2
348 // Expect col2 x DECOMP => col3
349 "A\\u0316\\u0300", "\\u00C0\\u0316", "A\\u0316\\u0300",
350 "A\\u0300\\u0316", "\\u00C0\\u0316", "A\\u0316\\u0300",
351 "A\\u0327\\u0300", "\\u00C0\\u0327", "A\\u0327\\u0300",
352 "c\\u0321\\u0327", "c\\u0321\\u0327", "c\\u0321\\u0327",
353 "c\\u0327\\u0321", "\\u00E7\\u0321", "c\\u0327\\u0321",
355 int32_t DATA_length
= UPRV_LENGTHOF(DATA
);
357 for (int32_t i
=0; i
<DATA_length
; i
+=3) {
358 UErrorCode status
= U_ZERO_ERROR
;
359 UnicodeString
a(DATA
[i
], "");
362 Normalizer::normalize(a
, UNORM_NFKC
, 0, b
, status
);
363 if (U_FAILURE(status
)) {
364 dataerrln("Error calling normalize UNORM_NFKC: %s", u_errorName(status
));
366 UnicodeString
exp(DATA
[i
+1], "");
367 exp
= exp
.unescape();
369 logln((UnicodeString
)"Ok: " + hex(a
) + " x COMPOSE_COMPAT => " + hex(b
));
371 errln((UnicodeString
)"FAIL: " + hex(a
) + " x COMPOSE_COMPAT => " + hex(b
) +
372 ", expect " + hex(exp
));
375 Normalizer::normalize(b
, UNORM_NFD
, 0, a
, status
);
376 if (U_FAILURE(status
)) {
377 dataerrln("Error calling normalize UNORM_NFD: %s", u_errorName(status
));
379 UnicodeString exp
= UnicodeString(DATA
[i
+2], "").unescape();
381 logln((UnicodeString
)"Ok: " + hex(b
) + " x DECOMP => " + hex(a
));
383 errln((UnicodeString
)"FAIL: " + hex(b
) + " x DECOMP => " + hex(a
) +
384 ", expect " + hex(exp
));
391 * Run a few specific cases that are failing for Verisign.
393 void BasicNormalizerTest::TestVerisign(void) {
396 > 05B8 05B9 05B1 0591 05C3 05B0 05AC 059F
397 > Their output (supposedly from ICU):
398 > 05B8 05B1 05B9 0591 05C3 05B0 05AC 059F
399 > My output from charlint:
400 > 05B1 05B8 05B9 0591 05C3 05B0 05AC 059F
402 05B8 05B9 05B1 0591 05C3 05B0 05AC 059F => 05B1 05B8 05B9 0591 05C3 05B0
405 U+05B8 18 E HEBREW POINT QAMATS
406 U+05B9 19 F HEBREW POINT HOLAM
407 U+05B1 11 HEBREW POINT HATAF SEGOL
408 U+0591 220 HEBREW ACCENT ETNAHTA
409 U+05C3 0 HEBREW PUNCTUATION SOF PASUQ
410 U+05B0 10 HEBREW POINT SHEVA
411 U+05AC 230 HEBREW ACCENT ILUY
412 U+059F 230 HEBREW ACCENT QARNEY PARA
414 U+05B1 11 HEBREW POINT HATAF SEGOL
415 U+05B8 18 HEBREW POINT QAMATS
416 U+05B9 19 HEBREW POINT HOLAM
417 U+0591 220 HEBREW ACCENT ETNAHTA
418 U+05C3 0 HEBREW PUNCTUATION SOF PASUQ
419 U+05B0 10 HEBREW POINT SHEVA
420 U+05AC 230 HEBREW ACCENT ILUY
421 U+059F 230 HEBREW ACCENT QARNEY PARA
424 U+05B8 18 HEBREW POINT QAMATS
425 U+05B1 11 HEBREW POINT HATAF SEGOL
426 U+05B9 19 HEBREW POINT HOLAM
427 U+0591 220 HEBREW ACCENT ETNAHTA
428 U+05C3 0 HEBREW PUNCTUATION SOF PASUQ
429 U+05B0 10 HEBREW POINT SHEVA
430 U+05AC 230 HEBREW ACCENT ILUY
431 U+059F 230 HEBREW ACCENT QARNEY PARA
435 >0592 05B7 05BC 05A5 05B0 05C0 05C4 05AD
436 >Their output (supposedly from ICU):
437 >0592 05B0 05B7 05BC 05A5 05C0 05AD 05C4
438 >My output from charlint:
439 >05B0 05B7 05BC 05A5 0592 05C0 05AD 05C4
441 0592 05B7 05BC 05A5 05B0 05C0 05C4 05AD => 05B0 05B7 05BC 05A5 0592 05C0
444 U+0592 230 HEBREW ACCENT SEGOL
445 U+05B7 17 HEBREW POINT PATAH
446 U+05BC 21 HEBREW POINT DAGESH OR MAPIQ
447 U+05A5 220 HEBREW ACCENT MERKHA
448 U+05B0 10 HEBREW POINT SHEVA
449 U+05C0 0 HEBREW PUNCTUATION PASEQ
450 U+05C4 230 HEBREW MARK UPPER DOT
451 U+05AD 222 HEBREW ACCENT DEHI
453 U+05B0 10 HEBREW POINT SHEVA
454 U+05B7 17 HEBREW POINT PATAH
455 U+05BC 21 HEBREW POINT DAGESH OR MAPIQ
456 U+05A5 220 HEBREW ACCENT MERKHA
457 U+0592 230 HEBREW ACCENT SEGOL
458 U+05C0 0 HEBREW PUNCTUATION PASEQ
459 U+05AD 222 HEBREW ACCENT DEHI
460 U+05C4 230 HEBREW MARK UPPER DOT
463 U+0592 230 HEBREW ACCENT SEGOL
464 U+05B0 10 HEBREW POINT SHEVA
465 U+05B7 17 HEBREW POINT PATAH
466 U+05BC 21 HEBREW POINT DAGESH OR MAPIQ
467 U+05A5 220 HEBREW ACCENT MERKHA
468 U+05C0 0 HEBREW PUNCTUATION PASEQ
469 U+05AD 222 HEBREW ACCENT DEHI
470 U+05C4 230 HEBREW MARK UPPER DOT
472 UnicodeString data
[2][3];
473 data
[0][0] = str("\\u05B8\\u05B9\\u05B1\\u0591\\u05C3\\u05B0\\u05AC\\u059F");
474 data
[0][1] = str("\\u05B1\\u05B8\\u05B9\\u0591\\u05C3\\u05B0\\u05AC\\u059F");
475 data
[0][2] = str("");
476 data
[1][0] = str("\\u0592\\u05B7\\u05BC\\u05A5\\u05B0\\u05C0\\u05C4\\u05AD");
477 data
[1][1] = str("\\u05B0\\u05B7\\u05BC\\u05A5\\u0592\\u05C0\\u05AD\\u05C4");
478 data
[1][2] = str("");
480 staticTest(UNORM_NFD
, 0, data
, ARRAY_LENGTH(data
), 1);
481 staticTest(UNORM_NFC
, 0, data
, ARRAY_LENGTH(data
), 1);
484 //------------------------------------------------------------------------
485 // Internal utilities
488 UnicodeString
BasicNormalizerTest::hex(UChar ch
) {
489 UnicodeString result
;
490 return appendHex(ch
, 4, result
);
493 UnicodeString
BasicNormalizerTest::hex(const UnicodeString
& s
) {
494 UnicodeString result
;
495 for (int i
= 0; i
< s
.length(); ++i
) {
496 if (i
!= 0) result
+= (UChar
)0x2c/*,*/;
497 appendHex(s
[i
], 4, result
);
503 inline static void insert(UnicodeString
& dest
, int pos
, UChar32 ch
)
505 dest
.replace(pos
, 0, ch
);
508 void BasicNormalizerTest::backAndForth(Normalizer
* iter
, const UnicodeString
& input
)
511 iter
->setText(input
, status
);
513 // Run through the iterator forwards and stick it into a StringBuffer
514 UnicodeString forward
;
515 for (ch
= iter
->first(); ch
!= iter
->DONE
; ch
= iter
->next()) {
519 // Now do it backwards
520 UnicodeString reverse
;
521 for (ch
= iter
->last(); ch
!= iter
->DONE
; ch
= iter
->previous()) {
522 insert(reverse
, 0, ch
);
525 if (forward
!= reverse
) {
526 errln("Forward/reverse mismatch for input " + hex(input
)
527 + ", forward: " + hex(forward
) + ", backward: " + hex(reverse
));
531 void BasicNormalizerTest::staticTest(UNormalizationMode mode
, int options
,
532 UnicodeString tests
[][3], int length
,
535 for (int i
= 0; i
< length
; i
++)
537 UnicodeString
& input
= tests
[i
][0];
538 UnicodeString
& expect
= tests
[i
][outCol
];
540 logln("Normalizing '" + input
+ "' (" + hex(input
) + ")" );
542 UnicodeString output
;
543 Normalizer::normalize(input
, mode
, options
, output
, status
);
545 if (output
!= expect
) {
546 dataerrln(UnicodeString("ERROR: case ") + i
+ " normalized " + hex(input
) + "\n"
547 + " expected " + hex(expect
) + "\n"
548 + " static got " + hex(output
) );
553 void BasicNormalizerTest::iterateTest(Normalizer
* iter
,
554 UnicodeString tests
[][3], int length
,
557 for (int i
= 0; i
< length
; i
++)
559 UnicodeString
& input
= tests
[i
][0];
560 UnicodeString
& expect
= tests
[i
][outCol
];
562 logln("Normalizing '" + input
+ "' (" + hex(input
) + ")" );
564 iter
->setText(input
, status
);
565 assertEqual(input
, expect
, iter
, UnicodeString("ERROR: case ") + i
+ " ");
569 void BasicNormalizerTest::assertEqual(const UnicodeString
& input
,
570 const UnicodeString
& expected
,
572 const UnicodeString
& errPrefix
)
574 UnicodeString result
;
576 for (UChar32 ch
= iter
->first(); ch
!= iter
->DONE
; ch
= iter
->next()) {
579 if (result
!= expected
) {
580 dataerrln(errPrefix
+ "normalized " + hex(input
) + "\n"
581 + " expected " + hex(expected
) + "\n"
582 + " iterate got " + hex(result
) );
586 // helper class for TestPreviousNext()
587 // simple UTF-32 character iterator
588 class UChar32Iterator
{
590 UChar32Iterator(const UChar32
*text
, int32_t len
, int32_t index
) :
591 s(text
), length(len
), i(index
) {}
626 BasicNormalizerTest::TestPreviousNext(const UChar
*src
, int32_t srcLength
,
627 const UChar32
*expect
, int32_t expectLength
,
628 const int32_t *expectIndex
, // its length=expectLength+1
629 int32_t srcMiddle
, int32_t expectMiddle
,
631 UNormalizationMode mode
,
634 Normalizer
iter(src
, srcLength
, mode
);
636 // test getStaticClassID and getDynamicClassID
637 if(iter
.getDynamicClassID() != Normalizer::getStaticClassID()) {
638 errln("getStaticClassID != getDynamicClassID for Normalizer.");
641 UChar32Iterator
iter32(expect
, expectLength
, expectMiddle
);
646 // initially set the indexes into the middle of the strings
647 iter
.setIndexOnly(srcMiddle
);
649 // move around and compare the iteration code points with
651 const char *move
=moves
;
652 while((m
=*move
++)!=0) {
655 c2
=iter32
.previous();
659 } else /* m=='+' */ {
666 // copy the moves until the current (m) move, and terminate
668 uprv_strcpy(history
, moves
);
669 history
[move
-moves
]=0;
670 dataerrln("error: mismatch in Normalizer iteration (%s) at %s: "
671 "got c1=U+%04lx != expected c2=U+%04lx",
672 name
, history
, c1
, c2
);
677 if(iter
.getIndex()!=expectIndex
[iter32
.getIndex()]) {
678 // copy the moves until the current (m) move, and terminate
680 uprv_strcpy(history
, moves
);
681 history
[move
-moves
]=0;
682 errln("error: index mismatch in Normalizer iteration (%s) at %s: "
683 "Normalizer index %ld expected %ld\n",
684 name
, history
, iter
.getIndex(), expectIndex
[iter32
.getIndex()]);
691 BasicNormalizerTest::TestPreviousNext() {
692 // src and expect strings
693 static const UChar src
[]={
694 U16_LEAD(0x2f999), U16_TRAIL(0x2f999),
695 U16_LEAD(0x1d15f), U16_TRAIL(0x1d15f),
699 static const UChar32 expect
[]={
706 // expected src indexes corresponding to expect indexes
707 static const int32_t expectIndex
[]={
712 6 // behind last character
715 // src and expect strings for regression test for j2911
716 static const UChar src_j2911
[]={
717 U16_LEAD(0x2f999), U16_TRAIL(0x2f999),
718 0xdd00, 0xd900, // unpaired surrogates - regression test for j2911
722 static const UChar32 expect_j2911
[]={
724 0xdd00, 0xd900, // unpaired surrogates - regression test for j2911
729 // expected src indexes corresponding to expect indexes
730 static const int32_t expectIndex_j2911
[]={
735 8 // behind last character
738 // initial indexes into the src and expect strings
739 // for both sets of test data
748 // - for previous(), 0 for current(), + for next()
749 // for both sets of test data
750 static const char *const moves
="0+0+0--0-0-+++0--+++++++0--------";
752 TestPreviousNext(src
, UPRV_LENGTHOF(src
),
753 expect
, UPRV_LENGTHOF(expect
),
755 SRC_MIDDLE
, EXPECT_MIDDLE
,
756 moves
, UNORM_NFD
, "basic");
758 TestPreviousNext(src_j2911
, UPRV_LENGTHOF(src_j2911
),
759 expect_j2911
, UPRV_LENGTHOF(expect_j2911
),
761 SRC_MIDDLE
, EXPECT_MIDDLE
,
762 moves
, UNORM_NFKC
, "j2911");
764 // try again from different "middle" indexes
765 TestPreviousNext(src
, UPRV_LENGTHOF(src
),
766 expect
, UPRV_LENGTHOF(expect
),
768 SRC_MIDDLE_2
, EXPECT_MIDDLE_2
,
769 moves
, UNORM_NFD
, "basic_2");
771 TestPreviousNext(src_j2911
, UPRV_LENGTHOF(src_j2911
),
772 expect_j2911
, UPRV_LENGTHOF(expect_j2911
),
774 SRC_MIDDLE_2
, EXPECT_MIDDLE_2
,
775 moves
, UNORM_NFKC
, "j2911_2");
778 void BasicNormalizerTest::TestConcatenate() {
779 static const char *const
781 /* mode, left, right, result */
794 /* ### TODO: add more interesting cases */
798 "\\u0C4D\\U000110BA\\U0001D169",
799 "\\u03B1\\U0001D169\\U000110BA\\u0C4D\\u0345"
803 UnicodeString left
, right
, expect
, result
, r
;
804 UErrorCode errorCode
;
805 UNormalizationMode mode
;
808 /* test concatenation */
809 for(i
=0; i
<UPRV_LENGTHOF(cases
); ++i
) {
810 switch(*cases
[i
][0]) {
811 case 'C': mode
=UNORM_NFC
; break;
812 case 'D': mode
=UNORM_NFD
; break;
813 case 'c': mode
=UNORM_NFKC
; break;
814 case 'd': mode
=UNORM_NFKD
; break;
815 default: mode
=UNORM_NONE
; break;
818 left
=UnicodeString(cases
[i
][1], "").unescape();
819 right
=UnicodeString(cases
[i
][2], "").unescape();
820 expect
=UnicodeString(cases
[i
][3], "").unescape();
822 //result=r=UnicodeString();
823 errorCode
=U_ZERO_ERROR
;
825 r
=Normalizer::concatenate(left
, right
, result
, mode
, 0, errorCode
);
826 if(U_FAILURE(errorCode
) || /*result!=r ||*/ result
!=expect
) {
827 dataerrln("error in Normalizer::concatenate(), cases[] fails with "+
828 UnicodeString(u_errorName(errorCode
))+", result==expect: expected: "+
829 hex(expect
)+" =========> got: " + hex(result
));
833 /* test error cases */
835 /* left.getBuffer()==result.getBuffer() */
836 result
=r
=expect
=UnicodeString("zz", "");
837 errorCode
=U_UNEXPECTED_TOKEN
;
838 r
=Normalizer::concatenate(left
, right
, result
, mode
, 0, errorCode
);
839 if(errorCode
!=U_UNEXPECTED_TOKEN
|| result
!=r
|| !result
.isBogus()) {
840 errln("error in Normalizer::concatenate(), violates UErrorCode protocol");
844 errorCode
=U_ZERO_ERROR
;
845 r
=Normalizer::concatenate(left
, right
, result
, mode
, 0, errorCode
);
846 if(errorCode
!=U_ILLEGAL_ARGUMENT_ERROR
|| result
!=r
|| !result
.isBogus()) {
847 errln("error in Normalizer::concatenate(), does not detect left.isBogus()");
851 // reference implementation of Normalizer::compare
853 ref_norm_compare(const UnicodeString
&s1
, const UnicodeString
&s2
, uint32_t options
, UErrorCode
&errorCode
) {
854 UnicodeString r1
, r2
, t1
, t2
;
855 int32_t normOptions
=(int32_t)(options
>>UNORM_COMPARE_NORM_OPTIONS_SHIFT
);
857 if(options
&U_COMPARE_IGNORE_CASE
) {
858 Normalizer::decompose(s1
, FALSE
, normOptions
, r1
, errorCode
);
859 Normalizer::decompose(s2
, FALSE
, normOptions
, r2
, errorCode
);
861 r1
.foldCase(options
);
862 r2
.foldCase(options
);
868 Normalizer::decompose(r1
, FALSE
, normOptions
, t1
, errorCode
);
869 Normalizer::decompose(r2
, FALSE
, normOptions
, t2
, errorCode
);
871 if(options
&U_COMPARE_CODE_POINT_ORDER
) {
872 return t1
.compareCodePointOrder(t2
);
874 return t1
.compare(t2
);
878 // test wrapper for Normalizer::compare, sets UNORM_INPUT_IS_FCD appropriately
880 _norm_compare(const UnicodeString
&s1
, const UnicodeString
&s2
, uint32_t options
, UErrorCode
&errorCode
) {
881 int32_t normOptions
=(int32_t)(options
>>UNORM_COMPARE_NORM_OPTIONS_SHIFT
);
883 if( UNORM_YES
==Normalizer::quickCheck(s1
, UNORM_FCD
, normOptions
, errorCode
) &&
884 UNORM_YES
==Normalizer::quickCheck(s2
, UNORM_FCD
, normOptions
, errorCode
)) {
885 options
|=UNORM_INPUT_IS_FCD
;
888 return Normalizer::compare(s1
, s2
, options
, errorCode
);
891 // reference implementation of UnicodeString::caseCompare
893 ref_case_compare(const UnicodeString
&s1
, const UnicodeString
&s2
, uint32_t options
) {
894 UnicodeString t1
, t2
;
899 t1
.foldCase(options
);
900 t2
.foldCase(options
);
902 if(options
&U_COMPARE_CODE_POINT_ORDER
) {
903 return t1
.compareCodePointOrder(t2
);
905 return t1
.compare(t2
);
909 // reduce an integer to -1/0/1
910 static inline int32_t
911 _sign(int32_t value
) {
915 return (value
>>31)|1;
920 _signString(int32_t value
) {
923 } else if(value
==0) {
925 } else /* value>0 */ {
931 BasicNormalizerTest::TestCompare() {
932 // test Normalizer::compare and unorm_compare (thinly wrapped by the former)
933 // by comparing it with its semantic equivalent
934 // since we trust the pieces, this is sufficient
936 // test each string with itself and each other
937 // each time with all options
938 static const char *const
940 // some cases from NormalizationTest.txt
942 "D\\u031B\\u0307\\u0323",
943 "\\u1E0C\\u031B\\u0307",
944 "D\\u031B\\u0323\\u0307",
945 "d\\u031B\\u0323\\u0307",
952 // Angstrom sign = A ring
960 "a\\u059A\\u0316\\u302A\\u032Fb",
961 "a\\u302A\\u0316\\u032F\\u059Ab",
962 "a\\u302A\\u0316\\u032F\\u059Ab",
963 "A\\u059A\\u0316\\u302A\\u032Fb",
965 // from ICU case folding tests
967 "A\\u00df\\u00b5\\ufb03\\U0001040c\\u0131",
968 "ass\\u03bcffi\\U00010434i",
969 "\\u0061\\u0042\\u0131\\u03a3\\u00df\\ufb03\\ud93f\\udfff",
970 "\\u0041\\u0062\\u0069\\u03c3\\u0073\\u0053\\u0046\\u0066\\u0049\\ud93f\\udfff",
971 "\\u0041\\u0062\\u0131\\u03c3\\u0053\\u0073\\u0066\\u0046\\u0069\\ud93f\\udfff",
972 "\\u0041\\u0062\\u0069\\u03c3\\u0073\\u0053\\u0046\\u0066\\u0049\\ud93f\\udffd",
974 // U+d800 U+10001 see implementation comment in unorm_cmpEquivFold
975 // vs. U+10000 at bottom - code point order
977 "\\ud800\\ud800\\udc01",
980 // other code point order tests from ustrtest.cpp
983 "\\u20ac\\ud800\\udc00",
988 "\\uff61\\ud800\\udc02",
992 // long strings, see cnormtst.c/TestNormCoverage()
993 // equivalent if case-insensitive
995 "\\uAD8B\\uAD8B\\uAD8B\\uAD8B"
996 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
997 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
998 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
999 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
1000 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
1001 "aaaaaaaaaaaaaaaaaazzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"
1002 "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
1003 "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
1004 "ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
1005 "\\uAD8B\\uAD8B\\uAD8B\\uAD8B"
1006 "d\\u031B\\u0307\\u0323",
1008 "\\u1100\\u116f\\u11aa\\uAD8B\\uAD8B\\u1100\\u116f\\u11aa"
1009 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
1010 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
1011 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
1012 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
1013 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
1014 "aaaaaaaaaaAAAAAAAAZZZZZZZZZZZZZZZZzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"
1015 "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
1016 "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
1017 "ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
1018 "\\u1100\\u116f\\u11aa\\uAD8B\\uAD8B\\u1100\\u116f\\u11aa"
1019 "\\u1E0C\\u031B\\u0307",
1021 // some strings that may make a difference whether the compare function
1022 // case-folds or decomposes first
1024 "\\u0360\\u0345\\u0334",
1025 "\\u0360\\u03b9\\u0334",
1027 "\\u0360\\u1f80\\u0334",
1028 "\\u0360\\u03b1\\u0313\\u03b9\\u0334",
1030 "\\u0360\\u1ffc\\u0334",
1031 "\\u0360\\u03c9\\u03b9\\u0334",
1033 "a\\u0360\\u0345\\u0360\\u0345b",
1034 "a\\u0345\\u0360\\u0345\\u0360b",
1036 // interesting cases for canonical caseless match with turkic i handling
1041 // strings with post-Unicode 3.2 normalization or normalization corrections
1043 "\\u00e4\\u193b\\U0002f868",
1044 "\\u0061\\u193b\\u0308\\u36fc",
1051 UnicodeString s
[100]; // at least as many items as in strings[] !
1053 // all combinations of options
1054 // UNORM_INPUT_IS_FCD is set automatically if both input strings fulfill FCD conditions
1055 // set UNORM_UNICODE_3_2 in one additional combination
1056 static const struct {
1061 { U_COMPARE_CODE_POINT_ORDER
, "c.p. order" },
1062 { U_COMPARE_IGNORE_CASE
, "ignore case" },
1063 { U_COMPARE_CODE_POINT_ORDER
|U_COMPARE_IGNORE_CASE
, "c.p. order & ignore case" },
1064 { U_COMPARE_IGNORE_CASE
|U_FOLD_CASE_EXCLUDE_SPECIAL_I
, "ignore case & special i" },
1065 { U_COMPARE_CODE_POINT_ORDER
|U_COMPARE_IGNORE_CASE
|U_FOLD_CASE_EXCLUDE_SPECIAL_I
, "c.p. order & ignore case & special i" },
1066 { UNORM_UNICODE_3_2
<<UNORM_COMPARE_NORM_OPTIONS_SHIFT
, "Unicode 3.2" }
1069 int32_t i
, j
, k
, count
=UPRV_LENGTHOF(strings
);
1070 int32_t result
, refResult
;
1072 UErrorCode errorCode
;
1074 // create the UnicodeStrings
1075 for(i
=0; i
<count
; ++i
) {
1076 s
[i
]=UnicodeString(strings
[i
], "").unescape();
1079 // test them each with each other
1080 for(i
=0; i
<count
; ++i
) {
1081 for(j
=i
; j
<count
; ++j
) {
1082 for(k
=0; k
<UPRV_LENGTHOF(opt
); ++k
) {
1083 // test Normalizer::compare
1084 errorCode
=U_ZERO_ERROR
;
1085 result
=_norm_compare(s
[i
], s
[j
], opt
[k
].options
, errorCode
);
1086 refResult
=ref_norm_compare(s
[i
], s
[j
], opt
[k
].options
, errorCode
);
1087 if(_sign(result
)!=_sign(refResult
)) {
1088 errln("Normalizer::compare(%d, %d, %s)%s should be %s %s",
1089 i
, j
, opt
[k
].name
, _signString(result
), _signString(refResult
),
1090 U_SUCCESS(errorCode
) ? "" : u_errorName(errorCode
));
1093 // test UnicodeString::caseCompare - same internal implementation function
1094 if(opt
[k
].options
&U_COMPARE_IGNORE_CASE
) {
1095 errorCode
=U_ZERO_ERROR
;
1096 result
=s
[i
].caseCompare(s
[j
], opt
[k
].options
);
1097 refResult
=ref_case_compare(s
[i
], s
[j
], opt
[k
].options
);
1098 if(_sign(result
)!=_sign(refResult
)) {
1099 errln("UniStr::caseCompare(%d, %d, %s)%s should be %s %s",
1100 i
, j
, opt
[k
].name
, _signString(result
), _signString(refResult
),
1101 U_SUCCESS(errorCode
) ? "" : u_errorName(errorCode
));
1108 // test cases with i and I to make sure Turkic works
1109 static const UChar iI
[]={ 0x49, 0x69, 0x130, 0x131 };
1110 UnicodeSet iSet
, set
;
1112 UnicodeString s1
, s2
;
1114 const Normalizer2Impl
*nfcImpl
=Normalizer2Factory::getNFCImpl(errorCode
);
1115 if(U_FAILURE(errorCode
) || !nfcImpl
->ensureCanonIterData(errorCode
)) {
1116 dataerrln("Normalizer2Factory::getNFCImpl().ensureCanonIterData() failed: %s",
1117 u_errorName(errorCode
));
1121 // collect all sets into one for contiguous output
1122 for(i
=0; i
<UPRV_LENGTHOF(iI
); ++i
) {
1123 if(nfcImpl
->getCanonStartSet(iI
[i
], iSet
)) {
1128 // test all of these precomposed characters
1129 const Normalizer2
*nfcNorm2
=Normalizer2::getNFCInstance(errorCode
);
1130 UnicodeSetIterator
it(set
);
1131 while(it
.next() && !it
.isString()) {
1132 UChar32 c
=it
.getCodepoint();
1133 if(!nfcNorm2
->getDecomposition(c
, s2
)) {
1134 dataerrln("NFC.getDecomposition(i-composite U+%04lx) failed", (long)c
);
1139 for(k
=0; k
<UPRV_LENGTHOF(opt
); ++k
) {
1140 // test Normalizer::compare
1141 errorCode
=U_ZERO_ERROR
;
1142 result
=_norm_compare(s1
, s2
, opt
[k
].options
, errorCode
);
1143 refResult
=ref_norm_compare(s1
, s2
, opt
[k
].options
, errorCode
);
1144 if(_sign(result
)!=_sign(refResult
)) {
1145 errln("Normalizer::compare(U+%04x with its NFD, %s)%s should be %s %s",
1146 c
, opt
[k
].name
, _signString(result
), _signString(refResult
),
1147 U_SUCCESS(errorCode
) ? "" : u_errorName(errorCode
));
1150 // test UnicodeString::caseCompare - same internal implementation function
1151 if(opt
[k
].options
&U_COMPARE_IGNORE_CASE
) {
1152 errorCode
=U_ZERO_ERROR
;
1153 result
=s1
.caseCompare(s2
, opt
[k
].options
);
1154 refResult
=ref_case_compare(s1
, s2
, opt
[k
].options
);
1155 if(_sign(result
)!=_sign(refResult
)) {
1156 errln("UniStr::caseCompare(U+%04x with its NFD, %s)%s should be %s %s",
1157 c
, opt
[k
].name
, _signString(result
), _signString(refResult
),
1158 U_SUCCESS(errorCode
) ? "" : u_errorName(errorCode
));
1164 // test getDecomposition() for some characters that do not decompose
1165 if( nfcNorm2
->getDecomposition(0x20, s2
) ||
1166 nfcNorm2
->getDecomposition(0x4e00, s2
) ||
1167 nfcNorm2
->getDecomposition(0x20002, s2
)
1169 errln("NFC.getDecomposition() returns TRUE for characters which do not have decompositions");
1172 // test getRawDecomposition() for some characters that do not decompose
1173 if( nfcNorm2
->getRawDecomposition(0x20, s2
) ||
1174 nfcNorm2
->getRawDecomposition(0x4e00, s2
) ||
1175 nfcNorm2
->getRawDecomposition(0x20002, s2
)
1177 errln("NFC.getRawDecomposition() returns TRUE for characters which do not have decompositions");
1180 // test composePair() for some pairs of characters that do not compose
1181 if( nfcNorm2
->composePair(0x20, 0x301)>=0 ||
1182 nfcNorm2
->composePair(0x61, 0x305)>=0 ||
1183 nfcNorm2
->composePair(0x1100, 0x1160)>=0 ||
1184 nfcNorm2
->composePair(0xac00, 0x11a7)>=0
1186 errln("NFC.composePair() incorrectly composes some pairs of characters");
1189 // test FilteredNormalizer2::getDecomposition()
1190 UnicodeSet
filter(UNICODE_STRING_SIMPLE("[^\\u00a0-\\u00ff]"), errorCode
);
1191 FilteredNormalizer2
fn2(*nfcNorm2
, filter
);
1192 if( fn2
.getDecomposition(0xe4, s1
) || !fn2
.getDecomposition(0x100, s2
) ||
1193 s2
.length()!=2 || s2
[0]!=0x41 || s2
[1]!=0x304
1195 errln("FilteredNormalizer2(NFC, ^A0-FF).getDecomposition() failed");
1198 // test FilteredNormalizer2::getRawDecomposition()
1199 if( fn2
.getRawDecomposition(0xe4, s1
) || !fn2
.getRawDecomposition(0x100, s2
) ||
1200 s2
.length()!=2 || s2
[0]!=0x41 || s2
[1]!=0x304
1202 errln("FilteredNormalizer2(NFC, ^A0-FF).getRawDecomposition() failed");
1205 // test FilteredNormalizer2::composePair()
1206 if( 0x100!=fn2
.composePair(0x41, 0x304) ||
1207 fn2
.composePair(0xc7, 0x301)>=0 // unfiltered result: U+1E08
1209 errln("FilteredNormalizer2(NFC, ^A0-FF).composePair() failed");
1213 // verify that case-folding does not un-FCD strings
1215 BasicNormalizerTest::countFoldFCDExceptions(uint32_t foldingOptions
) {
1216 UnicodeString s
, fold
, d
;
1219 uint8_t cc
, trailCC
, foldCC
, foldTrailCC
;
1220 UNormalizationCheckResult qcResult
;
1223 UErrorCode errorCode
;
1225 logln("Test if case folding may un-FCD a string (folding options %04lx)", foldingOptions
);
1228 for(c
=0; c
<=0x10ffff; ++c
) {
1229 errorCode
= U_ZERO_ERROR
;
1230 category
=u_charType(c
);
1231 if(category
==U_UNASSIGNED
) {
1232 continue; // skip unassigned code points
1235 c
=0xd7a3; // skip Hangul - no case folding there
1238 // skip Han blocks - no case folding there either
1254 // get leading and trailing cc for c
1255 Normalizer::decompose(s
, FALSE
, 0, d
, errorCode
);
1257 cc
=u_getCombiningClass(d
.char32At(0));
1258 trailCC
=u_getCombiningClass(d
.char32At(d
.length()-1));
1260 // get leading and trailing cc for the case-folding of c
1261 s
.foldCase(foldingOptions
);
1262 Normalizer::decompose(s
, FALSE
, 0, d
, errorCode
);
1263 foldCC
=u_getCombiningClass(d
.char32At(0));
1264 foldTrailCC
=u_getCombiningClass(d
.char32At(d
.length()-1));
1266 qcResult
=Normalizer::quickCheck(s
, UNORM_FCD
, errorCode
);
1268 if (U_FAILURE(errorCode
)) {
1270 dataerrln("U+%04lx: Failed with error %s", u_errorName(errorCode
));
1274 // - character maps to empty string: adjacent characters may then need reordering
1275 // - folding has different leading/trailing cc's, and they don't become just 0
1276 // - folding itself is not FCD
1277 if( qcResult
!=UNORM_YES
||
1279 (cc
!=foldCC
&& foldCC
!=0) || (trailCC
!=foldTrailCC
&& foldTrailCC
!=0)
1282 dataerrln("U+%04lx: case-folding may un-FCD a string (folding options %04lx)", c
, foldingOptions
);
1283 dataerrln(" cc %02x trailCC %02x foldCC(U+%04lx) %02x foldTrailCC(U+%04lx) %02x quickCheck(folded)=%d", cc
, trailCC
, d
.char32At(0), foldCC
, d
.char32At(d
.length()-1), foldTrailCC
, qcResult
);
1288 // if a code point is in NFD but its case folding is not, then
1289 // unorm_compare will also fail
1290 if(isNFD
&& UNORM_YES
!=Normalizer::quickCheck(s
, UNORM_NFD
, errorCode
)) {
1292 errln("U+%04lx: case-folding un-NFDs this character (folding options %04lx)", c
, foldingOptions
);
1296 logln("There are %ld code points for which case-folding may un-FCD a string (folding options %04lx)", count
, foldingOptions
);
1301 BasicNormalizerTest::FindFoldFCDExceptions() {
1304 count
=countFoldFCDExceptions(0);
1305 count
+=countFoldFCDExceptions(U_FOLD_CASE_EXCLUDE_SPECIAL_I
);
1308 * If case-folding un-FCDs any strings, then unorm_compare() must be
1310 * It currently assumes that one can check for FCD then case-fold
1311 * and then still have FCD strings for raw decomposition without reordering.
1313 dataerrln("error: There are %ld code points for which case-folding may un-FCD a string for all folding options.\n"
1314 "See comment in BasicNormalizerTest::FindFoldFCDExceptions()!", count
);
1319 initExpectedSkippables(UnicodeSet skipSets
[UNORM_MODE_COUNT
], UErrorCode
&errorCode
) {
1320 skipSets
[UNORM_NFD
].applyPattern(
1321 UNICODE_STRING_SIMPLE("[[:NFD_QC=Yes:]&[:ccc=0:]]"), errorCode
);
1322 skipSets
[UNORM_NFC
].applyPattern(
1323 UNICODE_STRING_SIMPLE("[[:NFC_QC=Yes:]&[:ccc=0:]-[:HST=LV:]]"), errorCode
);
1324 skipSets
[UNORM_NFKD
].applyPattern(
1325 UNICODE_STRING_SIMPLE("[[:NFKD_QC=Yes:]&[:ccc=0:]]"), errorCode
);
1326 skipSets
[UNORM_NFKC
].applyPattern(
1327 UNICODE_STRING_SIMPLE("[[:NFKC_QC=Yes:]&[:ccc=0:]-[:HST=LV:]]"), errorCode
);
1329 // Remove from the NFC and NFKC sets all those characters that change
1330 // when a back-combining character is added.
1331 // First, get all of the back-combining characters and their combining classes.
1332 UnicodeSet
combineBack("[:NFC_QC=Maybe:]", errorCode
);
1333 int32_t numCombineBack
=combineBack
.size();
1334 int32_t *combineBackCharsAndCc
=new int32_t[numCombineBack
*2];
1335 UnicodeSetIterator
iter(combineBack
);
1336 for(int32_t i
=0; i
<numCombineBack
; ++i
) {
1338 UChar32 c
=iter
.getCodepoint();
1339 combineBackCharsAndCc
[2*i
]=c
;
1340 combineBackCharsAndCc
[2*i
+1]=u_getCombiningClass(c
);
1343 // We need not look at control codes, Han characters nor Hangul LVT syllables because they
1344 // do not combine forward. LV syllables are already removed.
1345 UnicodeSet
notInteresting("[[:C:][:Unified_Ideograph:][:HST=LVT:]]", errorCode
);
1346 LocalPointer
<UnicodeSet
> unsure(&((UnicodeSet
*)(skipSets
[UNORM_NFC
].clone()))->removeAll(notInteresting
));
1347 // System.out.format("unsure.size()=%d\n", unsure.size());
1349 // For each character about which we are unsure, see if it changes when we add
1350 // one of the back-combining characters.
1351 const Normalizer2
*norm2
=Normalizer2::getNFCInstance(errorCode
);
1353 iter
.reset(*unsure
);
1354 while(iter
.next()) {
1355 UChar32 c
=iter
.getCodepoint();
1357 int32_t cLength
=s
.length();
1358 int32_t tccc
=u_getIntPropertyValue(c
, UCHAR_TRAIL_CANONICAL_COMBINING_CLASS
);
1359 for(int32_t i
=0; i
<numCombineBack
; ++i
) {
1360 // If c's decomposition ends with a character with non-zero combining class, then
1361 // c can only change if it combines with a character with a non-zero combining class.
1362 int32_t cc2
=combineBackCharsAndCc
[2*i
+1];
1363 if(tccc
==0 || cc2
!=0) {
1364 UChar32 c2
=combineBackCharsAndCc
[2*i
];
1366 if(!norm2
->isNormalized(s
, errorCode
)) {
1367 // System.out.format("remove U+%04x (tccc=%d) + U+%04x (cc=%d)\n", c, tccc, c2, cc2);
1368 skipSets
[UNORM_NFC
].remove(c
);
1369 skipSets
[UNORM_NFKC
].remove(c
);
1372 s
.truncate(cLength
);
1376 delete [] combineBackCharsAndCc
;
1380 BasicNormalizerTest::TestSkippable() {
1381 UnicodeSet diff
, skipSets
[UNORM_MODE_COUNT
], expectSets
[UNORM_MODE_COUNT
];
1382 UnicodeString s
, pattern
;
1384 /* build NF*Skippable sets from runtime data */
1385 IcuTestErrorCode
errorCode(*this, "TestSkippable");
1386 skipSets
[UNORM_NFD
].applyPattern(UNICODE_STRING_SIMPLE("[:NFD_Inert:]"), errorCode
);
1387 skipSets
[UNORM_NFKD
].applyPattern(UNICODE_STRING_SIMPLE("[:NFKD_Inert:]"), errorCode
);
1388 skipSets
[UNORM_NFC
].applyPattern(UNICODE_STRING_SIMPLE("[:NFC_Inert:]"), errorCode
);
1389 skipSets
[UNORM_NFKC
].applyPattern(UNICODE_STRING_SIMPLE("[:NFKC_Inert:]"), errorCode
);
1390 if(errorCode
.logDataIfFailureAndReset("UnicodeSet(NF..._Inert) failed")) {
1394 /* get expected sets from hardcoded patterns */
1395 initExpectedSkippables(expectSets
, errorCode
);
1396 errorCode
.assertSuccess();
1398 for(int32_t i
=UNORM_NONE
; i
<UNORM_MODE_COUNT
; ++i
) {
1399 if(skipSets
[i
]!=expectSets
[i
]) {
1400 errln("error: TestSkippable skipSets[%d]!=expectedSets[%d]\n", i
, i
);
1401 // Note: This used to depend on hardcoded UnicodeSet patterns generated by
1402 // Mark's unicodetools.com.ibm.text.UCD.NFSkippable, by
1403 // running com.ibm.text.UCD.Main with the option NFSkippable.
1404 // Since ICU 4.6/Unicode 6, we are generating the
1405 // expectSets ourselves in initSkippables().
1407 s
=UNICODE_STRING_SIMPLE("skip-expect=");
1408 (diff
=skipSets
[i
]).removeAll(expectSets
[i
]).toPattern(pattern
, TRUE
);
1412 s
.append(UNICODE_STRING_SIMPLE("\n\nexpect-skip="));
1413 (diff
=expectSets
[i
]).removeAll(skipSets
[i
]).toPattern(pattern
, TRUE
);
1415 s
.append(UNICODE_STRING_SIMPLE("\n\n"));
1422 struct StringPair
{ const char *input
, *expected
; };
1425 BasicNormalizerTest::TestCustomComp() {
1426 static const StringPair pairs
[]={
1427 { "\\uD801\\uE000\\uDFFE", "" },
1428 { "\\uD800\\uD801\\uE000\\uDFFE\\uDFFF", "\\uD7FF\\uFFFF" },
1429 { "\\uD800\\uD801\\uDFFE\\uDFFF", "\\uD7FF\\U000107FE\\uFFFF" },
1430 { "\\uE001\\U000110B9\\u0345\\u0308\\u0327", "\\uE002\\U000110B9\\u0327\\u0345" },
1431 { "\\uE010\\U000F0011\\uE012", "\\uE011\\uE012" },
1432 { "\\uE010\\U000F0011\\U000F0011\\uE012", "\\uE011\\U000F0010" },
1433 { "\\uE111\\u1161\\uE112\\u1162", "\\uAE4C\\u1102\\u0062\\u1162" },
1434 { "\\uFFF3\\uFFF7\\U00010036\\U00010077", "\\U00010037\\U00010037\\uFFF6\\U00010037" }
1436 IcuTestErrorCode
errorCode(*this, "BasicNormalizerTest/TestCustomComp");
1437 const Normalizer2
*customNorm2
=
1438 Normalizer2::getInstance(loadTestData(errorCode
), "testnorm",
1439 UNORM2_COMPOSE
, errorCode
);
1440 if(errorCode
.logDataIfFailureAndReset("unable to load testdata/testnorm.nrm")) {
1443 for(int32_t i
=0; i
<UPRV_LENGTHOF(pairs
); ++i
) {
1444 const StringPair
&pair
=pairs
[i
];
1445 UnicodeString input
=UnicodeString(pair
.input
, -1, US_INV
).unescape();
1446 UnicodeString expected
=UnicodeString(pair
.expected
, -1, US_INV
).unescape();
1447 UnicodeString result
=customNorm2
->normalize(input
, errorCode
);
1448 if(result
!=expected
) {
1449 errln("custom compose Normalizer2 did not normalize input %d as expected", i
);
1455 BasicNormalizerTest::TestCustomFCC() {
1456 static const StringPair pairs
[]={
1457 { "\\uD801\\uE000\\uDFFE", "" },
1458 { "\\uD800\\uD801\\uE000\\uDFFE\\uDFFF", "\\uD7FF\\uFFFF" },
1459 { "\\uD800\\uD801\\uDFFE\\uDFFF", "\\uD7FF\\U000107FE\\uFFFF" },
1460 // The following expected result is different from CustomComp
1461 // because of only-contiguous composition.
1462 { "\\uE001\\U000110B9\\u0345\\u0308\\u0327", "\\uE001\\U000110B9\\u0327\\u0308\\u0345" },
1463 { "\\uE010\\U000F0011\\uE012", "\\uE011\\uE012" },
1464 { "\\uE010\\U000F0011\\U000F0011\\uE012", "\\uE011\\U000F0010" },
1465 { "\\uE111\\u1161\\uE112\\u1162", "\\uAE4C\\u1102\\u0062\\u1162" },
1466 { "\\uFFF3\\uFFF7\\U00010036\\U00010077", "\\U00010037\\U00010037\\uFFF6\\U00010037" }
1468 IcuTestErrorCode
errorCode(*this, "BasicNormalizerTest/TestCustomFCC");
1469 const Normalizer2
*customNorm2
=
1470 Normalizer2::getInstance(loadTestData(errorCode
), "testnorm",
1471 UNORM2_COMPOSE_CONTIGUOUS
, errorCode
);
1472 if(errorCode
.logDataIfFailureAndReset("unable to load testdata/testnorm.nrm")) {
1475 for(int32_t i
=0; i
<UPRV_LENGTHOF(pairs
); ++i
) {
1476 const StringPair
&pair
=pairs
[i
];
1477 UnicodeString input
=UnicodeString(pair
.input
, -1, US_INV
).unescape();
1478 UnicodeString expected
=UnicodeString(pair
.expected
, -1, US_INV
).unescape();
1479 UnicodeString result
=customNorm2
->normalize(input
, errorCode
);
1480 if(result
!=expected
) {
1481 errln("custom FCC Normalizer2 did not normalize input %d as expected", i
);
1486 /* Improve code coverage of Normalizer2 */
1488 BasicNormalizerTest::TestFilteredNormalizer2Coverage() {
1489 UErrorCode errorCode
= U_ZERO_ERROR
;
1490 const Normalizer2
*nfcNorm2
=Normalizer2::getNFCInstance(errorCode
);
1491 if (U_FAILURE(errorCode
)) {
1492 dataerrln("Normalizer2::getNFCInstance() call failed - %s", u_errorName(status
));
1495 UnicodeSet
filter(UNICODE_STRING_SIMPLE("[^\\u00a0-\\u00ff\\u0310-\\u031f]"), errorCode
);
1496 FilteredNormalizer2
fn2(*nfcNorm2
, filter
);
1498 UChar32 char32
= 0x0054;
1500 if (fn2
.isInert(char32
)) {
1501 errln("FilteredNormalizer2.isInert() failed.");
1504 if (fn2
.hasBoundaryAfter(char32
)) {
1505 errln("FilteredNormalizer2.hasBoundaryAfter() failed.");
1509 for(c
=0; c
<=0x3ff; ++c
) {
1510 uint8_t expectedCC
= filter
.contains(c
) ? nfcNorm2
->getCombiningClass(c
) : 0;
1511 uint8_t cc
=fn2
.getCombiningClass(c
);
1512 if(cc
!=expectedCC
) {
1514 UnicodeString("FilteredNormalizer2(NFC, ^A0-FF,310-31F).getCombiningClass(U+")+
1516 ")==filtered NFC.getCC()");
1520 UnicodeString newString1
= UNICODE_STRING_SIMPLE("[^\\u0100-\\u01ff]");
1521 UnicodeString newString2
= UNICODE_STRING_SIMPLE("[^\\u0200-\\u02ff]");
1522 fn2
.append(newString1
, newString2
, errorCode
);
1523 if (U_FAILURE(errorCode
)) {
1524 errln("FilteredNormalizer2.append() failed.");
1528 #endif /* #if !UCONFIG_NO_NORMALIZATION */