]> git.saurik.com Git - apple/icu.git/blame - icuSources/test/intltest/tstnorm.cpp
projects / apple / icu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ICU-551.51.4.tar.gz
[apple/icu.git] / icuSources / test / intltest / tstnorm.cpp
CommitLineData
b75a7d8f
A		 1/********************************************************************
2 * COPYRIGHT:
b331163b 3 * Copyright (c) 1997-2014, International Business Machines Corporation and
b75a7d8f
A		 4 * others. All Rights Reserved.
5 ********************************************************************/
6
7#include "unicode/utypes.h"
8
9#if !UCONFIG_NO_NORMALIZATION
10
11#include "unicode/uchar.h"
729e4ab9 12#include "unicode/errorcode.h"
b75a7d8f
A		 13#include "unicode/normlzr.h"
14#include "unicode/uniset.h"
15#include "unicode/usetiter.h"
16#include "unicode/schriter.h"
4388f060 17#include "unicode/utf16.h"
b75a7d8f 18#include "cstring.h"
729e4ab9 19#include "normalizer2impl.h"
b75a7d8f
A		 20#include "tstnorm.h"
21
b331163b 22#define ARRAY_LENGTH(array) UPRV_LENGTHOF(array)
b75a7d8f
A		 23
24#define CASE(id,test) case id: \
25 name = #test; \
26 if (exec) { \
27 logln(#test "---"); \
28 logln((UnicodeString)""); \
29 test(); \
30 } \
31 break
32
33static UErrorCode status = U_ZERO_ERROR;
34
35void BasicNormalizerTest::runIndexedTest(int32_t index, UBool exec,
36 const char* &name, char* /*par*/) {
37 switch (index) {
38 CASE(0,TestDecomp);
39 CASE(1,TestCompatDecomp);
40 CASE(2,TestCanonCompose);
41 CASE(3,TestCompatCompose);
42 CASE(4,TestPrevious);
43 CASE(5,TestHangulDecomp);
44 CASE(6,TestHangulCompose);
45 CASE(7,TestTibetan);
46 CASE(8,TestCompositionExclusion);
47 CASE(9,TestZeroIndex);
48 CASE(10,TestVerisign);
49 CASE(11,TestPreviousNext);
50 CASE(12,TestNormalizerAPI);
51 CASE(13,TestConcatenate);
52 CASE(14,FindFoldFCDExceptions);
53 CASE(15,TestCompare);
54 CASE(16,TestSkippable);
729e4ab9
A		 55#if !UCONFIG_NO_FILE_IO && !UCONFIG_NO_LEGACY_CONVERSION
56 CASE(17,TestCustomComp);
57 CASE(18,TestCustomFCC);
58#endif
59 CASE(19,TestFilteredNormalizer2Coverage);
b75a7d8f
A		 60 default: name = ""; break;
61 }
62}
63
64/**
65 * Convert Java-style strings with \u Unicode escapes into UnicodeString objects
66 */
67static UnicodeString str(const char *input)
68{
69 UnicodeString str(input, ""); // Invariant conversion
70 return str.unescape();
71}
72
73
74BasicNormalizerTest::BasicNormalizerTest()
75{
76 // canonTest
77 // Input Decomposed Composed
78
79 canonTests[0][0] = str("cat"); canonTests[0][1] = str("cat"); canonTests[0][2] = str("cat");
80
81 canonTests[1][0] = str("\\u00e0ardvark"); canonTests[1][1] = str("a\\u0300ardvark"); canonTests[1][2] = str("\\u00e0ardvark");
82
83 canonTests[2][0] = str("\\u1e0a"); canonTests[2][1] = str("D\\u0307"); canonTests[2][2] = str("\\u1e0a"); // D-dot_above
84
85 canonTests[3][0] = str("D\\u0307"); canonTests[3][1] = str("D\\u0307"); canonTests[3][2] = str("\\u1e0a"); // D dot_above
86
87 canonTests[4][0] = str("\\u1e0c\\u0307"); canonTests[4][1] = str("D\\u0323\\u0307"); canonTests[4][2] = str("\\u1e0c\\u0307"); // D-dot_below dot_above
88
89 canonTests[5][0] = str("\\u1e0a\\u0323"); canonTests[5][1] = str("D\\u0323\\u0307"); canonTests[5][2] = str("\\u1e0c\\u0307"); // D-dot_above dot_below
90
91 canonTests[6][0] = str("D\\u0307\\u0323"); canonTests[6][1] = str("D\\u0323\\u0307"); canonTests[6][2] = str("\\u1e0c\\u0307"); // D dot_below dot_above
92
93 canonTests[7][0] = str("\\u1e10\\u0307\\u0323"); canonTests[7][1] = str("D\\u0327\\u0323\\u0307"); canonTests[7][2] = str("\\u1e10\\u0323\\u0307"); // D dot_below cedilla dot_above
94
95 canonTests[8][0] = str("D\\u0307\\u0328\\u0323"); canonTests[8][1] = str("D\\u0328\\u0323\\u0307"); canonTests[8][2] = str("\\u1e0c\\u0328\\u0307"); // D dot_above ogonek dot_below
96
97 canonTests[9][0] = str("\\u1E14"); canonTests[9][1] = str("E\\u0304\\u0300"); canonTests[9][2] = str("\\u1E14"); // E-macron-grave
98
99 canonTests[10][0] = str("\\u0112\\u0300"); canonTests[10][1] = str("E\\u0304\\u0300"); canonTests[10][2] = str("\\u1E14"); // E-macron + grave
100
101 canonTests[11][0] = str("\\u00c8\\u0304"); canonTests[11][1] = str("E\\u0300\\u0304"); canonTests[11][2] = str("\\u00c8\\u0304"); // E-grave + macron
102
103 canonTests[12][0] = str("\\u212b"); canonTests[12][1] = str("A\\u030a"); canonTests[12][2] = str("\\u00c5"); // angstrom_sign
104
105 canonTests[13][0] = str("\\u00c5"); canonTests[13][1] = str("A\\u030a"); canonTests[13][2] = str("\\u00c5"); // A-ring
106
107 canonTests[14][0] = str("\\u00C4ffin"); canonTests[14][1] = str("A\\u0308ffin"); canonTests[14][2] = str("\\u00C4ffin");
108
109 canonTests[15][0] = str("\\u00C4\\uFB03n"); canonTests[15][1] = str("A\\u0308\\uFB03n"); canonTests[15][2] = str("\\u00C4\\uFB03n");
110
111 canonTests[16][0] = str("Henry IV"); canonTests[16][1] = str("Henry IV"); canonTests[16][2] = str("Henry IV");
112
113 canonTests[17][0] = str("Henry \\u2163"); canonTests[17][1] = str("Henry \\u2163"); canonTests[17][2] = str("Henry \\u2163");
114
115 canonTests[18][0] = str("\\u30AC"); canonTests[18][1] = str("\\u30AB\\u3099"); canonTests[18][2] = str("\\u30AC"); // ga (Katakana)
116
117 canonTests[19][0] = str("\\u30AB\\u3099"); canonTests[19][1] = str("\\u30AB\\u3099"); canonTests[19][2] = str("\\u30AC"); // ka + ten
118
119 canonTests[20][0] = str("\\uFF76\\uFF9E"); canonTests[20][1] = str("\\uFF76\\uFF9E"); canonTests[20][2] = str("\\uFF76\\uFF9E"); // hw_ka + hw_ten
120
121 canonTests[21][0] = str("\\u30AB\\uFF9E"); canonTests[21][1] = str("\\u30AB\\uFF9E"); canonTests[21][2] = str("\\u30AB\\uFF9E"); // ka + hw_ten
122
123 canonTests[22][0] = str("\\uFF76\\u3099"); canonTests[22][1] = str("\\uFF76\\u3099"); canonTests[22][2] = str("\\uFF76\\u3099"); // hw_ka + ten
124
125 canonTests[23][0] = str("A\\u0300\\u0316"); canonTests[23][1] = str("A\\u0316\\u0300"); canonTests[23][2] = str("\\u00C0\\u0316");
126
127 /* compatTest */
128 // Input Decomposed Composed
129 compatTests[0][0] = str("cat"); compatTests[0][1] = str("cat"); compatTests[0][2] = str("cat") ;
130
131 compatTests[1][0] = str("\\uFB4f"); compatTests[1][1] = str("\\u05D0\\u05DC"); compatTests[1][2] = str("\\u05D0\\u05DC"); // Alef-Lamed vs. Alef, Lamed
132
133 compatTests[2][0] = str("\\u00C4ffin"); compatTests[2][1] = str("A\\u0308ffin"); compatTests[2][2] = str("\\u00C4ffin") ;
134
135 compatTests[3][0] = str("\\u00C4\\uFB03n"); compatTests[3][1] = str("A\\u0308ffin"); compatTests[3][2] = str("\\u00C4ffin") ; // ffi ligature -> f + f + i
136
137 compatTests[4][0] = str("Henry IV"); compatTests[4][1] = str("Henry IV"); compatTests[4][2] = str("Henry IV") ;
138
139 compatTests[5][0] = str("Henry \\u2163"); compatTests[5][1] = str("Henry IV"); compatTests[5][2] = str("Henry IV") ;
140
141 compatTests[6][0] = str("\\u30AC"); compatTests[6][1] = str("\\u30AB\\u3099"); compatTests[6][2] = str("\\u30AC") ; // ga (Katakana)
142
143 compatTests[7][0] = str("\\u30AB\\u3099"); compatTests[7][1] = str("\\u30AB\\u3099"); compatTests[7][2] = str("\\u30AC") ; // ka + ten
144
145 compatTests[8][0] = str("\\uFF76\\u3099"); compatTests[8][1] = str("\\u30AB\\u3099"); compatTests[8][2] = str("\\u30AC") ; // hw_ka + ten
146
147 /* These two are broken in Unicode 2.1.2 but fixed in 2.1.5 and later */
148 compatTests[9][0] = str("\\uFF76\\uFF9E"); compatTests[9][1] = str("\\u30AB\\u3099"); compatTests[9][2] = str("\\u30AC") ; // hw_ka + hw_ten
149
150 compatTests[10][0] = str("\\u30AB\\uFF9E"); compatTests[10][1] = str("\\u30AB\\u3099"); compatTests[10][2] = str("\\u30AC") ; // ka + hw_ten
151
152 /* Hangul Canonical */
153 // Input Decomposed Composed
154 hangulCanon[0][0] = str("\\ud4db"); hangulCanon[0][1] = str("\\u1111\\u1171\\u11b6"); hangulCanon[0][2] = str("\\ud4db") ;
155
156 hangulCanon[1][0] = str("\\u1111\\u1171\\u11b6"), hangulCanon[1][1] = str("\\u1111\\u1171\\u11b6"), hangulCanon[1][2] = str("\\ud4db");
157}
158
159BasicNormalizerTest::~BasicNormalizerTest()
160{
161}
162
163void BasicNormalizerTest::TestPrevious()
164{
165 Normalizer* norm = new Normalizer("", UNORM_NFD);
166
167 logln("testing decomp...");
168 uint32_t i;
169 for (i = 0; i < ARRAY_LENGTH(canonTests); i++) {
170 backAndForth(norm, canonTests[i][0]);
171 }
172
173 logln("testing compose...");
174 norm->setMode(UNORM_NFC);
175 for (i = 0; i < ARRAY_LENGTH(canonTests); i++) {
176 backAndForth(norm, canonTests[i][0]);
177 }
178
179 delete norm;
180}
181
182void BasicNormalizerTest::TestDecomp()
183{
184 Normalizer* norm = new Normalizer("", UNORM_NFD);
185 iterateTest(norm, canonTests, ARRAY_LENGTH(canonTests), 1);
186 staticTest(UNORM_NFD, 0, canonTests, ARRAY_LENGTH(canonTests), 1);
187 delete norm;
188}
189
190void BasicNormalizerTest::TestCompatDecomp()
191{
192 Normalizer* norm = new Normalizer("", UNORM_NFKD);
193 iterateTest(norm, compatTests, ARRAY_LENGTH(compatTests), 1);
194
195 staticTest(UNORM_NFKD, 0,
196 compatTests, ARRAY_LENGTH(compatTests), 1);
197 delete norm;
198}
199
200void BasicNormalizerTest::TestCanonCompose()
201{
202 Normalizer* norm = new Normalizer("", UNORM_NFC);
203 iterateTest(norm, canonTests, ARRAY_LENGTH(canonTests), 2);
204
205 staticTest(UNORM_NFC, 0, canonTests,
206 ARRAY_LENGTH(canonTests), 2);
207 delete norm;
208}
209
210void BasicNormalizerTest::TestCompatCompose()
211{
212 Normalizer* norm = new Normalizer("", UNORM_NFKC);
213 iterateTest(norm, compatTests, ARRAY_LENGTH(compatTests), 2);
214
215 staticTest(UNORM_NFKC, 0,
216 compatTests, ARRAY_LENGTH(compatTests), 2);
217 delete norm;
218}
219
220
221//-------------------------------------------------------------------------------
222
223void BasicNormalizerTest::TestHangulCompose()
224{
225 // Make sure that the static composition methods work
226 logln("Canonical composition...");
227 staticTest(UNORM_NFC, 0, hangulCanon, ARRAY_LENGTH(hangulCanon), 2);
228 logln("Compatibility composition...");
229
230 // Now try iterative composition....
231 logln("Static composition...");
232 Normalizer* norm = new Normalizer("", UNORM_NFC);
233 iterateTest(norm, hangulCanon, ARRAY_LENGTH(hangulCanon), 2);
234 norm->setMode(UNORM_NFKC);
235
236 // And finally, make sure you can do it in reverse too
237 logln("Reverse iteration...");
238 norm->setMode(UNORM_NFC);
239 for (uint32_t i = 0; i < ARRAY_LENGTH(hangulCanon); i++) {
240 backAndForth(norm, hangulCanon[i][0]);
241 }
242 delete norm;
243}
244
245void BasicNormalizerTest::TestHangulDecomp()
246{
247 // Make sure that the static decomposition methods work
248 logln("Canonical decomposition...");
249 staticTest(UNORM_NFD, 0, hangulCanon, ARRAY_LENGTH(hangulCanon), 1);
250 logln("Compatibility decomposition...");
251
252 // Now the iterative decomposition methods...
253 logln("Iterative decomposition...");
254 Normalizer* norm = new Normalizer("", UNORM_NFD);
255 iterateTest(norm, hangulCanon, ARRAY_LENGTH(hangulCanon), 1);
256 norm->setMode(UNORM_NFKD);
257
258 // And finally, make sure you can do it in reverse too
259 logln("Reverse iteration...");
260 norm->setMode(UNORM_NFD);
261 for (uint32_t i = 0; i < ARRAY_LENGTH(hangulCanon); i++) {
262 backAndForth(norm, hangulCanon[i][0]);
263 }
264 delete norm;
265}
266
267/**
268 * The Tibetan vowel sign AA, 0f71, was messed up prior to Unicode version 2.1.9.
269 */
270void BasicNormalizerTest::TestTibetan(void) {
271 UnicodeString decomp[1][3];
272 decomp[0][0] = str("\\u0f77");
273 decomp[0][1] = str("\\u0f77");
274 decomp[0][2] = str("\\u0fb2\\u0f71\\u0f80");
275
276 UnicodeString compose[1][3];
277 compose[0][0] = str("\\u0fb2\\u0f71\\u0f80");
278 compose[0][1] = str("\\u0fb2\\u0f71\\u0f80");
279 compose[0][2] = str("\\u0fb2\\u0f71\\u0f80");
280
281 staticTest(UNORM_NFD, 0, decomp, ARRAY_LENGTH(decomp), 1);
282 staticTest(UNORM_NFKD, 0, decomp, ARRAY_LENGTH(decomp), 2);
283 staticTest(UNORM_NFC, 0, compose, ARRAY_LENGTH(compose), 1);
284 staticTest(UNORM_NFKC, 0, compose, ARRAY_LENGTH(compose), 2);
285}
286
287/**
288 * Make sure characters in the CompositionExclusion.txt list do not get
289 * composed to.
290 */
291void BasicNormalizerTest::TestCompositionExclusion(void) {
292 // This list is generated from CompositionExclusion.txt.
293 // Update whenever the normalizer tables are updated. Note
294 // that we test all characters listed, even those that can be
295 // derived from the Unicode DB and are therefore commented
296 // out.
297 // ### TODO read composition exclusion from source/data/unidata file
298 // and test against that
299 UnicodeString EXCLUDED = str(
300 "\\u0340\\u0341\\u0343\\u0344\\u0374\\u037E\\u0387\\u0958"
301 "\\u0959\\u095A\\u095B\\u095C\\u095D\\u095E\\u095F\\u09DC"
302 "\\u09DD\\u09DF\\u0A33\\u0A36\\u0A59\\u0A5A\\u0A5B\\u0A5E"
303 "\\u0B5C\\u0B5D\\u0F43\\u0F4D\\u0F52\\u0F57\\u0F5C\\u0F69"
304 "\\u0F73\\u0F75\\u0F76\\u0F78\\u0F81\\u0F93\\u0F9D\\u0FA2"
305 "\\u0FA7\\u0FAC\\u0FB9\\u1F71\\u1F73\\u1F75\\u1F77\\u1F79"
306 "\\u1F7B\\u1F7D\\u1FBB\\u1FBE\\u1FC9\\u1FCB\\u1FD3\\u1FDB"
307 "\\u1FE3\\u1FEB\\u1FEE\\u1FEF\\u1FF9\\u1FFB\\u1FFD\\u2000"
308 "\\u2001\\u2126\\u212A\\u212B\\u2329\\u232A\\uF900\\uFA10"
309 "\\uFA12\\uFA15\\uFA20\\uFA22\\uFA25\\uFA26\\uFA2A\\uFB1F"
310 "\\uFB2A\\uFB2B\\uFB2C\\uFB2D\\uFB2E\\uFB2F\\uFB30\\uFB31"
311 "\\uFB32\\uFB33\\uFB34\\uFB35\\uFB36\\uFB38\\uFB39\\uFB3A"
312 "\\uFB3B\\uFB3C\\uFB3E\\uFB40\\uFB41\\uFB43\\uFB44\\uFB46"
313 "\\uFB47\\uFB48\\uFB49\\uFB4A\\uFB4B\\uFB4C\\uFB4D\\uFB4E"
314 );
315 for (int32_t i=0; i<EXCLUDED.length(); ++i) {
316 UnicodeString a(EXCLUDED.charAt(i));
317 UnicodeString b;
318 UnicodeString c;
319 Normalizer::normalize(a, UNORM_NFKD, 0, b, status);
320 Normalizer::normalize(b, UNORM_NFC, 0, c, status);
321 if (c == a) {
322 errln("FAIL: " + hex(a) + " x DECOMP_COMPAT => " +
323 hex(b) + " x COMPOSE => " +
324 hex(c));
325 } else if (verbose) {
326 logln("Ok: " + hex(a) + " x DECOMP_COMPAT => " +
327 hex(b) + " x COMPOSE => " +
328 hex(c));
329 }
330 }
331}
332
333/**
334 * Test for a problem that showed up just before ICU 1.6 release
335 * having to do with combining characters with an index of zero.
336 * Such characters do not participate in any canonical
337 * decompositions. However, having an index of zero means that
338 * they all share one typeMask[] entry, that is, they all have to
339 * map to the same canonical class, which is not the case, in
340 * reality.
341 */
342void BasicNormalizerTest::TestZeroIndex(void) {
343 const char* DATA[] = {
344 // Expect col1 x COMPOSE_COMPAT => col2
345 // Expect col2 x DECOMP => col3
346 "A\\u0316\\u0300", "\\u00C0\\u0316", "A\\u0316\\u0300",
347 "A\\u0300\\u0316", "\\u00C0\\u0316", "A\\u0316\\u0300",
348 "A\\u0327\\u0300", "\\u00C0\\u0327", "A\\u0327\\u0300",
349 "c\\u0321\\u0327", "c\\u0321\\u0327", "c\\u0321\\u0327",
350 "c\\u0327\\u0321", "\\u00E7\\u0321", "c\\u0327\\u0321",
351 };
352 int32_t DATA_length = (int32_t)(sizeof(DATA) / sizeof(DATA[0]));
353
354 for (int32_t i=0; i<DATA_length; i+=3) {
355 UErrorCode status = U_ZERO_ERROR;
356 UnicodeString a(DATA[i], "");
357 a = a.unescape();
358 UnicodeString b;
359 Normalizer::normalize(a, UNORM_NFKC, 0, b, status);
729e4ab9
A		 360 if (U_FAILURE(status)) {
361 dataerrln("Error calling normalize UNORM_NFKC: %s", u_errorName(status));
b75a7d8f 362 } else {
729e4ab9
A		 363 UnicodeString exp(DATA[i+1], "");
364 exp = exp.unescape();
365 if (b == exp) {
366 logln((UnicodeString)"Ok: " + hex(a) + " x COMPOSE_COMPAT => " + hex(b));
367 } else {
368 errln((UnicodeString)"FAIL: " + hex(a) + " x COMPOSE_COMPAT => " + hex(b) +
369 ", expect " + hex(exp));
370 }
b75a7d8f
A		 371 }
372 Normalizer::normalize(b, UNORM_NFD, 0, a, status);
729e4ab9
A		 373 if (U_FAILURE(status)) {
374 dataerrln("Error calling normalize UNORM_NFD: %s", u_errorName(status));
b75a7d8f 375 } else {
729e4ab9
A		 376 UnicodeString exp = UnicodeString(DATA[i+2], "").unescape();
377 if (a == exp) {
378 logln((UnicodeString)"Ok: " + hex(b) + " x DECOMP => " + hex(a));
379 } else {
380 errln((UnicodeString)"FAIL: " + hex(b) + " x DECOMP => " + hex(a) +
381 ", expect " + hex(exp));
382 }
b75a7d8f
A		 383 }
384 }
385}
386
387/**
388 * Run a few specific cases that are failing for Verisign.
389 */
390void BasicNormalizerTest::TestVerisign(void) {
391 /*
392 > Their input:
393 > 05B8 05B9 05B1 0591 05C3 05B0 05AC 059F
394 > Their output (supposedly from ICU):
395 > 05B8 05B1 05B9 0591 05C3 05B0 05AC 059F
396 > My output from charlint:
397 > 05B1 05B8 05B9 0591 05C3 05B0 05AC 059F
398
399 05B8 05B9 05B1 0591 05C3 05B0 05AC 059F => 05B1 05B8 05B9 0591 05C3 05B0
400 05AC 059F
401
402 U+05B8 18 E HEBREW POINT QAMATS
403 U+05B9 19 F HEBREW POINT HOLAM
404 U+05B1 11 HEBREW POINT HATAF SEGOL
405 U+0591 220 HEBREW ACCENT ETNAHTA
406 U+05C3 0 HEBREW PUNCTUATION SOF PASUQ
407 U+05B0 10 HEBREW POINT SHEVA
408 U+05AC 230 HEBREW ACCENT ILUY
409 U+059F 230 HEBREW ACCENT QARNEY PARA
410
411 U+05B1 11 HEBREW POINT HATAF SEGOL
412 U+05B8 18 HEBREW POINT QAMATS
413 U+05B9 19 HEBREW POINT HOLAM
414 U+0591 220 HEBREW ACCENT ETNAHTA
415 U+05C3 0 HEBREW PUNCTUATION SOF PASUQ
416 U+05B0 10 HEBREW POINT SHEVA
417 U+05AC 230 HEBREW ACCENT ILUY
418 U+059F 230 HEBREW ACCENT QARNEY PARA
419
420 Wrong result:
421 U+05B8 18 HEBREW POINT QAMATS
422 U+05B1 11 HEBREW POINT HATAF SEGOL
423 U+05B9 19 HEBREW POINT HOLAM
424 U+0591 220 HEBREW ACCENT ETNAHTA
425 U+05C3 0 HEBREW PUNCTUATION SOF PASUQ
426 U+05B0 10 HEBREW POINT SHEVA
427 U+05AC 230 HEBREW ACCENT ILUY
428 U+059F 230 HEBREW ACCENT QARNEY PARA
429
430
431 > Their input:
432 >0592 05B7 05BC 05A5 05B0 05C0 05C4 05AD
433 >Their output (supposedly from ICU):
434 >0592 05B0 05B7 05BC 05A5 05C0 05AD 05C4
435 >My output from charlint:
436 >05B0 05B7 05BC 05A5 0592 05C0 05AD 05C4
437
438 0592 05B7 05BC 05A5 05B0 05C0 05C4 05AD => 05B0 05B7 05BC 05A5 0592 05C0
439 05AD 05C4
440
441 U+0592 230 HEBREW ACCENT SEGOL
442 U+05B7 17 HEBREW POINT PATAH
443 U+05BC 21 HEBREW POINT DAGESH OR MAPIQ
444 U+05A5 220 HEBREW ACCENT MERKHA
445 U+05B0 10 HEBREW POINT SHEVA
446 U+05C0 0 HEBREW PUNCTUATION PASEQ
447 U+05C4 230 HEBREW MARK UPPER DOT
448 U+05AD 222 HEBREW ACCENT DEHI
449
450 U+05B0 10 HEBREW POINT SHEVA
451 U+05B7 17 HEBREW POINT PATAH
452 U+05BC 21 HEBREW POINT DAGESH OR MAPIQ
453 U+05A5 220 HEBREW ACCENT MERKHA
454 U+0592 230 HEBREW ACCENT SEGOL
455 U+05C0 0 HEBREW PUNCTUATION PASEQ
456 U+05AD 222 HEBREW ACCENT DEHI
457 U+05C4 230 HEBREW MARK UPPER DOT
458
459 Wrong result:
460 U+0592 230 HEBREW ACCENT SEGOL
461 U+05B0 10 HEBREW POINT SHEVA
462 U+05B7 17 HEBREW POINT PATAH
463 U+05BC 21 HEBREW POINT DAGESH OR MAPIQ
464 U+05A5 220 HEBREW ACCENT MERKHA
465 U+05C0 0 HEBREW PUNCTUATION PASEQ
466 U+05AD 222 HEBREW ACCENT DEHI
467 U+05C4 230 HEBREW MARK UPPER DOT
468 */
469 UnicodeString data[2][3];
470 data[0][0] = str("\\u05B8\\u05B9\\u05B1\\u0591\\u05C3\\u05B0\\u05AC\\u059F");
471 data[0][1] = str("\\u05B1\\u05B8\\u05B9\\u0591\\u05C3\\u05B0\\u05AC\\u059F");
472 data[0][2] = str("");
473 data[1][0] = str("\\u0592\\u05B7\\u05BC\\u05A5\\u05B0\\u05C0\\u05C4\\u05AD");
474 data[1][1] = str("\\u05B0\\u05B7\\u05BC\\u05A5\\u0592\\u05C0\\u05AD\\u05C4");
475 data[1][2] = str("");
476
477 staticTest(UNORM_NFD, 0, data, ARRAY_LENGTH(data), 1);
478 staticTest(UNORM_NFC, 0, data, ARRAY_LENGTH(data), 1);
479}
480
481//------------------------------------------------------------------------
482// Internal utilities
483//
484
485UnicodeString BasicNormalizerTest::hex(UChar ch) {
486 UnicodeString result;
487 return appendHex(ch, 4, result);
488}
489
490UnicodeString BasicNormalizerTest::hex(const UnicodeString& s) {
491 UnicodeString result;
492 for (int i = 0; i < s.length(); ++i) {
493 if (i != 0) result += (UChar)0x2c/*,*/;
494 appendHex(s[i], 4, result);
495 }
496 return result;
497}
498
499
500inline static void insert(UnicodeString& dest, int pos, UChar32 ch)
501{
502 dest.replace(pos, 0, ch);
503}
504
505void BasicNormalizerTest::backAndForth(Normalizer* iter, const UnicodeString& input)
506{
507 UChar32 ch;
508 iter->setText(input, status);
509
510 // Run through the iterator forwards and stick it into a StringBuffer
511 UnicodeString forward;
512 for (ch = iter->first(); ch != iter->DONE; ch = iter->next()) {
513 forward += ch;
514 }
515
516 // Now do it backwards
517 UnicodeString reverse;
518 for (ch = iter->last(); ch != iter->DONE; ch = iter->previous()) {
519 insert(reverse, 0, ch);
520 }
521
522 if (forward != reverse) {
523 errln("Forward/reverse mismatch for input " + hex(input)
524 + ", forward: " + hex(forward) + ", backward: " + hex(reverse));
525 }
526}
527
528void BasicNormalizerTest::staticTest(UNormalizationMode mode, int options,
529 UnicodeString tests[][3], int length,
530 int outCol)
531{
532 for (int i = 0; i < length; i++)
533 {
534 UnicodeString& input = tests[i][0];
535 UnicodeString& expect = tests[i][outCol];
536
537 logln("Normalizing '" + input + "' (" + hex(input) + ")" );
538
539 UnicodeString output;
540 Normalizer::normalize(input, mode, options, output, status);
541
542 if (output != expect) {
729e4ab9 543 dataerrln(UnicodeString("ERROR: case ") + i + " normalized " + hex(input) + "\n"
b75a7d8f
A		 544 + " expected " + hex(expect) + "\n"
545 + " static got " + hex(output) );
546 }
547 }
548}
549
550void BasicNormalizerTest::iterateTest(Normalizer* iter,
551 UnicodeString tests[][3], int length,
552 int outCol)
553{
554 for (int i = 0; i < length; i++)
555 {
556 UnicodeString& input = tests[i][0];
557 UnicodeString& expect = tests[i][outCol];
558
559 logln("Normalizing '" + input + "' (" + hex(input) + ")" );
560
561 iter->setText(input, status);
562 assertEqual(input, expect, iter, UnicodeString("ERROR: case ") + i + " ");
563 }
564}
565
566void BasicNormalizerTest::assertEqual(const UnicodeString& input,
567 const UnicodeString& expected,
568 Normalizer* iter,
569 const UnicodeString& errPrefix)
570{
571 UnicodeString result;
572
573 for (UChar32 ch = iter->first(); ch != iter->DONE; ch = iter->next()) {
574 result += ch;
575 }
576 if (result != expected) {
729e4ab9 577 dataerrln(errPrefix + "normalized " + hex(input) + "\n"
b75a7d8f
A		 578 + " expected " + hex(expected) + "\n"
579 + " iterate got " + hex(result) );
580 }
581}
582
583// helper class for TestPreviousNext()
584// simple UTF-32 character iterator
585class UChar32Iterator {
586public:
587 UChar32Iterator(const UChar32 *text, int32_t len, int32_t index) :
588 s(text), length(len), i(index) {}
589
590 UChar32 current() {
591 if(i<length) {
592 return s[i];
593 } else {
594 return 0xffff;
595 }
596 }
597
598 UChar32 next() {
599 if(i<length) {
600 return s[i++];
601 } else {
602 return 0xffff;
603 }
604 }
605
606 UChar32 previous() {
607 if(i>0) {
608 return s[--i];
609 } else {
610 return 0xffff;
611 }
612 }
613
614 int32_t getIndex() {
615 return i;
616 }
617private:
618 const UChar32 *s;
619 int32_t length, i;
620};
621
622void
623BasicNormalizerTest::TestPreviousNext(const UChar *src, int32_t srcLength,
624 const UChar32 *expect, int32_t expectLength,
625 const int32_t *expectIndex, // its length=expectLength+1
626 int32_t srcMiddle, int32_t expectMiddle,
627 const char *moves,
628 UNormalizationMode mode,
629 const char *name) {
630 // iterators
631 Normalizer iter(src, srcLength, mode);
632
633 // test getStaticClassID and getDynamicClassID
634 if(iter.getDynamicClassID() != Normalizer::getStaticClassID()) {
635 errln("getStaticClassID != getDynamicClassID for Normalizer.");
636 }
637
638 UChar32Iterator iter32(expect, expectLength, expectMiddle);
639
640 UChar32 c1, c2;
641 char m;
642
643 // initially set the indexes into the middle of the strings
644 iter.setIndexOnly(srcMiddle);
645
646 // move around and compare the iteration code points with
647 // the expected ones
648 const char *move=moves;
649 while((m=*move++)!=0) {
650 if(m=='-') {
651 c1=iter.previous();
652 c2=iter32.previous();
653 } else if(m=='0') {
654 c1=iter.current();
655 c2=iter32.current();
656 } else /* m=='+' */ {
657 c1=iter.next();
658 c2=iter32.next();
659 }
660
661 // compare results
662 if(c1!=c2) {
663 // copy the moves until the current (m) move, and terminate
664 char history[64];
665 uprv_strcpy(history, moves);
666 history[move-moves]=0;
729e4ab9
A		 667 dataerrln("error: mismatch in Normalizer iteration (%s) at %s: "
668 "got c1=U+%04lx != expected c2=U+%04lx",
b75a7d8f
A		 669 name, history, c1, c2);
670 break;
671 }
672
673 // compare indexes
674 if(iter.getIndex()!=expectIndex[iter32.getIndex()]) {
675 // copy the moves until the current (m) move, and terminate
676 char history[64];
677 uprv_strcpy(history, moves);
678 history[move-moves]=0;
679 errln("error: index mismatch in Normalizer iteration (%s) at %s: "
680 "Normalizer index %ld expected %ld\n",
681 name, history, iter.getIndex(), expectIndex[iter32.getIndex()]);
682 break;
683 }
684 }
685}
686
687void
688BasicNormalizerTest::TestPreviousNext() {
689 // src and expect strings
690 static const UChar src[]={
4388f060
A		 691 U16_LEAD(0x2f999), U16_TRAIL(0x2f999),
692 U16_LEAD(0x1d15f), U16_TRAIL(0x1d15f),
b75a7d8f
A		 693 0xc4,
694 0x1ed0
695 };
696 static const UChar32 expect[]={
697 0x831d,
698 0x1d158, 0x1d165,
699 0x41, 0x308,
700 0x4f, 0x302, 0x301
701 };
702
703 // expected src indexes corresponding to expect indexes
704 static const int32_t expectIndex[]={
705 0,
706 2, 2,
707 4, 4,
708 5, 5, 5,
709 6 // behind last character
710 };
711
712 // src and expect strings for regression test for j2911
713 static const UChar src_j2911[]={
4388f060 714 U16_LEAD(0x2f999), U16_TRAIL(0x2f999),
b75a7d8f
A		 715 0xdd00, 0xd900, // unpaired surrogates - regression test for j2911
716 0xc4,
717 0x4f, 0x302, 0x301
718 };
719 static const UChar32 expect_j2911[]={
720 0x831d,
721 0xdd00, 0xd900, // unpaired surrogates - regression test for j2911
722 0xc4,
723 0x1ed0
724 };
725
726 // expected src indexes corresponding to expect indexes
727 static const int32_t expectIndex_j2911[]={
728 0,
729 2, 3,
730 4,
731 5,
732 8 // behind last character
733 };
734
735 // initial indexes into the src and expect strings
736 // for both sets of test data
737 enum {
738 SRC_MIDDLE=4,
739 EXPECT_MIDDLE=3,
740 SRC_MIDDLE_2=2,
741 EXPECT_MIDDLE_2=1
742 };
743
744 // movement vector
745 // - for previous(), 0 for current(), + for next()
746 // for both sets of test data
747 static const char *const moves="0+0+0--0-0-+++0--+++++++0--------";
748
b331163b
A		 749 TestPreviousNext(src, UPRV_LENGTHOF(src),
750 expect, UPRV_LENGTHOF(expect),
b75a7d8f
A		 751 expectIndex,
752 SRC_MIDDLE, EXPECT_MIDDLE,
753 moves, UNORM_NFD, "basic");
754
b331163b
A		 755 TestPreviousNext(src_j2911, UPRV_LENGTHOF(src_j2911),
756 expect_j2911, UPRV_LENGTHOF(expect_j2911),
b75a7d8f
A		 757 expectIndex_j2911,
758 SRC_MIDDLE, EXPECT_MIDDLE,
759 moves, UNORM_NFKC, "j2911");
760
761 // try again from different "middle" indexes
b331163b
A		 762 TestPreviousNext(src, UPRV_LENGTHOF(src),
763 expect, UPRV_LENGTHOF(expect),
b75a7d8f
A		 764 expectIndex,
765 SRC_MIDDLE_2, EXPECT_MIDDLE_2,
766 moves, UNORM_NFD, "basic_2");
767
b331163b
A		 768 TestPreviousNext(src_j2911, UPRV_LENGTHOF(src_j2911),
769 expect_j2911, UPRV_LENGTHOF(expect_j2911),
b75a7d8f
A		 770 expectIndex_j2911,
771 SRC_MIDDLE_2, EXPECT_MIDDLE_2,
772 moves, UNORM_NFKC, "j2911_2");
773}
774
775void BasicNormalizerTest::TestConcatenate() {
776 static const char *const
777 cases[][4]={
778 /* mode, left, right, result */
779 {
780 "C",
781 "re",
782 "\\u0301sum\\u00e9",
783 "r\\u00e9sum\\u00e9"
784 },
785 {
786 "C",
787 "a\\u1100",
788 "\\u1161bcdefghijk",
789 "a\\uac00bcdefghijk"
790 },
791 /* ### TODO: add more interesting cases */
792 {
729e4ab9
A		 793 "D",
794 "\\u03B1\\u0345",
795 "\\u0C4D\\U000110BA\\U0001D169",
796 "\\u03B1\\U0001D169\\U000110BA\\u0C4D\\u0345"
b75a7d8f
A		 797 }
798 };
799
800 UnicodeString left, right, expect, result, r;
801 UErrorCode errorCode;
802 UNormalizationMode mode;
803 int32_t i;
804
805 /* test concatenation */
806 for(i=0; i<(int32_t)(sizeof(cases)/sizeof(cases[0])); ++i) {
807 switch(*cases[i][0]) {
808 case 'C': mode=UNORM_NFC; break;
809 case 'D': mode=UNORM_NFD; break;
810 case 'c': mode=UNORM_NFKC; break;
811 case 'd': mode=UNORM_NFKD; break;
812 default: mode=UNORM_NONE; break;
813 }
814
815 left=UnicodeString(cases[i][1], "").unescape();
816 right=UnicodeString(cases[i][2], "").unescape();
817 expect=UnicodeString(cases[i][3], "").unescape();
818
819 //result=r=UnicodeString();
820 errorCode=U_ZERO_ERROR;
821
822 r=Normalizer::concatenate(left, right, result, mode, 0, errorCode);
823 if(U_FAILURE(errorCode) || /*result!=r ||*/ result!=expect) {
729e4ab9 824 dataerrln("error in Normalizer::concatenate(), cases[] fails with "+
b75a7d8f
A		 825 UnicodeString(u_errorName(errorCode))+", result==expect: expected: "+
826 hex(expect)+" =========> got: " + hex(result));
827 }
828 }
829
830 /* test error cases */
831
832 /* left.getBuffer()==result.getBuffer() */
833 result=r=expect=UnicodeString("zz", "");
834 errorCode=U_UNEXPECTED_TOKEN;
835 r=Normalizer::concatenate(left, right, result, mode, 0, errorCode);
836 if(errorCode!=U_UNEXPECTED_TOKEN || result!=r || !result.isBogus()) {
837 errln("error in Normalizer::concatenate(), violates UErrorCode protocol");
838 }
839
840 left.setToBogus();
841 errorCode=U_ZERO_ERROR;
842 r=Normalizer::concatenate(left, right, result, mode, 0, errorCode);
843 if(errorCode!=U_ILLEGAL_ARGUMENT_ERROR || result!=r || !result.isBogus()) {
844 errln("error in Normalizer::concatenate(), does not detect left.isBogus()");
845 }
846}
847
848// reference implementation of Normalizer::compare
849static int32_t
850ref_norm_compare(const UnicodeString &s1, const UnicodeString &s2, uint32_t options, UErrorCode &errorCode) {
851 UnicodeString r1, r2, t1, t2;
852 int32_t normOptions=(int32_t)(options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT);
853
854 if(options&U_COMPARE_IGNORE_CASE) {
855 Normalizer::decompose(s1, FALSE, normOptions, r1, errorCode);
856 Normalizer::decompose(s2, FALSE, normOptions, r2, errorCode);
857
858 r1.foldCase(options);
859 r2.foldCase(options);
860 } else {
861 r1=s1;
862 r2=s2;
863 }
864
865 Normalizer::decompose(r1, FALSE, normOptions, t1, errorCode);
866 Normalizer::decompose(r2, FALSE, normOptions, t2, errorCode);
867
868 if(options&U_COMPARE_CODE_POINT_ORDER) {
869 return t1.compareCodePointOrder(t2);
870 } else {
871 return t1.compare(t2);
872 }
873}
874
875// test wrapper for Normalizer::compare, sets UNORM_INPUT_IS_FCD appropriately
876static int32_t
877_norm_compare(const UnicodeString &s1, const UnicodeString &s2, uint32_t options, UErrorCode &errorCode) {
878 int32_t normOptions=(int32_t)(options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT);
879
880 if( UNORM_YES==Normalizer::quickCheck(s1, UNORM_FCD, normOptions, errorCode) &&
881 UNORM_YES==Normalizer::quickCheck(s2, UNORM_FCD, normOptions, errorCode)) {
882 options|=UNORM_INPUT_IS_FCD;
883 }
884
885 return Normalizer::compare(s1, s2, options, errorCode);
886}
887
888// reference implementation of UnicodeString::caseCompare
889static int32_t
890ref_case_compare(const UnicodeString &s1, const UnicodeString &s2, uint32_t options) {
891 UnicodeString t1, t2;
892
893 t1=s1;
894 t2=s2;
895
896 t1.foldCase(options);
897 t2.foldCase(options);
898
899 if(options&U_COMPARE_CODE_POINT_ORDER) {
900 return t1.compareCodePointOrder(t2);
901 } else {
902 return t1.compare(t2);
903 }
904}
905
906// reduce an integer to -1/0/1
907static inline int32_t
908_sign(int32_t value) {
909 if(value==0) {
910 return 0;
911 } else {
912 return (value>>31)|1;
913 }
914}
915
916static const char *
917_signString(int32_t value) {
918 if(value<0) {
919 return "<0";
920 } else if(value==0) {
921 return "=0";
922 } else /* value>0 */ {
923 return ">0";
924 }
925}
926
927void
928BasicNormalizerTest::TestCompare() {
929 // test Normalizer::compare and unorm_compare (thinly wrapped by the former)
930 // by comparing it with its semantic equivalent
931 // since we trust the pieces, this is sufficient
932
933 // test each string with itself and each other
934 // each time with all options
935 static const char *const
936 strings[]={
937 // some cases from NormalizationTest.txt
938 // 0..3
939 "D\\u031B\\u0307\\u0323",
940 "\\u1E0C\\u031B\\u0307",
941 "D\\u031B\\u0323\\u0307",
942 "d\\u031B\\u0323\\u0307",
943
944 // 4..6
945 "\\u00E4",
946 "a\\u0308",
947 "A\\u0308",
948
949 // Angstrom sign = A ring
950 // 7..10
951 "\\u212B",
952 "\\u00C5",
953 "A\\u030A",
954 "a\\u030A",
955
956 // 11.14
957 "a\\u059A\\u0316\\u302A\\u032Fb",
958 "a\\u302A\\u0316\\u032F\\u059Ab",
959 "a\\u302A\\u0316\\u032F\\u059Ab",
960 "A\\u059A\\u0316\\u302A\\u032Fb",
961
962 // from ICU case folding tests
963 // 15..20
964 "A\\u00df\\u00b5\\ufb03\\U0001040c\\u0131",
965 "ass\\u03bcffi\\U00010434i",
966 "\\u0061\\u0042\\u0131\\u03a3\\u00df\\ufb03\\ud93f\\udfff",
967 "\\u0041\\u0062\\u0069\\u03c3\\u0073\\u0053\\u0046\\u0066\\u0049\\ud93f\\udfff",
968 "\\u0041\\u0062\\u0131\\u03c3\\u0053\\u0073\\u0066\\u0046\\u0069\\ud93f\\udfff",
969 "\\u0041\\u0062\\u0069\\u03c3\\u0073\\u0053\\u0046\\u0066\\u0049\\ud93f\\udffd",
970
971 // U+d800 U+10001 see implementation comment in unorm_cmpEquivFold
972 // vs. U+10000 at bottom - code point order
973 // 21..22
974 "\\ud800\\ud800\\udc01",
975 "\\ud800\\udc00",
976
977 // other code point order tests from ustrtest.cpp
978 // 23..31
979 "\\u20ac\\ud801",
980 "\\u20ac\\ud800\\udc00",
981 "\\ud800",
982 "\\ud800\\uff61",
983 "\\udfff",
984 "\\uff61\\udfff",
985 "\\uff61\\ud800\\udc02",
986 "\\ud800\\udc02",
987 "\\ud84d\\udc56",
988
989 // long strings, see cnormtst.c/TestNormCoverage()
990 // equivalent if case-insensitive
991 // 32..33
992 "\\uAD8B\\uAD8B\\uAD8B\\uAD8B"
993 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
994 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
995 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
996 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
997 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
998 "aaaaaaaaaaaaaaaaaazzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"
999 "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
1000 "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
1001 "ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
1002 "\\uAD8B\\uAD8B\\uAD8B\\uAD8B"
1003 "d\\u031B\\u0307\\u0323",
1004
1005 "\\u1100\\u116f\\u11aa\\uAD8B\\uAD8B\\u1100\\u116f\\u11aa"
1006 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
1007 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
1008 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
1009 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
1010 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e"
1011 "aaaaaaaaaaAAAAAAAAZZZZZZZZZZZZZZZZzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"
1012 "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
1013 "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
1014 "ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
1015 "\\u1100\\u116f\\u11aa\\uAD8B\\uAD8B\\u1100\\u116f\\u11aa"
1016 "\\u1E0C\\u031B\\u0307",
1017
1018 // some strings that may make a difference whether the compare function
1019 // case-folds or decomposes first
1020 // 34..41
1021 "\\u0360\\u0345\\u0334",
1022 "\\u0360\\u03b9\\u0334",
1023
1024 "\\u0360\\u1f80\\u0334",
1025 "\\u0360\\u03b1\\u0313\\u03b9\\u0334",
1026
1027 "\\u0360\\u1ffc\\u0334",
1028 "\\u0360\\u03c9\\u03b9\\u0334",
1029
1030 "a\\u0360\\u0345\\u0360\\u0345b",
1031 "a\\u0345\\u0360\\u0345\\u0360b",
1032
1033 // interesting cases for canonical caseless match with turkic i handling
1034 // 42..43
1035 "\\u00cc",
1036 "\\u0069\\u0300",
1037
1038 // strings with post-Unicode 3.2 normalization or normalization corrections
1039 // 44..45
1040 "\\u00e4\\u193b\\U0002f868",
1041 "\\u0061\\u193b\\u0308\\u36fc",
1042
1043 // empty string
1044 // 46
1045 ""
1046 };
1047
1048 UnicodeString s[100]; // at least as many items as in strings[] !
1049
1050 // all combinations of options
1051 // UNORM_INPUT_IS_FCD is set automatically if both input strings fulfill FCD conditions
1052 // set UNORM_UNICODE_3_2 in one additional combination
1053 static const struct {
1054 uint32_t options;
1055 const char *name;
1056 } opt[]={
1057 { 0, "default" },
1058 { U_COMPARE_CODE_POINT_ORDER, "c.p. order" },
1059 { U_COMPARE_IGNORE_CASE, "ignore case" },
1060 { U_COMPARE_CODE_POINT_ORDER|U_COMPARE_IGNORE_CASE, "c.p. order & ignore case" },
1061 { U_COMPARE_IGNORE_CASE|U_FOLD_CASE_EXCLUDE_SPECIAL_I, "ignore case & special i" },
1062 { U_COMPARE_CODE_POINT_ORDER|U_COMPARE_IGNORE_CASE|U_FOLD_CASE_EXCLUDE_SPECIAL_I, "c.p. order & ignore case & special i" },
1063 { UNORM_UNICODE_3_2<<UNORM_COMPARE_NORM_OPTIONS_SHIFT, "Unicode 3.2" }
1064 };
1065
b331163b 1066 int32_t i, j, k, count=UPRV_LENGTHOF(strings);
b75a7d8f
A		 1067 int32_t result, refResult;
1068
1069 UErrorCode errorCode;
1070
1071 // create the UnicodeStrings
1072 for(i=0; i<count; ++i) {
1073 s[i]=UnicodeString(strings[i], "").unescape();
1074 }
1075
1076 // test them each with each other
1077 for(i=0; i<count; ++i) {
1078 for(j=i; j<count; ++j) {
b331163b 1079 for(k=0; k<UPRV_LENGTHOF(opt); ++k) {
b75a7d8f
A		 1080 // test Normalizer::compare
1081 errorCode=U_ZERO_ERROR;
1082 result=_norm_compare(s[i], s[j], opt[k].options, errorCode);
1083 refResult=ref_norm_compare(s[i], s[j], opt[k].options, errorCode);
1084 if(_sign(result)!=_sign(refResult)) {
1085 errln("Normalizer::compare(%d, %d, %s)%s should be %s %s",
1086 i, j, opt[k].name, _signString(result), _signString(refResult),
1087 U_SUCCESS(errorCode) ? "" : u_errorName(errorCode));
1088 }
1089
1090 // test UnicodeString::caseCompare - same internal implementation function
1091 if(opt[k].options&U_COMPARE_IGNORE_CASE) {
1092 errorCode=U_ZERO_ERROR;
1093 result=s[i].caseCompare(s[j], opt[k].options);
1094 refResult=ref_case_compare(s[i], s[j], opt[k].options);
1095 if(_sign(result)!=_sign(refResult)) {
1096 errln("UniStr::caseCompare(%d, %d, %s)%s should be %s %s",
1097 i, j, opt[k].name, _signString(result), _signString(refResult),
1098 U_SUCCESS(errorCode) ? "" : u_errorName(errorCode));
1099 }
1100 }
1101 }
1102 }
1103 }
1104
1105 // test cases with i and I to make sure Turkic works
1106 static const UChar iI[]={ 0x49, 0x69, 0x130, 0x131 };
729e4ab9 1107 UnicodeSet iSet, set;
b75a7d8f
A		 1108
1109 UnicodeString s1, s2;
729e4ab9
A		 1110
1111 const Normalizer2Impl *nfcImpl=Normalizer2Factory::getNFCImpl(errorCode);
1112 if(U_FAILURE(errorCode) || !nfcImpl->ensureCanonIterData(errorCode)) {
1113 dataerrln("Normalizer2Factory::getNFCImpl().ensureCanonIterData() failed: %s",
1114 u_errorName(errorCode));
1115 return;
1116 }
b75a7d8f
A		 1117
1118 // collect all sets into one for contiguous output
b331163b 1119 for(i=0; i<UPRV_LENGTHOF(iI); ++i) {
729e4ab9
A		 1120 if(nfcImpl->getCanonStartSet(iI[i], iSet)) {
1121 set.addAll(iSet);
b75a7d8f
A		 1122 }
1123 }
1124
1125 // test all of these precomposed characters
b331163b 1126 const Normalizer2 *nfcNorm2=Normalizer2::getNFCInstance(errorCode);
b75a7d8f 1127 UnicodeSetIterator it(set);
729e4ab9
A		 1128 while(it.next() && !it.isString()) {
1129 UChar32 c=it.getCodepoint();
1130 if(!nfcNorm2->getDecomposition(c, s2)) {
1131 dataerrln("NFC.getDecomposition(i-composite U+%04lx) failed", (long)c);
1132 return;
1133 }
1134
1135 s1.setTo(c);
b331163b 1136 for(k=0; k<UPRV_LENGTHOF(opt); ++k) {
729e4ab9 1137 // test Normalizer::compare
b75a7d8f 1138 errorCode=U_ZERO_ERROR;
729e4ab9
A		 1139 result=_norm_compare(s1, s2, opt[k].options, errorCode);
1140 refResult=ref_norm_compare(s1, s2, opt[k].options, errorCode);
1141 if(_sign(result)!=_sign(refResult)) {
1142 errln("Normalizer::compare(U+%04x with its NFD, %s)%s should be %s %s",
1143 c, opt[k].name, _signString(result), _signString(refResult),
1144 U_SUCCESS(errorCode) ? "" : u_errorName(errorCode));
b75a7d8f
A		 1145 }
1146
729e4ab9
A		 1147 // test UnicodeString::caseCompare - same internal implementation function
1148 if(opt[k].options&U_COMPARE_IGNORE_CASE) {
b75a7d8f 1149 errorCode=U_ZERO_ERROR;
729e4ab9
A		 1150 result=s1.caseCompare(s2, opt[k].options);
1151 refResult=ref_case_compare(s1, s2, opt[k].options);
b75a7d8f 1152 if(_sign(result)!=_sign(refResult)) {
729e4ab9
A		 1153 errln("UniStr::caseCompare(U+%04x with its NFD, %s)%s should be %s %s",
1154 c, opt[k].name, _signString(result), _signString(refResult),
b75a7d8f
A		 1155 U_SUCCESS(errorCode) ? "" : u_errorName(errorCode));
1156 }
b75a7d8f 1157 }
b75a7d8f
A		 1158 }
1159 }
729e4ab9
A		 1160
1161 // test getDecomposition() for some characters that do not decompose
1162 if( nfcNorm2->getDecomposition(0x20, s2) ||
1163 nfcNorm2->getDecomposition(0x4e00, s2) ||
1164 nfcNorm2->getDecomposition(0x20002, s2)
1165 ) {
1166 errln("NFC.getDecomposition() returns TRUE for characters which do not have decompositions");
1167 }
1168
4388f060
A		 1169 // test getRawDecomposition() for some characters that do not decompose
1170 if( nfcNorm2->getRawDecomposition(0x20, s2) ||
1171 nfcNorm2->getRawDecomposition(0x4e00, s2) ||
1172 nfcNorm2->getRawDecomposition(0x20002, s2)
1173 ) {
1174 errln("NFC.getRawDecomposition() returns TRUE for characters which do not have decompositions");
1175 }
1176
1177 // test composePair() for some pairs of characters that do not compose
1178 if( nfcNorm2->composePair(0x20, 0x301)>=0 ||
1179 nfcNorm2->composePair(0x61, 0x305)>=0 ||
1180 nfcNorm2->composePair(0x1100, 0x1160)>=0 ||
1181 nfcNorm2->composePair(0xac00, 0x11a7)>=0
1182 ) {
1183 errln("NFC.composePair() incorrectly composes some pairs of characters");
1184 }
1185
729e4ab9
A		 1186 // test FilteredNormalizer2::getDecomposition()
1187 UnicodeSet filter(UNICODE_STRING_SIMPLE("[^\\u00a0-\\u00ff]"), errorCode);
1188 FilteredNormalizer2 fn2(*nfcNorm2, filter);
1189 if( fn2.getDecomposition(0xe4, s1) || !fn2.getDecomposition(0x100, s2) ||
1190 s2.length()!=2 || s2[0]!=0x41 || s2[1]!=0x304
1191 ) {
1192 errln("FilteredNormalizer2(NFC, ^A0-FF).getDecomposition() failed");
1193 }
4388f060
A		 1194
1195 // test FilteredNormalizer2::getRawDecomposition()
1196 if( fn2.getRawDecomposition(0xe4, s1) || !fn2.getRawDecomposition(0x100, s2) ||
1197 s2.length()!=2 || s2[0]!=0x41 || s2[1]!=0x304
1198 ) {
1199 errln("FilteredNormalizer2(NFC, ^A0-FF).getRawDecomposition() failed");
1200 }
1201
1202 // test FilteredNormalizer2::composePair()
1203 if( 0x100!=fn2.composePair(0x41, 0x304) ||
1204 fn2.composePair(0xc7, 0x301)>=0 // unfiltered result: U+1E08
1205 ) {
1206 errln("FilteredNormalizer2(NFC, ^A0-FF).composePair() failed");
1207 }
b75a7d8f
A		 1208}
1209
1210// verify that case-folding does not un-FCD strings
1211int32_t
1212BasicNormalizerTest::countFoldFCDExceptions(uint32_t foldingOptions) {
1213 UnicodeString s, fold, d;
1214 UChar32 c;
1215 int32_t count;
1216 uint8_t cc, trailCC, foldCC, foldTrailCC;
1217 UNormalizationCheckResult qcResult;
1218 int8_t category;
1219 UBool isNFD;
1220 UErrorCode errorCode;
1221
1222 logln("Test if case folding may un-FCD a string (folding options %04lx)", foldingOptions);
1223
1224 count=0;
1225 for(c=0; c<=0x10ffff; ++c) {
1226 errorCode = U_ZERO_ERROR;
1227 category=u_charType(c);
1228 if(category==U_UNASSIGNED) {
1229 continue; // skip unassigned code points
1230 }
1231 if(c==0xac00) {
1232 c=0xd7a3; // skip Hangul - no case folding there
1233 continue;
1234 }
1235 // skip Han blocks - no case folding there either
1236 if(c==0x3400) {
1237 c=0x4db5;
1238 continue;
1239 }
1240 if(c==0x4e00) {
1241 c=0x9fa5;
1242 continue;
1243 }
1244 if(c==0x20000) {
1245 c=0x2a6d6;
1246 continue;
1247 }
1248
1249 s.setTo(c);
1250
1251 // get leading and trailing cc for c
1252 Normalizer::decompose(s, FALSE, 0, d, errorCode);
1253 isNFD= s==d;
1254 cc=u_getCombiningClass(d.char32At(0));
1255 trailCC=u_getCombiningClass(d.char32At(d.length()-1));
1256
1257 // get leading and trailing cc for the case-folding of c
1258 s.foldCase(foldingOptions);
1259 Normalizer::decompose(s, FALSE, 0, d, errorCode);
1260 foldCC=u_getCombiningClass(d.char32At(0));
1261 foldTrailCC=u_getCombiningClass(d.char32At(d.length()-1));
1262
1263 qcResult=Normalizer::quickCheck(s, UNORM_FCD, errorCode);
1264
1265 if (U_FAILURE(errorCode)) {
1266 ++count;
729e4ab9 1267 dataerrln("U+%04lx: Failed with error %s", u_errorName(errorCode));
b75a7d8f
A		 1268 }
1269
1270 // bad:
1271 // - character maps to empty string: adjacent characters may then need reordering
1272 // - folding has different leading/trailing cc's, and they don't become just 0
1273 // - folding itself is not FCD
1274 if( qcResult!=UNORM_YES ||
1275 s.isEmpty() ||
1276 (cc!=foldCC && foldCC!=0) || (trailCC!=foldTrailCC && foldTrailCC!=0)
1277 ) {
1278 ++count;
729e4ab9
A		 1279 dataerrln("U+%04lx: case-folding may un-FCD a string (folding options %04lx)", c, foldingOptions);
1280 dataerrln(" cc %02x trailCC %02x foldCC(U+%04lx) %02x foldTrailCC(U+%04lx) %02x quickCheck(folded)=%d", cc, trailCC, d.char32At(0), foldCC, d.char32At(d.length()-1), foldTrailCC, qcResult);
b75a7d8f
A		 1281 continue;
1282 }
1283
1284 // also bad:
1285 // if a code point is in NFD but its case folding is not, then
1286 // unorm_compare will also fail
1287 if(isNFD && UNORM_YES!=Normalizer::quickCheck(s, UNORM_NFD, errorCode)) {
1288 ++count;
1289 errln("U+%04lx: case-folding un-NFDs this character (folding options %04lx)", c, foldingOptions);
1290 }
1291 }
1292
1293 logln("There are %ld code points for which case-folding may un-FCD a string (folding options %04lx)", count, foldingOptions);
1294 return count;
1295}
1296
1297void
1298BasicNormalizerTest::FindFoldFCDExceptions() {
1299 int32_t count;
1300
1301 count=countFoldFCDExceptions(0);
1302 count+=countFoldFCDExceptions(U_FOLD_CASE_EXCLUDE_SPECIAL_I);
1303 if(count>0) {
1304 /*
1305 * If case-folding un-FCDs any strings, then unorm_compare() must be
1306 * re-implemented.
1307 * It currently assumes that one can check for FCD then case-fold
1308 * and then still have FCD strings for raw decomposition without reordering.
1309 */
729e4ab9 1310 dataerrln("error: There are %ld code points for which case-folding may un-FCD a string for all folding options.\n"
b75a7d8f
A		 1311 "See comment in BasicNormalizerTest::FindFoldFCDExceptions()!", count);
1312 }
1313}
1314
b75a7d8f 1315static void
729e4ab9
A		 1316initExpectedSkippables(UnicodeSet skipSets[UNORM_MODE_COUNT], UErrorCode &errorCode) {
1317 skipSets[UNORM_NFD].applyPattern(
1318 UNICODE_STRING_SIMPLE("[[:NFD_QC=Yes:]&[:ccc=0:]]"), errorCode);
1319 skipSets[UNORM_NFC].applyPattern(
1320 UNICODE_STRING_SIMPLE("[[:NFC_QC=Yes:]&[:ccc=0:]-[:HST=LV:]]"), errorCode);
1321 skipSets[UNORM_NFKD].applyPattern(
1322 UNICODE_STRING_SIMPLE("[[:NFKD_QC=Yes:]&[:ccc=0:]]"), errorCode);
1323 skipSets[UNORM_NFKC].applyPattern(
1324 UNICODE_STRING_SIMPLE("[[:NFKC_QC=Yes:]&[:ccc=0:]-[:HST=LV:]]"), errorCode);
1325
1326 // Remove from the NFC and NFKC sets all those characters that change
1327 // when a back-combining character is added.
1328 // First, get all of the back-combining characters and their combining classes.
1329 UnicodeSet combineBack("[:NFC_QC=Maybe:]", errorCode);
1330 int32_t numCombineBack=combineBack.size();
1331 int32_t *combineBackCharsAndCc=new int32_t[numCombineBack*2];
1332 UnicodeSetIterator iter(combineBack);
1333 for(int32_t i=0; i<numCombineBack; ++i) {
1334 iter.next();
1335 UChar32 c=iter.getCodepoint();
1336 combineBackCharsAndCc[2*i]=c;
1337 combineBackCharsAndCc[2*i+1]=u_getCombiningClass(c);
1338 }
374ca955 1339
729e4ab9
A		 1340 // We need not look at control codes, Han characters nor Hangul LVT syllables because they
1341 // do not combine forward. LV syllables are already removed.
1342 UnicodeSet notInteresting("[[:C:][:Unified_Ideograph:][:HST=LVT:]]", errorCode);
1343 LocalPointer<UnicodeSet> unsure(&((UnicodeSet *)(skipSets[UNORM_NFC].clone()))->removeAll(notInteresting));
1344 // System.out.format("unsure.size()=%d\n", unsure.size());
1345
1346 // For each character about which we are unsure, see if it changes when we add
1347 // one of the back-combining characters.
4388f060 1348 const Normalizer2 *norm2=Normalizer2::getNFCInstance(errorCode);
729e4ab9
A		 1349 UnicodeString s;
1350 iter.reset(*unsure);
1351 while(iter.next()) {
1352 UChar32 c=iter.getCodepoint();
1353 s.setTo(c);
1354 int32_t cLength=s.length();
1355 int32_t tccc=u_getIntPropertyValue(c, UCHAR_TRAIL_CANONICAL_COMBINING_CLASS);
1356 for(int32_t i=0; i<numCombineBack; ++i) {
1357 // If c's decomposition ends with a character with non-zero combining class, then
1358 // c can only change if it combines with a character with a non-zero combining class.
1359 int32_t cc2=combineBackCharsAndCc[2*i+1];
1360 if(tccc==0 || cc2!=0) {
1361 UChar32 c2=combineBackCharsAndCc[2*i];
1362 s.append(c2);
1363 if(!norm2->isNormalized(s, errorCode)) {
1364 // System.out.format("remove U+%04x (tccc=%d) + U+%04x (cc=%d)\n", c, tccc, c2, cc2);
1365 skipSets[UNORM_NFC].remove(c);
1366 skipSets[UNORM_NFKC].remove(c);
1367 break;
1368 }
1369 s.truncate(cLength);
1370 }
1371 }
1372 }
1373 delete [] combineBackCharsAndCc;
374ca955
A		 1374}
1375
b75a7d8f
A		 1376void
1377BasicNormalizerTest::TestSkippable() {
729e4ab9 1378 UnicodeSet diff, skipSets[UNORM_MODE_COUNT], expectSets[UNORM_MODE_COUNT];
b75a7d8f 1379 UnicodeString s, pattern;
b75a7d8f
A		 1380
1381 /* build NF*Skippable sets from runtime data */
729e4ab9
A		 1382 IcuTestErrorCode errorCode(*this, "TestSkippable");
1383 skipSets[UNORM_NFD].applyPattern(UNICODE_STRING_SIMPLE("[:NFD_Inert:]"), errorCode);
1384 skipSets[UNORM_NFKD].applyPattern(UNICODE_STRING_SIMPLE("[:NFKD_Inert:]"), errorCode);
1385 skipSets[UNORM_NFC].applyPattern(UNICODE_STRING_SIMPLE("[:NFC_Inert:]"), errorCode);
1386 skipSets[UNORM_NFKC].applyPattern(UNICODE_STRING_SIMPLE("[:NFKC_Inert:]"), errorCode);
1387 if(errorCode.logDataIfFailureAndReset("UnicodeSet(NF..._Inert) failed")) {
b75a7d8f
A		 1388 return;
1389 }
b75a7d8f
A		 1390
1391 /* get expected sets from hardcoded patterns */
729e4ab9
A		 1392 initExpectedSkippables(expectSets, errorCode);
1393 errorCode.assertSuccess();
b75a7d8f 1394
729e4ab9 1395 for(int32_t i=UNORM_NONE; i<UNORM_MODE_COUNT; ++i) {
b75a7d8f 1396 if(skipSets[i]!=expectSets[i]) {
729e4ab9
A		 1397 errln("error: TestSkippable skipSets[%d]!=expectedSets[%d]\n", i, i);
1398 // Note: This used to depend on hardcoded UnicodeSet patterns generated by
1399 // Mark's unicodetools.com.ibm.text.UCD.NFSkippable, by
1400 // running com.ibm.text.UCD.Main with the option NFSkippable.
1401 // Since ICU 4.6/Unicode 6, we are generating the
1402 // expectSets ourselves in initSkippables().
b75a7d8f
A		 1403
1404 s=UNICODE_STRING_SIMPLE("skip-expect=");
1405 (diff=skipSets[i]).removeAll(expectSets[i]).toPattern(pattern, TRUE);
1406 s.append(pattern);
1407
1408 pattern.remove();
1409 s.append(UNICODE_STRING_SIMPLE("\n\nexpect-skip="));
1410 (diff=expectSets[i]).removeAll(skipSets[i]).toPattern(pattern, TRUE);
1411 s.append(pattern);
1412 s.append(UNICODE_STRING_SIMPLE("\n\n"));
1413
1414 errln(s);
1415 }
1416 }
1417}
1418
729e4ab9
A		 1419struct StringPair { const char *input, *expected; };
1420
1421void
1422BasicNormalizerTest::TestCustomComp() {
1423 static const StringPair pairs[]={
1424 { "\\uD801\\uE000\\uDFFE", "" },
1425 { "\\uD800\\uD801\\uE000\\uDFFE\\uDFFF", "\\uD7FF\\uFFFF" },
1426 { "\\uD800\\uD801\\uDFFE\\uDFFF", "\\uD7FF\\U000107FE\\uFFFF" },
1427 { "\\uE001\\U000110B9\\u0345\\u0308\\u0327", "\\uE002\\U000110B9\\u0327\\u0345" },
1428 { "\\uE010\\U000F0011\\uE012", "\\uE011\\uE012" },
1429 { "\\uE010\\U000F0011\\U000F0011\\uE012", "\\uE011\\U000F0010" },
1430 { "\\uE111\\u1161\\uE112\\u1162", "\\uAE4C\\u1102\\u0062\\u1162" },
1431 { "\\uFFF3\\uFFF7\\U00010036\\U00010077", "\\U00010037\\U00010037\\uFFF6\\U00010037" }
1432 };
1433 IcuTestErrorCode errorCode(*this, "BasicNormalizerTest/TestCustomComp");
1434 const Normalizer2 *customNorm2=
1435 Normalizer2::getInstance(loadTestData(errorCode), "testnorm",
1436 UNORM2_COMPOSE, errorCode);
1437 if(errorCode.logDataIfFailureAndReset("unable to load testdata/testnorm.nrm")) {
1438 return;
1439 }
b331163b 1440 for(int32_t i=0; i<UPRV_LENGTHOF(pairs); ++i) {
729e4ab9
A		 1441 const StringPair &pair=pairs[i];
1442 UnicodeString input=UnicodeString(pair.input, -1, US_INV).unescape();
1443 UnicodeString expected=UnicodeString(pair.expected, -1, US_INV).unescape();
1444 UnicodeString result=customNorm2->normalize(input, errorCode);
1445 if(result!=expected) {
1446 errln("custom compose Normalizer2 did not normalize input %d as expected", i);
1447 }
1448 }
1449}
1450
1451void
1452BasicNormalizerTest::TestCustomFCC() {
1453 static const StringPair pairs[]={
1454 { "\\uD801\\uE000\\uDFFE", "" },
1455 { "\\uD800\\uD801\\uE000\\uDFFE\\uDFFF", "\\uD7FF\\uFFFF" },
1456 { "\\uD800\\uD801\\uDFFE\\uDFFF", "\\uD7FF\\U000107FE\\uFFFF" },
1457 // The following expected result is different from CustomComp
1458 // because of only-contiguous composition.
1459 { "\\uE001\\U000110B9\\u0345\\u0308\\u0327", "\\uE001\\U000110B9\\u0327\\u0308\\u0345" },
1460 { "\\uE010\\U000F0011\\uE012", "\\uE011\\uE012" },
1461 { "\\uE010\\U000F0011\\U000F0011\\uE012", "\\uE011\\U000F0010" },
1462 { "\\uE111\\u1161\\uE112\\u1162", "\\uAE4C\\u1102\\u0062\\u1162" },
1463 { "\\uFFF3\\uFFF7\\U00010036\\U00010077", "\\U00010037\\U00010037\\uFFF6\\U00010037" }
1464 };
1465 IcuTestErrorCode errorCode(*this, "BasicNormalizerTest/TestCustomFCC");
1466 const Normalizer2 *customNorm2=
1467 Normalizer2::getInstance(loadTestData(errorCode), "testnorm",
1468 UNORM2_COMPOSE_CONTIGUOUS, errorCode);
1469 if(errorCode.logDataIfFailureAndReset("unable to load testdata/testnorm.nrm")) {
1470 return;
1471 }
b331163b 1472 for(int32_t i=0; i<UPRV_LENGTHOF(pairs); ++i) {
729e4ab9
A		 1473 const StringPair &pair=pairs[i];
1474 UnicodeString input=UnicodeString(pair.input, -1, US_INV).unescape();
1475 UnicodeString expected=UnicodeString(pair.expected, -1, US_INV).unescape();
1476 UnicodeString result=customNorm2->normalize(input, errorCode);
1477 if(result!=expected) {
1478 errln("custom FCC Normalizer2 did not normalize input %d as expected", i);
1479 }
1480 }
1481}
1482
1483/* Improve code coverage of Normalizer2 */
1484void
1485BasicNormalizerTest::TestFilteredNormalizer2Coverage() {
1486 UErrorCode errorCode = U_ZERO_ERROR;
b331163b 1487 const Normalizer2 *nfcNorm2=Normalizer2::getNFCInstance(errorCode);
729e4ab9 1488 if (U_FAILURE(errorCode)) {
b331163b 1489 dataerrln("Normalizer2::getNFCInstance() call failed - %s", u_errorName(status));
729e4ab9
A		 1490 return;
1491 }
4388f060 1492 UnicodeSet filter(UNICODE_STRING_SIMPLE("[^\\u00a0-\\u00ff\\u0310-\\u031f]"), errorCode);
729e4ab9
A		 1493 FilteredNormalizer2 fn2(*nfcNorm2, filter);
1494
1495 UChar32 char32 = 0x0054;
1496
1497 if (fn2.isInert(char32)) {
1498 errln("FilteredNormalizer2.isInert() failed.");
1499 }
1500
1501 if (fn2.hasBoundaryAfter(char32)) {
1502 errln("FilteredNormalizer2.hasBoundaryAfter() failed.");
1503 }
1504
4388f060
A		 1505 UChar32 c;
1506 for(c=0; c<=0x3ff; ++c) {
1507 uint8_t expectedCC= filter.contains(c) ? nfcNorm2->getCombiningClass(c) : 0;
1508 uint8_t cc=fn2.getCombiningClass(c);
1509 if(cc!=expectedCC) {
1510 errln(
1511 UnicodeString("FilteredNormalizer2(NFC, ^A0-FF,310-31F).getCombiningClass(U+")+
1512 hex(c)+
1513 ")==filtered NFC.getCC()");
1514 }
1515 }
1516
1517 UnicodeString newString1 = UNICODE_STRING_SIMPLE("[^\\u0100-\\u01ff]");
1518 UnicodeString newString2 = UNICODE_STRING_SIMPLE("[^\\u0200-\\u02ff]");
729e4ab9
A		 1519 fn2.append(newString1, newString2, errorCode);
1520 if (U_FAILURE(errorCode)) {
1521 errln("FilteredNormalizer2.append() failed.");
1522 }
1523}
1524
b75a7d8f 1525#endif /* #if !UCONFIG_NO_NORMALIZATION */
mirror of ICU