projects / apple / icu.git / blame
| Commit | Line | Data |
|---|---|---|
| b75a7d8f A |
1 | /******************************************************************** |
| 2 | * COPYRIGHT: | |
| 3 | * Copyright (c) 1997-2003, International Business Machines Corporation and | |
| 4 | * others. All Rights Reserved. | |
| 5 | ********************************************************************/ | |
| 6 | /******************************************************************************** | |
| 7 | * | |
| 8 | * File CITERTST.C | |
| 9 | * | |
| 10 | * Modification History: | |
| 11 | * Date Name Description | |
| 12 | * Madhu Katragadda Ported for C API | |
| 13 | * 02/19/01 synwee Modified test case for new collation iterator | |
| 14 | *********************************************************************************/ | |
| 15 | /* | |
| 16 | * Collation Iterator tests. | |
| 17 | * (Let me reiterate my position...) | |
| 18 | */ | |
| 19 | ||
| 20 | #include "unicode/utypes.h" | |
| 21 | ||
| 22 | #if !UCONFIG_NO_COLLATION | |
| 23 | ||
| 24 | #include "unicode/ucol.h" | |
| 25 | #include "unicode/uloc.h" | |
| 26 | #include "unicode/uchar.h" | |
| 27 | #include "unicode/ustring.h" | |
| 28 | #include "cmemory.h" | |
| 29 | #include "cintltst.h" | |
| 30 | #include "citertst.h" | |
| 31 | #include "ccolltst.h" | |
| 32 | #include "filestrm.h" | |
| 33 | #include "cstring.h" | |
| 34 | #include "ucol_imp.h" | |
| 35 | #include "ucol_tok.h" | |
| 36 | #include <stdio.h> | |
| 37 | ||
| 38 | extern uint8_t ucol_uprv_getCaseBits(const UChar *, uint32_t, UErrorCode *); | |
| 39 | ||
| 40 | void addCollIterTest(TestNode** root) | |
| 41 | { | |
| 42 | addTest(root, &TestPrevious, "tscoll/citertst/TestPrevious"); | |
| 43 | addTest(root, &TestOffset, "tscoll/citertst/TestOffset"); | |
| 44 | addTest(root, &TestSetText, "tscoll/citertst/TestSetText"); | |
| 45 | addTest(root, &TestMaxExpansion, "tscoll/citertst/TestMaxExpansion"); | |
| 46 | addTest(root, &TestUnicodeChar, "tscoll/citertst/TestUnicodeChar"); | |
| 47 | addTest(root, &TestNormalizedUnicodeChar, | |
| 48 | "tscoll/citertst/TestNormalizedUnicodeChar"); | |
| 49 | addTest(root, &TestNormalization, "tscoll/citertst/TestNormalization"); | |
| 50 | addTest(root, &TestBug672, "tscoll/citertst/TestBug672"); | |
| 51 | addTest(root, &TestBug672Normalize, "tscoll/citertst/TestBug672Normalize"); | |
| 52 | addTest(root, &TestSmallBuffer, "tscoll/citertst/TestSmallBuffer"); | |
| 53 | addTest(root, &TestCEs, "tscoll/citertst/TestCEs"); | |
| 54 | addTest(root, &TestDiscontiguos, "tscoll/citertst/TestDiscontiguos"); | |
| 55 | addTest(root, &TestCEBufferOverflow, "tscoll/citertst/TestCEBufferOverflow"); | |
| 56 | addTest(root, &TestCEValidity, "tscoll/citertst/TestCEValidity"); | |
| 57 | addTest(root, &TestSortKeyValidity, "tscoll/citertst/TestSortKeyValidity"); | |
| 58 | } | |
| 59 | ||
| 60 | /* The locales we support */ | |
| 61 | ||
| 62 | static const char * LOCALES[] = {"en_AU", "en_BE", "en_CA"}; | |
| 63 | ||
| 64 | static void TestBug672() { | |
| 65 | UErrorCode status = U_ZERO_ERROR; | |
| 66 | UChar pattern[20]; | |
| 67 | UChar text[50]; | |
| 68 | int i; | |
| 69 | int result[3][3]; | |
| 70 | ||
| 71 | u_uastrcpy(pattern, "resume"); | |
| 72 | u_uastrcpy(text, "Time to resume updating my resume."); | |
| 73 | ||
| 74 | for (i = 0; i < 3; ++ i) { | |
| 75 | UCollator *coll = ucol_open(LOCALES[i], &status); | |
| 76 | UCollationElements *pitr = ucol_openElements(coll, pattern, -1, | |
| 77 | &status); | |
| 78 | UCollationElements *titer = ucol_openElements(coll, text, -1, | |
| 79 | &status); | |
| 80 | if (U_FAILURE(status)) { | |
| 81 | log_err("ERROR: in creation of either the collator or the collation iterator :%s\n", | |
| 82 | myErrorName(status)); | |
| 83 | return; | |
| 84 | } | |
| 85 | ||
| 86 | log_verbose("locale tested %s\n", LOCALES[i]); | |
| 87 | ||
| 88 | while (ucol_next(pitr, &status) != UCOL_NULLORDER && | |
| 89 | U_SUCCESS(status)) { | |
| 90 | } | |
| 91 | if (U_FAILURE(status)) { | |
| 92 | log_err("ERROR: reversing collation iterator :%s\n", | |
| 93 | myErrorName(status)); | |
| 94 | return; | |
| 95 | } | |
| 96 | ucol_reset(pitr); | |
| 97 | ||
| 98 | ucol_setOffset(titer, u_strlen(pattern), &status); | |
| 99 | if (U_FAILURE(status)) { | |
| 100 | log_err("ERROR: setting offset in collator :%s\n", | |
| 101 | myErrorName(status)); | |
| 102 | return; | |
| 103 | } | |
| 104 | result[i][0] = ucol_getOffset(titer); | |
| 105 | log_verbose("Text iterator set to offset %d\n", result[i][0]); | |
| 106 | ||
| 107 | /* Use previous() */ | |
| 108 | ucol_previous(titer, &status); | |
| 109 | result[i][1] = ucol_getOffset(titer); | |
| 110 | log_verbose("Current offset %d after previous\n", result[i][1]); | |
| 111 | ||
| 112 | /* Add one to index */ | |
| 113 | log_verbose("Adding one to current offset...\n"); | |
| 114 | ucol_setOffset(titer, ucol_getOffset(titer) + 1, &status); | |
| 115 | if (U_FAILURE(status)) { | |
| 116 | log_err("ERROR: setting offset in collator :%s\n", | |
| 117 | myErrorName(status)); | |
| 118 | return; | |
| 119 | } | |
| 120 | result[i][2] = ucol_getOffset(titer); | |
| 121 | log_verbose("Current offset in text = %d\n", result[i][2]); | |
| 122 | ucol_closeElements(pitr); | |
| 123 | ucol_closeElements(titer); | |
| 124 | ucol_close(coll); | |
| 125 | } | |
| 126 | ||
| 127 | if (uprv_memcmp(result[0], result[1], 3) != 0 || | |
| 128 | uprv_memcmp(result[1], result[2], 3) != 0) { | |
| 129 | log_err("ERROR: Different locales have different offsets at the same character\n"); | |
| 130 | } | |
| 131 | } | |
| 132 | ||
| 133 | ||
| 134 | ||
| 135 | /* Running this test with normalization enabled showed up a bug in the incremental | |
| 136 | normalization code. */ | |
| 137 | static void TestBug672Normalize() { | |
| 138 | UErrorCode status = U_ZERO_ERROR; | |
| 139 | UChar pattern[20]; | |
| 140 | UChar text[50]; | |
| 141 | int i; | |
| 142 | int result[3][3]; | |
| 143 | ||
| 144 | u_uastrcpy(pattern, "resume"); | |
| 145 | u_uastrcpy(text, "Time to resume updating my resume."); | |
| 146 | ||
| 147 | for (i = 0; i < 3; ++ i) { | |
| 148 | UCollator *coll = ucol_open(LOCALES[i], &status); | |
| 149 | UCollationElements *pitr = NULL; | |
| 150 | UCollationElements *titer = NULL; | |
| 151 | ||
| 152 | ucol_setAttribute(coll, UCOL_NORMALIZATION_MODE, UCOL_ON, &status); | |
| 153 | ||
| 154 | pitr = ucol_openElements(coll, pattern, -1, &status); | |
| 155 | titer = ucol_openElements(coll, text, -1, &status); | |
| 156 | if (U_FAILURE(status)) { | |
| 157 | log_err("ERROR: in creation of either the collator or the collation iterator :%s\n", | |
| 158 | myErrorName(status)); | |
| 159 | return; | |
| 160 | } | |
| 161 | ||
| 162 | log_verbose("locale tested %s\n", LOCALES[i]); | |
| 163 | ||
| 164 | while (ucol_next(pitr, &status) != UCOL_NULLORDER && | |
| 165 | U_SUCCESS(status)) { | |
| 166 | } | |
| 167 | if (U_FAILURE(status)) { | |
| 168 | log_err("ERROR: reversing collation iterator :%s\n", | |
| 169 | myErrorName(status)); | |
| 170 | return; | |
| 171 | } | |
| 172 | ucol_reset(pitr); | |
| 173 | ||
| 174 | ucol_setOffset(titer, u_strlen(pattern), &status); | |
| 175 | if (U_FAILURE(status)) { | |
| 176 | log_err("ERROR: setting offset in collator :%s\n", | |
| 177 | myErrorName(status)); | |
| 178 | return; | |
| 179 | } | |
| 180 | result[i][0] = ucol_getOffset(titer); | |
| 181 | log_verbose("Text iterator set to offset %d\n", result[i][0]); | |
| 182 | ||
| 183 | /* Use previous() */ | |
| 184 | ucol_previous(titer, &status); | |
| 185 | result[i][1] = ucol_getOffset(titer); | |
| 186 | log_verbose("Current offset %d after previous\n", result[i][1]); | |
| 187 | ||
| 188 | /* Add one to index */ | |
| 189 | log_verbose("Adding one to current offset...\n"); | |
| 190 | ucol_setOffset(titer, ucol_getOffset(titer) + 1, &status); | |
| 191 | if (U_FAILURE(status)) { | |
| 192 | log_err("ERROR: setting offset in collator :%s\n", | |
| 193 | myErrorName(status)); | |
| 194 | return; | |
| 195 | } | |
| 196 | result[i][2] = ucol_getOffset(titer); | |
| 197 | log_verbose("Current offset in text = %d\n", result[i][2]); | |
| 198 | ucol_closeElements(pitr); | |
| 199 | ucol_closeElements(titer); | |
| 200 | ucol_close(coll); | |
| 201 | } | |
| 202 | ||
| 203 | if (uprv_memcmp(result[0], result[1], 3) != 0 || | |
| 204 | uprv_memcmp(result[1], result[2], 3) != 0) { | |
| 205 | log_err("ERROR: Different locales have different offsets at the same character\n"); | |
| 206 | } | |
| 207 | } | |
| 208 | ||
| 209 | ||
| 210 | ||
| 211 | ||
| 212 | /** | |
| 213 | * Test for CollationElementIterator previous and next for the whole set of | |
| 214 | * unicode characters. | |
| 215 | */ | |
| 216 | static void TestUnicodeChar() | |
| 217 | { | |
| 218 | UChar source[0x100]; | |
| 219 | UCollator *en_us; | |
| 220 | UCollationElements *iter; | |
| 221 | UErrorCode status = U_ZERO_ERROR; | |
| 222 | UChar codepoint; | |
| 223 | ||
| 224 | UChar *test; | |
| 225 | en_us = ucol_open("en_US", &status); | |
| 226 | if (U_FAILURE(status)){ | |
| 227 | log_err("ERROR: in creation of collation data using ucol_open()\n %s\n", | |
| 228 | myErrorName(status)); | |
| 229 | return; | |
| 230 | } | |
| 231 | ||
| 232 | for (codepoint = 1; codepoint < 0xFFFE;) | |
| 233 | { | |
| 234 | test = source; | |
| 235 | ||
| 236 | while (codepoint % 0xFF != 0) | |
| 237 | { | |
| 238 | if (u_isdefined(codepoint)) | |
| 239 | *(test ++) = codepoint; | |
| 240 | codepoint ++; | |
| 241 | } | |
| 242 | ||
| 243 | if (u_isdefined(codepoint)) | |
| 244 | *(test ++) = codepoint; | |
| 245 | ||
| 246 | if (codepoint != 0xFFFF) | |
| 247 | codepoint ++; | |
| 248 | ||
| 249 | *test = 0; | |
| 250 | iter=ucol_openElements(en_us, source, u_strlen(source), &status); | |
| 251 | if(U_FAILURE(status)){ | |
| 252 | log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", | |
| 253 | myErrorName(status)); | |
| 254 | ucol_close(en_us); | |
| 255 | return; | |
| 256 | } | |
| 257 | /* A basic test to see if it's working at all */ | |
| 258 | log_verbose("codepoint testing %x\n", codepoint); | |
| 259 | backAndForth(iter); | |
| 260 | ucol_closeElements(iter); | |
| 261 | ||
| 262 | /* null termination test */ | |
| 263 | iter=ucol_openElements(en_us, source, -1, &status); | |
| 264 | if(U_FAILURE(status)){ | |
| 265 | log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", | |
| 266 | myErrorName(status)); | |
| 267 | ucol_close(en_us); | |
| 268 | return; | |
| 269 | } | |
| 270 | /* A basic test to see if it's working at all */ | |
| 271 | backAndForth(iter); | |
| 272 | ucol_closeElements(iter); | |
| 273 | } | |
| 274 | ||
| 275 | ucol_close(en_us); | |
| 276 | } | |
| 277 | ||
| 278 | /** | |
| 279 | * Test for CollationElementIterator previous and next for the whole set of | |
| 280 | * unicode characters with normalization on. | |
| 281 | */ | |
| 282 | static void TestNormalizedUnicodeChar() | |
| 283 | { | |
| 284 | UChar source[0x100]; | |
| 285 | UCollator *th_th; | |
| 286 | UCollationElements *iter; | |
| 287 | UErrorCode status = U_ZERO_ERROR; | |
| 288 | UChar codepoint; | |
| 289 | ||
| 290 | UChar *test; | |
| 291 | /* thai should have normalization on */ | |
| 292 | th_th = ucol_open("th_TH", &status); | |
| 293 | if (U_FAILURE(status)){ | |
| 294 | log_err("ERROR: in creation of thai collation using ucol_open()\n %s\n", | |
| 295 | myErrorName(status)); | |
| 296 | return; | |
| 297 | } | |
| 298 | ||
| 299 | for (codepoint = 1; codepoint < 0xFFFE;) | |
| 300 | { | |
| 301 | test = source; | |
| 302 | ||
| 303 | while (codepoint % 0xFF != 0) | |
| 304 | { | |
| 305 | if (u_isdefined(codepoint)) | |
| 306 | *(test ++) = codepoint; | |
| 307 | codepoint ++; | |
| 308 | } | |
| 309 | ||
| 310 | if (u_isdefined(codepoint)) | |
| 311 | *(test ++) = codepoint; | |
| 312 | ||
| 313 | if (codepoint != 0xFFFF) | |
| 314 | codepoint ++; | |
| 315 | ||
| 316 | *test = 0; | |
| 317 | iter=ucol_openElements(th_th, source, u_strlen(source), &status); | |
| 318 | if(U_FAILURE(status)){ | |
| 319 | log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", | |
| 320 | myErrorName(status)); | |
| 321 | ucol_close(th_th); | |
| 322 | return; | |
| 323 | } | |
| 324 | ||
| 325 | backAndForth(iter); | |
| 326 | ucol_closeElements(iter); | |
| 327 | ||
| 328 | iter=ucol_openElements(th_th, source, -1, &status); | |
| 329 | if(U_FAILURE(status)){ | |
| 330 | log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", | |
| 331 | myErrorName(status)); | |
| 332 | ucol_close(th_th); | |
| 333 | return; | |
| 334 | } | |
| 335 | ||
| 336 | backAndForth(iter); | |
| 337 | ucol_closeElements(iter); | |
| 338 | } | |
| 339 | ||
| 340 | ucol_close(th_th); | |
| 341 | } | |
| 342 | ||
| 343 | /** | |
| 344 | * Test the incremental normalization | |
| 345 | */ | |
| 346 | static void TestNormalization() | |
| 347 | { | |
| 348 | UErrorCode status = U_ZERO_ERROR; | |
| 349 | const char *str = | |
| 350 | "&a < \\u0300\\u0315 < A\\u0300\\u0315 < \\u0316\\u0315B < \\u0316\\u0300\\u0315"; | |
| 351 | UCollator *coll; | |
| 352 | UChar rule[50]; | |
| 353 | int rulelen = u_unescape(str, rule, 50); | |
| 354 | int count = 0; | |
| 355 | const char *testdata[] = | |
| 356 | {"\\u1ED9", "o\\u0323\\u0302", | |
| 357 | "\\u0300\\u0315", "\\u0315\\u0300", | |
| 358 | "A\\u0300\\u0315B", "A\\u0315\\u0300B", | |
| 359 | "A\\u0316\\u0315B", "A\\u0315\\u0316B", | |
| 360 | "\\u0316\\u0300\\u0315", "\\u0315\\u0300\\u0316", | |
| 361 | "A\\u0316\\u0300\\u0315B", "A\\u0315\\u0300\\u0316B", | |
| 362 | "\\u0316\\u0315\\u0300", "A\\u0316\\u0315\\u0300B"}; | |
| 363 | int32_t srclen; | |
| 364 | UChar source[10]; | |
| 365 | UCollationElements *iter; | |
| 366 | ||
| 367 | coll = ucol_openRules(rule, rulelen, UCOL_ON, UCOL_TERTIARY, NULL, &status); | |
| 368 | ucol_setAttribute(coll, UCOL_NORMALIZATION_MODE, UCOL_ON, &status); | |
| 369 | if (U_FAILURE(status)){ | |
| 370 | log_err("ERROR: in creation of collator using ucol_openRules()\n %s\n", | |
| 371 | myErrorName(status)); | |
| 372 | return; | |
| 373 | } | |
| 374 | ||
| 375 | srclen = u_unescape(testdata[0], source, 10); | |
| 376 | iter = ucol_openElements(coll, source, srclen, &status); | |
| 377 | backAndForth(iter); | |
| 378 | ucol_closeElements(iter); | |
| 379 | ||
| 380 | srclen = u_unescape(testdata[1], source, 10); | |
| 381 | iter = ucol_openElements(coll, source, srclen, &status); | |
| 382 | backAndForth(iter); | |
| 383 | ucol_closeElements(iter); | |
| 384 | ||
| 385 | while (count < 12) { | |
| 386 | srclen = u_unescape(testdata[count], source, 10); | |
| 387 | iter = ucol_openElements(coll, source, srclen, &status); | |
| 388 | ||
| 389 | if (U_FAILURE(status)){ | |
| 390 | log_err("ERROR: in creation of collator element iterator\n %s\n", | |
| 391 | myErrorName(status)); | |
| 392 | return; | |
| 393 | } | |
| 394 | backAndForth(iter); | |
| 395 | ucol_closeElements(iter); | |
| 396 | ||
| 397 | iter = ucol_openElements(coll, source, -1, &status); | |
| 398 | ||
| 399 | if (U_FAILURE(status)){ | |
| 400 | log_err("ERROR: in creation of collator element iterator\n %s\n", | |
| 401 | myErrorName(status)); | |
| 402 | return; | |
| 403 | } | |
| 404 | backAndForth(iter); | |
| 405 | ucol_closeElements(iter); | |
| 406 | count ++; | |
| 407 | } | |
| 408 | ucol_close(coll); | |
| 409 | } | |
| 410 | ||
| 411 | /** | |
| 412 | * Test for CollationElementIterator.previous() | |
| 413 | * | |
| 414 | * @bug 4108758 - Make sure it works with contracting characters | |
| 415 | * | |
| 416 | */ | |
| 417 | static void TestPrevious() | |
| 418 | { | |
| 419 | UCollator *coll=NULL; | |
| 420 | UChar rule[50]; | |
| 421 | UChar *source; | |
| 422 | UCollator *c1, *c2, *c3; | |
| 423 | UCollationElements *iter; | |
| 424 | UErrorCode status = U_ZERO_ERROR; | |
| 425 | ||
| 426 | test1=(UChar*)malloc(sizeof(UChar) * 50); | |
| 427 | test2=(UChar*)malloc(sizeof(UChar) * 50); | |
| 428 | u_uastrcpy(test1, "What subset of all possible test cases?"); | |
| 429 | u_uastrcpy(test2, "has the highest probability of detecting"); | |
| 430 | coll = ucol_open("en_US", &status); | |
| 431 | ||
| 432 | iter=ucol_openElements(coll, test1, u_strlen(test1), &status); | |
| 433 | log_verbose("English locale testing back and forth\n"); | |
| 434 | if(U_FAILURE(status)){ | |
| 435 | log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", | |
| 436 | myErrorName(status)); | |
| 437 | ucol_close(coll); | |
| 438 | return; | |
| 439 | } | |
| 440 | /* A basic test to see if it's working at all */ | |
| 441 | backAndForth(iter); | |
| 442 | ucol_closeElements(iter); | |
| 443 | ucol_close(coll); | |
| 444 | ||
| 445 | /* Test with a contracting character sequence */ | |
| 446 | u_uastrcpy(rule, "&a,A < b,B < c,C, d,D < z,Z < ch,cH,Ch,CH"); | |
| 447 | c1 = ucol_openRules(rule, u_strlen(rule), UCOL_OFF, UCOL_DEFAULT_STRENGTH, NULL, &status); | |
| 448 | ||
| 449 | log_verbose("Contraction rule testing back and forth with no normalization\n"); | |
| 450 | ||
| 451 | if (c1 == NULL || U_FAILURE(status)) | |
| 452 | { | |
| 453 | log_err("Couldn't create a RuleBasedCollator with a contracting sequence\n %s\n", | |
| 454 | myErrorName(status)); | |
| 455 | return; | |
| 456 | } | |
| 457 | source=(UChar*)malloc(sizeof(UChar) * 20); | |
| 458 | u_uastrcpy(source, "abchdcba"); | |
| 459 | iter=ucol_openElements(c1, source, u_strlen(source), &status); | |
| 460 | if(U_FAILURE(status)){ | |
| 461 | log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", | |
| 462 | myErrorName(status)); | |
| 463 | return; | |
| 464 | } | |
| 465 | backAndForth(iter); | |
| 466 | ucol_closeElements(iter); | |
| 467 | ucol_close(c1); | |
| 468 | ||
| 469 | /* Test with an expanding character sequence */ | |
| 470 | u_uastrcpy(rule, "&a < b < c/abd < d"); | |
| 471 | c2 = ucol_openRules(rule, u_strlen(rule), UCOL_OFF, UCOL_DEFAULT_STRENGTH, NULL, &status); | |
| 472 | log_verbose("Expansion rule testing back and forth with no normalization\n"); | |
| 473 | if (c2 == NULL || U_FAILURE(status)) | |
| 474 | { | |
| 475 | log_err("Couldn't create a RuleBasedCollator with a contracting sequence.\n %s\n", | |
| 476 | myErrorName(status)); | |
| 477 | return; | |
| 478 | } | |
| 479 | u_uastrcpy(source, "abcd"); | |
| 480 | iter=ucol_openElements(c2, source, u_strlen(source), &status); | |
| 481 | if(U_FAILURE(status)){ | |
| 482 | log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", | |
| 483 | myErrorName(status)); | |
| 484 | return; | |
| 485 | } | |
| 486 | backAndForth(iter); | |
| 487 | ucol_closeElements(iter); | |
| 488 | ucol_close(c2); | |
| 489 | /* Now try both */ | |
| 490 | u_uastrcpy(rule, "&a < b < c/aba < d < z < ch"); | |
| 491 | c3 = ucol_openRules(rule, u_strlen(rule), UCOL_DEFAULT, UCOL_DEFAULT_STRENGTH,NULL, &status); | |
| 492 | log_verbose("Expansion/contraction rule testing back and forth with no normalization\n"); | |
| 493 | ||
| 494 | if (c3 == NULL || U_FAILURE(status)) | |
| 495 | { | |
| 496 | log_err("Couldn't create a RuleBasedCollator with a contracting sequence.\n %s\n", | |
| 497 | myErrorName(status)); | |
| 498 | return; | |
| 499 | } | |
| 500 | u_uastrcpy(source, "abcdbchdc"); | |
| 501 | iter=ucol_openElements(c3, source, u_strlen(source), &status); | |
| 502 | if(U_FAILURE(status)){ | |
| 503 | log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", | |
| 504 | myErrorName(status)); | |
| 505 | return; | |
| 506 | } | |
| 507 | backAndForth(iter); | |
| 508 | ucol_closeElements(iter); | |
| 509 | ucol_close(c3); | |
| 510 | source[0] = 0x0e41; | |
| 511 | source[1] = 0x0e02; | |
| 512 | source[2] = 0x0e41; | |
| 513 | source[3] = 0x0e02; | |
| 514 | source[4] = 0x0e27; | |
| 515 | source[5] = 0x61; | |
| 516 | source[6] = 0x62; | |
| 517 | source[7] = 0x63; | |
| 518 | source[8] = 0; | |
| 519 | ||
| 520 | coll = ucol_open("th_TH", &status); | |
| 521 | log_verbose("Thai locale testing back and forth with normalization\n"); | |
| 522 | iter=ucol_openElements(coll, source, u_strlen(source), &status); | |
| 523 | if(U_FAILURE(status)){ | |
| 524 | log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", | |
| 525 | myErrorName(status)); | |
| 526 | return; | |
| 527 | } | |
| 528 | backAndForth(iter); | |
| 529 | ucol_closeElements(iter); | |
| 530 | ucol_close(coll); | |
| 531 | ||
| 532 | /* prev test */ | |
| 533 | source[0] = 0x0061; | |
| 534 | source[1] = 0x30CF; | |
| 535 | source[2] = 0x3099; | |
| 536 | source[3] = 0x30FC; | |
| 537 | source[4] = 0; | |
| 538 | ||
| 539 | coll = ucol_open("ja_JP", &status); | |
| 540 | log_verbose("Japanese locale testing back and forth with normalization\n"); | |
| 541 | iter=ucol_openElements(coll, source, u_strlen(source), &status); | |
| 542 | if(U_FAILURE(status)){ | |
| 543 | log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", | |
| 544 | myErrorName(status)); | |
| 545 | return; | |
| 546 | } | |
| 547 | backAndForth(iter); | |
| 548 | ucol_closeElements(iter); | |
| 549 | ucol_close(coll); | |
| 550 | ||
| 551 | free(source); | |
| 552 | free(test1); | |
| 553 | free(test2); | |
| 554 | } | |
| 555 | ||
| 556 | /** | |
| 557 | * Test for getOffset() and setOffset() | |
| 558 | */ | |
| 559 | static void TestOffset() | |
| 560 | { | |
| 561 | UErrorCode status= U_ZERO_ERROR; | |
| 562 | UCollator *en_us=NULL; | |
| 563 | UCollationElements *iter, *pristine; | |
| 564 | int32_t offset; | |
| 565 | int32_t *orders; | |
| 566 | int32_t orderLength=0; | |
| 567 | int count = 0; | |
| 568 | test1=(UChar*)malloc(sizeof(UChar) * 50); | |
| 569 | test2=(UChar*)malloc(sizeof(UChar) * 50); | |
| 570 | u_uastrcpy(test1, "What subset of all possible test cases?"); | |
| 571 | u_uastrcpy(test2, "has the highest probability of detecting"); | |
| 572 | en_us = ucol_open("en_US", &status); | |
| 573 | log_verbose("Testing getOffset and setOffset for CollationElements\n"); | |
| 574 | iter = ucol_openElements(en_us, test1, u_strlen(test1), &status); | |
| 575 | if(U_FAILURE(status)){ | |
| 576 | log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", | |
| 577 | myErrorName(status)); | |
| 578 | ucol_close(en_us); | |
| 579 | return; | |
| 580 | } | |
| 581 | /* Run all the way through the iterator, then get the offset */ | |
| 582 | ||
| 583 | orders = getOrders(iter, &orderLength); | |
| 584 | ||
| 585 | offset = ucol_getOffset(iter); | |
| 586 | ||
| 587 | if (offset != u_strlen(test1)) | |
| 588 | { | |
| 589 | log_err("offset at end != length %d vs %d\n", offset, | |
| 590 | u_strlen(test1) ); | |
| 591 | } | |
| 592 | ||
| 593 | /* Now set the offset back to the beginning and see if it works */ | |
| 594 | pristine=ucol_openElements(en_us, test1, u_strlen(test1), &status); | |
| 595 | if(U_FAILURE(status)){ | |
| 596 | log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", | |
| 597 | myErrorName(status)); | |
| 598 | ucol_close(en_us); | |
| 599 | return; | |
| 600 | } | |
| 601 | status = U_ZERO_ERROR; | |
| 602 | ||
| 603 | ucol_setOffset(iter, 0, &status); | |
| 604 | if (U_FAILURE(status)) | |
| 605 | { | |
| 606 | log_err("setOffset failed. %s\n", myErrorName(status)); | |
| 607 | } | |
| 608 | else | |
| 609 | { | |
| 610 | assertEqual(iter, pristine); | |
| 611 | } | |
| 612 | ||
| 613 | ucol_closeElements(pristine); | |
| 614 | ucol_closeElements(iter); | |
| 615 | free(orders); | |
| 616 | ||
| 617 | /* testing offsets in normalization buffer */ | |
| 618 | test1[0] = 0x61; | |
| 619 | test1[1] = 0x300; | |
| 620 | test1[2] = 0x316; | |
| 621 | test1[3] = 0x62; | |
| 622 | test1[4] = 0; | |
| 623 | ucol_setAttribute(en_us, UCOL_NORMALIZATION_MODE, UCOL_ON, &status); | |
| 624 | iter = ucol_openElements(en_us, test1, 4, &status); | |
| 625 | if(U_FAILURE(status)){ | |
| 626 | log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", | |
| 627 | myErrorName(status)); | |
| 628 | ucol_close(en_us); | |
| 629 | return; | |
| 630 | } | |
| 631 | ||
| 632 | count = 0; | |
| 633 | while (ucol_next(iter, &status) != UCOL_NULLORDER && | |
| 634 | U_SUCCESS(status)) { | |
| 635 | switch (count) { | |
| 636 | case 0: | |
| 637 | if (ucol_getOffset(iter) != 1) { | |
| 638 | log_err("ERROR: Offset of iteration should be 0\n"); | |
| 639 | } | |
| 640 | break; | |
| 641 | case 3: | |
| 642 | if (ucol_getOffset(iter) != 4) { | |
| 643 | log_err("ERROR: Offset of iteration should be 4\n"); | |
| 644 | } | |
| 645 | break; | |
| 646 | default: | |
| 647 | if (ucol_getOffset(iter) != 3) { | |
| 648 | log_err("ERROR: Offset of iteration should be 3\n"); | |
| 649 | } | |
| 650 | } | |
| 651 | count ++; | |
| 652 | } | |
| 653 | ||
| 654 | ucol_reset(iter); | |
| 655 | count = 0; | |
| 656 | while (ucol_previous(iter, &status) != UCOL_NULLORDER && | |
| 657 | U_SUCCESS(status)) { | |
| 658 | switch (count) { | |
| 659 | case 0: | |
| 660 | if (ucol_getOffset(iter) != 3) { | |
| 661 | log_err("ERROR: Offset of iteration should be 3\n"); | |
| 662 | } | |
| 663 | break; | |
| 664 | default: | |
| 665 | if (ucol_getOffset(iter) != 0) { | |
| 666 | log_err("ERROR: Offset of iteration should be 0\n"); | |
| 667 | } | |
| 668 | } | |
| 669 | count ++; | |
| 670 | } | |
| 671 | ||
| 672 | if(U_FAILURE(status)){ | |
| 673 | log_err("ERROR: in iterating collation elements %s\n", | |
| 674 | myErrorName(status)); | |
| 675 | } | |
| 676 | ||
| 677 | ucol_closeElements(iter); | |
| 678 | ucol_close(en_us); | |
| 679 | free(test1); | |
| 680 | free(test2); | |
| 681 | } | |
| 682 | ||
| 683 | /** | |
| 684 | * Test for setText() | |
| 685 | */ | |
| 686 | static void TestSetText() | |
| 687 | { | |
| 688 | int32_t c,i; | |
| 689 | UErrorCode status = U_ZERO_ERROR; | |
| 690 | UCollator *en_us=NULL; | |
| 691 | UCollationElements *iter1, *iter2; | |
| 692 | test1=(UChar*)malloc(sizeof(UChar) * 50); | |
| 693 | test2=(UChar*)malloc(sizeof(UChar) * 50); | |
| 694 | u_uastrcpy(test1, "What subset of all possible test cases?"); | |
| 695 | u_uastrcpy(test2, "has the highest probability of detecting"); | |
| 696 | en_us = ucol_open("en_US", &status); | |
| 697 | log_verbose("testing setText for Collation elements\n"); | |
| 698 | iter1=ucol_openElements(en_us, test1, u_strlen(test1), &status); | |
| 699 | if(U_FAILURE(status)){ | |
| 700 | log_err("ERROR: in creation of collation element iterator1 using ucol_openElements()\n %s\n", | |
| 701 | myErrorName(status)); | |
| 702 | ucol_close(en_us); | |
| 703 | return; | |
| 704 | } | |
| 705 | iter2=ucol_openElements(en_us, test2, u_strlen(test2), &status); | |
| 706 | if(U_FAILURE(status)){ | |
| 707 | log_err("ERROR: in creation of collation element iterator2 using ucol_openElements()\n %s\n", | |
| 708 | myErrorName(status)); | |
| 709 | ucol_close(en_us); | |
| 710 | return; | |
| 711 | } | |
| 712 | ||
| 713 | /* Run through the second iterator just to exercise it */ | |
| 714 | c = ucol_next(iter2, &status); | |
| 715 | i = 0; | |
| 716 | ||
| 717 | while ( ++i < 10 && (c != UCOL_NULLORDER)) | |
| 718 | { | |
| 719 | if (U_FAILURE(status)) | |
| 720 | { | |
| 721 | log_err("iter2->next() returned an error. %s\n", myErrorName(status)); | |
| 722 | ucol_closeElements(iter2); | |
| 723 | ucol_closeElements(iter1); | |
| 724 | ucol_close(en_us); | |
| 725 | return; | |
| 726 | } | |
| 727 | ||
| 728 | c = ucol_next(iter2, &status); | |
| 729 | } | |
| 730 | ||
| 731 | /* Now set it to point to the same string as the first iterator */ | |
| 732 | ucol_setText(iter2, test1, u_strlen(test1), &status); | |
| 733 | if (U_FAILURE(status)) | |
| 734 | { | |
| 735 | log_err("call to iter2->setText(test1) failed. %s\n", myErrorName(status)); | |
| 736 | } | |
| 737 | else | |
| 738 | { | |
| 739 | assertEqual(iter1, iter2); | |
| 740 | } | |
| 741 | ||
| 742 | /* Now set it to point to a null string with fake length*/ | |
| 743 | ucol_setText(iter2, NULL, 2, &status); | |
| 744 | if (U_FAILURE(status)) | |
| 745 | { | |
| 746 | log_err("call to iter2->setText(null) failed. %s\n", myErrorName(status)); | |
| 747 | } | |
| 748 | else | |
| 749 | { | |
| 750 | if (ucol_next(iter2, &status) != UCOL_NULLORDER) { | |
| 751 | log_err("iter2 with null text expected to return UCOL_NULLORDER\n"); | |
| 752 | } | |
| 753 | } | |
| 754 | ||
| 755 | ucol_closeElements(iter2); | |
| 756 | ucol_closeElements(iter1); | |
| 757 | ucol_close(en_us); | |
| 758 | free(test1); | |
| 759 | free(test2); | |
| 760 | } | |
| 761 | ||
| 762 | ||
| 763 | ||
| 764 | static void backAndForth(UCollationElements *iter) | |
| 765 | { | |
| 766 | /* Run through the iterator forwards and stick it into an array */ | |
| 767 | int32_t index, o; | |
| 768 | UErrorCode status = U_ZERO_ERROR; | |
| 769 | int32_t orderLength = 0; | |
| 770 | int32_t *orders; | |
| 771 | orders= getOrders(iter, &orderLength); | |
| 772 | ||
| 773 | ||
| 774 | /* Now go through it backwards and make sure we get the same values */ | |
| 775 | index = orderLength; | |
| 776 | ucol_reset(iter); | |
| 777 | ||
| 778 | /* synwee : changed */ | |
| 779 | while ((o = ucol_previous(iter, &status)) != UCOL_NULLORDER) | |
| 780 | { | |
| 781 | if (o != orders[-- index]) | |
| 782 | { | |
| 783 | if (o == 0) | |
| 784 | index ++; | |
| 785 | else | |
| 786 | { | |
| 787 | while (index > 0 && orders[-- index] == 0) | |
| 788 | { | |
| 789 | } | |
| 790 | if (o != orders[index]) | |
| 791 | { | |
| 792 | log_err("Mismatch at index : 0x%x\n", index); | |
| 793 | return; | |
| 794 | } | |
| 795 | ||
| 796 | } | |
| 797 | } | |
| 798 | } | |
| 799 | ||
| 800 | while (index != 0 && orders[index - 1] == 0) { | |
| 801 | index --; | |
| 802 | } | |
| 803 | ||
| 804 | if (index != 0) | |
| 805 | { | |
| 806 | log_err("Didn't get back to beginning - index is %d\n", index); | |
| 807 | ||
| 808 | ucol_reset(iter); | |
| 809 | log_err("\nnext: "); | |
| 810 | if ((o = ucol_next(iter, &status)) != UCOL_NULLORDER) | |
| 811 | { | |
| 812 | log_err("Error at %x\n", o); | |
| 813 | } | |
| 814 | log_err("\nprev: "); | |
| 815 | if ((o = ucol_previous(iter, &status)) != UCOL_NULLORDER) | |
| 816 | { | |
| 817 | log_err("Error at %x\n", o); | |
| 818 | } | |
| 819 | log_verbose("\n"); | |
| 820 | } | |
| 821 | ||
| 822 | free(orders); | |
| 823 | } | |
| 824 | ||
| 825 | /** @bug 4108762 | |
| 826 | * Test for getMaxExpansion() | |
| 827 | */ | |
| 828 | static void TestMaxExpansion() | |
| 829 | { | |
| 830 | UErrorCode status = U_ZERO_ERROR; | |
| 831 | UCollator *coll ;/*= ucol_open("en_US", &status);*/ | |
| 832 | UChar ch = 0; | |
| 833 | UChar supplementary[2] = {0xD800, 0xDC00}; | |
| 834 | uint32_t sorder = 0; | |
| 835 | UCollationElements *iter ;/*= ucol_openElements(coll, &ch, 1, &status);*/ | |
| 836 | uint32_t temporder = 0; | |
| 837 | ||
| 838 | UChar rule[256]; | |
| 839 | u_uastrcpy(rule, "&a < ab < c/aba < d < z < ch"); | |
| 840 | coll = ucol_openRules(rule, u_strlen(rule), UCOL_DEFAULT, | |
| 841 | UCOL_DEFAULT_STRENGTH,NULL, &status); | |
| 842 | if(U_SUCCESS(status) && coll) { | |
| 843 | iter = ucol_openElements(coll, &ch, 1, &status); | |
| 844 | ||
| 845 | while (ch < 0xFFFF && U_SUCCESS(status)) { | |
| 846 | int count = 1; | |
| 847 | uint32_t order; | |
| 848 | int32_t size = 0; | |
| 849 | ||
| 850 | ch ++; | |
| 851 | ||
| 852 | ucol_setText(iter, &ch, 1, &status); | |
| 853 | order = ucol_previous(iter, &status); | |
| 854 | ||
| 855 | /* thai management */ | |
| 856 | if (order == 0) | |
| 857 | order = ucol_previous(iter, &status); | |
| 858 | ||
| 859 | while (U_SUCCESS(status) && | |
| 860 | ucol_previous(iter, &status) != UCOL_NULLORDER) { | |
| 861 | count ++; | |
| 862 | } | |
| 863 | ||
| 864 | size = ucol_getMaxExpansion(iter, order); | |
| 865 | if (U_FAILURE(status) || size < count) { | |
| 866 | log_err("Failure at codepoint %d, maximum expansion count < %d\n", | |
| 867 | ch, count); | |
| 868 | } | |
| 869 | } | |
| 870 | ||
| 871 | /* testing for exact max expansion */ | |
| 872 | ch = 0; | |
| 873 | while (ch < 0x61) { | |
| 874 | uint32_t order; | |
| 875 | int32_t size; | |
| 876 | ucol_setText(iter, &ch, 1, &status); | |
| 877 | order = ucol_previous(iter, &status); | |
| 878 | size = ucol_getMaxExpansion(iter, order); | |
| 879 | if (U_FAILURE(status) || size != 1) { | |
| 880 | log_err("Failure at codepoint %d, maximum expansion count < %d\n", | |
| 881 | ch, 1); | |
| 882 | } | |
| 883 | ch ++; | |
| 884 | } | |
| 885 | ||
| 886 | ch = 0x63; | |
| 887 | ucol_setText(iter, &ch, 1, &status); | |
| 888 | temporder = ucol_previous(iter, &status); | |
| 889 | ||
| 890 | if (U_FAILURE(status) || ucol_getMaxExpansion(iter, temporder) != 3) { | |
| 891 | log_err("Failure at codepoint %d, maximum expansion count != %d\n", | |
| 892 | ch, 3); | |
| 893 | } | |
| 894 | ||
| 895 | ch = 0x64; | |
| 896 | ucol_setText(iter, &ch, 1, &status); | |
| 897 | temporder = ucol_previous(iter, &status); | |
| 898 | ||
| 899 | if (U_FAILURE(status) || ucol_getMaxExpansion(iter, temporder) != 1) { | |
| 900 | log_err("Failure at codepoint %d, maximum expansion count != %d\n", | |
| 901 | ch, 3); | |
| 902 | } | |
| 903 | ||
| 904 | ucol_setText(iter, supplementary, 2, &status); | |
| 905 | sorder = ucol_previous(iter, &status); | |
| 906 | ||
| 907 | if (U_FAILURE(status) || ucol_getMaxExpansion(iter, sorder) != 2) { | |
| 908 | log_err("Failure at codepoint %d, maximum expansion count < %d\n", | |
| 909 | ch, 2); | |
| 910 | } | |
| 911 | ||
| 912 | /* testing jamo */ | |
| 913 | ch = 0x1165; | |
| 914 | ||
| 915 | ucol_setText(iter, &ch, 1, &status); | |
| 916 | temporder = ucol_previous(iter, &status); | |
| 917 | if (U_FAILURE(status) || ucol_getMaxExpansion(iter, temporder) > 3) { | |
| 918 | log_err("Failure at codepoint %d, maximum expansion count > %d\n", | |
| 919 | ch, 3); | |
| 920 | } | |
| 921 | ||
| 922 | ucol_closeElements(iter); | |
| 923 | ucol_close(coll); | |
| 924 | ||
| 925 | /* testing special jamo &a<\u1160 */ | |
| 926 | rule[0] = 0x26; | |
| 927 | rule[1] = 0x71; | |
| 928 | rule[2] = 0x3c; | |
| 929 | rule[3] = 0x1165; | |
| 930 | rule[4] = 0x2f; | |
| 931 | rule[5] = 0x71; | |
| 932 | rule[6] = 0x71; | |
| 933 | rule[7] = 0x71; | |
| 934 | rule[8] = 0x71; | |
| 935 | rule[9] = 0; | |
| 936 | ||
| 937 | coll = ucol_openRules(rule, u_strlen(rule), UCOL_DEFAULT, | |
| 938 | UCOL_DEFAULT_STRENGTH,NULL, &status); | |
| 939 | iter = ucol_openElements(coll, &ch, 1, &status); | |
| 940 | ||
| 941 | temporder = ucol_previous(iter, &status); | |
| 942 | if (U_FAILURE(status) || ucol_getMaxExpansion(iter, temporder) != 6) { | |
| 943 | log_err("Failure at codepoint %d, maximum expansion count > %d\n", | |
| 944 | ch, 5); | |
| 945 | } | |
| 946 | ||
| 947 | ucol_closeElements(iter); | |
| 948 | ucol_close(coll); | |
| 949 | } else { | |
| 950 | log_data_err("Couldn't open collator\n"); | |
| 951 | } | |
| 952 | ||
| 953 | } | |
| 954 | ||
| 955 | /** | |
| 956 | * Return an integer array containing all of the collation orders | |
| 957 | * returned by calls to next on the specified iterator | |
| 958 | */ | |
| 959 | static int32_t* getOrders(UCollationElements *iter, int32_t *orderLength) | |
| 960 | { | |
| 961 | UErrorCode status; | |
| 962 | int32_t order; | |
| 963 | int32_t maxSize = 100; | |
| 964 | int32_t size = 0; | |
| 965 | int32_t *temp; | |
| 966 | int32_t *orders =(int32_t*)malloc(sizeof(int32_t) * maxSize); | |
| 967 | status= U_ZERO_ERROR; | |
| 968 | ||
| 969 | ||
| 970 | while ((order=ucol_next(iter, &status)) != UCOL_NULLORDER) | |
| 971 | { | |
| 972 | if (size == maxSize) | |
| 973 | { | |
| 974 | maxSize *= 2; | |
| 975 | temp = (int32_t*)malloc(sizeof(int32_t) * maxSize); | |
| 976 | ||
| 977 | memcpy(temp, orders, size * sizeof(int32_t)); | |
| 978 | free(orders); | |
| 979 | orders = temp; | |
| 980 | ||
| 981 | } | |
| 982 | ||
| 983 | orders[size++] = order; | |
| 984 | } | |
| 985 | ||
| 986 | if (maxSize > size) | |
| 987 | { | |
| 988 | if (size == 0) { | |
| 989 | size = 1; | |
| 990 | temp = (int32_t*)malloc(sizeof(int32_t) * size); | |
| 991 | temp[0] = 0; | |
| 992 | } | |
| 993 | else { | |
| 994 | temp = (int32_t*)malloc(sizeof(int32_t) * size); | |
| 995 | memcpy(temp, orders, size * sizeof(int32_t)); | |
| 996 | } | |
| 997 | ||
| 998 | free(orders); | |
| 999 | orders = temp; | |
| 1000 | } | |
| 1001 | ||
| 1002 | *orderLength = size; | |
| 1003 | return orders; | |
| 1004 | } | |
| 1005 | ||
| 1006 | ||
| 1007 | static void assertEqual(UCollationElements *i1, UCollationElements *i2) | |
| 1008 | { | |
| 1009 | int32_t c1, c2; | |
| 1010 | int32_t count = 0; | |
| 1011 | UErrorCode status = U_ZERO_ERROR; | |
| 1012 | ||
| 1013 | do | |
| 1014 | { | |
| 1015 | c1 = ucol_next(i1, &status); | |
| 1016 | c2 = ucol_next(i2, &status); | |
| 1017 | ||
| 1018 | if (c1 != c2) | |
| 1019 | { | |
| 1020 | log_err("Error in iteration %d assetEqual between\n %d and %d, they are not equal\n", count, c1, c2); | |
| 1021 | break; | |
| 1022 | } | |
| 1023 | ||
| 1024 | count += 1; | |
| 1025 | } | |
| 1026 | while (c1 != UCOL_NULLORDER); | |
| 1027 | } | |
| 1028 | ||
| 1029 | /** | |
| 1030 | * Testing iterators with extremely small buffers | |
| 1031 | */ | |
| 1032 | static void TestSmallBuffer() | |
| 1033 | { | |
| 1034 | UErrorCode status = U_ZERO_ERROR; | |
| 1035 | UCollator *coll; | |
| 1036 | UCollationElements *testiter, | |
| 1037 | *iter; | |
| 1038 | int32_t count = 0; | |
| 1039 | int32_t *testorders, | |
| 1040 | *orders; | |
| 1041 | ||
| 1042 | UChar teststr[500]; | |
| 1043 | UChar str[] = {0x300, 0x31A, 0}; | |
| 1044 | /* | |
| 1045 | creating a long string of decomposable characters, | |
| 1046 | since by default the writable buffer is of size 256 | |
| 1047 | */ | |
| 1048 | while (count < 500) { | |
| 1049 | if ((count & 1) == 0) { | |
| 1050 | teststr[count ++] = 0x300; | |
| 1051 | } | |
| 1052 | else { | |
| 1053 | teststr[count ++] = 0x31A; | |
| 1054 | } | |
| 1055 | } | |
| 1056 | ||
| 1057 | coll = ucol_open("th_TH", &status); | |
| 1058 | if(U_SUCCESS(status) && coll) { | |
| 1059 | testiter = ucol_openElements(coll, teststr, 500, &status); | |
| 1060 | iter = ucol_openElements(coll, str, 2, &status); | |
| 1061 | ||
| 1062 | orders = getOrders(iter, &count); | |
| 1063 | if (count != 2) { | |
| 1064 | log_err("Error collation elements size is not 2 for \\u0300\\u031A\n"); | |
| 1065 | } | |
| 1066 | ||
| 1067 | /* | |
| 1068 | this will rearrange the string data to 250 characters of 0x300 first then | |
| 1069 | 250 characters of 0x031A | |
| 1070 | */ | |
| 1071 | testorders = getOrders(testiter, &count); | |
| 1072 | ||
| 1073 | if (count != 500) { | |
| 1074 | log_err("Error decomposition does not give the right sized collation elements\n"); | |
| 1075 | } | |
| 1076 | ||
| 1077 | while (count != 0) { | |
| 1078 | /* UCA collation element for 0x0F76 */ | |
| 1079 | if ((count > 250 && testorders[-- count] != orders[1]) || | |
| 1080 | (count <= 250 && testorders[-- count] != orders[0])) { | |
| 1081 | log_err("Error decomposition does not give the right collation element at %d count\n", count); | |
| 1082 | break; | |
| 1083 | } | |
| 1084 | } | |
| 1085 | ||
| 1086 | free(testorders); | |
| 1087 | free(orders); | |
| 1088 | ||
| 1089 | ucol_reset(testiter); | |
| 1090 | /* ensures that the writable buffer was cleared */ | |
| 1091 | if (testiter->iteratordata_.writableBuffer != | |
| 1092 | testiter->iteratordata_.stackWritableBuffer) { | |
| 1093 | log_err("Error Writable buffer in collation element iterator not reset\n"); | |
| 1094 | } | |
| 1095 | ||
| 1096 | /* ensures closing of elements done properly to clear writable buffer */ | |
| 1097 | ucol_next(testiter, &status); | |
| 1098 | ucol_next(testiter, &status); | |
| 1099 | ucol_closeElements(testiter); | |
| 1100 | ucol_closeElements(iter); | |
| 1101 | ucol_close(coll); | |
| 1102 | } else { | |
| 1103 | log_data_err("Couldn't open collator\n"); | |
| 1104 | } | |
| 1105 | } | |
| 1106 | ||
| 1107 | /** | |
| 1108 | * Sniplets of code from genuca | |
| 1109 | */ | |
| 1110 | static int32_t hex2num(char hex) { | |
| 1111 | if(hex>='0' && hex <='9') { | |
| 1112 | return hex-'0'; | |
| 1113 | } else if(hex>='a' && hex<='f') { | |
| 1114 | return hex-'a'+10; | |
| 1115 | } else if(hex>='A' && hex<='F') { | |
| 1116 | return hex-'A'+10; | |
| 1117 | } else { | |
| 1118 | return 0; | |
| 1119 | } | |
| 1120 | } | |
| 1121 | ||
| 1122 | /** | |
| 1123 | * Getting codepoints from a string | |
| 1124 | * @param str character string contain codepoints seperated by space and ended | |
| 1125 | * by a semicolon | |
| 1126 | * @param codepoints array for storage, assuming size > 5 | |
| 1127 | * @return position at the end of the codepoint section | |
| 1128 | */ | |
| 1129 | static char * getCodePoints(char *str, UChar *codepoints) { | |
| 1130 | char *pStartCP = str; | |
| 1131 | char *pEndCP = str + 4; | |
| 1132 | ||
| 1133 | *codepoints = (UChar)((hex2num(*pStartCP) << 12) | | |
| 1134 | (hex2num(*(pStartCP + 1)) << 8) | | |
| 1135 | (hex2num(*(pStartCP + 2)) << 4) | | |
| 1136 | (hex2num(*(pStartCP + 3)))); | |
| 1137 | codepoints ++; | |
| 1138 | while (*pEndCP != ';') { | |
| 1139 | pStartCP = pEndCP + 1; | |
| 1140 | *codepoints = (UChar)((hex2num(*pStartCP) << 12) | | |
| 1141 | (hex2num(*(pStartCP + 1)) << 8) | | |
| 1142 | (hex2num(*(pStartCP + 2)) << 4) | | |
| 1143 | (hex2num(*(pStartCP + 3)))); | |
| 1144 | codepoints ++; | |
| 1145 | pEndCP = pStartCP + 4; | |
| 1146 | } | |
| 1147 | *codepoints = 0; | |
| 1148 | return pEndCP + 1; | |
| 1149 | } | |
| 1150 | ||
| 1151 | /** | |
| 1152 | * Sniplets of code from genuca | |
| 1153 | */ | |
| 1154 | static int32_t | |
| 1155 | readElement(char **from, char *to, char separator, UErrorCode *status) | |
| 1156 | { | |
| 1157 | if (U_SUCCESS(*status)) { | |
| 1158 | char buffer[1024]; | |
| 1159 | int32_t i = 0; | |
| 1160 | while (**from != separator) { | |
| 1161 | if (**from != ' ') { | |
| 1162 | *(buffer+i++) = **from; | |
| 1163 | } | |
| 1164 | (*from)++; | |
| 1165 | } | |
| 1166 | (*from)++; | |
| 1167 | *(buffer + i) = 0; | |
| 1168 | strcpy(to, buffer); | |
| 1169 | return i/2; | |
| 1170 | } | |
| 1171 | ||
| 1172 | return 0; | |
| 1173 | } | |
| 1174 | ||
| 1175 | /** | |
| 1176 | * Sniplets of code from genuca | |
| 1177 | */ | |
| 1178 | static uint32_t | |
| 1179 | getSingleCEValue(char *primary, char *secondary, char *tertiary, | |
| 1180 | UErrorCode *status) | |
| 1181 | { | |
| 1182 | if (U_SUCCESS(*status)) { | |
| 1183 | uint32_t value = 0; | |
| 1184 | char primsave = '\0'; | |
| 1185 | char secsave = '\0'; | |
| 1186 | char tersave = '\0'; | |
| 1187 | char *primend = primary+4; | |
| 1188 | char *secend = secondary+2; | |
| 1189 | char *terend = tertiary+2; | |
| 1190 | uint32_t primvalue; | |
| 1191 | uint32_t secvalue; | |
| 1192 | uint32_t tervalue; | |
| 1193 | ||
| 1194 | if (uprv_strlen(primary) > 4) { | |
| 1195 | primsave = *primend; | |
| 1196 | *primend = '\0'; | |
| 1197 | } | |
| 1198 | ||
| 1199 | if (uprv_strlen(secondary) > 2) { | |
| 1200 | secsave = *secend; | |
| 1201 | *secend = '\0'; | |
| 1202 | } | |
| 1203 | ||
| 1204 | if (uprv_strlen(tertiary) > 2) { | |
| 1205 | tersave = *terend; | |
| 1206 | *terend = '\0'; | |
| 1207 | } | |
| 1208 | ||
| 1209 | primvalue = (*primary!='\0')?uprv_strtoul(primary, &primend, 16):0; | |
| 1210 | secvalue = (*secondary!='\0')?uprv_strtoul(secondary, &secend, 16):0; | |
| 1211 | tervalue = (*tertiary!='\0')?uprv_strtoul(tertiary, &terend, 16):0; | |
| 1212 | if(primvalue <= 0xFF) { | |
| 1213 | primvalue <<= 8; | |
| 1214 | } | |
| 1215 | ||
| 1216 | value = ((primvalue << UCOL_PRIMARYORDERSHIFT) & UCOL_PRIMARYORDERMASK) | |
| 1217 | | ((secvalue << UCOL_SECONDARYORDERSHIFT) & UCOL_SECONDARYORDERMASK) | |
| 1218 | | (tervalue & UCOL_TERTIARYORDERMASK); | |
| 1219 | ||
| 1220 | if(primsave!='\0') { | |
| 1221 | *primend = primsave; | |
| 1222 | } | |
| 1223 | if(secsave!='\0') { | |
| 1224 | *secend = secsave; | |
| 1225 | } | |
| 1226 | if(tersave!='\0') { | |
| 1227 | *terend = tersave; | |
| 1228 | } | |
| 1229 | return value; | |
| 1230 | } | |
| 1231 | return 0; | |
| 1232 | } | |
| 1233 | ||
| 1234 | /** | |
| 1235 | * Getting collation elements generated from a string | |
| 1236 | * @param str character string contain collation elements contained in [] and | |
| 1237 | * seperated by space | |
| 1238 | * @param ce array for storage, assuming size > 20 | |
| 1239 | * @param status error status | |
| 1240 | * @return position at the end of the codepoint section | |
| 1241 | */ | |
| 1242 | static char * getCEs(char *str, uint32_t *ces, UErrorCode *status) { | |
| 1243 | char *pStartCP = uprv_strchr(str, '['); | |
| 1244 | int count = 0; | |
| 1245 | char *pEndCP; | |
| 1246 | char primary[100]; | |
| 1247 | char secondary[100]; | |
| 1248 | char tertiary[100]; | |
| 1249 | ||
| 1250 | while (*pStartCP == '[') { | |
| 1251 | uint32_t primarycount = 0; | |
| 1252 | uint32_t secondarycount = 0; | |
| 1253 | uint32_t tertiarycount = 0; | |
| 1254 | uint32_t CEi = 1; | |
| 1255 | pEndCP = strchr(pStartCP, ']'); | |
| 1256 | if(pEndCP == NULL) { | |
| 1257 | break; | |
| 1258 | } | |
| 1259 | pStartCP ++; | |
| 1260 | ||
| 1261 | primarycount = readElement(&pStartCP, primary, ',', status); | |
| 1262 | secondarycount = readElement(&pStartCP, secondary, ',', status); | |
| 1263 | tertiarycount = readElement(&pStartCP, tertiary, ']', status); | |
| 1264 | ||
| 1265 | /* I want to get the CEs entered right here, including continuation */ | |
| 1266 | ces[count ++] = getSingleCEValue(primary, secondary, tertiary, status); | |
| 1267 | if (U_FAILURE(*status)) { | |
| 1268 | break; | |
| 1269 | } | |
| 1270 | ||
| 1271 | while (2 * CEi < primarycount || CEi < secondarycount || | |
| 1272 | CEi < tertiarycount) { | |
| 1273 | uint32_t value = UCOL_CONTINUATION_MARKER; /* Continuation marker */ | |
| 1274 | if (2 * CEi < primarycount) { | |
| 1275 | value |= ((hex2num(*(primary + 4 * CEi)) & 0xF) << 28); | |
| 1276 | value |= ((hex2num(*(primary + 4 * CEi + 1)) & 0xF) << 24); | |
| 1277 | } | |
| 1278 | ||
| 1279 | if (2 * CEi + 1 < primarycount) { | |
| 1280 | value |= ((hex2num(*(primary + 4 * CEi + 2)) & 0xF) << 20); | |
| 1281 | value |= ((hex2num(*(primary + 4 * CEi + 3)) &0xF) << 16); | |
| 1282 | } | |
| 1283 | ||
| 1284 | if (CEi < secondarycount) { | |
| 1285 | value |= ((hex2num(*(secondary + 2 * CEi)) & 0xF) << 12); | |
| 1286 | value |= ((hex2num(*(secondary + 2 * CEi + 1)) & 0xF) << 8); | |
| 1287 | } | |
| 1288 | ||
| 1289 | if (CEi < tertiarycount) { | |
| 1290 | value |= ((hex2num(*(tertiary + 2 * CEi)) & 0x3) << 4); | |
| 1291 | value |= (hex2num(*(tertiary + 2 * CEi + 1)) & 0xF); | |
| 1292 | } | |
| 1293 | ||
| 1294 | CEi ++; | |
| 1295 | ces[count ++] = value; | |
| 1296 | } | |
| 1297 | ||
| 1298 | pStartCP = pEndCP + 1; | |
| 1299 | } | |
| 1300 | ces[count] = 0; | |
| 1301 | return pStartCP; | |
| 1302 | } | |
| 1303 | ||
| 1304 | /** | |
| 1305 | * Getting the FractionalUCA.txt file stream | |
| 1306 | */ | |
| 1307 | static FileStream * getFractionalUCA(void) | |
| 1308 | { | |
| 1309 | char newPath[256]; | |
| 1310 | char backupPath[256]; | |
| 1311 | FileStream *result = NULL; | |
| 1312 | ||
| 1313 | /* Look inside ICU_DATA first */ | |
| 1314 | uprv_strcpy(newPath, u_getDataDirectory()); | |
| 1315 | uprv_strcat(newPath, "unidata" U_FILE_SEP_STRING ); | |
| 1316 | uprv_strcat(newPath, "FractionalUCA.txt"); | |
| 1317 | ||
| 1318 | /* As a fallback, try to guess where the source data was located | |
| 1319 | * at the time ICU was built, and look there. | |
| 1320 | */ | |
| 1321 | #if defined (U_TOPSRCDIR) | |
| 1322 | strcpy(backupPath, U_TOPSRCDIR U_FILE_SEP_STRING "data"); | |
| 1323 | #else | |
| 1324 | { | |
| 1325 | UErrorCode errorCode = U_ZERO_ERROR; | |
| 1326 | strcpy(backupPath, loadTestData(&errorCode)); | |
| 1327 | strcat(backupPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING "data"); | |
| 1328 | } | |
| 1329 | #endif | |
| 1330 | strcat(backupPath, U_FILE_SEP_STRING "unidata" U_FILE_SEP_STRING "FractionalUCA.txt"); | |
| 1331 | ||
| 1332 | result = T_FileStream_open(newPath, "rb"); | |
| 1333 | ||
| 1334 | if (result == NULL) { | |
| 1335 | result = T_FileStream_open(backupPath, "rb"); | |
| 1336 | if (result == NULL) { | |
| 1337 | log_err("Failed to open either %s or %s\n", newPath, backupPath); | |
| 1338 | } | |
| 1339 | } | |
| 1340 | return result; | |
| 1341 | } | |
| 1342 | ||
| 1343 | /** | |
| 1344 | * Testing the CEs returned by the iterator | |
| 1345 | */ | |
| 1346 | static void TestCEs() { | |
| 1347 | FileStream *file = NULL; | |
| 1348 | char line[1024]; | |
| 1349 | char *str; | |
| 1350 | UChar codepoints[5]; | |
| 1351 | uint32_t ces[20]; | |
| 1352 | UErrorCode status = U_ZERO_ERROR; | |
| 1353 | UCollator *coll = ucol_open("", &status); | |
| 1354 | uint32_t lineNo = 0; | |
| 1355 | ||
| 1356 | if (U_FAILURE(status)) { | |
| 1357 | log_err("Error in opening root collator\n"); | |
| 1358 | return; | |
| 1359 | } | |
| 1360 | ||
| 1361 | file = getFractionalUCA(); | |
| 1362 | ||
| 1363 | if (file == NULL) { | |
| 1364 | log_err("*** unable to open input FractionalUCA.txt file ***\n"); | |
| 1365 | return; | |
| 1366 | } | |
| 1367 | ||
| 1368 | ||
| 1369 | while (T_FileStream_readLine(file, line, sizeof(line)) != NULL) { | |
| 1370 | int count = 0; | |
| 1371 | UCollationElements *iter; | |
| 1372 | lineNo++; | |
| 1373 | /* skip this line if it is empty or a comment or is a return value | |
| 1374 | or start of some variable section */ | |
| 1375 | if(line[0] == 0 || line[0] == '#' || line[0] == '\n' || | |
| 1376 | line[0] == 0x000D || line[0] == '[') { | |
| 1377 | continue; | |
| 1378 | } | |
| 1379 | ||
| 1380 | str = getCodePoints(line, codepoints); | |
| 1381 | ||
| 1382 | /* these are 'fake' codepoints in the fractional UCA, and are used just | |
| 1383 | * for positioning of indirect values. They should not go through this | |
| 1384 | * test. | |
| 1385 | */ | |
| 1386 | if(*codepoints == 0xFDD0) { | |
| 1387 | continue; | |
| 1388 | } | |
| 1389 | ||
| 1390 | getCEs(str, ces, &status); | |
| 1391 | if (U_FAILURE(status)) { | |
| 1392 | log_err("Error in parsing collation elements in FractionalUCA.txt\n"); | |
| 1393 | break; | |
| 1394 | } | |
| 1395 | iter = ucol_openElements(coll, codepoints, -1, &status); | |
| 1396 | if (U_FAILURE(status)) { | |
| 1397 | log_err("Error in opening collation elements\n"); | |
| 1398 | break; | |
| 1399 | } | |
| 1400 | for (;;) { | |
| 1401 | uint32_t ce = (uint32_t)ucol_next(iter, &status); | |
| 1402 | if (ce == 0xFFFFFFFF) { | |
| 1403 | ce = 0; | |
| 1404 | } | |
| 1405 | /* we now unconditionally reorder Thai/Lao prevowels, so this | |
| 1406 | * test would fail if we don't skip here. | |
| 1407 | */ | |
| 1408 | if(UCOL_ISTHAIPREVOWEL(*codepoints) && ce == 0 && count == 0) { | |
| 1409 | continue; | |
| 1410 | } | |
| 1411 | if (ce != ces[count] || U_FAILURE(status)) { | |
| 1412 | log_err("Collation elements in FractionalUCA.txt and iterators do not match!\n"); | |
| 1413 | break; | |
| 1414 | } | |
| 1415 | if (ces[count] == 0) { | |
| 1416 | break; | |
| 1417 | } | |
| 1418 | count ++; | |
| 1419 | } | |
| 1420 | ucol_closeElements(iter); | |
| 1421 | } | |
| 1422 | ||
| 1423 | T_FileStream_close(file); | |
| 1424 | ucol_close(coll); | |
| 1425 | } | |
| 1426 | ||
| 1427 | /** | |
| 1428 | * Testing the discontigous contractions | |
| 1429 | */ | |
| 1430 | static void TestDiscontiguos() { | |
| 1431 | const char *rulestr = | |
| 1432 | "&z < AB < X\\u0300 < ABC < X\\u0300\\u0315"; | |
| 1433 | UChar rule[50]; | |
| 1434 | int rulelen = u_unescape(rulestr, rule, 50); | |
| 1435 | const char *src[] = { | |
| 1436 | "ADB", "ADBC", "A\\u0315B", "A\\u0315BC", | |
| 1437 | /* base character blocked */ | |
| 1438 | "XD\\u0300", "XD\\u0300\\u0315", | |
| 1439 | /* non blocking combining character */ | |
| 1440 | "X\\u0319\\u0300", "X\\u0319\\u0300\\u0315", | |
| 1441 | /* blocking combining character */ | |
| 1442 | "X\\u0314\\u0300", "X\\u0314\\u0300\\u0315", | |
| 1443 | /* contraction prefix */ | |
| 1444 | "ABDC", "AB\\u0315C","X\\u0300D\\u0315", "X\\u0300\\u0319\\u0315", | |
| 1445 | "X\\u0300\\u031A\\u0315", | |
| 1446 | /* ends not with a contraction character */ | |
| 1447 | "X\\u0319\\u0300D", "X\\u0319\\u0300\\u0315D", "X\\u0300D\\u0315D", | |
| 1448 | "X\\u0300\\u0319\\u0315D", "X\\u0300\\u031A\\u0315D" | |
| 1449 | }; | |
| 1450 | const char *tgt[] = { | |
| 1451 | /* non blocking combining character */ | |
| 1452 | "A D B", "A D BC", "A \\u0315 B", "A \\u0315 BC", | |
| 1453 | /* base character blocked */ | |
| 1454 | "X D \\u0300", "X D \\u0300\\u0315", | |
| 1455 | /* non blocking combining character */ | |
| 1456 | "X\\u0300 \\u0319", "X\\u0300\\u0315 \\u0319", | |
| 1457 | /* blocking combining character */ | |
| 1458 | "X \\u0314 \\u0300", "X \\u0314 \\u0300\\u0315", | |
| 1459 | /* contraction prefix */ | |
| 1460 | "AB DC", "AB \\u0315 C","X\\u0300 D \\u0315", "X\\u0300\\u0315 \\u0319", | |
| 1461 | "X\\u0300 \\u031A \\u0315", | |
| 1462 | /* ends not with a contraction character */ | |
| 1463 | "X\\u0300 \\u0319D", "X\\u0300\\u0315 \\u0319D", "X\\u0300 D\\u0315D", | |
| 1464 | "X\\u0300\\u0315 \\u0319D", "X\\u0300 \\u031A\\u0315D" | |
| 1465 | }; | |
| 1466 | int size = 20; | |
| 1467 | UCollator *coll; | |
| 1468 | UErrorCode status = U_ZERO_ERROR; | |
| 1469 | int count = 0; | |
| 1470 | UCollationElements *iter; | |
| 1471 | UCollationElements *resultiter; | |
| 1472 | ||
| 1473 | coll = ucol_openRules(rule, rulelen, UCOL_OFF, UCOL_DEFAULT_STRENGTH,NULL, &status); | |
| 1474 | iter = ucol_openElements(coll, rule, 1, &status); | |
| 1475 | resultiter = ucol_openElements(coll, rule, 1, &status); | |
| 1476 | ||
| 1477 | if (U_FAILURE(status)) { | |
| 1478 | log_err("Error opening collation rules\n"); | |
| 1479 | return; | |
| 1480 | } | |
| 1481 | ||
| 1482 | while (count < size) { | |
| 1483 | UChar str[20]; | |
| 1484 | UChar tstr[20]; | |
| 1485 | int strLen = u_unescape(src[count], str, 20); | |
| 1486 | UChar *s; | |
| 1487 | ||
| 1488 | ucol_setText(iter, str, strLen, &status); | |
| 1489 | if (U_FAILURE(status)) { | |
| 1490 | log_err("Error opening collation iterator\n"); | |
| 1491 | return; | |
| 1492 | } | |
| 1493 | ||
| 1494 | u_unescape(tgt[count], tstr, 20); | |
| 1495 | s = tstr; | |
| 1496 | ||
| 1497 | log_verbose("count %d\n", count); | |
| 1498 | ||
| 1499 | for (;;) { | |
| 1500 | uint32_t ce; | |
| 1501 | UChar *e = u_strchr(s, 0x20); | |
| 1502 | if (e == 0) { | |
| 1503 | e = u_strchr(s, 0); | |
| 1504 | } | |
| 1505 | ucol_setText(resultiter, s, (int32_t)(e - s), &status); | |
| 1506 | ce = ucol_next(resultiter, &status); | |
| 1507 | if (U_FAILURE(status)) { | |
| 1508 | log_err("Error manipulating collation iterator\n"); | |
| 1509 | return; | |
| 1510 | } | |
| 1511 | while (ce != UCOL_NULLORDER) { | |
| 1512 | if (ce != (uint32_t)ucol_next(iter, &status) || | |
| 1513 | U_FAILURE(status)) { | |
| 1514 | log_err("Discontiguos contraction test mismatch\n"); | |
| 1515 | return; | |
| 1516 | } | |
| 1517 | ce = ucol_next(resultiter, &status); | |
| 1518 | if (U_FAILURE(status)) { | |
| 1519 | log_err("Error getting next collation element\n"); | |
| 1520 | return; | |
| 1521 | } | |
| 1522 | } | |
| 1523 | s = e + 1; | |
| 1524 | if (*e == 0) { | |
| 1525 | break; | |
| 1526 | } | |
| 1527 | } | |
| 1528 | ucol_reset(iter); | |
| 1529 | backAndForth(iter); | |
| 1530 | count ++; | |
| 1531 | } | |
| 1532 | ucol_closeElements(resultiter); | |
| 1533 | ucol_closeElements(iter); | |
| 1534 | ucol_close(coll); | |
| 1535 | } | |
| 1536 | ||
| 1537 | static void TestCEBufferOverflow() | |
| 1538 | { | |
| 1539 | UChar str[UCOL_EXPAND_CE_BUFFER_SIZE + 1]; | |
| 1540 | UErrorCode status = U_ZERO_ERROR; | |
| 1541 | UChar rule[10]; | |
| 1542 | UCollator *coll; | |
| 1543 | UCollationElements *iter; | |
| 1544 | ||
| 1545 | u_uastrcpy(rule, "&z < AB"); | |
| 1546 | coll = ucol_openRules(rule, u_strlen(rule), UCOL_OFF, UCOL_DEFAULT_STRENGTH, NULL,&status); | |
| 1547 | if (U_FAILURE(status)) { | |
| 1548 | log_err("Rule based collator not created for testing ce buffer overflow\n"); | |
| 1549 | return; | |
| 1550 | } | |
| 1551 | ||
| 1552 | /* 0xDCDC is a trail surrogate hence deemed unsafe by the heuristic | |
| 1553 | test. this will cause an overflow in getPrev */ | |
| 1554 | str[0] = 0x0041; /* 'A' */ | |
| 1555 | /*uprv_memset(str + 1, 0xE0, sizeof(UChar) * UCOL_EXPAND_CE_BUFFER_SIZE);*/ | |
| 1556 | uprv_memset(str + 1, 0xDC, sizeof(UChar) * UCOL_EXPAND_CE_BUFFER_SIZE); | |
| 1557 | str[UCOL_EXPAND_CE_BUFFER_SIZE] = 0x0042; /* 'B' */ | |
| 1558 | iter = ucol_openElements(coll, str, UCOL_EXPAND_CE_BUFFER_SIZE + 1, | |
| 1559 | &status); | |
| 1560 | if (ucol_previous(iter, &status) != UCOL_NULLORDER || | |
| 1561 | status != U_BUFFER_OVERFLOW_ERROR) { | |
| 1562 | log_err("CE buffer expected to overflow with long string of trail surrogates\n"); | |
| 1563 | } | |
| 1564 | ucol_closeElements(iter); | |
| 1565 | ucol_close(coll); | |
| 1566 | } | |
| 1567 | ||
| 1568 | /** | |
| 1569 | * Byte bounds checks. Checks if each byte in data is between upper and lower | |
| 1570 | * inclusive. | |
| 1571 | */ | |
| 1572 | static UBool checkByteBounds(uint32_t data, char upper, char lower) | |
| 1573 | { | |
| 1574 | int count = 4; | |
| 1575 | while (count > 0) { | |
| 1576 | char b = (char)(data & 0xFF); | |
| 1577 | if (b > upper || b < lower) { | |
| 1578 | return FALSE; | |
| 1579 | } | |
| 1580 | data = data >> 8; | |
| 1581 | count --; | |
| 1582 | } | |
| 1583 | return TRUE; | |
| 1584 | } | |
| 1585 | ||
| 1586 | /** | |
| 1587 | * Determines case of the string of codepoints. | |
| 1588 | * If it is a multiple codepoints it has to treated as a contraction. | |
| 1589 | */ | |
| 1590 | #if 0 | |
| 1591 | static uint8_t getCase(const UChar *s, uint32_t len) { | |
| 1592 | UBool lower = FALSE; | |
| 1593 | UBool upper = FALSE; | |
| 1594 | UBool title = FALSE; | |
| 1595 | UErrorCode status = U_ZERO_ERROR; | |
| 1596 | UChar str[256]; | |
| 1597 | const UChar *ps = s; | |
| 1598 | ||
| 1599 | if (len == 0) { | |
| 1600 | return UCOL_LOWER_CASE; | |
| 1601 | } | |
| 1602 | ||
| 1603 | while (len > 0) { | |
| 1604 | UChar c = *ps ++; | |
| 1605 | ||
| 1606 | if (u_islower(c)) { | |
| 1607 | lower = TRUE; | |
| 1608 | } | |
| 1609 | if (u_isupper(c)) { | |
| 1610 | upper = TRUE; | |
| 1611 | } | |
| 1612 | if (u_istitle(c)) { | |
| 1613 | title = TRUE; | |
| 1614 | } | |
| 1615 | ||
| 1616 | len --; | |
| 1617 | } | |
| 1618 | if ((lower && !upper && !title) || (!lower && !upper && !title)){ | |
| 1619 | return UCOL_LOWER_CASE; | |
| 1620 | } | |
| 1621 | if (upper && !lower && !title) { | |
| 1622 | return UCOL_UPPER_CASE; | |
| 1623 | } | |
| 1624 | /* mix of cases here */ | |
| 1625 | /* len = unorm_normalize(s, len, UNORM_NFKD, 0, str, 256, &status); | |
| 1626 | if (U_FAILURE(status)) { | |
| 1627 | log_err("Error normalizing data string\n"); | |
| 1628 | return UCOL_LOWER_CASE; | |
| 1629 | }*/ | |
| 1630 | ||
| 1631 | if ((title && len >= 2) || (lower && upper)) { | |
| 1632 | return UCOL_MIXED_CASE; | |
| 1633 | } | |
| 1634 | if (u_isupper(s[0])) { | |
| 1635 | return UCOL_UPPER_CASE; | |
| 1636 | } | |
| 1637 | return UCOL_LOWER_CASE; | |
| 1638 | } | |
| 1639 | #endif | |
| 1640 | ||
| 1641 | /** | |
| 1642 | * Checking collation element validity given the boundary arguments. | |
| 1643 | */ | |
| 1644 | static UBool checkCEValidity(const UCollator *coll, const UChar *codepoints, | |
| 1645 | int length, uint32_t primarymax, | |
| 1646 | uint32_t secondarymax) | |
| 1647 | { | |
| 1648 | UErrorCode status = U_ZERO_ERROR; | |
| 1649 | UCollationElements *iter = ucol_openElements(coll, codepoints, length, | |
| 1650 | &status); | |
| 1651 | uint32_t ce; | |
| 1652 | UBool first = TRUE; | |
| 1653 | /* | |
| 1654 | UBool upper = FALSE; | |
| 1655 | UBool lower = FALSE; | |
| 1656 | */ | |
| 1657 | ||
| 1658 | if (U_FAILURE(status)) { | |
| 1659 | log_err("Error creating iterator for testing validity\n"); | |
| 1660 | } | |
| 1661 | ||
| 1662 | ce = ucol_next(iter, &status); | |
| 1663 | ||
| 1664 | while (ce != UCOL_NULLORDER) { | |
| 1665 | if (ce != 0) { | |
| 1666 | uint32_t primary = UCOL_PRIMARYORDER(ce); | |
| 1667 | uint32_t secondary = UCOL_SECONDARYORDER(ce); | |
| 1668 | uint32_t tertiary = UCOL_TERTIARYORDER(ce); | |
| 1669 | /* uint32_t scasebits = tertiary & 0xC0;*/ | |
| 1670 | ||
| 1671 | if ((tertiary == 0 && secondary != 0) || | |
| 1672 | (tertiary < 0xC0 && secondary == 0 && primary != 0)) { | |
| 1673 | /* n-1th level is not zero when the nth level is | |
| 1674 | except for continuations, this is wrong */ | |
| 1675 | log_err("Lower level weight not 0 when high level weight is 0\n"); | |
| 1676 | goto fail; | |
| 1677 | } | |
| 1678 | else { | |
| 1679 | /* checks if any byte is illegal ie = 01 02 03. */ | |
| 1680 | if (checkByteBounds(ce, 0x3, 0x1)) { | |
| 1681 | log_err("Byte range in CE lies in illegal bounds 0x1 - 0x3\n"); | |
| 1682 | goto fail; | |
| 1683 | } | |
| 1684 | } | |
| 1685 | if ((primary != 0 && primary < primarymax) || (primary >= 0xFF00 && !isContinuation(ce))) { | |
| 1686 | log_err("UCA primary weight out of bounds\n"); | |
| 1687 | goto fail; | |
| 1688 | } | |
| 1689 | /* case matching not done since data generated by ken */ | |
| 1690 | if (first) { | |
| 1691 | if (secondary >= 6 && secondary <= secondarymax) { | |
| 1692 | log_err("Secondary weight out of range\n"); | |
| 1693 | goto fail; | |
| 1694 | } | |
| 1695 | first = FALSE; | |
| 1696 | } | |
| 1697 | } | |
| 1698 | ce = ucol_next(iter, &status); | |
| 1699 | } | |
| 1700 | ucol_closeElements(iter); | |
| 1701 | return TRUE; | |
| 1702 | fail : | |
| 1703 | ucol_closeElements(iter); | |
| 1704 | return FALSE; | |
| 1705 | } | |
| 1706 | ||
| 1707 | static void TestCEValidity() | |
| 1708 | { | |
| 1709 | /* testing UCA collation elements */ | |
| 1710 | UErrorCode status = U_ZERO_ERROR; | |
| 1711 | /* en_US has no tailorings */ | |
| 1712 | UCollator *coll = ucol_open("en_US", &status); | |
| 1713 | /* tailored locales */ | |
| 1714 | char locale[][6] = {"fr_FR", "ko_KR", "sh_YU", "th_TH", "zh_CN"}; | |
| 1715 | FileStream *file = getFractionalUCA(); | |
| 1716 | char line[1024]; | |
| 1717 | UChar codepoints[10]; | |
| 1718 | int count = 0; | |
| 1719 | UParseError parseError; | |
| 1720 | if (U_FAILURE(status)) { | |
| 1721 | log_err("en_US collator creation failed\n"); | |
| 1722 | return; | |
| 1723 | } | |
| 1724 | log_verbose("Testing UCA elements\n"); | |
| 1725 | if (file == NULL) { | |
| 1726 | log_err("Fractional UCA data can not be opened\n"); | |
| 1727 | return; | |
| 1728 | } | |
| 1729 | ||
| 1730 | while (T_FileStream_readLine(file, line, sizeof(line)) != NULL) { | |
| 1731 | if(line[0] == 0 || line[0] == '#' || line[0] == '\n' || | |
| 1732 | line[0] == 0x000D || line[0] == '[') { | |
| 1733 | continue; | |
| 1734 | } | |
| 1735 | ||
| 1736 | getCodePoints(line, codepoints); | |
| 1737 | checkCEValidity(coll, codepoints, u_strlen(codepoints), 5, 86); | |
| 1738 | } | |
| 1739 | ||
| 1740 | log_verbose("Testing UCA elements for the whole range of unicode characters\n"); | |
| 1741 | codepoints[0] = 0; | |
| 1742 | while (codepoints[0] < 0xFFFF) { | |
| 1743 | if (u_isdefined((UChar32)codepoints[0])) { | |
| 1744 | checkCEValidity(coll, codepoints, 1, 5, 86); | |
| 1745 | } | |
| 1746 | codepoints[0] ++; | |
| 1747 | } | |
| 1748 | ||
| 1749 | ucol_close(coll); | |
| 1750 | ||
| 1751 | /* testing tailored collation elements */ | |
| 1752 | log_verbose("Testing tailored elements\n"); | |
| 1753 | while (count < 5) { | |
| 1754 | const UChar *rules = NULL, | |
| 1755 | *current = NULL; | |
| 1756 | UChar *rulesCopy = NULL; | |
| 1757 | int32_t ruleLen = 0; | |
| 1758 | ||
| 1759 | uint32_t chOffset = 0; | |
| 1760 | uint32_t chLen = 0; | |
| 1761 | uint32_t exOffset = 0; | |
| 1762 | uint32_t exLen = 0; | |
| 1763 | uint32_t prefixOffset = 0; | |
| 1764 | uint32_t prefixLen = 0; | |
| 1765 | UBool startOfRules = TRUE; | |
| 1766 | UColOptionSet opts; | |
| 1767 | ||
| 1768 | UColTokenParser src; | |
| 1769 | uint32_t strength = 0; | |
| 1770 | uint16_t specs = 0; | |
| 1771 | ||
| 1772 | coll = ucol_open(locale[count], &status); | |
| 1773 | if (U_FAILURE(status)) { | |
| 1774 | log_err("%s collator creation failed\n", locale[count]); | |
| 1775 | return; | |
| 1776 | } | |
| 1777 | ||
| 1778 | src.opts = &opts; | |
| 1779 | rules = ucol_getRules(coll, &ruleLen); | |
| 1780 | ||
| 1781 | if (ruleLen > 0) { | |
| 1782 | rulesCopy = (UChar *)malloc((ruleLen + | |
| 1783 | UCOL_TOK_EXTRA_RULE_SPACE_SIZE) * sizeof(UChar)); | |
| 1784 | uprv_memcpy(rulesCopy, rules, ruleLen * sizeof(UChar)); | |
| 1785 | src.current = src.source = rulesCopy; | |
| 1786 | src.end = rulesCopy + ruleLen; | |
| 1787 | src.extraCurrent = src.end; | |
| 1788 | src.extraEnd = src.end + UCOL_TOK_EXTRA_RULE_SPACE_SIZE; | |
| 1789 | ||
| 1790 | while ((current = ucol_tok_parseNextToken(&src, startOfRules, &parseError,&status)) != NULL) { | |
| 1791 | strength = src.parsedToken.strength; | |
| 1792 | chOffset = src.parsedToken.charsOffset; | |
| 1793 | chLen = src.parsedToken.charsLen; | |
| 1794 | exOffset = src.parsedToken.extensionOffset; | |
| 1795 | exLen = src.parsedToken.extensionLen; | |
| 1796 | prefixOffset = src.parsedToken.prefixOffset; | |
| 1797 | prefixLen = src.parsedToken.prefixLen; | |
| 1798 | specs = src.parsedToken.flags; | |
| 1799 | ||
| 1800 | startOfRules = FALSE; | |
| 1801 | uprv_memcpy(codepoints, src.source + chOffset, | |
| 1802 | chLen * sizeof(UChar)); | |
| 1803 | codepoints[chLen] = 0; | |
| 1804 | checkCEValidity(coll, codepoints, chLen, 4, 85); | |
| 1805 | } | |
| 1806 | free(rulesCopy); | |
| 1807 | } | |
| 1808 | ||
| 1809 | ucol_close(coll); | |
| 1810 | count ++; | |
| 1811 | } | |
| 1812 | T_FileStream_close(file); | |
| 1813 | } | |
| 1814 | ||
| 1815 | static void printSortKeyError(const UChar *codepoints, int length, | |
| 1816 | uint8_t *sortkey, int sklen) | |
| 1817 | { | |
| 1818 | int count = 0; | |
| 1819 | log_err("Sortkey not valid for "); | |
| 1820 | while (length > 0) { | |
| 1821 | log_err("0x%04x ", *codepoints); | |
| 1822 | length --; | |
| 1823 | codepoints ++; | |
| 1824 | } | |
| 1825 | log_err("\nSortkey : "); | |
| 1826 | while (count < sklen) { | |
| 1827 | log_err("0x%02x ", sortkey[count]); | |
| 1828 | count ++; | |
| 1829 | } | |
| 1830 | log_err("\n"); | |
| 1831 | } | |
| 1832 | ||
| 1833 | /** | |
| 1834 | * Checking sort key validity for all levels | |
| 1835 | */ | |
| 1836 | static UBool checkSortKeyValidity(UCollator *coll, | |
| 1837 | const UChar *codepoints, | |
| 1838 | int length) | |
| 1839 | { | |
| 1840 | UErrorCode status = U_ZERO_ERROR; | |
| 1841 | UCollationStrength strength[5] = {UCOL_PRIMARY, UCOL_SECONDARY, | |
| 1842 | UCOL_TERTIARY, UCOL_QUATERNARY, | |
| 1843 | UCOL_IDENTICAL}; | |
| 1844 | int strengthlen = 5; | |
| 1845 | int index = 0; | |
| 1846 | int caselevel = 0; | |
| 1847 | ||
| 1848 | while (caselevel < 1) { | |
| 1849 | if (caselevel == 0) { | |
| 1850 | ucol_setAttribute(coll, UCOL_CASE_LEVEL, UCOL_OFF, &status); | |
| 1851 | } | |
| 1852 | else { | |
| 1853 | ucol_setAttribute(coll, UCOL_CASE_LEVEL, UCOL_ON, &status); | |
| 1854 | } | |
| 1855 | ||
| 1856 | while (index < strengthlen) { | |
| 1857 | int count01 = 0; | |
| 1858 | uint32_t count = 0; | |
| 1859 | uint8_t sortkey[128]; | |
| 1860 | uint32_t sklen; | |
| 1861 | ||
| 1862 | ucol_setStrength(coll, strength[index]); | |
| 1863 | sklen = ucol_getSortKey(coll, codepoints, length, sortkey, 128); | |
| 1864 | while (sortkey[count] != 0) { | |
| 1865 | if (sortkey[count] == 2 || (sortkey[count] == 3 && count01 > 0 && index != 4)) { | |
| 1866 | printSortKeyError(codepoints, length, sortkey, sklen); | |
| 1867 | return FALSE; | |
| 1868 | } | |
| 1869 | if (sortkey[count] == 1) { | |
| 1870 | count01 ++; | |
| 1871 | } | |
| 1872 | count ++; | |
| 1873 | } | |
| 1874 | ||
| 1875 | if (count + 1 != sklen || (count01 != index + caselevel)) { | |
| 1876 | printSortKeyError(codepoints, length, sortkey, sklen); | |
| 1877 | return FALSE; | |
| 1878 | } | |
| 1879 | index ++; | |
| 1880 | } | |
| 1881 | caselevel ++; | |
| 1882 | } | |
| 1883 | return TRUE; | |
| 1884 | } | |
| 1885 | ||
| 1886 | static void TestSortKeyValidity(void) | |
| 1887 | { | |
| 1888 | /* testing UCA collation elements */ | |
| 1889 | UErrorCode status = U_ZERO_ERROR; | |
| 1890 | /* en_US has no tailorings */ | |
| 1891 | UCollator *coll = ucol_open("en_US", &status); | |
| 1892 | /* tailored locales */ | |
| 1893 | char locale[][6] = {"fr_FR\0", "ko_KR\0", "sh_YU\0", "th_TH\0", "zh_CN\0"}; | |
| 1894 | FileStream *file = getFractionalUCA(); | |
| 1895 | char line[1024]; | |
| 1896 | UChar codepoints[10]; | |
| 1897 | int count = 0; | |
| 1898 | UParseError parseError; | |
| 1899 | if (U_FAILURE(status)) { | |
| 1900 | log_err("en_US collator creation failed\n"); | |
| 1901 | return; | |
| 1902 | } | |
| 1903 | log_verbose("Testing UCA elements\n"); | |
| 1904 | if (file == NULL) { | |
| 1905 | log_err("Fractional UCA data can not be opened\n"); | |
| 1906 | return; | |
| 1907 | } | |
| 1908 | ||
| 1909 | while (T_FileStream_readLine(file, line, sizeof(line)) != NULL) { | |
| 1910 | if(line[0] == 0 || line[0] == '#' || line[0] == '\n' || | |
| 1911 | line[0] == 0x000D || line[0] == '[') { | |
| 1912 | continue; | |
| 1913 | } | |
| 1914 | ||
| 1915 | getCodePoints(line, codepoints); | |
| 1916 | checkSortKeyValidity(coll, codepoints, u_strlen(codepoints)); | |
| 1917 | } | |
| 1918 | ||
| 1919 | log_verbose("Testing UCA elements for the whole range of unicode characters\n"); | |
| 1920 | codepoints[0] = 0; | |
| 1921 | ||
| 1922 | while (codepoints[0] < 0xFFFF) { | |
| 1923 | if (u_isdefined((UChar32)codepoints[0])) { | |
| 1924 | checkSortKeyValidity(coll, codepoints, 1); | |
| 1925 | } | |
| 1926 | codepoints[0] ++; | |
| 1927 | } | |
| 1928 | ||
| 1929 | ucol_close(coll); | |
| 1930 | ||
| 1931 | /* testing tailored collation elements */ | |
| 1932 | log_verbose("Testing tailored elements\n"); | |
| 1933 | while (count < 5) { | |
| 1934 | const UChar *rules = NULL, | |
| 1935 | *current = NULL; | |
| 1936 | UChar *rulesCopy = NULL; | |
| 1937 | int32_t ruleLen = 0; | |
| 1938 | ||
| 1939 | uint32_t chOffset = 0; | |
| 1940 | uint32_t chLen = 0; | |
| 1941 | uint32_t exOffset = 0; | |
| 1942 | uint32_t exLen = 0; | |
| 1943 | uint32_t prefixOffset = 0; | |
| 1944 | uint32_t prefixLen = 0; | |
| 1945 | UBool startOfRules = TRUE; | |
| 1946 | UColOptionSet opts; | |
| 1947 | ||
| 1948 | UColTokenParser src; | |
| 1949 | uint32_t strength = 0; | |
| 1950 | uint16_t specs = 0; | |
| 1951 | ||
| 1952 | coll = ucol_open(locale[count], &status); | |
| 1953 | if (U_FAILURE(status)) { | |
| 1954 | log_err("%s collator creation failed\n", locale[count]); | |
| 1955 | return; | |
| 1956 | } | |
| 1957 | ||
| 1958 | src.opts = &opts; | |
| 1959 | rules = ucol_getRules(coll, &ruleLen); | |
| 1960 | ||
| 1961 | if (ruleLen > 0) { | |
| 1962 | rulesCopy = (UChar *)malloc((ruleLen + | |
| 1963 | UCOL_TOK_EXTRA_RULE_SPACE_SIZE) * sizeof(UChar)); | |
| 1964 | uprv_memcpy(rulesCopy, rules, ruleLen * sizeof(UChar)); | |
| 1965 | src.current = src.source = rulesCopy; | |
| 1966 | src.end = rulesCopy + ruleLen; | |
| 1967 | src.extraCurrent = src.end; | |
| 1968 | src.extraEnd = src.end + UCOL_TOK_EXTRA_RULE_SPACE_SIZE; | |
| 1969 | ||
| 1970 | while ((current = ucol_tok_parseNextToken(&src, startOfRules,&parseError, &status)) != NULL) { | |
| 1971 | strength = src.parsedToken.strength; | |
| 1972 | chOffset = src.parsedToken.charsOffset; | |
| 1973 | chLen = src.parsedToken.charsLen; | |
| 1974 | exOffset = src.parsedToken.extensionOffset; | |
| 1975 | exLen = src.parsedToken.extensionLen; | |
| 1976 | prefixOffset = src.parsedToken.prefixOffset; | |
| 1977 | prefixLen = src.parsedToken.prefixLen; | |
| 1978 | specs = src.parsedToken.flags; | |
| 1979 | ||
| 1980 | startOfRules = FALSE; | |
| 1981 | uprv_memcpy(codepoints, src.source + chOffset, | |
| 1982 | chLen * sizeof(UChar)); | |
| 1983 | codepoints[chLen] = 0; | |
| 1984 | checkSortKeyValidity(coll, codepoints, chLen); | |
| 1985 | } | |
| 1986 | free(rulesCopy); | |
| 1987 | } | |
| 1988 | ||
| 1989 | ucol_close(coll); | |
| 1990 | count ++; | |
| 1991 | } | |
| 1992 | T_FileStream_close(file); | |
| 1993 | } | |
| 1994 | ||
| 1995 | #endif /* #if !UCONFIG_NO_COLLATION */ |