icuSources/test/intltest/tscoll.cpp

   1 /********************************************************************
   2  * COPYRIGHT:
   3  * Copyright (c) 1997-2005, International Business Machines Corporation and
   4  * others. All Rights Reserved.
   5  ********************************************************************/
   6
   7 /**
   8  * IntlTestCollator is the medium level test class for everything in the directory "collate".
   9  */
  10
  11 /***********************************************************************
  12 * Modification history
  13 * Date        Name        Description
  14 * 02/14/2001  synwee      Compare with cintltst and commented away tests
  15 *                         that are not run.
  16 ***********************************************************************/
  17
  18 #include "unicode/utypes.h"
  19
  20 #if !UCONFIG_NO_COLLATION
  21
  22 #include "unicode/uchar.h"
  23 #include "unicode/ustring.h"
  24
  25 #include "dadrcoll.h"
  26
  27 #include "encoll.h"
  28 #include "frcoll.h"
  29 #include "decoll.h"
  30 #include "escoll.h"
  31 #include "ficoll.h"
  32 #include "jacoll.h"
  33 #include "trcoll.h"
  34 #include "allcoll.h"
  35 #include "g7coll.h"
  36 #include "mnkytst.h"
  37 #include "apicoll.h"
  38 #include "regcoll.h"
  39 #include "currcoll.h"
  40 #include "itercoll.h"
  41 #include "tstnorm.h"
  42 #include "normconf.h"
  43 #include "thcoll.h"
  44 #include "srchtest.h"
  45 #include "cntabcol.h"
  46 #include "lcukocol.h"
  47 #include "ucaconf.h"
  48 #include "svccoll.h"
  49 #include "cmemory.h"
  50 #include "rndmcoll.h"
  51
  52 #define TESTCLASS(n,classname)        \
  53     case n:                           \
  54         name = #classname;            \
  55         if (exec) {                   \
  56             logln(#classname "---");  \
  57             logln("");                \
  58             classname t;              \
  59             callTest(t, par);         \
  60         }                             \
  61         break
  62
  63 void IntlTestCollator::runIndexedTest( int32_t index, UBool exec, const char* &name, char* par )
  64 {
  65     if (exec) {
  66         logln("TestSuite Collator: ");
  67     }
  68
  69     switch (index) {
  70       TESTCLASS(0, CollationEnglishTest);
  71       TESTCLASS(1, CollationFrenchTest);
  72       TESTCLASS(2, CollationGermanTest);
  73       TESTCLASS(3, CollationSpanishTest);
  74       TESTCLASS(4, CollationKanaTest);
  75       TESTCLASS(5, CollationTurkishTest);
  76       TESTCLASS(6, CollationDummyTest);
  77       TESTCLASS(7, G7CollationTest);
  78       TESTCLASS(8, CollationMonkeyTest);
  79       TESTCLASS(9, CollationAPITest);
  80       TESTCLASS(10, CollationRegressionTest);
  81       TESTCLASS(11, CollationCurrencyTest);
  82       TESTCLASS(12, CollationIteratorTest);
  83       TESTCLASS(13, CollationThaiTest);
  84       TESTCLASS(14, LotusCollationKoreanTest);
  85       TESTCLASS(15, StringSearchTest);
  86       TESTCLASS(16, ContractionTableTest);
  87       TESTCLASS(17, DataDrivenCollatorTest);
  88       TESTCLASS(18, UCAConformanceTest);
  89       TESTCLASS(19, CollationServiceTest);
  90       TESTCLASS(20, CollationFinnishTest); // removed by weiv - we have changed Finnish collation
  91       TESTCLASS(21, RandomCollatorTest);
  92
  93       default: name = ""; break;
  94     }
  95 }
  96
  97 UCollationResult
  98 IntlTestCollator::compareUsingPartials(UCollator *coll, const UChar source[], int32_t sLen, const UChar target[], int32_t tLen, int32_t pieceSize, UErrorCode &status) {
  99   int32_t partialSKResult = 0;
 100   uint8_t sBuf[512], tBuf[512];
 101   UCharIterator sIter, tIter;
 102   uint32_t sState[2], tState[2];
 103   int32_t sSize = pieceSize, tSize = pieceSize;
 104   int32_t i = 0;
 105   status = U_ZERO_ERROR;
 106   sState[0] = 0; sState[1] = 0;
 107   tState[0] = 0; tState[1] = 0;
 108   while(sSize == pieceSize && tSize == pieceSize && partialSKResult == 0) {
 109     uiter_setString(&sIter, source, sLen);
 110     uiter_setString(&tIter, target, tLen);
 111     sSize = ucol_nextSortKeyPart(coll, &sIter, sState, sBuf, pieceSize, &status);
 112     tSize = ucol_nextSortKeyPart(coll, &tIter, tState, tBuf, pieceSize, &status);
 113
 114     if(sState[0] != 0 || tState[0] != 0) {
 115       log("State != 0 : %08X %08X\n", sState[0], tState[0]);
 116     }
 117     log("%i ", i++);
 118
 119     partialSKResult = memcmp(sBuf, tBuf, pieceSize);
 120   }
 121
 122   if(partialSKResult < 0) {
 123       return UCOL_LESS;
 124   } else if(partialSKResult > 0) {
 125     return UCOL_GREATER;
 126   } else {
 127     return UCOL_EQUAL;
 128   }
 129 }
 130
 131 void
 132 IntlTestCollator::doTestVariant(Collator* col, const UnicodeString &source, const UnicodeString &target, Collator::EComparisonResult result)
 133 {
 134   UErrorCode status = U_ZERO_ERROR;
 135
 136   UCollator *myCollation = (UCollator *)((RuleBasedCollator *)col)->getUCollator();
 137
 138   Collator::EComparisonResult compareResult = col->compare(source, target);
 139
 140   CollationKey srckey, tgtkey;
 141   col->getCollationKey(source, srckey, status);
 142   col->getCollationKey(target, tgtkey, status);
 143   if (U_FAILURE(status)){
 144     errln("Creation of collation keys failed\n");
 145   }
 146   Collator::EComparisonResult keyResult = srckey.compareTo(tgtkey);
 147
 148   reportCResult(source, target, srckey, tgtkey, compareResult, keyResult, result, result);
 149
 150     UColAttributeValue norm = ucol_getAttribute(myCollation, UCOL_NORMALIZATION_MODE, &status);
 151
 152     int32_t sLen = source.length(), tLen = target.length();
 153     const UChar* src = source.getBuffer();
 154     const UChar* trg = target.getBuffer();
 155     UCollationResult compareResultIter = (UCollationResult)result;
 156
 157     if(1) {
 158       UCharIterator sIter, tIter;
 159       uiter_setString(&sIter, src, sLen);
 160       uiter_setString(&tIter, trg, tLen);
 161       compareResultIter = ucol_strcollIter(myCollation, &sIter, &tIter, &status);
 162       if(compareResultIter != (UCollationResult)result) {
 163         errln("Different result for iterative comparison "+source+" "+target);
 164       }
 165     }
 166     /* convert the strings to UTF-8 and do try comparing with char iterator */
 167     if(!quick) { /*!QUICK*/
 168       char utf8Source[256], utf8Target[256];
 169       int32_t utf8SourceLen = 0, utf8TargetLen = 0;
 170       u_strToUTF8(utf8Source, 256, &utf8SourceLen, src, sLen, &status);
 171       if(U_FAILURE(status)) { /* probably buffer is not big enough */
 172         log("Src UTF-8 buffer too small! Will not compare!\n");
 173       } else {
 174         u_strToUTF8(utf8Target, 256, &utf8TargetLen, trg, tLen, &status);
 175         if(U_SUCCESS(status)) { /* probably buffer is not big enough */
 176           UCollationResult compareResultUTF8 = (UCollationResult)result, compareResultUTF8Norm = (UCollationResult)result;
 177           UCharIterator sIter, tIter;
 178           /*log_verbose("Strings converted to UTF-8:%s, %s\n", aescstrdup(source,-1), aescstrdup(target,-1));*/
 179           uiter_setUTF8(&sIter, utf8Source, utf8SourceLen);
 180           uiter_setUTF8(&tIter, utf8Target, utf8TargetLen);
 181        /*uiter_setString(&sIter, source, sLen);
 182       uiter_setString(&tIter, target, tLen);*/
 183           compareResultUTF8 = ucol_strcollIter(myCollation, &sIter, &tIter, &status);
 184           ucol_setAttribute(myCollation, UCOL_NORMALIZATION_MODE, UCOL_ON, &status);
 185           sIter.move(&sIter, 0, UITER_START);
 186           tIter.move(&tIter, 0, UITER_START);
 187           compareResultUTF8Norm = ucol_strcollIter(myCollation, &sIter, &tIter, &status);
 188           ucol_setAttribute(myCollation, UCOL_NORMALIZATION_MODE, norm, &status);
 189           if(compareResultUTF8 != compareResultIter) {
 190             errln("different results in iterative comparison for UTF-16 and UTF-8 encoded strings. "+source+", "+target);
 191           }
 192           if(compareResultUTF8 != compareResultUTF8Norm) {
 193             errln("different results in iterative when normalization is turned on with UTF-8 strings. "+source+", "+target);
 194           }
 195         } else {
 196           log("Target UTF-8 buffer too small! Did not compare!\n");
 197         }
 198         if(U_FAILURE(status)) {
 199           log("UTF-8 strcoll failed! Ignoring result\n");
 200         }
 201       }
 202     }
 203
 204     /* testing the partial sortkeys */
 205     if(1) { /*!QUICK*/
 206       int32_t partialSizes[] = { 3, 1, 2, 4, 8, 20, 80 }; /* just size 3 in the quick mode */
 207       int32_t partialSizesSize = 1;
 208       if(!quick) {
 209         partialSizesSize = 7;
 210       }
 211       int32_t i = 0;
 212       log("partial sortkey test piecesize=");
 213       for(i = 0; i < partialSizesSize; i++) {
 214         UCollationResult partialSKResult = (UCollationResult)result, partialNormalizedSKResult = (UCollationResult)result;
 215         log("%i ", partialSizes[i]);
 216
 217         partialSKResult = compareUsingPartials(myCollation, src, sLen, trg, tLen, partialSizes[i], status);
 218         if(partialSKResult != (UCollationResult)result) {
 219           errln("Partial sortkey comparison returned wrong result: "+source+", "+target+" (size "+partialSizes[i]+")");
 220         }
 221
 222         if(norm != UCOL_ON && !quick) {
 223           log("N ");
 224           ucol_setAttribute(myCollation, UCOL_NORMALIZATION_MODE, UCOL_ON, &status);
 225           partialNormalizedSKResult = compareUsingPartials(myCollation, src, sLen, trg, tLen, partialSizes[i], status);
 226           ucol_setAttribute(myCollation, UCOL_NORMALIZATION_MODE, norm, &status);
 227           if(partialSKResult != partialNormalizedSKResult) {
 228             errln("Partial sortkey comparison gets different result when normalization is on: "+source+", "+target+" (size "+partialSizes[i]+")");
 229           }
 230         }
 231       }
 232       log("\n");
 233     }
 234 /*
 235   if (compareResult != result) {
 236     errln("String comparison failed in variant test\n");
 237   }
 238   if (keyResult != result) {
 239     errln("Collation key comparison failed in variant test\n");
 240   }
 241 */
 242 }
 243
 244 void
 245 IntlTestCollator::doTest(Collator* col, const UChar *source, const UChar *target, Collator::EComparisonResult result) {
 246   doTest(col, UnicodeString(source), UnicodeString(target), result);
 247 }
 248
 249 void
 250 IntlTestCollator::doTest(Collator* col, const UnicodeString &source, const UnicodeString &target, Collator::EComparisonResult result)
 251 {
 252   if(col) {
 253     doTestVariant(col, source, target, result);
 254     if(result == Collator::LESS) {
 255       doTestVariant(col, target, source, Collator::GREATER);
 256     } else if (result == Collator::GREATER) {
 257       doTestVariant(col, target, source, Collator::LESS);
 258     }
 259
 260     UErrorCode status = U_ZERO_ERROR;
 261     CollationElementIterator* c = ((RuleBasedCollator *)col)->createCollationElementIterator( source );
 262     logln("Testing iterating source: "+source);
 263     backAndForth(*c);
 264     c->setText(target, status);
 265     logln("Testing iterating target: "+target);
 266     backAndForth(*c);
 267     delete c;
 268   }
 269 }
 270
 271
 272 // used for collation result reporting, defined here for convenience
 273 // (maybe moved later)
 274 void
 275 IntlTestCollator::reportCResult( const UnicodeString &source, const UnicodeString &target,
 276              CollationKey &sourceKey, CollationKey &targetKey,
 277              Collator::EComparisonResult compareResult,
 278              Collator::EComparisonResult keyResult,
 279                                 Collator::EComparisonResult incResult,
 280                          Collator::EComparisonResult expectedResult )
 281 {
 282     if (expectedResult < -1 || expectedResult > 1)
 283     {
 284         errln("***** invalid call to reportCResult ****");
 285         return;
 286     }
 287
 288     UBool ok1 = (compareResult == expectedResult);
 289     UBool ok2 = (keyResult == expectedResult);
 290     UBool ok3 = (incResult == expectedResult);
 291
 292
 293     if (ok1 && ok2 && ok3 && !verbose) {
 294         // Keep non-verbose, passing tests fast
 295         return;
 296     } else {
 297         UnicodeString msg1(ok1 ? "Ok: compare(" : "FAIL: compare(");
 298         UnicodeString msg2(", "), msg3(") returned "), msg4("; expected ");
 299         UnicodeString prettySource, prettyTarget, sExpect, sResult;
 300
 301         IntlTest::prettify(source, prettySource);
 302         IntlTest::prettify(target, prettyTarget);
 303         appendCompareResult(compareResult, sResult);
 304         appendCompareResult(expectedResult, sExpect);
 305
 306         if (ok1) {
 307             logln(msg1 + prettySource + msg2 + prettyTarget + msg3 + sResult);
 308         } else {
 309             errln(msg1 + prettySource + msg2 + prettyTarget + msg3 + sResult + msg4 + sExpect);
 310         }
 311
 312         msg1 = UnicodeString(ok2 ? "Ok: key(" : "FAIL: key(");
 313         msg2 = ").compareTo(key(";
 314         msg3 = ")) returned ";
 315
 316         appendCompareResult(keyResult, sResult);
 317
 318         if (ok2) {
 319             logln(msg1 + prettySource + msg2 + prettyTarget + msg3 + sResult);
 320         } else {
 321             errln(msg1 + prettySource + msg2 + prettyTarget + msg3 + sResult + msg4 + sExpect);
 322
 323             msg1 = "  ";
 324             msg2 = " vs. ";
 325
 326             prettify(sourceKey, prettySource);
 327             prettify(targetKey, prettyTarget);
 328
 329             errln(msg1 + prettySource + msg2 + prettyTarget);
 330         }
 331         msg1 = UnicodeString (ok3 ? "Ok: incCompare(" : "FAIL: incCompare(");
 332         msg2 = ", ";
 333         msg3 = ") returned ";
 334
 335         appendCompareResult(incResult, sResult);
 336
 337         if (ok3) {
 338             logln(msg1 + prettySource + msg2 + prettyTarget + msg3 + sResult);
 339         } else {
 340             errln(msg1 + prettySource + msg2 + prettyTarget + msg3 + sResult + msg4 + sExpect);
 341         }
 342     }
 343 }
 344
 345 UnicodeString&
 346 IntlTestCollator::appendCompareResult(Collator::EComparisonResult result,
 347                   UnicodeString& target)
 348 {
 349     if (result == Collator::LESS)
 350     {
 351         target += "LESS";
 352     }
 353     else if (result == Collator::EQUAL)
 354     {
 355         target += "EQUAL";
 356     }
 357     else if (result == Collator::GREATER)
 358     {
 359         target += "GREATER";
 360     }
 361     else
 362     {
 363         UnicodeString huh = "?";
 364
 365         target += (huh + (int32_t)result);
 366     }
 367
 368     return target;
 369 }
 370
 371 // Produce a printable representation of a CollationKey
 372 UnicodeString &IntlTestCollator::prettify(const CollationKey &source, UnicodeString &target)
 373 {
 374     int32_t i, byteCount;
 375     const uint8_t *bytes = source.getByteArray(byteCount);
 376
 377     target.remove();
 378     target += "[";
 379
 380     for (i = 0; i < byteCount; i += 1)
 381     {
 382         appendHex(bytes[i], 2, target);
 383         target += " ";
 384     }
 385
 386     target += "]";
 387
 388     return target;
 389 }
 390
 391 void IntlTestCollator::backAndForth(CollationElementIterator &iter)
 392 {
 393     // Run through the iterator forwards and stick it into an array
 394     int32_t orderLength = 0;
 395     int32_t *orders = getOrders(iter, orderLength);
 396     UErrorCode status = U_ZERO_ERROR;
 397
 398     // Now go through it backwards and make sure we get the same values
 399     int32_t index = orderLength;
 400     int32_t o;
 401
 402     // reset the iterator
 403     iter.reset();
 404
 405     while ((o = iter.previous(status)) != CollationElementIterator::NULLORDER)
 406     {
 407         if (index == 0) {
 408           if(o == 0) {
 409             continue;
 410           } else { // this is an error, orders exhausted but there are non-ignorable CEs from
 411             // going backwards
 412             errln("Backward iteration returned a non ignorable after orders are exhausted");
 413             break;
 414         }
 415         }
 416         if (o != orders[--index])
 417         {
 418             if (o == 0)
 419                 index ++;
 420             else
 421             {
 422                 while (index > 0 && orders[--index] == 0)
 423                 {
 424                 }
 425                 if (o != orders[index])
 426                 {
 427             errln("Mismatch at index %d: 0x%X vs 0x%X", index,
 428                   orders[index], o);
 429                     break;
 430                 }
 431             }
 432         }
 433     }
 434
 435     while (index != 0 && orders[index - 1] == 0)
 436     {
 437       index --;
 438     }
 439
 440     if (index != 0)
 441     {
 442         UnicodeString msg("Didn't get back to beginning - index is ");
 443         errln(msg + index);
 444
 445         iter.reset();
 446         err("next: ");
 447         while ((o = iter.next(status)) != CollationElementIterator::NULLORDER)
 448         {
 449             UnicodeString hexString("0x");
 450
 451             appendHex(o, 8, hexString);
 452             hexString += " ";
 453             err(hexString);
 454         }
 455         errln("");
 456
 457         err("prev: ");
 458         while ((o = iter.previous(status)) != CollationElementIterator::NULLORDER)
 459         {
 460             UnicodeString hexString("0x");
 461
 462             appendHex(o, 8, hexString);
 463             hexString += " ";
 464              err(hexString);
 465         }
 466         errln("");
 467     }
 468
 469     delete[] orders;
 470 }
 471
 472
 473 /**
 474  * Return an integer array containing all of the collation orders
 475  * returned by calls to next on the specified iterator
 476  */
 477 int32_t *IntlTestCollator::getOrders(CollationElementIterator &iter, int32_t &orderLength)
 478 {
 479     int32_t maxSize = 100;
 480     int32_t size = 0;
 481     int32_t *orders = new int32_t[maxSize];
 482     UErrorCode status = U_ZERO_ERROR;
 483
 484     int32_t order;
 485     while ((order = iter.next(status)) != CollationElementIterator::NULLORDER)
 486     {
 487         if (size == maxSize)
 488         {
 489             maxSize *= 2;
 490             int32_t *temp = new int32_t[maxSize];
 491
 492             uprv_memcpy(temp, orders, size * sizeof(int32_t));
 493             delete[] orders;
 494             orders = temp;
 495         }
 496
 497         orders[size++] = order;
 498     }
 499
 500     if (maxSize > size)
 501     {
 502         int32_t *temp = new int32_t[size];
 503
 504         uprv_memcpy(temp, orders, size * sizeof(int32_t));
 505         delete[] orders;
 506         orders = temp;
 507     }
 508
 509     orderLength = size;
 510     return orders;
 511 }
 512
 513 #endif /* #if !UCONFIG_NO_COLLATION */