icuSources/test/intltest/convtest.cpp

   1 /*
   2 *******************************************************************************
   3 *
   4 *   Copyright (C) 2003-2013, International Business Machines
   5 *   Corporation and others.  All Rights Reserved.
   6 *
   7 *******************************************************************************
   8 *   file name:  convtest.cpp
   9 *   encoding:   US-ASCII
  10 *   tab size:   8 (not used)
  11 *   indentation:4
  12 *
  13 *   created on: 2003jul15
  14 *   created by: Markus W. Scherer
  15 *
  16 *   Test file for data-driven conversion tests.
  17 */
  18
  19 #include "unicode/utypes.h"
  20
  21 #if !UCONFIG_NO_LEGACY_CONVERSION
  22 /*
  23  * Note: Turning off all of convtest.cpp if !UCONFIG_NO_LEGACY_CONVERSION
  24  * is slightly unnecessary - it removes tests for Unicode charsets
  25  * like UTF-8 that should work.
  26  * However, there is no easy way for the test to detect whether a test case
  27  * is for a Unicode charset, so it would be difficult to only exclude those.
  28  * Also, regular testing of ICU is done with all modules on, therefore
  29  * not testing conversion for a custom configuration like this should be ok.
  30  */
  31
  32 #include "unicode/ucnv.h"
  33 #include "unicode/unistr.h"
  34 #include "unicode/parsepos.h"
  35 #include "unicode/uniset.h"
  36 #include "unicode/ustring.h"
  37 #include "unicode/ures.h"
  38 #include "convtest.h"
  39 #include "unicode/tstdtmod.h"
  40 #include <string.h>
  41 #include <stdlib.h>
  42
  43 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
  44
  45 enum {
  46     // characters used in test data for callbacks
  47     SUB_CB='?',
  48     SKIP_CB='0',
  49     STOP_CB='.',
  50     ESC_CB='&'
  51 };
  52
  53 ConversionTest::ConversionTest() {
  54     UErrorCode errorCode=U_ZERO_ERROR;
  55     utf8Cnv=ucnv_open("UTF-8", &errorCode);
  56     ucnv_setToUCallBack(utf8Cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
  57     if(U_FAILURE(errorCode)) {
  58         errln("unable to open UTF-8 converter");
  59     }
  60 }
  61
  62 ConversionTest::~ConversionTest() {
  63     ucnv_close(utf8Cnv);
  64 }
  65
  66 void
  67 ConversionTest::runIndexedTest(int32_t index, UBool exec, const char *&name, char * /*par*/) {
  68     if (exec) logln("TestSuite ConversionTest: ");
  69     switch (index) {
  70 #if !UCONFIG_NO_FILE_IO
  71         case 0: name="TestToUnicode"; if (exec) TestToUnicode(); break;
  72         case 1: name="TestFromUnicode"; if (exec) TestFromUnicode(); break;
  73         case 2: name="TestGetUnicodeSet"; if (exec) TestGetUnicodeSet(); break;
  74 #else
  75         case 0:
  76         case 1:
  77         case 2: name="skip"; break;
  78 #endif
  79         case 3: name="TestGetUnicodeSet2"; if (exec) TestGetUnicodeSet2(); break;
  80         default: name=""; break; //needed to end loop
  81     }
  82 }
  83
  84 // test data interface ----------------------------------------------------- ***
  85
  86 void
  87 ConversionTest::TestToUnicode() {
  88     ConversionCase cc;
  89     char charset[100], cbopt[4];
  90     const char *option;
  91     UnicodeString s, unicode;
  92     int32_t offsetsLength;
  93     UConverterToUCallback callback;
  94
  95     TestDataModule *dataModule;
  96     TestData *testData;
  97     const DataMap *testCase;
  98     UErrorCode errorCode;
  99     int32_t i;
 100
 101     errorCode=U_ZERO_ERROR;
 102     dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
 103     if(U_SUCCESS(errorCode)) {
 104         testData=dataModule->createTestData("toUnicode", errorCode);
 105         if(U_SUCCESS(errorCode)) {
 106             for(i=0; testData->nextCase(testCase, errorCode); ++i) {
 107                 if(U_FAILURE(errorCode)) {
 108                     errln("error retrieving conversion/toUnicode test case %d - %s",
 109                             i, u_errorName(errorCode));
 110                     errorCode=U_ZERO_ERROR;
 111                     continue;
 112                 }
 113
 114                 cc.caseNr=i;
 115
 116                 s=testCase->getString("charset", errorCode);
 117                 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
 118                 cc.charset=charset;
 119
 120                 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
 121                 unicode=testCase->getString("unicode", errorCode);
 122                 cc.unicode=unicode.getBuffer();
 123                 cc.unicodeLength=unicode.length();
 124
 125                 offsetsLength=0;
 126                 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
 127                 if(offsetsLength==0) {
 128                     cc.offsets=NULL;
 129                 } else if(offsetsLength!=unicode.length()) {
 130                     errln("toUnicode[%d] unicode[%d] and offsets[%d] must have the same length",
 131                             i, unicode.length(), offsetsLength);
 132                     errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 133                 }
 134
 135                 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
 136                 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
 137
 138                 s=testCase->getString("errorCode", errorCode);
 139                 if(s==UNICODE_STRING("invalid", 7)) {
 140                     cc.outErrorCode=U_INVALID_CHAR_FOUND;
 141                 } else if(s==UNICODE_STRING("illegal", 7)) {
 142                     cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
 143                 } else if(s==UNICODE_STRING("truncated", 9)) {
 144                     cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
 145                 } else if(s==UNICODE_STRING("illesc", 6)) {
 146                     cc.outErrorCode=U_ILLEGAL_ESCAPE_SEQUENCE;
 147                 } else if(s==UNICODE_STRING("unsuppesc", 9)) {
 148                     cc.outErrorCode=U_UNSUPPORTED_ESCAPE_SEQUENCE;
 149                 } else {
 150                     cc.outErrorCode=U_ZERO_ERROR;
 151                 }
 152
 153                 s=testCase->getString("callback", errorCode);
 154                 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
 155                 cc.cbopt=cbopt;
 156                 switch(cbopt[0]) {
 157                 case SUB_CB:
 158                     callback=UCNV_TO_U_CALLBACK_SUBSTITUTE;
 159                     break;
 160                 case SKIP_CB:
 161                     callback=UCNV_TO_U_CALLBACK_SKIP;
 162                     break;
 163                 case STOP_CB:
 164                     callback=UCNV_TO_U_CALLBACK_STOP;
 165                     break;
 166                 case ESC_CB:
 167                     callback=UCNV_TO_U_CALLBACK_ESCAPE;
 168                     break;
 169                 default:
 170                     callback=NULL;
 171                     break;
 172                 }
 173                 option=callback==NULL ? cbopt : cbopt+1;
 174                 if(*option==0) {
 175                     option=NULL;
 176                 }
 177
 178                 cc.invalidChars=testCase->getBinary(cc.invalidLength, "invalidChars", errorCode);
 179
 180                 if(U_FAILURE(errorCode)) {
 181                     errln("error parsing conversion/toUnicode test case %d - %s",
 182                             i, u_errorName(errorCode));
 183                     errorCode=U_ZERO_ERROR;
 184                 } else {
 185                     logln("TestToUnicode[%d] %s", i, charset);
 186                     ToUnicodeCase(cc, callback, option);
 187                 }
 188             }
 189             delete testData;
 190         }
 191         delete dataModule;
 192     }
 193     else {
 194         dataerrln("Could not load test conversion data");
 195     }
 196 }
 197
 198 void
 199 ConversionTest::TestFromUnicode() {
 200     ConversionCase cc;
 201     char charset[100], cbopt[4];
 202     const char *option;
 203     UnicodeString s, unicode, invalidUChars;
 204     int32_t offsetsLength, index;
 205     UConverterFromUCallback callback;
 206
 207     TestDataModule *dataModule;
 208     TestData *testData;
 209     const DataMap *testCase;
 210     const UChar *p;
 211     UErrorCode errorCode;
 212     int32_t i, length;
 213
 214     errorCode=U_ZERO_ERROR;
 215     dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
 216     if(U_SUCCESS(errorCode)) {
 217         testData=dataModule->createTestData("fromUnicode", errorCode);
 218         if(U_SUCCESS(errorCode)) {
 219             for(i=0; testData->nextCase(testCase, errorCode); ++i) {
 220                 if(U_FAILURE(errorCode)) {
 221                     errln("error retrieving conversion/fromUnicode test case %d - %s",
 222                             i, u_errorName(errorCode));
 223                     errorCode=U_ZERO_ERROR;
 224                     continue;
 225                 }
 226
 227                 cc.caseNr=i;
 228
 229                 s=testCase->getString("charset", errorCode);
 230                 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
 231                 cc.charset=charset;
 232
 233                 unicode=testCase->getString("unicode", errorCode);
 234                 cc.unicode=unicode.getBuffer();
 235                 cc.unicodeLength=unicode.length();
 236                 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
 237
 238                 offsetsLength=0;
 239                 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
 240                 if(offsetsLength==0) {
 241                     cc.offsets=NULL;
 242                 } else if(offsetsLength!=cc.bytesLength) {
 243                     errln("fromUnicode[%d] bytes[%d] and offsets[%d] must have the same length",
 244                             i, cc.bytesLength, offsetsLength);
 245                     errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 246                 }
 247
 248                 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
 249                 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
 250
 251                 s=testCase->getString("errorCode", errorCode);
 252                 if(s==UNICODE_STRING("invalid", 7)) {
 253                     cc.outErrorCode=U_INVALID_CHAR_FOUND;
 254                 } else if(s==UNICODE_STRING("illegal", 7)) {
 255                     cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
 256                 } else if(s==UNICODE_STRING("truncated", 9)) {
 257                     cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
 258                 } else {
 259                     cc.outErrorCode=U_ZERO_ERROR;
 260                 }
 261
 262                 s=testCase->getString("callback", errorCode);
 263                 cc.setSub=0; // default: no subchar
 264
 265                 if((index=s.indexOf((UChar)0))>0) {
 266                     // read NUL-separated subchar first, if any
 267                     // copy the subchar from Latin-1 characters
 268                     // start after the NUL
 269                     p=s.getTerminatedBuffer();
 270                     length=index+1;
 271                     p+=length;
 272                     length=s.length()-length;
 273                     if(length<=0 || length>=(int32_t)sizeof(cc.subchar)) {
 274                         errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 275                     } else {
 276                         int32_t j;
 277
 278                         for(j=0; j<length; ++j) {
 279                             cc.subchar[j]=(char)p[j];
 280                         }
 281                         // NUL-terminate the subchar
 282                         cc.subchar[j]=0;
 283                         cc.setSub=1;
 284                     }
 285
 286                     // remove the NUL and subchar from s
 287                     s.truncate(index);
 288                 } else if((index=s.indexOf((UChar)0x3d))>0) /* '=' */ {
 289                     // read a substitution string, separated by an equal sign
 290                     p=s.getBuffer()+index+1;
 291                     length=s.length()-(index+1);
 292                     if(length<0 || length>=LENGTHOF(cc.subString)) {
 293                         errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 294                     } else {
 295                         u_memcpy(cc.subString, p, length);
 296                         // NUL-terminate the subString
 297                         cc.subString[length]=0;
 298                         cc.setSub=-1;
 299                     }
 300
 301                     // remove the equal sign and subString from s
 302                     s.truncate(index);
 303                 }
 304
 305                 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
 306                 cc.cbopt=cbopt;
 307                 switch(cbopt[0]) {
 308                 case SUB_CB:
 309                     callback=UCNV_FROM_U_CALLBACK_SUBSTITUTE;
 310                     break;
 311                 case SKIP_CB:
 312                     callback=UCNV_FROM_U_CALLBACK_SKIP;
 313                     break;
 314                 case STOP_CB:
 315                     callback=UCNV_FROM_U_CALLBACK_STOP;
 316                     break;
 317                 case ESC_CB:
 318                     callback=UCNV_FROM_U_CALLBACK_ESCAPE;
 319                     break;
 320                 default:
 321                     callback=NULL;
 322                     break;
 323                 }
 324                 option=callback==NULL ? cbopt : cbopt+1;
 325                 if(*option==0) {
 326                     option=NULL;
 327                 }
 328
 329                 invalidUChars=testCase->getString("invalidUChars", errorCode);
 330                 cc.invalidUChars=invalidUChars.getBuffer();
 331                 cc.invalidLength=invalidUChars.length();
 332
 333                 if(U_FAILURE(errorCode)) {
 334                     errln("error parsing conversion/fromUnicode test case %d - %s",
 335                             i, u_errorName(errorCode));
 336                     errorCode=U_ZERO_ERROR;
 337                 } else {
 338                     logln("TestFromUnicode[%d] %s", i, charset);
 339                     FromUnicodeCase(cc, callback, option);
 340                 }
 341             }
 342             delete testData;
 343         }
 344         delete dataModule;
 345     }
 346     else {
 347         dataerrln("Could not load test conversion data");
 348     }
 349 }
 350
 351 static const UChar ellipsis[]={ 0x2e, 0x2e, 0x2e };
 352
 353 void
 354 ConversionTest::TestGetUnicodeSet() {
 355     char charset[100];
 356     UnicodeString s, map, mapnot;
 357     int32_t which;
 358
 359     ParsePosition pos;
 360     UnicodeSet cnvSet, mapSet, mapnotSet, diffSet;
 361     UnicodeSet *cnvSetPtr = &cnvSet;
 362     LocalUConverterPointer cnv;
 363
 364     TestDataModule *dataModule;
 365     TestData *testData;
 366     const DataMap *testCase;
 367     UErrorCode errorCode;
 368     int32_t i;
 369
 370     errorCode=U_ZERO_ERROR;
 371     dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
 372     if(U_SUCCESS(errorCode)) {
 373         testData=dataModule->createTestData("getUnicodeSet", errorCode);
 374         if(U_SUCCESS(errorCode)) {
 375             for(i=0; testData->nextCase(testCase, errorCode); ++i) {
 376                 if(U_FAILURE(errorCode)) {
 377                     errln("error retrieving conversion/getUnicodeSet test case %d - %s",
 378                             i, u_errorName(errorCode));
 379                     errorCode=U_ZERO_ERROR;
 380                     continue;
 381                 }
 382
 383                 s=testCase->getString("charset", errorCode);
 384                 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
 385
 386                 map=testCase->getString("map", errorCode);
 387                 mapnot=testCase->getString("mapnot", errorCode);
 388
 389                 which=testCase->getInt28("which", errorCode);
 390
 391                 if(U_FAILURE(errorCode)) {
 392                     errln("error parsing conversion/getUnicodeSet test case %d - %s",
 393                             i, u_errorName(errorCode));
 394                     errorCode=U_ZERO_ERROR;
 395                     continue;
 396                 }
 397
 398                 // test this test case
 399                 mapSet.clear();
 400                 mapnotSet.clear();
 401
 402                 pos.setIndex(0);
 403                 mapSet.applyPattern(map, pos, 0, NULL, errorCode);
 404                 if(U_FAILURE(errorCode) || pos.getIndex()!=map.length()) {
 405                     errln("error creating the map set for conversion/getUnicodeSet test case %d - %s\n"
 406                           "    error index %d  index %d  U+%04x",
 407                             i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), map.char32At(pos.getIndex()));
 408                     errorCode=U_ZERO_ERROR;
 409                     continue;
 410                 }
 411
 412                 pos.setIndex(0);
 413                 mapnotSet.applyPattern(mapnot, pos, 0, NULL, errorCode);
 414                 if(U_FAILURE(errorCode) || pos.getIndex()!=mapnot.length()) {
 415                     errln("error creating the mapnot set for conversion/getUnicodeSet test case %d - %s\n"
 416                           "    error index %d  index %d  U+%04x",
 417                             i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), mapnot.char32At(pos.getIndex()));
 418                     errorCode=U_ZERO_ERROR;
 419                     continue;
 420                 }
 421
 422                 logln("TestGetUnicodeSet[%d] %s", i, charset);
 423
 424                 cnv.adoptInstead(cnv_open(charset, errorCode));
 425                 if(U_FAILURE(errorCode)) {
 426                     errcheckln(errorCode, "error opening \"%s\" for conversion/getUnicodeSet test case %d - %s",
 427                             charset, i, u_errorName(errorCode));
 428                     errorCode=U_ZERO_ERROR;
 429                     continue;
 430                 }
 431
 432                 ucnv_getUnicodeSet(cnv.getAlias(), cnvSetPtr->toUSet(), (UConverterUnicodeSet)which, &errorCode);
 433
 434                 if(U_FAILURE(errorCode)) {
 435                     errln("error in ucnv_getUnicodeSet(\"%s\") for conversion/getUnicodeSet test case %d - %s",
 436                             charset, i, u_errorName(errorCode));
 437                     errorCode=U_ZERO_ERROR;
 438                     continue;
 439                 }
 440
 441                 // are there items that must be in cnvSet but are not?
 442                 (diffSet=mapSet).removeAll(cnvSet);
 443                 if(!diffSet.isEmpty()) {
 444                     diffSet.toPattern(s, TRUE);
 445                     if(s.length()>100) {
 446                         s.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
 447                     }
 448                     errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - conversion/getUnicodeSet test case %d",
 449                             charset, i);
 450                     errln(s);
 451                 }
 452
 453                 // are there items that must not be in cnvSet but are?
 454                 (diffSet=mapnotSet).retainAll(cnvSet);
 455                 if(!diffSet.isEmpty()) {
 456                     diffSet.toPattern(s, TRUE);
 457                     if(s.length()>100) {
 458                         s.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
 459                     }
 460                     errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - conversion/getUnicodeSet test case %d",
 461                             charset, i);
 462                     errln(s);
 463                 }
 464             }
 465             delete testData;
 466         }
 467         delete dataModule;
 468     }
 469     else {
 470         dataerrln("Could not load test conversion data");
 471     }
 472 }
 473
 474 U_CDECL_BEGIN
 475 static void U_CALLCONV
 476 getUnicodeSetCallback(const void *context,
 477                       UConverterFromUnicodeArgs * /*fromUArgs*/,
 478                       const UChar* /*codeUnits*/,
 479                       int32_t /*length*/,
 480                       UChar32 codePoint,
 481                       UConverterCallbackReason reason,
 482                       UErrorCode *pErrorCode) {
 483     if(reason<=UCNV_IRREGULAR) {
 484         ((UnicodeSet *)context)->remove(codePoint);  // the converter cannot convert this code point
 485         *pErrorCode=U_ZERO_ERROR;                    // skip
 486     }  // else ignore the reset, close and clone calls.
 487 }
 488 U_CDECL_END
 489
 490 // Compare ucnv_getUnicodeSet() with the set of characters that can be converted.
 491 void
 492 ConversionTest::TestGetUnicodeSet2() {
 493     // Build a string with all code points.
 494     UChar32 cpLimit;
 495     int32_t s0Length;
 496     if(quick) {
 497         cpLimit=s0Length=0x10000;  // BMP only
 498     } else {
 499         cpLimit=0x110000;
 500         s0Length=0x10000+0x200000;  // BMP + surrogate pairs
 501     }
 502     UChar *s0=new UChar[s0Length];
 503     if(s0==NULL) {
 504         return;
 505     }
 506     UChar *s=s0;
 507     UChar32 c;
 508     UChar c2;
 509     // low BMP
 510     for(c=0; c<=0xd7ff; ++c) {
 511         *s++=(UChar)c;
 512     }
 513     // trail surrogates
 514     for(c=0xdc00; c<=0xdfff; ++c) {
 515         *s++=(UChar)c;
 516     }
 517     // lead surrogates
 518     // (after trails so that there is not even one surrogate pair in between)
 519     for(c=0xd800; c<=0xdbff; ++c) {
 520         *s++=(UChar)c;
 521     }
 522     // high BMP
 523     for(c=0xe000; c<=0xffff; ++c) {
 524         *s++=(UChar)c;
 525     }
 526     // supplementary code points = surrogate pairs
 527     if(cpLimit==0x110000) {
 528         for(c=0xd800; c<=0xdbff; ++c) {
 529             for(c2=0xdc00; c2<=0xdfff; ++c2) {
 530                 *s++=(UChar)c;
 531                 *s++=c2;
 532             }
 533         }
 534     }
 535
 536     static const char *const cnvNames[]={
 537         "UTF-8",
 538         "UTF-7",
 539         "UTF-16",
 540         "US-ASCII",
 541         "ISO-8859-1",
 542         "windows-1252",
 543         "Shift-JIS",
 544         "ibm-1390",  // EBCDIC_STATEFUL table
 545         "ibm-16684",  // DBCS-only extension table based on EBCDIC_STATEFUL table
 546         "HZ",
 547         "ISO-2022-JP",
 548         "JIS7",
 549         "ISO-2022-CN",
 550         "ISO-2022-CN-EXT",
 551         "LMBCS"
 552     };
 553     LocalUConverterPointer cnv;
 554     char buffer[1024];
 555     int32_t i;
 556     for(i=0; i<LENGTHOF(cnvNames); ++i) {
 557         UErrorCode errorCode=U_ZERO_ERROR;
 558         cnv.adoptInstead(cnv_open(cnvNames[i], errorCode));
 559         if(U_FAILURE(errorCode)) {
 560             errcheckln(errorCode, "failed to open converter %s - %s", cnvNames[i], u_errorName(errorCode));
 561             continue;
 562         }
 563         UnicodeSet expected;
 564         ucnv_setFromUCallBack(cnv.getAlias(), getUnicodeSetCallback, &expected, NULL, NULL, &errorCode);
 565         if(U_FAILURE(errorCode)) {
 566             errln("failed to set the callback on converter %s - %s", cnvNames[i], u_errorName(errorCode));
 567             continue;
 568         }
 569         UConverterUnicodeSet which;
 570         for(which=UCNV_ROUNDTRIP_SET; which<UCNV_SET_COUNT; which=(UConverterUnicodeSet)((int)which+1)) {
 571             if(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET) {
 572                 ucnv_setFallback(cnv.getAlias(), TRUE);
 573             }
 574             expected.add(0, cpLimit-1);
 575             s=s0;
 576             UBool flush;
 577             do {
 578                 char *t=buffer;
 579                 flush=(UBool)(s==s0+s0Length);
 580                 ucnv_fromUnicode(cnv.getAlias(), &t, buffer+sizeof(buffer), (const UChar **)&s, s0+s0Length, NULL, flush, &errorCode);
 581                 if(U_FAILURE(errorCode)) {
 582                     if(errorCode==U_BUFFER_OVERFLOW_ERROR) {
 583                         errorCode=U_ZERO_ERROR;
 584                         continue;
 585                     } else {
 586                         break;  // unexpected error, should not occur
 587                     }
 588                 }
 589             } while(!flush);
 590             UnicodeSet set;
 591             ucnv_getUnicodeSet(cnv.getAlias(), set.toUSet(), which, &errorCode);
 592             if(cpLimit<0x110000) {
 593                 set.remove(cpLimit, 0x10ffff);
 594             }
 595             if(which==UCNV_ROUNDTRIP_SET) {
 596                 // ignore PUA code points because they will be converted even if they
 597                 // are fallbacks and when other fallbacks are turned off,
 598                 // but ucnv_getUnicodeSet(UCNV_ROUNDTRIP_SET) delivers true roundtrips
 599                 expected.remove(0xe000, 0xf8ff);
 600                 expected.remove(0xf0000, 0xffffd);
 601                 expected.remove(0x100000, 0x10fffd);
 602                 set.remove(0xe000, 0xf8ff);
 603                 set.remove(0xf0000, 0xffffd);
 604                 set.remove(0x100000, 0x10fffd);
 605             }
 606             if(set!=expected) {
 607                 // First try to see if we have different sets because ucnv_getUnicodeSet()
 608                 // added strings: The above conversion method does not tell us what strings might be convertible.
 609                 // Remove strings from the set and compare again.
 610                 // Unfortunately, there are no good, direct set methods for finding out whether there are strings
 611                 // in the set, nor for enumerating or removing just them.
 612                 // Intersect all code points with the set. The intersection will not contain strings.
 613                 UnicodeSet temp(0, 0x10ffff);
 614                 temp.retainAll(set);
 615                 set=temp;
 616             }
 617             if(set!=expected) {
 618                 UnicodeSet diffSet;
 619                 UnicodeString out;
 620
 621                 // are there items that must be in the set but are not?
 622                 (diffSet=expected).removeAll(set);
 623                 if(!diffSet.isEmpty()) {
 624                     diffSet.toPattern(out, TRUE);
 625                     if(out.length()>100) {
 626                         out.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
 627                     }
 628                     errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - which set: %d",
 629                             cnvNames[i], which);
 630                     errln(out);
 631                 }
 632
 633                 // are there items that must not be in the set but are?
 634                 (diffSet=set).removeAll(expected);
 635                 if(!diffSet.isEmpty()) {
 636                     diffSet.toPattern(out, TRUE);
 637                     if(out.length()>100) {
 638                         out.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
 639                     }
 640                     errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - which set: %d",
 641                             cnvNames[i], which);
 642                     errln(out);
 643                 }
 644             }
 645         }
 646     }
 647
 648     delete [] s0;
 649 }
 650
 651 // open testdata or ICU data converter ------------------------------------- ***
 652
 653 UConverter *
 654 ConversionTest::cnv_open(const char *name, UErrorCode &errorCode) {
 655     if(name!=NULL && *name=='+') {
 656         // Converter names that start with '+' are ignored in ICU4J tests.
 657         ++name;
 658     }
 659     if(name!=NULL && *name=='*') {
 660         /* loadTestData(): set the data directory */
 661         return ucnv_openPackage(loadTestData(errorCode), name+1, &errorCode);
 662     } else {
 663         return ucnv_open(name, &errorCode);
 664     }
 665 }
 666
 667 // output helpers ---------------------------------------------------------- ***
 668
 669 static inline char
 670 hexDigit(uint8_t digit) {
 671     return digit<=9 ? (char)('0'+digit) : (char)('a'-10+digit);
 672 }
 673
 674 static char *
 675 printBytes(const uint8_t *bytes, int32_t length, char *out) {
 676     uint8_t b;
 677
 678     if(length>0) {
 679         b=*bytes++;
 680         --length;
 681         *out++=hexDigit((uint8_t)(b>>4));
 682         *out++=hexDigit((uint8_t)(b&0xf));
 683     }
 684
 685     while(length>0) {
 686         b=*bytes++;
 687         --length;
 688         *out++=' ';
 689         *out++=hexDigit((uint8_t)(b>>4));
 690         *out++=hexDigit((uint8_t)(b&0xf));
 691     }
 692     *out++=0;
 693     return out;
 694 }
 695
 696 static char *
 697 printUnicode(const UChar *unicode, int32_t length, char *out) {
 698     UChar32 c;
 699     int32_t i;
 700
 701     for(i=0; i<length;) {
 702         if(i>0) {
 703             *out++=' ';
 704         }
 705         U16_NEXT(unicode, i, length, c);
 706         // write 4..6 digits
 707         if(c>=0x100000) {
 708             *out++='1';
 709         }
 710         if(c>=0x10000) {
 711             *out++=hexDigit((uint8_t)((c>>16)&0xf));
 712         }
 713         *out++=hexDigit((uint8_t)((c>>12)&0xf));
 714         *out++=hexDigit((uint8_t)((c>>8)&0xf));
 715         *out++=hexDigit((uint8_t)((c>>4)&0xf));
 716         *out++=hexDigit((uint8_t)(c&0xf));
 717     }
 718     *out++=0;
 719     return out;
 720 }
 721
 722 static char *
 723 printOffsets(const int32_t *offsets, int32_t length, char *out) {
 724     int32_t i, o, d;
 725
 726     if(offsets==NULL) {
 727         length=0;
 728     }
 729
 730     for(i=0; i<length; ++i) {
 731         if(i>0) {
 732             *out++=' ';
 733         }
 734         o=offsets[i];
 735
 736         // print all offsets with 2 characters each (-x, -9..99, xx)
 737         if(o<-9) {
 738             *out++='-';
 739             *out++='x';
 740         } else if(o<0) {
 741             *out++='-';
 742             *out++=(char)('0'-o);
 743         } else if(o<=99) {
 744             *out++=(d=o/10)==0 ? ' ' : (char)('0'+d);
 745             *out++=(char)('0'+o%10);
 746         } else /* o>99 */ {
 747             *out++='x';
 748             *out++='x';
 749         }
 750     }
 751     *out++=0;
 752     return out;
 753 }
 754
 755 // toUnicode test worker functions ----------------------------------------- ***
 756
 757 static int32_t
 758 stepToUnicode(ConversionCase &cc, UConverter *cnv,
 759               UChar *result, int32_t resultCapacity,
 760               int32_t *resultOffsets, /* also resultCapacity */
 761               int32_t step,
 762               UErrorCode *pErrorCode) {
 763     const char *source, *sourceLimit, *bytesLimit;
 764     UChar *target, *targetLimit, *resultLimit;
 765     UBool flush;
 766
 767     source=(const char *)cc.bytes;
 768     target=result;
 769     bytesLimit=source+cc.bytesLength;
 770     resultLimit=result+resultCapacity;
 771
 772     if(step>=0) {
 773         // call ucnv_toUnicode() with in/out buffers no larger than (step) at a time
 774         // move only one buffer (in vs. out) at a time to be extra mean
 775         // step==0 performs bulk conversion and generates offsets
 776
 777         // initialize the partial limits for the loop
 778         if(step==0) {
 779             // use the entire buffers
 780             sourceLimit=bytesLimit;
 781             targetLimit=resultLimit;
 782             flush=cc.finalFlush;
 783         } else {
 784             // start with empty partial buffers
 785             sourceLimit=source;
 786             targetLimit=target;
 787             flush=FALSE;
 788
 789             // output offsets only for bulk conversion
 790             resultOffsets=NULL;
 791         }
 792
 793         for(;;) {
 794             // resetting the opposite conversion direction must not affect this one
 795             ucnv_resetFromUnicode(cnv);
 796
 797             // convert
 798             ucnv_toUnicode(cnv,
 799                 &target, targetLimit,
 800                 &source, sourceLimit,
 801                 resultOffsets,
 802                 flush, pErrorCode);
 803
 804             // check pointers and errors
 805             if(source>sourceLimit || target>targetLimit) {
 806                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
 807                 break;
 808             } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
 809                 if(target!=targetLimit) {
 810                     // buffer overflow must only be set when the target is filled
 811                     *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
 812                     break;
 813                 } else if(targetLimit==resultLimit) {
 814                     // not just a partial overflow
 815                     break;
 816                 }
 817
 818                 // the partial target is filled, set a new limit, reset the error and continue
 819                 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
 820                 *pErrorCode=U_ZERO_ERROR;
 821             } else if(U_FAILURE(*pErrorCode)) {
 822                 // some other error occurred, done
 823                 break;
 824             } else {
 825                 if(source!=sourceLimit) {
 826                     // when no error occurs, then the input must be consumed
 827                     *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
 828                     break;
 829                 }
 830
 831                 if(sourceLimit==bytesLimit) {
 832                     // we are done
 833                     break;
 834                 }
 835
 836                 // the partial conversion succeeded, set a new limit and continue
 837                 sourceLimit=(bytesLimit-source)>=step ? source+step : bytesLimit;
 838                 flush=(UBool)(cc.finalFlush && sourceLimit==bytesLimit);
 839             }
 840         }
 841     } else /* step<0 */ {
 842         /*
 843          * step==-1: call only ucnv_getNextUChar()
 844          * otherwise alternate between ucnv_toUnicode() and ucnv_getNextUChar()
 845          *   if step==-2 or -3, then give ucnv_toUnicode() the whole remaining input,
 846          *   else give it at most (-step-2)/2 bytes
 847          */
 848         UChar32 c;
 849
 850         // end the loop by getting an index out of bounds error
 851         for(;;) {
 852             // resetting the opposite conversion direction must not affect this one
 853             ucnv_resetFromUnicode(cnv);
 854
 855             // convert
 856             if((step&1)!=0 /* odd: -1, -3, -5, ... */) {
 857                 sourceLimit=source; // use sourceLimit not as a real limit
 858                                     // but to remember the pre-getNextUChar source pointer
 859                 c=ucnv_getNextUChar(cnv, &source, bytesLimit, pErrorCode);
 860
 861                 // check pointers and errors
 862                 if(*pErrorCode==U_INDEX_OUTOFBOUNDS_ERROR) {
 863                     if(source!=bytesLimit) {
 864                         *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
 865                     } else {
 866                         *pErrorCode=U_ZERO_ERROR;
 867                     }
 868                     break;
 869                 } else if(U_FAILURE(*pErrorCode)) {
 870                     break;
 871                 }
 872                 // source may not move if c is from previous overflow
 873
 874                 if(target==resultLimit) {
 875                     *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
 876                     break;
 877                 }
 878                 if(c<=0xffff) {
 879                     *target++=(UChar)c;
 880                 } else {
 881                     *target++=U16_LEAD(c);
 882                     if(target==resultLimit) {
 883                         *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
 884                         break;
 885                     }
 886                     *target++=U16_TRAIL(c);
 887                 }
 888
 889                 // alternate between -n-1 and -n but leave -1 alone
 890                 if(step<-1) {
 891                     ++step;
 892                 }
 893             } else /* step is even */ {
 894                 // allow only one UChar output
 895                 targetLimit=target<resultLimit ? target+1 : resultLimit;
 896
 897                 // as with ucnv_getNextUChar(), we always flush (if we go to bytesLimit)
 898                 // and never output offsets
 899                 if(step==-2) {
 900                     sourceLimit=bytesLimit;
 901                 } else {
 902                     sourceLimit=source+(-step-2)/2;
 903                     if(sourceLimit>bytesLimit) {
 904                         sourceLimit=bytesLimit;
 905                     }
 906                 }
 907
 908                 ucnv_toUnicode(cnv,
 909                     &target, targetLimit,
 910                     &source, sourceLimit,
 911                     NULL, (UBool)(sourceLimit==bytesLimit), pErrorCode);
 912
 913                 // check pointers and errors
 914                 if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
 915                     if(target!=targetLimit) {
 916                         // buffer overflow must only be set when the target is filled
 917                         *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
 918                         break;
 919                     } else if(targetLimit==resultLimit) {
 920                         // not just a partial overflow
 921                         break;
 922                     }
 923
 924                     // the partial target is filled, set a new limit and continue
 925                     *pErrorCode=U_ZERO_ERROR;
 926                 } else if(U_FAILURE(*pErrorCode)) {
 927                     // some other error occurred, done
 928                     break;
 929                 } else {
 930                     if(source!=sourceLimit) {
 931                         // when no error occurs, then the input must be consumed
 932                         *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
 933                         break;
 934                     }
 935
 936                     // we are done (flush==TRUE) but we continue, to get the index out of bounds error above
 937                 }
 938
 939                 --step;
 940             }
 941         }
 942     }
 943
 944     return (int32_t)(target-result);
 945 }
 946
 947 UBool
 948 ConversionTest::ToUnicodeCase(ConversionCase &cc, UConverterToUCallback callback, const char *option) {
 949     // open the converter
 950     IcuTestErrorCode errorCode(*this, "ToUnicodeCase");
 951     LocalUConverterPointer cnv(cnv_open(cc.charset, errorCode));
 952     if(errorCode.isFailure()) {
 953         errcheckln(errorCode, "toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
 954                 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, errorCode.errorName());
 955         errorCode.reset();
 956         return FALSE;
 957     }
 958
 959     // set the callback
 960     if(callback!=NULL) {
 961         ucnv_setToUCallBack(cnv.getAlias(), callback, option, NULL, NULL, errorCode);
 962         if(U_FAILURE(errorCode)) {
 963             errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setToUCallBack() failed - %s",
 964                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
 965             return FALSE;
 966         }
 967     }
 968
 969     int32_t resultOffsets[256];
 970     UChar result[256];
 971     int32_t resultLength;
 972     UBool ok;
 973
 974     static const struct {
 975         int32_t step;
 976         const char *name;
 977     } steps[]={
 978         { 0, "bulk" }, // must be first for offsets to be checked
 979         { 1, "step=1" },
 980         { 3, "step=3" },
 981         { 7, "step=7" },
 982         { -1, "getNext" },
 983         { -2, "toU(bulk)+getNext" },
 984         { -3, "getNext+toU(bulk)" },
 985         { -4, "toU(1)+getNext" },
 986         { -5, "getNext+toU(1)" },
 987         { -12, "toU(5)+getNext" },
 988         { -13, "getNext+toU(5)" },
 989     };
 990     int32_t i, step;
 991
 992     ok=TRUE;
 993     for(i=0; i<LENGTHOF(steps) && ok; ++i) {
 994         step=steps[i].step;
 995         if(step<0 && !cc.finalFlush) {
 996             // skip ucnv_getNextUChar() if !finalFlush because
 997             // ucnv_getNextUChar() always implies flush
 998             continue;
 999         }
1000         if(step!=0) {
1001             // bulk test is first, then offsets are not checked any more
1002             cc.offsets=NULL;
1003         }
1004         else {
1005             memset(resultOffsets, -1, LENGTHOF(resultOffsets));
1006         }
1007         memset(result, -1, LENGTHOF(result));
1008         errorCode.reset();
1009         resultLength=stepToUnicode(cc, cnv.getAlias(),
1010                                 result, LENGTHOF(result),
1011                                 step==0 ? resultOffsets : NULL,
1012                                 step, errorCode);
1013         ok=checkToUnicode(
1014                 cc, cnv.getAlias(), steps[i].name,
1015                 result, resultLength,
1016                 cc.offsets!=NULL ? resultOffsets : NULL,
1017                 errorCode);
1018         if(errorCode.isFailure() || !cc.finalFlush) {
1019             // reset if an error occurred or we did not flush
1020             // otherwise do nothing to make sure that flushing resets
1021             ucnv_resetToUnicode(cnv.getAlias());
1022         }
1023         if (cc.offsets != NULL && resultOffsets[resultLength] != -1) {
1024             errln("toUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1025                 cc.caseNr, cc.charset, resultLength);
1026         }
1027         if (result[resultLength] != (UChar)-1) {
1028             errln("toUnicode[%d](%s) Conversion wrote too much to result at index %d",
1029                 cc.caseNr, cc.charset, resultLength);
1030         }
1031     }
1032
1033     // not a real loop, just a convenience for breaking out of the block
1034     while(ok && cc.finalFlush) {
1035         // test ucnv_toUChars()
1036         memset(result, 0, sizeof(result));
1037
1038         errorCode.reset();
1039         resultLength=ucnv_toUChars(cnv.getAlias(),
1040                         result, LENGTHOF(result),
1041                         (const char *)cc.bytes, cc.bytesLength,
1042                         errorCode);
1043         ok=checkToUnicode(
1044                 cc, cnv.getAlias(), "toUChars",
1045                 result, resultLength,
1046                 NULL,
1047                 errorCode);
1048         if(!ok) {
1049             break;
1050         }
1051
1052         // test preflighting
1053         // keep the correct result for simple checking
1054         errorCode.reset();
1055         resultLength=ucnv_toUChars(cnv.getAlias(),
1056                         NULL, 0,
1057                         (const char *)cc.bytes, cc.bytesLength,
1058                         errorCode);
1059         if(errorCode.get()==U_STRING_NOT_TERMINATED_WARNING || errorCode.get()==U_BUFFER_OVERFLOW_ERROR) {
1060             errorCode.reset();
1061         }
1062         ok=checkToUnicode(
1063                 cc, cnv.getAlias(), "preflight toUChars",
1064                 result, resultLength,
1065                 NULL,
1066                 errorCode);
1067         break;
1068     }
1069
1070     errorCode.reset();  // all errors have already been reported
1071     return ok;
1072 }
1073
1074 UBool
1075 ConversionTest::checkToUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1076                                const UChar *result, int32_t resultLength,
1077                                const int32_t *resultOffsets,
1078                                UErrorCode resultErrorCode) {
1079     char resultInvalidChars[8];
1080     int8_t resultInvalidLength;
1081     UErrorCode errorCode;
1082
1083     const char *msg;
1084
1085     // reset the message; NULL will mean "ok"
1086     msg=NULL;
1087
1088     errorCode=U_ZERO_ERROR;
1089     resultInvalidLength=sizeof(resultInvalidChars);
1090     ucnv_getInvalidChars(cnv, resultInvalidChars, &resultInvalidLength, &errorCode);
1091     if(U_FAILURE(errorCode)) {
1092         errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidChars() failed - %s",
1093                 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1094         return FALSE;
1095     }
1096
1097     // check everything that might have gone wrong
1098     if(cc.unicodeLength!=resultLength) {
1099         msg="wrong result length";
1100     } else if(0!=u_memcmp(cc.unicode, result, cc.unicodeLength)) {
1101         msg="wrong result string";
1102     } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.unicodeLength*sizeof(*cc.offsets))) {
1103         msg="wrong offsets";
1104     } else if(cc.outErrorCode!=resultErrorCode) {
1105         msg="wrong error code";
1106     } else if(cc.invalidLength!=resultInvalidLength) {
1107         msg="wrong length of last invalid input";
1108     } else if(0!=memcmp(cc.invalidChars, resultInvalidChars, cc.invalidLength)) {
1109         msg="wrong last invalid input";
1110     }
1111
1112     if(msg==NULL) {
1113         return TRUE;
1114     } else {
1115         char buffer[2000]; // one buffer for all strings
1116         char *s, *bytesString, *unicodeString, *resultString,
1117             *offsetsString, *resultOffsetsString,
1118             *invalidCharsString, *resultInvalidCharsString;
1119
1120         bytesString=s=buffer;
1121         s=printBytes(cc.bytes, cc.bytesLength, bytesString);
1122         s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString=s);
1123         s=printUnicode(result, resultLength, resultString=s);
1124         s=printOffsets(cc.offsets, cc.unicodeLength, offsetsString=s);
1125         s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1126         s=printBytes(cc.invalidChars, cc.invalidLength, invalidCharsString=s);
1127         s=printBytes((uint8_t *)resultInvalidChars, resultInvalidLength, resultInvalidCharsString=s);
1128
1129         if((s-buffer)>(int32_t)sizeof(buffer)) {
1130             errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkToUnicode() test output buffer overflow writing %d chars\n",
1131                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1132             exit(1);
1133         }
1134
1135         errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1136               "  bytes <%s>[%d]\n"
1137               " expected <%s>[%d]\n"
1138               "  result  <%s>[%d]\n"
1139               " offsets         <%s>\n"
1140               "  result offsets <%s>\n"
1141               " error code expected %s got %s\n"
1142               "  invalidChars expected <%s> got <%s>\n",
1143               cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1144               bytesString, cc.bytesLength,
1145               unicodeString, cc.unicodeLength,
1146               resultString, resultLength,
1147               offsetsString,
1148               resultOffsetsString,
1149               u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1150               invalidCharsString, resultInvalidCharsString);
1151
1152         return FALSE;
1153     }
1154 }
1155
1156 // fromUnicode test worker functions --------------------------------------- ***
1157
1158 static int32_t
1159 stepFromUTF8(ConversionCase &cc,
1160              UConverter *utf8Cnv, UConverter *cnv,
1161              char *result, int32_t resultCapacity,
1162              int32_t step,
1163              UErrorCode *pErrorCode) {
1164     const char *source, *sourceLimit, *utf8Limit;
1165     UChar pivotBuffer[32];
1166     UChar *pivotSource, *pivotTarget, *pivotLimit;
1167     char *target, *targetLimit, *resultLimit;
1168     UBool flush;
1169
1170     source=cc.utf8;
1171     pivotSource=pivotTarget=pivotBuffer;
1172     target=result;
1173     utf8Limit=source+cc.utf8Length;
1174     resultLimit=result+resultCapacity;
1175
1176     // call ucnv_convertEx() with in/out buffers no larger than (step) at a time
1177     // move only one buffer (in vs. out) at a time to be extra mean
1178     // step==0 performs bulk conversion
1179
1180     // initialize the partial limits for the loop
1181     if(step==0) {
1182         // use the entire buffers
1183         sourceLimit=utf8Limit;
1184         targetLimit=resultLimit;
1185         flush=cc.finalFlush;
1186
1187         pivotLimit=pivotBuffer+LENGTHOF(pivotBuffer);
1188     } else {
1189         // start with empty partial buffers
1190         sourceLimit=source;
1191         targetLimit=target;
1192         flush=FALSE;
1193
1194         // empty pivot is not allowed, make it of length step
1195         pivotLimit=pivotBuffer+step;
1196     }
1197
1198     for(;;) {
1199         // resetting the opposite conversion direction must not affect this one
1200         ucnv_resetFromUnicode(utf8Cnv);
1201         ucnv_resetToUnicode(cnv);
1202
1203         // convert
1204         ucnv_convertEx(cnv, utf8Cnv,
1205             &target, targetLimit,
1206             &source, sourceLimit,
1207             pivotBuffer, &pivotSource, &pivotTarget, pivotLimit,
1208             FALSE, flush, pErrorCode);
1209
1210         // check pointers and errors
1211         if(source>sourceLimit || target>targetLimit) {
1212             *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1213             break;
1214         } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1215             if(target!=targetLimit) {
1216                 // buffer overflow must only be set when the target is filled
1217                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1218                 break;
1219             } else if(targetLimit==resultLimit) {
1220                 // not just a partial overflow
1221                 break;
1222             }
1223
1224             // the partial target is filled, set a new limit, reset the error and continue
1225             targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1226             *pErrorCode=U_ZERO_ERROR;
1227         } else if(U_FAILURE(*pErrorCode)) {
1228             if(pivotSource==pivotBuffer) {
1229                 // toUnicode error, should not occur
1230                 // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1231                 break;
1232             } else {
1233                 // fromUnicode error
1234                 // some other error occurred, done
1235                 break;
1236             }
1237         } else {
1238             if(source!=sourceLimit) {
1239                 // when no error occurs, then the input must be consumed
1240                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1241                 break;
1242             }
1243
1244             if(sourceLimit==utf8Limit) {
1245                 // we are done
1246                 if(*pErrorCode==U_STRING_NOT_TERMINATED_WARNING) {
1247                     // ucnv_convertEx() warns about not terminating the output
1248                     // but ucnv_fromUnicode() does not and so
1249                     // checkFromUnicode() does not expect it
1250                     *pErrorCode=U_ZERO_ERROR;
1251                 }
1252                 break;
1253             }
1254
1255             // the partial conversion succeeded, set a new limit and continue
1256             sourceLimit=(utf8Limit-source)>=step ? source+step : utf8Limit;
1257             flush=(UBool)(cc.finalFlush && sourceLimit==utf8Limit);
1258         }
1259     }
1260
1261     return (int32_t)(target-result);
1262 }
1263
1264 static int32_t
1265 stepFromUnicode(ConversionCase &cc, UConverter *cnv,
1266                 char *result, int32_t resultCapacity,
1267                 int32_t *resultOffsets, /* also resultCapacity */
1268                 int32_t step,
1269                 UErrorCode *pErrorCode) {
1270     const UChar *source, *sourceLimit, *unicodeLimit;
1271     char *target, *targetLimit, *resultLimit;
1272     UBool flush;
1273
1274     source=cc.unicode;
1275     target=result;
1276     unicodeLimit=source+cc.unicodeLength;
1277     resultLimit=result+resultCapacity;
1278
1279     // call ucnv_fromUnicode() with in/out buffers no larger than (step) at a time
1280     // move only one buffer (in vs. out) at a time to be extra mean
1281     // step==0 performs bulk conversion and generates offsets
1282
1283     // initialize the partial limits for the loop
1284     if(step==0) {
1285         // use the entire buffers
1286         sourceLimit=unicodeLimit;
1287         targetLimit=resultLimit;
1288         flush=cc.finalFlush;
1289     } else {
1290         // start with empty partial buffers
1291         sourceLimit=source;
1292         targetLimit=target;
1293         flush=FALSE;
1294
1295         // output offsets only for bulk conversion
1296         resultOffsets=NULL;
1297     }
1298
1299     for(;;) {
1300         // resetting the opposite conversion direction must not affect this one
1301         ucnv_resetToUnicode(cnv);
1302
1303         // convert
1304         ucnv_fromUnicode(cnv,
1305             &target, targetLimit,
1306             &source, sourceLimit,
1307             resultOffsets,
1308             flush, pErrorCode);
1309
1310         // check pointers and errors
1311         if(source>sourceLimit || target>targetLimit) {
1312             *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1313             break;
1314         } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1315             if(target!=targetLimit) {
1316                 // buffer overflow must only be set when the target is filled
1317                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1318                 break;
1319             } else if(targetLimit==resultLimit) {
1320                 // not just a partial overflow
1321                 break;
1322             }
1323
1324             // the partial target is filled, set a new limit, reset the error and continue
1325             targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1326             *pErrorCode=U_ZERO_ERROR;
1327         } else if(U_FAILURE(*pErrorCode)) {
1328             // some other error occurred, done
1329             break;
1330         } else {
1331             if(source!=sourceLimit) {
1332                 // when no error occurs, then the input must be consumed
1333                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1334                 break;
1335             }
1336
1337             if(sourceLimit==unicodeLimit) {
1338                 // we are done
1339                 break;
1340             }
1341
1342             // the partial conversion succeeded, set a new limit and continue
1343             sourceLimit=(unicodeLimit-source)>=step ? source+step : unicodeLimit;
1344             flush=(UBool)(cc.finalFlush && sourceLimit==unicodeLimit);
1345         }
1346     }
1347
1348     return (int32_t)(target-result);
1349 }
1350
1351 UBool
1352 ConversionTest::FromUnicodeCase(ConversionCase &cc, UConverterFromUCallback callback, const char *option) {
1353     UConverter *cnv;
1354     UErrorCode errorCode;
1355
1356     // open the converter
1357     errorCode=U_ZERO_ERROR;
1358     cnv=cnv_open(cc.charset, errorCode);
1359     if(U_FAILURE(errorCode)) {
1360         errcheckln(errorCode, "fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
1361                 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1362         return FALSE;
1363     }
1364     ucnv_resetToUnicode(utf8Cnv);
1365
1366     // set the callback
1367     if(callback!=NULL) {
1368         ucnv_setFromUCallBack(cnv, callback, option, NULL, NULL, &errorCode);
1369         if(U_FAILURE(errorCode)) {
1370             errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setFromUCallBack() failed - %s",
1371                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1372             ucnv_close(cnv);
1373             return FALSE;
1374         }
1375     }
1376
1377     // set the fallbacks flag
1378     // TODO change with Jitterbug 2401, then add a similar call for toUnicode too
1379     ucnv_setFallback(cnv, cc.fallbacks);
1380
1381     // set the subchar
1382     int32_t length;
1383
1384     if(cc.setSub>0) {
1385         length=(int32_t)strlen(cc.subchar);
1386         ucnv_setSubstChars(cnv, cc.subchar, (int8_t)length, &errorCode);
1387         if(U_FAILURE(errorCode)) {
1388             errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstChars() failed - %s",
1389                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1390             ucnv_close(cnv);
1391             return FALSE;
1392         }
1393     } else if(cc.setSub<0) {
1394         ucnv_setSubstString(cnv, cc.subString, -1, &errorCode);
1395         if(U_FAILURE(errorCode)) {
1396             errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstString() failed - %s",
1397                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1398             ucnv_close(cnv);
1399             return FALSE;
1400         }
1401     }
1402
1403     // convert unicode to utf8
1404     char utf8[256];
1405     cc.utf8=utf8;
1406     u_strToUTF8(utf8, LENGTHOF(utf8), &cc.utf8Length,
1407                 cc.unicode, cc.unicodeLength,
1408                 &errorCode);
1409     if(U_FAILURE(errorCode)) {
1410         // skip UTF-8 testing of a string with an unpaired surrogate,
1411         // or of one that's too long
1412         // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1413         cc.utf8Length=-1;
1414     }
1415
1416     int32_t resultOffsets[256];
1417     char result[256];
1418     int32_t resultLength;
1419     UBool ok;
1420
1421     static const struct {
1422         int32_t step;
1423         const char *name, *utf8Name;
1424     } steps[]={
1425         { 0, "bulk",   "utf8" }, // must be first for offsets to be checked
1426         { 1, "step=1", "utf8 step=1" },
1427         { 3, "step=3", "utf8 step=3" },
1428         { 7, "step=7", "utf8 step=7" }
1429     };
1430     int32_t i, step;
1431
1432     ok=TRUE;
1433     for(i=0; i<LENGTHOF(steps) && ok; ++i) {
1434         step=steps[i].step;
1435         memset(resultOffsets, -1, LENGTHOF(resultOffsets));
1436         memset(result, -1, LENGTHOF(result));
1437         errorCode=U_ZERO_ERROR;
1438         resultLength=stepFromUnicode(cc, cnv,
1439                                 result, LENGTHOF(result),
1440                                 step==0 ? resultOffsets : NULL,
1441                                 step, &errorCode);
1442         ok=checkFromUnicode(
1443                 cc, cnv, steps[i].name,
1444                 (uint8_t *)result, resultLength,
1445                 cc.offsets!=NULL ? resultOffsets : NULL,
1446                 errorCode);
1447         if(U_FAILURE(errorCode) || !cc.finalFlush) {
1448             // reset if an error occurred or we did not flush
1449             // otherwise do nothing to make sure that flushing resets
1450             ucnv_resetFromUnicode(cnv);
1451         }
1452         if (resultOffsets[resultLength] != -1) {
1453             errln("fromUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1454                 cc.caseNr, cc.charset, resultLength);
1455         }
1456         if (result[resultLength] != (char)-1) {
1457             errln("fromUnicode[%d](%s) Conversion wrote too much to result at index %d",
1458                 cc.caseNr, cc.charset, resultLength);
1459         }
1460
1461         // bulk test is first, then offsets are not checked any more
1462         cc.offsets=NULL;
1463
1464         // test direct conversion from UTF-8
1465         if(cc.utf8Length>=0) {
1466             errorCode=U_ZERO_ERROR;
1467             resultLength=stepFromUTF8(cc, utf8Cnv, cnv,
1468                                     result, LENGTHOF(result),
1469                                     step, &errorCode);
1470             ok=checkFromUnicode(
1471                     cc, cnv, steps[i].utf8Name,
1472                     (uint8_t *)result, resultLength,
1473                     NULL,
1474                     errorCode);
1475             if(U_FAILURE(errorCode) || !cc.finalFlush) {
1476                 // reset if an error occurred or we did not flush
1477                 // otherwise do nothing to make sure that flushing resets
1478                 ucnv_resetToUnicode(utf8Cnv);
1479                 ucnv_resetFromUnicode(cnv);
1480             }
1481         }
1482     }
1483
1484     // not a real loop, just a convenience for breaking out of the block
1485     while(ok && cc.finalFlush) {
1486         // test ucnv_fromUChars()
1487         memset(result, 0, sizeof(result));
1488
1489         errorCode=U_ZERO_ERROR;
1490         resultLength=ucnv_fromUChars(cnv,
1491                         result, LENGTHOF(result),
1492                         cc.unicode, cc.unicodeLength,
1493                         &errorCode);
1494         ok=checkFromUnicode(
1495                 cc, cnv, "fromUChars",
1496                 (uint8_t *)result, resultLength,
1497                 NULL,
1498                 errorCode);
1499         if(!ok) {
1500             break;
1501         }
1502
1503         // test preflighting
1504         // keep the correct result for simple checking
1505         errorCode=U_ZERO_ERROR;
1506         resultLength=ucnv_fromUChars(cnv,
1507                         NULL, 0,
1508                         cc.unicode, cc.unicodeLength,
1509                         &errorCode);
1510         if(errorCode==U_STRING_NOT_TERMINATED_WARNING || errorCode==U_BUFFER_OVERFLOW_ERROR) {
1511             errorCode=U_ZERO_ERROR;
1512         }
1513         ok=checkFromUnicode(
1514                 cc, cnv, "preflight fromUChars",
1515                 (uint8_t *)result, resultLength,
1516                 NULL,
1517                 errorCode);
1518         break;
1519     }
1520
1521     ucnv_close(cnv);
1522     return ok;
1523 }
1524
1525 UBool
1526 ConversionTest::checkFromUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1527                                  const uint8_t *result, int32_t resultLength,
1528                                  const int32_t *resultOffsets,
1529                                  UErrorCode resultErrorCode) {
1530     UChar resultInvalidUChars[8];
1531     int8_t resultInvalidLength;
1532     UErrorCode errorCode;
1533
1534     const char *msg;
1535
1536     // reset the message; NULL will mean "ok"
1537     msg=NULL;
1538
1539     errorCode=U_ZERO_ERROR;
1540     resultInvalidLength=LENGTHOF(resultInvalidUChars);
1541     ucnv_getInvalidUChars(cnv, resultInvalidUChars, &resultInvalidLength, &errorCode);
1542     if(U_FAILURE(errorCode)) {
1543         errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidUChars() failed - %s",
1544                 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1545         return FALSE;
1546     }
1547
1548     // check everything that might have gone wrong
1549     if(cc.bytesLength!=resultLength) {
1550         msg="wrong result length";
1551     } else if(0!=memcmp(cc.bytes, result, cc.bytesLength)) {
1552         msg="wrong result string";
1553     } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.bytesLength*sizeof(*cc.offsets))) {
1554         msg="wrong offsets";
1555     } else if(cc.outErrorCode!=resultErrorCode) {
1556         msg="wrong error code";
1557     } else if(cc.invalidLength!=resultInvalidLength) {
1558         msg="wrong length of last invalid input";
1559     } else if(0!=u_memcmp(cc.invalidUChars, resultInvalidUChars, cc.invalidLength)) {
1560         msg="wrong last invalid input";
1561     }
1562
1563     if(msg==NULL) {
1564         return TRUE;
1565     } else {
1566         char buffer[2000]; // one buffer for all strings
1567         char *s, *unicodeString, *bytesString, *resultString,
1568             *offsetsString, *resultOffsetsString,
1569             *invalidCharsString, *resultInvalidUCharsString;
1570
1571         unicodeString=s=buffer;
1572         s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString);
1573         s=printBytes(cc.bytes, cc.bytesLength, bytesString=s);
1574         s=printBytes(result, resultLength, resultString=s);
1575         s=printOffsets(cc.offsets, cc.bytesLength, offsetsString=s);
1576         s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1577         s=printUnicode(cc.invalidUChars, cc.invalidLength, invalidCharsString=s);
1578         s=printUnicode(resultInvalidUChars, resultInvalidLength, resultInvalidUCharsString=s);
1579
1580         if((s-buffer)>(int32_t)sizeof(buffer)) {
1581             errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkFromUnicode() test output buffer overflow writing %d chars\n",
1582                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1583             exit(1);
1584         }
1585
1586         errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1587               "  unicode <%s>[%d]\n"
1588               " expected <%s>[%d]\n"
1589               "  result  <%s>[%d]\n"
1590               " offsets         <%s>\n"
1591               "  result offsets <%s>\n"
1592               " error code expected %s got %s\n"
1593               "  invalidChars expected <%s> got <%s>\n",
1594               cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1595               unicodeString, cc.unicodeLength,
1596               bytesString, cc.bytesLength,
1597               resultString, resultLength,
1598               offsetsString,
1599               resultOffsetsString,
1600               u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1601               invalidCharsString, resultInvalidUCharsString);
1602
1603         return FALSE;
1604     }
1605 }
1606
1607 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */