icuSources/test/intltest/convtest.cpp

   1 /*
   2 *******************************************************************************
   3 *
   4 *   Copyright (C) 2003-2010, International Business Machines
   5 *   Corporation and others.  All Rights Reserved.
   6 *
   7 *******************************************************************************
   8 *   file name:  convtest.cpp
   9 *   encoding:   US-ASCII
  10 *   tab size:   8 (not used)
  11 *   indentation:4
  12 *
  13 *   created on: 2003jul15
  14 *   created by: Markus W. Scherer
  15 *
  16 *   Test file for data-driven conversion tests.
  17 */
  18
  19 #include "unicode/utypes.h"
  20
  21 #if !UCONFIG_NO_LEGACY_CONVERSION
  22 /*
  23  * Note: Turning off all of convtest.cpp if !UCONFIG_NO_LEGACY_CONVERSION
  24  * is slightly unnecessary - it removes tests for Unicode charsets
  25  * like UTF-8 that should work.
  26  * However, there is no easy way for the test to detect whether a test case
  27  * is for a Unicode charset, so it would be difficult to only exclude those.
  28  * Also, regular testing of ICU is done with all modules on, therefore
  29  * not testing conversion for a custom configuration like this should be ok.
  30  */
  31
  32 #include "unicode/ucnv.h"
  33 #include "unicode/unistr.h"
  34 #include "unicode/parsepos.h"
  35 #include "unicode/uniset.h"
  36 #include "unicode/ustring.h"
  37 #include "unicode/ures.h"
  38 #include "convtest.h"
  39 #include "unicode/tstdtmod.h"
  40 #include <string.h>
  41 #include <stdlib.h>
  42
  43 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
  44
  45 enum {
  46     // characters used in test data for callbacks
  47     SUB_CB='?',
  48     SKIP_CB='0',
  49     STOP_CB='.',
  50     ESC_CB='&'
  51 };
  52
  53 ConversionTest::ConversionTest() {
  54     UErrorCode errorCode=U_ZERO_ERROR;
  55     utf8Cnv=ucnv_open("UTF-8", &errorCode);
  56     ucnv_setToUCallBack(utf8Cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
  57     if(U_FAILURE(errorCode)) {
  58         errln("unable to open UTF-8 converter");
  59     }
  60 }
  61
  62 ConversionTest::~ConversionTest() {
  63     ucnv_close(utf8Cnv);
  64 }
  65
  66 void
  67 ConversionTest::runIndexedTest(int32_t index, UBool exec, const char *&name, char * /*par*/) {
  68     if (exec) logln("TestSuite ConversionTest: ");
  69     switch (index) {
  70 #if !UCONFIG_NO_FILE_IO
  71         case 0: name="TestToUnicode"; if (exec) TestToUnicode(); break;
  72         case 1: name="TestFromUnicode"; if (exec) TestFromUnicode(); break;
  73         case 2: name="TestGetUnicodeSet"; if (exec) TestGetUnicodeSet(); break;
  74 #else
  75         case 0:
  76         case 1:
  77         case 2: name="skip"; break;
  78 #endif
  79         case 3: name="TestGetUnicodeSet2"; if (exec) TestGetUnicodeSet2(); break;
  80         default: name=""; break; //needed to end loop
  81     }
  82 }
  83
  84 // test data interface ----------------------------------------------------- ***
  85
  86 void
  87 ConversionTest::TestToUnicode() {
  88     ConversionCase cc;
  89     char charset[100], cbopt[4];
  90     const char *option;
  91     UnicodeString s, unicode;
  92     int32_t offsetsLength;
  93     UConverterToUCallback callback;
  94
  95     TestDataModule *dataModule;
  96     TestData *testData;
  97     const DataMap *testCase;
  98     UErrorCode errorCode;
  99     int32_t i;
 100
 101     errorCode=U_ZERO_ERROR;
 102     dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
 103     if(U_SUCCESS(errorCode)) {
 104         testData=dataModule->createTestData("toUnicode", errorCode);
 105         if(U_SUCCESS(errorCode)) {
 106             for(i=0; testData->nextCase(testCase, errorCode); ++i) {
 107                 if(U_FAILURE(errorCode)) {
 108                     errln("error retrieving conversion/toUnicode test case %d - %s",
 109                             i, u_errorName(errorCode));
 110                     errorCode=U_ZERO_ERROR;
 111                     continue;
 112                 }
 113
 114                 cc.caseNr=i;
 115
 116                 s=testCase->getString("charset", errorCode);
 117                 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
 118                 cc.charset=charset;
 119
 120                 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
 121                 unicode=testCase->getString("unicode", errorCode);
 122                 cc.unicode=unicode.getBuffer();
 123                 cc.unicodeLength=unicode.length();
 124
 125                 offsetsLength=0;
 126                 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
 127                 if(offsetsLength==0) {
 128                     cc.offsets=NULL;
 129                 } else if(offsetsLength!=unicode.length()) {
 130                     errln("toUnicode[%d] unicode[%d] and offsets[%d] must have the same length",
 131                             i, unicode.length(), offsetsLength);
 132                     errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 133                 }
 134
 135                 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
 136                 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
 137
 138                 s=testCase->getString("errorCode", errorCode);
 139                 if(s==UNICODE_STRING("invalid", 7)) {
 140                     cc.outErrorCode=U_INVALID_CHAR_FOUND;
 141                 } else if(s==UNICODE_STRING("illegal", 7)) {
 142                     cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
 143                 } else if(s==UNICODE_STRING("truncated", 9)) {
 144                     cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
 145                 } else if(s==UNICODE_STRING("illesc", 6)) {
 146                     cc.outErrorCode=U_ILLEGAL_ESCAPE_SEQUENCE;
 147                 } else if(s==UNICODE_STRING("unsuppesc", 9)) {
 148                     cc.outErrorCode=U_UNSUPPORTED_ESCAPE_SEQUENCE;
 149                 } else {
 150                     cc.outErrorCode=U_ZERO_ERROR;
 151                 }
 152
 153                 s=testCase->getString("callback", errorCode);
 154                 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
 155                 cc.cbopt=cbopt;
 156                 switch(cbopt[0]) {
 157                 case SUB_CB:
 158                     callback=UCNV_TO_U_CALLBACK_SUBSTITUTE;
 159                     break;
 160                 case SKIP_CB:
 161                     callback=UCNV_TO_U_CALLBACK_SKIP;
 162                     break;
 163                 case STOP_CB:
 164                     callback=UCNV_TO_U_CALLBACK_STOP;
 165                     break;
 166                 case ESC_CB:
 167                     callback=UCNV_TO_U_CALLBACK_ESCAPE;
 168                     break;
 169                 default:
 170                     callback=NULL;
 171                     break;
 172                 }
 173                 option=callback==NULL ? cbopt : cbopt+1;
 174                 if(*option==0) {
 175                     option=NULL;
 176                 }
 177
 178                 cc.invalidChars=testCase->getBinary(cc.invalidLength, "invalidChars", errorCode);
 179
 180                 if(U_FAILURE(errorCode)) {
 181                     errln("error parsing conversion/toUnicode test case %d - %s",
 182                             i, u_errorName(errorCode));
 183                     errorCode=U_ZERO_ERROR;
 184                 } else {
 185                     logln("TestToUnicode[%d] %s", i, charset);
 186                     ToUnicodeCase(cc, callback, option);
 187                 }
 188             }
 189             delete testData;
 190         }
 191         delete dataModule;
 192     }
 193     else {
 194         dataerrln("Could not load test conversion data");
 195     }
 196 }
 197
 198 void
 199 ConversionTest::TestFromUnicode() {
 200     ConversionCase cc;
 201     char charset[100], cbopt[4];
 202     const char *option;
 203     UnicodeString s, unicode, invalidUChars;
 204     int32_t offsetsLength, index;
 205     UConverterFromUCallback callback;
 206
 207     TestDataModule *dataModule;
 208     TestData *testData;
 209     const DataMap *testCase;
 210     const UChar *p;
 211     UErrorCode errorCode;
 212     int32_t i, length;
 213
 214     errorCode=U_ZERO_ERROR;
 215     dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
 216     if(U_SUCCESS(errorCode)) {
 217         testData=dataModule->createTestData("fromUnicode", errorCode);
 218         if(U_SUCCESS(errorCode)) {
 219             for(i=0; testData->nextCase(testCase, errorCode); ++i) {
 220                 if(U_FAILURE(errorCode)) {
 221                     errln("error retrieving conversion/fromUnicode test case %d - %s",
 222                             i, u_errorName(errorCode));
 223                     errorCode=U_ZERO_ERROR;
 224                     continue;
 225                 }
 226
 227                 cc.caseNr=i;
 228
 229                 s=testCase->getString("charset", errorCode);
 230                 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
 231                 cc.charset=charset;
 232
 233                 unicode=testCase->getString("unicode", errorCode);
 234                 cc.unicode=unicode.getBuffer();
 235                 cc.unicodeLength=unicode.length();
 236                 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
 237
 238                 offsetsLength=0;
 239                 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
 240                 if(offsetsLength==0) {
 241                     cc.offsets=NULL;
 242                 } else if(offsetsLength!=cc.bytesLength) {
 243                     errln("fromUnicode[%d] bytes[%d] and offsets[%d] must have the same length",
 244                             i, cc.bytesLength, offsetsLength);
 245                     errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 246                 }
 247
 248                 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
 249                 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
 250
 251                 s=testCase->getString("errorCode", errorCode);
 252                 if(s==UNICODE_STRING("invalid", 7)) {
 253                     cc.outErrorCode=U_INVALID_CHAR_FOUND;
 254                 } else if(s==UNICODE_STRING("illegal", 7)) {
 255                     cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
 256                 } else if(s==UNICODE_STRING("truncated", 9)) {
 257                     cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
 258                 } else {
 259                     cc.outErrorCode=U_ZERO_ERROR;
 260                 }
 261
 262                 s=testCase->getString("callback", errorCode);
 263                 cc.setSub=0; // default: no subchar
 264
 265                 if((index=s.indexOf((UChar)0))>0) {
 266                     // read NUL-separated subchar first, if any
 267                     // copy the subchar from Latin-1 characters
 268                     // start after the NUL
 269                     p=s.getTerminatedBuffer();
 270                     length=index+1;
 271                     p+=length;
 272                     length=s.length()-length;
 273                     if(length<=0 || length>=(int32_t)sizeof(cc.subchar)) {
 274                         errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 275                     } else {
 276                         int32_t j;
 277
 278                         for(j=0; j<length; ++j) {
 279                             cc.subchar[j]=(char)p[j];
 280                         }
 281                         // NUL-terminate the subchar
 282                         cc.subchar[j]=0;
 283                         cc.setSub=1;
 284                     }
 285
 286                     // remove the NUL and subchar from s
 287                     s.truncate(index);
 288                 } else if((index=s.indexOf((UChar)0x3d))>0) /* '=' */ {
 289                     // read a substitution string, separated by an equal sign
 290                     p=s.getBuffer()+index+1;
 291                     length=s.length()-(index+1);
 292                     if(length<0 || length>=LENGTHOF(cc.subString)) {
 293                         errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 294                     } else {
 295                         u_memcpy(cc.subString, p, length);
 296                         // NUL-terminate the subString
 297                         cc.subString[length]=0;
 298                         cc.setSub=-1;
 299                     }
 300
 301                     // remove the equal sign and subString from s
 302                     s.truncate(index);
 303                 }
 304
 305                 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
 306                 cc.cbopt=cbopt;
 307                 switch(cbopt[0]) {
 308                 case SUB_CB:
 309                     callback=UCNV_FROM_U_CALLBACK_SUBSTITUTE;
 310                     break;
 311                 case SKIP_CB:
 312                     callback=UCNV_FROM_U_CALLBACK_SKIP;
 313                     break;
 314                 case STOP_CB:
 315                     callback=UCNV_FROM_U_CALLBACK_STOP;
 316                     break;
 317                 case ESC_CB:
 318                     callback=UCNV_FROM_U_CALLBACK_ESCAPE;
 319                     break;
 320                 default:
 321                     callback=NULL;
 322                     break;
 323                 }
 324                 option=callback==NULL ? cbopt : cbopt+1;
 325                 if(*option==0) {
 326                     option=NULL;
 327                 }
 328
 329                 invalidUChars=testCase->getString("invalidUChars", errorCode);
 330                 cc.invalidUChars=invalidUChars.getBuffer();
 331                 cc.invalidLength=invalidUChars.length();
 332
 333                 if(U_FAILURE(errorCode)) {
 334                     errln("error parsing conversion/fromUnicode test case %d - %s",
 335                             i, u_errorName(errorCode));
 336                     errorCode=U_ZERO_ERROR;
 337                 } else {
 338                     logln("TestFromUnicode[%d] %s", i, charset);
 339                     FromUnicodeCase(cc, callback, option);
 340                 }
 341             }
 342             delete testData;
 343         }
 344         delete dataModule;
 345     }
 346     else {
 347         dataerrln("Could not load test conversion data");
 348     }
 349 }
 350
 351 static const UChar ellipsis[]={ 0x2e, 0x2e, 0x2e };
 352
 353 void
 354 ConversionTest::TestGetUnicodeSet() {
 355     char charset[100];
 356     UnicodeString s, map, mapnot;
 357     int32_t which;
 358
 359     ParsePosition pos;
 360     UnicodeSet cnvSet, mapSet, mapnotSet, diffSet;
 361     UnicodeSet *cnvSetPtr = &cnvSet;
 362     LocalUConverterPointer cnv;
 363
 364     TestDataModule *dataModule;
 365     TestData *testData;
 366     const DataMap *testCase;
 367     UErrorCode errorCode;
 368     int32_t i;
 369
 370     errorCode=U_ZERO_ERROR;
 371     dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
 372     if(U_SUCCESS(errorCode)) {
 373         testData=dataModule->createTestData("getUnicodeSet", errorCode);
 374         if(U_SUCCESS(errorCode)) {
 375             for(i=0; testData->nextCase(testCase, errorCode); ++i) {
 376                 if(U_FAILURE(errorCode)) {
 377                     errln("error retrieving conversion/getUnicodeSet test case %d - %s",
 378                             i, u_errorName(errorCode));
 379                     errorCode=U_ZERO_ERROR;
 380                     continue;
 381                 }
 382
 383                 s=testCase->getString("charset", errorCode);
 384                 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
 385
 386                 map=testCase->getString("map", errorCode);
 387                 mapnot=testCase->getString("mapnot", errorCode);
 388
 389                 which=testCase->getInt28("which", errorCode);
 390
 391                 if(U_FAILURE(errorCode)) {
 392                     errln("error parsing conversion/getUnicodeSet test case %d - %s",
 393                             i, u_errorName(errorCode));
 394                     errorCode=U_ZERO_ERROR;
 395                     continue;
 396                 }
 397
 398                 // test this test case
 399                 mapSet.clear();
 400                 mapnotSet.clear();
 401
 402                 pos.setIndex(0);
 403                 mapSet.applyPattern(map, pos, 0, NULL, errorCode);
 404                 if(U_FAILURE(errorCode) || pos.getIndex()!=map.length()) {
 405                     errln("error creating the map set for conversion/getUnicodeSet test case %d - %s\n"
 406                           "    error index %d  index %d  U+%04x",
 407                             i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), map.char32At(pos.getIndex()));
 408                     errorCode=U_ZERO_ERROR;
 409                     continue;
 410                 }
 411
 412                 pos.setIndex(0);
 413                 mapnotSet.applyPattern(mapnot, pos, 0, NULL, errorCode);
 414                 if(U_FAILURE(errorCode) || pos.getIndex()!=mapnot.length()) {
 415                     errln("error creating the mapnot set for conversion/getUnicodeSet test case %d - %s\n"
 416                           "    error index %d  index %d  U+%04x",
 417                             i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), mapnot.char32At(pos.getIndex()));
 418                     errorCode=U_ZERO_ERROR;
 419                     continue;
 420                 }
 421
 422                 logln("TestGetUnicodeSet[%d] %s", i, charset);
 423
 424                 cnv.adoptInstead(cnv_open(charset, errorCode));
 425                 if(U_FAILURE(errorCode)) {
 426                     errcheckln(errorCode, "error opening \"%s\" for conversion/getUnicodeSet test case %d - %s",
 427                             charset, i, u_errorName(errorCode));
 428                     errorCode=U_ZERO_ERROR;
 429                     continue;
 430                 }
 431
 432                 ucnv_getUnicodeSet(cnv.getAlias(), cnvSetPtr->toUSet(), (UConverterUnicodeSet)which, &errorCode);
 433
 434                 if(U_FAILURE(errorCode)) {
 435                     errln("error in ucnv_getUnicodeSet(\"%s\") for conversion/getUnicodeSet test case %d - %s",
 436                             charset, i, u_errorName(errorCode));
 437                     errorCode=U_ZERO_ERROR;
 438                     continue;
 439                 }
 440
 441                 // are there items that must be in cnvSet but are not?
 442                 (diffSet=mapSet).removeAll(cnvSet);
 443                 if(!diffSet.isEmpty()) {
 444                     diffSet.toPattern(s, TRUE);
 445                     if(s.length()>100) {
 446                         s.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
 447                     }
 448                     errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - conversion/getUnicodeSet test case %d",
 449                             charset, i);
 450                     errln(s);
 451                 }
 452
 453                 // are there items that must not be in cnvSet but are?
 454                 (diffSet=mapnotSet).retainAll(cnvSet);
 455                 if(!diffSet.isEmpty()) {
 456                     diffSet.toPattern(s, TRUE);
 457                     if(s.length()>100) {
 458                         s.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
 459                     }
 460                     errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - conversion/getUnicodeSet test case %d",
 461                             charset, i);
 462                     errln(s);
 463                 }
 464             }
 465             delete testData;
 466         }
 467         delete dataModule;
 468     }
 469     else {
 470         dataerrln("Could not load test conversion data");
 471     }
 472 }
 473
 474 U_CDECL_BEGIN
 475 static void U_CALLCONV
 476 getUnicodeSetCallback(const void *context,
 477                       UConverterFromUnicodeArgs * /*fromUArgs*/,
 478                       const UChar* /*codeUnits*/,
 479                       int32_t /*length*/,
 480                       UChar32 codePoint,
 481                       UConverterCallbackReason reason,
 482                       UErrorCode *pErrorCode) {
 483     if(reason<=UCNV_IRREGULAR) {
 484         ((UnicodeSet *)context)->remove(codePoint);  // the converter cannot convert this code point
 485         *pErrorCode=U_ZERO_ERROR;                    // skip
 486     }  // else ignore the reset, close and clone calls.
 487 }
 488 U_CDECL_END
 489
 490 // Compare ucnv_getUnicodeSet() with the set of characters that can be converted.
 491 void
 492 ConversionTest::TestGetUnicodeSet2() {
 493     // Build a string with all code points.
 494     UChar32 cpLimit;
 495     int32_t s0Length;
 496     if(quick) {
 497         cpLimit=s0Length=0x10000;  // BMP only
 498     } else {
 499         cpLimit=0x110000;
 500         s0Length=0x10000+0x200000;  // BMP + surrogate pairs
 501     }
 502     UChar *s0=new UChar[s0Length];
 503     if(s0==NULL) {
 504         return;
 505     }
 506     UChar *s=s0;
 507     UChar32 c;
 508     UChar c2;
 509     // low BMP
 510     for(c=0; c<=0xd7ff; ++c) {
 511         *s++=(UChar)c;
 512     }
 513     // trail surrogates
 514     for(c=0xdc00; c<=0xdfff; ++c) {
 515         *s++=(UChar)c;
 516     }
 517     // lead surrogates
 518     // (after trails so that there is not even one surrogate pair in between)
 519     for(c=0xd800; c<=0xdbff; ++c) {
 520         *s++=(UChar)c;
 521     }
 522     // high BMP
 523     for(c=0xe000; c<=0xffff; ++c) {
 524         *s++=(UChar)c;
 525     }
 526     // supplementary code points = surrogate pairs
 527     if(cpLimit==0x110000) {
 528         for(c=0xd800; c<=0xdbff; ++c) {
 529             for(c2=0xdc00; c2<=0xdfff; ++c2) {
 530                 *s++=(UChar)c;
 531                 *s++=c2;
 532             }
 533         }
 534     }
 535
 536     static const char *const cnvNames[]={
 537         "UTF-8",
 538         "UTF-7",
 539         "UTF-16",
 540         "US-ASCII",
 541         "ISO-8859-1",
 542         "windows-1252",
 543         "Shift-JIS",
 544         "ibm-1390",  // EBCDIC_STATEFUL table
 545         "ibm-16684",  // DBCS-only extension table based on EBCDIC_STATEFUL table
 546         "HZ",
 547         "ISO-2022-JP",
 548         "JIS7",
 549         "ISO-2022-CN",
 550         "ISO-2022-CN-EXT",
 551         "LMBCS"
 552     };
 553     LocalUConverterPointer cnv;
 554     char buffer[1024];
 555     int32_t i;
 556     for(i=0; i<LENGTHOF(cnvNames); ++i) {
 557         UErrorCode errorCode=U_ZERO_ERROR;
 558         cnv.adoptInstead(cnv_open(cnvNames[i], errorCode));
 559         if(U_FAILURE(errorCode)) {
 560             errcheckln(errorCode, "failed to open converter %s - %s", cnvNames[i], u_errorName(errorCode));
 561             continue;
 562         }
 563         UnicodeSet expected;
 564         ucnv_setFromUCallBack(cnv.getAlias(), getUnicodeSetCallback, &expected, NULL, NULL, &errorCode);
 565         if(U_FAILURE(errorCode)) {
 566             errln("failed to set the callback on converter %s - %s", cnvNames[i], u_errorName(errorCode));
 567             continue;
 568         }
 569         UConverterUnicodeSet which;
 570         for(which=UCNV_ROUNDTRIP_SET; which<UCNV_SET_COUNT; which=(UConverterUnicodeSet)((int)which+1)) {
 571             if(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET) {
 572                 ucnv_setFallback(cnv.getAlias(), TRUE);
 573             }
 574             expected.add(0, cpLimit-1);
 575             s=s0;
 576             UBool flush;
 577             do {
 578                 char *t=buffer;
 579                 flush=(UBool)(s==s0+s0Length);
 580                 ucnv_fromUnicode(cnv.getAlias(), &t, buffer+sizeof(buffer), (const UChar **)&s, s0+s0Length, NULL, flush, &errorCode);
 581                 if(U_FAILURE(errorCode)) {
 582                     if(errorCode==U_BUFFER_OVERFLOW_ERROR) {
 583                         errorCode=U_ZERO_ERROR;
 584                         continue;
 585                     } else {
 586                         break;  // unexpected error, should not occur
 587                     }
 588                 }
 589             } while(!flush);
 590             UnicodeSet set;
 591             ucnv_getUnicodeSet(cnv.getAlias(), set.toUSet(), which, &errorCode);
 592             if(cpLimit<0x110000) {
 593                 set.remove(cpLimit, 0x10ffff);
 594             }
 595             if(which==UCNV_ROUNDTRIP_SET) {
 596                 // ignore PUA code points because they will be converted even if they
 597                 // are fallbacks and when other fallbacks are turned off,
 598                 // but ucnv_getUnicodeSet(UCNV_ROUNDTRIP_SET) delivers true roundtrips
 599                 expected.remove(0xe000, 0xf8ff);
 600                 expected.remove(0xf0000, 0xffffd);
 601                 expected.remove(0x100000, 0x10fffd);
 602                 set.remove(0xe000, 0xf8ff);
 603                 set.remove(0xf0000, 0xffffd);
 604                 set.remove(0x100000, 0x10fffd);
 605             }
 606             if(set!=expected) {
 607                 // First try to see if we have different sets because ucnv_getUnicodeSet()
 608                 // added strings: The above conversion method does not tell us what strings might be convertible.
 609                 // Remove strings from the set and compare again.
 610                 // Unfortunately, there are no good, direct set methods for finding out whether there are strings
 611                 // in the set, nor for enumerating or removing just them.
 612                 // Intersect all code points with the set. The intersection will not contain strings.
 613                 UnicodeSet temp(0, 0x10ffff);
 614                 temp.retainAll(set);
 615                 set=temp;
 616             }
 617             if(set!=expected) {
 618                 UnicodeSet diffSet;
 619                 UnicodeString out;
 620
 621                 // are there items that must be in the set but are not?
 622                 (diffSet=expected).removeAll(set);
 623                 if(!diffSet.isEmpty()) {
 624                     diffSet.toPattern(out, TRUE);
 625                     if(out.length()>100) {
 626                         out.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
 627                     }
 628                     errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - which set: %d",
 629                             cnvNames[i], which);
 630                     errln(out);
 631                 }
 632
 633                 // are there items that must not be in the set but are?
 634                 (diffSet=set).removeAll(expected);
 635                 if(!diffSet.isEmpty()) {
 636                     diffSet.toPattern(out, TRUE);
 637                     if(out.length()>100) {
 638                         out.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
 639                     }
 640                     errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - which set: %d",
 641                             cnvNames[i], which);
 642                     errln(out);
 643                 }
 644             }
 645         }
 646     }
 647
 648     delete [] s0;
 649 }
 650
 651 // open testdata or ICU data converter ------------------------------------- ***
 652
 653 UConverter *
 654 ConversionTest::cnv_open(const char *name, UErrorCode &errorCode) {
 655     if(name!=NULL && *name=='*') {
 656         /* loadTestData(): set the data directory */
 657         return ucnv_openPackage(loadTestData(errorCode), name+1, &errorCode);
 658     } else if(name!=NULL && *name=='+') {
 659         return ucnv_open((name+1), &errorCode);
 660     } else {
 661         return ucnv_open(name, &errorCode);
 662     }
 663 }
 664
 665 // output helpers ---------------------------------------------------------- ***
 666
 667 static inline char
 668 hexDigit(uint8_t digit) {
 669     return digit<=9 ? (char)('0'+digit) : (char)('a'-10+digit);
 670 }
 671
 672 static char *
 673 printBytes(const uint8_t *bytes, int32_t length, char *out) {
 674     uint8_t b;
 675
 676     if(length>0) {
 677         b=*bytes++;
 678         --length;
 679         *out++=hexDigit((uint8_t)(b>>4));
 680         *out++=hexDigit((uint8_t)(b&0xf));
 681     }
 682
 683     while(length>0) {
 684         b=*bytes++;
 685         --length;
 686         *out++=' ';
 687         *out++=hexDigit((uint8_t)(b>>4));
 688         *out++=hexDigit((uint8_t)(b&0xf));
 689     }
 690     *out++=0;
 691     return out;
 692 }
 693
 694 static char *
 695 printUnicode(const UChar *unicode, int32_t length, char *out) {
 696     UChar32 c;
 697     int32_t i;
 698
 699     for(i=0; i<length;) {
 700         if(i>0) {
 701             *out++=' ';
 702         }
 703         U16_NEXT(unicode, i, length, c);
 704         // write 4..6 digits
 705         if(c>=0x100000) {
 706             *out++='1';
 707         }
 708         if(c>=0x10000) {
 709             *out++=hexDigit((uint8_t)((c>>16)&0xf));
 710         }
 711         *out++=hexDigit((uint8_t)((c>>12)&0xf));
 712         *out++=hexDigit((uint8_t)((c>>8)&0xf));
 713         *out++=hexDigit((uint8_t)((c>>4)&0xf));
 714         *out++=hexDigit((uint8_t)(c&0xf));
 715     }
 716     *out++=0;
 717     return out;
 718 }
 719
 720 static char *
 721 printOffsets(const int32_t *offsets, int32_t length, char *out) {
 722     int32_t i, o, d;
 723
 724     if(offsets==NULL) {
 725         length=0;
 726     }
 727
 728     for(i=0; i<length; ++i) {
 729         if(i>0) {
 730             *out++=' ';
 731         }
 732         o=offsets[i];
 733
 734         // print all offsets with 2 characters each (-x, -9..99, xx)
 735         if(o<-9) {
 736             *out++='-';
 737             *out++='x';
 738         } else if(o<0) {
 739             *out++='-';
 740             *out++=(char)('0'-o);
 741         } else if(o<=99) {
 742             *out++=(d=o/10)==0 ? ' ' : (char)('0'+d);
 743             *out++=(char)('0'+o%10);
 744         } else /* o>99 */ {
 745             *out++='x';
 746             *out++='x';
 747         }
 748     }
 749     *out++=0;
 750     return out;
 751 }
 752
 753 // toUnicode test worker functions ----------------------------------------- ***
 754
 755 static int32_t
 756 stepToUnicode(ConversionCase &cc, UConverter *cnv,
 757               UChar *result, int32_t resultCapacity,
 758               int32_t *resultOffsets, /* also resultCapacity */
 759               int32_t step,
 760               UErrorCode *pErrorCode) {
 761     const char *source, *sourceLimit, *bytesLimit;
 762     UChar *target, *targetLimit, *resultLimit;
 763     UBool flush;
 764
 765     source=(const char *)cc.bytes;
 766     target=result;
 767     bytesLimit=source+cc.bytesLength;
 768     resultLimit=result+resultCapacity;
 769
 770     if(step>=0) {
 771         // call ucnv_toUnicode() with in/out buffers no larger than (step) at a time
 772         // move only one buffer (in vs. out) at a time to be extra mean
 773         // step==0 performs bulk conversion and generates offsets
 774
 775         // initialize the partial limits for the loop
 776         if(step==0) {
 777             // use the entire buffers
 778             sourceLimit=bytesLimit;
 779             targetLimit=resultLimit;
 780             flush=cc.finalFlush;
 781         } else {
 782             // start with empty partial buffers
 783             sourceLimit=source;
 784             targetLimit=target;
 785             flush=FALSE;
 786
 787             // output offsets only for bulk conversion
 788             resultOffsets=NULL;
 789         }
 790
 791         for(;;) {
 792             // resetting the opposite conversion direction must not affect this one
 793             ucnv_resetFromUnicode(cnv);
 794
 795             // convert
 796             ucnv_toUnicode(cnv,
 797                 &target, targetLimit,
 798                 &source, sourceLimit,
 799                 resultOffsets,
 800                 flush, pErrorCode);
 801
 802             // check pointers and errors
 803             if(source>sourceLimit || target>targetLimit) {
 804                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
 805                 break;
 806             } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
 807                 if(target!=targetLimit) {
 808                     // buffer overflow must only be set when the target is filled
 809                     *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
 810                     break;
 811                 } else if(targetLimit==resultLimit) {
 812                     // not just a partial overflow
 813                     break;
 814                 }
 815
 816                 // the partial target is filled, set a new limit, reset the error and continue
 817                 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
 818                 *pErrorCode=U_ZERO_ERROR;
 819             } else if(U_FAILURE(*pErrorCode)) {
 820                 // some other error occurred, done
 821                 break;
 822             } else {
 823                 if(source!=sourceLimit) {
 824                     // when no error occurs, then the input must be consumed
 825                     *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
 826                     break;
 827                 }
 828
 829                 if(sourceLimit==bytesLimit) {
 830                     // we are done
 831                     break;
 832                 }
 833
 834                 // the partial conversion succeeded, set a new limit and continue
 835                 sourceLimit=(bytesLimit-source)>=step ? source+step : bytesLimit;
 836                 flush=(UBool)(cc.finalFlush && sourceLimit==bytesLimit);
 837             }
 838         }
 839     } else /* step<0 */ {
 840         /*
 841          * step==-1: call only ucnv_getNextUChar()
 842          * otherwise alternate between ucnv_toUnicode() and ucnv_getNextUChar()
 843          *   if step==-2 or -3, then give ucnv_toUnicode() the whole remaining input,
 844          *   else give it at most (-step-2)/2 bytes
 845          */
 846         UChar32 c;
 847
 848         // end the loop by getting an index out of bounds error
 849         for(;;) {
 850             // resetting the opposite conversion direction must not affect this one
 851             ucnv_resetFromUnicode(cnv);
 852
 853             // convert
 854             if((step&1)!=0 /* odd: -1, -3, -5, ... */) {
 855                 sourceLimit=source; // use sourceLimit not as a real limit
 856                                     // but to remember the pre-getNextUChar source pointer
 857                 c=ucnv_getNextUChar(cnv, &source, bytesLimit, pErrorCode);
 858
 859                 // check pointers and errors
 860                 if(*pErrorCode==U_INDEX_OUTOFBOUNDS_ERROR) {
 861                     if(source!=bytesLimit) {
 862                         *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
 863                     } else {
 864                         *pErrorCode=U_ZERO_ERROR;
 865                     }
 866                     break;
 867                 } else if(U_FAILURE(*pErrorCode)) {
 868                     break;
 869                 }
 870                 // source may not move if c is from previous overflow
 871
 872                 if(target==resultLimit) {
 873                     *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
 874                     break;
 875                 }
 876                 if(c<=0xffff) {
 877                     *target++=(UChar)c;
 878                 } else {
 879                     *target++=U16_LEAD(c);
 880                     if(target==resultLimit) {
 881                         *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
 882                         break;
 883                     }
 884                     *target++=U16_TRAIL(c);
 885                 }
 886
 887                 // alternate between -n-1 and -n but leave -1 alone
 888                 if(step<-1) {
 889                     ++step;
 890                 }
 891             } else /* step is even */ {
 892                 // allow only one UChar output
 893                 targetLimit=target<resultLimit ? target+1 : resultLimit;
 894
 895                 // as with ucnv_getNextUChar(), we always flush (if we go to bytesLimit)
 896                 // and never output offsets
 897                 if(step==-2) {
 898                     sourceLimit=bytesLimit;
 899                 } else {
 900                     sourceLimit=source+(-step-2)/2;
 901                     if(sourceLimit>bytesLimit) {
 902                         sourceLimit=bytesLimit;
 903                     }
 904                 }
 905
 906                 ucnv_toUnicode(cnv,
 907                     &target, targetLimit,
 908                     &source, sourceLimit,
 909                     NULL, (UBool)(sourceLimit==bytesLimit), pErrorCode);
 910
 911                 // check pointers and errors
 912                 if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
 913                     if(target!=targetLimit) {
 914                         // buffer overflow must only be set when the target is filled
 915                         *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
 916                         break;
 917                     } else if(targetLimit==resultLimit) {
 918                         // not just a partial overflow
 919                         break;
 920                     }
 921
 922                     // the partial target is filled, set a new limit and continue
 923                     *pErrorCode=U_ZERO_ERROR;
 924                 } else if(U_FAILURE(*pErrorCode)) {
 925                     // some other error occurred, done
 926                     break;
 927                 } else {
 928                     if(source!=sourceLimit) {
 929                         // when no error occurs, then the input must be consumed
 930                         *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
 931                         break;
 932                     }
 933
 934                     // we are done (flush==TRUE) but we continue, to get the index out of bounds error above
 935                 }
 936
 937                 --step;
 938             }
 939         }
 940     }
 941
 942     return (int32_t)(target-result);
 943 }
 944
 945 UBool
 946 ConversionTest::ToUnicodeCase(ConversionCase &cc, UConverterToUCallback callback, const char *option) {
 947     // open the converter
 948     IcuTestErrorCode errorCode(*this, "ToUnicodeCase");
 949     LocalUConverterPointer cnv(cnv_open(cc.charset, errorCode));
 950     if(errorCode.isFailure()) {
 951         errcheckln(errorCode, "toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
 952                 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, errorCode.errorName());
 953         errorCode.reset();
 954         return FALSE;
 955     }
 956
 957     // set the callback
 958     if(callback!=NULL) {
 959         ucnv_setToUCallBack(cnv.getAlias(), callback, option, NULL, NULL, errorCode);
 960         if(U_FAILURE(errorCode)) {
 961             errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setToUCallBack() failed - %s",
 962                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
 963             return FALSE;
 964         }
 965     }
 966
 967     int32_t resultOffsets[256];
 968     UChar result[256];
 969     int32_t resultLength;
 970     UBool ok;
 971
 972     static const struct {
 973         int32_t step;
 974         const char *name;
 975     } steps[]={
 976         { 0, "bulk" }, // must be first for offsets to be checked
 977         { 1, "step=1" },
 978         { 3, "step=3" },
 979         { 7, "step=7" },
 980         { -1, "getNext" },
 981         { -2, "toU(bulk)+getNext" },
 982         { -3, "getNext+toU(bulk)" },
 983         { -4, "toU(1)+getNext" },
 984         { -5, "getNext+toU(1)" },
 985         { -12, "toU(5)+getNext" },
 986         { -13, "getNext+toU(5)" },
 987     };
 988     int32_t i, step;
 989
 990     ok=TRUE;
 991     for(i=0; i<LENGTHOF(steps) && ok; ++i) {
 992         step=steps[i].step;
 993         if(step<0 && !cc.finalFlush) {
 994             // skip ucnv_getNextUChar() if !finalFlush because
 995             // ucnv_getNextUChar() always implies flush
 996             continue;
 997         }
 998         if(step!=0) {
 999             // bulk test is first, then offsets are not checked any more
1000             cc.offsets=NULL;
1001         }
1002         else {
1003             memset(resultOffsets, -1, LENGTHOF(resultOffsets));
1004         }
1005         memset(result, -1, LENGTHOF(result));
1006         errorCode.reset();
1007         resultLength=stepToUnicode(cc, cnv.getAlias(),
1008                                 result, LENGTHOF(result),
1009                                 step==0 ? resultOffsets : NULL,
1010                                 step, errorCode);
1011         ok=checkToUnicode(
1012                 cc, cnv.getAlias(), steps[i].name,
1013                 result, resultLength,
1014                 cc.offsets!=NULL ? resultOffsets : NULL,
1015                 errorCode);
1016         if(errorCode.isFailure() || !cc.finalFlush) {
1017             // reset if an error occurred or we did not flush
1018             // otherwise do nothing to make sure that flushing resets
1019             ucnv_resetToUnicode(cnv.getAlias());
1020         }
1021         if (cc.offsets != NULL && resultOffsets[resultLength] != -1) {
1022             errln("toUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1023                 cc.caseNr, cc.charset, resultLength);
1024         }
1025         if (result[resultLength] != (UChar)-1) {
1026             errln("toUnicode[%d](%s) Conversion wrote too much to result at index %d",
1027                 cc.caseNr, cc.charset, resultLength);
1028         }
1029     }
1030
1031     // not a real loop, just a convenience for breaking out of the block
1032     while(ok && cc.finalFlush) {
1033         // test ucnv_toUChars()
1034         memset(result, 0, sizeof(result));
1035
1036         errorCode.reset();
1037         resultLength=ucnv_toUChars(cnv.getAlias(),
1038                         result, LENGTHOF(result),
1039                         (const char *)cc.bytes, cc.bytesLength,
1040                         errorCode);
1041         ok=checkToUnicode(
1042                 cc, cnv.getAlias(), "toUChars",
1043                 result, resultLength,
1044                 NULL,
1045                 errorCode);
1046         if(!ok) {
1047             break;
1048         }
1049
1050         // test preflighting
1051         // keep the correct result for simple checking
1052         errorCode.reset();
1053         resultLength=ucnv_toUChars(cnv.getAlias(),
1054                         NULL, 0,
1055                         (const char *)cc.bytes, cc.bytesLength,
1056                         errorCode);
1057         if(errorCode.get()==U_STRING_NOT_TERMINATED_WARNING || errorCode.get()==U_BUFFER_OVERFLOW_ERROR) {
1058             errorCode.reset();
1059         }
1060         ok=checkToUnicode(
1061                 cc, cnv.getAlias(), "preflight toUChars",
1062                 result, resultLength,
1063                 NULL,
1064                 errorCode);
1065         break;
1066     }
1067
1068     errorCode.reset();  // all errors have already been reported
1069     return ok;
1070 }
1071
1072 UBool
1073 ConversionTest::checkToUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1074                                const UChar *result, int32_t resultLength,
1075                                const int32_t *resultOffsets,
1076                                UErrorCode resultErrorCode) {
1077     char resultInvalidChars[8];
1078     int8_t resultInvalidLength;
1079     UErrorCode errorCode;
1080
1081     const char *msg;
1082
1083     // reset the message; NULL will mean "ok"
1084     msg=NULL;
1085
1086     errorCode=U_ZERO_ERROR;
1087     resultInvalidLength=sizeof(resultInvalidChars);
1088     ucnv_getInvalidChars(cnv, resultInvalidChars, &resultInvalidLength, &errorCode);
1089     if(U_FAILURE(errorCode)) {
1090         errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidChars() failed - %s",
1091                 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1092         return FALSE;
1093     }
1094
1095     // check everything that might have gone wrong
1096     if(cc.unicodeLength!=resultLength) {
1097         msg="wrong result length";
1098     } else if(0!=u_memcmp(cc.unicode, result, cc.unicodeLength)) {
1099         msg="wrong result string";
1100     } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.unicodeLength*sizeof(*cc.offsets))) {
1101         msg="wrong offsets";
1102     } else if(cc.outErrorCode!=resultErrorCode) {
1103         msg="wrong error code";
1104     } else if(cc.invalidLength!=resultInvalidLength) {
1105         msg="wrong length of last invalid input";
1106     } else if(0!=memcmp(cc.invalidChars, resultInvalidChars, cc.invalidLength)) {
1107         msg="wrong last invalid input";
1108     }
1109
1110     if(msg==NULL) {
1111         return TRUE;
1112     } else {
1113         char buffer[2000]; // one buffer for all strings
1114         char *s, *bytesString, *unicodeString, *resultString,
1115             *offsetsString, *resultOffsetsString,
1116             *invalidCharsString, *resultInvalidCharsString;
1117
1118         bytesString=s=buffer;
1119         s=printBytes(cc.bytes, cc.bytesLength, bytesString);
1120         s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString=s);
1121         s=printUnicode(result, resultLength, resultString=s);
1122         s=printOffsets(cc.offsets, cc.unicodeLength, offsetsString=s);
1123         s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1124         s=printBytes(cc.invalidChars, cc.invalidLength, invalidCharsString=s);
1125         s=printBytes((uint8_t *)resultInvalidChars, resultInvalidLength, resultInvalidCharsString=s);
1126
1127         if((s-buffer)>(int32_t)sizeof(buffer)) {
1128             errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkToUnicode() test output buffer overflow writing %d chars\n",
1129                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1130             exit(1);
1131         }
1132
1133         errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1134               "  bytes <%s>[%d]\n"
1135               " expected <%s>[%d]\n"
1136               "  result  <%s>[%d]\n"
1137               " offsets         <%s>\n"
1138               "  result offsets <%s>\n"
1139               " error code expected %s got %s\n"
1140               "  invalidChars expected <%s> got <%s>\n",
1141               cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1142               bytesString, cc.bytesLength,
1143               unicodeString, cc.unicodeLength,
1144               resultString, resultLength,
1145               offsetsString,
1146               resultOffsetsString,
1147               u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1148               invalidCharsString, resultInvalidCharsString);
1149
1150         return FALSE;
1151     }
1152 }
1153
1154 // fromUnicode test worker functions --------------------------------------- ***
1155
1156 static int32_t
1157 stepFromUTF8(ConversionCase &cc,
1158              UConverter *utf8Cnv, UConverter *cnv,
1159              char *result, int32_t resultCapacity,
1160              int32_t step,
1161              UErrorCode *pErrorCode) {
1162     const char *source, *sourceLimit, *utf8Limit;
1163     UChar pivotBuffer[32];
1164     UChar *pivotSource, *pivotTarget, *pivotLimit;
1165     char *target, *targetLimit, *resultLimit;
1166     UBool flush;
1167
1168     source=cc.utf8;
1169     pivotSource=pivotTarget=pivotBuffer;
1170     target=result;
1171     utf8Limit=source+cc.utf8Length;
1172     resultLimit=result+resultCapacity;
1173
1174     // call ucnv_convertEx() with in/out buffers no larger than (step) at a time
1175     // move only one buffer (in vs. out) at a time to be extra mean
1176     // step==0 performs bulk conversion
1177
1178     // initialize the partial limits for the loop
1179     if(step==0) {
1180         // use the entire buffers
1181         sourceLimit=utf8Limit;
1182         targetLimit=resultLimit;
1183         flush=cc.finalFlush;
1184
1185         pivotLimit=pivotBuffer+LENGTHOF(pivotBuffer);
1186     } else {
1187         // start with empty partial buffers
1188         sourceLimit=source;
1189         targetLimit=target;
1190         flush=FALSE;
1191
1192         // empty pivot is not allowed, make it of length step
1193         pivotLimit=pivotBuffer+step;
1194     }
1195
1196     for(;;) {
1197         // resetting the opposite conversion direction must not affect this one
1198         ucnv_resetFromUnicode(utf8Cnv);
1199         ucnv_resetToUnicode(cnv);
1200
1201         // convert
1202         ucnv_convertEx(cnv, utf8Cnv,
1203             &target, targetLimit,
1204             &source, sourceLimit,
1205             pivotBuffer, &pivotSource, &pivotTarget, pivotLimit,
1206             FALSE, flush, pErrorCode);
1207
1208         // check pointers and errors
1209         if(source>sourceLimit || target>targetLimit) {
1210             *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1211             break;
1212         } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1213             if(target!=targetLimit) {
1214                 // buffer overflow must only be set when the target is filled
1215                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1216                 break;
1217             } else if(targetLimit==resultLimit) {
1218                 // not just a partial overflow
1219                 break;
1220             }
1221
1222             // the partial target is filled, set a new limit, reset the error and continue
1223             targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1224             *pErrorCode=U_ZERO_ERROR;
1225         } else if(U_FAILURE(*pErrorCode)) {
1226             if(pivotSource==pivotBuffer) {
1227                 // toUnicode error, should not occur
1228                 // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1229                 break;
1230             } else {
1231                 // fromUnicode error
1232                 // some other error occurred, done
1233                 break;
1234             }
1235         } else {
1236             if(source!=sourceLimit) {
1237                 // when no error occurs, then the input must be consumed
1238                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1239                 break;
1240             }
1241
1242             if(sourceLimit==utf8Limit) {
1243                 // we are done
1244                 if(*pErrorCode==U_STRING_NOT_TERMINATED_WARNING) {
1245                     // ucnv_convertEx() warns about not terminating the output
1246                     // but ucnv_fromUnicode() does not and so
1247                     // checkFromUnicode() does not expect it
1248                     *pErrorCode=U_ZERO_ERROR;
1249                 }
1250                 break;
1251             }
1252
1253             // the partial conversion succeeded, set a new limit and continue
1254             sourceLimit=(utf8Limit-source)>=step ? source+step : utf8Limit;
1255             flush=(UBool)(cc.finalFlush && sourceLimit==utf8Limit);
1256         }
1257     }
1258
1259     return (int32_t)(target-result);
1260 }
1261
1262 static int32_t
1263 stepFromUnicode(ConversionCase &cc, UConverter *cnv,
1264                 char *result, int32_t resultCapacity,
1265                 int32_t *resultOffsets, /* also resultCapacity */
1266                 int32_t step,
1267                 UErrorCode *pErrorCode) {
1268     const UChar *source, *sourceLimit, *unicodeLimit;
1269     char *target, *targetLimit, *resultLimit;
1270     UBool flush;
1271
1272     source=cc.unicode;
1273     target=result;
1274     unicodeLimit=source+cc.unicodeLength;
1275     resultLimit=result+resultCapacity;
1276
1277     // call ucnv_fromUnicode() with in/out buffers no larger than (step) at a time
1278     // move only one buffer (in vs. out) at a time to be extra mean
1279     // step==0 performs bulk conversion and generates offsets
1280
1281     // initialize the partial limits for the loop
1282     if(step==0) {
1283         // use the entire buffers
1284         sourceLimit=unicodeLimit;
1285         targetLimit=resultLimit;
1286         flush=cc.finalFlush;
1287     } else {
1288         // start with empty partial buffers
1289         sourceLimit=source;
1290         targetLimit=target;
1291         flush=FALSE;
1292
1293         // output offsets only for bulk conversion
1294         resultOffsets=NULL;
1295     }
1296
1297     for(;;) {
1298         // resetting the opposite conversion direction must not affect this one
1299         ucnv_resetToUnicode(cnv);
1300
1301         // convert
1302         ucnv_fromUnicode(cnv,
1303             &target, targetLimit,
1304             &source, sourceLimit,
1305             resultOffsets,
1306             flush, pErrorCode);
1307
1308         // check pointers and errors
1309         if(source>sourceLimit || target>targetLimit) {
1310             *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1311             break;
1312         } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1313             if(target!=targetLimit) {
1314                 // buffer overflow must only be set when the target is filled
1315                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1316                 break;
1317             } else if(targetLimit==resultLimit) {
1318                 // not just a partial overflow
1319                 break;
1320             }
1321
1322             // the partial target is filled, set a new limit, reset the error and continue
1323             targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1324             *pErrorCode=U_ZERO_ERROR;
1325         } else if(U_FAILURE(*pErrorCode)) {
1326             // some other error occurred, done
1327             break;
1328         } else {
1329             if(source!=sourceLimit) {
1330                 // when no error occurs, then the input must be consumed
1331                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1332                 break;
1333             }
1334
1335             if(sourceLimit==unicodeLimit) {
1336                 // we are done
1337                 break;
1338             }
1339
1340             // the partial conversion succeeded, set a new limit and continue
1341             sourceLimit=(unicodeLimit-source)>=step ? source+step : unicodeLimit;
1342             flush=(UBool)(cc.finalFlush && sourceLimit==unicodeLimit);
1343         }
1344     }
1345
1346     return (int32_t)(target-result);
1347 }
1348
1349 UBool
1350 ConversionTest::FromUnicodeCase(ConversionCase &cc, UConverterFromUCallback callback, const char *option) {
1351     UConverter *cnv;
1352     UErrorCode errorCode;
1353
1354     // open the converter
1355     errorCode=U_ZERO_ERROR;
1356     cnv=cnv_open(cc.charset, errorCode);
1357     if(U_FAILURE(errorCode)) {
1358         errcheckln(errorCode, "fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
1359                 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1360         return FALSE;
1361     }
1362     ucnv_resetToUnicode(utf8Cnv);
1363
1364     // set the callback
1365     if(callback!=NULL) {
1366         ucnv_setFromUCallBack(cnv, callback, option, NULL, NULL, &errorCode);
1367         if(U_FAILURE(errorCode)) {
1368             errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setFromUCallBack() failed - %s",
1369                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1370             ucnv_close(cnv);
1371             return FALSE;
1372         }
1373     }
1374
1375     // set the fallbacks flag
1376     // TODO change with Jitterbug 2401, then add a similar call for toUnicode too
1377     ucnv_setFallback(cnv, cc.fallbacks);
1378
1379     // set the subchar
1380     int32_t length;
1381
1382     if(cc.setSub>0) {
1383         length=(int32_t)strlen(cc.subchar);
1384         ucnv_setSubstChars(cnv, cc.subchar, (int8_t)length, &errorCode);
1385         if(U_FAILURE(errorCode)) {
1386             errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstChars() failed - %s",
1387                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1388             ucnv_close(cnv);
1389             return FALSE;
1390         }
1391     } else if(cc.setSub<0) {
1392         ucnv_setSubstString(cnv, cc.subString, -1, &errorCode);
1393         if(U_FAILURE(errorCode)) {
1394             errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstString() failed - %s",
1395                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1396             ucnv_close(cnv);
1397             return FALSE;
1398         }
1399     }
1400
1401     // convert unicode to utf8
1402     char utf8[256];
1403     cc.utf8=utf8;
1404     u_strToUTF8(utf8, LENGTHOF(utf8), &cc.utf8Length,
1405                 cc.unicode, cc.unicodeLength,
1406                 &errorCode);
1407     if(U_FAILURE(errorCode)) {
1408         // skip UTF-8 testing of a string with an unpaired surrogate,
1409         // or of one that's too long
1410         // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1411         cc.utf8Length=-1;
1412     }
1413
1414     int32_t resultOffsets[256];
1415     char result[256];
1416     int32_t resultLength;
1417     UBool ok;
1418
1419     static const struct {
1420         int32_t step;
1421         const char *name, *utf8Name;
1422     } steps[]={
1423         { 0, "bulk",   "utf8" }, // must be first for offsets to be checked
1424         { 1, "step=1", "utf8 step=1" },
1425         { 3, "step=3", "utf8 step=3" },
1426         { 7, "step=7", "utf8 step=7" }
1427     };
1428     int32_t i, step;
1429
1430     ok=TRUE;
1431     for(i=0; i<LENGTHOF(steps) && ok; ++i) {
1432         step=steps[i].step;
1433         memset(resultOffsets, -1, LENGTHOF(resultOffsets));
1434         memset(result, -1, LENGTHOF(result));
1435         errorCode=U_ZERO_ERROR;
1436         resultLength=stepFromUnicode(cc, cnv,
1437                                 result, LENGTHOF(result),
1438                                 step==0 ? resultOffsets : NULL,
1439                                 step, &errorCode);
1440         ok=checkFromUnicode(
1441                 cc, cnv, steps[i].name,
1442                 (uint8_t *)result, resultLength,
1443                 cc.offsets!=NULL ? resultOffsets : NULL,
1444                 errorCode);
1445         if(U_FAILURE(errorCode) || !cc.finalFlush) {
1446             // reset if an error occurred or we did not flush
1447             // otherwise do nothing to make sure that flushing resets
1448             ucnv_resetFromUnicode(cnv);
1449         }
1450         if (resultOffsets[resultLength] != -1) {
1451             errln("fromUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1452                 cc.caseNr, cc.charset, resultLength);
1453         }
1454         if (result[resultLength] != (char)-1) {
1455             errln("fromUnicode[%d](%s) Conversion wrote too much to result at index %d",
1456                 cc.caseNr, cc.charset, resultLength);
1457         }
1458
1459         // bulk test is first, then offsets are not checked any more
1460         cc.offsets=NULL;
1461
1462         // test direct conversion from UTF-8
1463         if(cc.utf8Length>=0) {
1464             errorCode=U_ZERO_ERROR;
1465             resultLength=stepFromUTF8(cc, utf8Cnv, cnv,
1466                                     result, LENGTHOF(result),
1467                                     step, &errorCode);
1468             ok=checkFromUnicode(
1469                     cc, cnv, steps[i].utf8Name,
1470                     (uint8_t *)result, resultLength,
1471                     NULL,
1472                     errorCode);
1473             if(U_FAILURE(errorCode) || !cc.finalFlush) {
1474                 // reset if an error occurred or we did not flush
1475                 // otherwise do nothing to make sure that flushing resets
1476                 ucnv_resetToUnicode(utf8Cnv);
1477                 ucnv_resetFromUnicode(cnv);
1478             }
1479         }
1480     }
1481
1482     // not a real loop, just a convenience for breaking out of the block
1483     while(ok && cc.finalFlush) {
1484         // test ucnv_fromUChars()
1485         memset(result, 0, sizeof(result));
1486
1487         errorCode=U_ZERO_ERROR;
1488         resultLength=ucnv_fromUChars(cnv,
1489                         result, LENGTHOF(result),
1490                         cc.unicode, cc.unicodeLength,
1491                         &errorCode);
1492         ok=checkFromUnicode(
1493                 cc, cnv, "fromUChars",
1494                 (uint8_t *)result, resultLength,
1495                 NULL,
1496                 errorCode);
1497         if(!ok) {
1498             break;
1499         }
1500
1501         // test preflighting
1502         // keep the correct result for simple checking
1503         errorCode=U_ZERO_ERROR;
1504         resultLength=ucnv_fromUChars(cnv,
1505                         NULL, 0,
1506                         cc.unicode, cc.unicodeLength,
1507                         &errorCode);
1508         if(errorCode==U_STRING_NOT_TERMINATED_WARNING || errorCode==U_BUFFER_OVERFLOW_ERROR) {
1509             errorCode=U_ZERO_ERROR;
1510         }
1511         ok=checkFromUnicode(
1512                 cc, cnv, "preflight fromUChars",
1513                 (uint8_t *)result, resultLength,
1514                 NULL,
1515                 errorCode);
1516         break;
1517     }
1518
1519     ucnv_close(cnv);
1520     return ok;
1521 }
1522
1523 UBool
1524 ConversionTest::checkFromUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1525                                  const uint8_t *result, int32_t resultLength,
1526                                  const int32_t *resultOffsets,
1527                                  UErrorCode resultErrorCode) {
1528     UChar resultInvalidUChars[8];
1529     int8_t resultInvalidLength;
1530     UErrorCode errorCode;
1531
1532     const char *msg;
1533
1534     // reset the message; NULL will mean "ok"
1535     msg=NULL;
1536
1537     errorCode=U_ZERO_ERROR;
1538     resultInvalidLength=LENGTHOF(resultInvalidUChars);
1539     ucnv_getInvalidUChars(cnv, resultInvalidUChars, &resultInvalidLength, &errorCode);
1540     if(U_FAILURE(errorCode)) {
1541         errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidUChars() failed - %s",
1542                 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1543         return FALSE;
1544     }
1545
1546     // check everything that might have gone wrong
1547     if(cc.bytesLength!=resultLength) {
1548         msg="wrong result length";
1549     } else if(0!=memcmp(cc.bytes, result, cc.bytesLength)) {
1550         msg="wrong result string";
1551     } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.bytesLength*sizeof(*cc.offsets))) {
1552         msg="wrong offsets";
1553     } else if(cc.outErrorCode!=resultErrorCode) {
1554         msg="wrong error code";
1555     } else if(cc.invalidLength!=resultInvalidLength) {
1556         msg="wrong length of last invalid input";
1557     } else if(0!=u_memcmp(cc.invalidUChars, resultInvalidUChars, cc.invalidLength)) {
1558         msg="wrong last invalid input";
1559     }
1560
1561     if(msg==NULL) {
1562         return TRUE;
1563     } else {
1564         char buffer[2000]; // one buffer for all strings
1565         char *s, *unicodeString, *bytesString, *resultString,
1566             *offsetsString, *resultOffsetsString,
1567             *invalidCharsString, *resultInvalidUCharsString;
1568
1569         unicodeString=s=buffer;
1570         s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString);
1571         s=printBytes(cc.bytes, cc.bytesLength, bytesString=s);
1572         s=printBytes(result, resultLength, resultString=s);
1573         s=printOffsets(cc.offsets, cc.bytesLength, offsetsString=s);
1574         s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1575         s=printUnicode(cc.invalidUChars, cc.invalidLength, invalidCharsString=s);
1576         s=printUnicode(resultInvalidUChars, resultInvalidLength, resultInvalidUCharsString=s);
1577
1578         if((s-buffer)>(int32_t)sizeof(buffer)) {
1579             errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkFromUnicode() test output buffer overflow writing %d chars\n",
1580                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1581             exit(1);
1582         }
1583
1584         errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1585               "  unicode <%s>[%d]\n"
1586               " expected <%s>[%d]\n"
1587               "  result  <%s>[%d]\n"
1588               " offsets         <%s>\n"
1589               "  result offsets <%s>\n"
1590               " error code expected %s got %s\n"
1591               "  invalidChars expected <%s> got <%s>\n",
1592               cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1593               unicodeString, cc.unicodeLength,
1594               bytesString, cc.bytesLength,
1595               resultString, resultLength,
1596               offsetsString,
1597               resultOffsetsString,
1598               u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1599               invalidCharsString, resultInvalidUCharsString);
1600
1601         return FALSE;
1602     }
1603 }
1604
1605 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */