icuSources/test/intltest/convtest.cpp

   1 // © 2016 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3 /*
   4 *******************************************************************************
   5 *
   6 *   Copyright (C) 2003-2014, International Business Machines
   7 *   Corporation and others.  All Rights Reserved.
   8 *
   9 *******************************************************************************
  10 *   file name:  convtest.cpp
  11 *   encoding:   UTF-8
  12 *   tab size:   8 (not used)
  13 *   indentation:4
  14 *
  15 *   created on: 2003jul15
  16 *   created by: Markus W. Scherer
  17 *
  18 *   Test file for data-driven conversion tests.
  19 */
  20
  21 #include "unicode/utypes.h"
  22
  23 #if !UCONFIG_NO_LEGACY_CONVERSION
  24 /*
  25  * Note: Turning off all of convtest.cpp if !UCONFIG_NO_LEGACY_CONVERSION
  26  * is slightly unnecessary - it removes tests for Unicode charsets
  27  * like UTF-8 that should work.
  28  * However, there is no easy way for the test to detect whether a test case
  29  * is for a Unicode charset, so it would be difficult to only exclude those.
  30  * Also, regular testing of ICU is done with all modules on, therefore
  31  * not testing conversion for a custom configuration like this should be ok.
  32  */
  33
  34 #include "unicode/ucnv.h"
  35 #include "unicode/unistr.h"
  36 #include "unicode/parsepos.h"
  37 #include "unicode/uniset.h"
  38 #include "unicode/ustring.h"
  39 #include "unicode/ures.h"
  40 #include "unicode/utf16.h"
  41 #include "convtest.h"
  42 #include "cmemory.h"
  43 #include "unicode/tstdtmod.h"
  44 #include <string.h>
  45 #include <stdlib.h>
  46
  47 enum {
  48     // characters used in test data for callbacks
  49     SUB_CB='?',
  50     SKIP_CB='0',
  51     STOP_CB='.',
  52     ESC_CB='&'
  53 };
  54
  55 ConversionTest::ConversionTest() {
  56     UErrorCode errorCode=U_ZERO_ERROR;
  57     utf8Cnv=ucnv_open("UTF-8", &errorCode);
  58     ucnv_setToUCallBack(utf8Cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
  59     if(U_FAILURE(errorCode)) {
  60         errln("unable to open UTF-8 converter");
  61     }
  62 }
  63
  64 ConversionTest::~ConversionTest() {
  65     ucnv_close(utf8Cnv);
  66 }
  67
  68 void
  69 ConversionTest::runIndexedTest(int32_t index, UBool exec, const char *&name, char * /*par*/) {
  70     if (exec) logln("TestSuite ConversionTest: ");
  71     TESTCASE_AUTO_BEGIN;
  72 #if !UCONFIG_NO_FILE_IO
  73     TESTCASE_AUTO(TestToUnicode);
  74     TESTCASE_AUTO(TestFromUnicode);
  75     TESTCASE_AUTO(TestGetUnicodeSet);
  76 #endif
  77     TESTCASE_AUTO(TestGetUnicodeSet2);
  78     TESTCASE_AUTO(TestDefaultIgnorableCallback);
  79     TESTCASE_AUTO(TestUTF8ToUTF8Overflow);
  80     TESTCASE_AUTO_END;
  81 }
  82
  83 // test data interface ----------------------------------------------------- ***
  84
  85 void
  86 ConversionTest::TestToUnicode() {
  87     ConversionCase cc;
  88     char charset[100], cbopt[4];
  89     const char *option;
  90     UnicodeString s, unicode;
  91     int32_t offsetsLength;
  92     UConverterToUCallback callback;
  93
  94     TestDataModule *dataModule;
  95     TestData *testData;
  96     const DataMap *testCase;
  97     UErrorCode errorCode;
  98     int32_t i;
  99
 100     errorCode=U_ZERO_ERROR;
 101     dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
 102     if(U_SUCCESS(errorCode)) {
 103         testData=dataModule->createTestData("toUnicode", errorCode);
 104         if(U_SUCCESS(errorCode)) {
 105             for(i=0; testData->nextCase(testCase, errorCode); ++i) {
 106                 if(U_FAILURE(errorCode)) {
 107                     errln("error retrieving conversion/toUnicode test case %d - %s",
 108                             i, u_errorName(errorCode));
 109                     errorCode=U_ZERO_ERROR;
 110                     continue;
 111                 }
 112
 113                 cc.caseNr=i;
 114
 115                 s=testCase->getString("charset", errorCode);
 116                 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
 117                 cc.charset=charset;
 118
 119                 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
 120                 unicode=testCase->getString("unicode", errorCode);
 121                 cc.unicode=unicode.getBuffer();
 122                 cc.unicodeLength=unicode.length();
 123
 124                 offsetsLength=0;
 125                 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
 126                 if(offsetsLength==0) {
 127                     cc.offsets=NULL;
 128                 } else if(offsetsLength!=unicode.length()) {
 129                     errln("toUnicode[%d] unicode[%d] and offsets[%d] must have the same length",
 130                             i, unicode.length(), offsetsLength);
 131                     errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 132                 }
 133
 134                 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
 135                 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
 136
 137                 s=testCase->getString("errorCode", errorCode);
 138                 if(s==UNICODE_STRING("invalid", 7)) {
 139                     cc.outErrorCode=U_INVALID_CHAR_FOUND;
 140                 } else if(s==UNICODE_STRING("illegal", 7)) {
 141                     cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
 142                 } else if(s==UNICODE_STRING("truncated", 9)) {
 143                     cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
 144                 } else if(s==UNICODE_STRING("illesc", 6)) {
 145                     cc.outErrorCode=U_ILLEGAL_ESCAPE_SEQUENCE;
 146                 } else if(s==UNICODE_STRING("unsuppesc", 9)) {
 147                     cc.outErrorCode=U_UNSUPPORTED_ESCAPE_SEQUENCE;
 148                 } else {
 149                     cc.outErrorCode=U_ZERO_ERROR;
 150                 }
 151
 152                 s=testCase->getString("callback", errorCode);
 153                 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
 154                 cc.cbopt=cbopt;
 155                 switch(cbopt[0]) {
 156                 case SUB_CB:
 157                     callback=UCNV_TO_U_CALLBACK_SUBSTITUTE;
 158                     break;
 159                 case SKIP_CB:
 160                     callback=UCNV_TO_U_CALLBACK_SKIP;
 161                     break;
 162                 case STOP_CB:
 163                     callback=UCNV_TO_U_CALLBACK_STOP;
 164                     break;
 165                 case ESC_CB:
 166                     callback=UCNV_TO_U_CALLBACK_ESCAPE;
 167                     break;
 168                 default:
 169                     callback=NULL;
 170                     break;
 171                 }
 172                 option=callback==NULL ? cbopt : cbopt+1;
 173                 if(*option==0) {
 174                     option=NULL;
 175                 }
 176
 177                 cc.invalidChars=testCase->getBinary(cc.invalidLength, "invalidChars", errorCode);
 178
 179                 if(U_FAILURE(errorCode)) {
 180                     errln("error parsing conversion/toUnicode test case %d - %s",
 181                             i, u_errorName(errorCode));
 182                     errorCode=U_ZERO_ERROR;
 183                 } else {
 184                     logln("TestToUnicode[%d] %s", i, charset);
 185                     ToUnicodeCase(cc, callback, option);
 186                 }
 187             }
 188             delete testData;
 189         }
 190         delete dataModule;
 191     }
 192     else {
 193         dataerrln("Could not load test conversion data");
 194     }
 195 }
 196
 197 void
 198 ConversionTest::TestFromUnicode() {
 199     ConversionCase cc;
 200     char charset[100], cbopt[4];
 201     const char *option;
 202     UnicodeString s, unicode, invalidUChars;
 203     int32_t offsetsLength, index;
 204     UConverterFromUCallback callback;
 205
 206     TestDataModule *dataModule;
 207     TestData *testData;
 208     const DataMap *testCase;
 209     const UChar *p;
 210     UErrorCode errorCode;
 211     int32_t i, length;
 212
 213     errorCode=U_ZERO_ERROR;
 214     dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
 215     if(U_SUCCESS(errorCode)) {
 216         testData=dataModule->createTestData("fromUnicode", errorCode);
 217         if(U_SUCCESS(errorCode)) {
 218             for(i=0; testData->nextCase(testCase, errorCode); ++i) {
 219                 if(U_FAILURE(errorCode)) {
 220                     errln("error retrieving conversion/fromUnicode test case %d - %s",
 221                             i, u_errorName(errorCode));
 222                     errorCode=U_ZERO_ERROR;
 223                     continue;
 224                 }
 225
 226                 cc.caseNr=i;
 227
 228                 s=testCase->getString("charset", errorCode);
 229                 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
 230                 cc.charset=charset;
 231
 232                 unicode=testCase->getString("unicode", errorCode);
 233                 cc.unicode=unicode.getBuffer();
 234                 cc.unicodeLength=unicode.length();
 235                 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
 236
 237                 offsetsLength=0;
 238                 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
 239                 if(offsetsLength==0) {
 240                     cc.offsets=NULL;
 241                 } else if(offsetsLength!=cc.bytesLength) {
 242                     errln("fromUnicode[%d] bytes[%d] and offsets[%d] must have the same length",
 243                             i, cc.bytesLength, offsetsLength);
 244                     errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 245                 }
 246
 247                 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
 248                 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
 249
 250                 s=testCase->getString("errorCode", errorCode);
 251                 if(s==UNICODE_STRING("invalid", 7)) {
 252                     cc.outErrorCode=U_INVALID_CHAR_FOUND;
 253                 } else if(s==UNICODE_STRING("illegal", 7)) {
 254                     cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
 255                 } else if(s==UNICODE_STRING("truncated", 9)) {
 256                     cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
 257                 } else {
 258                     cc.outErrorCode=U_ZERO_ERROR;
 259                 }
 260
 261                 s=testCase->getString("callback", errorCode);
 262                 cc.setSub=0; // default: no subchar
 263
 264                 if((index=s.indexOf((UChar)0))>0) {
 265                     // read NUL-separated subchar first, if any
 266                     // copy the subchar from Latin-1 characters
 267                     // start after the NUL
 268                     p=s.getTerminatedBuffer();
 269                     length=index+1;
 270                     p+=length;
 271                     length=s.length()-length;
 272                     if(length<=0 || length>=(int32_t)sizeof(cc.subchar)) {
 273                         errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 274                     } else {
 275                         int32_t j;
 276
 277                         for(j=0; j<length; ++j) {
 278                             cc.subchar[j]=(char)p[j];
 279                         }
 280                         // NUL-terminate the subchar
 281                         cc.subchar[j]=0;
 282                         cc.setSub=1;
 283                     }
 284
 285                     // remove the NUL and subchar from s
 286                     s.truncate(index);
 287                 } else if((index=s.indexOf((UChar)0x3d))>0) /* '=' */ {
 288                     // read a substitution string, separated by an equal sign
 289                     p=s.getBuffer()+index+1;
 290                     length=s.length()-(index+1);
 291                     if(length<0 || length>=UPRV_LENGTHOF(cc.subString)) {
 292                         errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 293                     } else {
 294                         u_memcpy(cc.subString, p, length);
 295                         // NUL-terminate the subString
 296                         cc.subString[length]=0;
 297                         cc.setSub=-1;
 298                     }
 299
 300                     // remove the equal sign and subString from s
 301                     s.truncate(index);
 302                 }
 303
 304                 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
 305                 cc.cbopt=cbopt;
 306                 switch(cbopt[0]) {
 307                 case SUB_CB:
 308                     callback=UCNV_FROM_U_CALLBACK_SUBSTITUTE;
 309                     break;
 310                 case SKIP_CB:
 311                     callback=UCNV_FROM_U_CALLBACK_SKIP;
 312                     break;
 313                 case STOP_CB:
 314                     callback=UCNV_FROM_U_CALLBACK_STOP;
 315                     break;
 316                 case ESC_CB:
 317                     callback=UCNV_FROM_U_CALLBACK_ESCAPE;
 318                     break;
 319                 default:
 320                     callback=NULL;
 321                     break;
 322                 }
 323                 option=callback==NULL ? cbopt : cbopt+1;
 324                 if(*option==0) {
 325                     option=NULL;
 326                 }
 327
 328                 invalidUChars=testCase->getString("invalidUChars", errorCode);
 329                 cc.invalidUChars=invalidUChars.getBuffer();
 330                 cc.invalidLength=invalidUChars.length();
 331
 332                 if(U_FAILURE(errorCode)) {
 333                     errln("error parsing conversion/fromUnicode test case %d - %s",
 334                             i, u_errorName(errorCode));
 335                     errorCode=U_ZERO_ERROR;
 336                 } else {
 337                     logln("TestFromUnicode[%d] %s", i, charset);
 338                     FromUnicodeCase(cc, callback, option);
 339                 }
 340             }
 341             delete testData;
 342         }
 343         delete dataModule;
 344     }
 345     else {
 346         dataerrln("Could not load test conversion data");
 347     }
 348 }
 349
 350 static const UChar ellipsis[]={ 0x2e, 0x2e, 0x2e };
 351
 352 void
 353 ConversionTest::TestGetUnicodeSet() {
 354     char charset[100];
 355     UnicodeString s, map, mapnot;
 356     int32_t which;
 357
 358     ParsePosition pos;
 359     UnicodeSet cnvSet, mapSet, mapnotSet, diffSet;
 360     UnicodeSet *cnvSetPtr = &cnvSet;
 361     LocalUConverterPointer cnv;
 362
 363     TestDataModule *dataModule;
 364     TestData *testData;
 365     const DataMap *testCase;
 366     UErrorCode errorCode;
 367     int32_t i;
 368
 369     errorCode=U_ZERO_ERROR;
 370     dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
 371     if(U_SUCCESS(errorCode)) {
 372         testData=dataModule->createTestData("getUnicodeSet", errorCode);
 373         if(U_SUCCESS(errorCode)) {
 374             for(i=0; testData->nextCase(testCase, errorCode); ++i) {
 375                 if(U_FAILURE(errorCode)) {
 376                     errln("error retrieving conversion/getUnicodeSet test case %d - %s",
 377                             i, u_errorName(errorCode));
 378                     errorCode=U_ZERO_ERROR;
 379                     continue;
 380                 }
 381
 382                 s=testCase->getString("charset", errorCode);
 383                 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
 384
 385                 map=testCase->getString("map", errorCode);
 386                 mapnot=testCase->getString("mapnot", errorCode);
 387
 388                 which=testCase->getInt28("which", errorCode);
 389
 390                 if(U_FAILURE(errorCode)) {
 391                     errln("error parsing conversion/getUnicodeSet test case %d - %s",
 392                             i, u_errorName(errorCode));
 393                     errorCode=U_ZERO_ERROR;
 394                     continue;
 395                 }
 396
 397                 // test this test case
 398                 mapSet.clear();
 399                 mapnotSet.clear();
 400
 401                 pos.setIndex(0);
 402                 mapSet.applyPattern(map, pos, 0, NULL, errorCode);
 403                 if(U_FAILURE(errorCode) || pos.getIndex()!=map.length()) {
 404                     errln("error creating the map set for conversion/getUnicodeSet test case %d - %s\n"
 405                           "    error index %d  index %d  U+%04x",
 406                             i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), map.char32At(pos.getIndex()));
 407                     errorCode=U_ZERO_ERROR;
 408                     continue;
 409                 }
 410
 411                 pos.setIndex(0);
 412                 mapnotSet.applyPattern(mapnot, pos, 0, NULL, errorCode);
 413                 if(U_FAILURE(errorCode) || pos.getIndex()!=mapnot.length()) {
 414                     errln("error creating the mapnot set for conversion/getUnicodeSet test case %d - %s\n"
 415                           "    error index %d  index %d  U+%04x",
 416                             i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), mapnot.char32At(pos.getIndex()));
 417                     errorCode=U_ZERO_ERROR;
 418                     continue;
 419                 }
 420
 421                 logln("TestGetUnicodeSet[%d] %s", i, charset);
 422
 423                 cnv.adoptInstead(cnv_open(charset, errorCode));
 424                 if(U_FAILURE(errorCode)) {
 425                     errcheckln(errorCode, "error opening \"%s\" for conversion/getUnicodeSet test case %d - %s",
 426                             charset, i, u_errorName(errorCode));
 427                     errorCode=U_ZERO_ERROR;
 428                     continue;
 429                 }
 430
 431                 ucnv_getUnicodeSet(cnv.getAlias(), cnvSetPtr->toUSet(), (UConverterUnicodeSet)which, &errorCode);
 432
 433                 if(U_FAILURE(errorCode)) {
 434                     errln("error in ucnv_getUnicodeSet(\"%s\") for conversion/getUnicodeSet test case %d - %s",
 435                             charset, i, u_errorName(errorCode));
 436                     errorCode=U_ZERO_ERROR;
 437                     continue;
 438                 }
 439
 440                 // are there items that must be in cnvSet but are not?
 441                 (diffSet=mapSet).removeAll(cnvSet);
 442                 if(!diffSet.isEmpty()) {
 443                     diffSet.toPattern(s, TRUE);
 444                     if(s.length()>100) {
 445                         s.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
 446                     }
 447                     errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - conversion/getUnicodeSet test case %d",
 448                             charset, i);
 449                     errln(s);
 450                 }
 451
 452                 // are there items that must not be in cnvSet but are?
 453                 (diffSet=mapnotSet).retainAll(cnvSet);
 454                 if(!diffSet.isEmpty()) {
 455                     diffSet.toPattern(s, TRUE);
 456                     if(s.length()>100) {
 457                         s.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
 458                     }
 459                     errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - conversion/getUnicodeSet test case %d",
 460                             charset, i);
 461                     errln(s);
 462                 }
 463             }
 464             delete testData;
 465         }
 466         delete dataModule;
 467     }
 468     else {
 469         dataerrln("Could not load test conversion data");
 470     }
 471 }
 472
 473 U_CDECL_BEGIN
 474 static void U_CALLCONV
 475 getUnicodeSetCallback(const void *context,
 476                       UConverterFromUnicodeArgs * /*fromUArgs*/,
 477                       const UChar* /*codeUnits*/,
 478                       int32_t /*length*/,
 479                       UChar32 codePoint,
 480                       UConverterCallbackReason reason,
 481                       UErrorCode *pErrorCode) {
 482     if(reason<=UCNV_IRREGULAR) {
 483         ((UnicodeSet *)context)->remove(codePoint);  // the converter cannot convert this code point
 484         *pErrorCode=U_ZERO_ERROR;                    // skip
 485     }  // else ignore the reset, close and clone calls.
 486 }
 487 U_CDECL_END
 488
 489 // Compare ucnv_getUnicodeSet() with the set of characters that can be converted.
 490 void
 491 ConversionTest::TestGetUnicodeSet2() {
 492     // Build a string with all code points.
 493     UChar32 cpLimit;
 494     int32_t s0Length;
 495     if(quick) {
 496         cpLimit=s0Length=0x10000;  // BMP only
 497     } else {
 498         cpLimit=0x110000;
 499         s0Length=0x10000+0x200000;  // BMP + surrogate pairs
 500     }
 501     UChar *s0=new UChar[s0Length];
 502     if(s0==NULL) {
 503         return;
 504     }
 505     UChar *s=s0;
 506     UChar32 c;
 507     UChar c2;
 508     // low BMP
 509     for(c=0; c<=0xd7ff; ++c) {
 510         *s++=(UChar)c;
 511     }
 512     // trail surrogates
 513     for(c=0xdc00; c<=0xdfff; ++c) {
 514         *s++=(UChar)c;
 515     }
 516     // lead surrogates
 517     // (after trails so that there is not even one surrogate pair in between)
 518     for(c=0xd800; c<=0xdbff; ++c) {
 519         *s++=(UChar)c;
 520     }
 521     // high BMP
 522     for(c=0xe000; c<=0xffff; ++c) {
 523         *s++=(UChar)c;
 524     }
 525     // supplementary code points = surrogate pairs
 526     if(cpLimit==0x110000) {
 527         for(c=0xd800; c<=0xdbff; ++c) {
 528             for(c2=0xdc00; c2<=0xdfff; ++c2) {
 529                 *s++=(UChar)c;
 530                 *s++=c2;
 531             }
 532         }
 533     }
 534
 535     static const char *const cnvNames[]={
 536         "UTF-8",
 537         "UTF-7",
 538         "UTF-16",
 539         "US-ASCII",
 540         "ISO-8859-1",
 541         "windows-1252",
 542         "Shift-JIS",
 543         "ibm-1390",  // EBCDIC_STATEFUL table
 544         "ibm-16684",  // DBCS-only extension table based on EBCDIC_STATEFUL table
 545         "HZ",
 546         "ISO-2022-JP",
 547         "JIS7",
 548         "ISO-2022-CN",
 549         "ISO-2022-CN-EXT",
 550         "LMBCS"
 551     };
 552     LocalUConverterPointer cnv;
 553     char buffer[1024];
 554     int32_t i;
 555     for(i=0; i<UPRV_LENGTHOF(cnvNames); ++i) {
 556         UErrorCode errorCode=U_ZERO_ERROR;
 557         cnv.adoptInstead(cnv_open(cnvNames[i], errorCode));
 558         if(U_FAILURE(errorCode)) {
 559             errcheckln(errorCode, "failed to open converter %s - %s", cnvNames[i], u_errorName(errorCode));
 560             continue;
 561         }
 562         UnicodeSet expected;
 563         ucnv_setFromUCallBack(cnv.getAlias(), getUnicodeSetCallback, &expected, NULL, NULL, &errorCode);
 564         if(U_FAILURE(errorCode)) {
 565             errln("failed to set the callback on converter %s - %s", cnvNames[i], u_errorName(errorCode));
 566             continue;
 567         }
 568         UConverterUnicodeSet which;
 569         for(which=UCNV_ROUNDTRIP_SET; which<UCNV_SET_COUNT; which=(UConverterUnicodeSet)((int)which+1)) {
 570             if(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET) {
 571                 ucnv_setFallback(cnv.getAlias(), TRUE);
 572             }
 573             expected.add(0, cpLimit-1);
 574             s=s0;
 575             UBool flush;
 576             do {
 577                 char *t=buffer;
 578                 flush=(UBool)(s==s0+s0Length);
 579                 ucnv_fromUnicode(cnv.getAlias(), &t, buffer+sizeof(buffer), (const UChar **)&s, s0+s0Length, NULL, flush, &errorCode);
 580                 if(U_FAILURE(errorCode)) {
 581                     if(errorCode==U_BUFFER_OVERFLOW_ERROR) {
 582                         errorCode=U_ZERO_ERROR;
 583                         continue;
 584                     } else {
 585                         break;  // unexpected error, should not occur
 586                     }
 587                 }
 588             } while(!flush);
 589             UnicodeSet set;
 590             ucnv_getUnicodeSet(cnv.getAlias(), set.toUSet(), which, &errorCode);
 591             if(cpLimit<0x110000) {
 592                 set.remove(cpLimit, 0x10ffff);
 593             }
 594             if(which==UCNV_ROUNDTRIP_SET) {
 595                 // ignore PUA code points because they will be converted even if they
 596                 // are fallbacks and when other fallbacks are turned off,
 597                 // but ucnv_getUnicodeSet(UCNV_ROUNDTRIP_SET) delivers true roundtrips
 598                 expected.remove(0xe000, 0xf8ff);
 599                 expected.remove(0xf0000, 0xffffd);
 600                 expected.remove(0x100000, 0x10fffd);
 601                 set.remove(0xe000, 0xf8ff);
 602                 set.remove(0xf0000, 0xffffd);
 603                 set.remove(0x100000, 0x10fffd);
 604             }
 605             if(set!=expected) {
 606                 // First try to see if we have different sets because ucnv_getUnicodeSet()
 607                 // added strings: The above conversion method does not tell us what strings might be convertible.
 608                 // Remove strings from the set and compare again.
 609                 set.removeAllStrings();
 610             }
 611             if(set!=expected) {
 612                 UnicodeSet diffSet;
 613                 UnicodeString out;
 614
 615                 // are there items that must be in the set but are not?
 616                 (diffSet=expected).removeAll(set);
 617                 if(!diffSet.isEmpty()) {
 618                     diffSet.toPattern(out, TRUE);
 619                     if(out.length()>100) {
 620                         out.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
 621                     }
 622                     errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - which set: %d",
 623                             cnvNames[i], which);
 624                     errln(out);
 625                 }
 626
 627                 // are there items that must not be in the set but are?
 628                 (diffSet=set).removeAll(expected);
 629                 if(!diffSet.isEmpty()) {
 630                     diffSet.toPattern(out, TRUE);
 631                     if(out.length()>100) {
 632                         out.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
 633                     }
 634                     errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - which set: %d",
 635                             cnvNames[i], which);
 636                     errln(out);
 637                 }
 638             }
 639         }
 640     }
 641
 642     delete [] s0;
 643 }
 644
 645 // Test all codepoints which has the default ignorable Unicode property are ignored if they have no mapping
 646 // If there are any failures, the hard coded list (IS_DEFAULT_IGNORABLE_CODE_POINT) in ucnv_err.c should be updated
 647 void
 648 ConversionTest::TestDefaultIgnorableCallback() {
 649     UErrorCode status = U_ZERO_ERROR;
 650     const char *cnv_name = "euc-jp-2007";
 651     const char *pattern_ignorable = "[:Default_Ignorable_Code_Point:]";
 652     const char *pattern_not_ignorable = "[:^Default_Ignorable_Code_Point:]";
 653
 654     UnicodeSet *set_ignorable = new UnicodeSet(pattern_ignorable, status);
 655     if (U_FAILURE(status)) {
 656         dataerrln("Unable to create Unicodeset: %s - %s\n", pattern_ignorable, u_errorName(status));
 657         return;
 658     }
 659
 660     UnicodeSet *set_not_ignorable = new UnicodeSet(pattern_not_ignorable, status);
 661     if (U_FAILURE(status)) {
 662         dataerrln("Unable to create Unicodeset: %s - %s\n", pattern_not_ignorable, u_errorName(status));
 663         return;
 664     }
 665
 666     UConverter *cnv = cnv_open(cnv_name, status);
 667     if (U_FAILURE(status)) {
 668         dataerrln("Unable to open converter: %s - %s\n", cnv_name, u_errorName(status));
 669         return;
 670     }
 671
 672     // set callback for the converter
 673     ucnv_setFromUCallBack(cnv, UCNV_FROM_U_CALLBACK_SUBSTITUTE, NULL, NULL, NULL, &status);
 674
 675     UChar32 input[1];
 676     char output[10];
 677     int32_t outputLength;
 678
 679     // test default ignorables are ignored
 680     int size = set_ignorable->size();
 681     for (int i = 0; i < size; i++) {
 682         status = U_ZERO_ERROR;
 683         outputLength= 0;
 684
 685         input[0] = set_ignorable->charAt(i);
 686
 687         outputLength = ucnv_fromUChars(cnv, output, 10, UnicodeString::fromUTF32(input, 1).getTerminatedBuffer(), -1, &status);
 688         if (U_FAILURE(status) || outputLength != 0) {
 689             errln("Ignorable code point: U+%04X not skipped as expected - %s", input[0], u_errorName(status));
 690         }
 691     }
 692
 693     // test non-ignorables are not ignored
 694     size = set_not_ignorable->size();
 695     for (int i = 0; i < size; i++) {
 696         status = U_ZERO_ERROR;
 697         outputLength= 0;
 698
 699         input[0] = set_not_ignorable->charAt(i);
 700
 701         if (input[0] == 0) {
 702             continue;
 703         }
 704
 705         outputLength = ucnv_fromUChars(cnv, output, 10, UnicodeString::fromUTF32(input, 1).getTerminatedBuffer(), -1, &status);
 706         if (U_FAILURE(status) || outputLength <= 0) {
 707             errln("Non-ignorable code point: U+%04X skipped unexpectedly - %s", input[0], u_errorName(status));
 708         }
 709     }
 710
 711     ucnv_close(cnv);
 712     delete set_not_ignorable;
 713     delete set_ignorable;
 714 }
 715
 716 void
 717 ConversionTest::TestUTF8ToUTF8Overflow() {
 718     IcuTestErrorCode errorCode(*this, "TestUTF8ToUTF8Overflow");
 719     LocalUConverterPointer cnv1(ucnv_open("UTF-8", errorCode));
 720     LocalUConverterPointer cnv2(ucnv_open("UTF-8", errorCode));
 721     static const char *text = "aä";  // ä: 2 bytes
 722     const char *source = text;
 723     const char *sourceLimit = text + strlen(text);
 724     char result[20];
 725     char *target = result;
 726     const char *targetLimit = result + sizeof(result);
 727     UChar buffer16[20];
 728     UChar *pivotSource = buffer16;
 729     UChar *pivotTarget = buffer16;
 730     const UChar *pivotLimit = buffer16 + UPRV_LENGTHOF(buffer16);
 731     int32_t length;
 732
 733     // Convert with insufficient target capacity.
 734     result[2] = 5;
 735     ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
 736                    &target, result + 2, &source, sourceLimit,
 737                    buffer16, &pivotSource, &pivotTarget, pivotLimit,
 738                    FALSE, FALSE, errorCode);
 739     assertEquals("overflow", U_BUFFER_OVERFLOW_ERROR, errorCode.reset());
 740     length = (int32_t)(target - result);
 741     assertEquals("number of bytes written", 2, length);
 742     assertEquals("next byte not clobbered", 5, result[2]);
 743
 744     // Convert the rest and flush.
 745     ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
 746                    &target, targetLimit, &source, sourceLimit,
 747                    buffer16, &pivotSource, &pivotTarget, pivotLimit,
 748                    FALSE, TRUE, errorCode);
 749
 750     assertSuccess("UTF-8->UTF-8", errorCode);
 751     length = (int32_t)(target - result);
 752     assertEquals("3 bytes", 3, length);
 753     if (length == 3) {
 754         assertTrue("result same as input", memcmp(text, result, length) == 0);
 755     }
 756
 757     ucnv_reset(cnv1.getAlias());
 758     ucnv_reset(cnv2.getAlias());
 759     memset(result, 0, sizeof(result));
 760     static const char *text2 = "a🚲";  // U+1F6B2 bicycle: 4 bytes
 761     source = text2;
 762     sourceLimit = text2 + strlen(text2);
 763     target = result;
 764     pivotSource = pivotTarget = buffer16;
 765
 766     // Convert with insufficient target capacity.
 767     result[3] = 5;
 768     ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
 769                    &target, result + 3, &source, sourceLimit,
 770                    buffer16, &pivotSource, &pivotTarget, pivotLimit,
 771                    FALSE, FALSE, errorCode);
 772     assertEquals("text2 overflow", U_BUFFER_OVERFLOW_ERROR, errorCode.reset());
 773     length = (int32_t)(target - result);
 774     assertEquals("text2 number of bytes written", 3, length);
 775     assertEquals("text2 next byte not clobbered", 5, result[3]);
 776
 777     // Convert the rest and flush.
 778     ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
 779                    &target, targetLimit, &source, sourceLimit,
 780                    buffer16, &pivotSource, &pivotTarget, pivotLimit,
 781                    FALSE, TRUE, errorCode);
 782
 783     assertSuccess("text2 UTF-8->UTF-8", errorCode);
 784     length = (int32_t)(target - result);
 785     assertEquals("text2 5 bytes", 5, length);
 786     if (length == 5) {
 787         assertTrue("text2 result same as input", memcmp(text2, result, length) == 0);
 788     }
 789
 790     ucnv_reset(cnv1.getAlias());
 791     ucnv_reset(cnv2.getAlias());
 792     memset(result, 0, sizeof(result));
 793     static const char *illFormed = "\xf1\x91\x93\x96\x91\x94";  // U+514D6 + two more trail bytes
 794     source = illFormed;
 795     sourceLimit = illFormed + strlen(illFormed);
 796     target = result;
 797     pivotSource = pivotTarget = buffer16;
 798
 799     ucnv_setToUCallBack(cnv1.getAlias(), UCNV_TO_U_CALLBACK_STOP, nullptr, nullptr, nullptr, errorCode);
 800
 801     // Convert only two bytes and flush (but expect failure).
 802     char errorBytes[10];
 803     int8_t errorLength;
 804     result[0] = 5;
 805     ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
 806                    &target, targetLimit, &source, source + 2,
 807                    buffer16, &pivotSource, &pivotTarget, pivotLimit,
 808                    FALSE, TRUE, errorCode);
 809     assertEquals("illFormed truncated", U_TRUNCATED_CHAR_FOUND, errorCode.reset());
 810     length = (int32_t)(target - result);
 811     assertEquals("illFormed number of bytes written", 0, length);
 812     errorLength = UPRV_LENGTHOF(errorBytes);
 813     ucnv_getInvalidChars(cnv1.getAlias(), errorBytes, &errorLength, errorCode);
 814     assertEquals("illFormed truncated errorLength", 2, (int32_t)errorLength);
 815     if (errorLength == 2) {
 816         assertEquals("illFormed truncated errorBytes", 0xf191,
 817                      ((int32_t)(uint8_t)errorBytes[0] << 8) | (uint8_t)errorBytes[1]);
 818     }
 819
 820     // Continue conversion starting with a trail byte.
 821     ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
 822                    &target, targetLimit, &source, sourceLimit,
 823                    buffer16, &pivotSource, &pivotTarget, pivotLimit,
 824                    FALSE, TRUE, errorCode);
 825
 826     assertEquals("illFormed trail byte", U_ILLEGAL_CHAR_FOUND, errorCode.reset());
 827     length = (int32_t)(target - result);
 828     assertEquals("illFormed trail byte number of bytes written", 0, length);
 829     errorLength = UPRV_LENGTHOF(errorBytes);
 830     ucnv_getInvalidChars(cnv1.getAlias(), errorBytes, &errorLength, errorCode);
 831     assertEquals("illFormed trail byte errorLength", 1, (int32_t)errorLength);
 832     if (errorLength == 1) {
 833         assertEquals("illFormed trail byte errorBytes", 0x93, (int32_t)(uint8_t)errorBytes[0]);
 834     }
 835 }
 836
 837 // open testdata or ICU data converter ------------------------------------- ***
 838
 839 UConverter *
 840 ConversionTest::cnv_open(const char *name, UErrorCode &errorCode) {
 841     if(name!=NULL && *name=='+') {
 842         // Converter names that start with '+' are ignored in ICU4J tests.
 843         ++name;
 844     }
 845     if(name!=NULL && *name=='*') {
 846         /* loadTestData(): set the data directory */
 847         return ucnv_openPackage(loadTestData(errorCode), name+1, &errorCode);
 848     } else {
 849         return ucnv_open(name, &errorCode);
 850     }
 851 }
 852
 853 // output helpers ---------------------------------------------------------- ***
 854
 855 static inline char
 856 hexDigit(uint8_t digit) {
 857     return digit<=9 ? (char)('0'+digit) : (char)('a'-10+digit);
 858 }
 859
 860 static char *
 861 printBytes(const uint8_t *bytes, int32_t length, char *out) {
 862     uint8_t b;
 863
 864     if(length>0) {
 865         b=*bytes++;
 866         --length;
 867         *out++=hexDigit((uint8_t)(b>>4));
 868         *out++=hexDigit((uint8_t)(b&0xf));
 869     }
 870
 871     while(length>0) {
 872         b=*bytes++;
 873         --length;
 874         *out++=' ';
 875         *out++=hexDigit((uint8_t)(b>>4));
 876         *out++=hexDigit((uint8_t)(b&0xf));
 877     }
 878     *out++=0;
 879     return out;
 880 }
 881
 882 static char *
 883 printUnicode(const UChar *unicode, int32_t length, char *out) {
 884     UChar32 c;
 885     int32_t i;
 886
 887     for(i=0; i<length;) {
 888         if(i>0) {
 889             *out++=' ';
 890         }
 891         U16_NEXT(unicode, i, length, c);
 892         // write 4..6 digits
 893         if(c>=0x100000) {
 894             *out++='1';
 895         }
 896         if(c>=0x10000) {
 897             *out++=hexDigit((uint8_t)((c>>16)&0xf));
 898         }
 899         *out++=hexDigit((uint8_t)((c>>12)&0xf));
 900         *out++=hexDigit((uint8_t)((c>>8)&0xf));
 901         *out++=hexDigit((uint8_t)((c>>4)&0xf));
 902         *out++=hexDigit((uint8_t)(c&0xf));
 903     }
 904     *out++=0;
 905     return out;
 906 }
 907
 908 static char *
 909 printOffsets(const int32_t *offsets, int32_t length, char *out) {
 910     int32_t i, o, d;
 911
 912     if(offsets==NULL) {
 913         length=0;
 914     }
 915
 916     for(i=0; i<length; ++i) {
 917         if(i>0) {
 918             *out++=' ';
 919         }
 920         o=offsets[i];
 921
 922         // print all offsets with 2 characters each (-x, -9..99, xx)
 923         if(o<-9) {
 924             *out++='-';
 925             *out++='x';
 926         } else if(o<0) {
 927             *out++='-';
 928             *out++=(char)('0'-o);
 929         } else if(o<=99) {
 930             *out++=(d=o/10)==0 ? ' ' : (char)('0'+d);
 931             *out++=(char)('0'+o%10);
 932         } else /* o>99 */ {
 933             *out++='x';
 934             *out++='x';
 935         }
 936     }
 937     *out++=0;
 938     return out;
 939 }
 940
 941 // toUnicode test worker functions ----------------------------------------- ***
 942
 943 static int32_t
 944 stepToUnicode(ConversionCase &cc, UConverter *cnv,
 945               UChar *result, int32_t resultCapacity,
 946               int32_t *resultOffsets, /* also resultCapacity */
 947               int32_t step,
 948               UErrorCode *pErrorCode) {
 949     const char *source, *sourceLimit, *bytesLimit;
 950     UChar *target, *targetLimit, *resultLimit;
 951     UBool flush;
 952
 953     source=(const char *)cc.bytes;
 954     target=result;
 955     bytesLimit=source+cc.bytesLength;
 956     resultLimit=result+resultCapacity;
 957
 958     if(step>=0) {
 959         // call ucnv_toUnicode() with in/out buffers no larger than (step) at a time
 960         // move only one buffer (in vs. out) at a time to be extra mean
 961         // step==0 performs bulk conversion and generates offsets
 962
 963         // initialize the partial limits for the loop
 964         if(step==0) {
 965             // use the entire buffers
 966             sourceLimit=bytesLimit;
 967             targetLimit=resultLimit;
 968             flush=cc.finalFlush;
 969         } else {
 970             // start with empty partial buffers
 971             sourceLimit=source;
 972             targetLimit=target;
 973             flush=FALSE;
 974
 975             // output offsets only for bulk conversion
 976             resultOffsets=NULL;
 977         }
 978
 979         for(;;) {
 980             // resetting the opposite conversion direction must not affect this one
 981             ucnv_resetFromUnicode(cnv);
 982
 983             // convert
 984             ucnv_toUnicode(cnv,
 985                 &target, targetLimit,
 986                 &source, sourceLimit,
 987                 resultOffsets,
 988                 flush, pErrorCode);
 989
 990             // check pointers and errors
 991             if(source>sourceLimit || target>targetLimit) {
 992                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
 993                 break;
 994             } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
 995                 if(target!=targetLimit) {
 996                     // buffer overflow must only be set when the target is filled
 997                     *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
 998                     break;
 999                 } else if(targetLimit==resultLimit) {
1000                     // not just a partial overflow
1001                     break;
1002                 }
1003
1004                 // the partial target is filled, set a new limit, reset the error and continue
1005                 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1006                 *pErrorCode=U_ZERO_ERROR;
1007             } else if(U_FAILURE(*pErrorCode)) {
1008                 // some other error occurred, done
1009                 break;
1010             } else {
1011                 if(source!=sourceLimit) {
1012                     // when no error occurs, then the input must be consumed
1013                     *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1014                     break;
1015                 }
1016
1017                 if(sourceLimit==bytesLimit) {
1018                     // we are done
1019                     break;
1020                 }
1021
1022                 // the partial conversion succeeded, set a new limit and continue
1023                 sourceLimit=(bytesLimit-source)>=step ? source+step : bytesLimit;
1024                 flush=(UBool)(cc.finalFlush && sourceLimit==bytesLimit);
1025             }
1026         }
1027     } else /* step<0 */ {
1028         /*
1029          * step==-1: call only ucnv_getNextUChar()
1030          * otherwise alternate between ucnv_toUnicode() and ucnv_getNextUChar()
1031          *   if step==-2 or -3, then give ucnv_toUnicode() the whole remaining input,
1032          *   else give it at most (-step-2)/2 bytes
1033          */
1034         UChar32 c;
1035
1036         // end the loop by getting an index out of bounds error
1037         for(;;) {
1038             // resetting the opposite conversion direction must not affect this one
1039             ucnv_resetFromUnicode(cnv);
1040
1041             // convert
1042             if((step&1)!=0 /* odd: -1, -3, -5, ... */) {
1043                 sourceLimit=source; // use sourceLimit not as a real limit
1044                                     // but to remember the pre-getNextUChar source pointer
1045                 c=ucnv_getNextUChar(cnv, &source, bytesLimit, pErrorCode);
1046
1047                 // check pointers and errors
1048                 if(*pErrorCode==U_INDEX_OUTOFBOUNDS_ERROR) {
1049                     if(source!=bytesLimit) {
1050                         *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1051                     } else {
1052                         *pErrorCode=U_ZERO_ERROR;
1053                     }
1054                     break;
1055                 } else if(U_FAILURE(*pErrorCode)) {
1056                     break;
1057                 }
1058                 // source may not move if c is from previous overflow
1059
1060                 if(target==resultLimit) {
1061                     *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
1062                     break;
1063                 }
1064                 if(c<=0xffff) {
1065                     *target++=(UChar)c;
1066                 } else {
1067                     *target++=U16_LEAD(c);
1068                     if(target==resultLimit) {
1069                         *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
1070                         break;
1071                     }
1072                     *target++=U16_TRAIL(c);
1073                 }
1074
1075                 // alternate between -n-1 and -n but leave -1 alone
1076                 if(step<-1) {
1077                     ++step;
1078                 }
1079             } else /* step is even */ {
1080                 // allow only one UChar output
1081                 targetLimit=target<resultLimit ? target+1 : resultLimit;
1082
1083                 // as with ucnv_getNextUChar(), we always flush (if we go to bytesLimit)
1084                 // and never output offsets
1085                 if(step==-2) {
1086                     sourceLimit=bytesLimit;
1087                 } else {
1088                     sourceLimit=source+(-step-2)/2;
1089                     if(sourceLimit>bytesLimit) {
1090                         sourceLimit=bytesLimit;
1091                     }
1092                 }
1093
1094                 ucnv_toUnicode(cnv,
1095                     &target, targetLimit,
1096                     &source, sourceLimit,
1097                     NULL, (UBool)(sourceLimit==bytesLimit), pErrorCode);
1098
1099                 // check pointers and errors
1100                 if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1101                     if(target!=targetLimit) {
1102                         // buffer overflow must only be set when the target is filled
1103                         *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1104                         break;
1105                     } else if(targetLimit==resultLimit) {
1106                         // not just a partial overflow
1107                         break;
1108                     }
1109
1110                     // the partial target is filled, set a new limit and continue
1111                     *pErrorCode=U_ZERO_ERROR;
1112                 } else if(U_FAILURE(*pErrorCode)) {
1113                     // some other error occurred, done
1114                     break;
1115                 } else {
1116                     if(source!=sourceLimit) {
1117                         // when no error occurs, then the input must be consumed
1118                         *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1119                         break;
1120                     }
1121
1122                     // we are done (flush==TRUE) but we continue, to get the index out of bounds error above
1123                 }
1124
1125                 --step;
1126             }
1127         }
1128     }
1129
1130     return (int32_t)(target-result);
1131 }
1132
1133 UBool
1134 ConversionTest::ToUnicodeCase(ConversionCase &cc, UConverterToUCallback callback, const char *option) {
1135     // open the converter
1136     IcuTestErrorCode errorCode(*this, "ToUnicodeCase");
1137     LocalUConverterPointer cnv(cnv_open(cc.charset, errorCode));
1138     // with no data, the above crashes with "pointer being freed was not allocated" for charset "x11-compound-text", see #13078
1139     if(errorCode.isFailure()) {
1140         errcheckln(errorCode, "toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
1141                 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, errorCode.errorName());
1142         errorCode.reset();
1143         return FALSE;
1144     }
1145
1146     // set the callback
1147     if(callback!=NULL) {
1148         ucnv_setToUCallBack(cnv.getAlias(), callback, option, NULL, NULL, errorCode);
1149         if(U_FAILURE(errorCode)) {
1150             errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setToUCallBack() failed - %s",
1151                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1152             return FALSE;
1153         }
1154     }
1155
1156     int32_t resultOffsets[256];
1157     UChar result[256];
1158     int32_t resultLength;
1159     UBool ok;
1160
1161     static const struct {
1162         int32_t step;
1163         const char *name;
1164     } steps[]={
1165         { 0, "bulk" }, // must be first for offsets to be checked
1166         { 1, "step=1" },
1167         { 3, "step=3" },
1168         { 7, "step=7" },
1169         { -1, "getNext" },
1170         { -2, "toU(bulk)+getNext" },
1171         { -3, "getNext+toU(bulk)" },
1172         { -4, "toU(1)+getNext" },
1173         { -5, "getNext+toU(1)" },
1174         { -12, "toU(5)+getNext" },
1175         { -13, "getNext+toU(5)" },
1176     };
1177     int32_t i, step;
1178
1179     ok=TRUE;
1180     for(i=0; i<UPRV_LENGTHOF(steps) && ok; ++i) {
1181         step=steps[i].step;
1182         if(step<0 && !cc.finalFlush) {
1183             // skip ucnv_getNextUChar() if !finalFlush because
1184             // ucnv_getNextUChar() always implies flush
1185             continue;
1186         }
1187         if(step!=0) {
1188             // bulk test is first, then offsets are not checked any more
1189             cc.offsets=NULL;
1190         }
1191         else {
1192             memset(resultOffsets, -1, UPRV_LENGTHOF(resultOffsets));
1193         }
1194         memset(result, -1, UPRV_LENGTHOF(result));
1195         errorCode.reset();
1196         resultLength=stepToUnicode(cc, cnv.getAlias(),
1197                                 result, UPRV_LENGTHOF(result),
1198                                 step==0 ? resultOffsets : NULL,
1199                                 step, errorCode);
1200         ok=checkToUnicode(
1201                 cc, cnv.getAlias(), steps[i].name,
1202                 result, resultLength,
1203                 cc.offsets!=NULL ? resultOffsets : NULL,
1204                 errorCode);
1205         if(errorCode.isFailure() || !cc.finalFlush) {
1206             // reset if an error occurred or we did not flush
1207             // otherwise do nothing to make sure that flushing resets
1208             ucnv_resetToUnicode(cnv.getAlias());
1209         }
1210         if (cc.offsets != NULL && resultOffsets[resultLength] != -1) {
1211             errln("toUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1212                 cc.caseNr, cc.charset, resultLength);
1213         }
1214         if (result[resultLength] != (UChar)-1) {
1215             errln("toUnicode[%d](%s) Conversion wrote too much to result at index %d",
1216                 cc.caseNr, cc.charset, resultLength);
1217         }
1218     }
1219
1220     // not a real loop, just a convenience for breaking out of the block
1221     while(ok && cc.finalFlush) {
1222         // test ucnv_toUChars()
1223         memset(result, 0, sizeof(result));
1224
1225         errorCode.reset();
1226         resultLength=ucnv_toUChars(cnv.getAlias(),
1227                         result, UPRV_LENGTHOF(result),
1228                         (const char *)cc.bytes, cc.bytesLength,
1229                         errorCode);
1230         ok=checkToUnicode(
1231                 cc, cnv.getAlias(), "toUChars",
1232                 result, resultLength,
1233                 NULL,
1234                 errorCode);
1235         if(!ok) {
1236             break;
1237         }
1238
1239         // test preflighting
1240         // keep the correct result for simple checking
1241         errorCode.reset();
1242         resultLength=ucnv_toUChars(cnv.getAlias(),
1243                         NULL, 0,
1244                         (const char *)cc.bytes, cc.bytesLength,
1245                         errorCode);
1246         if(errorCode.get()==U_STRING_NOT_TERMINATED_WARNING || errorCode.get()==U_BUFFER_OVERFLOW_ERROR) {
1247             errorCode.reset();
1248         }
1249         ok=checkToUnicode(
1250                 cc, cnv.getAlias(), "preflight toUChars",
1251                 result, resultLength,
1252                 NULL,
1253                 errorCode);
1254         break;
1255     }
1256
1257     errorCode.reset();  // all errors have already been reported
1258     return ok;
1259 }
1260
1261 UBool
1262 ConversionTest::checkToUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1263                                const UChar *result, int32_t resultLength,
1264                                const int32_t *resultOffsets,
1265                                UErrorCode resultErrorCode) {
1266     char resultInvalidChars[8];
1267     int8_t resultInvalidLength;
1268     UErrorCode errorCode;
1269
1270     const char *msg;
1271
1272     // reset the message; NULL will mean "ok"
1273     msg=NULL;
1274
1275     errorCode=U_ZERO_ERROR;
1276     resultInvalidLength=sizeof(resultInvalidChars);
1277     ucnv_getInvalidChars(cnv, resultInvalidChars, &resultInvalidLength, &errorCode);
1278     if(U_FAILURE(errorCode)) {
1279         errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidChars() failed - %s",
1280                 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1281         return FALSE;
1282     }
1283
1284     // check everything that might have gone wrong
1285     if(cc.unicodeLength!=resultLength) {
1286         msg="wrong result length";
1287     } else if(0!=u_memcmp(cc.unicode, result, cc.unicodeLength)) {
1288         msg="wrong result string";
1289     } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.unicodeLength*sizeof(*cc.offsets))) {
1290         msg="wrong offsets";
1291     } else if(cc.outErrorCode!=resultErrorCode) {
1292         msg="wrong error code";
1293     } else if(cc.invalidLength!=resultInvalidLength) {
1294         msg="wrong length of last invalid input";
1295     } else if(0!=memcmp(cc.invalidChars, resultInvalidChars, cc.invalidLength)) {
1296         msg="wrong last invalid input";
1297     }
1298
1299     if(msg==NULL) {
1300         return TRUE;
1301     } else {
1302         char buffer[2000]; // one buffer for all strings
1303         char *s, *bytesString, *unicodeString, *resultString,
1304             *offsetsString, *resultOffsetsString,
1305             *invalidCharsString, *resultInvalidCharsString;
1306
1307         bytesString=s=buffer;
1308         s=printBytes(cc.bytes, cc.bytesLength, bytesString);
1309         s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString=s);
1310         s=printUnicode(result, resultLength, resultString=s);
1311         s=printOffsets(cc.offsets, cc.unicodeLength, offsetsString=s);
1312         s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1313         s=printBytes(cc.invalidChars, cc.invalidLength, invalidCharsString=s);
1314         s=printBytes((uint8_t *)resultInvalidChars, resultInvalidLength, resultInvalidCharsString=s);
1315
1316         if((s-buffer)>(int32_t)sizeof(buffer)) {
1317             errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkToUnicode() test output buffer overflow writing %d chars\n",
1318                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1319             exit(1);
1320         }
1321
1322         errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1323               "  bytes <%s>[%d]\n"
1324               " expected <%s>[%d]\n"
1325               "  result  <%s>[%d]\n"
1326               " offsets         <%s>\n"
1327               "  result offsets <%s>\n"
1328               " error code expected %s got %s\n"
1329               "  invalidChars expected <%s> got <%s>\n",
1330               cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1331               bytesString, cc.bytesLength,
1332               unicodeString, cc.unicodeLength,
1333               resultString, resultLength,
1334               offsetsString,
1335               resultOffsetsString,
1336               u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1337               invalidCharsString, resultInvalidCharsString);
1338
1339         return FALSE;
1340     }
1341 }
1342
1343 // fromUnicode test worker functions --------------------------------------- ***
1344
1345 static int32_t
1346 stepFromUTF8(ConversionCase &cc,
1347              UConverter *utf8Cnv, UConverter *cnv,
1348              char *result, int32_t resultCapacity,
1349              int32_t step,
1350              UErrorCode *pErrorCode) {
1351     const char *source, *sourceLimit, *utf8Limit;
1352     UChar pivotBuffer[32];
1353     UChar *pivotSource, *pivotTarget, *pivotLimit;
1354     char *target, *targetLimit, *resultLimit;
1355     UBool flush;
1356
1357     source=cc.utf8;
1358     pivotSource=pivotTarget=pivotBuffer;
1359     target=result;
1360     utf8Limit=source+cc.utf8Length;
1361     resultLimit=result+resultCapacity;
1362
1363     // call ucnv_convertEx() with in/out buffers no larger than (step) at a time
1364     // move only one buffer (in vs. out) at a time to be extra mean
1365     // step==0 performs bulk conversion
1366
1367     // initialize the partial limits for the loop
1368     if(step==0) {
1369         // use the entire buffers
1370         sourceLimit=utf8Limit;
1371         targetLimit=resultLimit;
1372         flush=cc.finalFlush;
1373
1374         pivotLimit=pivotBuffer+UPRV_LENGTHOF(pivotBuffer);
1375     } else {
1376         // start with empty partial buffers
1377         sourceLimit=source;
1378         targetLimit=target;
1379         flush=FALSE;
1380
1381         // empty pivot is not allowed, make it of length step
1382         pivotLimit=pivotBuffer+step;
1383     }
1384
1385     for(;;) {
1386         // resetting the opposite conversion direction must not affect this one
1387         ucnv_resetFromUnicode(utf8Cnv);
1388         ucnv_resetToUnicode(cnv);
1389
1390         // convert
1391         ucnv_convertEx(cnv, utf8Cnv,
1392             &target, targetLimit,
1393             &source, sourceLimit,
1394             pivotBuffer, &pivotSource, &pivotTarget, pivotLimit,
1395             FALSE, flush, pErrorCode);
1396
1397         // check pointers and errors
1398         if(source>sourceLimit || target>targetLimit) {
1399             *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1400             break;
1401         } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1402             if(target!=targetLimit) {
1403                 // buffer overflow must only be set when the target is filled
1404                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1405                 break;
1406             } else if(targetLimit==resultLimit) {
1407                 // not just a partial overflow
1408                 break;
1409             }
1410
1411             // the partial target is filled, set a new limit, reset the error and continue
1412             targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1413             *pErrorCode=U_ZERO_ERROR;
1414         } else if(U_FAILURE(*pErrorCode)) {
1415             if(pivotSource==pivotBuffer) {
1416                 // toUnicode error, should not occur
1417                 // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1418                 break;
1419             } else {
1420                 // fromUnicode error
1421                 // some other error occurred, done
1422                 break;
1423             }
1424         } else {
1425             if(source!=sourceLimit) {
1426                 // when no error occurs, then the input must be consumed
1427                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1428                 break;
1429             }
1430
1431             if(sourceLimit==utf8Limit) {
1432                 // we are done
1433                 if(*pErrorCode==U_STRING_NOT_TERMINATED_WARNING) {
1434                     // ucnv_convertEx() warns about not terminating the output
1435                     // but ucnv_fromUnicode() does not and so
1436                     // checkFromUnicode() does not expect it
1437                     *pErrorCode=U_ZERO_ERROR;
1438                 }
1439                 break;
1440             }
1441
1442             // the partial conversion succeeded, set a new limit and continue
1443             sourceLimit=(utf8Limit-source)>=step ? source+step : utf8Limit;
1444             flush=(UBool)(cc.finalFlush && sourceLimit==utf8Limit);
1445         }
1446     }
1447
1448     return (int32_t)(target-result);
1449 }
1450
1451 static int32_t
1452 stepFromUnicode(ConversionCase &cc, UConverter *cnv,
1453                 char *result, int32_t resultCapacity,
1454                 int32_t *resultOffsets, /* also resultCapacity */
1455                 int32_t step,
1456                 UErrorCode *pErrorCode) {
1457     const UChar *source, *sourceLimit, *unicodeLimit;
1458     char *target, *targetLimit, *resultLimit;
1459     UBool flush;
1460
1461     source=cc.unicode;
1462     target=result;
1463     unicodeLimit=source+cc.unicodeLength;
1464     resultLimit=result+resultCapacity;
1465
1466     // call ucnv_fromUnicode() with in/out buffers no larger than (step) at a time
1467     // move only one buffer (in vs. out) at a time to be extra mean
1468     // step==0 performs bulk conversion and generates offsets
1469
1470     // initialize the partial limits for the loop
1471     if(step==0) {
1472         // use the entire buffers
1473         sourceLimit=unicodeLimit;
1474         targetLimit=resultLimit;
1475         flush=cc.finalFlush;
1476     } else {
1477         // start with empty partial buffers
1478         sourceLimit=source;
1479         targetLimit=target;
1480         flush=FALSE;
1481
1482         // output offsets only for bulk conversion
1483         resultOffsets=NULL;
1484     }
1485
1486     for(;;) {
1487         // resetting the opposite conversion direction must not affect this one
1488         ucnv_resetToUnicode(cnv);
1489
1490         // convert
1491         ucnv_fromUnicode(cnv,
1492             &target, targetLimit,
1493             &source, sourceLimit,
1494             resultOffsets,
1495             flush, pErrorCode);
1496
1497         // check pointers and errors
1498         if(source>sourceLimit || target>targetLimit) {
1499             *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1500             break;
1501         } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1502             if(target!=targetLimit) {
1503                 // buffer overflow must only be set when the target is filled
1504                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1505                 break;
1506             } else if(targetLimit==resultLimit) {
1507                 // not just a partial overflow
1508                 break;
1509             }
1510
1511             // the partial target is filled, set a new limit, reset the error and continue
1512             targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1513             *pErrorCode=U_ZERO_ERROR;
1514         } else if(U_FAILURE(*pErrorCode)) {
1515             // some other error occurred, done
1516             break;
1517         } else {
1518             if(source!=sourceLimit) {
1519                 // when no error occurs, then the input must be consumed
1520                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1521                 break;
1522             }
1523
1524             if(sourceLimit==unicodeLimit) {
1525                 // we are done
1526                 break;
1527             }
1528
1529             // the partial conversion succeeded, set a new limit and continue
1530             sourceLimit=(unicodeLimit-source)>=step ? source+step : unicodeLimit;
1531             flush=(UBool)(cc.finalFlush && sourceLimit==unicodeLimit);
1532         }
1533     }
1534
1535     return (int32_t)(target-result);
1536 }
1537
1538 UBool
1539 ConversionTest::FromUnicodeCase(ConversionCase &cc, UConverterFromUCallback callback, const char *option) {
1540     UConverter *cnv;
1541     UErrorCode errorCode;
1542
1543     // open the converter
1544     errorCode=U_ZERO_ERROR;
1545     cnv=cnv_open(cc.charset, errorCode);
1546     if(U_FAILURE(errorCode)) {
1547         errcheckln(errorCode, "fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
1548                 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1549         return FALSE;
1550     }
1551     ucnv_resetToUnicode(utf8Cnv);
1552
1553     // set the callback
1554     if(callback!=NULL) {
1555         ucnv_setFromUCallBack(cnv, callback, option, NULL, NULL, &errorCode);
1556         if(U_FAILURE(errorCode)) {
1557             errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setFromUCallBack() failed - %s",
1558                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1559             ucnv_close(cnv);
1560             return FALSE;
1561         }
1562     }
1563
1564     // set the fallbacks flag
1565     // TODO change with Jitterbug 2401, then add a similar call for toUnicode too
1566     ucnv_setFallback(cnv, cc.fallbacks);
1567
1568     // set the subchar
1569     int32_t length;
1570
1571     if(cc.setSub>0) {
1572         length=(int32_t)strlen(cc.subchar);
1573         ucnv_setSubstChars(cnv, cc.subchar, (int8_t)length, &errorCode);
1574         if(U_FAILURE(errorCode)) {
1575             errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstChars() failed - %s",
1576                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1577             ucnv_close(cnv);
1578             return FALSE;
1579         }
1580     } else if(cc.setSub<0) {
1581         ucnv_setSubstString(cnv, cc.subString, -1, &errorCode);
1582         if(U_FAILURE(errorCode)) {
1583             errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstString() failed - %s",
1584                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1585             ucnv_close(cnv);
1586             return FALSE;
1587         }
1588     }
1589
1590     // convert unicode to utf8
1591     char utf8[256];
1592     cc.utf8=utf8;
1593     u_strToUTF8(utf8, UPRV_LENGTHOF(utf8), &cc.utf8Length,
1594                 cc.unicode, cc.unicodeLength,
1595                 &errorCode);
1596     if(U_FAILURE(errorCode)) {
1597         // skip UTF-8 testing of a string with an unpaired surrogate,
1598         // or of one that's too long
1599         // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1600         cc.utf8Length=-1;
1601     }
1602
1603     int32_t resultOffsets[256];
1604     char result[256];
1605     int32_t resultLength;
1606     UBool ok;
1607
1608     static const struct {
1609         int32_t step;
1610         const char *name, *utf8Name;
1611     } steps[]={
1612         { 0, "bulk",   "utf8" }, // must be first for offsets to be checked
1613         { 1, "step=1", "utf8 step=1" },
1614         { 3, "step=3", "utf8 step=3" },
1615         { 7, "step=7", "utf8 step=7" }
1616     };
1617     int32_t i, step;
1618
1619     ok=TRUE;
1620     for(i=0; i<UPRV_LENGTHOF(steps) && ok; ++i) {
1621         step=steps[i].step;
1622         memset(resultOffsets, -1, UPRV_LENGTHOF(resultOffsets));
1623         memset(result, -1, UPRV_LENGTHOF(result));
1624         errorCode=U_ZERO_ERROR;
1625         resultLength=stepFromUnicode(cc, cnv,
1626                                 result, UPRV_LENGTHOF(result),
1627                                 step==0 ? resultOffsets : NULL,
1628                                 step, &errorCode);
1629         ok=checkFromUnicode(
1630                 cc, cnv, steps[i].name,
1631                 (uint8_t *)result, resultLength,
1632                 cc.offsets!=NULL ? resultOffsets : NULL,
1633                 errorCode);
1634         if(U_FAILURE(errorCode) || !cc.finalFlush) {
1635             // reset if an error occurred or we did not flush
1636             // otherwise do nothing to make sure that flushing resets
1637             ucnv_resetFromUnicode(cnv);
1638         }
1639         if (resultOffsets[resultLength] != -1) {
1640             errln("fromUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1641                 cc.caseNr, cc.charset, resultLength);
1642         }
1643         if (result[resultLength] != (char)-1) {
1644             errln("fromUnicode[%d](%s) Conversion wrote too much to result at index %d",
1645                 cc.caseNr, cc.charset, resultLength);
1646         }
1647
1648         // bulk test is first, then offsets are not checked any more
1649         cc.offsets=NULL;
1650
1651         // test direct conversion from UTF-8
1652         if(cc.utf8Length>=0) {
1653             errorCode=U_ZERO_ERROR;
1654             resultLength=stepFromUTF8(cc, utf8Cnv, cnv,
1655                                     result, UPRV_LENGTHOF(result),
1656                                     step, &errorCode);
1657             ok=checkFromUnicode(
1658                     cc, cnv, steps[i].utf8Name,
1659                     (uint8_t *)result, resultLength,
1660                     NULL,
1661                     errorCode);
1662             if(U_FAILURE(errorCode) || !cc.finalFlush) {
1663                 // reset if an error occurred or we did not flush
1664                 // otherwise do nothing to make sure that flushing resets
1665                 ucnv_resetToUnicode(utf8Cnv);
1666                 ucnv_resetFromUnicode(cnv);
1667             }
1668         }
1669     }
1670
1671     // not a real loop, just a convenience for breaking out of the block
1672     while(ok && cc.finalFlush) {
1673         // test ucnv_fromUChars()
1674         memset(result, 0, sizeof(result));
1675
1676         errorCode=U_ZERO_ERROR;
1677         resultLength=ucnv_fromUChars(cnv,
1678                         result, UPRV_LENGTHOF(result),
1679                         cc.unicode, cc.unicodeLength,
1680                         &errorCode);
1681         ok=checkFromUnicode(
1682                 cc, cnv, "fromUChars",
1683                 (uint8_t *)result, resultLength,
1684                 NULL,
1685                 errorCode);
1686         if(!ok) {
1687             break;
1688         }
1689
1690         // test preflighting
1691         // keep the correct result for simple checking
1692         errorCode=U_ZERO_ERROR;
1693         resultLength=ucnv_fromUChars(cnv,
1694                         NULL, 0,
1695                         cc.unicode, cc.unicodeLength,
1696                         &errorCode);
1697         if(errorCode==U_STRING_NOT_TERMINATED_WARNING || errorCode==U_BUFFER_OVERFLOW_ERROR) {
1698             errorCode=U_ZERO_ERROR;
1699         }
1700         ok=checkFromUnicode(
1701                 cc, cnv, "preflight fromUChars",
1702                 (uint8_t *)result, resultLength,
1703                 NULL,
1704                 errorCode);
1705         break;
1706     }
1707
1708     ucnv_close(cnv);
1709     return ok;
1710 }
1711
1712 UBool
1713 ConversionTest::checkFromUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1714                                  const uint8_t *result, int32_t resultLength,
1715                                  const int32_t *resultOffsets,
1716                                  UErrorCode resultErrorCode) {
1717     UChar resultInvalidUChars[8];
1718     int8_t resultInvalidLength;
1719     UErrorCode errorCode;
1720
1721     const char *msg;
1722
1723     // reset the message; NULL will mean "ok"
1724     msg=NULL;
1725
1726     errorCode=U_ZERO_ERROR;
1727     resultInvalidLength=UPRV_LENGTHOF(resultInvalidUChars);
1728     ucnv_getInvalidUChars(cnv, resultInvalidUChars, &resultInvalidLength, &errorCode);
1729     if(U_FAILURE(errorCode)) {
1730         errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidUChars() failed - %s",
1731                 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1732         return FALSE;
1733     }
1734
1735     // check everything that might have gone wrong
1736     if(cc.bytesLength!=resultLength) {
1737         msg="wrong result length";
1738     } else if(0!=memcmp(cc.bytes, result, cc.bytesLength)) {
1739         msg="wrong result string";
1740     } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.bytesLength*sizeof(*cc.offsets))) {
1741         msg="wrong offsets";
1742     } else if(cc.outErrorCode!=resultErrorCode) {
1743         msg="wrong error code";
1744     } else if(cc.invalidLength!=resultInvalidLength) {
1745         msg="wrong length of last invalid input";
1746     } else if(0!=u_memcmp(cc.invalidUChars, resultInvalidUChars, cc.invalidLength)) {
1747         msg="wrong last invalid input";
1748     }
1749
1750     if(msg==NULL) {
1751         return TRUE;
1752     } else {
1753         char buffer[2000]; // one buffer for all strings
1754         char *s, *unicodeString, *bytesString, *resultString,
1755             *offsetsString, *resultOffsetsString,
1756             *invalidCharsString, *resultInvalidUCharsString;
1757
1758         unicodeString=s=buffer;
1759         s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString);
1760         s=printBytes(cc.bytes, cc.bytesLength, bytesString=s);
1761         s=printBytes(result, resultLength, resultString=s);
1762         s=printOffsets(cc.offsets, cc.bytesLength, offsetsString=s);
1763         s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1764         s=printUnicode(cc.invalidUChars, cc.invalidLength, invalidCharsString=s);
1765         s=printUnicode(resultInvalidUChars, resultInvalidLength, resultInvalidUCharsString=s);
1766
1767         if((s-buffer)>(int32_t)sizeof(buffer)) {
1768             errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkFromUnicode() test output buffer overflow writing %d chars\n",
1769                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1770             exit(1);
1771         }
1772
1773         errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1774               "  unicode <%s>[%d]\n"
1775               " expected <%s>[%d]\n"
1776               "  result  <%s>[%d]\n"
1777               " offsets         <%s>\n"
1778               "  result offsets <%s>\n"
1779               " error code expected %s got %s\n"
1780               "  invalidChars expected <%s> got <%s>\n",
1781               cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1782               unicodeString, cc.unicodeLength,
1783               bytesString, cc.bytesLength,
1784               resultString, resultLength,
1785               offsetsString,
1786               resultOffsetsString,
1787               u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1788               invalidCharsString, resultInvalidUCharsString);
1789
1790         return FALSE;
1791     }
1792 }
1793
1794 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */