icuSources/test/intltest/convtest.cpp

   1 // © 2016 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3 /*
   4 *******************************************************************************
   5 *
   6 *   Copyright (C) 2003-2014, International Business Machines
   7 *   Corporation and others.  All Rights Reserved.
   8 *
   9 *******************************************************************************
  10 *   file name:  convtest.cpp
  11 *   encoding:   UTF-8
  12 *   tab size:   8 (not used)
  13 *   indentation:4
  14 *
  15 *   created on: 2003jul15
  16 *   created by: Markus W. Scherer
  17 *
  18 *   Test file for data-driven conversion tests.
  19 */
  20
  21 #include "unicode/utypes.h"
  22
  23 #if !UCONFIG_NO_LEGACY_CONVERSION
  24 /*
  25  * Note: Turning off all of convtest.cpp if !UCONFIG_NO_LEGACY_CONVERSION
  26  * is slightly unnecessary - it removes tests for Unicode charsets
  27  * like UTF-8 that should work.
  28  * However, there is no easy way for the test to detect whether a test case
  29  * is for a Unicode charset, so it would be difficult to only exclude those.
  30  * Also, regular testing of ICU is done with all modules on, therefore
  31  * not testing conversion for a custom configuration like this should be ok.
  32  */
  33
  34 #include "unicode/ucnv.h"
  35 #include "unicode/unistr.h"
  36 #include "unicode/parsepos.h"
  37 #include "unicode/uniset.h"
  38 #include "unicode/ustring.h"
  39 #include "unicode/ures.h"
  40 #include "unicode/utf16.h"
  41 #include "convtest.h"
  42 #include "cmemory.h"
  43 #include "unicode/tstdtmod.h"
  44 #include <string.h>
  45 #include <stdlib.h>
  46
  47 enum {
  48     // characters used in test data for callbacks
  49     SUB_CB='?',
  50     SKIP_CB='0',
  51     STOP_CB='.',
  52     ESC_CB='&'
  53 };
  54
  55 ConversionTest::ConversionTest() {
  56     UErrorCode errorCode=U_ZERO_ERROR;
  57     utf8Cnv=ucnv_open("UTF-8", &errorCode);
  58     ucnv_setToUCallBack(utf8Cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
  59     if(U_FAILURE(errorCode)) {
  60         errln("unable to open UTF-8 converter");
  61     }
  62 }
  63
  64 ConversionTest::~ConversionTest() {
  65     ucnv_close(utf8Cnv);
  66 }
  67
  68 void
  69 ConversionTest::runIndexedTest(int32_t index, UBool exec, const char *&name, char * /*par*/) {
  70     if (exec) logln("TestSuite ConversionTest: ");
  71     TESTCASE_AUTO_BEGIN;
  72 #if !UCONFIG_NO_FILE_IO
  73     TESTCASE_AUTO(TestToUnicode);
  74     TESTCASE_AUTO(TestFromUnicode);
  75     TESTCASE_AUTO(TestGetUnicodeSet);
  76 #endif
  77     TESTCASE_AUTO(TestGetUnicodeSet2);
  78     TESTCASE_AUTO(TestDefaultIgnorableCallback);
  79     TESTCASE_AUTO(TestUTF8ToUTF8Overflow);
  80     TESTCASE_AUTO_END;
  81 }
  82
  83 // test data interface ----------------------------------------------------- ***
  84
  85 void
  86 ConversionTest::TestToUnicode() {
  87     ConversionCase cc;
  88     char charset[100], cbopt[4];
  89     const char *option;
  90     UnicodeString s, unicode;
  91     int32_t offsetsLength;
  92     UConverterToUCallback callback;
  93
  94     TestDataModule *dataModule;
  95     TestData *testData;
  96     const DataMap *testCase;
  97     UErrorCode errorCode;
  98     int32_t i;
  99
 100     errorCode=U_ZERO_ERROR;
 101     dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
 102     if(U_SUCCESS(errorCode)) {
 103         testData=dataModule->createTestData("toUnicode", errorCode);
 104         if(U_SUCCESS(errorCode)) {
 105             for(i=0; testData->nextCase(testCase, errorCode); ++i) {
 106                 if(U_FAILURE(errorCode)) {
 107                     errln("error retrieving conversion/toUnicode test case %d - %s",
 108                             i, u_errorName(errorCode));
 109                     errorCode=U_ZERO_ERROR;
 110                     continue;
 111                 }
 112
 113                 cc.caseNr=i;
 114
 115                 s=testCase->getString("charset", errorCode);
 116                 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
 117                 cc.charset=charset;
 118
 119                 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
 120                 unicode=testCase->getString("unicode", errorCode);
 121                 cc.unicode=unicode.getBuffer();
 122                 cc.unicodeLength=unicode.length();
 123
 124                 offsetsLength=0;
 125                 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
 126                 if(offsetsLength==0) {
 127                     cc.offsets=NULL;
 128                 } else if(offsetsLength!=unicode.length()) {
 129                     errln("toUnicode[%d] unicode[%d] and offsets[%d] must have the same length",
 130                             i, unicode.length(), offsetsLength);
 131                     errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 132                 }
 133
 134                 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
 135                 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
 136
 137                 s=testCase->getString("errorCode", errorCode);
 138                 if(s==UNICODE_STRING("invalid", 7)) {
 139                     cc.outErrorCode=U_INVALID_CHAR_FOUND;
 140                 } else if(s==UNICODE_STRING("illegal", 7)) {
 141                     cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
 142                 } else if(s==UNICODE_STRING("truncated", 9)) {
 143                     cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
 144                 } else if(s==UNICODE_STRING("illesc", 6)) {
 145                     cc.outErrorCode=U_ILLEGAL_ESCAPE_SEQUENCE;
 146                 } else if(s==UNICODE_STRING("unsuppesc", 9)) {
 147                     cc.outErrorCode=U_UNSUPPORTED_ESCAPE_SEQUENCE;
 148                 } else {
 149                     cc.outErrorCode=U_ZERO_ERROR;
 150                 }
 151
 152                 s=testCase->getString("callback", errorCode);
 153                 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
 154                 cc.cbopt=cbopt;
 155                 switch(cbopt[0]) {
 156                 case SUB_CB:
 157                     callback=UCNV_TO_U_CALLBACK_SUBSTITUTE;
 158                     break;
 159                 case SKIP_CB:
 160                     callback=UCNV_TO_U_CALLBACK_SKIP;
 161                     break;
 162                 case STOP_CB:
 163                     callback=UCNV_TO_U_CALLBACK_STOP;
 164                     break;
 165                 case ESC_CB:
 166                     callback=UCNV_TO_U_CALLBACK_ESCAPE;
 167                     break;
 168                 default:
 169                     callback=NULL;
 170                     break;
 171                 }
 172                 option=callback==NULL ? cbopt : cbopt+1;
 173                 if(*option==0) {
 174                     option=NULL;
 175                 }
 176
 177                 cc.invalidChars=testCase->getBinary(cc.invalidLength, "invalidChars", errorCode);
 178
 179                 if(U_FAILURE(errorCode)) {
 180                     errln("error parsing conversion/toUnicode test case %d - %s",
 181                             i, u_errorName(errorCode));
 182                     errorCode=U_ZERO_ERROR;
 183                 } else {
 184                     logln("TestToUnicode[%d] %s", i, charset);
 185                     ToUnicodeCase(cc, callback, option);
 186                 }
 187             }
 188             delete testData;
 189         }
 190         delete dataModule;
 191     }
 192     else {
 193         dataerrln("Could not load test conversion data");
 194     }
 195 }
 196
 197 void
 198 ConversionTest::TestFromUnicode() {
 199     ConversionCase cc;
 200     char charset[100], cbopt[4];
 201     const char *option;
 202     UnicodeString s, unicode, invalidUChars;
 203     int32_t offsetsLength, index;
 204     UConverterFromUCallback callback;
 205
 206     TestDataModule *dataModule;
 207     TestData *testData;
 208     const DataMap *testCase;
 209     const UChar *p;
 210     UErrorCode errorCode;
 211     int32_t i, length;
 212
 213     errorCode=U_ZERO_ERROR;
 214     dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
 215     if(U_SUCCESS(errorCode)) {
 216         testData=dataModule->createTestData("fromUnicode", errorCode);
 217         if(U_SUCCESS(errorCode)) {
 218             for(i=0; testData->nextCase(testCase, errorCode); ++i) {
 219                 if(U_FAILURE(errorCode)) {
 220                     errln("error retrieving conversion/fromUnicode test case %d - %s",
 221                             i, u_errorName(errorCode));
 222                     errorCode=U_ZERO_ERROR;
 223                     continue;
 224                 }
 225
 226                 cc.caseNr=i;
 227
 228                 s=testCase->getString("charset", errorCode);
 229                 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
 230                 cc.charset=charset;
 231
 232                 unicode=testCase->getString("unicode", errorCode);
 233                 cc.unicode=unicode.getBuffer();
 234                 cc.unicodeLength=unicode.length();
 235                 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
 236
 237                 offsetsLength=0;
 238                 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
 239                 if(offsetsLength==0) {
 240                     cc.offsets=NULL;
 241                 } else if(offsetsLength!=cc.bytesLength) {
 242                     errln("fromUnicode[%d] bytes[%d] and offsets[%d] must have the same length",
 243                             i, cc.bytesLength, offsetsLength);
 244                     errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 245                 }
 246
 247                 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
 248                 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
 249
 250                 s=testCase->getString("errorCode", errorCode);
 251                 if(s==UNICODE_STRING("invalid", 7)) {
 252                     cc.outErrorCode=U_INVALID_CHAR_FOUND;
 253                 } else if(s==UNICODE_STRING("illegal", 7)) {
 254                     cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
 255                 } else if(s==UNICODE_STRING("truncated", 9)) {
 256                     cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
 257                 } else {
 258                     cc.outErrorCode=U_ZERO_ERROR;
 259                 }
 260
 261                 s=testCase->getString("callback", errorCode);
 262                 cc.setSub=0; // default: no subchar
 263
 264                 if((index=s.indexOf((UChar)0))>0) {
 265                     // read NUL-separated subchar first, if any
 266                     // copy the subchar from Latin-1 characters
 267                     // start after the NUL
 268                     p=s.getTerminatedBuffer();
 269                     length=index+1;
 270                     p+=length;
 271                     length=s.length()-length;
 272                     if(length<=0 || length>=(int32_t)sizeof(cc.subchar)) {
 273                         errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 274                     } else {
 275                         int32_t j;
 276
 277                         for(j=0; j<length; ++j) {
 278                             cc.subchar[j]=(char)p[j];
 279                         }
 280                         // NUL-terminate the subchar
 281                         cc.subchar[j]=0;
 282                         cc.setSub=1;
 283                     }
 284
 285                     // remove the NUL and subchar from s
 286                     s.truncate(index);
 287                 } else if((index=s.indexOf((UChar)0x3d))>0) /* '=' */ {
 288                     // read a substitution string, separated by an equal sign
 289                     p=s.getBuffer()+index+1;
 290                     length=s.length()-(index+1);
 291                     if(length<0 || length>=UPRV_LENGTHOF(cc.subString)) {
 292                         errorCode=U_ILLEGAL_ARGUMENT_ERROR;
 293                     } else {
 294                         u_memcpy(cc.subString, p, length);
 295                         // NUL-terminate the subString
 296                         cc.subString[length]=0;
 297                         cc.setSub=-1;
 298                     }
 299
 300                     // remove the equal sign and subString from s
 301                     s.truncate(index);
 302                 }
 303
 304                 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
 305                 cc.cbopt=cbopt;
 306                 switch(cbopt[0]) {
 307                 case SUB_CB:
 308                     callback=UCNV_FROM_U_CALLBACK_SUBSTITUTE;
 309                     break;
 310                 case SKIP_CB:
 311                     callback=UCNV_FROM_U_CALLBACK_SKIP;
 312                     break;
 313                 case STOP_CB:
 314                     callback=UCNV_FROM_U_CALLBACK_STOP;
 315                     break;
 316                 case ESC_CB:
 317                     callback=UCNV_FROM_U_CALLBACK_ESCAPE;
 318                     break;
 319                 default:
 320                     callback=NULL;
 321                     break;
 322                 }
 323                 option=callback==NULL ? cbopt : cbopt+1;
 324                 if(*option==0) {
 325                     option=NULL;
 326                 }
 327
 328                 invalidUChars=testCase->getString("invalidUChars", errorCode);
 329                 cc.invalidUChars=invalidUChars.getBuffer();
 330                 cc.invalidLength=invalidUChars.length();
 331
 332                 if(U_FAILURE(errorCode)) {
 333                     errln("error parsing conversion/fromUnicode test case %d - %s",
 334                             i, u_errorName(errorCode));
 335                     errorCode=U_ZERO_ERROR;
 336                 } else {
 337                     logln("TestFromUnicode[%d] %s", i, charset);
 338                     FromUnicodeCase(cc, callback, option);
 339                 }
 340             }
 341             delete testData;
 342         }
 343         delete dataModule;
 344     }
 345     else {
 346         dataerrln("Could not load test conversion data");
 347     }
 348 }
 349
 350 static const UChar ellipsis[]={ 0x2e, 0x2e, 0x2e };
 351
 352 void
 353 ConversionTest::TestGetUnicodeSet() {
 354     char charset[100];
 355     UnicodeString s, map, mapnot;
 356     int32_t which;
 357
 358     ParsePosition pos;
 359     UnicodeSet cnvSet, mapSet, mapnotSet, diffSet;
 360     UnicodeSet *cnvSetPtr = &cnvSet;
 361     LocalUConverterPointer cnv;
 362
 363     TestDataModule *dataModule;
 364     TestData *testData;
 365     const DataMap *testCase;
 366     UErrorCode errorCode;
 367     int32_t i;
 368
 369     errorCode=U_ZERO_ERROR;
 370     dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
 371     if(U_SUCCESS(errorCode)) {
 372         testData=dataModule->createTestData("getUnicodeSet", errorCode);
 373         if(U_SUCCESS(errorCode)) {
 374             for(i=0; testData->nextCase(testCase, errorCode); ++i) {
 375                 if(U_FAILURE(errorCode)) {
 376                     errln("error retrieving conversion/getUnicodeSet test case %d - %s",
 377                             i, u_errorName(errorCode));
 378                     errorCode=U_ZERO_ERROR;
 379                     continue;
 380                 }
 381
 382                 s=testCase->getString("charset", errorCode);
 383                 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
 384
 385                 map=testCase->getString("map", errorCode);
 386                 mapnot=testCase->getString("mapnot", errorCode);
 387
 388                 which=testCase->getInt28("which", errorCode);
 389
 390                 if(U_FAILURE(errorCode)) {
 391                     errln("error parsing conversion/getUnicodeSet test case %d - %s",
 392                             i, u_errorName(errorCode));
 393                     errorCode=U_ZERO_ERROR;
 394                     continue;
 395                 }
 396
 397                 // test this test case
 398                 mapSet.clear();
 399                 mapnotSet.clear();
 400
 401                 pos.setIndex(0);
 402                 mapSet.applyPattern(map, pos, 0, NULL, errorCode);
 403                 if(U_FAILURE(errorCode) || pos.getIndex()!=map.length()) {
 404                     errln("error creating the map set for conversion/getUnicodeSet test case %d - %s\n"
 405                           "    error index %d  index %d  U+%04x",
 406                             i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), map.char32At(pos.getIndex()));
 407                     errorCode=U_ZERO_ERROR;
 408                     continue;
 409                 }
 410
 411                 pos.setIndex(0);
 412                 mapnotSet.applyPattern(mapnot, pos, 0, NULL, errorCode);
 413                 if(U_FAILURE(errorCode) || pos.getIndex()!=mapnot.length()) {
 414                     errln("error creating the mapnot set for conversion/getUnicodeSet test case %d - %s\n"
 415                           "    error index %d  index %d  U+%04x",
 416                             i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), mapnot.char32At(pos.getIndex()));
 417                     errorCode=U_ZERO_ERROR;
 418                     continue;
 419                 }
 420
 421                 logln("TestGetUnicodeSet[%d] %s", i, charset);
 422
 423                 cnv.adoptInstead(cnv_open(charset, errorCode));
 424                 if(U_FAILURE(errorCode)) {
 425                     errcheckln(errorCode, "error opening \"%s\" for conversion/getUnicodeSet test case %d - %s",
 426                             charset, i, u_errorName(errorCode));
 427                     errorCode=U_ZERO_ERROR;
 428                     continue;
 429                 }
 430
 431                 ucnv_getUnicodeSet(cnv.getAlias(), cnvSetPtr->toUSet(), (UConverterUnicodeSet)which, &errorCode);
 432
 433                 if(U_FAILURE(errorCode)) {
 434                     errln("error in ucnv_getUnicodeSet(\"%s\") for conversion/getUnicodeSet test case %d - %s",
 435                             charset, i, u_errorName(errorCode));
 436                     errorCode=U_ZERO_ERROR;
 437                     continue;
 438                 }
 439
 440                 // are there items that must be in cnvSet but are not?
 441                 (diffSet=mapSet).removeAll(cnvSet);
 442                 if(!diffSet.isEmpty()) {
 443                     diffSet.toPattern(s, TRUE);
 444                     if(s.length()>100) {
 445                         s.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
 446                     }
 447                     errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - conversion/getUnicodeSet test case %d",
 448                             charset, i);
 449                     errln(s);
 450                 }
 451
 452                 // are there items that must not be in cnvSet but are?
 453                 (diffSet=mapnotSet).retainAll(cnvSet);
 454                 if(!diffSet.isEmpty()) {
 455                     diffSet.toPattern(s, TRUE);
 456                     if(s.length()>100) {
 457                         s.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
 458                     }
 459                     errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - conversion/getUnicodeSet test case %d",
 460                             charset, i);
 461                     errln(s);
 462                 }
 463             }
 464             delete testData;
 465         }
 466         delete dataModule;
 467     }
 468     else {
 469         dataerrln("Could not load test conversion data");
 470     }
 471 }
 472
 473 U_CDECL_BEGIN
 474 static void U_CALLCONV
 475 getUnicodeSetCallback(const void *context,
 476                       UConverterFromUnicodeArgs * /*fromUArgs*/,
 477                       const UChar* /*codeUnits*/,
 478                       int32_t /*length*/,
 479                       UChar32 codePoint,
 480                       UConverterCallbackReason reason,
 481                       UErrorCode *pErrorCode) {
 482     if(reason<=UCNV_IRREGULAR) {
 483         ((UnicodeSet *)context)->remove(codePoint);  // the converter cannot convert this code point
 484         *pErrorCode=U_ZERO_ERROR;                    // skip
 485     }  // else ignore the reset, close and clone calls.
 486 }
 487 U_CDECL_END
 488
 489 // Compare ucnv_getUnicodeSet() with the set of characters that can be converted.
 490 void
 491 ConversionTest::TestGetUnicodeSet2() {
 492     // Build a string with all code points.
 493     UChar32 cpLimit;
 494     int32_t s0Length;
 495     if(quick) {
 496         cpLimit=s0Length=0x10000;  // BMP only
 497     } else {
 498         cpLimit=0x110000;
 499         s0Length=0x10000+0x200000;  // BMP + surrogate pairs
 500     }
 501     UChar *s0=new UChar[s0Length];
 502     if(s0==NULL) {
 503         return;
 504     }
 505     UChar *s=s0;
 506     UChar32 c;
 507     UChar c2;
 508     // low BMP
 509     for(c=0; c<=0xd7ff; ++c) {
 510         *s++=(UChar)c;
 511     }
 512     // trail surrogates
 513     for(c=0xdc00; c<=0xdfff; ++c) {
 514         *s++=(UChar)c;
 515     }
 516     // lead surrogates
 517     // (after trails so that there is not even one surrogate pair in between)
 518     for(c=0xd800; c<=0xdbff; ++c) {
 519         *s++=(UChar)c;
 520     }
 521     // high BMP
 522     for(c=0xe000; c<=0xffff; ++c) {
 523         *s++=(UChar)c;
 524     }
 525     // supplementary code points = surrogate pairs
 526     if(cpLimit==0x110000) {
 527         for(c=0xd800; c<=0xdbff; ++c) {
 528             for(c2=0xdc00; c2<=0xdfff; ++c2) {
 529                 *s++=(UChar)c;
 530                 *s++=c2;
 531             }
 532         }
 533     }
 534
 535     static const char *const cnvNames[]={
 536         "UTF-8",
 537         "UTF-7",
 538         "UTF-16",
 539         "US-ASCII",
 540         "ISO-8859-1",
 541         "windows-1252",
 542         "Shift-JIS",
 543         "ibm-1390",  // EBCDIC_STATEFUL table
 544         "ibm-16684",  // DBCS-only extension table based on EBCDIC_STATEFUL table
 545         "HZ",
 546         "ISO-2022-JP",
 547         "JIS7",
 548         "ISO-2022-CN",
 549         "ISO-2022-CN-EXT",
 550         "LMBCS"
 551     };
 552     LocalUConverterPointer cnv;
 553     char buffer[1024];
 554     int32_t i;
 555     for(i=0; i<UPRV_LENGTHOF(cnvNames); ++i) {
 556         UErrorCode errorCode=U_ZERO_ERROR;
 557         cnv.adoptInstead(cnv_open(cnvNames[i], errorCode));
 558         if(U_FAILURE(errorCode)) {
 559             errcheckln(errorCode, "failed to open converter %s - %s", cnvNames[i], u_errorName(errorCode));
 560             continue;
 561         }
 562         UnicodeSet expected;
 563         ucnv_setFromUCallBack(cnv.getAlias(), getUnicodeSetCallback, &expected, NULL, NULL, &errorCode);
 564         if(U_FAILURE(errorCode)) {
 565             errln("failed to set the callback on converter %s - %s", cnvNames[i], u_errorName(errorCode));
 566             continue;
 567         }
 568         UConverterUnicodeSet which;
 569         for(which=UCNV_ROUNDTRIP_SET; which<UCNV_SET_COUNT; which=(UConverterUnicodeSet)((int)which+1)) {
 570             if(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET) {
 571                 ucnv_setFallback(cnv.getAlias(), TRUE);
 572             }
 573             expected.add(0, cpLimit-1);
 574             s=s0;
 575             UBool flush;
 576             do {
 577                 char *t=buffer;
 578                 flush=(UBool)(s==s0+s0Length);
 579                 ucnv_fromUnicode(cnv.getAlias(), &t, buffer+sizeof(buffer), (const UChar **)&s, s0+s0Length, NULL, flush, &errorCode);
 580                 if(U_FAILURE(errorCode)) {
 581                     if(errorCode==U_BUFFER_OVERFLOW_ERROR) {
 582                         errorCode=U_ZERO_ERROR;
 583                         continue;
 584                     } else {
 585                         break;  // unexpected error, should not occur
 586                     }
 587                 }
 588             } while(!flush);
 589             UnicodeSet set;
 590             ucnv_getUnicodeSet(cnv.getAlias(), set.toUSet(), which, &errorCode);
 591             if(cpLimit<0x110000) {
 592                 set.remove(cpLimit, 0x10ffff);
 593             }
 594             if(which==UCNV_ROUNDTRIP_SET) {
 595                 // ignore PUA code points because they will be converted even if they
 596                 // are fallbacks and when other fallbacks are turned off,
 597                 // but ucnv_getUnicodeSet(UCNV_ROUNDTRIP_SET) delivers true roundtrips
 598                 expected.remove(0xe000, 0xf8ff);
 599                 expected.remove(0xf0000, 0xffffd);
 600                 expected.remove(0x100000, 0x10fffd);
 601                 set.remove(0xe000, 0xf8ff);
 602                 set.remove(0xf0000, 0xffffd);
 603                 set.remove(0x100000, 0x10fffd);
 604             }
 605             if(set!=expected) {
 606                 // First try to see if we have different sets because ucnv_getUnicodeSet()
 607                 // added strings: The above conversion method does not tell us what strings might be convertible.
 608                 // Remove strings from the set and compare again.
 609                 // Unfortunately, there are no good, direct set methods for finding out whether there are strings
 610                 // in the set, nor for enumerating or removing just them.
 611                 // Intersect all code points with the set. The intersection will not contain strings.
 612                 UnicodeSet temp(0, 0x10ffff);
 613                 temp.retainAll(set);
 614                 set=temp;
 615             }
 616             if(set!=expected) {
 617                 UnicodeSet diffSet;
 618                 UnicodeString out;
 619
 620                 // are there items that must be in the set but are not?
 621                 (diffSet=expected).removeAll(set);
 622                 if(!diffSet.isEmpty()) {
 623                     diffSet.toPattern(out, TRUE);
 624                     if(out.length()>100) {
 625                         out.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
 626                     }
 627                     errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - which set: %d",
 628                             cnvNames[i], which);
 629                     errln(out);
 630                 }
 631
 632                 // are there items that must not be in the set but are?
 633                 (diffSet=set).removeAll(expected);
 634                 if(!diffSet.isEmpty()) {
 635                     diffSet.toPattern(out, TRUE);
 636                     if(out.length()>100) {
 637                         out.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
 638                     }
 639                     errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - which set: %d",
 640                             cnvNames[i], which);
 641                     errln(out);
 642                 }
 643             }
 644         }
 645     }
 646
 647     delete [] s0;
 648 }
 649
 650 // Test all codepoints which has the default ignorable Unicode property are ignored if they have no mapping
 651 // If there are any failures, the hard coded list (IS_DEFAULT_IGNORABLE_CODE_POINT) in ucnv_err.c should be updated
 652 void
 653 ConversionTest::TestDefaultIgnorableCallback() {
 654     UErrorCode status = U_ZERO_ERROR;
 655     const char *cnv_name = "euc-jp-2007";
 656     const char *pattern_ignorable = "[:Default_Ignorable_Code_Point:]";
 657     const char *pattern_not_ignorable = "[:^Default_Ignorable_Code_Point:]";
 658
 659     UnicodeSet *set_ignorable = new UnicodeSet(pattern_ignorable, status);
 660     if (U_FAILURE(status)) {
 661         dataerrln("Unable to create Unicodeset: %s - %s\n", pattern_ignorable, u_errorName(status));
 662         return;
 663     }
 664
 665     UnicodeSet *set_not_ignorable = new UnicodeSet(pattern_not_ignorable, status);
 666     if (U_FAILURE(status)) {
 667         dataerrln("Unable to create Unicodeset: %s - %s\n", pattern_not_ignorable, u_errorName(status));
 668         return;
 669     }
 670
 671     UConverter *cnv = cnv_open(cnv_name, status);
 672     if (U_FAILURE(status)) {
 673         dataerrln("Unable to open converter: %s - %s\n", cnv_name, u_errorName(status));
 674         return;
 675     }
 676
 677     // set callback for the converter
 678     ucnv_setFromUCallBack(cnv, UCNV_FROM_U_CALLBACK_SUBSTITUTE, NULL, NULL, NULL, &status);
 679
 680     UChar32 input[1];
 681     char output[10];
 682     int32_t outputLength;
 683
 684     // test default ignorables are ignored
 685     int size = set_ignorable->size();
 686     for (int i = 0; i < size; i++) {
 687         status = U_ZERO_ERROR;
 688         outputLength= 0;
 689
 690         input[0] = set_ignorable->charAt(i);
 691
 692         outputLength = ucnv_fromUChars(cnv, output, 10, UnicodeString::fromUTF32(input, 1).getTerminatedBuffer(), -1, &status);
 693         if (U_FAILURE(status) || outputLength != 0) {
 694             errln("Ignorable code point: U+%04X not skipped as expected - %s", input[0], u_errorName(status));
 695         }
 696     }
 697
 698     // test non-ignorables are not ignored
 699     size = set_not_ignorable->size();
 700     for (int i = 0; i < size; i++) {
 701         status = U_ZERO_ERROR;
 702         outputLength= 0;
 703
 704         input[0] = set_not_ignorable->charAt(i);
 705
 706         if (input[0] == 0) {
 707             continue;
 708         }
 709
 710         outputLength = ucnv_fromUChars(cnv, output, 10, UnicodeString::fromUTF32(input, 1).getTerminatedBuffer(), -1, &status);
 711         if (U_FAILURE(status) || outputLength <= 0) {
 712             errln("Non-ignorable code point: U+%04X skipped unexpectedly - %s", input[0], u_errorName(status));
 713         }
 714     }
 715
 716     ucnv_close(cnv);
 717     delete set_not_ignorable;
 718     delete set_ignorable;
 719 }
 720
 721 void
 722 ConversionTest::TestUTF8ToUTF8Overflow() {
 723     IcuTestErrorCode errorCode(*this, "TestUTF8ToUTF8Overflow");
 724     LocalUConverterPointer cnv1(ucnv_open("UTF-8", errorCode));
 725     LocalUConverterPointer cnv2(ucnv_open("UTF-8", errorCode));
 726     static const char *text = "aä";  // ä: 2 bytes
 727     const char *source = text;
 728     const char *sourceLimit = text + strlen(text);
 729     char result[20];
 730     char *target = result;
 731     const char *targetLimit = result + sizeof(result);
 732     UChar buffer16[20];
 733     UChar *pivotSource = buffer16;
 734     UChar *pivotTarget = buffer16;
 735     const UChar *pivotLimit = buffer16 + UPRV_LENGTHOF(buffer16);
 736     int32_t length;
 737
 738     // Convert with insufficient target capacity.
 739     result[2] = 5;
 740     ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
 741                    &target, result + 2, &source, sourceLimit,
 742                    buffer16, &pivotSource, &pivotTarget, pivotLimit,
 743                    FALSE, FALSE, errorCode);
 744     assertEquals("overflow", U_BUFFER_OVERFLOW_ERROR, errorCode.reset());
 745     length = (int32_t)(target - result);
 746     assertEquals("number of bytes written", 2, length);
 747     assertEquals("next byte not clobbered", 5, result[2]);
 748
 749     // Convert the rest and flush.
 750     ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
 751                    &target, targetLimit, &source, sourceLimit,
 752                    buffer16, &pivotSource, &pivotTarget, pivotLimit,
 753                    FALSE, TRUE, errorCode);
 754
 755     assertSuccess("UTF-8->UTF-8", errorCode);
 756     length = (int32_t)(target - result);
 757     assertEquals("3 bytes", 3, length);
 758     if (length == 3) {
 759         assertTrue("result same as input", memcmp(text, result, length) == 0);
 760     }
 761
 762     ucnv_reset(cnv1.getAlias());
 763     ucnv_reset(cnv2.getAlias());
 764     memset(result, 0, sizeof(result));
 765     static const char *text2 = "a🚲";  // U+1F6B2 bicycle: 4 bytes
 766     source = text2;
 767     sourceLimit = text2 + strlen(text2);
 768     target = result;
 769     pivotSource = pivotTarget = buffer16;
 770
 771     // Convert with insufficient target capacity.
 772     result[3] = 5;
 773     ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
 774                    &target, result + 3, &source, sourceLimit,
 775                    buffer16, &pivotSource, &pivotTarget, pivotLimit,
 776                    FALSE, FALSE, errorCode);
 777     assertEquals("text2 overflow", U_BUFFER_OVERFLOW_ERROR, errorCode.reset());
 778     length = (int32_t)(target - result);
 779     assertEquals("text2 number of bytes written", 3, length);
 780     assertEquals("text2 next byte not clobbered", 5, result[3]);
 781
 782     // Convert the rest and flush.
 783     ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
 784                    &target, targetLimit, &source, sourceLimit,
 785                    buffer16, &pivotSource, &pivotTarget, pivotLimit,
 786                    FALSE, TRUE, errorCode);
 787
 788     assertSuccess("text2 UTF-8->UTF-8", errorCode);
 789     length = (int32_t)(target - result);
 790     assertEquals("text2 5 bytes", 5, length);
 791     if (length == 5) {
 792         assertTrue("text2 result same as input", memcmp(text2, result, length) == 0);
 793     }
 794
 795     ucnv_reset(cnv1.getAlias());
 796     ucnv_reset(cnv2.getAlias());
 797     memset(result, 0, sizeof(result));
 798     static const char *illFormed = "\xf1\x91\x93\x96\x91\x94";  // U+514D6 + two more trail bytes
 799     source = illFormed;
 800     sourceLimit = illFormed + strlen(illFormed);
 801     target = result;
 802     pivotSource = pivotTarget = buffer16;
 803
 804     ucnv_setToUCallBack(cnv1.getAlias(), UCNV_TO_U_CALLBACK_STOP, nullptr, nullptr, nullptr, errorCode);
 805
 806     // Convert only two bytes and flush (but expect failure).
 807     char errorBytes[10];
 808     int8_t errorLength;
 809     result[0] = 5;
 810     ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
 811                    &target, targetLimit, &source, source + 2,
 812                    buffer16, &pivotSource, &pivotTarget, pivotLimit,
 813                    FALSE, TRUE, errorCode);
 814     assertEquals("illFormed truncated", U_TRUNCATED_CHAR_FOUND, errorCode.reset());
 815     length = (int32_t)(target - result);
 816     assertEquals("illFormed number of bytes written", 0, length);
 817     errorLength = UPRV_LENGTHOF(errorBytes);
 818     ucnv_getInvalidChars(cnv1.getAlias(), errorBytes, &errorLength, errorCode);
 819     assertEquals("illFormed truncated errorLength", 2, (int32_t)errorLength);
 820     if (errorLength == 2) {
 821         assertEquals("illFormed truncated errorBytes", 0xf191,
 822                      ((int32_t)(uint8_t)errorBytes[0] << 8) | (uint8_t)errorBytes[1]);
 823     }
 824
 825     // Continue conversion starting with a trail byte.
 826     ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
 827                    &target, targetLimit, &source, sourceLimit,
 828                    buffer16, &pivotSource, &pivotTarget, pivotLimit,
 829                    FALSE, TRUE, errorCode);
 830
 831     assertEquals("illFormed trail byte", U_ILLEGAL_CHAR_FOUND, errorCode.reset());
 832     length = (int32_t)(target - result);
 833     assertEquals("illFormed trail byte number of bytes written", 0, length);
 834     errorLength = UPRV_LENGTHOF(errorBytes);
 835     ucnv_getInvalidChars(cnv1.getAlias(), errorBytes, &errorLength, errorCode);
 836     assertEquals("illFormed trail byte errorLength", 1, (int32_t)errorLength);
 837     if (errorLength == 1) {
 838         assertEquals("illFormed trail byte errorBytes", 0x93, (int32_t)(uint8_t)errorBytes[0]);
 839     }
 840 }
 841
 842 // open testdata or ICU data converter ------------------------------------- ***
 843
 844 UConverter *
 845 ConversionTest::cnv_open(const char *name, UErrorCode &errorCode) {
 846     if(name!=NULL && *name=='+') {
 847         // Converter names that start with '+' are ignored in ICU4J tests.
 848         ++name;
 849     }
 850     if(name!=NULL && *name=='*') {
 851         /* loadTestData(): set the data directory */
 852         return ucnv_openPackage(loadTestData(errorCode), name+1, &errorCode);
 853     } else {
 854         return ucnv_open(name, &errorCode);
 855     }
 856 }
 857
 858 // output helpers ---------------------------------------------------------- ***
 859
 860 static inline char
 861 hexDigit(uint8_t digit) {
 862     return digit<=9 ? (char)('0'+digit) : (char)('a'-10+digit);
 863 }
 864
 865 static char *
 866 printBytes(const uint8_t *bytes, int32_t length, char *out) {
 867     uint8_t b;
 868
 869     if(length>0) {
 870         b=*bytes++;
 871         --length;
 872         *out++=hexDigit((uint8_t)(b>>4));
 873         *out++=hexDigit((uint8_t)(b&0xf));
 874     }
 875
 876     while(length>0) {
 877         b=*bytes++;
 878         --length;
 879         *out++=' ';
 880         *out++=hexDigit((uint8_t)(b>>4));
 881         *out++=hexDigit((uint8_t)(b&0xf));
 882     }
 883     *out++=0;
 884     return out;
 885 }
 886
 887 static char *
 888 printUnicode(const UChar *unicode, int32_t length, char *out) {
 889     UChar32 c;
 890     int32_t i;
 891
 892     for(i=0; i<length;) {
 893         if(i>0) {
 894             *out++=' ';
 895         }
 896         U16_NEXT(unicode, i, length, c);
 897         // write 4..6 digits
 898         if(c>=0x100000) {
 899             *out++='1';
 900         }
 901         if(c>=0x10000) {
 902             *out++=hexDigit((uint8_t)((c>>16)&0xf));
 903         }
 904         *out++=hexDigit((uint8_t)((c>>12)&0xf));
 905         *out++=hexDigit((uint8_t)((c>>8)&0xf));
 906         *out++=hexDigit((uint8_t)((c>>4)&0xf));
 907         *out++=hexDigit((uint8_t)(c&0xf));
 908     }
 909     *out++=0;
 910     return out;
 911 }
 912
 913 static char *
 914 printOffsets(const int32_t *offsets, int32_t length, char *out) {
 915     int32_t i, o, d;
 916
 917     if(offsets==NULL) {
 918         length=0;
 919     }
 920
 921     for(i=0; i<length; ++i) {
 922         if(i>0) {
 923             *out++=' ';
 924         }
 925         o=offsets[i];
 926
 927         // print all offsets with 2 characters each (-x, -9..99, xx)
 928         if(o<-9) {
 929             *out++='-';
 930             *out++='x';
 931         } else if(o<0) {
 932             *out++='-';
 933             *out++=(char)('0'-o);
 934         } else if(o<=99) {
 935             *out++=(d=o/10)==0 ? ' ' : (char)('0'+d);
 936             *out++=(char)('0'+o%10);
 937         } else /* o>99 */ {
 938             *out++='x';
 939             *out++='x';
 940         }
 941     }
 942     *out++=0;
 943     return out;
 944 }
 945
 946 // toUnicode test worker functions ----------------------------------------- ***
 947
 948 static int32_t
 949 stepToUnicode(ConversionCase &cc, UConverter *cnv,
 950               UChar *result, int32_t resultCapacity,
 951               int32_t *resultOffsets, /* also resultCapacity */
 952               int32_t step,
 953               UErrorCode *pErrorCode) {
 954     const char *source, *sourceLimit, *bytesLimit;
 955     UChar *target, *targetLimit, *resultLimit;
 956     UBool flush;
 957
 958     source=(const char *)cc.bytes;
 959     target=result;
 960     bytesLimit=source+cc.bytesLength;
 961     resultLimit=result+resultCapacity;
 962
 963     if(step>=0) {
 964         // call ucnv_toUnicode() with in/out buffers no larger than (step) at a time
 965         // move only one buffer (in vs. out) at a time to be extra mean
 966         // step==0 performs bulk conversion and generates offsets
 967
 968         // initialize the partial limits for the loop
 969         if(step==0) {
 970             // use the entire buffers
 971             sourceLimit=bytesLimit;
 972             targetLimit=resultLimit;
 973             flush=cc.finalFlush;
 974         } else {
 975             // start with empty partial buffers
 976             sourceLimit=source;
 977             targetLimit=target;
 978             flush=FALSE;
 979
 980             // output offsets only for bulk conversion
 981             resultOffsets=NULL;
 982         }
 983
 984         for(;;) {
 985             // resetting the opposite conversion direction must not affect this one
 986             ucnv_resetFromUnicode(cnv);
 987
 988             // convert
 989             ucnv_toUnicode(cnv,
 990                 &target, targetLimit,
 991                 &source, sourceLimit,
 992                 resultOffsets,
 993                 flush, pErrorCode);
 994
 995             // check pointers and errors
 996             if(source>sourceLimit || target>targetLimit) {
 997                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
 998                 break;
 999             } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1000                 if(target!=targetLimit) {
1001                     // buffer overflow must only be set when the target is filled
1002                     *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1003                     break;
1004                 } else if(targetLimit==resultLimit) {
1005                     // not just a partial overflow
1006                     break;
1007                 }
1008
1009                 // the partial target is filled, set a new limit, reset the error and continue
1010                 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1011                 *pErrorCode=U_ZERO_ERROR;
1012             } else if(U_FAILURE(*pErrorCode)) {
1013                 // some other error occurred, done
1014                 break;
1015             } else {
1016                 if(source!=sourceLimit) {
1017                     // when no error occurs, then the input must be consumed
1018                     *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1019                     break;
1020                 }
1021
1022                 if(sourceLimit==bytesLimit) {
1023                     // we are done
1024                     break;
1025                 }
1026
1027                 // the partial conversion succeeded, set a new limit and continue
1028                 sourceLimit=(bytesLimit-source)>=step ? source+step : bytesLimit;
1029                 flush=(UBool)(cc.finalFlush && sourceLimit==bytesLimit);
1030             }
1031         }
1032     } else /* step<0 */ {
1033         /*
1034          * step==-1: call only ucnv_getNextUChar()
1035          * otherwise alternate between ucnv_toUnicode() and ucnv_getNextUChar()
1036          *   if step==-2 or -3, then give ucnv_toUnicode() the whole remaining input,
1037          *   else give it at most (-step-2)/2 bytes
1038          */
1039         UChar32 c;
1040
1041         // end the loop by getting an index out of bounds error
1042         for(;;) {
1043             // resetting the opposite conversion direction must not affect this one
1044             ucnv_resetFromUnicode(cnv);
1045
1046             // convert
1047             if((step&1)!=0 /* odd: -1, -3, -5, ... */) {
1048                 sourceLimit=source; // use sourceLimit not as a real limit
1049                                     // but to remember the pre-getNextUChar source pointer
1050                 c=ucnv_getNextUChar(cnv, &source, bytesLimit, pErrorCode);
1051
1052                 // check pointers and errors
1053                 if(*pErrorCode==U_INDEX_OUTOFBOUNDS_ERROR) {
1054                     if(source!=bytesLimit) {
1055                         *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1056                     } else {
1057                         *pErrorCode=U_ZERO_ERROR;
1058                     }
1059                     break;
1060                 } else if(U_FAILURE(*pErrorCode)) {
1061                     break;
1062                 }
1063                 // source may not move if c is from previous overflow
1064
1065                 if(target==resultLimit) {
1066                     *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
1067                     break;
1068                 }
1069                 if(c<=0xffff) {
1070                     *target++=(UChar)c;
1071                 } else {
1072                     *target++=U16_LEAD(c);
1073                     if(target==resultLimit) {
1074                         *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
1075                         break;
1076                     }
1077                     *target++=U16_TRAIL(c);
1078                 }
1079
1080                 // alternate between -n-1 and -n but leave -1 alone
1081                 if(step<-1) {
1082                     ++step;
1083                 }
1084             } else /* step is even */ {
1085                 // allow only one UChar output
1086                 targetLimit=target<resultLimit ? target+1 : resultLimit;
1087
1088                 // as with ucnv_getNextUChar(), we always flush (if we go to bytesLimit)
1089                 // and never output offsets
1090                 if(step==-2) {
1091                     sourceLimit=bytesLimit;
1092                 } else {
1093                     sourceLimit=source+(-step-2)/2;
1094                     if(sourceLimit>bytesLimit) {
1095                         sourceLimit=bytesLimit;
1096                     }
1097                 }
1098
1099                 ucnv_toUnicode(cnv,
1100                     &target, targetLimit,
1101                     &source, sourceLimit,
1102                     NULL, (UBool)(sourceLimit==bytesLimit), pErrorCode);
1103
1104                 // check pointers and errors
1105                 if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1106                     if(target!=targetLimit) {
1107                         // buffer overflow must only be set when the target is filled
1108                         *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1109                         break;
1110                     } else if(targetLimit==resultLimit) {
1111                         // not just a partial overflow
1112                         break;
1113                     }
1114
1115                     // the partial target is filled, set a new limit and continue
1116                     *pErrorCode=U_ZERO_ERROR;
1117                 } else if(U_FAILURE(*pErrorCode)) {
1118                     // some other error occurred, done
1119                     break;
1120                 } else {
1121                     if(source!=sourceLimit) {
1122                         // when no error occurs, then the input must be consumed
1123                         *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1124                         break;
1125                     }
1126
1127                     // we are done (flush==TRUE) but we continue, to get the index out of bounds error above
1128                 }
1129
1130                 --step;
1131             }
1132         }
1133     }
1134
1135     return (int32_t)(target-result);
1136 }
1137
1138 UBool
1139 ConversionTest::ToUnicodeCase(ConversionCase &cc, UConverterToUCallback callback, const char *option) {
1140     // open the converter
1141     IcuTestErrorCode errorCode(*this, "ToUnicodeCase");
1142     LocalUConverterPointer cnv(cnv_open(cc.charset, errorCode));
1143     // with no data, the above crashes with "pointer being freed was not allocated" for charset "x11-compound-text", see #13078
1144     if(errorCode.isFailure()) {
1145         errcheckln(errorCode, "toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
1146                 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, errorCode.errorName());
1147         errorCode.reset();
1148         return FALSE;
1149     }
1150
1151     // set the callback
1152     if(callback!=NULL) {
1153         ucnv_setToUCallBack(cnv.getAlias(), callback, option, NULL, NULL, errorCode);
1154         if(U_FAILURE(errorCode)) {
1155             errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setToUCallBack() failed - %s",
1156                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1157             return FALSE;
1158         }
1159     }
1160
1161     int32_t resultOffsets[256];
1162     UChar result[256];
1163     int32_t resultLength;
1164     UBool ok;
1165
1166     static const struct {
1167         int32_t step;
1168         const char *name;
1169     } steps[]={
1170         { 0, "bulk" }, // must be first for offsets to be checked
1171         { 1, "step=1" },
1172         { 3, "step=3" },
1173         { 7, "step=7" },
1174         { -1, "getNext" },
1175         { -2, "toU(bulk)+getNext" },
1176         { -3, "getNext+toU(bulk)" },
1177         { -4, "toU(1)+getNext" },
1178         { -5, "getNext+toU(1)" },
1179         { -12, "toU(5)+getNext" },
1180         { -13, "getNext+toU(5)" },
1181     };
1182     int32_t i, step;
1183
1184     ok=TRUE;
1185     for(i=0; i<UPRV_LENGTHOF(steps) && ok; ++i) {
1186         step=steps[i].step;
1187         if(step<0 && !cc.finalFlush) {
1188             // skip ucnv_getNextUChar() if !finalFlush because
1189             // ucnv_getNextUChar() always implies flush
1190             continue;
1191         }
1192         if(step!=0) {
1193             // bulk test is first, then offsets are not checked any more
1194             cc.offsets=NULL;
1195         }
1196         else {
1197             memset(resultOffsets, -1, UPRV_LENGTHOF(resultOffsets));
1198         }
1199         memset(result, -1, UPRV_LENGTHOF(result));
1200         errorCode.reset();
1201         resultLength=stepToUnicode(cc, cnv.getAlias(),
1202                                 result, UPRV_LENGTHOF(result),
1203                                 step==0 ? resultOffsets : NULL,
1204                                 step, errorCode);
1205         ok=checkToUnicode(
1206                 cc, cnv.getAlias(), steps[i].name,
1207                 result, resultLength,
1208                 cc.offsets!=NULL ? resultOffsets : NULL,
1209                 errorCode);
1210         if(errorCode.isFailure() || !cc.finalFlush) {
1211             // reset if an error occurred or we did not flush
1212             // otherwise do nothing to make sure that flushing resets
1213             ucnv_resetToUnicode(cnv.getAlias());
1214         }
1215         if (cc.offsets != NULL && resultOffsets[resultLength] != -1) {
1216             errln("toUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1217                 cc.caseNr, cc.charset, resultLength);
1218         }
1219         if (result[resultLength] != (UChar)-1) {
1220             errln("toUnicode[%d](%s) Conversion wrote too much to result at index %d",
1221                 cc.caseNr, cc.charset, resultLength);
1222         }
1223     }
1224
1225     // not a real loop, just a convenience for breaking out of the block
1226     while(ok && cc.finalFlush) {
1227         // test ucnv_toUChars()
1228         memset(result, 0, sizeof(result));
1229
1230         errorCode.reset();
1231         resultLength=ucnv_toUChars(cnv.getAlias(),
1232                         result, UPRV_LENGTHOF(result),
1233                         (const char *)cc.bytes, cc.bytesLength,
1234                         errorCode);
1235         ok=checkToUnicode(
1236                 cc, cnv.getAlias(), "toUChars",
1237                 result, resultLength,
1238                 NULL,
1239                 errorCode);
1240         if(!ok) {
1241             break;
1242         }
1243
1244         // test preflighting
1245         // keep the correct result for simple checking
1246         errorCode.reset();
1247         resultLength=ucnv_toUChars(cnv.getAlias(),
1248                         NULL, 0,
1249                         (const char *)cc.bytes, cc.bytesLength,
1250                         errorCode);
1251         if(errorCode.get()==U_STRING_NOT_TERMINATED_WARNING || errorCode.get()==U_BUFFER_OVERFLOW_ERROR) {
1252             errorCode.reset();
1253         }
1254         ok=checkToUnicode(
1255                 cc, cnv.getAlias(), "preflight toUChars",
1256                 result, resultLength,
1257                 NULL,
1258                 errorCode);
1259         break;
1260     }
1261
1262     errorCode.reset();  // all errors have already been reported
1263     return ok;
1264 }
1265
1266 UBool
1267 ConversionTest::checkToUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1268                                const UChar *result, int32_t resultLength,
1269                                const int32_t *resultOffsets,
1270                                UErrorCode resultErrorCode) {
1271     char resultInvalidChars[8];
1272     int8_t resultInvalidLength;
1273     UErrorCode errorCode;
1274
1275     const char *msg;
1276
1277     // reset the message; NULL will mean "ok"
1278     msg=NULL;
1279
1280     errorCode=U_ZERO_ERROR;
1281     resultInvalidLength=sizeof(resultInvalidChars);
1282     ucnv_getInvalidChars(cnv, resultInvalidChars, &resultInvalidLength, &errorCode);
1283     if(U_FAILURE(errorCode)) {
1284         errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidChars() failed - %s",
1285                 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1286         return FALSE;
1287     }
1288
1289     // check everything that might have gone wrong
1290     if(cc.unicodeLength!=resultLength) {
1291         msg="wrong result length";
1292     } else if(0!=u_memcmp(cc.unicode, result, cc.unicodeLength)) {
1293         msg="wrong result string";
1294     } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.unicodeLength*sizeof(*cc.offsets))) {
1295         msg="wrong offsets";
1296     } else if(cc.outErrorCode!=resultErrorCode) {
1297         msg="wrong error code";
1298     } else if(cc.invalidLength!=resultInvalidLength) {
1299         msg="wrong length of last invalid input";
1300     } else if(0!=memcmp(cc.invalidChars, resultInvalidChars, cc.invalidLength)) {
1301         msg="wrong last invalid input";
1302     }
1303
1304     if(msg==NULL) {
1305         return TRUE;
1306     } else {
1307         char buffer[2000]; // one buffer for all strings
1308         char *s, *bytesString, *unicodeString, *resultString,
1309             *offsetsString, *resultOffsetsString,
1310             *invalidCharsString, *resultInvalidCharsString;
1311
1312         bytesString=s=buffer;
1313         s=printBytes(cc.bytes, cc.bytesLength, bytesString);
1314         s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString=s);
1315         s=printUnicode(result, resultLength, resultString=s);
1316         s=printOffsets(cc.offsets, cc.unicodeLength, offsetsString=s);
1317         s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1318         s=printBytes(cc.invalidChars, cc.invalidLength, invalidCharsString=s);
1319         s=printBytes((uint8_t *)resultInvalidChars, resultInvalidLength, resultInvalidCharsString=s);
1320
1321         if((s-buffer)>(int32_t)sizeof(buffer)) {
1322             errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkToUnicode() test output buffer overflow writing %d chars\n",
1323                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1324             exit(1);
1325         }
1326
1327         errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1328               "  bytes <%s>[%d]\n"
1329               " expected <%s>[%d]\n"
1330               "  result  <%s>[%d]\n"
1331               " offsets         <%s>\n"
1332               "  result offsets <%s>\n"
1333               " error code expected %s got %s\n"
1334               "  invalidChars expected <%s> got <%s>\n",
1335               cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1336               bytesString, cc.bytesLength,
1337               unicodeString, cc.unicodeLength,
1338               resultString, resultLength,
1339               offsetsString,
1340               resultOffsetsString,
1341               u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1342               invalidCharsString, resultInvalidCharsString);
1343
1344         return FALSE;
1345     }
1346 }
1347
1348 // fromUnicode test worker functions --------------------------------------- ***
1349
1350 static int32_t
1351 stepFromUTF8(ConversionCase &cc,
1352              UConverter *utf8Cnv, UConverter *cnv,
1353              char *result, int32_t resultCapacity,
1354              int32_t step,
1355              UErrorCode *pErrorCode) {
1356     const char *source, *sourceLimit, *utf8Limit;
1357     UChar pivotBuffer[32];
1358     UChar *pivotSource, *pivotTarget, *pivotLimit;
1359     char *target, *targetLimit, *resultLimit;
1360     UBool flush;
1361
1362     source=cc.utf8;
1363     pivotSource=pivotTarget=pivotBuffer;
1364     target=result;
1365     utf8Limit=source+cc.utf8Length;
1366     resultLimit=result+resultCapacity;
1367
1368     // call ucnv_convertEx() with in/out buffers no larger than (step) at a time
1369     // move only one buffer (in vs. out) at a time to be extra mean
1370     // step==0 performs bulk conversion
1371
1372     // initialize the partial limits for the loop
1373     if(step==0) {
1374         // use the entire buffers
1375         sourceLimit=utf8Limit;
1376         targetLimit=resultLimit;
1377         flush=cc.finalFlush;
1378
1379         pivotLimit=pivotBuffer+UPRV_LENGTHOF(pivotBuffer);
1380     } else {
1381         // start with empty partial buffers
1382         sourceLimit=source;
1383         targetLimit=target;
1384         flush=FALSE;
1385
1386         // empty pivot is not allowed, make it of length step
1387         pivotLimit=pivotBuffer+step;
1388     }
1389
1390     for(;;) {
1391         // resetting the opposite conversion direction must not affect this one
1392         ucnv_resetFromUnicode(utf8Cnv);
1393         ucnv_resetToUnicode(cnv);
1394
1395         // convert
1396         ucnv_convertEx(cnv, utf8Cnv,
1397             &target, targetLimit,
1398             &source, sourceLimit,
1399             pivotBuffer, &pivotSource, &pivotTarget, pivotLimit,
1400             FALSE, flush, pErrorCode);
1401
1402         // check pointers and errors
1403         if(source>sourceLimit || target>targetLimit) {
1404             *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1405             break;
1406         } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1407             if(target!=targetLimit) {
1408                 // buffer overflow must only be set when the target is filled
1409                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1410                 break;
1411             } else if(targetLimit==resultLimit) {
1412                 // not just a partial overflow
1413                 break;
1414             }
1415
1416             // the partial target is filled, set a new limit, reset the error and continue
1417             targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1418             *pErrorCode=U_ZERO_ERROR;
1419         } else if(U_FAILURE(*pErrorCode)) {
1420             if(pivotSource==pivotBuffer) {
1421                 // toUnicode error, should not occur
1422                 // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1423                 break;
1424             } else {
1425                 // fromUnicode error
1426                 // some other error occurred, done
1427                 break;
1428             }
1429         } else {
1430             if(source!=sourceLimit) {
1431                 // when no error occurs, then the input must be consumed
1432                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1433                 break;
1434             }
1435
1436             if(sourceLimit==utf8Limit) {
1437                 // we are done
1438                 if(*pErrorCode==U_STRING_NOT_TERMINATED_WARNING) {
1439                     // ucnv_convertEx() warns about not terminating the output
1440                     // but ucnv_fromUnicode() does not and so
1441                     // checkFromUnicode() does not expect it
1442                     *pErrorCode=U_ZERO_ERROR;
1443                 }
1444                 break;
1445             }
1446
1447             // the partial conversion succeeded, set a new limit and continue
1448             sourceLimit=(utf8Limit-source)>=step ? source+step : utf8Limit;
1449             flush=(UBool)(cc.finalFlush && sourceLimit==utf8Limit);
1450         }
1451     }
1452
1453     return (int32_t)(target-result);
1454 }
1455
1456 static int32_t
1457 stepFromUnicode(ConversionCase &cc, UConverter *cnv,
1458                 char *result, int32_t resultCapacity,
1459                 int32_t *resultOffsets, /* also resultCapacity */
1460                 int32_t step,
1461                 UErrorCode *pErrorCode) {
1462     const UChar *source, *sourceLimit, *unicodeLimit;
1463     char *target, *targetLimit, *resultLimit;
1464     UBool flush;
1465
1466     source=cc.unicode;
1467     target=result;
1468     unicodeLimit=source+cc.unicodeLength;
1469     resultLimit=result+resultCapacity;
1470
1471     // call ucnv_fromUnicode() with in/out buffers no larger than (step) at a time
1472     // move only one buffer (in vs. out) at a time to be extra mean
1473     // step==0 performs bulk conversion and generates offsets
1474
1475     // initialize the partial limits for the loop
1476     if(step==0) {
1477         // use the entire buffers
1478         sourceLimit=unicodeLimit;
1479         targetLimit=resultLimit;
1480         flush=cc.finalFlush;
1481     } else {
1482         // start with empty partial buffers
1483         sourceLimit=source;
1484         targetLimit=target;
1485         flush=FALSE;
1486
1487         // output offsets only for bulk conversion
1488         resultOffsets=NULL;
1489     }
1490
1491     for(;;) {
1492         // resetting the opposite conversion direction must not affect this one
1493         ucnv_resetToUnicode(cnv);
1494
1495         // convert
1496         ucnv_fromUnicode(cnv,
1497             &target, targetLimit,
1498             &source, sourceLimit,
1499             resultOffsets,
1500             flush, pErrorCode);
1501
1502         // check pointers and errors
1503         if(source>sourceLimit || target>targetLimit) {
1504             *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1505             break;
1506         } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1507             if(target!=targetLimit) {
1508                 // buffer overflow must only be set when the target is filled
1509                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1510                 break;
1511             } else if(targetLimit==resultLimit) {
1512                 // not just a partial overflow
1513                 break;
1514             }
1515
1516             // the partial target is filled, set a new limit, reset the error and continue
1517             targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1518             *pErrorCode=U_ZERO_ERROR;
1519         } else if(U_FAILURE(*pErrorCode)) {
1520             // some other error occurred, done
1521             break;
1522         } else {
1523             if(source!=sourceLimit) {
1524                 // when no error occurs, then the input must be consumed
1525                 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1526                 break;
1527             }
1528
1529             if(sourceLimit==unicodeLimit) {
1530                 // we are done
1531                 break;
1532             }
1533
1534             // the partial conversion succeeded, set a new limit and continue
1535             sourceLimit=(unicodeLimit-source)>=step ? source+step : unicodeLimit;
1536             flush=(UBool)(cc.finalFlush && sourceLimit==unicodeLimit);
1537         }
1538     }
1539
1540     return (int32_t)(target-result);
1541 }
1542
1543 UBool
1544 ConversionTest::FromUnicodeCase(ConversionCase &cc, UConverterFromUCallback callback, const char *option) {
1545     UConverter *cnv;
1546     UErrorCode errorCode;
1547
1548     // open the converter
1549     errorCode=U_ZERO_ERROR;
1550     cnv=cnv_open(cc.charset, errorCode);
1551     if(U_FAILURE(errorCode)) {
1552         errcheckln(errorCode, "fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
1553                 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1554         return FALSE;
1555     }
1556     ucnv_resetToUnicode(utf8Cnv);
1557
1558     // set the callback
1559     if(callback!=NULL) {
1560         ucnv_setFromUCallBack(cnv, callback, option, NULL, NULL, &errorCode);
1561         if(U_FAILURE(errorCode)) {
1562             errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setFromUCallBack() failed - %s",
1563                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1564             ucnv_close(cnv);
1565             return FALSE;
1566         }
1567     }
1568
1569     // set the fallbacks flag
1570     // TODO change with Jitterbug 2401, then add a similar call for toUnicode too
1571     ucnv_setFallback(cnv, cc.fallbacks);
1572
1573     // set the subchar
1574     int32_t length;
1575
1576     if(cc.setSub>0) {
1577         length=(int32_t)strlen(cc.subchar);
1578         ucnv_setSubstChars(cnv, cc.subchar, (int8_t)length, &errorCode);
1579         if(U_FAILURE(errorCode)) {
1580             errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstChars() failed - %s",
1581                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1582             ucnv_close(cnv);
1583             return FALSE;
1584         }
1585     } else if(cc.setSub<0) {
1586         ucnv_setSubstString(cnv, cc.subString, -1, &errorCode);
1587         if(U_FAILURE(errorCode)) {
1588             errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstString() failed - %s",
1589                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1590             ucnv_close(cnv);
1591             return FALSE;
1592         }
1593     }
1594
1595     // convert unicode to utf8
1596     char utf8[256];
1597     cc.utf8=utf8;
1598     u_strToUTF8(utf8, UPRV_LENGTHOF(utf8), &cc.utf8Length,
1599                 cc.unicode, cc.unicodeLength,
1600                 &errorCode);
1601     if(U_FAILURE(errorCode)) {
1602         // skip UTF-8 testing of a string with an unpaired surrogate,
1603         // or of one that's too long
1604         // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1605         cc.utf8Length=-1;
1606     }
1607
1608     int32_t resultOffsets[256];
1609     char result[256];
1610     int32_t resultLength;
1611     UBool ok;
1612
1613     static const struct {
1614         int32_t step;
1615         const char *name, *utf8Name;
1616     } steps[]={
1617         { 0, "bulk",   "utf8" }, // must be first for offsets to be checked
1618         { 1, "step=1", "utf8 step=1" },
1619         { 3, "step=3", "utf8 step=3" },
1620         { 7, "step=7", "utf8 step=7" }
1621     };
1622     int32_t i, step;
1623
1624     ok=TRUE;
1625     for(i=0; i<UPRV_LENGTHOF(steps) && ok; ++i) {
1626         step=steps[i].step;
1627         memset(resultOffsets, -1, UPRV_LENGTHOF(resultOffsets));
1628         memset(result, -1, UPRV_LENGTHOF(result));
1629         errorCode=U_ZERO_ERROR;
1630         resultLength=stepFromUnicode(cc, cnv,
1631                                 result, UPRV_LENGTHOF(result),
1632                                 step==0 ? resultOffsets : NULL,
1633                                 step, &errorCode);
1634         ok=checkFromUnicode(
1635                 cc, cnv, steps[i].name,
1636                 (uint8_t *)result, resultLength,
1637                 cc.offsets!=NULL ? resultOffsets : NULL,
1638                 errorCode);
1639         if(U_FAILURE(errorCode) || !cc.finalFlush) {
1640             // reset if an error occurred or we did not flush
1641             // otherwise do nothing to make sure that flushing resets
1642             ucnv_resetFromUnicode(cnv);
1643         }
1644         if (resultOffsets[resultLength] != -1) {
1645             errln("fromUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1646                 cc.caseNr, cc.charset, resultLength);
1647         }
1648         if (result[resultLength] != (char)-1) {
1649             errln("fromUnicode[%d](%s) Conversion wrote too much to result at index %d",
1650                 cc.caseNr, cc.charset, resultLength);
1651         }
1652
1653         // bulk test is first, then offsets are not checked any more
1654         cc.offsets=NULL;
1655
1656         // test direct conversion from UTF-8
1657         if(cc.utf8Length>=0) {
1658             errorCode=U_ZERO_ERROR;
1659             resultLength=stepFromUTF8(cc, utf8Cnv, cnv,
1660                                     result, UPRV_LENGTHOF(result),
1661                                     step, &errorCode);
1662             ok=checkFromUnicode(
1663                     cc, cnv, steps[i].utf8Name,
1664                     (uint8_t *)result, resultLength,
1665                     NULL,
1666                     errorCode);
1667             if(U_FAILURE(errorCode) || !cc.finalFlush) {
1668                 // reset if an error occurred or we did not flush
1669                 // otherwise do nothing to make sure that flushing resets
1670                 ucnv_resetToUnicode(utf8Cnv);
1671                 ucnv_resetFromUnicode(cnv);
1672             }
1673         }
1674     }
1675
1676     // not a real loop, just a convenience for breaking out of the block
1677     while(ok && cc.finalFlush) {
1678         // test ucnv_fromUChars()
1679         memset(result, 0, sizeof(result));
1680
1681         errorCode=U_ZERO_ERROR;
1682         resultLength=ucnv_fromUChars(cnv,
1683                         result, UPRV_LENGTHOF(result),
1684                         cc.unicode, cc.unicodeLength,
1685                         &errorCode);
1686         ok=checkFromUnicode(
1687                 cc, cnv, "fromUChars",
1688                 (uint8_t *)result, resultLength,
1689                 NULL,
1690                 errorCode);
1691         if(!ok) {
1692             break;
1693         }
1694
1695         // test preflighting
1696         // keep the correct result for simple checking
1697         errorCode=U_ZERO_ERROR;
1698         resultLength=ucnv_fromUChars(cnv,
1699                         NULL, 0,
1700                         cc.unicode, cc.unicodeLength,
1701                         &errorCode);
1702         if(errorCode==U_STRING_NOT_TERMINATED_WARNING || errorCode==U_BUFFER_OVERFLOW_ERROR) {
1703             errorCode=U_ZERO_ERROR;
1704         }
1705         ok=checkFromUnicode(
1706                 cc, cnv, "preflight fromUChars",
1707                 (uint8_t *)result, resultLength,
1708                 NULL,
1709                 errorCode);
1710         break;
1711     }
1712
1713     ucnv_close(cnv);
1714     return ok;
1715 }
1716
1717 UBool
1718 ConversionTest::checkFromUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1719                                  const uint8_t *result, int32_t resultLength,
1720                                  const int32_t *resultOffsets,
1721                                  UErrorCode resultErrorCode) {
1722     UChar resultInvalidUChars[8];
1723     int8_t resultInvalidLength;
1724     UErrorCode errorCode;
1725
1726     const char *msg;
1727
1728     // reset the message; NULL will mean "ok"
1729     msg=NULL;
1730
1731     errorCode=U_ZERO_ERROR;
1732     resultInvalidLength=UPRV_LENGTHOF(resultInvalidUChars);
1733     ucnv_getInvalidUChars(cnv, resultInvalidUChars, &resultInvalidLength, &errorCode);
1734     if(U_FAILURE(errorCode)) {
1735         errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidUChars() failed - %s",
1736                 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1737         return FALSE;
1738     }
1739
1740     // check everything that might have gone wrong
1741     if(cc.bytesLength!=resultLength) {
1742         msg="wrong result length";
1743     } else if(0!=memcmp(cc.bytes, result, cc.bytesLength)) {
1744         msg="wrong result string";
1745     } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.bytesLength*sizeof(*cc.offsets))) {
1746         msg="wrong offsets";
1747     } else if(cc.outErrorCode!=resultErrorCode) {
1748         msg="wrong error code";
1749     } else if(cc.invalidLength!=resultInvalidLength) {
1750         msg="wrong length of last invalid input";
1751     } else if(0!=u_memcmp(cc.invalidUChars, resultInvalidUChars, cc.invalidLength)) {
1752         msg="wrong last invalid input";
1753     }
1754
1755     if(msg==NULL) {
1756         return TRUE;
1757     } else {
1758         char buffer[2000]; // one buffer for all strings
1759         char *s, *unicodeString, *bytesString, *resultString,
1760             *offsetsString, *resultOffsetsString,
1761             *invalidCharsString, *resultInvalidUCharsString;
1762
1763         unicodeString=s=buffer;
1764         s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString);
1765         s=printBytes(cc.bytes, cc.bytesLength, bytesString=s);
1766         s=printBytes(result, resultLength, resultString=s);
1767         s=printOffsets(cc.offsets, cc.bytesLength, offsetsString=s);
1768         s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1769         s=printUnicode(cc.invalidUChars, cc.invalidLength, invalidCharsString=s);
1770         s=printUnicode(resultInvalidUChars, resultInvalidLength, resultInvalidUCharsString=s);
1771
1772         if((s-buffer)>(int32_t)sizeof(buffer)) {
1773             errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkFromUnicode() test output buffer overflow writing %d chars\n",
1774                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1775             exit(1);
1776         }
1777
1778         errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1779               "  unicode <%s>[%d]\n"
1780               " expected <%s>[%d]\n"
1781               "  result  <%s>[%d]\n"
1782               " offsets         <%s>\n"
1783               "  result offsets <%s>\n"
1784               " error code expected %s got %s\n"
1785               "  invalidChars expected <%s> got <%s>\n",
1786               cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1787               unicodeString, cc.unicodeLength,
1788               bytesString, cc.bytesLength,
1789               resultString, resultLength,
1790               offsetsString,
1791               resultOffsetsString,
1792               u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1793               invalidCharsString, resultInvalidUCharsString);
1794
1795         return FALSE;
1796     }
1797 }
1798
1799 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */