icuSources/test/intltest/uts46test.cpp

   1 // © 2016 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3 /*
   4 *******************************************************************************
   5 *   Copyright (C) 2010-2014, International Business Machines
   6 *   Corporation and others.  All Rights Reserved.
   7 *******************************************************************************
   8 *   file name:  uts46test.cpp
   9 *   encoding:   UTF-8
  10 *   tab size:   8 (not used)
  11 *   indentation:4
  12 *
  13 *   created on: 2010may05
  14 *   created by: Markus W. Scherer
  15 */
  16
  17 #include "unicode/utypes.h"
  18
  19 #if !UCONFIG_NO_IDNA
  20
  21 #include <string.h>
  22 #include "unicode/bytestream.h"
  23 #include "unicode/idna.h"
  24 #include "unicode/localpointer.h"
  25 #include "unicode/std_string.h"
  26 #include "unicode/stringpiece.h"
  27 #include "unicode/uidna.h"
  28 #include "unicode/unistr.h"
  29 #include "charstr.h"
  30 #include "cmemory.h"
  31 #include "intltest.h"
  32 #include "uparse.h"
  33
  34 class UTS46Test : public IntlTest {
  35 public:
  36     UTS46Test() : trans(NULL), nontrans(NULL) {}
  37     virtual ~UTS46Test();
  38
  39     void runIndexedTest(int32_t index, UBool exec, const char *&name, char *par=NULL);
  40     void TestAPI();
  41     void TestNotSTD3();
  42     void TestSomeCases();
  43     void IdnaTest();
  44
  45     void checkIdnaTestResult(const char *line, const char *type,
  46                              const UnicodeString &expected, const UnicodeString &result,
  47                              const char *status, const IDNAInfo &info);
  48     void idnaTestOneLine(char *fields[][2], UErrorCode &errorCode);
  49
  50 private:
  51     IDNA *trans, *nontrans;
  52 };
  53
  54 extern IntlTest *createUTS46Test() {
  55     return new UTS46Test();
  56 }
  57
  58 UTS46Test::~UTS46Test() {
  59     delete trans;
  60     delete nontrans;
  61 }
  62
  63 void UTS46Test::runIndexedTest(int32_t index, UBool exec, const char *&name, char * /*par*/) {
  64     if(exec) {
  65         logln("TestSuite UTS46Test: ");
  66         if(trans==NULL) {
  67             IcuTestErrorCode errorCode(*this, "init/createUTS46Instance()");
  68             uint32_t commonOptions=
  69                 UIDNA_USE_STD3_RULES|UIDNA_CHECK_BIDI|
  70                 UIDNA_CHECK_CONTEXTJ|UIDNA_CHECK_CONTEXTO;
  71             trans=IDNA::createUTS46Instance(commonOptions, errorCode);
  72             nontrans=IDNA::createUTS46Instance(
  73                 commonOptions|
  74                 UIDNA_NONTRANSITIONAL_TO_ASCII|UIDNA_NONTRANSITIONAL_TO_UNICODE,
  75                 errorCode);
  76             if(errorCode.errDataIfFailureAndReset("createUTS46Instance()")) {
  77                 name="";
  78                 return;
  79             }
  80         }
  81     }
  82     TESTCASE_AUTO_BEGIN;
  83     TESTCASE_AUTO(TestAPI);
  84     TESTCASE_AUTO(TestNotSTD3);
  85     TESTCASE_AUTO(TestSomeCases);
  86     TESTCASE_AUTO(IdnaTest);
  87     TESTCASE_AUTO_END;
  88 }
  89
  90 const uint32_t severeErrors=
  91     UIDNA_ERROR_LEADING_COMBINING_MARK|
  92     UIDNA_ERROR_DISALLOWED|
  93     UIDNA_ERROR_PUNYCODE|
  94     UIDNA_ERROR_LABEL_HAS_DOT|
  95     UIDNA_ERROR_INVALID_ACE_LABEL;
  96
  97 static UBool isASCII(const UnicodeString &str) {
  98     const UChar *s=str.getBuffer();
  99     int32_t length=str.length();
 100     for(int32_t i=0; i<length; ++i) {
 101         if(s[i]>=0x80) {
 102             return FALSE;
 103         }
 104     }
 105     return TRUE;
 106 }
 107
 108 class TestCheckedArrayByteSink : public CheckedArrayByteSink {
 109 public:
 110     TestCheckedArrayByteSink(char* outbuf, int32_t capacity)
 111             : CheckedArrayByteSink(outbuf, capacity), calledFlush(FALSE) {}
 112     virtual CheckedArrayByteSink& Reset() {
 113         CheckedArrayByteSink::Reset();
 114         calledFlush = FALSE;
 115         return *this;
 116     }
 117     virtual void Flush() { calledFlush = TRUE; }
 118     UBool calledFlush;
 119 };
 120
 121 void UTS46Test::TestAPI() {
 122     UErrorCode errorCode=U_ZERO_ERROR;
 123     UnicodeString result;
 124     IDNAInfo info;
 125     UnicodeString input=UNICODE_STRING_SIMPLE("www.eXample.cOm");
 126     UnicodeString expected=UNICODE_STRING_SIMPLE("www.example.com");
 127     trans->nameToASCII(input, result, info, errorCode);
 128     if(U_FAILURE(errorCode) || info.hasErrors() || result!=expected) {
 129         errln("T.nameToASCII(www.example.com) info.errors=%04lx result matches=%d %s",
 130               (long)info.getErrors(), result==expected, u_errorName(errorCode));
 131     }
 132     errorCode=U_USELESS_COLLATOR_ERROR;
 133     trans->nameToUnicode(input, result, info, errorCode);
 134     if(errorCode!=U_USELESS_COLLATOR_ERROR || !result.isBogus()) {
 135         errln("T.nameToUnicode(U_FAILURE) did not preserve the errorCode "
 136               "or not result.setToBogus() - %s",
 137               u_errorName(errorCode));
 138     }
 139     errorCode=U_ZERO_ERROR;
 140     input.setToBogus();
 141     result=UNICODE_STRING_SIMPLE("quatsch");
 142     nontrans->labelToASCII(input, result, info, errorCode);
 143     if(errorCode!=U_ILLEGAL_ARGUMENT_ERROR || !result.isBogus()) {
 144         errln("N.labelToASCII(bogus) did not set illegal-argument-error "
 145               "or not result.setToBogus() - %s",
 146               u_errorName(errorCode));
 147     }
 148     errorCode=U_ZERO_ERROR;
 149     input=UNICODE_STRING_SIMPLE("xn--bcher.de-65a");
 150     expected=UNICODE_STRING_SIMPLE("xn--bcher\\uFFFDde-65a").unescape();
 151     nontrans->labelToASCII(input, result, info, errorCode);
 152     if( U_FAILURE(errorCode) ||
 153         info.getErrors()!=(UIDNA_ERROR_LABEL_HAS_DOT|UIDNA_ERROR_INVALID_ACE_LABEL) ||
 154         result!=expected
 155     ) {
 156         errln("N.labelToASCII(label-with-dot) failed with errors %04lx - %s",
 157               info.getErrors(), u_errorName(errorCode));
 158     }
 159     // UTF-8
 160     char buffer[100];
 161     TestCheckedArrayByteSink sink(buffer, UPRV_LENGTHOF(buffer));
 162     errorCode=U_ZERO_ERROR;
 163     nontrans->labelToUnicodeUTF8(StringPiece(NULL, 5), sink, info, errorCode);
 164     if(errorCode!=U_ILLEGAL_ARGUMENT_ERROR || sink.NumberOfBytesWritten()!=0) {
 165         errln("N.labelToUnicodeUTF8(StringPiece(NULL, 5)) did not set illegal-argument-error ",
 166               "or did output something - %s",
 167               u_errorName(errorCode));
 168     }
 169
 170     sink.Reset();
 171     errorCode=U_ZERO_ERROR;
 172     nontrans->nameToASCII_UTF8(StringPiece(), sink, info, errorCode);
 173     if(U_FAILURE(errorCode) || sink.NumberOfBytesWritten()!=0 || !sink.calledFlush) {
 174         errln("N.nameToASCII_UTF8(empty) failed - %s",
 175               u_errorName(errorCode));
 176     }
 177
 178     static const char s[]={ 0x61, (char)0xc3, (char)0x9f };
 179     sink.Reset();
 180     errorCode=U_USELESS_COLLATOR_ERROR;
 181     nontrans->nameToUnicodeUTF8(StringPiece(s, 3), sink, info, errorCode);
 182     if(errorCode!=U_USELESS_COLLATOR_ERROR || sink.NumberOfBytesWritten()!=0) {
 183         errln("N.nameToUnicode_UTF8(U_FAILURE) did not preserve the errorCode "
 184               "or did output something - %s",
 185               u_errorName(errorCode));
 186     }
 187
 188     sink.Reset();
 189     errorCode=U_ZERO_ERROR;
 190     trans->labelToUnicodeUTF8(StringPiece(s, 3), sink, info, errorCode);
 191     if( U_FAILURE(errorCode) || sink.NumberOfBytesWritten()!=3 ||
 192         buffer[0]!=0x61 || buffer[1]!=0x73 || buffer[2]!=0x73 ||
 193         !sink.calledFlush
 194     ) {
 195         errln("T.labelToUnicodeUTF8(a sharp-s) failed - %s",
 196               u_errorName(errorCode));
 197     }
 198
 199     sink.Reset();
 200     errorCode=U_ZERO_ERROR;
 201     // "eXampLe.cOm"
 202     static const char eX[]={ 0x65, 0x58, 0x61, 0x6d, 0x70, 0x4c, 0x65, 0x2e, 0x63, 0x4f, 0x6d, 0 };
 203     // "example.com"
 204     static const char ex[]={ 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d };
 205     trans->nameToUnicodeUTF8(eX, sink, info, errorCode);
 206     if( U_FAILURE(errorCode) || sink.NumberOfBytesWritten()!=11 ||
 207         0!=memcmp(ex, buffer, 11) || !sink.calledFlush
 208     ) {
 209         errln("T.nameToUnicodeUTF8(eXampLe.cOm) failed - %s",
 210               u_errorName(errorCode));
 211     }
 212 }
 213
 214 void UTS46Test::TestNotSTD3() {
 215     IcuTestErrorCode errorCode(*this, "TestNotSTD3()");
 216     char buffer[400];
 217     LocalPointer<IDNA> not3(IDNA::createUTS46Instance(UIDNA_CHECK_BIDI, errorCode));
 218     if(errorCode.isFailure()) {
 219         return;
 220     }
 221     UnicodeString input=UNICODE_STRING_SIMPLE("\\u0000A_2+2=4\\u000A.e\\u00DFen.net").unescape();
 222     UnicodeString result;
 223     IDNAInfo info;
 224     if( not3->nameToUnicode(input, result, info, errorCode)!=
 225             UNICODE_STRING_SIMPLE("\\u0000a_2+2=4\\u000A.essen.net").unescape() ||
 226         info.hasErrors()
 227     ) {
 228         prettify(result).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer));
 229         errln("notSTD3.nameToUnicode(non-LDH ASCII) unexpected errors %04lx string %s",
 230               (long)info.getErrors(), buffer);
 231     }
 232     // A space (BiDi class WS) is not allowed in a BiDi domain name.
 233     input=UNICODE_STRING_SIMPLE("a z.xn--4db.edu");
 234     not3->nameToASCII(input, result, info, errorCode);
 235     if(result!=input || info.getErrors()!=UIDNA_ERROR_BIDI) {
 236         errln("notSTD3.nameToASCII(ASCII-with-space.alef.edu) failed");
 237     }
 238     // Characters that are canonically equivalent to sequences with non-LDH ASCII.
 239     input=UNICODE_STRING_SIMPLE("a\\u2260b\\u226Ec\\u226Fd").unescape();
 240     not3->nameToUnicode(input, result, info, errorCode);
 241     if(result!=input || info.hasErrors()) {
 242         prettify(result).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer));
 243         errln("notSTD3.nameToUnicode(equiv to non-LDH ASCII) unexpected errors %04lx string %s",
 244               (long)info.getErrors(), buffer);
 245     }
 246 }
 247
 248 struct TestCase {
 249     // Input string and options string (Nontransitional/Transitional/Both).
 250     const char *s, *o;
 251     // Expected Unicode result string.
 252     const char *u;
 253     uint32_t errors;
 254 };
 255
 256 static const TestCase testCases[]={
 257     { "www.eXample.cOm", "B",  // all ASCII
 258       "www.example.com", 0 },
 259     { "B\\u00FCcher.de", "B",  // u-umlaut
 260       "b\\u00FCcher.de", 0 },
 261     { "\\u00D6BB", "B",  // O-umlaut
 262       "\\u00F6bb", 0 },
 263     { "fa\\u00DF.de", "N",  // sharp s
 264       "fa\\u00DF.de", 0 },
 265     { "fa\\u00DF.de", "T",  // sharp s
 266       "fass.de", 0 },
 267     { "XN--fA-hia.dE", "B",  // sharp s in Punycode
 268       "fa\\u00DF.de", 0 },
 269     { "\\u03B2\\u03CC\\u03BB\\u03BF\\u03C2.com", "N",  // Greek with final sigma
 270       "\\u03B2\\u03CC\\u03BB\\u03BF\\u03C2.com", 0 },
 271     { "\\u03B2\\u03CC\\u03BB\\u03BF\\u03C2.com", "T",  // Greek with final sigma
 272       "\\u03B2\\u03CC\\u03BB\\u03BF\\u03C3.com", 0 },
 273     { "xn--nxasmm1c", "B",  // Greek with final sigma in Punycode
 274       "\\u03B2\\u03CC\\u03BB\\u03BF\\u03C2", 0 },
 275     { "www.\\u0DC1\\u0DCA\\u200D\\u0DBB\\u0DD3.com", "N",  // "Sri" in "Sri Lanka" has a ZWJ
 276       "www.\\u0DC1\\u0DCA\\u200D\\u0DBB\\u0DD3.com", 0 },
 277     { "www.\\u0DC1\\u0DCA\\u200D\\u0DBB\\u0DD3.com", "T",  // "Sri" in "Sri Lanka" has a ZWJ
 278       "www.\\u0DC1\\u0DCA\\u0DBB\\u0DD3.com", 0 },
 279     { "www.xn--10cl1a0b660p.com", "B",  // "Sri" in Punycode
 280       "www.\\u0DC1\\u0DCA\\u200D\\u0DBB\\u0DD3.com", 0 },
 281     { "\\u0646\\u0627\\u0645\\u0647\\u200C\\u0627\\u06CC", "N",  // ZWNJ
 282       "\\u0646\\u0627\\u0645\\u0647\\u200C\\u0627\\u06CC", 0 },
 283     { "\\u0646\\u0627\\u0645\\u0647\\u200C\\u0627\\u06CC", "T",  // ZWNJ
 284       "\\u0646\\u0627\\u0645\\u0647\\u0627\\u06CC", 0 },
 285     { "xn--mgba3gch31f060k.com", "B",  // ZWNJ in Punycode
 286       "\\u0646\\u0627\\u0645\\u0647\\u200C\\u0627\\u06CC.com", 0 },
 287     { "a.b\\uFF0Ec\\u3002d\\uFF61", "B",
 288       "a.b.c.d.", 0 },
 289     { "U\\u0308.xn--tda", "B",  // U+umlaut.u-umlaut
 290       "\\u00FC.\\u00FC", 0 },
 291     { "xn--u-ccb", "B",  // u+umlaut in Punycode
 292       "xn--u-ccb\\uFFFD", UIDNA_ERROR_INVALID_ACE_LABEL },
 293     { "a\\u2488com", "B",  // contains 1-dot
 294       "a\\uFFFDcom", UIDNA_ERROR_DISALLOWED },
 295     { "xn--a-ecp.ru", "B",  // contains 1-dot in Punycode
 296       "xn--a-ecp\\uFFFD.ru", UIDNA_ERROR_INVALID_ACE_LABEL },
 297     { "xn--0.pt", "B",  // invalid Punycode
 298       "xn--0\\uFFFD.pt", UIDNA_ERROR_PUNYCODE },
 299     { "xn--a.pt", "B",  // U+0080
 300       "xn--a\\uFFFD.pt", UIDNA_ERROR_INVALID_ACE_LABEL },
 301     { "xn--a-\\u00C4.pt", "B",  // invalid Punycode
 302       "xn--a-\\u00E4.pt", UIDNA_ERROR_PUNYCODE },
 303     { "\\u65E5\\u672C\\u8A9E\\u3002\\uFF2A\\uFF30", "B",  // Japanese with fullwidth ".jp"
 304       "\\u65E5\\u672C\\u8A9E.jp", 0 },
 305     { "\\u2615", "B", "\\u2615", 0 },  // Unicode 4.0 HOT BEVERAGE
 306     // some characters are disallowed because they are canonically equivalent
 307     // to sequences with non-LDH ASCII
 308     { "a\\u2260b\\u226Ec\\u226Fd", "B",
 309       "a\\uFFFDb\\uFFFDc\\uFFFDd", UIDNA_ERROR_DISALLOWED },
 310     // many deviation characters, test the special mapping code
 311     { "1.a\\u00DF\\u200C\\u200Db\\u200C\\u200Dc\\u00DF\\u00DF\\u00DF\\u00DFd"
 312       "\\u03C2\\u03C3\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFe"
 313       "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFx"
 314       "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFy"
 315       "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u0302\\u00DFz", "N",
 316       "1.a\\u00DF\\u200C\\u200Db\\u200C\\u200Dc\\u00DF\\u00DF\\u00DF\\u00DFd"
 317       "\\u03C2\\u03C3\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFe"
 318       "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFx"
 319       "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFy"
 320       "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u0302\\u00DFz",
 321       UIDNA_ERROR_LABEL_TOO_LONG|UIDNA_ERROR_CONTEXTJ },
 322     { "1.a\\u00DF\\u200C\\u200Db\\u200C\\u200Dc\\u00DF\\u00DF\\u00DF\\u00DFd"
 323       "\\u03C2\\u03C3\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFe"
 324       "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFx"
 325       "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFy"
 326       "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u0302\\u00DFz", "T",
 327       "1.assbcssssssssd"
 328       "\\u03C3\\u03C3sssssssssssssssse"
 329       "ssssssssssssssssssssx"
 330       "ssssssssssssssssssssy"
 331       "sssssssssssssss\\u015Dssz", UIDNA_ERROR_LABEL_TOO_LONG },
 332     // "xn--bss" with deviation characters
 333     { "\\u200Cx\\u200Dn\\u200C-\\u200D-b\\u00DF", "N",
 334       "\\u200Cx\\u200Dn\\u200C-\\u200D-b\\u00DF", UIDNA_ERROR_CONTEXTJ },
 335     { "\\u200Cx\\u200Dn\\u200C-\\u200D-b\\u00DF", "T",
 336       "\\u5919", 0 },
 337     // "xn--bssffl" written as:
 338     // 02E3 MODIFIER LETTER SMALL X
 339     // 034F COMBINING GRAPHEME JOINER (ignored)
 340     // 2115 DOUBLE-STRUCK CAPITAL N
 341     // 200B ZERO WIDTH SPACE (ignored)
 342     // FE63 SMALL HYPHEN-MINUS
 343     // 00AD SOFT HYPHEN (ignored)
 344     // FF0D FULLWIDTH HYPHEN-MINUS
 345     // 180C MONGOLIAN FREE VARIATION SELECTOR TWO (ignored)
 346     // 212C SCRIPT CAPITAL B
 347     // FE00 VARIATION SELECTOR-1 (ignored)
 348     // 017F LATIN SMALL LETTER LONG S
 349     // 2064 INVISIBLE PLUS (ignored)
 350     // 1D530 MATHEMATICAL FRAKTUR SMALL S
 351     // E01EF VARIATION SELECTOR-256 (ignored)
 352     // FB04 LATIN SMALL LIGATURE FFL
 353     { "\\u02E3\\u034F\\u2115\\u200B\\uFE63\\u00AD\\uFF0D\\u180C"
 354       "\\u212C\\uFE00\\u017F\\u2064\\U0001D530\\U000E01EF\\uFB04", "B",
 355       "\\u5921\\u591E\\u591C\\u5919", 0 },
 356     { "123456789012345678901234567890123456789012345678901234567890123."
 357       "123456789012345678901234567890123456789012345678901234567890123."
 358       "123456789012345678901234567890123456789012345678901234567890123."
 359       "1234567890123456789012345678901234567890123456789012345678901", "B",
 360       "123456789012345678901234567890123456789012345678901234567890123."
 361       "123456789012345678901234567890123456789012345678901234567890123."
 362       "123456789012345678901234567890123456789012345678901234567890123."
 363       "1234567890123456789012345678901234567890123456789012345678901", 0 },
 364     { "123456789012345678901234567890123456789012345678901234567890123."
 365       "123456789012345678901234567890123456789012345678901234567890123."
 366       "123456789012345678901234567890123456789012345678901234567890123."
 367       "1234567890123456789012345678901234567890123456789012345678901.", "B",
 368       "123456789012345678901234567890123456789012345678901234567890123."
 369       "123456789012345678901234567890123456789012345678901234567890123."
 370       "123456789012345678901234567890123456789012345678901234567890123."
 371       "1234567890123456789012345678901234567890123456789012345678901.", 0 },
 372     // Domain name >256 characters, forces slow path in UTF-8 processing.
 373     { "123456789012345678901234567890123456789012345678901234567890123."
 374       "123456789012345678901234567890123456789012345678901234567890123."
 375       "123456789012345678901234567890123456789012345678901234567890123."
 376       "123456789012345678901234567890123456789012345678901234567890123."
 377       "12345678901234567890123456789012345678901234567890123456789012", "B",
 378       "123456789012345678901234567890123456789012345678901234567890123."
 379       "123456789012345678901234567890123456789012345678901234567890123."
 380       "123456789012345678901234567890123456789012345678901234567890123."
 381       "123456789012345678901234567890123456789012345678901234567890123."
 382       "12345678901234567890123456789012345678901234567890123456789012",
 383       UIDNA_ERROR_DOMAIN_NAME_TOO_LONG },
 384     { "123456789012345678901234567890123456789012345678901234567890123."
 385       "123456789012345678901234567890123456789012345678901234567890123."
 386       "123456789012345678901234567890123456789012345678901234567890123."
 387       "123456789012345678901234567890123456789012345678901234567890123."
 388       "1234567890123456789012345678901234567890123456789\\u05D0", "B",
 389       "123456789012345678901234567890123456789012345678901234567890123."
 390       "123456789012345678901234567890123456789012345678901234567890123."
 391       "123456789012345678901234567890123456789012345678901234567890123."
 392       "123456789012345678901234567890123456789012345678901234567890123."
 393       "1234567890123456789012345678901234567890123456789\\u05D0",
 394       UIDNA_ERROR_DOMAIN_NAME_TOO_LONG|UIDNA_ERROR_BIDI },
 395     { "123456789012345678901234567890123456789012345678901234567890123."
 396       "1234567890123456789012345678901234567890123456789012345678901234."
 397       "123456789012345678901234567890123456789012345678901234567890123."
 398       "123456789012345678901234567890123456789012345678901234567890", "B",
 399       "123456789012345678901234567890123456789012345678901234567890123."
 400       "1234567890123456789012345678901234567890123456789012345678901234."
 401       "123456789012345678901234567890123456789012345678901234567890123."
 402       "123456789012345678901234567890123456789012345678901234567890",
 403       UIDNA_ERROR_LABEL_TOO_LONG },
 404     { "123456789012345678901234567890123456789012345678901234567890123."
 405       "1234567890123456789012345678901234567890123456789012345678901234."
 406       "123456789012345678901234567890123456789012345678901234567890123."
 407       "123456789012345678901234567890123456789012345678901234567890.", "B",
 408       "123456789012345678901234567890123456789012345678901234567890123."
 409       "1234567890123456789012345678901234567890123456789012345678901234."
 410       "123456789012345678901234567890123456789012345678901234567890123."
 411       "123456789012345678901234567890123456789012345678901234567890.",
 412       UIDNA_ERROR_LABEL_TOO_LONG },
 413     { "123456789012345678901234567890123456789012345678901234567890123."
 414       "1234567890123456789012345678901234567890123456789012345678901234."
 415       "123456789012345678901234567890123456789012345678901234567890123."
 416       "1234567890123456789012345678901234567890123456789012345678901", "B",
 417       "123456789012345678901234567890123456789012345678901234567890123."
 418       "1234567890123456789012345678901234567890123456789012345678901234."
 419       "123456789012345678901234567890123456789012345678901234567890123."
 420       "1234567890123456789012345678901234567890123456789012345678901",
 421       UIDNA_ERROR_LABEL_TOO_LONG|UIDNA_ERROR_DOMAIN_NAME_TOO_LONG },
 422     // label length 63: xn--1234567890123456789012345678901234567890123456789012345-9te
 423     { "\\u00E41234567890123456789012345678901234567890123456789012345", "B",
 424       "\\u00E41234567890123456789012345678901234567890123456789012345", 0 },
 425     { "1234567890\\u00E41234567890123456789012345678901234567890123456", "B",
 426       "1234567890\\u00E41234567890123456789012345678901234567890123456", UIDNA_ERROR_LABEL_TOO_LONG },
 427     { "123456789012345678901234567890123456789012345678901234567890123."
 428       "1234567890\\u00E4123456789012345678901234567890123456789012345."
 429       "123456789012345678901234567890123456789012345678901234567890123."
 430       "1234567890123456789012345678901234567890123456789012345678901", "B",
 431       "123456789012345678901234567890123456789012345678901234567890123."
 432       "1234567890\\u00E4123456789012345678901234567890123456789012345."
 433       "123456789012345678901234567890123456789012345678901234567890123."
 434       "1234567890123456789012345678901234567890123456789012345678901", 0 },
 435     { "123456789012345678901234567890123456789012345678901234567890123."
 436       "1234567890\\u00E4123456789012345678901234567890123456789012345."
 437       "123456789012345678901234567890123456789012345678901234567890123."
 438       "1234567890123456789012345678901234567890123456789012345678901.", "B",
 439       "123456789012345678901234567890123456789012345678901234567890123."
 440       "1234567890\\u00E4123456789012345678901234567890123456789012345."
 441       "123456789012345678901234567890123456789012345678901234567890123."
 442       "1234567890123456789012345678901234567890123456789012345678901.", 0 },
 443     { "123456789012345678901234567890123456789012345678901234567890123."
 444       "1234567890\\u00E4123456789012345678901234567890123456789012345."
 445       "123456789012345678901234567890123456789012345678901234567890123."
 446       "12345678901234567890123456789012345678901234567890123456789012", "B",
 447       "123456789012345678901234567890123456789012345678901234567890123."
 448       "1234567890\\u00E4123456789012345678901234567890123456789012345."
 449       "123456789012345678901234567890123456789012345678901234567890123."
 450       "12345678901234567890123456789012345678901234567890123456789012",
 451       UIDNA_ERROR_DOMAIN_NAME_TOO_LONG },
 452     { "123456789012345678901234567890123456789012345678901234567890123."
 453       "1234567890\\u00E41234567890123456789012345678901234567890123456."
 454       "123456789012345678901234567890123456789012345678901234567890123."
 455       "123456789012345678901234567890123456789012345678901234567890", "B",
 456       "123456789012345678901234567890123456789012345678901234567890123."
 457       "1234567890\\u00E41234567890123456789012345678901234567890123456."
 458       "123456789012345678901234567890123456789012345678901234567890123."
 459       "123456789012345678901234567890123456789012345678901234567890",
 460       UIDNA_ERROR_LABEL_TOO_LONG },
 461     { "123456789012345678901234567890123456789012345678901234567890123."
 462       "1234567890\\u00E41234567890123456789012345678901234567890123456."
 463       "123456789012345678901234567890123456789012345678901234567890123."
 464       "123456789012345678901234567890123456789012345678901234567890.", "B",
 465       "123456789012345678901234567890123456789012345678901234567890123."
 466       "1234567890\\u00E41234567890123456789012345678901234567890123456."
 467       "123456789012345678901234567890123456789012345678901234567890123."
 468       "123456789012345678901234567890123456789012345678901234567890.",
 469       UIDNA_ERROR_LABEL_TOO_LONG },
 470     { "123456789012345678901234567890123456789012345678901234567890123."
 471       "1234567890\\u00E41234567890123456789012345678901234567890123456."
 472       "123456789012345678901234567890123456789012345678901234567890123."
 473       "1234567890123456789012345678901234567890123456789012345678901", "B",
 474       "123456789012345678901234567890123456789012345678901234567890123."
 475       "1234567890\\u00E41234567890123456789012345678901234567890123456."
 476       "123456789012345678901234567890123456789012345678901234567890123."
 477       "1234567890123456789012345678901234567890123456789012345678901",
 478       UIDNA_ERROR_LABEL_TOO_LONG|UIDNA_ERROR_DOMAIN_NAME_TOO_LONG },
 479     // hyphen errors and empty-label errors
 480     // Ticket #10883: ToUnicode also checks for empty labels.
 481     { ".", "B", ".", UIDNA_ERROR_EMPTY_LABEL },
 482     { "\\uFF0E", "B", ".", UIDNA_ERROR_EMPTY_LABEL },
 483     // "xn---q----jra"=="-q--a-umlaut-"
 484     { "a.b..-q--a-.e", "B", "a.b..-q--a-.e",
 485       UIDNA_ERROR_EMPTY_LABEL|UIDNA_ERROR_LEADING_HYPHEN|UIDNA_ERROR_TRAILING_HYPHEN|
 486       UIDNA_ERROR_HYPHEN_3_4 },
 487     { "a.b..-q--\\u00E4-.e", "B", "a.b..-q--\\u00E4-.e",
 488       UIDNA_ERROR_EMPTY_LABEL|UIDNA_ERROR_LEADING_HYPHEN|UIDNA_ERROR_TRAILING_HYPHEN|
 489       UIDNA_ERROR_HYPHEN_3_4 },
 490     { "a.b..xn---q----jra.e", "B", "a.b..-q--\\u00E4-.e",
 491       UIDNA_ERROR_EMPTY_LABEL|UIDNA_ERROR_LEADING_HYPHEN|UIDNA_ERROR_TRAILING_HYPHEN|
 492       UIDNA_ERROR_HYPHEN_3_4 },
 493     { "a..c", "B", "a..c", UIDNA_ERROR_EMPTY_LABEL },
 494     { "a.xn--.c", "B", "a..c", UIDNA_ERROR_EMPTY_LABEL },
 495     { "a.-b.", "B", "a.-b.", UIDNA_ERROR_LEADING_HYPHEN },
 496     { "a.b-.c", "B", "a.b-.c", UIDNA_ERROR_TRAILING_HYPHEN },
 497     { "a.-.c", "B", "a.-.c", UIDNA_ERROR_LEADING_HYPHEN|UIDNA_ERROR_TRAILING_HYPHEN },
 498     { "a.bc--de.f", "B", "a.bc--de.f", UIDNA_ERROR_HYPHEN_3_4 },
 499     { "\\u00E4.\\u00AD.c", "B", "\\u00E4..c", UIDNA_ERROR_EMPTY_LABEL },
 500     { "\\u00E4.xn--.c", "B", "\\u00E4..c", UIDNA_ERROR_EMPTY_LABEL },
 501     { "\\u00E4.-b.", "B", "\\u00E4.-b.", UIDNA_ERROR_LEADING_HYPHEN },
 502     { "\\u00E4.b-.c", "B", "\\u00E4.b-.c", UIDNA_ERROR_TRAILING_HYPHEN },
 503     { "\\u00E4.-.c", "B", "\\u00E4.-.c", UIDNA_ERROR_LEADING_HYPHEN|UIDNA_ERROR_TRAILING_HYPHEN },
 504     { "\\u00E4.bc--de.f", "B", "\\u00E4.bc--de.f", UIDNA_ERROR_HYPHEN_3_4 },
 505     { "a.b.\\u0308c.d", "B", "a.b.\\uFFFDc.d", UIDNA_ERROR_LEADING_COMBINING_MARK },
 506     { "a.b.xn--c-bcb.d", "B",
 507       "a.b.xn--c-bcb\\uFFFD.d", UIDNA_ERROR_LEADING_COMBINING_MARK|UIDNA_ERROR_INVALID_ACE_LABEL },
 508     // BiDi
 509     { "A0", "B", "a0", 0 },
 510     { "0A", "B", "0a", 0 },  // all-LTR is ok to start with a digit (EN)
 511     { "0A.\\u05D0", "B",  // ASCII label does not start with L/R/AL
 512       "0a.\\u05D0", UIDNA_ERROR_BIDI },
 513     { "c.xn--0-eha.xn--4db", "B",  // 2nd label does not start with L/R/AL
 514       "c.0\\u00FC.\\u05D0", UIDNA_ERROR_BIDI },
 515     { "b-.\\u05D0", "B",  // label does not end with L/EN
 516       "b-.\\u05D0", UIDNA_ERROR_TRAILING_HYPHEN|UIDNA_ERROR_BIDI },
 517     { "d.xn----dha.xn--4db", "B",  // 2nd label does not end with L/EN
 518       "d.\\u00FC-.\\u05D0", UIDNA_ERROR_TRAILING_HYPHEN|UIDNA_ERROR_BIDI },
 519     { "a\\u05D0", "B", "a\\u05D0", UIDNA_ERROR_BIDI },  // first dir != last dir
 520     { "\\u05D0\\u05C7", "B", "\\u05D0\\u05C7", 0 },
 521     { "\\u05D09\\u05C7", "B", "\\u05D09\\u05C7", 0 },
 522     { "\\u05D0a\\u05C7", "B", "\\u05D0a\\u05C7", UIDNA_ERROR_BIDI },  // first dir != last dir
 523     { "\\u05D0\\u05EA", "B", "\\u05D0\\u05EA", 0 },
 524     { "\\u05D0\\u05F3\\u05EA", "B", "\\u05D0\\u05F3\\u05EA", 0 },
 525     { "a\\u05D0Tz", "B", "a\\u05D0tz", UIDNA_ERROR_BIDI },  // mixed dir
 526     { "\\u05D0T\\u05EA", "B", "\\u05D0t\\u05EA", UIDNA_ERROR_BIDI },  // mixed dir
 527     { "\\u05D07\\u05EA", "B", "\\u05D07\\u05EA", 0 },
 528     { "\\u05D0\\u0667\\u05EA", "B", "\\u05D0\\u0667\\u05EA", 0 },  // Arabic 7 in the middle
 529     { "a7\\u0667z", "B", "a7\\u0667z", UIDNA_ERROR_BIDI },  // AN digit in LTR
 530     { "a7\\u0667", "B", "a7\\u0667", UIDNA_ERROR_BIDI },  // AN digit in LTR
 531     { "\\u05D07\\u0667\\u05EA", "B",  // mixed EN/AN digits in RTL
 532       "\\u05D07\\u0667\\u05EA", UIDNA_ERROR_BIDI },
 533     { "\\u05D07\\u0667", "B",  // mixed EN/AN digits in RTL
 534       "\\u05D07\\u0667", UIDNA_ERROR_BIDI },
 535     // ZWJ
 536     { "\\u0BB9\\u0BCD\\u200D", "N", "\\u0BB9\\u0BCD\\u200D", 0 },  // Virama+ZWJ
 537     { "\\u0BB9\\u200D", "N", "\\u0BB9\\u200D", UIDNA_ERROR_CONTEXTJ },  // no Virama
 538     { "\\u200D", "N", "\\u200D", UIDNA_ERROR_CONTEXTJ },  // no Virama
 539     // ZWNJ
 540     { "\\u0BB9\\u0BCD\\u200C", "N", "\\u0BB9\\u0BCD\\u200C", 0 },  // Virama+ZWNJ
 541     { "\\u0BB9\\u200C", "N", "\\u0BB9\\u200C", UIDNA_ERROR_CONTEXTJ },  // no Virama
 542     { "\\u200C", "N", "\\u200C", UIDNA_ERROR_CONTEXTJ },  // no Virama
 543     { "\\u0644\\u0670\\u200C\\u06ED\\u06EF", "N",  // Joining types D T ZWNJ T R
 544       "\\u0644\\u0670\\u200C\\u06ED\\u06EF", 0 },
 545     { "\\u0644\\u0670\\u200C\\u06EF", "N",  // D T ZWNJ R
 546       "\\u0644\\u0670\\u200C\\u06EF", 0 },
 547     { "\\u0644\\u200C\\u06ED\\u06EF", "N",  // D ZWNJ T R
 548       "\\u0644\\u200C\\u06ED\\u06EF", 0 },
 549     { "\\u0644\\u200C\\u06EF", "N",  // D ZWNJ R
 550       "\\u0644\\u200C\\u06EF", 0 },
 551     { "\\u0644\\u0670\\u200C\\u06ED", "N",  // D T ZWNJ T
 552       "\\u0644\\u0670\\u200C\\u06ED", UIDNA_ERROR_BIDI|UIDNA_ERROR_CONTEXTJ },
 553     { "\\u06EF\\u200C\\u06EF", "N",  // R ZWNJ R
 554       "\\u06EF\\u200C\\u06EF", UIDNA_ERROR_CONTEXTJ },
 555     { "\\u0644\\u200C", "N",  // D ZWNJ
 556       "\\u0644\\u200C", UIDNA_ERROR_BIDI|UIDNA_ERROR_CONTEXTJ },
 557     { "\\u0660\\u0661", "B",  // Arabic-Indic Digits alone
 558       "\\u0660\\u0661", UIDNA_ERROR_BIDI },
 559     { "\\u06F0\\u06F1", "B",  // Extended Arabic-Indic Digits alone
 560       "\\u06F0\\u06F1", 0 },
 561     { "\\u0660\\u06F1", "B",  // Mixed Arabic-Indic Digits
 562       "\\u0660\\u06F1", UIDNA_ERROR_CONTEXTO_DIGITS|UIDNA_ERROR_BIDI },
 563     // All of the CONTEXTO "Would otherwise have been DISALLOWED" characters
 564     // in their correct contexts,
 565     // then each in incorrect context.
 566     { "l\\u00B7l\\u4E00\\u0375\\u03B1\\u05D0\\u05F3\\u05F4\\u30FB", "B",
 567       "l\\u00B7l\\u4E00\\u0375\\u03B1\\u05D0\\u05F3\\u05F4\\u30FB", UIDNA_ERROR_BIDI },
 568     { "l\\u00B7", "B",
 569       "l\\u00B7", UIDNA_ERROR_CONTEXTO_PUNCTUATION },
 570     { "\\u00B7l", "B",
 571       "\\u00B7l", UIDNA_ERROR_CONTEXTO_PUNCTUATION },
 572     { "\\u0375", "B",
 573       "\\u0375", UIDNA_ERROR_CONTEXTO_PUNCTUATION },
 574     { "\\u03B1\\u05F3", "B",
 575       "\\u03B1\\u05F3", UIDNA_ERROR_CONTEXTO_PUNCTUATION|UIDNA_ERROR_BIDI },
 576     { "\\u05F4", "B",
 577       "\\u05F4", UIDNA_ERROR_CONTEXTO_PUNCTUATION },
 578     { "l\\u30FB", "B",
 579       "l\\u30FB", UIDNA_ERROR_CONTEXTO_PUNCTUATION },
 580     // Ticket #8137: UTS #46 toUnicode() fails with non-ASCII labels that turn
 581     // into 15 characters (UChars).
 582     // The bug was in u_strFromPunycode() which did not write the last character
 583     // if it just so fit into the end of the destination buffer.
 584     // The UTS #46 code gives a default-capacity UnicodeString as the destination buffer,
 585     // and the internal UnicodeString capacity is currently 15 UChars on 64-bit machines
 586     // but 13 on 32-bit machines.
 587     // Label with 15 UChars, for 64-bit-machine testing:
 588     { "aaaaaaaaaaaaa\\u00FCa.de", "B", "aaaaaaaaaaaaa\\u00FCa.de", 0 },
 589     { "xn--aaaaaaaaaaaaaa-ssb.de", "B", "aaaaaaaaaaaaa\\u00FCa.de", 0 },
 590     { "abschlu\\u00DFpr\\u00FCfung.de", "N", "abschlu\\u00DFpr\\u00FCfung.de", 0 },
 591     { "xn--abschluprfung-hdb15b.de", "B", "abschlu\\u00DFpr\\u00FCfung.de", 0 },
 592     // Label with 13 UChars, for 32-bit-machine testing:
 593     { "xn--aaaaaaaaaaaa-nlb.de", "B", "aaaaaaaaaaa\\u00FCa.de", 0 },
 594     { "xn--schluprfung-z6a39a.de", "B", "schlu\\u00DFpr\\u00FCfung.de", 0 },
 595     // { "", "B",
 596     //   "", 0 },
 597 };
 598
 599 void UTS46Test::TestSomeCases() {
 600     IcuTestErrorCode errorCode(*this, "TestSomeCases");
 601     char buffer[400], buffer2[400];
 602     int32_t i;
 603     for(i=0; i<UPRV_LENGTHOF(testCases); ++i) {
 604         const TestCase &testCase=testCases[i];
 605         UnicodeString input(ctou(testCase.s));
 606         UnicodeString expected(ctou(testCase.u));
 607         // ToASCII/ToUnicode, transitional/nontransitional
 608         UnicodeString aT, uT, aN, uN;
 609         IDNAInfo aTInfo, uTInfo, aNInfo, uNInfo;
 610         trans->nameToASCII(input, aT, aTInfo, errorCode);
 611         trans->nameToUnicode(input, uT, uTInfo, errorCode);
 612         nontrans->nameToASCII(input, aN, aNInfo, errorCode);
 613         nontrans->nameToUnicode(input, uN, uNInfo, errorCode);
 614         if(errorCode.errIfFailureAndReset("first-level processing [%d/%s] %s",
 615                                           (int)i, testCase.o, testCase.s)
 616         ) {
 617             continue;
 618         }
 619         // ToUnicode does not set length-overflow errors.
 620         uint32_t uniErrors=testCase.errors&~
 621             (UIDNA_ERROR_LABEL_TOO_LONG|
 622              UIDNA_ERROR_DOMAIN_NAME_TOO_LONG);
 623         char mode=testCase.o[0];
 624         if(mode=='B' || mode=='N') {
 625             if(uNInfo.getErrors()!=uniErrors) {
 626                 errln("N.nameToUnicode([%d] %s) unexpected errors %04lx",
 627                       (int)i, testCase.s, (long)uNInfo.getErrors());
 628                 continue;
 629             }
 630             if(uN!=expected) {
 631                 prettify(uN).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer));
 632                 errln("N.nameToUnicode([%d] %s) unexpected string %s",
 633                       (int)i, testCase.s, buffer);
 634                 continue;
 635             }
 636             if(aNInfo.getErrors()!=testCase.errors) {
 637                 errln("N.nameToASCII([%d] %s) unexpected errors %04lx",
 638                       (int)i, testCase.s, (long)aNInfo.getErrors());
 639                 continue;
 640             }
 641         }
 642         if(mode=='B' || mode=='T') {
 643             if(uTInfo.getErrors()!=uniErrors) {
 644                 errln("T.nameToUnicode([%d] %s) unexpected errors %04lx",
 645                       (int)i, testCase.s, (long)uTInfo.getErrors());
 646                 continue;
 647             }
 648             if(uT!=expected) {
 649                 prettify(uT).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer));
 650                 errln("T.nameToUnicode([%d] %s) unexpected string %s",
 651                       (int)i, testCase.s, buffer);
 652                 continue;
 653             }
 654             if(aTInfo.getErrors()!=testCase.errors) {
 655                 errln("T.nameToASCII([%d] %s) unexpected errors %04lx",
 656                       (int)i, testCase.s, (long)aTInfo.getErrors());
 657                 continue;
 658             }
 659         }
 660         // ToASCII is all-ASCII if no severe errors
 661         if((aNInfo.getErrors()&severeErrors)==0 && !isASCII(aN)) {
 662             prettify(aN).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer));
 663             errln("N.nameToASCII([%d] %s) (errors %04lx) result is not ASCII %s",
 664                   (int)i, testCase.s, aNInfo.getErrors(), buffer);
 665             continue;
 666         }
 667         if((aTInfo.getErrors()&severeErrors)==0 && !isASCII(aT)) {
 668             prettify(aT).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer));
 669             errln("T.nameToASCII([%d] %s) (errors %04lx) result is not ASCII %s",
 670                   (int)i, testCase.s, aTInfo.getErrors(), buffer);
 671             continue;
 672         }
 673         if(verbose) {
 674             char m= mode=='B' ? mode : 'N';
 675             prettify(aN).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer));
 676             logln("%c.nameToASCII([%d] %s) (errors %04lx) result string: %s",
 677                   m, (int)i, testCase.s, aNInfo.getErrors(), buffer);
 678             if(mode!='B') {
 679                 prettify(aT).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer));
 680                 logln("T.nameToASCII([%d] %s) (errors %04lx) result string: %s",
 681                       (int)i, testCase.s, aTInfo.getErrors(), buffer);
 682             }
 683         }
 684         // second-level processing
 685         UnicodeString aTuN, uTaN, aNuN, uNaN;
 686         IDNAInfo aTuNInfo, uTaNInfo, aNuNInfo, uNaNInfo;
 687         nontrans->nameToUnicode(aT, aTuN, aTuNInfo, errorCode);
 688         nontrans->nameToASCII(uT, uTaN, uTaNInfo, errorCode);
 689         nontrans->nameToUnicode(aN, aNuN, aNuNInfo, errorCode);
 690         nontrans->nameToASCII(uN, uNaN, uNaNInfo, errorCode);
 691         if(errorCode.errIfFailureAndReset("second-level processing [%d/%s] %s",
 692                                           (int)i, testCase.o, testCase.s)
 693         ) {
 694             continue;
 695         }
 696         if(aN!=uNaN) {
 697             prettify(aN).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer));
 698             prettify(uNaN).extract(0, 0x7fffffff, buffer2, UPRV_LENGTHOF(buffer2));
 699             errln("N.nameToASCII([%d] %s)!=N.nameToUnicode().N.nameToASCII() "
 700                   "(errors %04lx) %s vs. %s",
 701                   (int)i, testCase.s, aNInfo.getErrors(), buffer, buffer2);
 702             continue;
 703         }
 704         if(aT!=uTaN) {
 705             prettify(aT).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer));
 706             prettify(uTaN).extract(0, 0x7fffffff, buffer2, UPRV_LENGTHOF(buffer2));
 707             errln("T.nameToASCII([%d] %s)!=T.nameToUnicode().N.nameToASCII() "
 708                   "(errors %04lx) %s vs. %s",
 709                   (int)i, testCase.s, aNInfo.getErrors(), buffer, buffer2);
 710             continue;
 711         }
 712         if(uN!=aNuN) {
 713             prettify(uN).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer));
 714             prettify(aNuN).extract(0, 0x7fffffff, buffer2, UPRV_LENGTHOF(buffer2));
 715             errln("N.nameToUnicode([%d] %s)!=N.nameToASCII().N.nameToUnicode() "
 716                   "(errors %04lx) %s vs. %s",
 717                   (int)i, testCase.s, uNInfo.getErrors(), buffer, buffer2);
 718             continue;
 719         }
 720         if(uT!=aTuN) {
 721             prettify(uT).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer));
 722             prettify(aTuN).extract(0, 0x7fffffff, buffer2, UPRV_LENGTHOF(buffer2));
 723             errln("T.nameToUnicode([%d] %s)!=T.nameToASCII().N.nameToUnicode() "
 724                   "(errors %04lx) %s vs. %s",
 725                   (int)i, testCase.s, uNInfo.getErrors(), buffer, buffer2);
 726             continue;
 727         }
 728         // labelToUnicode
 729         UnicodeString aTL, uTL, aNL, uNL;
 730         IDNAInfo aTLInfo, uTLInfo, aNLInfo, uNLInfo;
 731         trans->labelToASCII(input, aTL, aTLInfo, errorCode);
 732         trans->labelToUnicode(input, uTL, uTLInfo, errorCode);
 733         nontrans->labelToASCII(input, aNL, aNLInfo, errorCode);
 734         nontrans->labelToUnicode(input, uNL, uNLInfo, errorCode);
 735         if(errorCode.errIfFailureAndReset("labelToXYZ processing [%d/%s] %s",
 736                                           (int)i, testCase.o, testCase.s)
 737         ) {
 738             continue;
 739         }
 740         if(aN.indexOf((UChar)0x2e)<0) {
 741             if(aN!=aNL || aNInfo.getErrors()!=aNLInfo.getErrors()) {
 742                 prettify(aN).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer));
 743                 prettify(aNL).extract(0, 0x7fffffff, buffer2, UPRV_LENGTHOF(buffer2));
 744                 errln("N.nameToASCII([%d] %s)!=N.labelToASCII() "
 745                       "(errors %04lx vs %04lx) %s vs. %s",
 746                       (int)i, testCase.s, aNInfo.getErrors(), aNLInfo.getErrors(), buffer, buffer2);
 747                 continue;
 748             }
 749         } else {
 750             if((aNLInfo.getErrors()&UIDNA_ERROR_LABEL_HAS_DOT)==0) {
 751                 errln("N.labelToASCII([%d] %s) errors %04lx missing UIDNA_ERROR_LABEL_HAS_DOT",
 752                       (int)i, testCase.s, (long)aNLInfo.getErrors());
 753                 continue;
 754             }
 755         }
 756         if(aT.indexOf((UChar)0x2e)<0) {
 757             if(aT!=aTL || aTInfo.getErrors()!=aTLInfo.getErrors()) {
 758                 prettify(aT).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer));
 759                 prettify(aTL).extract(0, 0x7fffffff, buffer2, UPRV_LENGTHOF(buffer2));
 760                 errln("T.nameToASCII([%d] %s)!=T.labelToASCII() "
 761                       "(errors %04lx vs %04lx) %s vs. %s",
 762                       (int)i, testCase.s, aTInfo.getErrors(), aTLInfo.getErrors(), buffer, buffer2);
 763                 continue;
 764             }
 765         } else {
 766             if((aTLInfo.getErrors()&UIDNA_ERROR_LABEL_HAS_DOT)==0) {
 767                 errln("T.labelToASCII([%d] %s) errors %04lx missing UIDNA_ERROR_LABEL_HAS_DOT",
 768                       (int)i, testCase.s, (long)aTLInfo.getErrors());
 769                 continue;
 770             }
 771         }
 772         if(uN.indexOf((UChar)0x2e)<0) {
 773             if(uN!=uNL || uNInfo.getErrors()!=uNLInfo.getErrors()) {
 774                 prettify(uN).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer));
 775                 prettify(uNL).extract(0, 0x7fffffff, buffer2, UPRV_LENGTHOF(buffer2));
 776                 errln("N.nameToUnicode([%d] %s)!=N.labelToUnicode() "
 777                       "(errors %04lx vs %04lx) %s vs. %s",
 778                       (int)i, testCase.s, uNInfo.getErrors(), uNLInfo.getErrors(), buffer, buffer2);
 779                 continue;
 780             }
 781         } else {
 782             if((uNLInfo.getErrors()&UIDNA_ERROR_LABEL_HAS_DOT)==0) {
 783                 errln("N.labelToUnicode([%d] %s) errors %04lx missing UIDNA_ERROR_LABEL_HAS_DOT",
 784                       (int)i, testCase.s, (long)uNLInfo.getErrors());
 785                 continue;
 786             }
 787         }
 788         if(uT.indexOf((UChar)0x2e)<0) {
 789             if(uT!=uTL || uTInfo.getErrors()!=uTLInfo.getErrors()) {
 790                 prettify(uT).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer));
 791                 prettify(uTL).extract(0, 0x7fffffff, buffer2, UPRV_LENGTHOF(buffer2));
 792                 errln("T.nameToUnicode([%d] %s)!=T.labelToUnicode() "
 793                       "(errors %04lx vs %04lx) %s vs. %s",
 794                       (int)i, testCase.s, uTInfo.getErrors(), uTLInfo.getErrors(), buffer, buffer2);
 795                 continue;
 796             }
 797         } else {
 798             if((uTLInfo.getErrors()&UIDNA_ERROR_LABEL_HAS_DOT)==0) {
 799                 errln("T.labelToUnicode([%d] %s) errors %04lx missing UIDNA_ERROR_LABEL_HAS_DOT",
 800                       (int)i, testCase.s, (long)uTLInfo.getErrors());
 801                 continue;
 802             }
 803         }
 804         // Differences between transitional and nontransitional processing
 805         if(mode=='B') {
 806             if( aNInfo.isTransitionalDifferent() ||
 807                 aTInfo.isTransitionalDifferent() ||
 808                 uNInfo.isTransitionalDifferent() ||
 809                 uTInfo.isTransitionalDifferent() ||
 810                 aNLInfo.isTransitionalDifferent() ||
 811                 aTLInfo.isTransitionalDifferent() ||
 812                 uNLInfo.isTransitionalDifferent() ||
 813                 uTLInfo.isTransitionalDifferent()
 814             ) {
 815                 errln("B.process([%d] %s) isTransitionalDifferent()", (int)i, testCase.s);
 816                 continue;
 817             }
 818             if( aN!=aT || uN!=uT || aNL!=aTL || uNL!=uTL ||
 819                 aNInfo.getErrors()!=aTInfo.getErrors() || uNInfo.getErrors()!=uTInfo.getErrors() ||
 820                 aNLInfo.getErrors()!=aTLInfo.getErrors() || uNLInfo.getErrors()!=uTLInfo.getErrors()
 821             ) {
 822                 errln("N.process([%d] %s) vs. T.process() different errors or result strings",
 823                       (int)i, testCase.s);
 824                 continue;
 825             }
 826         } else {
 827             if( !aNInfo.isTransitionalDifferent() ||
 828                 !aTInfo.isTransitionalDifferent() ||
 829                 !uNInfo.isTransitionalDifferent() ||
 830                 !uTInfo.isTransitionalDifferent() ||
 831                 !aNLInfo.isTransitionalDifferent() ||
 832                 !aTLInfo.isTransitionalDifferent() ||
 833                 !uNLInfo.isTransitionalDifferent() ||
 834                 !uTLInfo.isTransitionalDifferent()
 835             ) {
 836                 errln("%s.process([%d] %s) !isTransitionalDifferent()",
 837                       testCase.o, (int)i, testCase.s);
 838                 continue;
 839             }
 840             if(aN==aT || uN==uT || aNL==aTL || uNL==uTL) {
 841                 errln("N.process([%d] %s) vs. T.process() same result strings",
 842                       (int)i, testCase.s);
 843                 continue;
 844             }
 845         }
 846         // UTF-8
 847         std::string input8, aT8, uT8, aN8, uN8;
 848         StringByteSink<std::string> aT8Sink(&aT8), uT8Sink(&uT8), aN8Sink(&aN8), uN8Sink(&uN8);
 849         IDNAInfo aT8Info, uT8Info, aN8Info, uN8Info;
 850         input.toUTF8String(input8);
 851         trans->nameToASCII_UTF8(input8, aT8Sink, aT8Info, errorCode);
 852         trans->nameToUnicodeUTF8(input8, uT8Sink, uT8Info, errorCode);
 853         nontrans->nameToASCII_UTF8(input8, aN8Sink, aN8Info, errorCode);
 854         nontrans->nameToUnicodeUTF8(input8, uN8Sink, uN8Info, errorCode);
 855         if(errorCode.errIfFailureAndReset("UTF-8 processing [%d/%s] %s",
 856                                           (int)i, testCase.o, testCase.s)
 857         ) {
 858             continue;
 859         }
 860         UnicodeString aT16(UnicodeString::fromUTF8(aT8));
 861         UnicodeString uT16(UnicodeString::fromUTF8(uT8));
 862         UnicodeString aN16(UnicodeString::fromUTF8(aN8));
 863         UnicodeString uN16(UnicodeString::fromUTF8(uN8));
 864         if( aN8Info.getErrors()!=aNInfo.getErrors() ||
 865             uN8Info.getErrors()!=uNInfo.getErrors()
 866         ) {
 867             errln("N.xyzUTF8([%d] %s) vs. UTF-16 processing different errors %04lx vs. %04lx",
 868                   (int)i, testCase.s,
 869                   (long)aN8Info.getErrors(), (long)aNInfo.getErrors());
 870             continue;
 871         }
 872         if( aT8Info.getErrors()!=aTInfo.getErrors() ||
 873             uT8Info.getErrors()!=uTInfo.getErrors()
 874         ) {
 875             errln("T.xyzUTF8([%d] %s) vs. UTF-16 processing different errors %04lx vs. %04lx",
 876                   (int)i, testCase.s,
 877                   (long)aT8Info.getErrors(), (long)aTInfo.getErrors());
 878             continue;
 879         }
 880         if(aT16!=aT || uT16!=uT || aN16!=aN || uN16!=uN) {
 881             errln("%s.xyzUTF8([%d] %s) vs. UTF-16 processing different string results",
 882                   testCase.o, (int)i, testCase.s, (long)aTInfo.getErrors());
 883             continue;
 884         }
 885         if( aT8Info.isTransitionalDifferent()!=aTInfo.isTransitionalDifferent() ||
 886             uT8Info.isTransitionalDifferent()!=uTInfo.isTransitionalDifferent() ||
 887             aN8Info.isTransitionalDifferent()!=aNInfo.isTransitionalDifferent() ||
 888             uN8Info.isTransitionalDifferent()!=uNInfo.isTransitionalDifferent()
 889         ) {
 890             errln("%s.xyzUTF8([%d] %s) vs. UTF-16 processing different isTransitionalDifferent()",
 891                   testCase.o, (int)i, testCase.s);
 892             continue;
 893         }
 894     }
 895 }
 896
 897 namespace {
 898
 899 const int32_t kNumFields = 7;
 900
 901 void U_CALLCONV
 902 idnaTestLineFn(void *context,
 903                char *fields[][2], int32_t /* fieldCount */,
 904                UErrorCode *pErrorCode) {
 905     reinterpret_cast<UTS46Test *>(context)->idnaTestOneLine(fields, *pErrorCode);
 906 }
 907
 908 UnicodeString s16FromField(char *(&field)[2]) {
 909     int32_t length = (int32_t)(field[1] - field[0]);
 910     return UnicodeString::fromUTF8(StringPiece(field[0], length)).trim().unescape();
 911 }
 912
 913 std::string statusFromField(char *(&field)[2]) {
 914     const char *start = u_skipWhitespace(field[0]);
 915     std::string status;
 916     if (start != field[1]) {
 917         int32_t length = (int32_t)(field[1] - start);
 918         while (length > 0 && (start[length - 1] == u' ' || start[length - 1] == u'\t')) {
 919             --length;
 920         }
 921         status.assign(start, length);
 922     }
 923     return status;
 924 }
 925
 926 }  // namespace
 927
 928 void UTS46Test::checkIdnaTestResult(const char *line, const char *type,
 929                                     const UnicodeString &expected, const UnicodeString &result,
 930                                     const char *status, const IDNAInfo &info) {
 931     // An error in toUnicode or toASCII is indicated by a value in square brackets,
 932     // such as "[B5 B6]".
 933     UBool expectedHasErrors = FALSE;
 934     if (*status != 0) {
 935         if (*status != u'[') {
 936             errln("%s  status field does not start with '[': %s\n    %s", type, status, line);
 937         }
 938         if (strcmp(status, u8"[]") != 0) {
 939             expectedHasErrors = TRUE;
 940         }
 941     }
 942     if (expectedHasErrors != info.hasErrors()) {
 943         errln("%s  expected errors %s %d != %d = actual has errors: %04lx\n    %s",
 944               type, status, expectedHasErrors, info.hasErrors(), (long)info.getErrors(), line);
 945     }
 946     if (!expectedHasErrors && expected != result) {
 947         errln("%s  expected != actual\n    %s", type, line);
 948         errln(UnicodeString(u"    ") + expected);
 949         errln(UnicodeString(u"    ") + result);
 950     }
 951 }
 952
 953 void UTS46Test::idnaTestOneLine(char *fields[][2], UErrorCode &errorCode) {
 954     // IdnaTestV2.txt (since Unicode 11)
 955     // Column 1: source
 956     // The source string to be tested
 957     UnicodeString source = s16FromField(fields[0]);
 958
 959     // Column 2: toUnicode
 960     // The result of applying toUnicode to the source, with Transitional_Processing=false.
 961     // A blank value means the same as the source value.
 962     UnicodeString toUnicode = s16FromField(fields[1]);
 963     if (toUnicode.isEmpty()) {
 964         toUnicode = source;
 965     }
 966
 967     // Column 3: toUnicodeStatus
 968     // A set of status codes, each corresponding to a particular test.
 969     // A blank value means [].
 970     std::string toUnicodeStatus = statusFromField(fields[2]);
 971
 972     // Column 4: toAsciiN
 973     // The result of applying toASCII to the source, with Transitional_Processing=false.
 974     // A blank value means the same as the toUnicode value.
 975     UnicodeString toAsciiN = s16FromField(fields[3]);
 976     if (toAsciiN.isEmpty()) {
 977         toAsciiN = toUnicode;
 978     }
 979
 980     // Column 5: toAsciiNStatus
 981     // A set of status codes, each corresponding to a particular test.
 982     // A blank value means the same as the toUnicodeStatus value.
 983     std::string toAsciiNStatus = statusFromField(fields[4]);
 984     if (toAsciiNStatus.empty()) {
 985         toAsciiNStatus = toUnicodeStatus;
 986     }
 987
 988     // Column 6: toAsciiT
 989     // The result of applying toASCII to the source, with Transitional_Processing=true.
 990     // A blank value means the same as the toAsciiN value.
 991     UnicodeString toAsciiT = s16FromField(fields[5]);
 992     if (toAsciiT.isEmpty()) {
 993         toAsciiT = toAsciiN;
 994     }
 995
 996     // Column 7: toAsciiTStatus
 997     // A set of status codes, each corresponding to a particular test.
 998     // A blank value means the same as the toAsciiNStatus value.
 999     std::string toAsciiTStatus = statusFromField(fields[6]);
1000     if (toAsciiTStatus.empty()) {
1001         toAsciiTStatus = toAsciiNStatus;
1002     }
1003
1004     // ToASCII/ToUnicode, transitional/nontransitional
1005     UnicodeString uN, aN, aT;
1006     IDNAInfo uNInfo, aNInfo, aTInfo;
1007     nontrans->nameToUnicode(source, uN, uNInfo, errorCode);
1008     checkIdnaTestResult(fields[0][0], "toUnicodeNontrans", toUnicode, uN,
1009                         toUnicodeStatus.c_str(), uNInfo);
1010     nontrans->nameToASCII(source, aN, aNInfo, errorCode);
1011     checkIdnaTestResult(fields[0][0], "toASCIINontrans", toAsciiN, aN,
1012                         toAsciiNStatus.c_str(), aNInfo);
1013     trans->nameToASCII(source, aT, aTInfo, errorCode);
1014     checkIdnaTestResult(fields[0][0], "toASCIITrans", toAsciiT, aT,
1015                         toAsciiTStatus.c_str(), aTInfo);
1016 }
1017
1018 // TODO: de-duplicate
1019 U_DEFINE_LOCAL_OPEN_POINTER(LocalStdioFilePointer, FILE, fclose);
1020
1021 // http://www.unicode.org/Public/idna/latest/IdnaTest.txt
1022 void UTS46Test::IdnaTest() {
1023     IcuTestErrorCode errorCode(*this, "IdnaTest");
1024     const char *sourceTestDataPath = getSourceTestData(errorCode);
1025     if (errorCode.errIfFailureAndReset("unable to find the source/test/testdata "
1026                                        "folder (getSourceTestData())")) {
1027         return;
1028     }
1029     CharString path(sourceTestDataPath, errorCode);
1030     path.appendPathPart("IdnaTestV2.txt", errorCode);
1031     LocalStdioFilePointer idnaTestFile(fopen(path.data(), "r"));
1032     if (idnaTestFile.isNull()) {
1033         errln("unable to open %s", path.data());
1034         return;
1035     }
1036
1037     // Columns (c1, c2,...) are separated by semicolons.
1038     // Leading and trailing spaces and tabs in each column are ignored.
1039     // Comments are indicated with hash marks.
1040     char *fields[kNumFields][2];
1041     u_parseDelimitedFile(path.data(), ';', fields, kNumFields, idnaTestLineFn, this, errorCode);
1042     if (errorCode.errIfFailureAndReset("error parsing IdnaTest.txt")) {
1043         return;
1044     }
1045 }
1046
1047 #endif  // UCONFIG_NO_IDNA