X-Git-Url: https://git.saurik.com/apple/icu.git/blobdiff_plain/b75a7d8f3b4adbae880cab104ce2c6a50eee4db2..151279e3792e85d0417b499c229886b3af724f55:/icuSources/test/intltest/usettest.cpp diff --git a/icuSources/test/intltest/usettest.cpp b/icuSources/test/intltest/usettest.cpp index 8acedd34..abc40039 100644 --- a/icuSources/test/intltest/usettest.cpp +++ b/icuSources/test/intltest/usettest.cpp @@ -1,20 +1,37 @@ /* -********************************************************************** -* Copyright (C) 1999-2003 Alan Liu ,International Business Machines Corporation and +******************************************************************************** +* Copyright (C) 1999-2012 International Business Machines Corporation and * others. All Rights Reserved. -********************************************************************** +******************************************************************************** * Date Name Description * 10/20/99 alan Creation. * 03/22/2000 Madhu Added additional tests -********************************************************************** +******************************************************************************** */ +#include + +#include #include "unicode/utypes.h" #include "usettest.h" +#include "unicode/ucnv.h" #include "unicode/uniset.h" #include "unicode/uchar.h" #include "unicode/usetiter.h" #include "unicode/ustring.h" +#include "unicode/parsepos.h" +#include "unicode/symtable.h" +#include "unicode/uversion.h" +#include "hash.h" + +#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) + +#define TEST_ASSERT_SUCCESS(status) {if (U_FAILURE(status)) { \ + dataerrln("fail in file \"%s\", line %d: \"%s\"", __FILE__, __LINE__, \ + u_errorName(status));}} + +#define TEST_ASSERT(expr) {if (!(expr)) { \ + dataerrln("fail in file \"%s\", line %d", __FILE__, __LINE__); }} UnicodeString operator+(const UnicodeString& left, const UnicodeSet& set) { UnicodeString pat; @@ -26,11 +43,26 @@ UnicodeString operator+(const UnicodeString& left, const UnicodeSet& set) { name = #test; \ if (exec) { \ logln(#test "---"); \ - logln((UnicodeString)""); \ + logln(); \ test(); \ } \ break +UnicodeSetTest::UnicodeSetTest() : utf8Cnv(NULL) { +} + +UConverter *UnicodeSetTest::openUTF8Converter() { + if(utf8Cnv==NULL) { + UErrorCode errorCode=U_ZERO_ERROR; + utf8Cnv=ucnv_open("UTF-8", &errorCode); + } + return utf8Cnv; +} + +UnicodeSetTest::~UnicodeSetTest() { + ucnv_close(utf8Cnv); +} + void UnicodeSetTest::runIndexedTest(int32_t index, UBool exec, const char* &name, char* /*par*/) { @@ -49,15 +81,23 @@ UnicodeSetTest::runIndexedTest(int32_t index, UBool exec, CASE(10,TestToPattern); CASE(11,TestIndexOf); CASE(12,TestStrings); - CASE(13,TestStringPatterns); - CASE(14,Testj2268); - CASE(15,TestCloseOver); - CASE(16,TestEscapePattern); - CASE(17,TestInvalidCodePoint); + CASE(13,Testj2268); + CASE(14,TestCloseOver); + CASE(15,TestEscapePattern); + CASE(16,TestInvalidCodePoint); + CASE(17,TestSymbolTable); + CASE(18,TestSurrogate); + CASE(19,TestPosixClasses); + CASE(20,TestIteration); + CASE(21,TestFreezable); + CASE(22,TestSpan); + CASE(23,TestStringSpan); default: name = ""; break; } } +static const char NOT[] = "%%%%"; + /** * UVector was improperly copying contents * This code will crash this is still true @@ -71,42 +111,97 @@ void UnicodeSetTest::Testj2268() { } /** - * Test that toPattern() round trips with syntax characters and - * whitespace. + * Test toPattern(). */ void UnicodeSetTest::TestToPattern() { - static const char* OTHER_TOPATTERN_TESTS[] = { - "[[:latin:]&[:greek:]]", - "[[:latin:]-[:greek:]]", - "[:nonspacing mark:]", - NULL - }; - - for (int32_t j=0; OTHER_TOPATTERN_TESTS[j]!=NULL; ++j) { - UErrorCode ec = U_ZERO_ERROR; - UnicodeSet s(OTHER_TOPATTERN_TESTS[j], ec); - if (U_FAILURE(ec)) { - errln((UnicodeString)"FAIL: bad pattern " + OTHER_TOPATTERN_TESTS[j]); - continue; - } - checkPat(OTHER_TOPATTERN_TESTS[j], s); - } - - for (UChar32 i = 0; i <= 0x10FFFF; ++i) { - if ((i <= 0xFF && !u_isalpha(i)) || u_isspace(i)) { + UErrorCode ec = U_ZERO_ERROR; - // check various combinations to make sure they all work. - if (i != 0 && !toPatternAux(i, i)){ - continue; - } - if (!toPatternAux(0, i)){ + // Test that toPattern() round trips with syntax characters and + // whitespace. + { + static const char* OTHER_TOPATTERN_TESTS[] = { + "[[:latin:]&[:greek:]]", + "[[:latin:]-[:greek:]]", + "[:nonspacing mark:]", + NULL + }; + + for (int32_t j=0; OTHER_TOPATTERN_TESTS[j]!=NULL; ++j) { + ec = U_ZERO_ERROR; + UnicodeSet s(OTHER_TOPATTERN_TESTS[j], ec); + if (U_FAILURE(ec)) { + dataerrln((UnicodeString)"FAIL: bad pattern " + OTHER_TOPATTERN_TESTS[j] + " - " + UnicodeString(u_errorName(ec))); continue; } - if (!toPatternAux(i, 0xFFFF)){ - continue; + checkPat(OTHER_TOPATTERN_TESTS[j], s); + } + + for (UChar32 i = 0; i <= 0x10FFFF; ++i) { + if ((i <= 0xFF && !u_isalpha(i)) || u_isspace(i)) { + + // check various combinations to make sure they all work. + if (i != 0 && !toPatternAux(i, i)){ + continue; + } + if (!toPatternAux(0, i)){ + continue; + } + if (!toPatternAux(i, 0xFFFF)){ + continue; + } } } } + + // Test pattern behavior of multicharacter strings. + { + ec = U_ZERO_ERROR; + UnicodeSet* s = new UnicodeSet("[a-z {aa} {ab}]", ec); + + // This loop isn't a loop. It's here to make the compiler happy. + // If you're curious, try removing it and changing the 'break' + // statements (except for the last) to goto's. + for (;;) { + if (U_FAILURE(ec)) break; + const char* exp1[] = {"aa", "ab", NOT, "ac", NULL}; + expectToPattern(*s, "[a-z{aa}{ab}]", exp1); + + s->add("ac"); + const char* exp2[] = {"aa", "ab", "ac", NOT, "xy", NULL}; + expectToPattern(*s, "[a-z{aa}{ab}{ac}]", exp2); + + s->applyPattern(UNICODE_STRING_SIMPLE("[a-z {\\{l} {r\\}}]"), ec); + if (U_FAILURE(ec)) break; + const char* exp3[] = {"{l", "r}", NOT, "xy", NULL}; + expectToPattern(*s, UNICODE_STRING_SIMPLE("[a-z{r\\}}{\\{l}]"), exp3); + + s->add("[]"); + const char* exp4[] = {"{l", "r}", "[]", NOT, "xy", NULL}; + expectToPattern(*s, UNICODE_STRING_SIMPLE("[a-z{\\[\\]}{r\\}}{\\{l}]"), exp4); + + s->applyPattern(UNICODE_STRING_SIMPLE("[a-z {\\u4E01\\u4E02}{\\n\\r}]"), ec); + if (U_FAILURE(ec)) break; + const char* exp5[] = {"\\u4E01\\u4E02", "\n\r", NULL}; + expectToPattern(*s, UNICODE_STRING_SIMPLE("[a-z{\\u000A\\u000D}{\\u4E01\\u4E02}]"), exp5); + + // j2189 + s->clear(); + s->add(UnicodeString("abc", "")); + s->add(UnicodeString("abc", "")); + const char* exp6[] = {"abc", NOT, "ab", NULL}; + expectToPattern(*s, "[{abc}]", exp6); + + break; + } + + if (U_FAILURE(ec)) errln("FAIL: pattern parse error"); + delete s; + } + + // JB#3400: For 2 character ranges prefer [ab] to [a-b] + UnicodeSet s; + s.add((UChar)97, (UChar)98); // 'a', 'b' + expectToPattern(s, "[ab]", NULL); } UBool UnicodeSetTest::toPatternAux(UChar32 start, UChar32 end) { @@ -186,7 +281,8 @@ UnicodeSetTest::TestCategories(void) { const char* pat = " [:Lu:] "; // Whitespace ok outside [:..:] UnicodeSet set(pat, status); if (U_FAILURE(status)) { - errln((UnicodeString)"Fail: Can't construct set with " + pat); + dataerrln((UnicodeString)"Fail: Can't construct set with " + pat + " - " + UnicodeString(u_errorName(status))); + return; } else { expectContainment(set, pat, "ABC", "abc"); } @@ -223,14 +319,14 @@ UnicodeSetTest::TestCloneEqualHash(void) { // set1 and set2 used to be built with the obsolete constructor taking // UCharCategory values; replaced with pattern constructors // markus 20030502 - UnicodeSet *set1=new UnicodeSet("\\p{Lowercase Letter}", status); // :Ll: Letter, lowercase - UnicodeSet *set1a=new UnicodeSet("[:Ll:]", status); // Letter, lowercase + UnicodeSet *set1=new UnicodeSet(UNICODE_STRING_SIMPLE("\\p{Lowercase Letter}"), status); // :Ll: Letter, lowercase + UnicodeSet *set1a=new UnicodeSet(UNICODE_STRING_SIMPLE("[:Ll:]"), status); // Letter, lowercase if (U_FAILURE(status)){ - errln((UnicodeString)"FAIL: Can't construst set with category->Ll"); + dataerrln((UnicodeString)"FAIL: Can't construst set with category->Ll" + " - " + UnicodeString(u_errorName(status))); return; } - UnicodeSet *set2=new UnicodeSet("\\p{Decimal Number}", status); //Number, Decimal digit - UnicodeSet *set2a=new UnicodeSet("[:Nd:]", status); //Number, Decimal digit + UnicodeSet *set2=new UnicodeSet(UNICODE_STRING_SIMPLE("\\p{Decimal Number}"), status); //Number, Decimal digit + UnicodeSet *set2a=new UnicodeSet(UNICODE_STRING_SIMPLE("[:Nd:]"), status); //Number, Decimal digit if (U_FAILURE(status)){ errln((UnicodeString)"FAIL: Can't construct set with category->Nd"); return; @@ -293,6 +389,9 @@ UnicodeSetTest::TestAddRemove(void) { UnicodeSet set; // Construct empty set doAssert(set.isEmpty() == TRUE, "set should be empty"); doAssert(set.size() == 0, "size should be 0"); + set.complement(); + doAssert(set.size() == 0x110000, "size should be 0x110000"); + set.clear(); set.add(0x0061, 0x007a); expectPairs(set, "az"); doAssert(set.isEmpty() == FALSE, "set should not be empty"); @@ -482,7 +581,8 @@ void UnicodeSetTest::TestAPI() { logln((UnicodeString)"a [3-10]: " + a); logln((UnicodeString)"b [7-15]: " + b); - c = a; c.addAll(b); + c = a; + c.addAll(b); UnicodeSet exp((UChar32)3,(UChar32)15); if (c == exp) { logln((UnicodeString)"c.set(a).add(b): " + c); @@ -504,7 +604,8 @@ void UnicodeSetTest::TestAPI() { } else { errln((UnicodeString)"FAIL: c.complement() = " + c + ", expect " + exp); } - c = a; c.complementAll(b); + c = a; + c.complementAll(b); exp.set((UChar32)3,(UChar32)6); exp.add((UChar32)11,(UChar32) 15); if (c == exp) { @@ -596,7 +697,131 @@ void UnicodeSetTest::TestAPI() { errln("FAIL: serialize"); return; } + + // Conversions to and from USet + UnicodeSet *uniset = &set; + USet *uset = uniset->toUSet(); + TEST_ASSERT((void *)uset == (void *)uniset); + UnicodeSet *setx = UnicodeSet::fromUSet(uset); + TEST_ASSERT((void *)setx == (void *)uset); + const UnicodeSet *constSet = uniset; + const USet *constUSet = constSet->toUSet(); + TEST_ASSERT((void *)constUSet == (void *)constSet); + const UnicodeSet *constSetx = UnicodeSet::fromUSet(constUSet); + TEST_ASSERT((void *)constSetx == (void *)constUSet); + + // span(UnicodeString) and spanBack(UnicodeString) convenience methods + UnicodeString longString=UNICODE_STRING_SIMPLE("aaaaaaaaaabbbbbbbbbbcccccccccc"); + UnicodeSet ac(0x61, 0x63); + ac.remove(0x62).freeze(); + if( ac.span(longString, -5, USET_SPAN_CONTAINED)!=10 || + ac.span(longString, 0, USET_SPAN_CONTAINED)!=10 || + ac.span(longString, 5, USET_SPAN_CONTAINED)!=10 || + ac.span(longString, 10, USET_SPAN_CONTAINED)!=10 || + ac.span(longString, 15, USET_SPAN_CONTAINED)!=15 || + ac.span(longString, 20, USET_SPAN_CONTAINED)!=30 || + ac.span(longString, 25, USET_SPAN_CONTAINED)!=30 || + ac.span(longString, 30, USET_SPAN_CONTAINED)!=30 || + ac.span(longString, 35, USET_SPAN_CONTAINED)!=30 || + ac.span(longString, INT32_MAX, USET_SPAN_CONTAINED)!=30 + ) { + errln("UnicodeSet.span(UnicodeString, ...) returns incorrect end indexes"); + } + if( ac.spanBack(longString, -5, USET_SPAN_CONTAINED)!=0 || + ac.spanBack(longString, 0, USET_SPAN_CONTAINED)!=0 || + ac.spanBack(longString, 5, USET_SPAN_CONTAINED)!=0 || + ac.spanBack(longString, 10, USET_SPAN_CONTAINED)!=0 || + ac.spanBack(longString, 15, USET_SPAN_CONTAINED)!=15 || + ac.spanBack(longString, 20, USET_SPAN_CONTAINED)!=20 || + ac.spanBack(longString, 25, USET_SPAN_CONTAINED)!=20 || + ac.spanBack(longString, 30, USET_SPAN_CONTAINED)!=20 || + ac.spanBack(longString, 35, USET_SPAN_CONTAINED)!=20 || + ac.spanBack(longString, INT32_MAX, USET_SPAN_CONTAINED)!=20 + ) { + errln("UnicodeSet.spanBack(UnicodeString, ...) returns incorrect start indexes"); + } +} + +void UnicodeSetTest::TestIteration() { + UErrorCode ec = U_ZERO_ERROR; + int i = 0; + int outerLoop; + + // 6 code points, 3 ranges, 2 strings, 8 total elements + // Iteration will access them in sorted order - a, b, c, y, z, U0001abcd, "str1", "str2" + UnicodeSet set(UNICODE_STRING_SIMPLE("[zabyc\\U0001abcd{str1}{str2}]"), ec); + TEST_ASSERT_SUCCESS(ec); + UnicodeSetIterator it(set); + + for (outerLoop=0; outerLoop<3; outerLoop++) { + // Run the test multiple times, to check that iterator.reset() is working. + for (i=0; i<10; i++) { + UBool nextv = it.next(); + UBool isString = it.isString(); + int32_t codePoint = it.getCodepoint(); + //int32_t codePointEnd = it.getCodepointEnd(); + UnicodeString s = it.getString(); + switch (i) { + case 0: + TEST_ASSERT(nextv == TRUE); + TEST_ASSERT(isString == FALSE); + TEST_ASSERT(codePoint==0x61); + TEST_ASSERT(s == "a"); + break; + case 1: + TEST_ASSERT(nextv == TRUE); + TEST_ASSERT(isString == FALSE); + TEST_ASSERT(codePoint==0x62); + TEST_ASSERT(s == "b"); + break; + case 2: + TEST_ASSERT(nextv == TRUE); + TEST_ASSERT(isString == FALSE); + TEST_ASSERT(codePoint==0x63); + TEST_ASSERT(s == "c"); + break; + case 3: + TEST_ASSERT(nextv == TRUE); + TEST_ASSERT(isString == FALSE); + TEST_ASSERT(codePoint==0x79); + TEST_ASSERT(s == "y"); + break; + case 4: + TEST_ASSERT(nextv == TRUE); + TEST_ASSERT(isString == FALSE); + TEST_ASSERT(codePoint==0x7a); + TEST_ASSERT(s == "z"); + break; + case 5: + TEST_ASSERT(nextv == TRUE); + TEST_ASSERT(isString == FALSE); + TEST_ASSERT(codePoint==0x1abcd); + TEST_ASSERT(s == UnicodeString((UChar32)0x1abcd)); + break; + case 6: + TEST_ASSERT(nextv == TRUE); + TEST_ASSERT(isString == TRUE); + TEST_ASSERT(s == "str1"); + break; + case 7: + TEST_ASSERT(nextv == TRUE); + TEST_ASSERT(isString == TRUE); + TEST_ASSERT(s == "str2"); + break; + case 8: + TEST_ASSERT(nextv == FALSE); + break; + case 9: + TEST_ASSERT(nextv == FALSE); + break; + } + } + it.reset(); // prepare to run the iteration again. + } } + + + void UnicodeSetTest::TestStrings() { UErrorCode ec = U_ZERO_ERROR; @@ -637,65 +862,16 @@ void UnicodeSetTest::TestStrings() { } } -static const char NOT[] = "%%%%"; - -/** - * Test pattern behavior of multicharacter strings. - */ -void UnicodeSetTest::TestStringPatterns() { - UErrorCode ec = U_ZERO_ERROR; - UnicodeSet* s = new UnicodeSet("[a-z {aa} {ab}]", ec); - - // This loop isn't a loop. It's here to make the compiler happy. - // If you're curious, try removing it and changing the 'break' - // statements (except for the last) to goto's. - for (;;) { - if (U_FAILURE(ec)) break; - const char* exp1[] = {"aa", "ab", NOT, "ac", NULL}; - expectToPattern(*s, "[a-z{aa}{ab}]", exp1); - - s->add("ac"); - const char* exp2[] = {"aa", "ab", "ac", NOT, "xy", NULL}; - expectToPattern(*s, "[a-z{aa}{ab}{ac}]", exp2); - - s->applyPattern("[a-z {\\{l} {r\\}}]", ec); - if (U_FAILURE(ec)) break; - const char* exp3[] = {"{l", "r}", NOT, "xy", NULL}; - expectToPattern(*s, "[a-z{\\{l}{r\\}}]", exp3); - - s->add("[]"); - const char* exp4[] = {"{l", "r}", "[]", NOT, "xy", NULL}; - expectToPattern(*s, "[a-z{\\[\\]}{r\\}}{\\{l}]", exp4); - - s->applyPattern("[a-z {\\u4E01\\u4E02}{\\n\\r}]", ec); - if (U_FAILURE(ec)) break; - const char* exp5[] = {"\\u4E01\\u4E02", "\n\r", NULL}; - expectToPattern(*s, "[a-z{\\u4E01\\u4E02}{\\n\\r}]", exp5); - - // j2189 - s->clear(); - s->add(UnicodeString("abc", "")); - s->add(UnicodeString("abc", "")); - const char* exp6[] = {"abc", NOT, "ab", NULL}; - expectToPattern(*s, "[{abc}]", exp6); - - break; - } - - if (U_FAILURE(ec)) errln("FAIL: pattern parse error"); - delete s; -} - /** * Test the [:Latin:] syntax. */ void UnicodeSetTest::TestScriptSet() { - expectContainment("[:Latin:]", "aA", CharsToUnicodeString("\\u0391\\u03B1")); + expectContainment(UNICODE_STRING_SIMPLE("[:Latin:]"), "aA", CharsToUnicodeString("\\u0391\\u03B1")); - expectContainment("[:Greek:]", CharsToUnicodeString("\\u0391\\u03B1"), "aA"); + expectContainment(UNICODE_STRING_SIMPLE("[:Greek:]"), CharsToUnicodeString("\\u0391\\u03B1"), "aA"); /* Jitterbug 1423 */ - expectContainment("[[:Common:][:Inherited:]]", CharsToUnicodeString("\\U00003099\\U0001D169\\u0000"), "aA"); + expectContainment(UNICODE_STRING_SIMPLE("[[:Common:][:Inherited:]]"), CharsToUnicodeString("\\U00003099\\U0001D169\\u0000"), "aA"); } @@ -703,7 +879,7 @@ void UnicodeSetTest::TestScriptSet() { * Test the [:Latin:] syntax. */ void UnicodeSetTest::TestPropertySet() { - static const char* DATA[] = { + static const char* const DATA[] = { // Pattern, Chars IN, Chars NOT in "[:Latin:]", @@ -718,6 +894,7 @@ void UnicodeSetTest::TestPropertySet() { "abc", "ABC", +#if !UCONFIG_NO_NORMALIZATION // Combining class: @since ICU 2.2 // Check both symbolic and numeric "\\p{ccc=Nukta}", @@ -731,6 +908,7 @@ void UnicodeSetTest::TestPropertySet() { "[:c c c = iota subscript :]", "\\u0345", "xyz", +#endif // Bidi class: @since ICU 2.2 "\\p{bidiclass=lefttoright}", @@ -743,8 +921,10 @@ void UnicodeSetTest::TestPropertySet() { "x", "[:math=false:]", - "q", - "(*+)", + "q)*(", + // weiv: )(and * were removed from math in Unicode 4.0.1 + //"(*+)", + "+<>^", // JB#1767 \N{}, \p{ASCII} "[:Ascii:]", @@ -810,17 +990,177 @@ void UnicodeSetTest::TestPropertySet() { "[^b-]", // trailing '-' is literal "ac", - "-b" + "-b", + + "[a-b-]", // trailing '-' is literal + "ab-", + "c=", + + "[[a-q]&[p-z]-]", // trailing '-' is literal + "pq-", + "or=", + + "[\\s|\\)|:|$|\\>]", // from regex tests + "s|):$>", + "abc", + + "[\\uDC00cd]", // JB#2906: isolated trail at start + "cd\\uDC00", + "ab\\uD800\\U00010000", + + "[ab\\uD800]", // JB#2906: isolated trail at start + "ab\\uD800", + "cd\\uDC00\\U00010000", + + "[ab\\uD800cd]", // JB#2906: isolated lead in middle + "abcd\\uD800", + "ef\\uDC00\\U00010000", + + "[ab\\uDC00cd]", // JB#2906: isolated trail in middle + "abcd\\uDC00", + "ef\\uD800\\U00010000", + +#if !UCONFIG_NO_NORMALIZATION + "[:^lccc=0:]", // Lead canonical class + "\\u0300\\u0301", + "abcd\\u00c0\\u00c5", + + "[:^tccc=0:]", // Trail canonical class + "\\u0300\\u0301\\u00c0\\u00c5", + "abcd", + + "[[:^lccc=0:][:^tccc=0:]]", // Lead and trail canonical class + "\\u0300\\u0301\\u00c0\\u00c5", + "abcd", + + "[[:^lccc=0:]-[:^tccc=0:]]", // Stuff that starts with an accent but ends with a base (none right now) + "", + "abcd\\u0300\\u0301\\u00c0\\u00c5", + + "[[:ccc=0:]-[:lccc=0:]-[:tccc=0:]]", // Weirdos. Complete canonical class is zero, but both lead and trail are not + "\\u0F73\\u0F75\\u0F81", + "abcd\\u0300\\u0301\\u00c0\\u00c5", +#endif /* !UCONFIG_NO_NORMALIZATION */ + + "[:Assigned:]", + "A\\uE000\\uF8FF\\uFDC7\\U00010000\\U0010FFFD", + "\\u0888\\uFDD3\\uFFFE\\U00050005", + + // Script_Extensions, new in Unicode 6.0 + "[:scx=Arab:]", + "\\u061E\\u061F\\u0620\\u0621\\u063F\\u0640\\u0650\\u065E\\uFDF1\\uFDF2\\uFDF3", + "\\u061D\\uFDEF\\uFDFE", + + // U+FDF2 has Script=Arabic and also Arab in its Script_Extensions, + // so scx-sc is missing U+FDF2. + "[[:Script_Extensions=Arabic:]-[:Arab:]]", + "\\u0640\\u064B\\u0650\\u0655\\uFDFD", + "\\uFDF2" }; static const int32_t DATA_LEN = sizeof(DATA)/sizeof(DATA[0]); for (int32_t i=0; i " + exp); } else { - UnicodeString buf; - errln((UnicodeString)"FAIL: " + pat + ".closeOver(" + selector + ") => " + + dataerrln((UnicodeString)"FAIL: " + pat + ".closeOver(" + selector + ") => " + s.toPattern(buf, TRUE) + ", expected " + exp); } } +#if 0 + /* + * Unused test code. + * This was used to compare the old implementation (using USET_CASE) + * with the new one (using 0x100 temporarily) + * while transitioning from hardcoded case closure tables in uniset.cpp + * (moved to uniset_props.cpp) to building the data by gencase into ucase.icu. + * and using ucase.c functions for closure. + * See Jitterbug 3432 RFE: Move uniset.cpp data to a data file + * + * Note: The old and new implementation never fully matched because + * the old implementation turned out to not map U+0130 and U+0131 correctly + * (dotted I and dotless i) and because the old implementation's data tables + * were outdated compared to Unicode 4.0.1 at the time of the change to the + * new implementation. (So sigmas and some other characters were not handled + * according to the newer Unicode version.) + */ + UnicodeSet sens("[:case_sensitive:]", ec), sens2, s2; + UnicodeSetIterator si(sens); + UnicodeString str, buf2; + const UnicodeString *pStr; + UChar32 c; + while(si.next()) { + if(!si.isString()) { + c=si.getCodepoint(); + s.clear(); + s.add(c); + + str.setTo(c); + str.foldCase(); + sens2.add(str); + + t=s; + s.closeOver(USET_CASE); + t.closeOver(0x100); + if(s!=t) { + errln("FAIL: closeOver(U+%04x) differs: ", c); + errln((UnicodeString)"old "+s.toPattern(buf, TRUE)+" new: "+t.toPattern(buf2, TRUE)); + } + } + } + // remove all code points + // should contain all full case folding mapping strings + sens2.remove(0, 0x10ffff); + si.reset(sens2); + while(si.next()) { + if(si.isString()) { + pStr=&si.getString(); + s.clear(); + s.add(*pStr); + t=s2=s; + s.closeOver(USET_CASE); + t.closeOver(0x100); + if(s!=t) { + errln((UnicodeString)"FAIL: closeOver("+s2.toPattern(buf, TRUE)+") differs: "); + errln((UnicodeString)"old "+s.toPattern(buf, TRUE)+" new: "+t.toPattern(buf2, TRUE)); + } + } + } +#endif + // Test the pattern API - s.applyPattern("[abc]", USET_CASE_INSENSITIVE, ec); + s.applyPattern("[abc]", USET_CASE_INSENSITIVE, NULL, ec); if (U_FAILURE(ec)) { errln("FAIL: applyPattern failed"); } else { expectContainment(s, "abcABC", "defDEF"); } - UnicodeSet v("[^abc]", USET_CASE_INSENSITIVE, ec); + UnicodeSet v("[^abc]", USET_CASE_INSENSITIVE, NULL, ec); if (U_FAILURE(ec)) { errln("FAIL: constructor failed"); } else { expectContainment(v, "defDEF", "abcABC"); } + UnicodeSet cm("[abck]", USET_ADD_CASE_MAPPINGS, NULL, ec); + if (U_FAILURE(ec)) { + errln("FAIL: construct w/case mappings failed"); + } else { + expectContainment(cm, "abckABCK", CharsToUnicodeString("defDEF\\u212A")); + } } void UnicodeSetTest::TestEscapePattern() { const char pattern[] = - "[\\uFEFF \\uFFF9-\\uFFFC \\U0001D173-\\U0001D17A \\U000F0000-\\U000FFFFD ]"; + "[\\uFEFF \\u200A-\\u200E \\U0001D173-\\U0001D17A \\U000F0000-\\U000FFFFD ]"; const char exp[] = - "[\\uFEFF\\uFFF9-\\uFFFC\\U0001D173-\\U0001D17A\\U000F0000-\\U000FFFFD]"; + "[\\u200A-\\u200E\\uFEFF\\U0001D173-\\U0001D17A\\U000F0000-\\U000FFFFD]"; // We test this with two passes; in the second pass we - // pre-unescape the pattern. Since U+FEFF and several other code - // points are rule whitespace, this fails -- which is what we - // expect. + // pre-unescape the pattern. Since U+200E is Pattern_White_Space, + // this fails -- which is what we expect. for (int32_t pass=1; pass<=2; ++pass) { UErrorCode ec = U_ZERO_ERROR; - UnicodeString pat(pattern); + UnicodeString pat(pattern, -1, US_INV); if (pass==2) { pat = pat.unescape(); } @@ -968,7 +1415,7 @@ void UnicodeSetTest::TestEscapePattern() { UnicodeString newpat; set.toPattern(newpat, TRUE); - if (newpat == exp) { + if (newpat == UnicodeString(exp, -1, US_INV)) { logln(escape(pat) + " => " + newpat); } else { errln((UnicodeString)"FAIL: " + escape(pat) + " => " + newpat); @@ -1042,7 +1489,7 @@ void UnicodeSetTest::TestInvalidCodePoint() { b = set.containsNone(start, end); b = set.containsSome(start, end); - int32_t index = set.indexOf(start); + /*int32_t index = set.indexOf(start);*/ set.clear(); set.add(start); @@ -1120,6 +1567,157 @@ void UnicodeSetTest::TestInvalidCodePoint() { } } +// Used by TestSymbolTable +class TokenSymbolTable : public SymbolTable { +public: + Hashtable contents; + + TokenSymbolTable(UErrorCode& ec) : contents(FALSE, ec) { + contents.setValueDeleter(uprv_deleteUObject); + } + + ~TokenSymbolTable() {} + + /** + * (Non-SymbolTable API) Add the given variable and value to + * the table. Variable should NOT contain leading '$'. + */ + void add(const UnicodeString& var, const UnicodeString& value, + UErrorCode& ec) { + if (U_SUCCESS(ec)) { + contents.put(var, new UnicodeString(value), ec); + } + } + + /** + * SymbolTable API + */ + virtual const UnicodeString* lookup(const UnicodeString& s) const { + return (const UnicodeString*) contents.get(s); + } + + /** + * SymbolTable API + */ + virtual const UnicodeFunctor* lookupMatcher(UChar32 /*ch*/) const { + return NULL; + } + + /** + * SymbolTable API + */ + virtual UnicodeString parseReference(const UnicodeString& text, + ParsePosition& pos, int32_t limit) const { + int32_t start = pos.getIndex(); + int32_t i = start; + UnicodeString result; + while (i < limit) { + UChar c = text.charAt(i); + if ((i==start && !u_isIDStart(c)) || !u_isIDPart(c)) { + break; + } + ++i; + } + if (i == start) { // No valid name chars + return result; // Indicate failure with empty string + } + pos.setIndex(i); + text.extractBetween(start, i, result); + return result; + } +}; + +void UnicodeSetTest::TestSymbolTable() { + // Multiple test cases can be set up here. Each test case + // is terminated by null: + // var, value, var, value,..., input pat., exp. output pat., null + const char* DATA[] = { + "us", "a-z", "[0-1$us]", "[0-1a-z]", NULL, + "us", "[a-z]", "[0-1$us]", "[0-1[a-z]]", NULL, + "us", "\\[a\\-z\\]", "[0-1$us]", "[-01\\[\\]az]", NULL, + NULL + }; + + for (int32_t i=0; DATA[i]!=NULL; ++i) { + UErrorCode ec = U_ZERO_ERROR; + TokenSymbolTable sym(ec); + if (U_FAILURE(ec)) { + errln("FAIL: couldn't construct TokenSymbolTable"); + continue; + } + + // Set up variables + while (DATA[i+2] != NULL) { + sym.add(UnicodeString(DATA[i], -1, US_INV), UnicodeString(DATA[i+1], -1, US_INV), ec); + if (U_FAILURE(ec)) { + errln("FAIL: couldn't add to TokenSymbolTable"); + continue; + } + i += 2; + } + + // Input pattern and expected output pattern + UnicodeString inpat = UnicodeString(DATA[i], -1, US_INV), exppat = UnicodeString(DATA[i+1], -1, US_INV); + i += 2; + + ParsePosition pos(0); + UnicodeSet us(inpat, pos, USET_IGNORE_SPACE, &sym, ec); + if (U_FAILURE(ec)) { + errln("FAIL: couldn't construct UnicodeSet"); + continue; + } + + // results + if (pos.getIndex() != inpat.length()) { + errln((UnicodeString)"Failed to read to end of string \"" + + inpat + "\": read to " + + pos.getIndex() + ", length is " + + inpat.length()); + } + + UnicodeSet us2(exppat, ec); + if (U_FAILURE(ec)) { + errln("FAIL: couldn't construct expected UnicodeSet"); + continue; + } + + UnicodeString a, b; + if (us != us2) { + errln((UnicodeString)"Failed, got " + us.toPattern(a, TRUE) + + ", expected " + us2.toPattern(b, TRUE)); + } else { + logln((UnicodeString)"Ok, got " + us.toPattern(a, TRUE)); + } + } +} + +void UnicodeSetTest::TestSurrogate() { + const char* DATA[] = { + // These should all behave identically + "[abc\\uD800\\uDC00]", + // "[abc\uD800\uDC00]", // Can't do this on C -- only Java + "[abc\\U00010000]", + 0 + }; + for (int i=0; DATA[i] != 0; ++i) { + UErrorCode ec = U_ZERO_ERROR; + logln((UnicodeString)"Test pattern " + i + " :" + UnicodeString(DATA[i], -1, US_INV)); + UnicodeString str = UnicodeString(DATA[i], -1, US_INV); + UnicodeSet set(str, ec); + if (U_FAILURE(ec)) { + errln("FAIL: UnicodeSet constructor"); + continue; + } + expectContainment(set, + CharsToUnicodeString("abc\\U00010000"), + CharsToUnicodeString("\\uD800;\\uDC00")); // split apart surrogate-pair + if (set.size() != 4) { + errln((UnicodeString)"FAIL: " + UnicodeString(DATA[i], -1, US_INV) + ".size() == " + + set.size() + ", expected 4"); + } + } +} + void UnicodeSetTest::TestExhaustive() { // exhaustive tests. Simulate UnicodeSets with integers. // That gives us very solid tests (except for large memory tests). @@ -1373,7 +1971,7 @@ UnicodeSetTest::expectContainment(const UnicodeString& pat, UErrorCode ec = U_ZERO_ERROR; UnicodeSet set(pat, ec); if (U_FAILURE(ec)) { - errln((UnicodeString)"FAIL: pattern \"" + + dataerrln((UnicodeString)"FAIL: pattern \"" + pat + "\" => " + u_errorName(ec)); return; } @@ -1489,6 +2087,9 @@ void UnicodeSetTest::expectToPattern(const UnicodeSet& set, errln((UnicodeString)"FAIL: toPattern() => \"" + pat + "\", expected \"" + expPat + "\""); return; } + if (expStrings == NULL) { + return; + } UBool in = TRUE; for (int32_t i=0; expStrings[i] != NULL; ++i) { if (expStrings[i] == NOT) { // sic; pointer comparison @@ -1546,3 +2147,1671 @@ UnicodeSetTest::escape(const UnicodeString& s) { } return buf; } + +void UnicodeSetTest::TestFreezable() { + UErrorCode errorCode=U_ZERO_ERROR; + UnicodeString idPattern=UNICODE_STRING("[:ID_Continue:]", 15); + UnicodeSet idSet(idPattern, errorCode); + if(U_FAILURE(errorCode)) { + dataerrln("FAIL: unable to create UnicodeSet([:ID_Continue:]) - %s", u_errorName(errorCode)); + return; + } + + UnicodeString wsPattern=UNICODE_STRING("[:White_Space:]", 15); + UnicodeSet wsSet(wsPattern, errorCode); + if(U_FAILURE(errorCode)) { + dataerrln("FAIL: unable to create UnicodeSet([:White_Space:]) - %s", u_errorName(errorCode)); + return; + } + + idSet.add(idPattern); + UnicodeSet frozen(idSet); + frozen.freeze(); + + if(idSet.isFrozen() || !frozen.isFrozen()) { + errln("FAIL: isFrozen() is wrong"); + } + if(frozen!=idSet || !(frozen==idSet)) { + errln("FAIL: a copy-constructed frozen set differs from its original"); + } + + frozen=wsSet; + if(frozen!=idSet || !(frozen==idSet)) { + errln("FAIL: a frozen set was modified by operator="); + } + + UnicodeSet frozen2(frozen); + if(frozen2!=frozen || frozen2!=idSet) { + errln("FAIL: a copied frozen set differs from its frozen original"); + } + if(!frozen2.isFrozen()) { + errln("FAIL: copy-constructing a frozen set results in a thawed one"); + } + UnicodeSet frozen3(5, 55); // Set to some values to really test assignment below, not copy construction. + if(frozen3.contains(0, 4) || !frozen3.contains(5, 55) || frozen3.contains(56, 0x10ffff)) { + errln("FAIL: UnicodeSet(5, 55) failed"); + } + frozen3=frozen; + if(!frozen3.isFrozen()) { + errln("FAIL: copying a frozen set results in a thawed one"); + } + + UnicodeSet *cloned=(UnicodeSet *)frozen.clone(); + if(!cloned->isFrozen() || *cloned!=frozen || cloned->containsSome(0xd802, 0xd805)) { + errln("FAIL: clone() failed"); + } + cloned->add(0xd802, 0xd805); + if(cloned->containsSome(0xd802, 0xd805)) { + errln("FAIL: unable to modify clone"); + } + delete cloned; + + UnicodeSet *thawed=(UnicodeSet *)frozen.cloneAsThawed(); + if(thawed->isFrozen() || *thawed!=frozen || thawed->containsSome(0xd802, 0xd805)) { + errln("FAIL: cloneAsThawed() failed"); + } + thawed->add(0xd802, 0xd805); + if(!thawed->contains(0xd802, 0xd805)) { + errln("FAIL: unable to modify thawed clone"); + } + delete thawed; + + frozen.set(5, 55); + if(frozen!=idSet || !(frozen==idSet)) { + errln("FAIL: UnicodeSet::set() modified a frozen set"); + } + + frozen.clear(); + if(frozen!=idSet || !(frozen==idSet)) { + errln("FAIL: UnicodeSet::clear() modified a frozen set"); + } + + frozen.closeOver(USET_CASE_INSENSITIVE); + if(frozen!=idSet || !(frozen==idSet)) { + errln("FAIL: UnicodeSet::closeOver() modified a frozen set"); + } + + frozen.compact(); + if(frozen!=idSet || !(frozen==idSet)) { + errln("FAIL: UnicodeSet::compact() modified a frozen set"); + } + + ParsePosition pos; + frozen. + applyPattern(wsPattern, errorCode). + applyPattern(wsPattern, USET_IGNORE_SPACE, NULL, errorCode). + applyPattern(wsPattern, pos, USET_IGNORE_SPACE, NULL, errorCode). + applyIntPropertyValue(UCHAR_CANONICAL_COMBINING_CLASS, 230, errorCode). + applyPropertyAlias(UNICODE_STRING_SIMPLE("Assigned"), UnicodeString(), errorCode); + if(frozen!=idSet || !(frozen==idSet)) { + errln("FAIL: UnicodeSet::applyXYZ() modified a frozen set"); + } + + frozen. + add(0xd800). + add(0xd802, 0xd805). + add(wsPattern). + addAll(idPattern). + addAll(wsSet); + if(frozen!=idSet || !(frozen==idSet)) { + errln("FAIL: UnicodeSet::addXYZ() modified a frozen set"); + } + + frozen. + retain(0x62). + retain(0x64, 0x69). + retainAll(wsPattern). + retainAll(wsSet); + if(frozen!=idSet || !(frozen==idSet)) { + errln("FAIL: UnicodeSet::retainXYZ() modified a frozen set"); + } + + frozen. + remove(0x62). + remove(0x64, 0x69). + remove(idPattern). + removeAll(idPattern). + removeAll(idSet); + if(frozen!=idSet || !(frozen==idSet)) { + errln("FAIL: UnicodeSet::removeXYZ() modified a frozen set"); + } + + frozen. + complement(). + complement(0x62). + complement(0x64, 0x69). + complement(idPattern). + complementAll(idPattern). + complementAll(idSet); + if(frozen!=idSet || !(frozen==idSet)) { + errln("FAIL: UnicodeSet::complementXYZ() modified a frozen set"); + } +} + +// Test span() etc. -------------------------------------------------------- *** + +// Append the UTF-8 version of the string to t and return the appended UTF-8 length. +static int32_t +appendUTF8(const UChar *s, int32_t length, char *t, int32_t capacity) { + UErrorCode errorCode=U_ZERO_ERROR; + int32_t length8=0; + u_strToUTF8(t, capacity, &length8, s, length, &errorCode); + if(U_SUCCESS(errorCode)) { + return length8; + } else { + // The string contains an unpaired surrogate. + // Ignore this string. + return 0; + } +} + +class UnicodeSetWithStringsIterator; + +// Make the strings in a UnicodeSet easily accessible. +class UnicodeSetWithStrings { +public: + UnicodeSetWithStrings(const UnicodeSet &normalSet) : + set(normalSet), stringsLength(0), hasSurrogates(FALSE) { + int32_t size=set.size(); + if(size>0 && set.charAt(size-1)<0) { + // If a set's last element is not a code point, then it must contain strings. + // Iterate over the set, skip all code point ranges, and cache the strings. + // Convert them to UTF-8 for spanUTF8(). + UnicodeSetIterator iter(set); + const UnicodeString *s; + char *s8=utf8; + int32_t length8, utf8Count=0; + while(iter.nextRange() && stringsLengthgetBuffer(), s->length(), + s8, (int32_t)(sizeof(utf8)-utf8Count)); + if(length8==0) { + hasSurrogates=TRUE; // Contains unpaired surrogates. + } + s8+=length8; + ++stringsLength; + } + } + } + } + + const UnicodeSet &getSet() const { + return set; + } + + UBool hasStrings() const { + return (UBool)(stringsLength>0); + } + + UBool hasStringsWithSurrogates() const { + return hasSurrogates; + } + +private: + friend class UnicodeSetWithStringsIterator; + + const UnicodeSet &set; + + const UnicodeString *strings[20]; + int32_t stringsLength; + UBool hasSurrogates; + + char utf8[1024]; + int32_t utf8Lengths[20]; + + int32_t nextStringIndex; + int32_t nextUTF8Start; +}; + +class UnicodeSetWithStringsIterator { +public: + UnicodeSetWithStringsIterator(const UnicodeSetWithStrings &set) : + fSet(set), nextStringIndex(0), nextUTF8Start(0) { + } + + void reset() { + nextStringIndex=nextUTF8Start=0; + } + + const UnicodeString *nextString() { + if(nextStringIndexlength()<=(length-start) && matches16CPB(s, start, length, *str)) { + // spanNeedsStrings=TRUE; + return start; + } + } + start=next; + } + return start; + } else /* USET_SPAN_CONTAINED or USET_SPAN_SIMPLE */ { + UnicodeSetWithStringsIterator iter(set); + UChar32 c; + int32_t start, next, maxSpanLimit=0; + for(start=next=0; startlength()<=(length-start) && matches16CPB(s, start, length, *str)) { + // spanNeedsStrings=TRUE; + int32_t matchLimit=start+str->length(); + if(matchLimit==length) { + return length; + } + if(spanCondition==USET_SPAN_CONTAINED) { + // Iterate for the shortest match at each position. + // Recurse for each but the shortest match. + if(next==start) { + next=matchLimit; // First match from start. + } else { + if(matchLimitmaxSpanLimit) { + maxSpanLimit=matchLimit+spanLength; + if(maxSpanLimit==length) { + return length; + } + } + } + } else /* spanCondition==USET_SPAN_SIMPLE */ { + if(matchLimit>next) { + // Remember longest match from start. + next=matchLimit; + } + } + } + } + if(next==start) { + break; // No match from start. + } + start=next; + } + if(start>maxSpanLimit) { + return start; + } else { + return maxSpanLimit; + } + } +} + +static int32_t containsSpanBackUTF16(const UnicodeSetWithStrings &set, const UChar *s, int32_t length, + USetSpanCondition spanCondition) { + if(length==0) { + return 0; + } + const UnicodeSet &realSet(set.getSet()); + if(!set.hasStrings()) { + if(spanCondition!=USET_SPAN_NOT_CONTAINED) { + spanCondition=USET_SPAN_CONTAINED; // Pin to 0/1 values. + } + + UChar32 c; + int32_t prev=length; + do { + U16_PREV(s, 0, length, c); + if(realSet.contains(c)!=spanCondition) { + break; + } + } while((prev=length)>0); + return prev; + } else if(spanCondition==USET_SPAN_NOT_CONTAINED) { + UnicodeSetWithStringsIterator iter(set); + UChar32 c; + int32_t prev=length, length0=length; + do { + U16_PREV(s, 0, length, c); + if(realSet.contains(c)) { + break; + } + const UnicodeString *str; + iter.reset(); + while((str=iter.nextString())!=NULL) { + if(str->length()<=prev && matches16CPB(s, prev-str->length(), length0, *str)) { + // spanNeedsStrings=TRUE; + return prev; + } + } + } while((prev=length)>0); + return prev; + } else /* USET_SPAN_CONTAINED or USET_SPAN_SIMPLE */ { + UnicodeSetWithStringsIterator iter(set); + UChar32 c; + int32_t prev=length, minSpanStart=length, length0=length; + do { + U16_PREV(s, 0, length, c); + if(!realSet.contains(c)) { + length=prev; // Do not span this single, not-contained code point. + } + const UnicodeString *str; + iter.reset(); + while((str=iter.nextString())!=NULL) { + if(str->length()<=prev && matches16CPB(s, prev-str->length(), length0, *str)) { + // spanNeedsStrings=TRUE; + int32_t matchStart=prev-str->length(); + if(matchStart==0) { + return 0; + } + if(spanCondition==USET_SPAN_CONTAINED) { + // Iterate for the shortest match at each position. + // Recurse for each but the shortest match. + if(length==prev) { + length=matchStart; // First match from prev. + } else { + if(matchStart>length) { + // Remember shortest match from prev for iteration. + int32_t temp=length; + length=matchStart; + matchStart=temp; + } + // Recurse for non-shortest match from prev. + int32_t spanStart=containsSpanBackUTF16(set, s, matchStart, + USET_SPAN_CONTAINED); + if(spanStart0); + if(prevmaxSpanLimit) { + maxSpanLimit=matchLimit+spanLength; + if(maxSpanLimit==length) { + return length; + } + } + } + } else /* spanCondition==USET_SPAN_SIMPLE */ { + if(matchLimit>next) { + // Remember longest match from start. + next=matchLimit; + } + } + } + } + if(next==start) { + break; // No match from start. + } + start=next; + } + if(start>maxSpanLimit) { + return start; + } else { + return maxSpanLimit; + } + } +} + +static int32_t containsSpanBackUTF8(const UnicodeSetWithStrings &set, const char *s, int32_t length, + USetSpanCondition spanCondition) { + if(length==0) { + return 0; + } + const UnicodeSet &realSet(set.getSet()); + if(!set.hasStrings()) { + if(spanCondition!=USET_SPAN_NOT_CONTAINED) { + spanCondition=USET_SPAN_CONTAINED; // Pin to 0/1 values. + } + + UChar32 c; + int32_t prev=length; + do { + U8_PREV_OR_FFFD(s, 0, length, c); + if(realSet.contains(c)!=spanCondition) { + break; + } + } while((prev=length)>0); + return prev; + } else if(spanCondition==USET_SPAN_NOT_CONTAINED) { + UnicodeSetWithStringsIterator iter(set); + UChar32 c; + int32_t prev=length; + do { + U8_PREV_OR_FFFD(s, 0, length, c); + if(realSet.contains(c)) { + break; + } + const char *s8; + int32_t length8; + iter.reset(); + while((s8=iter.nextUTF8(length8))!=NULL) { + if(length8!=0 && length8<=prev && 0==memcmp(s+prev-length8, s8, length8)) { + // spanNeedsStrings=TRUE; + return prev; + } + } + } while((prev=length)>0); + return prev; + } else /* USET_SPAN_CONTAINED or USET_SPAN_SIMPLE */ { + UnicodeSetWithStringsIterator iter(set); + UChar32 c; + int32_t prev=length, minSpanStart=length; + do { + U8_PREV_OR_FFFD(s, 0, length, c); + if(!realSet.contains(c)) { + length=prev; // Do not span this single, not-contained code point. + } + const char *s8; + int32_t length8; + iter.reset(); + while((s8=iter.nextUTF8(length8))!=NULL) { + if(length8!=0 && length8<=prev && 0==memcmp(s+prev-length8, s8, length8)) { + // spanNeedsStrings=TRUE; + int32_t matchStart=prev-length8; + if(matchStart==0) { + return 0; + } + if(spanCondition==USET_SPAN_CONTAINED) { + // Iterate for the shortest match at each position. + // Recurse for each but the shortest match. + if(length==prev) { + length=matchStart; // First match from prev. + } else { + if(matchStart>length) { + // Remember shortest match from prev for iteration. + int32_t temp=length; + length=matchStart; + matchStart=temp; + } + // Recurse for non-shortest match from prev. + int32_t spanStart=containsSpanBackUTF8(set, s, matchStart, + USET_SPAN_CONTAINED); + if(spanStart0); + if(prev=length) { + break; + } + spanCondition=invertSpanCondition(spanCondition, contained); + } + break; + case 2: + case 3: + start=0; + for(;;) { + start+= isUTF16 ? realSet.span((const UChar *)s+start, length>=0 ? length-start : length, spanCondition) : + realSet.spanUTF8((const char *)s+start, length>=0 ? length-start : length, spanCondition); + if(count=0 ? start>=length : + isUTF16 ? ((const UChar *)s)[start]==0 : + ((const char *)s)[start]==0 + ) { + break; + } + spanCondition=invertSpanCondition(spanCondition, contained); + } + break; + case 4: + case 5: + if(length<0) { + length=slen(s, isUTF16); + } + for(;;) { + ++count; + if(count<=limitsCapacity) { + limits[limitsCapacity-count]=length; + } + length= isUTF16 ? containsSpanBackUTF16(set, (const UChar *)s, length, spanCondition) : + containsSpanBackUTF8(set, (const char *)s, length, spanCondition); + if(length==0 && spanCondition==firstSpanCondition) { + break; + } + spanCondition=invertSpanCondition(spanCondition, contained); + } + if(count=0 ? length : slen(s, isUTF16); + } + // Note: Length<0 is tested only for the first spanBack(). + // If we wanted to keep length<0 for all spanBack()s, we would have to + // temporarily modify the string by placing a NUL where the previous spanBack() stopped. + length= isUTF16 ? realSet.spanBack((const UChar *)s, length, spanCondition) : + realSet.spanBackUTF8((const char *)s, length, spanCondition); + if(length==0 && spanCondition==firstSpanCondition) { + break; + } + spanCondition=invertSpanCondition(spanCondition, contained); + } + if(count=0) + * or returned to the caller (with an input expectCount<0). + */ +void UnicodeSetTest::testSpan(const UnicodeSetWithStrings *sets[4], + const void *s, int32_t length, UBool isUTF16, + uint32_t whichSpans, + int32_t expectLimits[], int32_t &expectCount, + const char *testName, int32_t index) { + int32_t limits[500]; + int32_t limitsCount; + int i, j; + + const char *typeName; + int type; + + for(i=0; iLENGTHOF(limits)) { + errln("FAIL: %s[0x%lx].%s.%s span count=%ld > %ld capacity - too many spans", + testName, (long)index, setNames[i], typeName, (long)limitsCount, (long)LENGTHOF(limits)); + return; + } + memcpy(expectLimits, limits, limitsCount*4); + } else if(limitsCount!=expectCount) { + errln("FAIL: %s[0x%lx].%s.%s span count=%ld != %ld", + testName, (long)index, setNames[i], typeName, (long)limitsCount, (long)expectCount); + } else { + for(j=0; j0) { + string.setTo(FALSE, s16+prev, length); // read-only alias + if(i&1) { + if(!sets[SLOW]->getSet().containsAll(string)) { + errln("FAIL: %s[0x%lx].%s.containsAll(%ld..%ld)==FALSE contradicts span()", + testName, (long)index, setNames[SLOW], (long)prev, (long)limit); + return; + } + if(!sets[FAST]->getSet().containsAll(string)) { + errln("FAIL: %s[0x%lx].%s.containsAll(%ld..%ld)==FALSE contradicts span()", + testName, (long)index, setNames[FAST], (long)prev, (long)limit); + return; + } + } else { + if(!sets[SLOW]->getSet().containsNone(string)) { + errln("FAIL: %s[0x%lx].%s.containsNone(%ld..%ld)==FALSE contradicts span()", + testName, (long)index, setNames[SLOW], (long)prev, (long)limit); + return; + } + if(!sets[FAST]->getSet().containsNone(string)) { + errln("FAIL: %s[0x%lx].%s.containsNone(%ld..%ld)==FALSE contradicts span()", + testName, (long)index, setNames[FAST], (long)prev, (long)limit); + return; + } + } + } + prev=limit; + } + } +} + +// Specifically test either UTF-16 or UTF-8. +void UnicodeSetTest::testSpan(const UnicodeSetWithStrings *sets[4], + const void *s, int32_t length, UBool isUTF16, + uint32_t whichSpans, + const char *testName, int32_t index) { + int32_t expectLimits[500]; + int32_t expectCount=-1; + testSpan(sets, s, length, isUTF16, whichSpans, expectLimits, expectCount, testName, index); +} + +UBool stringContainsUnpairedSurrogate(const UChar *s, int32_t length) { + UChar c, c2; + + if(length>=0) { + while(length>0) { + c=*s++; + --length; + if(0xd800<=c && c<0xe000) { + if(c>=0xdc00 || length==0 || !U16_IS_TRAIL(c2=*s++)) { + return TRUE; + } + --length; + } + } + } else { + while((c=*s++)!=0) { + if(0xd800<=c && c<0xe000) { + if(c>=0xdc00 || !U16_IS_TRAIL(c2=*s++)) { + return TRUE; + } + } + } + } + return FALSE; +} + +// Test both UTF-16 and UTF-8 versions of span() etc. on the same sets and text, +// unless either UTF is turned off in whichSpans. +// Testing UTF-16 and UTF-8 together requires that surrogate code points +// have the same contains(c) value as U+FFFD. +void UnicodeSetTest::testSpanBothUTFs(const UnicodeSetWithStrings *sets[4], + const UChar *s16, int32_t length16, + uint32_t whichSpans, + const char *testName, int32_t index) { + int32_t expectLimits[500]; + int32_t expectCount; + + expectCount=-1; // Get expectLimits[] from testSpan(). + + if((whichSpans&SPAN_UTF16)!=0) { + testSpan(sets, s16, length16, TRUE, whichSpans, expectLimits, expectCount, testName, index); + } + if((whichSpans&SPAN_UTF8)==0) { + return; + } + + // Convert s16[] and expectLimits[] to UTF-8. + uint8_t s8[3000]; + int32_t offsets[3000]; + + const UChar *s16Limit=s16+length16; + char *t=(char *)s8; + char *tLimit=t+sizeof(s8); + int32_t *o=offsets; + UErrorCode errorCode=U_ZERO_ERROR; + + // Convert with substitution: Turn unpaired surrogates into U+FFFD. + ucnv_fromUnicode(openUTF8Converter(), &t, tLimit, &s16, s16Limit, o, TRUE, &errorCode); + if(U_FAILURE(errorCode)) { + errln("FAIL: %s[0x%lx] ucnv_fromUnicode(to UTF-8) fails with %s", + testName, (long)index, u_errorName(errorCode)); + ucnv_resetFromUnicode(utf8Cnv); + return; + } + int32_t length8=(int32_t)(t-(char *)s8); + + // Convert expectLimits[]. + int32_t i, j, expect; + for(i=j=0; igetSet().contains(0xfffd) ? + sets[0]->getSet().contains(0xd800, 0xdfff) : + sets[0]->getSet().containsNone(0xd800, 0xdfff)) || + sets[0]->hasStringsWithSurrogates()); + + UChar s[1000]; + int32_t length=0; + uint32_t localWhichSpans; + + UChar32 c, first; + for(first=c=0;; c=nextCodePoint(c)) { + if(c>0x10ffff || length>(LENGTHOF(s)-U16_MAX_LENGTH)) { + localWhichSpans=whichSpans; + if(stringContainsUnpairedSurrogate(s, length) && inconsistentSurrogates) { + localWhichSpans&=~SPAN_UTF8; + } + testSpanBothUTFs(sets, s, length, localWhichSpans, testName, first); + if(c>0x10ffff) { + break; + } + length=0; + first=c; + } + U16_APPEND_UNSAFE(s, length, c); + } +} + +// Test with a particular, interesting string. +// Specify length and try NUL-termination. +void UnicodeSetTest::testSpanUTF16String(const UnicodeSetWithStrings *sets[4], uint32_t whichSpans, const char *testName) { + static const UChar s[]={ + 0x61, 0x62, 0x20, // Latin, space + 0x3b1, 0x3b2, 0x3b3, // Greek + 0xd900, // lead surrogate + 0x3000, 0x30ab, 0x30ad, // wide space, Katakana + 0xdc05, // trail surrogate + 0xa0, 0xac00, 0xd7a3, // nbsp, Hangul + 0xd900, 0xdc05, // unassigned supplementary + 0xd840, 0xdfff, 0xd860, 0xdffe, // Han supplementary + 0xd7a4, 0xdc05, 0xd900, 0x2028, // unassigned, surrogates in wrong order, LS + 0 // NUL + }; + + if((whichSpans&SPAN_UTF16)==0) { + return; + } + testSpan(sets, s, -1, TRUE, (whichSpans&~SPAN_UTF8), testName, 0); + testSpan(sets, s, LENGTHOF(s)-1, TRUE, (whichSpans&~SPAN_UTF8), testName, 1); +} + +void UnicodeSetTest::testSpanUTF8String(const UnicodeSetWithStrings *sets[4], uint32_t whichSpans, const char *testName) { + static const char s[]={ + "abc" // Latin + + /* trail byte in lead position */ + "\x80" + + " " // space + + /* truncated multi-byte sequences */ + "\xd0" + "\xe0" + "\xe1" + "\xed" + "\xee" + "\xf0" + "\xf1" + "\xf4" + "\xf8" + "\xfc" + + "\xCE\xB1\xCE\xB2\xCE\xB3" // Greek + + /* trail byte in lead position */ + "\x80" + + "\xe0\x80" + "\xe0\xa0" + "\xe1\x80" + "\xed\x80" + "\xed\xa0" + "\xee\x80" + "\xf0\x80" + "\xf0\x90" + "\xf1\x80" + "\xf4\x80" + "\xf4\x90" + "\xf8\x80" + "\xfc\x80" + + "\xE3\x80\x80\xE3\x82\xAB\xE3\x82\xAD" // wide space, Katakana + + /* trail byte in lead position */ + "\x80" + + "\xf0\x80\x80" + "\xf0\x90\x80" + "\xf1\x80\x80" + "\xf4\x80\x80" + "\xf4\x90\x80" + "\xf8\x80\x80" + "\xfc\x80\x80" + + "\xC2\xA0\xEA\xB0\x80\xED\x9E\xA3" // nbsp, Hangul + + /* trail byte in lead position */ + "\x80" + + "\xf8\x80\x80\x80" + "\xfc\x80\x80\x80" + + "\xF1\x90\x80\x85" // unassigned supplementary + + /* trail byte in lead position */ + "\x80" + + "\xfc\x80\x80\x80\x80" + + "\xF0\xA0\x8F\xBF\xF0\xA8\x8F\xBE" // Han supplementary + + /* trail byte in lead position */ + "\x80" + + /* complete sequences but non-shortest forms or out of range etc. */ + "\xc0\x80" + "\xe0\x80\x80" + "\xed\xa0\x80" + "\xf0\x80\x80\x80" + "\xf4\x90\x80\x80" + "\xf8\x80\x80\x80\x80" + "\xfc\x80\x80\x80\x80\x80" + "\xfe" + "\xff" + + /* trail byte in lead position */ + "\x80" + + "\xED\x9E\xA4\xE2\x80\xA8" // unassigned, LS, NUL-terminated + }; + + if((whichSpans&SPAN_UTF8)==0) { + return; + } + testSpan(sets, s, -1, FALSE, (whichSpans&~SPAN_UTF16), testName, 0); + testSpan(sets, s, LENGTHOF(s)-1, FALSE, (whichSpans&~SPAN_UTF16), testName, 1); +} + +// Take a set of span options and multiply them so that +// each portion only has one of the options a, b and c. +// If b==0, then the set of options is just modified with mask and a. +// If b!=0 and c==0, then the set of options is just modified with mask, a and b. +static int32_t +addAlternative(uint32_t whichSpans[], int32_t whichSpansCount, + uint32_t mask, uint32_t a, uint32_t b, uint32_t c) { + uint32_t s; + int32_t i; + + for(i=0; i { 4, 7, 8 } spanBack() -> { 5, 8 } + "-c", + "byayaxy", // span() -> { 4, 7 } complement.span() -> { 7 } + "byayax", // span() -> { 4, 6 } complement.span() -> { 6 } + "-", + "byaya", // span() -> { 5 } + "byay", // span() -> { 4 } + "bya", // span() -> { 3 } + + // span(longest match) will not span the whole string. + "[a{ab}{bc}]", + "-cl", + // test_string 0x21 + "abc", + + "[a{ab}{abc}{cd}]", + "-cl", + "acdabcdabccd", + + // spanBack(longest match) will not span the whole string. + "[c{ab}{bc}]", + "-cl", + "abc", + + "[d{cd}{bcd}{ab}]", + "-cl", + "abbcdabcdabd", + + // Test with non-ASCII set strings - test proper handling of surrogate pairs + // and UTF-8 trail bytes. + // Copies of above test sets and strings, but transliterated to have + // different code points with similar trail units. + // Previous: a b c d + // Unicode: 042B 30AB 200AB 204AB + // UTF-16: 042B 30AB D840 DCAB D841 DCAB + // UTF-8: D0 AB E3 82 AB F0 A0 82 AB F0 A0 92 AB + "[\\u042B{\\u042B\\u30AB}{\\u042B\\u30AB\\U000200AB}{\\U000200AB\\U000204AB}]", + "-cl", + "\\u042B\\U000200AB\\U000204AB\\u042B\\u30AB\\U000200AB\\U000204AB\\u042B\\u30AB\\U000200AB\\U000200AB\\U000204AB", + + "[\\U000204AB{\\U000200AB\\U000204AB}{\\u30AB\\U000200AB\\U000204AB}{\\u042B\\u30AB}]", + "-cl", + "\\u042B\\u30AB\\u30AB\\U000200AB\\U000204AB\\u042B\\u30AB\\U000200AB\\U000204AB\\u042B\\u30AB\\U000204AB", + + // Stress bookkeeping and recursion. + // The following strings are barely doable with the recursive + // reference implementation. + // The not-contained character at the end prevents an early exit from the span(). + "[b{bb}]", + "-c", + // test_string 0x33 + "bbbbbbbbbbbbbbbbbbbbbbbb-", + // On complement sets, span() and spanBack() get different results + // because b is not in the complement set and there is an odd number of b's + // in the test string. + "-bc", + "bbbbbbbbbbbbbbbbbbbbbbbbb-", + + // Test with set strings with an initial or final code point span + // longer than 254. + "[a{" _64_a _64_a _64_a _64_a "b}" + "{a" _64_b _64_b _64_b _64_b "}]", + "-c", + _64_a _64_a _64_a _63_a "b", + _64_a _64_a _64_a _64_a "b", + _64_a _64_a _64_a _64_a "aaaabbbb", + "a" _64_b _64_b _64_b _63_b, + "a" _64_b _64_b _64_b _64_b, + "aaaabbbb" _64_b _64_b _64_b _64_b, + + // Test with strings containing unpaired surrogates. + // They are not representable in UTF-8, and a leading trail surrogate + // and a trailing lead surrogate must not match in the middle of a proper surrogate pair. + // U+20001 == \\uD840\\uDC01 + // U+20400 == \\uD841\\uDC00 + "[a\\U00020001\\U00020400{ab}{b\\uD840}{\\uDC00a}]", + "-8cl", + "aaab\\U00020001ba\\U00020400aba\\uD840ab\\uD840\\U00020000b\\U00020000a\\U00020000\\uDC00a\\uDC00babbb" + }; + uint32_t whichSpans[96]={ SPAN_ALL }; + int32_t whichSpansCount=1; + + UnicodeSet *sets[SET_COUNT]={ NULL }; + const UnicodeSetWithStrings *sets_with_str[SET_COUNT]={ NULL }; + + char testName[1024]; + char *testNameLimit=testName; + + int32_t i, j; + for(i=0; icomplement(); + // Intermediate set: Test cloning of a frozen set. + UnicodeSet *fast=new UnicodeSet(*sets[SLOW]); + fast->freeze(); + sets[FAST]=(UnicodeSet *)fast->clone(); + delete fast; + UnicodeSet *fastNot=new UnicodeSet(*sets[SLOW_NOT]); + fastNot->freeze(); + sets[FAST_NOT]=(UnicodeSet *)fastNot->clone(); + delete fastNot; + + for(j=0; j1) { + sprintf(testNameLimit+10 /* strlen("bad_string") */, + "%%0x%3x", + whichSpans[j]); + } + testSpanUTF16String(sets_with_str, whichSpans[j], testName); + testSpanUTF8String(sets_with_str, whichSpans[j], testName); + } + + strcpy(testNameLimit, "contents"); + for(j=0; j1) { + sprintf(testNameLimit+8 /* strlen("contents") */, + "%%0x%3x", + whichSpans[j]); + } + testSpanContents(sets_with_str, whichSpans[j], testName); + } + } else { + UnicodeString string=UnicodeString(s, -1, US_INV).unescape(); + strcpy(testNameLimit, "test_string"); + for(j=0; j1) { + sprintf(testNameLimit+11 /* strlen("test_string") */, + "%%0x%3x", + whichSpans[j]); + } + testSpanBothUTFs(sets_with_str, string.getBuffer(), string.length(), whichSpans[j], testName, i); + } + } + } + for(j=0; j