+ result.truncate(0);
+ utext_openUnicodeString(&resultText, &result, &status);
+ m.reset();
+ m.find(10, status);
+ m.find();
+ m.appendReplacement(&resultText, &replText, status);
+ REGEX_CHECK_STATUS;
+ const char str_blah8[] = { 0x54, 0x68, 0x65, 0x20, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x20, 0x73, 0x74, 0x61, 0x72, 0x74, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x73, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x65, 0x6e, 0x64, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x65, 0x65, 0x20, 0x6f, 0x6f, 0x68, 0x00 }; /* The matches start with ss and end with ee ooh */
+ REGEX_ASSERT_UTEXT_UTF8(str_blah8, &resultText);
+
+ m.appendTail(&resultText, status);
+ const char str_blah9[] = { 0x54, 0x68, 0x65, 0x20, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x20, 0x73, 0x74, 0x61, 0x72, 0x74, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x73, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x65, 0x6e, 0x64, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x65, 0x65, 0x20, 0x6f, 0x6f, 0x68, 0x20, 0x66, 0x69, 0x6e, 0x00 }; /* The matches start with ss and end with ee ooh fin */
+ REGEX_ASSERT_UTEXT_UTF8(str_blah9, &resultText);
+
+ utext_close(&resultText);
+ }
+
+ delete matcher2;
+ delete pat2;
+ delete matcher;
+ delete pat;
+
+ utext_close(&dataText);
+ utext_close(&replText);
+ utext_close(&destText);
+ utext_close(&re);
+}
+
+
+//---------------------------------------------------------------------------
+//
+// API_Pattern_UTF8 Test that the API for class RegexPattern is
+// present and nominally working.
+//
+//---------------------------------------------------------------------------
+void RegexTest::API_Pattern_UTF8() {
+ RegexPattern pata; // Test default constructor to not crash.
+ RegexPattern patb;
+
+ REGEX_ASSERT(pata == patb);
+ REGEX_ASSERT(pata == pata);
+
+ UText re1 = UTEXT_INITIALIZER;
+ UText re2 = UTEXT_INITIALIZER;
+ UErrorCode status = U_ZERO_ERROR;
+ UParseError pe;
+
+ const char str_abcalmz[] = { 0x61, 0x62, 0x63, 0x5b, 0x61, 0x2d, 0x6c, 0x5d, 0x5b, 0x6d, 0x2d, 0x7a, 0x5d, 0x00 }; /* abc[a-l][m-z] */
+ const char str_def[] = { 0x64, 0x65, 0x66, 0x00 }; /* def */
+ utext_openUTF8(&re1, str_abcalmz, -1, &status);
+ utext_openUTF8(&re2, str_def, -1, &status);
+
+ RegexPattern *pat1 = RegexPattern::compile(&re1, 0, pe, status);
+ RegexPattern *pat2 = RegexPattern::compile(&re2, 0, pe, status);
+ REGEX_CHECK_STATUS;
+ REGEX_ASSERT(*pat1 == *pat1);
+ REGEX_ASSERT(*pat1 != pata);
+
+ // Assign
+ patb = *pat1;
+ REGEX_ASSERT(patb == *pat1);
+
+ // Copy Construct
+ RegexPattern patc(*pat1);
+ REGEX_ASSERT(patc == *pat1);
+ REGEX_ASSERT(patb == patc);
+ REGEX_ASSERT(pat1 != pat2);
+ patb = *pat2;
+ REGEX_ASSERT(patb != patc);
+ REGEX_ASSERT(patb == *pat2);
+
+ // Compile with no flags.
+ RegexPattern *pat1a = RegexPattern::compile(&re1, pe, status);
+ REGEX_ASSERT(*pat1a == *pat1);
+
+ REGEX_ASSERT(pat1a->flags() == 0);
+
+ // Compile with different flags should be not equal
+ RegexPattern *pat1b = RegexPattern::compile(&re1, UREGEX_CASE_INSENSITIVE, pe, status);
+ REGEX_CHECK_STATUS;
+
+ REGEX_ASSERT(*pat1b != *pat1a);
+ REGEX_ASSERT(pat1b->flags() == UREGEX_CASE_INSENSITIVE);
+ REGEX_ASSERT(pat1a->flags() == 0);
+ delete pat1b;
+
+ // clone
+ RegexPattern *pat1c = pat1->clone();
+ REGEX_ASSERT(*pat1c == *pat1);
+ REGEX_ASSERT(*pat1c != *pat2);
+
+ delete pat1c;
+ delete pat1a;
+ delete pat1;
+ delete pat2;
+
+ utext_close(&re1);
+ utext_close(&re2);
+
+
+ //
+ // Verify that a matcher created from a cloned pattern works.
+ // (Jitterbug 3423)
+ //
+ {
+ UErrorCode status = U_ZERO_ERROR;
+ UText pattern = UTEXT_INITIALIZER;
+ const char str_pL[] = { 0x5c, 0x70, 0x7b, 0x4c, 0x7d, 0x2b, 0x00 }; /* \p{L}+ */
+ utext_openUTF8(&pattern, str_pL, -1, &status);
+
+ RegexPattern *pSource = RegexPattern::compile(&pattern, 0, status);
+ RegexPattern *pClone = pSource->clone();
+ delete pSource;
+ RegexMatcher *mFromClone = pClone->matcher(status);
+ REGEX_CHECK_STATUS;
+
+ UText input = UTEXT_INITIALIZER;
+ const char str_HelloWorld[] = { 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x00 }; /* Hello World */
+ utext_openUTF8(&input, str_HelloWorld, -1, &status);
+ mFromClone->reset(&input);
+ REGEX_ASSERT(mFromClone->find() == TRUE);
+ REGEX_ASSERT(mFromClone->group(status) == "Hello");
+ REGEX_ASSERT(mFromClone->find() == TRUE);
+ REGEX_ASSERT(mFromClone->group(status) == "World");
+ REGEX_ASSERT(mFromClone->find() == FALSE);
+ delete mFromClone;
+ delete pClone;
+
+ utext_close(&input);
+ utext_close(&pattern);
+ }
+
+ //
+ // matches convenience API
+ //
+ {
+ UErrorCode status = U_ZERO_ERROR;
+ UText pattern = UTEXT_INITIALIZER;
+ UText input = UTEXT_INITIALIZER;
+
+ const char str_randominput[] = { 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x20, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x00 }; /* random input */
+ utext_openUTF8(&input, str_randominput, -1, &status);
+
+ const char str_dotstar[] = { 0x2e, 0x2a, 0x00 }; /* .* */
+ utext_openUTF8(&pattern, str_dotstar, -1, &status);
+ REGEX_ASSERT(RegexPattern::matches(&pattern, &input, pe, status) == TRUE);
+ REGEX_CHECK_STATUS;
+
+ const char str_abc[] = { 0x61, 0x62, 0x63, 0x00 }; /* abc */
+ utext_openUTF8(&pattern, str_abc, -1, &status);
+ REGEX_ASSERT(RegexPattern::matches("abc", "random input", pe, status) == FALSE);
+ REGEX_CHECK_STATUS;
+
+ const char str_nput[] = { 0x2e, 0x2a, 0x6e, 0x70, 0x75, 0x74, 0x00 }; /* .*nput */
+ utext_openUTF8(&pattern, str_nput, -1, &status);
+ REGEX_ASSERT(RegexPattern::matches(".*nput", "random input", pe, status) == TRUE);
+ REGEX_CHECK_STATUS;
+
+ utext_openUTF8(&pattern, str_randominput, -1, &status);
+ REGEX_ASSERT(RegexPattern::matches("random input", "random input", pe, status) == TRUE);
+ REGEX_CHECK_STATUS;
+
+ const char str_u[] = { 0x2e, 0x2a, 0x75, 0x00 }; /* .*u */
+ utext_openUTF8(&pattern, str_u, -1, &status);
+ REGEX_ASSERT(RegexPattern::matches(".*u", "random input", pe, status) == FALSE);
+ REGEX_CHECK_STATUS;
+
+ utext_openUTF8(&input, str_abc, -1, &status);
+ utext_openUTF8(&pattern, str_abc, -1, &status);
+ status = U_INDEX_OUTOFBOUNDS_ERROR;
+ REGEX_ASSERT(RegexPattern::matches("abc", "abc", pe, status) == FALSE);
+ REGEX_ASSERT(status == U_INDEX_OUTOFBOUNDS_ERROR);
+
+ utext_close(&input);
+ utext_close(&pattern);
+ }
+
+
+ //
+ // Split()
+ //
+ status = U_ZERO_ERROR;
+ const char str_spaceplus[] = { 0x20, 0x2b, 0x00 }; /* + */
+ utext_openUTF8(&re1, str_spaceplus, -1, &status);
+ pat1 = RegexPattern::compile(&re1, pe, status);
+ REGEX_CHECK_STATUS;
+ UnicodeString fields[10];
+
+ int32_t n;
+ n = pat1->split("Now is the time", fields, 10, status);
+ REGEX_CHECK_STATUS;
+ REGEX_ASSERT(n==4);
+ REGEX_ASSERT(fields[0]=="Now");
+ REGEX_ASSERT(fields[1]=="is");
+ REGEX_ASSERT(fields[2]=="the");
+ REGEX_ASSERT(fields[3]=="time");
+ REGEX_ASSERT(fields[4]=="");
+
+ n = pat1->split("Now is the time", fields, 2, status);
+ REGEX_CHECK_STATUS;
+ REGEX_ASSERT(n==2);
+ REGEX_ASSERT(fields[0]=="Now");
+ REGEX_ASSERT(fields[1]=="is the time");
+ REGEX_ASSERT(fields[2]=="the"); // left over from previous test
+
+ fields[1] = "*";
+ status = U_ZERO_ERROR;
+ n = pat1->split("Now is the time", fields, 1, status);
+ REGEX_CHECK_STATUS;
+ REGEX_ASSERT(n==1);
+ REGEX_ASSERT(fields[0]=="Now is the time");
+ REGEX_ASSERT(fields[1]=="*");
+ status = U_ZERO_ERROR;
+
+ n = pat1->split(" Now is the time ", fields, 10, status);
+ REGEX_CHECK_STATUS;
+ REGEX_ASSERT(n==6);
+ REGEX_ASSERT(fields[0]=="");
+ REGEX_ASSERT(fields[1]=="Now");
+ REGEX_ASSERT(fields[2]=="is");
+ REGEX_ASSERT(fields[3]=="the");
+ REGEX_ASSERT(fields[4]=="time");
+ REGEX_ASSERT(fields[5]=="");
+ REGEX_ASSERT(fields[6]=="");
+
+ fields[2] = "*";
+ n = pat1->split(" ", fields, 10, status);
+ REGEX_CHECK_STATUS;
+ REGEX_ASSERT(n==2);
+ REGEX_ASSERT(fields[0]=="");
+ REGEX_ASSERT(fields[1]=="");
+ REGEX_ASSERT(fields[2]=="*");
+
+ fields[0] = "foo";
+ n = pat1->split("", fields, 10, status);
+ REGEX_CHECK_STATUS;
+ REGEX_ASSERT(n==0);
+ REGEX_ASSERT(fields[0]=="foo");
+
+ delete pat1;
+
+ // split, with a pattern with (capture)
+ regextst_openUTF8FromInvariant(&re1, "<(\\w*)>", -1, &status);
+ pat1 = RegexPattern::compile(&re1, pe, status);
+ REGEX_CHECK_STATUS;
+
+ status = U_ZERO_ERROR;
+ fields[6] = fields[7] = "*";
+ n = pat1->split("<a>Now is <b>the time<c>", fields, 10, status);
+ REGEX_CHECK_STATUS;
+ REGEX_ASSERT(n==7);
+ REGEX_ASSERT(fields[0]=="");
+ REGEX_ASSERT(fields[1]=="a");
+ REGEX_ASSERT(fields[2]=="Now is ");
+ REGEX_ASSERT(fields[3]=="b");
+ REGEX_ASSERT(fields[4]=="the time");
+ REGEX_ASSERT(fields[5]=="c");
+ REGEX_ASSERT(fields[6]=="");
+ REGEX_ASSERT(fields[7]=="*");
+ REGEX_ASSERT(status==U_ZERO_ERROR);
+
+ fields[6] = fields[7] = "*";
+ n = pat1->split(" <a>Now is <b>the time<c>", fields, 10, status);
+ REGEX_CHECK_STATUS;
+ REGEX_ASSERT(n==7);
+ REGEX_ASSERT(fields[0]==" ");
+ REGEX_ASSERT(fields[1]=="a");
+ REGEX_ASSERT(fields[2]=="Now is ");
+ REGEX_ASSERT(fields[3]=="b");
+ REGEX_ASSERT(fields[4]=="the time");
+ REGEX_ASSERT(fields[5]=="c");
+ REGEX_ASSERT(fields[6]=="");
+ REGEX_ASSERT(fields[7]=="*");
+
+ status = U_ZERO_ERROR;
+ fields[6] = "foo";
+ n = pat1->split(" <a>Now is <b>the time<c> ", fields, 6, status);
+ REGEX_CHECK_STATUS;
+ REGEX_ASSERT(n==6);
+ REGEX_ASSERT(fields[0]==" ");
+ REGEX_ASSERT(fields[1]=="a");
+ REGEX_ASSERT(fields[2]=="Now is ");
+ REGEX_ASSERT(fields[3]=="b");
+ REGEX_ASSERT(fields[4]=="the time");
+ REGEX_ASSERT(fields[5]==" ");
+ REGEX_ASSERT(fields[6]=="foo");
+
+ status = U_ZERO_ERROR;
+ fields[5] = "foo";
+ n = pat1->split(" <a>Now is <b>the time<c>", fields, 5, status);
+ REGEX_CHECK_STATUS;
+ REGEX_ASSERT(n==5);
+ REGEX_ASSERT(fields[0]==" ");
+ REGEX_ASSERT(fields[1]=="a");
+ REGEX_ASSERT(fields[2]=="Now is ");
+ REGEX_ASSERT(fields[3]=="b");
+ REGEX_ASSERT(fields[4]=="the time<c>");
+ REGEX_ASSERT(fields[5]=="foo");
+
+ status = U_ZERO_ERROR;
+ fields[5] = "foo";
+ n = pat1->split(" <a>Now is <b>the time", fields, 5, status);
+ REGEX_CHECK_STATUS;
+ REGEX_ASSERT(n==5);
+ REGEX_ASSERT(fields[0]==" ");
+ REGEX_ASSERT(fields[1]=="a");
+ REGEX_ASSERT(fields[2]=="Now is ");
+ REGEX_ASSERT(fields[3]=="b");
+ REGEX_ASSERT(fields[4]=="the time");
+ REGEX_ASSERT(fields[5]=="foo");
+
+ status = U_ZERO_ERROR;
+ n = pat1->split(" <a>Now is <b>the time<c>", fields, 4, status);
+ REGEX_CHECK_STATUS;
+ REGEX_ASSERT(n==4);
+ REGEX_ASSERT(fields[0]==" ");
+ REGEX_ASSERT(fields[1]=="a");
+ REGEX_ASSERT(fields[2]=="Now is ");
+ REGEX_ASSERT(fields[3]=="the time<c>");
+ status = U_ZERO_ERROR;
+ delete pat1;
+
+ regextst_openUTF8FromInvariant(&re1, "([-,])", -1, &status);
+ pat1 = RegexPattern::compile(&re1, pe, status);
+ REGEX_CHECK_STATUS;
+ n = pat1->split("1-10,20", fields, 10, status);
+ REGEX_CHECK_STATUS;
+ REGEX_ASSERT(n==5);
+ REGEX_ASSERT(fields[0]=="1");
+ REGEX_ASSERT(fields[1]=="-");
+ REGEX_ASSERT(fields[2]=="10");
+ REGEX_ASSERT(fields[3]==",");
+ REGEX_ASSERT(fields[4]=="20");
+ delete pat1;
+
+
+ //
+ // RegexPattern::pattern() and patternText()
+ //
+ pat1 = new RegexPattern();
+ REGEX_ASSERT(pat1->pattern() == "");
+ REGEX_ASSERT_UTEXT_UTF8("", pat1->patternText(status));
+ delete pat1;
+ const char *helloWorldInvariant = "(Hello, world)*";
+ regextst_openUTF8FromInvariant(&re1, helloWorldInvariant, -1, &status);
+ pat1 = RegexPattern::compile(&re1, pe, status);
+ REGEX_CHECK_STATUS;
+ REGEX_ASSERT_UNISTR(pat1->pattern(),"(Hello, world)*");
+ REGEX_ASSERT_UTEXT_INVARIANT("(Hello, world)*", pat1->patternText(status));
+ delete pat1;
+
+ utext_close(&re1);
+}
+
+
+//---------------------------------------------------------------------------
+//
+// Extended A more thorough check for features of regex patterns
+// The test cases are in a separate data file,
+// source/tests/testdata/regextst.txt
+// A description of the test data format is included in that file.
+//
+//---------------------------------------------------------------------------
+
+const char *
+RegexTest::getPath(char buffer[2048], const char *filename) {
+ UErrorCode status=U_ZERO_ERROR;
+ const char *testDataDirectory = IntlTest::getSourceTestData(status);
+ if (U_FAILURE(status)) {
+ errln("ERROR: loadTestData() failed - %s", u_errorName(status));
+ return NULL;
+ }
+
+ strcpy(buffer, testDataDirectory);
+ strcat(buffer, filename);
+ return buffer;
+}
+
+void RegexTest::Extended() {
+ char tdd[2048];
+ const char *srcPath;
+ UErrorCode status = U_ZERO_ERROR;
+ int32_t lineNum = 0;
+
+ //
+ // Open and read the test data file.
+ //
+ srcPath=getPath(tdd, "regextst.txt");
+ if(srcPath==NULL) {
+ return; /* something went wrong, error already output */
+ }
+
+ int32_t len;
+ UChar *testData = ReadAndConvertFile(srcPath, len, "utf-8", status);
+ if (U_FAILURE(status)) {
+ return; /* something went wrong, error already output */
+ }
+
+ //
+ // Put the test data into a UnicodeString
+ //
+ UnicodeString testString(FALSE, testData, len);
+
+ RegexMatcher quotedStuffMat(UNICODE_STRING_SIMPLE("\\s*([\\'\\\"/])(.*?)\\1"), 0, status);
+ RegexMatcher commentMat (UNICODE_STRING_SIMPLE("\\s*(#.*)?$"), 0, status);
+ RegexMatcher flagsMat (UNICODE_STRING_SIMPLE("\\s*([ixsmdteDEGLMQvabtyYzZ2-9]*)([:letter:]*)"), 0, status);
+
+ RegexMatcher lineMat(UNICODE_STRING_SIMPLE("(.*?)\\r?\\n"), testString, 0, status);
+ UnicodeString testPattern; // The pattern for test from the test file.
+ UnicodeString testFlags; // the flags for a test.
+ UnicodeString matchString; // The marked up string to be used as input
+
+ if (U_FAILURE(status)){
+ dataerrln("Construct RegexMatcher() error.");
+ delete [] testData;
+ return;
+ }
+
+ //
+ // Loop over the test data file, once per line.
+ //
+ while (lineMat.find()) {
+ lineNum++;
+ if (U_FAILURE(status)) {
+ errln("%s:%d: ICU Error \"%s\"", srcPath, lineNum, u_errorName(status));
+ }
+
+ status = U_ZERO_ERROR;
+ UnicodeString testLine = lineMat.group(1, status);
+ if (testLine.length() == 0) {
+ continue;
+ }
+
+ //
+ // Parse the test line. Skip blank and comment only lines.
+ // Separate out the three main fields - pattern, flags, target.
+ //
+
+ commentMat.reset(testLine);
+ if (commentMat.lookingAt(status)) {
+ // This line is a comment, or blank.
+ continue;
+ }
+
+ //
+ // Pull out the pattern field, remove it from the test file line.
+ //
+ quotedStuffMat.reset(testLine);
+ if (quotedStuffMat.lookingAt(status)) {
+ testPattern = quotedStuffMat.group(2, status);
+ testLine.remove(0, quotedStuffMat.end(0, status));
+ } else {
+ errln("Bad pattern (missing quotes?) at %s:%d", srcPath, lineNum);
+ continue;
+ }
+
+
+ //
+ // Pull out the flags from the test file line.
+ //
+ flagsMat.reset(testLine);
+ flagsMat.lookingAt(status); // Will always match, possibly an empty string.
+ testFlags = flagsMat.group(1, status);
+ if (flagsMat.group(2, status).length() > 0) {
+ errln("Bad Match flag at line %d. Scanning %c\n",
+ lineNum, flagsMat.group(2, status).charAt(0));
+ continue;
+ }
+ testLine.remove(0, flagsMat.end(0, status));
+
+ //
+ // Pull out the match string, as a whole.
+ // We'll process the <tags> later.
+ //
+ quotedStuffMat.reset(testLine);
+ if (quotedStuffMat.lookingAt(status)) {
+ matchString = quotedStuffMat.group(2, status);
+ testLine.remove(0, quotedStuffMat.end(0, status));
+ } else {
+ errln("Bad match string at test file line %d", lineNum);
+ continue;
+ }
+
+ //
+ // The only thing left from the input line should be an optional trailing comment.
+ //
+ commentMat.reset(testLine);
+ if (commentMat.lookingAt(status) == FALSE) {
+ errln("Line %d: unexpected characters at end of test line.", lineNum);
+ continue;
+ }
+
+ //
+ // Run the test
+ //
+ regex_find(testPattern, testFlags, matchString, srcPath, lineNum);
+ }
+
+ delete [] testData;
+
+}
+
+
+
+//---------------------------------------------------------------------------
+//
+// regex_find(pattern, flags, inputString, lineNumber)
+//
+// Function to run a single test from the Extended (data driven) tests.
+// See file test/testdata/regextst.txt for a description of the
+// pattern and inputString fields, and the allowed flags.
+// lineNumber is the source line in regextst.txt of the test.
+//
+//---------------------------------------------------------------------------
+
+
+// Set a value into a UVector at position specified by a decimal number in
+// a UnicodeString. This is a utility function needed by the actual test function,
+// which follows.
+static void set(UVector &vec, int32_t val, UnicodeString index) {
+ UErrorCode status=U_ZERO_ERROR;
+ int32_t idx = 0;
+ for (int32_t i=0; i<index.length(); i++) {
+ int32_t d=u_charDigitValue(index.charAt(i));
+ if (d<0) {return;}
+ idx = idx*10 + d;
+ }
+ while (vec.size()<idx+1) {vec.addElement(-1, status);}
+ vec.setElementAt(val, idx);
+}
+
+static void setInt(UVector &vec, int32_t val, int32_t idx) {
+ UErrorCode status=U_ZERO_ERROR;
+ while (vec.size()<idx+1) {vec.addElement(-1, status);}
+ vec.setElementAt(val, idx);
+}
+
+static UBool utextOffsetToNative(UText *utext, int32_t unistrOffset, int32_t& nativeIndex)
+{
+ UBool couldFind = TRUE;
+ UTEXT_SETNATIVEINDEX(utext, 0);
+ int32_t i = 0;
+ while (i < unistrOffset) {
+ UChar32 c = UTEXT_NEXT32(utext);
+ if (c != U_SENTINEL) {
+ i += U16_LENGTH(c);
+ } else {
+ couldFind = FALSE;
+ break;
+ }
+ }
+ nativeIndex = (int32_t)UTEXT_GETNATIVEINDEX(utext);
+ return couldFind;
+}
+
+
+void RegexTest::regex_find(const UnicodeString &pattern,
+ const UnicodeString &flags,
+ const UnicodeString &inputString,
+ const char *srcPath,
+ int32_t line) {
+ UnicodeString unEscapedInput;
+ UnicodeString deTaggedInput;
+
+ int32_t patternUTF8Length, inputUTF8Length;
+ char *patternChars = NULL, *inputChars = NULL;
+ UText patternText = UTEXT_INITIALIZER;
+ UText inputText = UTEXT_INITIALIZER;
+ UConverter *UTF8Converter = NULL;
+
+ UErrorCode status = U_ZERO_ERROR;
+ UParseError pe;
+ RegexPattern *parsePat = NULL;
+ RegexMatcher *parseMatcher = NULL;
+ RegexPattern *callerPattern = NULL, *UTF8Pattern = NULL;
+ RegexMatcher *matcher = NULL, *UTF8Matcher = NULL;
+ UVector groupStarts(status);
+ UVector groupEnds(status);
+ UVector groupStartsUTF8(status);
+ UVector groupEndsUTF8(status);
+ UBool isMatch = FALSE, isUTF8Match = FALSE;
+ UBool failed = FALSE;
+ int32_t numFinds;
+ int32_t i;
+ UBool useMatchesFunc = FALSE;
+ UBool useLookingAtFunc = FALSE;
+ int32_t regionStart = -1;
+ int32_t regionEnd = -1;
+ int32_t regionStartUTF8 = -1;
+ int32_t regionEndUTF8 = -1;
+
+
+ //
+ // Compile the caller's pattern
+ //
+ uint32_t bflags = 0;
+ if (flags.indexOf((UChar)0x69) >= 0) { // 'i' flag
+ bflags |= UREGEX_CASE_INSENSITIVE;
+ }
+ if (flags.indexOf((UChar)0x78) >= 0) { // 'x' flag
+ bflags |= UREGEX_COMMENTS;
+ }
+ if (flags.indexOf((UChar)0x73) >= 0) { // 's' flag
+ bflags |= UREGEX_DOTALL;
+ }
+ if (flags.indexOf((UChar)0x6d) >= 0) { // 'm' flag
+ bflags |= UREGEX_MULTILINE;
+ }
+
+ if (flags.indexOf((UChar)0x65) >= 0) { // 'e' flag
+ bflags |= UREGEX_ERROR_ON_UNKNOWN_ESCAPES;
+ }
+ if (flags.indexOf((UChar)0x44) >= 0) { // 'D' flag
+ bflags |= UREGEX_UNIX_LINES;
+ }
+ if (flags.indexOf((UChar)0x51) >= 0) { // 'Q' flag
+ bflags |= UREGEX_LITERAL;
+ }
+
+
+ callerPattern = RegexPattern::compile(pattern, bflags, pe, status);
+ if (status != U_ZERO_ERROR) {
+ #if UCONFIG_NO_BREAK_ITERATION==1
+ // 'v' test flag means that the test pattern should not compile if ICU was configured
+ // to not include break iteration. RBBI is needed for Unicode word boundaries.
+ if (flags.indexOf((UChar)0x76) >= 0 /*'v'*/ && status == U_UNSUPPORTED_ERROR) {
+ goto cleanupAndReturn;
+ }
+ #endif
+ if (flags.indexOf((UChar)0x45) >= 0) { // flags contain 'E'
+ // Expected pattern compilation error.
+ if (flags.indexOf((UChar)0x64) >= 0) { // flags contain 'd'
+ logln("Pattern Compile returns \"%s\"", u_errorName(status));
+ }
+ goto cleanupAndReturn;
+ } else {
+ // Unexpected pattern compilation error.
+ dataerrln("Line %d: error %s compiling pattern.", line, u_errorName(status));
+ goto cleanupAndReturn;
+ }
+ }
+
+ UTF8Converter = ucnv_open("UTF8", &status);
+ ucnv_setFromUCallBack(UTF8Converter, UCNV_FROM_U_CALLBACK_STOP, NULL, NULL, NULL, &status);
+
+ patternUTF8Length = pattern.extract(NULL, 0, UTF8Converter, status);
+ status = U_ZERO_ERROR; // buffer overflow
+ patternChars = new char[patternUTF8Length+1];
+ pattern.extract(patternChars, patternUTF8Length+1, UTF8Converter, status);
+ utext_openUTF8(&patternText, patternChars, patternUTF8Length, &status);
+
+ if (status == U_ZERO_ERROR) {
+ UTF8Pattern = RegexPattern::compile(&patternText, bflags, pe, status);
+
+ if (status != U_ZERO_ERROR) {
+#if UCONFIG_NO_BREAK_ITERATION==1
+ // 'v' test flag means that the test pattern should not compile if ICU was configured
+ // to not include break iteration. RBBI is needed for Unicode word boundaries.
+ if (flags.indexOf((UChar)0x76) >= 0 /*'v'*/ && status == U_UNSUPPORTED_ERROR) {
+ goto cleanupAndReturn;
+ }
+#endif
+ if (flags.indexOf((UChar)0x45) >= 0) { // flags contain 'E'
+ // Expected pattern compilation error.
+ if (flags.indexOf((UChar)0x64) >= 0) { // flags contain 'd'
+ logln("Pattern Compile returns \"%s\" (UTF8)", u_errorName(status));
+ }
+ goto cleanupAndReturn;
+ } else {
+ // Unexpected pattern compilation error.
+ errln("Line %d: error %s compiling pattern. (UTF8)", line, u_errorName(status));
+ goto cleanupAndReturn;
+ }
+ }
+ }
+
+ if (UTF8Pattern == NULL) {
+ // UTF-8 does not allow unpaired surrogates, so this could actually happen without being a failure of the engine
+ logln("Unable to create UTF-8 pattern, skipping UTF-8 tests for %s:%d", srcPath, line);
+ status = U_ZERO_ERROR;
+ }
+
+ if (flags.indexOf((UChar)0x64) >= 0) { // 'd' flag
+ RegexPatternDump(callerPattern);
+ }
+
+ if (flags.indexOf((UChar)0x45) >= 0) { // 'E' flag
+ errln("%s, Line %d: Expected, but did not get, a pattern compilation error.", srcPath, line);
+ goto cleanupAndReturn;
+ }
+
+
+ //
+ // Number of times find() should be called on the test string, default to 1
+ //
+ numFinds = 1;
+ for (i=2; i<=9; i++) {
+ if (flags.indexOf((UChar)(0x30 + i)) >= 0) { // digit flag
+ if (numFinds != 1) {
+ errln("Line %d: more than one digit flag. Scanning %d.", line, i);
+ goto cleanupAndReturn;
+ }
+ numFinds = i;
+ }
+ }
+
+ // 'M' flag. Use matches() instead of find()
+ if (flags.indexOf((UChar)0x4d) >= 0) {
+ useMatchesFunc = TRUE;
+ }
+ if (flags.indexOf((UChar)0x4c) >= 0) {
+ useLookingAtFunc = TRUE;
+ }
+
+ //
+ // Find the tags in the input data, remove them, and record the group boundary
+ // positions.
+ //
+ parsePat = RegexPattern::compile("<(/?)(r|[0-9]+)>", 0, pe, status);
+ REGEX_CHECK_STATUS_L(line);
+
+ unEscapedInput = inputString.unescape();
+ parseMatcher = parsePat->matcher(unEscapedInput, status);
+ REGEX_CHECK_STATUS_L(line);
+ while(parseMatcher->find()) {
+ parseMatcher->appendReplacement(deTaggedInput, "", status);
+ REGEX_CHECK_STATUS;
+ UnicodeString groupNum = parseMatcher->group(2, status);
+ if (groupNum == "r") {
+ // <r> or </r>, a region specification within the string
+ if (parseMatcher->group(1, status) == "/") {
+ regionEnd = deTaggedInput.length();
+ } else {
+ regionStart = deTaggedInput.length();
+ }
+ } else {
+ // <digits> or </digits>, a group match boundary tag.
+ if (parseMatcher->group(1, status) == "/") {
+ set(groupEnds, deTaggedInput.length(), groupNum);
+ } else {
+ set(groupStarts, deTaggedInput.length(), groupNum);
+ }
+ }
+ }
+ parseMatcher->appendTail(deTaggedInput);
+ REGEX_ASSERT_L(groupStarts.size() == groupEnds.size(), line);
+ if ((regionStart>=0 || regionEnd>=0) && (regionStart<0 || regionStart>regionEnd)) {
+ errln("mismatched <r> tags");
+ failed = TRUE;
+ goto cleanupAndReturn;
+ }
+
+ //
+ // Configure the matcher according to the flags specified with this test.
+ //
+ matcher = callerPattern->matcher(deTaggedInput, status);
+ REGEX_CHECK_STATUS_L(line);
+ if (flags.indexOf((UChar)0x74) >= 0) { // 't' trace flag
+ matcher->setTrace(TRUE);
+ }
+
+ if (UTF8Pattern != NULL) {
+ inputUTF8Length = deTaggedInput.extract(NULL, 0, UTF8Converter, status);
+ status = U_ZERO_ERROR; // buffer overflow
+ inputChars = new char[inputUTF8Length+1];
+ deTaggedInput.extract(inputChars, inputUTF8Length+1, UTF8Converter, status);
+ utext_openUTF8(&inputText, inputChars, inputUTF8Length, &status);
+
+ if (status == U_ZERO_ERROR) {
+ UTF8Matcher = &UTF8Pattern->matcher(status)->reset(&inputText);
+ REGEX_CHECK_STATUS_L(line);
+ }
+
+ if (UTF8Matcher == NULL) {
+ // UTF-8 does not allow unpaired surrogates, so this could actually happen without being a failure of the engine
+ logln("Unable to create UTF-8 matcher, skipping UTF-8 tests for %s:%d", srcPath, line);
+ status = U_ZERO_ERROR;
+ }
+ }
+
+ //
+ // Generate native indices for UTF8 versions of region and capture group info
+ //
+ if (UTF8Matcher != NULL) {
+ if (regionStart>=0) (void) utextOffsetToNative(&inputText, regionStart, regionStartUTF8);
+ if (regionEnd>=0) (void) utextOffsetToNative(&inputText, regionEnd, regionEndUTF8);
+
+ // Fill out the native index UVector info.
+ // Only need 1 loop, from above we know groupStarts.size() = groupEnds.size()
+ for (i=0; i<groupStarts.size(); i++) {
+ int32_t start = groupStarts.elementAti(i);
+ // -1 means there was no UVector slot and we won't be requesting that capture group for this test, don't bother inserting
+ if (start >= 0) {
+ int32_t startUTF8;
+ if (!utextOffsetToNative(&inputText, start, startUTF8)) {
+ errln("Error at line %d: could not find native index for group start %d. UTF16 index %d", line, i, start);
+ failed = TRUE;
+ goto cleanupAndReturn; // Good chance of subsequent bogus errors. Stop now.
+ }
+ setInt(groupStartsUTF8, startUTF8, i);
+ }
+
+ int32_t end = groupEnds.elementAti(i);
+ // -1 means there was no UVector slot and we won't be requesting that capture group for this test, don't bother inserting
+ if (end >= 0) {
+ int32_t endUTF8;
+ if (!utextOffsetToNative(&inputText, end, endUTF8)) {
+ errln("Error at line %d: could not find native index for group end %d. UTF16 index %d", line, i, end);
+ failed = TRUE;
+ goto cleanupAndReturn; // Good chance of subsequent bogus errors. Stop now.
+ }
+ setInt(groupEndsUTF8, endUTF8, i);
+ }
+ }
+ }
+
+ if (regionStart>=0) {
+ matcher->region(regionStart, regionEnd, status);
+ REGEX_CHECK_STATUS_L(line);
+ if (UTF8Matcher != NULL) {
+ UTF8Matcher->region(regionStartUTF8, regionEndUTF8, status);
+ REGEX_CHECK_STATUS_L(line);
+ }
+ }
+ if (flags.indexOf((UChar)0x61) >= 0) { // 'a' anchoring bounds flag
+ matcher->useAnchoringBounds(FALSE);
+ if (UTF8Matcher != NULL) {
+ UTF8Matcher->useAnchoringBounds(FALSE);
+ }
+ }
+ if (flags.indexOf((UChar)0x62) >= 0) { // 'b' transparent bounds flag
+ matcher->useTransparentBounds(TRUE);
+ if (UTF8Matcher != NULL) {
+ UTF8Matcher->useTransparentBounds(TRUE);
+ }
+ }
+
+
+
+ //
+ // Do a find on the de-tagged input using the caller's pattern
+ // TODO: error on count>1 and not find().
+ // error on both matches() and lookingAt().
+ //
+ for (i=0; i<numFinds; i++) {
+ if (useMatchesFunc) {
+ isMatch = matcher->matches(status);
+ if (UTF8Matcher != NULL) {
+ isUTF8Match = UTF8Matcher->matches(status);
+ }
+ } else if (useLookingAtFunc) {
+ isMatch = matcher->lookingAt(status);
+ if (UTF8Matcher != NULL) {
+ isUTF8Match = UTF8Matcher->lookingAt(status);
+ }
+ } else {
+ isMatch = matcher->find();
+ if (UTF8Matcher != NULL) {
+ isUTF8Match = UTF8Matcher->find();
+ }
+ }
+ }
+ matcher->setTrace(FALSE);
+
+ //
+ // Match up the groups from the find() with the groups from the tags
+ //
+
+ // number of tags should match number of groups from find operation.
+ // matcher->groupCount does not include group 0, the entire match, hence the +1.
+ // G option in test means that capture group data is not available in the
+ // expected results, so the check needs to be suppressed.
+ if (isMatch == FALSE && groupStarts.size() != 0) {
+ dataerrln("Error at line %d: Match expected, but none found.", line);
+ failed = TRUE;
+ goto cleanupAndReturn;
+ } else if (UTF8Matcher != NULL && isUTF8Match == FALSE && groupStarts.size() != 0) {
+ errln("Error at line %d: Match expected, but none found. (UTF8)", line);
+ failed = TRUE;
+ goto cleanupAndReturn;
+ }
+
+ if (flags.indexOf((UChar)0x47 /*G*/) >= 0) {
+ // Only check for match / no match. Don't check capture groups.
+ if (isMatch && groupStarts.size() == 0) {
+ errln("Error at line %d: No match expected, but one found.", line);
+ failed = TRUE;
+ } else if (UTF8Matcher != NULL && isUTF8Match && groupStarts.size() == 0) {
+ errln("Error at line %d: No match expected, but one found. (UTF8)", line);
+ failed = TRUE;
+ }
+ goto cleanupAndReturn;
+ }
+
+ REGEX_CHECK_STATUS_L(line);
+ for (i=0; i<=matcher->groupCount(); i++) {
+ int32_t expectedStart = (i >= groupStarts.size()? -1 : groupStarts.elementAti(i));
+ int32_t expectedStartUTF8 = (i >= groupStartsUTF8.size()? -1 : groupStartsUTF8.elementAti(i));
+ if (matcher->start(i, status) != expectedStart) {
+ errln("Error at line %d: incorrect start position for group %d. Expected %d, got %d",
+ line, i, expectedStart, matcher->start(i, status));
+ failed = TRUE;
+ goto cleanupAndReturn; // Good chance of subsequent bogus errors. Stop now.
+ } else if (UTF8Matcher != NULL && UTF8Matcher->start(i, status) != expectedStartUTF8) {
+ errln("Error at line %d: incorrect start position for group %d. Expected %d, got %d (UTF8)",
+ line, i, expectedStartUTF8, UTF8Matcher->start(i, status));
+ failed = TRUE;
+ goto cleanupAndReturn; // Good chance of subsequent bogus errors. Stop now.
+ }
+
+ int32_t expectedEnd = (i >= groupEnds.size()? -1 : groupEnds.elementAti(i));
+ int32_t expectedEndUTF8 = (i >= groupEndsUTF8.size()? -1 : groupEndsUTF8.elementAti(i));
+ if (matcher->end(i, status) != expectedEnd) {
+ errln("Error at line %d: incorrect end position for group %d. Expected %d, got %d",
+ line, i, expectedEnd, matcher->end(i, status));
+ failed = TRUE;
+ // Error on end position; keep going; real error is probably yet to come as group
+ // end positions work from end of the input data towards the front.
+ } else if (UTF8Matcher != NULL && UTF8Matcher->end(i, status) != expectedEndUTF8) {
+ errln("Error at line %d: incorrect end position for group %d. Expected %d, got %d (UTF8)",
+ line, i, expectedEndUTF8, UTF8Matcher->end(i, status));
+ failed = TRUE;
+ // Error on end position; keep going; real error is probably yet to come as group
+ // end positions work from end of the input data towards the front.
+ }
+ }
+ if ( matcher->groupCount()+1 < groupStarts.size()) {
+ errln("Error at line %d: Expected %d capture groups, found %d.",
+ line, groupStarts.size()-1, matcher->groupCount());
+ failed = TRUE;
+ }
+ else if (UTF8Matcher != NULL && UTF8Matcher->groupCount()+1 < groupStarts.size()) {
+ errln("Error at line %d: Expected %d capture groups, found %d. (UTF8)",
+ line, groupStarts.size()-1, UTF8Matcher->groupCount());
+ failed = TRUE;
+ }
+
+ if ((flags.indexOf((UChar)0x59) >= 0) && // 'Y' flag: RequireEnd() == false
+ matcher->requireEnd() == TRUE) {
+ errln("Error at line %d: requireEnd() returned TRUE. Expected FALSE", line);
+ failed = TRUE;
+ } else if (UTF8Matcher != NULL && (flags.indexOf((UChar)0x59) >= 0) && // 'Y' flag: RequireEnd() == false
+ UTF8Matcher->requireEnd() == TRUE) {
+ errln("Error at line %d: requireEnd() returned TRUE. Expected FALSE (UTF8)", line);
+ failed = TRUE;
+ }
+
+ if ((flags.indexOf((UChar)0x79) >= 0) && // 'y' flag: RequireEnd() == true
+ matcher->requireEnd() == FALSE) {
+ errln("Error at line %d: requireEnd() returned FALSE. Expected TRUE", line);
+ failed = TRUE;
+ } else if (UTF8Matcher != NULL && (flags.indexOf((UChar)0x79) >= 0) && // 'Y' flag: RequireEnd() == false
+ UTF8Matcher->requireEnd() == FALSE) {
+ errln("Error at line %d: requireEnd() returned FALSE. Expected TRUE (UTF8)", line);
+ failed = TRUE;
+ }
+
+ if ((flags.indexOf((UChar)0x5A) >= 0) && // 'Z' flag: hitEnd() == false
+ matcher->hitEnd() == TRUE) {
+ errln("Error at line %d: hitEnd() returned TRUE. Expected FALSE", line);
+ failed = TRUE;
+ } else if (UTF8Matcher != NULL && (flags.indexOf((UChar)0x5A) >= 0) && // 'Z' flag: hitEnd() == false
+ UTF8Matcher->hitEnd() == TRUE) {
+ errln("Error at line %d: hitEnd() returned TRUE. Expected FALSE (UTF8)", line);
+ failed = TRUE;
+ }
+
+ if ((flags.indexOf((UChar)0x7A) >= 0) && // 'z' flag: hitEnd() == true
+ matcher->hitEnd() == FALSE) {
+ errln("Error at line %d: hitEnd() returned FALSE. Expected TRUE", line);
+ failed = TRUE;
+ } else if (UTF8Matcher != NULL && (flags.indexOf((UChar)0x7A) >= 0) && // 'z' flag: hitEnd() == true
+ UTF8Matcher->hitEnd() == FALSE) {
+ errln("Error at line %d: hitEnd() returned FALSE. Expected TRUE (UTF8)", line);
+ failed = TRUE;
+ }
+
+
+cleanupAndReturn:
+ if (failed) {
+ infoln((UnicodeString)"\""+pattern+(UnicodeString)"\" "
+ +flags+(UnicodeString)" \""+inputString+(UnicodeString)"\"");
+ // callerPattern->dump();
+ }
+ delete parseMatcher;
+ delete parsePat;
+ delete UTF8Matcher;
+ delete UTF8Pattern;
+ delete matcher;
+ delete callerPattern;
+
+ utext_close(&inputText);
+ delete[] inputChars;
+ utext_close(&patternText);
+ delete[] patternChars;
+ ucnv_close(UTF8Converter);
+}
+
+
+
+
+//---------------------------------------------------------------------------
+//
+// Errors Check for error handling in patterns.
+//
+//---------------------------------------------------------------------------
+void RegexTest::Errors() {
+ // \escape sequences that aren't implemented yet.
+ //REGEX_ERR("hex format \\x{abcd} not implemented", 1, 13, U_REGEX_UNIMPLEMENTED);
+
+ // Missing close parentheses
+ REGEX_ERR("Comment (?# with no close", 1, 25, U_REGEX_MISMATCHED_PAREN);
+ REGEX_ERR("Capturing Parenthesis(...", 1, 25, U_REGEX_MISMATCHED_PAREN);
+ REGEX_ERR("Grouping only parens (?: blah blah", 1, 34, U_REGEX_MISMATCHED_PAREN);
+
+ // Extra close paren
+ REGEX_ERR("Grouping only parens (?: blah)) blah", 1, 31, U_REGEX_MISMATCHED_PAREN);
+ REGEX_ERR(")))))))", 1, 1, U_REGEX_MISMATCHED_PAREN);
+ REGEX_ERR("(((((((", 1, 7, U_REGEX_MISMATCHED_PAREN);
+
+ // Look-ahead, Look-behind
+ // TODO: add tests for unbounded length look-behinds.
+ REGEX_ERR("abc(?<@xyz).*", 1, 7, U_REGEX_RULE_SYNTAX); // illegal construct
+
+ // Attempt to use non-default flags
+ {
+ UParseError pe;
+ UErrorCode status = U_ZERO_ERROR;
+ int32_t flags = UREGEX_CANON_EQ |
+ UREGEX_COMMENTS | UREGEX_DOTALL |
+ UREGEX_MULTILINE;
+ RegexPattern *pat1= RegexPattern::compile(".*", flags, pe, status);
+ REGEX_ASSERT(status == U_REGEX_UNIMPLEMENTED);
+ delete pat1;
+ }
+
+
+ // Quantifiers are allowed only after something that can be quantified.
+ REGEX_ERR("+", 1, 1, U_REGEX_RULE_SYNTAX);
+ REGEX_ERR("abc\ndef(*2)", 2, 5, U_REGEX_RULE_SYNTAX);
+ REGEX_ERR("abc**", 1, 5, U_REGEX_RULE_SYNTAX);
+
+ // Mal-formed {min,max} quantifiers
+ REGEX_ERR("abc{a,2}",1,5, U_REGEX_BAD_INTERVAL);
+ REGEX_ERR("abc{4,2}",1,8, U_REGEX_MAX_LT_MIN);
+ REGEX_ERR("abc{1,b}",1,7, U_REGEX_BAD_INTERVAL);
+ REGEX_ERR("abc{1,,2}",1,7, U_REGEX_BAD_INTERVAL);
+ REGEX_ERR("abc{1,2a}",1,8, U_REGEX_BAD_INTERVAL);
+ REGEX_ERR("abc{222222222222222222222}",1,14, U_REGEX_NUMBER_TOO_BIG);
+ REGEX_ERR("abc{5,50000000000}", 1, 17, U_REGEX_NUMBER_TOO_BIG); // Overflows int during scan
+ REGEX_ERR("abc{5,687865858}", 1, 16, U_REGEX_NUMBER_TOO_BIG); // Overflows regex binary format
+ REGEX_ERR("abc{687865858,687865859}", 1, 24, U_REGEX_NUMBER_TOO_BIG);
+
+ // Ticket 5389
+ REGEX_ERR("*c", 1, 1, U_REGEX_RULE_SYNTAX);
+
+ // Invalid Back Reference \0
+ // For ICU 3.8 and earlier
+ // For ICU versions newer than 3.8, \0 introduces an octal escape.
+ //
+ REGEX_ERR("(ab)\\0", 1, 6, U_REGEX_BAD_ESCAPE_SEQUENCE);
+
+}
+
+
+//-------------------------------------------------------------------------------
+//
+// Read a text data file, convert it to UChars, and return the data
+// in one big UChar * buffer, which the caller must delete.
+//
+//--------------------------------------------------------------------------------
+UChar *RegexTest::ReadAndConvertFile(const char *fileName, int32_t &ulen,
+ const char *defEncoding, UErrorCode &status) {
+ UChar *retPtr = NULL;
+ char *fileBuf = NULL;
+ UConverter* conv = NULL;
+ FILE *f = NULL;
+
+ ulen = 0;
+ if (U_FAILURE(status)) {
+ return retPtr;
+ }
+
+ //
+ // Open the file.
+ //
+ f = fopen(fileName, "rb");
+ if (f == 0) {
+ dataerrln("Error opening test data file %s\n", fileName);
+ status = U_FILE_ACCESS_ERROR;
+ return NULL;
+ }
+ //
+ // Read it in
+ //
+ int32_t fileSize;
+ int32_t amt_read;
+
+ fseek( f, 0, SEEK_END);
+ fileSize = ftell(f);
+ fileBuf = new char[fileSize];
+ fseek(f, 0, SEEK_SET);
+ amt_read = fread(fileBuf, 1, fileSize, f);
+ if (amt_read != fileSize || fileSize <= 0) {
+ errln("Error reading test data file.");
+ goto cleanUpAndReturn;
+ }
+
+ //
+ // Look for a Unicode Signature (BOM) on the data just read
+ //
+ int32_t signatureLength;
+ const char * fileBufC;
+ const char* encoding;
+
+ fileBufC = fileBuf;
+ encoding = ucnv_detectUnicodeSignature(
+ fileBuf, fileSize, &signatureLength, &status);
+ if(encoding!=NULL ){
+ fileBufC += signatureLength;
+ fileSize -= signatureLength;
+ } else {
+ encoding = defEncoding;
+ if (strcmp(encoding, "utf-8") == 0) {
+ errln("file %s is missing its BOM", fileName);
+ }
+ }
+
+ //
+ // Open a converter to take the rule file to UTF-16
+ //
+ conv = ucnv_open(encoding, &status);
+ if (U_FAILURE(status)) {
+ goto cleanUpAndReturn;
+ }
+
+ //
+ // Convert the rules to UChar.
+ // Preflight first to determine required buffer size.
+ //
+ ulen = ucnv_toUChars(conv,
+ NULL, // dest,
+ 0, // destCapacity,
+ fileBufC,
+ fileSize,
+ &status);
+ if (status == U_BUFFER_OVERFLOW_ERROR) {
+ // Buffer Overflow is expected from the preflight operation.
+ status = U_ZERO_ERROR;
+
+ retPtr = new UChar[ulen+1];
+ ucnv_toUChars(conv,
+ retPtr, // dest,
+ ulen+1,
+ fileBufC,
+ fileSize,
+ &status);
+ }
+
+cleanUpAndReturn:
+ fclose(f);
+ delete[] fileBuf;
+ ucnv_close(conv);
+ if (U_FAILURE(status)) {
+ errln("ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status));
+ delete []retPtr;
+ retPtr = 0;
+ ulen = 0;
+ };
+ return retPtr;
+}
+
+
+//-------------------------------------------------------------------------------
+//
+// PerlTests - Run Perl's regular expression tests
+// The input file for this test is re_tests, the standard regular
+// expression test data distributed with the Perl source code.
+//
+// Here is Perl's description of the test data file:
+//
+// # The tests are in a separate file 't/op/re_tests'.
+// # Each line in that file is a separate test.
+// # There are five columns, separated by tabs.
+// #
+// # Column 1 contains the pattern, optionally enclosed in C<''>.
+// # Modifiers can be put after the closing C<'>.
+// #
+// # Column 2 contains the string to be matched.
+// #
+// # Column 3 contains the expected result:
+// # y expect a match
+// # n expect no match
+// # c expect an error
+// # B test exposes a known bug in Perl, should be skipped
+// # b test exposes a known bug in Perl, should be skipped if noamp
+// #
+// # Columns 4 and 5 are used only if column 3 contains C<y> or C<c>.
+// #
+// # Column 4 contains a string, usually C<$&>.
+// #
+// # Column 5 contains the expected result of double-quote
+// # interpolating that string after the match, or start of error message.
+// #
+// # Column 6, if present, contains a reason why the test is skipped.
+// # This is printed with "skipped", for harness to pick up.
+// #
+// # \n in the tests are interpolated, as are variables of the form ${\w+}.
+// #
+// # If you want to add a regular expression test that can't be expressed
+// # in this format, don't add it here: put it in op/pat.t instead.
+//
+// For ICU, if field 3 contains an 'i', the test will be skipped.
+// The test exposes is some known incompatibility between ICU and Perl regexps.
+// (The i is in addition to whatever was there before.)
+//
+//-------------------------------------------------------------------------------
+void RegexTest::PerlTests() {
+ char tdd[2048];
+ const char *srcPath;
+ UErrorCode status = U_ZERO_ERROR;
+ UParseError pe;
+
+ //
+ // Open and read the test data file.
+ //
+ srcPath=getPath(tdd, "re_tests.txt");
+ if(srcPath==NULL) {
+ return; /* something went wrong, error already output */
+ }
+
+ int32_t len;
+ UChar *testData = ReadAndConvertFile(srcPath, len, "iso-8859-1", status);
+ if (U_FAILURE(status)) {
+ return; /* something went wrong, error already output */
+ }
+
+ //
+ // Put the test data into a UnicodeString
+ //
+ UnicodeString testDataString(FALSE, testData, len);
+
+ //
+ // Regex to break the input file into lines, and strip the new lines.
+ // One line per match, capture group one is the desired data.
+ //
+ RegexPattern* linePat = RegexPattern::compile(UNICODE_STRING_SIMPLE("(.+?)[\\r\\n]+"), 0, pe, status);
+ if (U_FAILURE(status)) {
+ dataerrln("RegexPattern::compile() error");
+ return;
+ }
+ RegexMatcher* lineMat = linePat->matcher(testDataString, status);
+
+ //
+ // Regex to split a test file line into fields.
+ // There are six fields, separated by tabs.
+ //
+ RegexPattern* fieldPat = RegexPattern::compile(UNICODE_STRING_SIMPLE("\\t"), 0, pe, status);
+
+ //
+ // Regex to identify test patterns with flag settings, and to separate them.
+ // Test patterns with flags look like 'pattern'i
+ // Test patterns without flags are not quoted: pattern
+ // Coming out, capture group 2 is the pattern, capture group 3 is the flags.
+ //
+ RegexPattern *flagPat = RegexPattern::compile(UNICODE_STRING_SIMPLE("('?)(.*)\\1(.*)"), 0, pe, status);
+ RegexMatcher* flagMat = flagPat->matcher(status);
+
+ //
+ // The Perl tests reference several perl-isms, which are evaluated/substituted
+ // in the test data. Not being perl, this must be done explicitly. Here
+ // are string constants and REs for these constructs.
+ //
+ UnicodeString nulnulSrc("${nulnul}");
+ UnicodeString nulnul("\\u0000\\u0000", -1, US_INV);
+ nulnul = nulnul.unescape();
+
+ UnicodeString ffffSrc("${ffff}");
+ UnicodeString ffff("\\uffff", -1, US_INV);
+ ffff = ffff.unescape();
+
+ // regexp for $-[0], $+[2], etc.
+ RegexPattern *groupsPat = RegexPattern::compile(UNICODE_STRING_SIMPLE("\\$([+\\-])\\[(\\d+)\\]"), 0, pe, status);
+ RegexMatcher *groupsMat = groupsPat->matcher(status);
+
+ // regexp for $0, $1, $2, etc.
+ RegexPattern *cgPat = RegexPattern::compile(UNICODE_STRING_SIMPLE("\\$(\\d+)"), 0, pe, status);
+ RegexMatcher *cgMat = cgPat->matcher(status);
+
+
+ //
+ // Main Loop for the Perl Tests, runs once per line from the
+ // test data file.
+ //
+ int32_t lineNum = 0;
+ int32_t skippedUnimplementedCount = 0;
+ while (lineMat->find()) {
+ lineNum++;
+
+ //
+ // Get a line, break it into its fields, do the Perl
+ // variable substitutions.
+ //
+ UnicodeString line = lineMat->group(1, status);
+ UnicodeString fields[7];
+ fieldPat->split(line, fields, 7, status);
+
+ flagMat->reset(fields[0]);
+ flagMat->matches(status);
+ UnicodeString pattern = flagMat->group(2, status);
+ pattern.findAndReplace("${bang}", "!");
+ pattern.findAndReplace(nulnulSrc, UNICODE_STRING_SIMPLE("\\u0000\\u0000"));
+ pattern.findAndReplace(ffffSrc, ffff);
+
+ //
+ // Identify patterns that include match flag settings,
+ // split off the flags, remove the extra quotes.
+ //
+ UnicodeString flagStr = flagMat->group(3, status);
+ if (U_FAILURE(status)) {
+ errln("ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status));
+ return;
+ }
+ int32_t flags = 0;
+ const UChar UChar_c = 0x63; // Char constants for the flag letters.
+ const UChar UChar_i = 0x69; // (Damn the lack of Unicode support in C)
+ const UChar UChar_m = 0x6d;
+ const UChar UChar_x = 0x78;
+ const UChar UChar_y = 0x79;
+ if (flagStr.indexOf(UChar_i) != -1) {
+ flags |= UREGEX_CASE_INSENSITIVE;
+ }
+ if (flagStr.indexOf(UChar_m) != -1) {
+ flags |= UREGEX_MULTILINE;
+ }
+ if (flagStr.indexOf(UChar_x) != -1) {
+ flags |= UREGEX_COMMENTS;
+ }
+
+ //
+ // Compile the test pattern.
+ //
+ status = U_ZERO_ERROR;
+ RegexPattern *testPat = RegexPattern::compile(pattern, flags, pe, status);
+ if (status == U_REGEX_UNIMPLEMENTED) {
+ //
+ // Test of a feature that is planned for ICU, but not yet implemented.
+ // skip the test.
+ skippedUnimplementedCount++;
+ delete testPat;
+ status = U_ZERO_ERROR;
+ continue;
+ }
+
+ if (U_FAILURE(status)) {
+ // Some tests are supposed to generate errors.
+ // Only report an error for tests that are supposed to succeed.
+ if (fields[2].indexOf(UChar_c) == -1 && // Compilation is not supposed to fail AND
+ fields[2].indexOf(UChar_i) == -1) // it's not an accepted ICU incompatibility
+ {
+ errln("line %d: ICU Error \"%s\"\n", lineNum, u_errorName(status));
+ }
+ status = U_ZERO_ERROR;
+ delete testPat;
+ continue;
+ }
+
+ if (fields[2].indexOf(UChar_i) >= 0) {
+ // ICU should skip this test.
+ delete testPat;
+ continue;
+ }
+
+ if (fields[2].indexOf(UChar_c) >= 0) {
+ // This pattern should have caused a compilation error, but didn't/
+ errln("line %d: Expected a pattern compile error, got success.", lineNum);
+ delete testPat;
+ continue;
+ }
+
+ //
+ // replace the Perl variables that appear in some of the
+ // match data strings.
+ //
+ UnicodeString matchString = fields[1];
+ matchString.findAndReplace(nulnulSrc, nulnul);
+ matchString.findAndReplace(ffffSrc, ffff);
+
+ // Replace any \n in the match string with an actual new-line char.
+ // Don't do full unescape, as this unescapes more than Perl does, which
+ // causes other spurious failures in the tests.
+ matchString.findAndReplace(UNICODE_STRING_SIMPLE("\\n"), "\n");
+
+
+
+ //
+ // Run the test, check for expected match/don't match result.
+ //
+ RegexMatcher *testMat = testPat->matcher(matchString, status);
+ UBool found = testMat->find();
+ UBool expected = FALSE;
+ if (fields[2].indexOf(UChar_y) >=0) {
+ expected = TRUE;
+ }
+ if (expected != found) {
+ errln("line %d: Expected %smatch, got %smatch",
+ lineNum, expected?"":"no ", found?"":"no " );
+ continue;
+ }
+
+ // Don't try to check expected results if there is no match.
+ // (Some have stuff in the expected fields)
+ if (!found) {
+ delete testMat;
+ delete testPat;
+ continue;
+ }
+
+ //
+ // Interpret the Perl expression from the fourth field of the data file,
+ // building up an ICU string from the results of the ICU match.
+ // The Perl expression will contain references to the results of
+ // a regex match, including the matched string, capture group strings,
+ // group starting and ending indicies, etc.
+ //
+ UnicodeString resultString;
+ UnicodeString perlExpr = fields[3];
+#if SUPPORT_MUTATING_INPUT_STRING
+ groupsMat->reset(perlExpr);
+ cgMat->reset(perlExpr);
+#endif
+
+ while (perlExpr.length() > 0) {
+#if !SUPPORT_MUTATING_INPUT_STRING
+ // Perferred usage. Reset after any modification to input string.
+ groupsMat->reset(perlExpr);
+ cgMat->reset(perlExpr);
+#endif
+
+ if (perlExpr.startsWith("$&")) {
+ resultString.append(testMat->group(status));
+ perlExpr.remove(0, 2);
+ }
+
+ else if (groupsMat->lookingAt(status)) {
+ // $-[0] $+[2] etc.
+ UnicodeString digitString = groupsMat->group(2, status);
+ int32_t t = 0;
+ int32_t groupNum = ICU_Utility::parseNumber(digitString, t, 10);
+ UnicodeString plusOrMinus = groupsMat->group(1, status);
+ int32_t matchPosition;
+ if (plusOrMinus.compare("+") == 0) {
+ matchPosition = testMat->end(groupNum, status);
+ } else {
+ matchPosition = testMat->start(groupNum, status);
+ }
+ if (matchPosition != -1) {
+ ICU_Utility::appendNumber(resultString, matchPosition);
+ }
+ perlExpr.remove(0, groupsMat->end(status));
+ }
+
+ else if (cgMat->lookingAt(status)) {
+ // $1, $2, $3, etc.
+ UnicodeString digitString = cgMat->group(1, status);
+ int32_t t = 0;
+ int32_t groupNum = ICU_Utility::parseNumber(digitString, t, 10);
+ if (U_SUCCESS(status)) {
+ resultString.append(testMat->group(groupNum, status));
+ status = U_ZERO_ERROR;
+ }
+ perlExpr.remove(0, cgMat->end(status));
+ }
+
+ else if (perlExpr.startsWith("@-")) {
+ int32_t i;
+ for (i=0; i<=testMat->groupCount(); i++) {
+ if (i>0) {
+ resultString.append(" ");
+ }
+ ICU_Utility::appendNumber(resultString, testMat->start(i, status));
+ }
+ perlExpr.remove(0, 2);
+ }
+
+ else if (perlExpr.startsWith("@+")) {
+ int32_t i;
+ for (i=0; i<=testMat->groupCount(); i++) {
+ if (i>0) {
+ resultString.append(" ");
+ }
+ ICU_Utility::appendNumber(resultString, testMat->end(i, status));
+ }
+ perlExpr.remove(0, 2);
+ }
+
+ else if (perlExpr.startsWith(UNICODE_STRING_SIMPLE("\\"))) { // \Escape. Take following char as a literal.
+ // or as an escaped sequence (e.g. \n)
+ if (perlExpr.length() > 1) {
+ perlExpr.remove(0, 1); // Remove the '\', but only if not last char.
+ }
+ UChar c = perlExpr.charAt(0);
+ switch (c) {
+ case 'n': c = '\n'; break;
+ // add any other escape sequences that show up in the test expected results.
+ }
+ resultString.append(c);
+ perlExpr.remove(0, 1);
+ }
+
+ else {
+ // Any characters from the perl expression that we don't explicitly
+ // recognize before here are assumed to be literals and copied
+ // as-is to the expected results.
+ resultString.append(perlExpr.charAt(0));
+ perlExpr.remove(0, 1);
+ }
+
+ if (U_FAILURE(status)) {
+ errln("Line %d: ICU Error \"%s\"", lineNum, u_errorName(status));
+ break;
+ }
+ }
+
+ //
+ // Expected Results Compare
+ //
+ UnicodeString expectedS(fields[4]);
+ expectedS.findAndReplace(nulnulSrc, nulnul);
+ expectedS.findAndReplace(ffffSrc, ffff);
+ expectedS.findAndReplace(UNICODE_STRING_SIMPLE("\\n"), "\n");
+
+
+ if (expectedS.compare(resultString) != 0) {
+ err("Line %d: Incorrect perl expression results.", lineNum);
+ infoln((UnicodeString)"Expected \""+expectedS+(UnicodeString)"\"; got \""+resultString+(UnicodeString)"\"");
+ }
+
+ delete testMat;
+ delete testPat;
+ }
+
+ //
+ // All done. Clean up allocated stuff.
+ //
+ delete cgMat;
+ delete cgPat;
+
+ delete groupsMat;
+ delete groupsPat;
+
+ delete flagMat;
+ delete flagPat;
projects / apple / icu.git / blobdiff