X-Git-Url: https://git.saurik.com/apple/icu.git/blobdiff_plain/73c04bcfe1096173b00431f0cdc742894b15eef0..2be6500137966bba13903b24204a44804f9e133a:/icuSources/test/intltest/rbbitst.cpp diff --git a/icuSources/test/intltest/rbbitst.cpp b/icuSources/test/intltest/rbbitst.cpp index 34deb932..eddb99a4 100644 --- a/icuSources/test/intltest/rbbitst.cpp +++ b/icuSources/test/intltest/rbbitst.cpp @@ -1,6 +1,6 @@ /******************************************************************** * COPYRIGHT: - * Copyright (c) 1999-2006, International Business Machines Corporation and + * Copyright (c) 1999-2013, International Business Machines Corporation and * others. All Rights Reserved. ********************************************************************/ /************************************************************************ @@ -9,6 +9,8 @@ * 01/12/2000 Madhu Updated for changed API and added new tests ************************************************************************/ +#include "utypeinfo.h" // for 'typeid' to work + #include "unicode/utypes.h" #if !UCONFIG_NO_BREAK_ITERATION @@ -21,7 +23,9 @@ #include "unicode/ucnv.h" #include "unicode/schriter.h" #include "unicode/uniset.h" -#include "unicode/regex.h" // TODO: make conditional on regexp being built. +#if !UCONFIG_NO_REGULAR_EXPRESSIONS +#include "unicode/regex.h" +#endif #include "unicode/ustring.h" #include "unicode/utext.h" #include "intltest.h" @@ -29,16 +33,124 @@ #include #include "uvector.h" #include "uvectr32.h" -#include "triedict.h" #include #include #include +#include "unicode/numfmt.h" +#include "unicode/uscript.h" #define TEST_ASSERT(x) {if (!(x)) { \ errln("Failure in file %s, line %d", __FILE__, __LINE__);}} -#define TEST_ASSERT_SUCCESS(errcode) {if (U_FAILURE(errcode)) { \ - errln("Failure in file %s, line %d, status = \"%s\"", __FILE__, __LINE__, u_errorName(errcode));}} +#define TEST_ASSERT_SUCCESS(errcode) { if (U_FAILURE(errcode)) { \ + errcheckln(errcode, "Failure in file %s, line %d, status = \"%s\"", __FILE__, __LINE__, u_errorName(errcode));}} + + +//--------------------------------------------- +// runIndexedTest +//--------------------------------------------- + + +// Note: Before adding new tests to this file, check whether the desired test data can +// simply be added to the file testdata/rbbitest.txt. In most cases it can, +// it's much less work than writing a new test, diagnostic output in the event of failures +// is good, and the test data file will is shared with ICU4J, so eventually the test +// will run there as well, without additional effort. + +void RBBITest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* params ) +{ + if (exec) logln("TestSuite RuleBasedBreakIterator: "); + + switch (index) { +#if !UCONFIG_NO_FILE_IO + case 0: name = "TestBug4153072"; + if(exec) TestBug4153072(); break; +#else + case 0: name = "skip"; + break; +#endif + + case 1: name = "skip"; + break; + case 2: name = "TestStatusReturn"; + if(exec) TestStatusReturn(); break; + +#if !UCONFIG_NO_FILE_IO + case 3: name = "TestUnicodeFiles"; + if(exec) TestUnicodeFiles(); break; + case 4: name = "TestEmptyString"; + if(exec) TestEmptyString(); break; +#else + case 3: case 4: name = "skip"; + break; +#endif + + case 5: name = "TestGetAvailableLocales"; + if(exec) TestGetAvailableLocales(); break; + + case 6: name = "TestGetDisplayName"; + if(exec) TestGetDisplayName(); break; + +#if !UCONFIG_NO_FILE_IO + case 7: name = "TestEndBehaviour"; + if(exec) TestEndBehaviour(); break; + case 8: case 9: case 10: name = "skip"; + break; + case 11: name = "TestWordBreaks"; + if(exec) TestWordBreaks(); break; + case 12: name = "TestWordBoundary"; + if(exec) TestWordBoundary(); break; + case 13: name = "TestLineBreaks"; + if(exec) TestLineBreaks(); break; + case 14: name = "TestSentBreaks"; + if(exec) TestSentBreaks(); break; + case 15: name = "TestExtended"; + if(exec) TestExtended(); break; +#else + case 7: case 8: case 9: case 10: case 11: case 12: case 13: case 14: case 15: name = "skip"; + break; +#endif + +#if !UCONFIG_NO_REGULAR_EXPRESSIONS && !UCONFIG_NO_FILE_IO + case 16: + name = "TestMonkey"; if(exec) TestMonkey(params); break; +#else + case 16: + name = "skip"; break; +#endif + +#if !UCONFIG_NO_FILE_IO + case 17: name = "TestBug3818"; + if(exec) TestBug3818(); break; +#else + case 17: name = "skip"; + break; +#endif + + case 18: name = "skip"; + break; + case 19: name = "TestDebug"; + if(exec) TestDebug(); break; + case 20: name = "skip"; + break; + +#if !UCONFIG_NO_FILE_IO + case 21: name = "TestBug5775"; + if (exec) TestBug5775(); break; +#else + case 21: name = "skip"; + break; +#endif + + case 22: name = "TestBug9983"; + if (exec) TestBug9983(); break; + case 23: name = "TestDictRules"; + if (exec) TestDictRules(); break; + case 24: name = "TestBug5532"; + if (exec) TestBug5532(); break; + default: name = ""; break; //needed to end loop + } +} //--------------------------------------------------------------------------- @@ -169,9 +281,9 @@ void BITestData::err(const char *heading, RBBITest *test, int32_t expectedIdx o = actual - fExpectedBreakPositions.elementAti(expectedIdx-1); } if (actual < expected) { - test->errln("%s unexpected break at offset %d in test item from line %d", heading, o, line); + test->errln("%s unexpected break at offset %d in test item from line %d. actual break: %d expected break: %d", heading, o, line, actual, expected); } else { - test->errln("%s Failed to find break at end of item from line %d", heading, line); + test->errln("%s Failed to find break at end of item from line %d. actual break: %d expected break: %d", heading, line, actual, expected); } } @@ -182,31 +294,6 @@ void BITestData::clearResults() { } -//----------------------------------------------------------------------------------- -// -// Cannned Test Characters -// -//----------------------------------------------------------------------------------- - -static const UChar cannedTestArray[] = { - 0x0001, 0x0002, 0x0003, 0x0004, 0x0020, 0x0021, '\\', 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0028, 0x0029, 0x002b, 0x002d, 0x0030, 0x0031, - 0x0032, 0x0033, 0x0034, 0x003c, 0x003d, 0x003e, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x005b, 0x005d, 0x005e, 0x005f, 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x007b, - 0x007d, 0x007c, 0x002c, 0x00a0, 0x00a2, - 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00ab, 0x00ad, 0x00ae, 0x00af, 0x00b0, 0x00b2, 0x00b3, - 0x00b4, 0x00b9, 0x00bb, 0x00bc, 0x00bd, 0x02b0, 0x02b1, 0x02b2, 0x02b3, 0x02b4, 0x0300, 0x0301, 0x0302, 0x0303, - 0x0304, 0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x0903, 0x093e, 0x093f, 0x0940, 0x0949, 0x0f3a, 0x0f3b, 0x2000, - 0x2001, 0x2002, 0x200c, 0x200d, 0x200e, 0x200f, 0x2010, 0x2011, 0x2012, 0x2028, 0x2029, 0x202a, 0x203e, 0x203f, - 0x2040, 0x20dd, 0x20de, 0x20df, 0x20e0, 0x2160, 0x2161, 0x2162, 0x2163, 0x2164, 0x0000 -}; - -static UnicodeString* cannedTestChars = 0; - -#define halfNA "\\u0928\\u094d\\u200d" -#define halfSA "\\u0938\\u094d\\u200d" -#define halfCHA "\\u091a\\u094d\\u200d" -#define halfKA "\\u0915\\u094d\\u200d" -#define deadTA "\\u0924\\u094d" - //-------------------------------------------------------------------------------------- // // RBBITest constructor and destructor @@ -214,44 +301,12 @@ static UnicodeString* cannedTestChars = 0; //-------------------------------------------------------------------------------------- RBBITest::RBBITest() { - UnicodeString temp(cannedTestArray); - cannedTestChars = new UnicodeString(); - *cannedTestChars += (UChar)0x0000; - *cannedTestChars += temp; } RBBITest::~RBBITest() { - delete cannedTestChars; } - -static const int T_NUMBER = 100; -static const int T_LETTER = 200; -static const int T_H_OR_K = 300; -static const int T_IDEO = 400; - - - - - - -//-------------------------------------------------------------------- -//Testing the BreakIterator for devanagari script -//-------------------------------------------------------------------- - -#define deadRA "\\u0930\\u094d" /*deadform RA = devanagari RA + virama*/ -#define deadPHA "\\u092b\\u094d" /*deadform PHA = devanagari PHA + virama*/ -#define deadTTHA "\\u0920\\u094d" -#define deadPA "\\u092a\\u094d" -#define deadSA "\\u0938\\u094d" -#define visarga "\\u0903" /*devanagari visarga looks like a english colon*/ - - - - - - //----------------------------------------------------------------------------------- // // Test for status {tag} return value from break rules. @@ -259,13 +314,13 @@ static const int T_IDEO = 400; // //----------------------------------------------------------------------------------- void RBBITest::TestStatusReturn() { - UnicodeString rulesString1 = "$Letters = [:L:];\n" + UnicodeString rulesString1("$Letters = [:L:];\n" "$Numbers = [:N:];\n" "$Letters+{1};\n" "$Numbers+{2};\n" "Help\\ {4}/me\\!;\n" "[^$Letters $Numbers];\n" - "!.*;\n"; + "!.*;\n", -1, US_INV); UnicodeString testString1 = "abc123..abc Help me Help me!"; // 01234567890123456789012345678 int32_t bounds1[] = {0, 3, 6, 7, 8, 11, 12, 16, 17, 19, 20, 25, 27, 28, -1}; @@ -276,7 +331,7 @@ void RBBITest::TestStatusReturn() { RuleBasedBreakIterator *bi = new RuleBasedBreakIterator(rulesString1, parseError, status); if(U_FAILURE(status)) { - errln("FAIL : in construction"); + dataerrln("FAIL : in construction - %s", u_errorName(status)); } else { int32_t pos; int32_t i = 0; @@ -344,127 +399,6 @@ static void printStringBreaks(UnicodeString ustr, int expected[], } } -void RBBITest::TestThaiLineBreak() { - UErrorCode status = U_ZERO_ERROR; - BITestData thaiLineSelection(status); - - // \u0e2f-- the Thai paiyannoi character-- isn't a letter. It's a symbol that - // represents elided letters at the end of a long word. It should be bound to - // the end of the word and not treated as an independent punctuation mark. - - - ADD_DATACHUNK(thaiLineSelection, NULL, 0, status); // Break at start of data - ADD_DATACHUNK(thaiLineSelection, "\\u0e2a\\u0e16\\u0e32\\u0e19\\u0e35\\u0e2f", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e08\\u0e30", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e23\\u0e30\\u0e14\\u0e21", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e08\\u0e49\\u0e32", 0, status); -// ADD_DATACHUNK(thaiLineSelection, "\\u0e2b\\u0e19\\u0e49\\u0e32", 0, status); -// ADD_DATACHUNK(thaiLineSelection, "\\u0e17\\u0e35\\u0e48", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e2b\\u0e19\\u0e49\\u0e32\\u0e17\\u0e35\\u0e48", 0, status); - // the commented-out lines (I think) are the preferred result; this line is what our current dictionary is giving us - ADD_DATACHUNK(thaiLineSelection, "\\u0e2d\\u0e2d\\u0e01", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e21\\u0e32", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e23\\u0e48\\u0e07", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e23\\u0e30\\u0e1a\\u0e32\\u0e22", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e2d\\u0e22\\u0e48\\u0e32\\u0e07", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e15\\u0e47\\u0e21", 0, status); - - // the one time where the paiyannoi occurs somewhere other than at the end - // of a word is in the Thai abbrevation for "etc.", which both begins and - // ends with a paiyannoi - ADD_DATACHUNK(thaiLineSelection, "\\u0e2f\\u0e25\\u0e2f", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e17\\u0e35\\u0e48", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e19\\u0e31\\u0e49\\u0e19", 0, status); - - RuleBasedBreakIterator* e = (RuleBasedBreakIterator *)BreakIterator::createLineInstance( - Locale("th"), status); - if (U_FAILURE(status)) - { - errln("Failed to create the BreakIterator for Thai locale in TestThaiLineBreak.\n"); - return; - } - - generalIteratorTest(*e, thaiLineSelection); - delete e; -} - - - -void RBBITest::TestMixedThaiLineBreak() -{ - UErrorCode status = U_ZERO_ERROR; - BITestData thaiLineSelection(status); - - ADD_DATACHUNK(thaiLineSelection, NULL, 0, status); // Break at start of data - - - // @suwit -- Test Arabic numerals, Thai numerals, Punctuation and English characters - // start - - ADD_DATACHUNK(thaiLineSelection, "\\u0E1B\\u0E35", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0E1E\\u0E38\\u0E17\\u0E18\\u0E28\\u0E31\\u0E01\\u0E23\\u0E32\\u0E0A ", 0, status); - ADD_DATACHUNK(thaiLineSelection, "2545 ", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0E40\\u0E1B\\u0E47\\u0E19", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0E1B\\u0E35", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0E09\\u0E25\\u0E2D\\u0E07", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0E04\\u0E23\\u0E1A", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0E23\\u0E2D\\u0E1A ", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\"\\u0E52\\u0E52\\u0E50 ", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0E1b\\u0E35\" ", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0E02\\u0E2d\\u0E07", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0E01\\u0E23\\u0E38\\u0E07", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0E23\\u0E31\\u0E15\\u0E19\\u0E42\\u0E01\\u0E2A\\u0E34\\u0E19\\u0E17\\u0E23\\u0E4C ", 0, status); - ADD_DATACHUNK(thaiLineSelection, "(\\u0E01\\u0E23\\u0E38\\u0E07\\u0E40\\u0E17\\u0E1e\\u0E2F", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0E2B\\u0E23\\u0E37\\u0E2D ", 0, status); - ADD_DATACHUNK(thaiLineSelection, "Bangkok)", 0, status); - - // @suwit - end of changes - - - RuleBasedBreakIterator* e = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(Locale("th"), status); - if (U_FAILURE(status)) - { - errln("Failed to create the BreakIterator for Thai locale in TestMixedThaiLineBreak.\n"); - return; - } - - - generalIteratorTest(*e, thaiLineSelection); - delete e; -} - - -void RBBITest::TestMaiyamok() -{ - UErrorCode status = U_ZERO_ERROR; - BITestData thaiLineSelection(status); - ADD_DATACHUNK(thaiLineSelection, NULL, 0, status); // Break at start of data - // the Thai maiyamok character is a shorthand symbol that means "repeat the previous - // word". Instead of appearing as a word unto itself, however, it's kept together - // with the word before it - ADD_DATACHUNK(thaiLineSelection, "\\u0e44\\u0e1b\\u0e46", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e21\\u0e32\\u0e46", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e23\\u0e30\\u0e2b\\u0e27\\u0e48\\u0e32\\u0e07", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e01\\u0e23\\u0e38\\u0e07", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e17\\u0e1e", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e41\\u0e25\\u0e30", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e03\\u0e35", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e22\\u0e07", 0, status); - ADD_DATACHUNK(thaiLineSelection, "\\u0e43\\u0e2b\\u0e21\\u0e48", 0, status); - - RuleBasedBreakIterator* e = (RuleBasedBreakIterator *)BreakIterator::createLineInstance( - Locale("th"), status); - - if (U_FAILURE(status)) - { - errln("Failed to create the BreakIterator for Thai locale in TestMaiyamok.\n"); - return; - } - generalIteratorTest(*e, thaiLineSelection); - delete e; -} - - void RBBITest::TestBug3818() { UErrorCode status = U_ZERO_ERROR; @@ -477,7 +411,7 @@ void RBBITest::TestBug3818() { RuleBasedBreakIterator* bi = (RuleBasedBreakIterator *)BreakIterator::createWordInstance(Locale("th"), status); if (U_FAILURE(status) || bi == NULL) { - errln("Fail at file %s, line %d, status = %s", __FILE__, __LINE__, u_errorName(status)); + errcheckln(status, "Fail at file %s, line %d, status = %s", __FILE__, __LINE__, u_errorName(status)); return; } bi->setText(thaiStr); @@ -495,315 +429,6 @@ void RBBITest::TestBug3818() { delete bi; } - -void RBBITest::TestJapaneseWordBreak() { - UErrorCode status = U_ZERO_ERROR; - BITestData japaneseWordSelection(status); - - ADD_DATACHUNK(japaneseWordSelection, NULL, 0, status); // Break at start of data - ADD_DATACHUNK(japaneseWordSelection, "\\u4ECA\\u65E5", 400, status); //2 - ADD_DATACHUNK(japaneseWordSelection, "\\u306F\\u3044\\u3044", 300, status); //5 - ADD_DATACHUNK(japaneseWordSelection, "\\u5929\\u6C17", 400, status); //7 - ADD_DATACHUNK(japaneseWordSelection, "\\u3067\\u3059\\u306D", 300, status); //10 - ADD_DATACHUNK(japaneseWordSelection, "\\u3002", 0, status); //11 - ADD_DATACHUNK(japaneseWordSelection, "\\u000D\\u000A", 0, status); //12 - - RuleBasedBreakIterator* e = (RuleBasedBreakIterator *)BreakIterator::createWordInstance( - Locale("ja"), status); - if (U_FAILURE(status)) - { - errln("Failed to create the BreakIterator for Japanese locale in TestJapaneseWordBreak.\n"); - return; - } - - generalIteratorTest(*e, japaneseWordSelection); - delete e; -} - -void RBBITest::TestTrieDict() { - UErrorCode status = U_ZERO_ERROR; - - // - // Open and read the test data file. - // - const char *testDataDirectory = IntlTest::getSourceTestData(status); - char testFileName[1000]; - if (testDataDirectory == NULL || strlen(testDataDirectory) + strlen("riwords.txt") + 10 >= sizeof(testFileName)) { - errln("Can't open test data. Path too long."); - return; - } - strcpy(testFileName, testDataDirectory); - strcat(testFileName, "riwords.txt"); - - // Items needing deleting at the end - MutableTrieDictionary *mutableDict = NULL; - CompactTrieDictionary *compactDict = NULL; - UnicodeSet *breaks = NULL; - UChar *testFile = NULL; - StringEnumeration *enumer = NULL; - MutableTrieDictionary *mutable2 = NULL; - StringEnumeration *cloneEnum = NULL; - CompactTrieDictionary *compact2 = NULL; - - - const UnicodeString *originalWord = NULL; - const UnicodeString *cloneWord = NULL; - UChar *current; - UChar *word; - UChar uc; - int32_t wordLen; - int32_t wordCount; - int32_t testCount; - - int len; - testFile = ReadAndConvertFile(testFileName, len, status); - if (U_FAILURE(status)) { - goto cleanup; /* something went wrong, error already output */ - } - - mutableDict = new MutableTrieDictionary(0x0E1C, status); - if (U_FAILURE(status)) { - errln("Error creating MutableTrieDictionary: %s\n", u_errorName(status)); - goto cleanup; - } - - breaks = new UnicodeSet; - breaks->add(0x000A); // Line Feed - breaks->add(0x000D); // Carriage Return - breaks->add(0x2028); // Line Separator - breaks->add(0x2029); // Paragraph Separator - - // Now add each non-comment line of the file as a word. - current = testFile; - word = current; - uc = *current++; - wordLen = 0; - wordCount = 0; - - while (uc) { - if (uc == 0x0023) { // #comment line, skip - while (uc && !breaks->contains(uc)) { - uc = *current++; - } - } - else while (uc && !breaks->contains(uc)) { - ++wordLen; - uc = *current++; - } - if (wordLen > 0) { - mutableDict->addWord(word, wordLen, status); - if (U_FAILURE(status)) { - errln("Could not add word to mutable dictionary; status %s\n", u_errorName(status)); - goto cleanup; - } - wordCount += 1; - } - - // Find beginning of next line - while (uc && breaks->contains(uc)) { - uc = *current++; - } - word = current-1; - wordLen = 0; - } - - if (wordCount < 50) { - errln("Word count (%d) unreasonably small\n", wordCount); - goto cleanup; - } - - enumer = mutableDict->openWords(status); - if (U_FAILURE(status)) { - errln("Could not open mutable dictionary enumerator: %s\n", u_errorName(status)); - goto cleanup; - } - - testCount = 0; - if (wordCount != (testCount = enumer->count(status))) { - errln("MutableTrieDictionary word count (%d) differs from file word count (%d), with status %s\n", - testCount, wordCount, u_errorName(status)); - goto cleanup; - } - - delete enumer; - enumer = NULL; - - // Now compact it - compactDict = new CompactTrieDictionary(*mutableDict, status); - if (U_FAILURE(status)) { - errln("Failed to create CompactTrieDictionary: %s\n", u_errorName(status)); - goto cleanup; - } - - enumer = compactDict->openWords(status); - if (U_FAILURE(status)) { - errln("Could not open compact trie dictionary enumerator: %s\n", u_errorName(status)); - goto cleanup; - } - - if (wordCount != (testCount = enumer->count(status))) { - errln("CompactTrieDictionary word count (%d) differs from file word count (%d), with status %s\n", - testCount, wordCount, u_errorName(status)); - goto cleanup; - } - - delete enumer; - enumer = NULL; - - // Now un-compact it - mutable2 = compactDict->cloneMutable(status); - if (U_FAILURE(status)) { - errln("Could not clone CompactTrieDictionary to MutableTrieDictionary: %s\n", u_errorName(status)); - goto cleanup; - } - - cloneEnum = mutable2->openWords(status); - if (U_FAILURE(status)) { - errln("Could not create cloned mutable enumerator: %s\n", u_errorName(status)); - goto cleanup; - } - - if (wordCount != (testCount = cloneEnum->count(status))) { - errln("Cloned MutableTrieDictionary word count (%d) differs from file word count (%d), with status %s\n", - testCount, wordCount, u_errorName(status)); - goto cleanup; - } - - // Compact original dictionary to clone. Note that we can only compare the same kind of - // dictionary as the order of the enumerators is not guaranteed to be the same between - // different kinds - enumer = mutableDict->openWords(status); - if (U_FAILURE(status)) { - errln("Could not re-open mutable dictionary enumerator: %s\n", u_errorName(status)); - goto cleanup; - } - - originalWord = enumer->snext(status); - cloneWord = cloneEnum->snext(status); - while (U_SUCCESS(status) && originalWord != NULL && cloneWord != NULL) { - if (*originalWord != *cloneWord) { - errln("Original and cloned MutableTrieDictionary word mismatch\n"); - goto cleanup; - } - originalWord = enumer->snext(status); - cloneWord = cloneEnum->snext(status); - } - - if (U_FAILURE(status)) { - errln("Enumeration failed: %s\n", u_errorName(status)); - goto cleanup; - } - - if (originalWord != cloneWord) { - errln("Original and cloned MutableTrieDictionary ended enumeration at different points\n"); - goto cleanup; - } - - // Test the data copying constructor for CompactTrieDict, and the data access APIs. - compact2 = new CompactTrieDictionary(compactDict->data(), status); - if (U_FAILURE(status)) { - errln("CompactTrieDictionary(const void *,...) failed\n"); - goto cleanup; - } - - if (compact2->dataSize() == 0) { - errln("CompactTrieDictionary->dataSize() == 0\n"); - goto cleanup; - } - - // Now count the words via the second dictionary - delete enumer; - enumer = compact2->openWords(status); - if (U_FAILURE(status)) { - errln("Could not open compact trie dictionary 2 enumerator: %s\n", u_errorName(status)); - goto cleanup; - } - - if (wordCount != (testCount = enumer->count(status))) { - errln("CompactTrieDictionary 2 word count (%d) differs from file word count (%d), with status %s\n", - testCount, wordCount, u_errorName(status)); - goto cleanup; - } - -cleanup: - delete compactDict; - delete mutableDict; - delete breaks; - delete[] testFile; - delete enumer; - delete mutable2; - delete cloneEnum; - delete compact2; -} - -//--------------------------------------------- -// runIndexedTest -//--------------------------------------------- - -void RBBITest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* params ) -{ - if (exec) logln("TestSuite RuleBasedBreakIterator: "); - - switch (index) { - case 0: name = "TestBug4153072"; - if(exec) TestBug4153072(); break; - case 1: name = "TestJapaneseLineBreak"; - if(exec) TestJapaneseLineBreak(); break; - case 2: name = "TestStatusReturn"; - if(exec) TestStatusReturn(); break; - - case 3: name = "TestLineBreakData"; - if(exec) TestLineBreakData(); break; - case 4: name = "TestEmptyString"; - if(exec) TestEmptyString(); break; - - case 5: name = "TestGetAvailableLocales"; - if(exec) TestGetAvailableLocales(); break; - - case 6: name = "TestGetDisplayName"; - if(exec) TestGetDisplayName(); break; - - case 7: name = "TestEndBehaviour"; - if(exec) TestEndBehaviour(); break; - case 8: name = "TestMixedThaiLineBreak"; - if(exec) TestMixedThaiLineBreak(); break; - case 9: name = "TestThaiLineBreak"; - if(exec) TestThaiLineBreak(); break; - case 10: name = "TestMaiyamok"; - if(exec) TestMaiyamok(); break; - case 11: name = "TestWordBreaks"; - if(exec) TestWordBreaks(); break; - case 12: name = "TestWordBoundary"; - if(exec) TestWordBoundary(); break; - case 13: name = "TestLineBreaks"; - if(exec) TestLineBreaks(); break; - case 14: name = "TestSentBreaks"; - if(exec) TestSentBreaks(); break; - case 15: name = "TestExtended"; - if(exec) TestExtended(); break; - case 16: name = "TestMonkey"; - if(exec) { - #if !UCONFIG_NO_REGULAR_EXPRESSIONS - TestMonkey(params); - #else - logln("skipping TestMonkey (UCONFIG_NO_REGULAR_EXPRESSIONS)"); - #endif - } - break; - case 17: name = "TestBug3818"; - if(exec) TestBug3818(); break; - case 18: name = "TestJapaneseWordBreak"; - if(exec) TestJapaneseWordBreak(); break; - case 19: name = "TestDebug"; - if(exec) TestDebug(); break; - case 20: name = "TestTrieDict"; - if(exec) TestTrieDict(); break; - - default: name = ""; break; //needed to end loop - } -} - - //---------------------------------------------------------------------------- // // generalIteratorTest Given a break iterator and a set of test data, @@ -866,7 +491,7 @@ void RBBITest::testLastAndPrevious(RuleBasedBreakIterator& bi, BITestData &td) int32_t lastP = 0x7ffffffe; int32_t tag; - logln("Test first and next"); + logln("Test last and previous"); bi.setText(td.fDataToBreak); td.clearResults(); @@ -1069,7 +694,7 @@ void RBBITest::TestEmptyString() RuleBasedBreakIterator* bi = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(Locale::getDefault(), status); if (U_FAILURE(status)) { - errln("Failed to create the BreakIterator for default locale in TestEmptyString.\n"); + errcheckln(status, "Failed to create the BreakIterator for default locale in TestEmptyString. - %s", u_errorName(status)); return; } generalIteratorTest(*bi, x); @@ -1082,7 +707,7 @@ void RBBITest::TestGetAvailableLocales() const Locale* locList = BreakIterator::getAvailableLocales(locCount); if (locCount == 0) - errln("getAvailableLocales() returned an empty list!"); + dataerrln("getAvailableLocales() returned an empty list!"); // Just make sure that it's returning good memory. int32_t i; for (i = 0; i < locCount; ++i) { @@ -1097,12 +722,12 @@ void RBBITest::TestGetDisplayName() BreakIterator::getDisplayName(Locale::getUS(), result); if (Locale::getDefault() == Locale::getUS() && result != "English (United States)") - errln("BreakIterator::getDisplayName() failed: expected \"English (United States)\", got \"" + dataerrln("BreakIterator::getDisplayName() failed: expected \"English (United States)\", got \"" + result); BreakIterator::getDisplayName(Locale::getFrance(), Locale::getUS(), result); if (result != "French (France)") - errln("BreakIterator::getDisplayName() failed: expected \"French (France)\", got \"" + dataerrln("BreakIterator::getDisplayName() failed: expected \"French (France)\", got \"" + result); } /** @@ -1116,7 +741,7 @@ void RBBITest::TestEndBehaviour() BreakIterator *wb = BreakIterator::createWordInstance(Locale::getDefault(), status); if (U_FAILURE(status)) { - errln("Failed to create the BreakIterator for default locale in TestEndBehaviour.\n"); + errcheckln(status, "Failed to create the BreakIterator for default locale in TestEndBehaviour. - %s", u_errorName(status)); return; } wb->setText(testString); @@ -1137,7 +762,7 @@ void RBBITest::TestBug4153072() { BreakIterator *iter = BreakIterator::createWordInstance(Locale::getDefault(), status); if (U_FAILURE(status)) { - errln("Failed to create the BreakIterator for default locale in TestBug4153072\n"); + errcheckln(status, "Failed to create the BreakIterator for default locale in TestBug4153072 - %s", u_errorName(status)); return; } UnicodeString str("...Hello, World!..."); @@ -1163,77 +788,42 @@ void RBBITest::TestBug4153072() { } -/** - * Test Japanese Line Break - * @bug 4095322 - */ -void RBBITest::TestJapaneseLineBreak() -{ -#if 0 - // Test needs updating some more... Dump it for now. - - - // Change for Unicode TR 14: Punctuation characters with categories Pi and Pf do not count - // as opening and closing punctuation for line breaking. - // Also, \u30fc and \u30fe are not counted as hyphens. Remove these chars - // from these tests. 6-13-2002 - // +// +// Test for problem reported by Ashok Matoria on 9 July 2007 +// One.Two. +// +// Sentence break at start (0) and then on calling next() it breaks at +// 'T' of "Two". Now, at this point if I do next() and +// then previous(), it breaks at instead of 'T' of "Two". +// +void RBBITest::TestBug5775() { UErrorCode status = U_ZERO_ERROR; - UnicodeString testString = CharsToUnicodeString("\\u4e00x\\u4e8c"); - UnicodeString precedingChars = CharsToUnicodeString( - //"([{\\u00ab$\\u00a5\\u00a3\\u00a4\\u2018\\u201a\\u201c\\u201e\\u201b\\u201f"); - "([{$\\u00a5\\u00a3\\u00a4\\u201a\\u201e"); - UnicodeString followingChars = CharsToUnicodeString( - // ")]}\\u00bb!%,.\\u3001\\u3002\\u3063\\u3083\\u3085\\u3087\\u30c3\\u30e3\\u30e5\\u30e7\\u30fc" - ")]}!%,.\\u3001\\u3002\\u3063\\u3083\\u3085\\u3087\\u30c3\\u30e3\\u30e5\\u30e7" - // ":;\\u309b\\u309c\\u3005\\u309d\\u309e\\u30fd\\u30fe\\u2019\\u201d\\u00b0\\u2032\\u2033\\u2034" - ":;\\u309b\\u309c\\u3005\\u309d\\u309e\\u30fd\\u00b0\\u2032\\u2033\\u2034" - "\\u2030\\u2031\\u2103\\u2109\\u00a2\\u0300\\u0301\\u0302"); - BreakIterator *iter = BreakIterator::createLineInstance(Locale::getJapan(), status); - - int32_t i; - if (U_FAILURE(status)) - { - errln("Failed to create the BreakIterator for Japanese locale in TestJapaneseLineBreak.\n"); + BreakIterator *bi = BreakIterator::createSentenceInstance(Locale::getEnglish(), status); + TEST_ASSERT_SUCCESS(status); + if (U_FAILURE(status)) { return; } - - for (i = 0; i < precedingChars.length(); i++) { - testString.setCharAt(1, precedingChars[i]); - iter->setText(testString); - int32_t j = iter->first(); - if (j != 0) - errln("ja line break failure: failed to start at 0"); - j = iter->next(); - if (j != 1) - errln("ja line break failure: failed to stop before '" + UCharToUnicodeString(precedingChars[i]) - + "' (" + ((int)(precedingChars[i])) + ")"); - j = iter->next(); - if (j != 3) - errln("ja line break failure: failed to skip position after '" + UCharToUnicodeString(precedingChars[i]) - + "' (" + ((int)(precedingChars[i])) + ")"); - } - - for (i = 0; i < followingChars.length(); i++) { - testString.setCharAt(1, followingChars[i]); - iter->setText(testString); - int j = iter->first(); - if (j != 0) - errln("ja line break failure: failed to start at 0"); - j = iter->next(); - if (j != 2) - errln("ja line break failure: failed to skip position before '" + UCharToUnicodeString(followingChars[i]) - + "' (" + ((int)(followingChars[i])) + ")"); - j = iter->next(); - if (j != 3) - errln("ja line break failure: failed to stop after '" + UCharToUnicodeString(followingChars[i]) - + "' (" + ((int)(followingChars[i])) + ")"); +// Check for status first for better handling of no data errors. + TEST_ASSERT(bi != NULL); + if (bi == NULL) { + return; } - delete iter; -#endif + + UnicodeString s("One.\\u00ad Two.", -1, US_INV); + // 01234 56789 + s = s.unescape(); + bi->setText(s); + int pos = bi->next(); + TEST_ASSERT(pos == 6); + pos = bi->next(); + TEST_ASSERT(pos == 10); + pos = bi->previous(); + TEST_ASSERT(pos == 6); + delete bi; } + //------------------------------------------------------------------------------ // // RBBITest::Extended Run RBBI Tests from an external test data file @@ -1366,6 +956,53 @@ void RBBITest::executeTest(TestParams *t) { i, t->srcLine->elementAti(i), t->srcCol->elementAti(i)); } } + + // Check isBoundary() + for (i=0; iexpectedBreaks->size(); i++) { + UBool boundaryExpected = (t->expectedBreaks->elementAti(i) != 0); + UBool boundaryFound = t->bi->isBoundary(i); + if (boundaryExpected != boundaryFound) { + errln("isBoundary(%d) incorrect. File line,col= %4d,%4d\n" + " Expected, Actual= %s, %s", + i, t->srcLine->elementAti(i), t->srcCol->elementAti(i), + boundaryExpected ? "true":"false", boundaryFound? "true" : "false"); + } + } + + // Check following() + for (i=0; iexpectedBreaks->size(); i++) { + int32_t actualBreak = t->bi->following(i); + int32_t expectedBreak = BreakIterator::DONE; + for (int32_t j=i+1; j < t->expectedBreaks->size(); j++) { + if (t->expectedBreaks->elementAti(j) != 0) { + expectedBreak = j; + break; + } + } + if (expectedBreak != actualBreak) { + errln("following(%d) incorrect. File line,col= %4d,%4d\n" + " Expected, Actual= %d, %d", + i, t->srcLine->elementAti(i), t->srcCol->elementAti(i), expectedBreak, actualBreak); + } + } + + // Check preceding() + for (i=t->expectedBreaks->size(); i>=0; i--) { + int32_t actualBreak = t->bi->preceding(i); + int32_t expectedBreak = BreakIterator::DONE; + + for (int32_t j=i-1; j >= 0; j--) { + if (t->expectedBreaks->elementAti(j) != 0) { + expectedBreak = j; + break; + } + } + if (expectedBreak != actualBreak) { + errln("preceding(%d) incorrect. File line,col= %4d,%4d\n" + " Expected, Actual= %d, %d", + i, t->srcLine->elementAti(i), t->srcCol->elementAti(i), expectedBreak, actualBreak); + } + } } @@ -1381,8 +1018,10 @@ void RBBITest::TestExtended() { tp.srcLine = new UVector32(status); tp.srcCol = new UVector32(status); - RegexMatcher localeMatcher("", 0, status); - TEST_ASSERT_SUCCESS(status); + RegexMatcher localeMatcher(UNICODE_STRING_SIMPLE(""), 0, status); + if (U_FAILURE(status)) { + dataerrln("Failure in file %s, line %d, status = \"%s\"", __FILE__, __LINE__, u_errorName(status)); + } // @@ -1398,13 +1037,14 @@ void RBBITest::TestExtended() { strcat(testFileName, "rbbitst.txt"); int len; - UChar *testFile = ReadAndConvertFile(testFileName, len, status); + UChar *testFile = ReadAndConvertFile(testFileName, len, "UTF-8", status); if (U_FAILURE(status)) { return; /* something went wrong, error already output */ } + // // Put the test data into a UnicodeString // @@ -1489,7 +1129,7 @@ void RBBITest::TestExtended() { if (testString.compare(charIdx-1, 6, "") == 0) { delete tp.bi; tp.bi = NULL; - tp.bi = BreakIterator::createSentenceInstance(locale, status); + tp.bi = BreakIterator::createSentenceInstance(locale, status); charIdx += 5; break; } @@ -1499,6 +1139,7 @@ void RBBITest::TestExtended() { charIdx += 6; break; } + // localeMatcher.reset(testString); if (localeMatcher.lookingAt(charIdx-1, status)) { @@ -1506,7 +1147,7 @@ void RBBITest::TestExtended() { char localeName8[100]; localeName.extract(0, localeName.length(), localeName8, sizeof(localeName8), 0); locale = Locale::createFromName(localeName8); - charIdx += localeMatcher.group(0, status).length(); + charIdx += localeMatcher.group(0, status).length() - 1; TEST_ASSERT_SUCCESS(status); break; } @@ -1521,9 +1162,9 @@ void RBBITest::TestExtended() { } errln("line %d: Tag expected in test file.", lineNum); - goto end_test; parseState = PARSE_COMMENT; savedState = PARSE_DATA; + goto end_test; // Stop the test. } break; @@ -1554,7 +1195,7 @@ void RBBITest::TestExtended() { break; } - if (testString.compare(charIdx-1, 3, "\\N{") == 0) { + if (testString.compare(charIdx-1, 3, UNICODE_STRING_SIMPLE("\\N{")) == 0) { // Named character, e.g. \N{COMBINING GRAVE ACCENT} // Get the code point from the name and insert it into the test data. // (Damn, no API takes names in Unicode !!! @@ -1700,17 +1341,17 @@ void RBBITest::TestExtended() { errln("Syntax Error in test file at line %d, col %d", lineNum, column); - goto end_test; parseState = PARSE_COMMENT; + goto end_test; // Stop the test break; } if (U_FAILURE(status)) { - errln("ICU Error %s while parsing test file at line %d.", + dataerrln("ICU Error %s while parsing test file at line %d.", u_errorName(status), lineNum); - goto end_test; status = U_ZERO_ERROR; + goto end_test; // Stop the test } } @@ -1725,16 +1366,67 @@ end_test: } +//------------------------------------------------------------------------------- +// +// TestDictRules create a break iterator from source rules that includes a +// dictionary range. Regression for bug #7130. Source rules +// do not declare a break iterator type (word, line, sentence, etc. +// but the dictionary code, without a type, would loop. +// +//------------------------------------------------------------------------------- +void RBBITest::TestDictRules() { + const char *rules = "$dictionary = [a-z]; \n" + "!!forward; \n" + "$dictionary $dictionary; \n" + "!!reverse; \n" + "$dictionary $dictionary; \n"; + const char *text = "aa"; + UErrorCode status = U_ZERO_ERROR; + UParseError parseError; + + RuleBasedBreakIterator bi(rules, parseError, status); + if (U_SUCCESS(status)) { + UnicodeString utext = text; + bi.setText(utext); + int32_t position; + int32_t loops; + for (loops = 0; loops<10; loops++) { + position = bi.next(); + if (position == RuleBasedBreakIterator::DONE) { + break; + } + } + TEST_ASSERT(loops == 1); + } else { + dataerrln("Error creating RuleBasedBreakIterator: %s", u_errorName(status)); + } +} + + + //------------------------------------------------------------------------------- // // ReadAndConvertFile Read a text data file, convert it to UChars, and // return the datain one big UChar * buffer, which the caller must delete. // +// parameters: +// fileName: the name of the file, with no directory part. The test data directory +// is assumed. +// ulen an out parameter, receives the actual length (in UChars) of the file data. +// encoding The file encoding. If the file contains a BOM, that will override the encoding +// specified here. The BOM, if it exists, will be stripped from the returned data. +// Pass NULL for the system default encoding. +// status +// returns: +// The file data, converted to UChar. +// The caller must delete this when done with +// delete [] theBuffer; +// // TODO: This is a clone of RegexTest::ReadAndConvertFile. // Move this function to some common place. // //-------------------------------------------------------------------------------- -UChar *RBBITest::ReadAndConvertFile(const char *fileName, int &ulen, UErrorCode &status) { +UChar *RBBITest::ReadAndConvertFile(const char *fileName, int &ulen, const char *encoding, UErrorCode &status) { UChar *retPtr = NULL; char *fileBuf = NULL; UConverter* conv = NULL; @@ -1750,7 +1442,7 @@ UChar *RBBITest::ReadAndConvertFile(const char *fileName, int &ulen, UErrorCode // f = fopen(fileName, "rb"); if (f == 0) { - errln("Error opening test data file %s\n", fileName); + dataerrln("Error opening test data file %s\n", fileName); status = U_FILE_ACCESS_ERROR; return NULL; } @@ -1775,14 +1467,15 @@ UChar *RBBITest::ReadAndConvertFile(const char *fileName, int &ulen, UErrorCode // int32_t signatureLength; const char * fileBufC; - const char* encoding; + const char* bomEncoding; fileBufC = fileBuf; - encoding = ucnv_detectUnicodeSignature( + bomEncoding = ucnv_detectUnicodeSignature( fileBuf, fileSize, &signatureLength, &status); - if(encoding!=NULL ){ + if(bomEncoding!=NULL ){ fileBufC += signatureLength; fileSize -= signatureLength; + encoding = bomEncoding; } // @@ -1822,7 +1515,7 @@ cleanUpAndReturn: ucnv_close(conv); if (U_FAILURE(status)) { errln("ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status)); - delete retPtr; + delete []retPtr; retPtr = 0; ulen = 0; }; @@ -1830,219 +1523,242 @@ cleanUpAndReturn: } -//-------------------------------------------------------------------------------------------- -// -// Exhaustive Tests, using Unicode Data Files. -// -//-------------------------------------------------------------------------------------------- +//-------------------------------------------------------------------------------------------- // -// Token level scanner for the Unicode Line Break Test Data file. -// Return the next token, as follows: -// >= 0: a UChar32 character, scanned from hex in the file. -// -1: a break position, a division sign in the file. -// -2: end of rule. A new line in the file. -// -3: end of file. No more rules. -// -4: Error -// -// The scanner -// strips comments, ('#' to end of line) -// Recognizes CR, CR/LF and LF as new lines. -// Skips over spaces and Xs (don't break here) in the data. +// Run tests from each of the boundary test data files distributed by the Unicode Consortium // -struct ScanState { - int32_t fPeekChar; - UBool fPeeked; - int32_t fLineNum; - FILE *fFile; - ScanState() :fPeeked(FALSE), fLineNum(0), fFile(NULL) {}; -}; - -// Literal characters that are of interest. In hex to keep EBCDIC based machines happy. -// The data itself is latin-1 on all platforms. -static const int32_t chSpace = 0x20; -static const int32_t chTab = 0x09; -static const int32_t chCR = 0x0D; -static const int32_t chLF = 0x0A; -static const int32_t chHash = 0x23; -static const int32_t chMult = 0xD7; -static const int32_t chDivide = 0xF7; - -static int32_t nextLBDToken(ScanState *s) { - int32_t c; - - // Read characters from the input file until we get something interesting - // to return. The file is in latin-1 encoding. - for (;;) { - // Get the next character to look at, - if (s->fPeeked) { - c = s->fPeekChar; - s->fPeeked = FALSE; - } else { - c = getc(s->fFile); - } - - // EOF. Return immediately. - if (c == EOF) { - return -3; - } - - // Spaces. Treat the multiply sign as a space - it indicates a no-break position - // in the data, and the test program doesn't want to see them. - // Continue the next char loop, looking for something significant. - if (c == chSpace || c == chTab || c == chMult) { - continue; - } - - // Divide sign. Indicates an expected break position. - if (c == chDivide) { - return -1; - } +//------------------------------------------------------------------------------------------- +void RBBITest::TestUnicodeFiles() { + RuleBasedBreakIterator *bi; + UErrorCode status = U_ZERO_ERROR; - // New Line Handling. Keep track of line number in the file, which in turn - // requires keeping track of CR/LF as a single new line. - if (c == chCR) { - s->fLineNum++; - s->fPeekChar = getc(s->fFile); - if (s->fPeekChar != chLF) {s->fPeeked = TRUE;}; - return -2; - } - if (c == chLF) { - s->fLineNum++; - return -2; - } + bi = (RuleBasedBreakIterator *)BreakIterator::createCharacterInstance(Locale::getEnglish(), status); + TEST_ASSERT_SUCCESS(status); + if (U_SUCCESS(status)) { + runUnicodeTestData("GraphemeBreakTest.txt", bi); + } + delete bi; - // Comments. Consume everything up to the next new line. - if (c == chHash) { - do { - c = getc(s->fFile); - } while (!(c == EOF || c == chCR || c == chLF)); - s->fPeekChar = c; - s->fPeeked = TRUE; - return nextLBDToken(s); - } + bi = (RuleBasedBreakIterator *)BreakIterator::createWordInstance(Locale::getEnglish(), status); + TEST_ASSERT_SUCCESS(status); + if (U_SUCCESS(status)) { + runUnicodeTestData("WordBreakTest.txt", bi); + } + delete bi; - // Scan a hex character (UChar32) value. - if (u_digit(c, 16) >= 0) { - int32_t v = u_digit(c, 16); - for (;;) { - c = getc(s->fFile); - if (u_digit(c, 16) < 0) {break;}; - v <<= 4; - v += u_digit(c, 16); - } - s->fPeekChar = c; - s->fPeeked = TRUE; - return v; - } + bi = (RuleBasedBreakIterator *)BreakIterator::createSentenceInstance(Locale::getEnglish(), status); + TEST_ASSERT_SUCCESS(status); + if (U_SUCCESS(status)) { + runUnicodeTestData("SentenceBreakTest.txt", bi); + } + delete bi; - // Error. Character was something unexpected. - return -4; + bi = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(Locale::getEnglish(), status); + TEST_ASSERT_SUCCESS(status); + if (U_SUCCESS(status)) { + runUnicodeTestData("LineBreakTest.txt", bi); } + delete bi; } +//-------------------------------------------------------------------------------------------- +// +// Run tests from one of the boundary test data files distributed by the Unicode Consortium +// +//------------------------------------------------------------------------------------------- +void RBBITest::runUnicodeTestData(const char *fileName, RuleBasedBreakIterator *bi) { +#if !UCONFIG_NO_REGULAR_EXPRESSIONS + // TODO(andy): Match line break behavior to Unicode 6.0 and remove this time bomb. Ticket #7270 + UBool isTicket7270Fixed = isICUVersionAtLeast(52, 1); + UBool isLineBreak = 0 == strcmp(fileName, "LineBreakTest.txt"); + UErrorCode status = U_ZERO_ERROR; -void RBBITest::TestLineBreakData() { - - UErrorCode status = U_ZERO_ERROR; - UnicodeString testString; - UVector expectedBreaks(status); - ScanState ss; - int32_t tok; - - BreakIterator *bi = BreakIterator::createLineInstance(Locale::getDefault(), status); - if (U_FAILURE(status)) { - errln("Failure creating break iterator"); + // + // Open and read the test data file, put it into a UnicodeString. + // + const char *testDataDirectory = IntlTest::getSourceTestData(status); + char testFileName[1000]; + if (testDataDirectory == NULL || strlen(testDataDirectory) >= sizeof(testFileName)) { + dataerrln("Can't open test data. Path too long."); return; } + strcpy(testFileName, testDataDirectory); + strcat(testFileName, fileName); + + logln("Opening data file %s\n", fileName); - const char * lbdfName = "LBTest.txt"; + int len; + UChar *testFile = ReadAndConvertFile(testFileName, len, "UTF-8", status); + if (status != U_FILE_ACCESS_ERROR) { + TEST_ASSERT_SUCCESS(status); + TEST_ASSERT(testFile != NULL); + } + if (U_FAILURE(status) || testFile == NULL) { + return; /* something went wrong, error already output */ + } + UnicodeString testFileAsString(TRUE, testFile, len); - // Open the test data file. - // TODO: a proper way to handle this data. - ss.fFile = fopen(lbdfName, "rb"); - if (ss.fFile == NULL) { - logln("Unable to open Line Break Test Data file. Skipping test."); - delete bi; + // + // Parse the test data file using a regular expression. + // Each kind of token is recognized in its own capture group; what type of item was scanned + // is identified by which group had a match. + // + // Caputure Group # 1 2 3 4 5 + // Parses this item: divide x hex digits comment \n unrecognized \n + // + UnicodeString tokenExpr("[ \t]*(?:(\\u00F7)|(\\u00D7)|([0-9a-fA-F]+)|((?:#.*?)?$.)|(.*?$.))", -1, US_INV); + RegexMatcher tokenMatcher(tokenExpr, testFileAsString, UREGEX_MULTILINE | UREGEX_DOTALL, status); + UnicodeString testString; + UVector32 breakPositions(status); + int lineNumber = 1; + TEST_ASSERT_SUCCESS(status); + if (U_FAILURE(status)) { return; } - // Loop once per line from the test data file. - for (;;) { - // Zero out test data from previous line. - testString.truncate(0); - expectedBreaks.removeAllElements(); - - // Read one test's (line's) worth of data from the file. - // Loop once per token on the input file line. - for(;;) { - tok = nextLBDToken(&ss); - - // If we scanned a character number in the file. - // save it in the test data array. - if (tok >= 0) { - testString.append((UChar32)tok); - continue; + // + // Scan through each test case, building up the string to be broken in testString, + // and the positions that should be boundaries in the breakPositions vector. + // + int spin = 0; + while (tokenMatcher.find()) { + if(tokenMatcher.hitEnd()) { + /* Shouldnt Happen(TM). This means we didn't find the symbols we were looking for. + This occurred when the text file was corrupt (wasn't marked as UTF-8) + and caused an infinite loop here on EBCDIC systems! + */ + fprintf(stderr,"FAIL: hit end of file %s for the %8dth time- corrupt data file?\r", fileName, ++spin); + // return; + } + if (tokenMatcher.start(1, status) >= 0) { + // Scanned a divide sign, indicating a break position in the test data. + if (testString.length()>0) { + breakPositions.addElement(testString.length(), status); } - - // If we scanned a break position in the data, record it. - if (tok == -1) { - expectedBreaks.addElement(testString.length(), status); - continue; + } + else if (tokenMatcher.start(2, status) >= 0) { + // Scanned an 'x', meaning no break at this position in the test data + // Nothing to be done here. + } + else if (tokenMatcher.start(3, status) >= 0) { + // Scanned Hex digits. Convert them to binary, append to the character data string. + const UnicodeString &hexNumber = tokenMatcher.group(3, status); + int length = hexNumber.length(); + if (length<=8) { + char buf[10]; + hexNumber.extract (0, length, buf, sizeof(buf), US_INV); + UChar32 c = (UChar32)strtol(buf, NULL, 16); + if (c<=0x10ffff) { + testString.append(c); + } else { + errln("Error: Unicode Character value out of range. \'%s\', line %d.\n", + fileName, lineNumber); + } + } else { + errln("Syntax Error: Hex Unicode Character value must have no more than 8 digits at \'%s\', line %d.\n", + fileName, lineNumber); + } + } + else if (tokenMatcher.start(4, status) >= 0) { + // Scanned to end of a line, possibly skipping over a comment in the process. + // If the line from the file contained test data, run the test now. + // + if (testString.length() > 0) { +// TODO(andy): Remove this time bomb code. Note: Failing line numbers may change when updating to new Unicode data. +// Rule 8 +// ZW SP* +// is not yet implemented. +if (!(isLineBreak && !isTicket7270Fixed && (5198 == lineNumber || + 5202 == lineNumber || + 5214 == lineNumber || + 5246 == lineNumber || + 5298 == lineNumber || + 5302 == lineNumber ))) { + checkUnicodeTestCase(fileName, lineNumber, testString, &breakPositions, bi); +} } - // If we scanned a new line, or EOF - // drop out of scan loop and run the test case. - if (tok == -2 || tok == -3) {break;}; - - // None of above. Error. - errln("Failure: Unrecognized data format, test file line %d", ss.fLineNum); + // Clear out this test case. + // The string and breakPositions vector will be refilled as the next + // test case is parsed. + testString.remove(); + breakPositions.removeAllElements(); + lineNumber++; + } else { + // Scanner catchall. Something unrecognized appeared on the line. + char token[16]; + UnicodeString uToken = tokenMatcher.group(0, status); + uToken.extract(0, uToken.length(), token, (uint32_t)sizeof(token)); + token[sizeof(token)-1] = 0; + errln("Syntax error in test data file \'%s\', line %d. Scanning \"%s\"\n", fileName, lineNumber, token); + + // Clean up, in preparation for continuing with the next line. + testString.remove(); + breakPositions.removeAllElements(); + lineNumber++; + } + TEST_ASSERT_SUCCESS(status); + if (U_FAILURE(status)) { break; } + } - // If this line from the test data file actually contained test data, - // run the test. - if (testString.length() > 0) { - int32_t pos; // Break Position in the test string - int32_t expectedI = 0; // Index of expected break position in vector of same. - int32_t expectedPos; // Expected break position (index into test string) - - bi->setText(testString); - pos = bi->first(); - pos = bi->next(); + delete [] testFile; + #endif // !UCONFIG_NO_REGULAR_EXPRESSIONS +} - for (; pos != BreakIterator::DONE; ) { - expectedPos = expectedBreaks.elementAti(expectedI); - if (pos < expectedPos) { - errln("Failure: Test file line %d, unexpected break found at position %d", - ss.fLineNum, pos); - break; - } - if (pos > expectedPos) { - errln("Failure: Test file line %d, failed to find break at position %d", - ss.fLineNum, expectedPos); - break; - } - pos = bi->next(); - expectedI++; - } +//-------------------------------------------------------------------------------------------- +// +// checkUnicodeTestCase() Run one test case from one of the Unicode Consortium +// test data files. Do only a simple, forward-only check - +// this test is mostly to check that ICU and the Unicode +// data agree with each other. +// +//-------------------------------------------------------------------------------------------- +void RBBITest::checkUnicodeTestCase(const char *testFileName, int lineNumber, + const UnicodeString &testString, // Text data to be broken + UVector32 *breakPositions, // Positions where breaks should be found. + RuleBasedBreakIterator *bi) { + int32_t pos; // Break Position in the test string + int32_t expectedI = 0; // Index of expected break position in the vector of expected results. + int32_t expectedPos; // Expected break position (index into test string) + + bi->setText(testString); + pos = bi->first(); + pos = bi->next(); + + while (pos != BreakIterator::DONE) { + if (expectedI >= breakPositions->size()) { + errln("Test file \"%s\", line %d, unexpected break found at position %d", + testFileName, lineNumber, pos); + break; } - - // If we've hit EOF on the input file, we're done. - if (tok == -3) { + expectedPos = breakPositions->elementAti(expectedI); + if (pos < expectedPos) { + errln("Test file \"%s\", line %d, unexpected break found at position %d", + testFileName, lineNumber, pos); + break; + } + if (pos > expectedPos) { + errln("Test file \"%s\", line %d, failed to find expected break at position %d", + testFileName, lineNumber, expectedPos); break; } + pos = bi->next(); + expectedI++; + } + if (pos==BreakIterator::DONE && expectedIsize()) { + errln("Test file \"%s\", line %d, failed to find expected break at position %d", + testFileName, lineNumber, breakPositions->elementAti(expectedI)); } +} - fclose(ss.fFile); - delete bi; -} #if !UCONFIG_NO_REGULAR_EXPRESSIONS - //--------------------------------------------------------------------------------------- // // classs RBBIMonkeyKind @@ -2122,10 +1838,17 @@ private: UnicodeSet *fCRLFSet; UnicodeSet *fControlSet; UnicodeSet *fExtendSet; + UnicodeSet *fRegionalIndicatorSet; + UnicodeSet *fPrependSet; + UnicodeSet *fSpacingSet; + UnicodeSet *fLSet; + UnicodeSet *fVSet; + UnicodeSet *fTSet; + UnicodeSet *fLVSet; + UnicodeSet *fLVTSet; UnicodeSet *fHangulSet; UnicodeSet *fAnySet; - RegexMatcher *fMatcher; const UnicodeString *fText; }; @@ -2134,20 +1857,35 @@ RBBICharMonkey::RBBICharMonkey() { UErrorCode status = U_ZERO_ERROR; fText = NULL; - fMatcher = new RegexMatcher("\\X", 0, status); // Pattern to match a grampheme cluster - fCRLFSet = new UnicodeSet("[\\r\\n]", status); - fControlSet = new UnicodeSet("[[\\p{Zl}\\p{Zp}\\p{Cc}\\p{Cf}]-[\\n]-[\\r]-\\p{Grapheme_Extend}]", status); - fExtendSet = new UnicodeSet("[\\p{Grapheme_Extend}]", status); - fHangulSet = new UnicodeSet( - "[\\p{Hangul_Syllable_Type=L}\\p{Hangul_Syllable_Type=L}\\p{Hangul_Syllable_Type=T}" - "\\p{Hangul_Syllable_Type=LV}\\p{Hangul_Syllable_Type=LVT}]", status); - fAnySet = new UnicodeSet("[\\u0000-\\U0010ffff]", status); + fCRLFSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\r\\n]"), status); + fControlSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = Control}]"), status); + fExtendSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = Extend}]"), status); + fRegionalIndicatorSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = Regional_Indicator}]"), status); + fPrependSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = Prepend}]"), status); + fSpacingSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = SpacingMark}]"), status); + fLSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = L}]"), status); + fVSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = V}]"), status); + fTSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = T}]"), status); + fLVSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = LV}]"), status); + fLVTSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = LVT}]"), status); + fHangulSet = new UnicodeSet(); + fHangulSet->addAll(*fLSet); + fHangulSet->addAll(*fVSet); + fHangulSet->addAll(*fTSet); + fHangulSet->addAll(*fLVSet); + fHangulSet->addAll(*fLVTSet); + fAnySet = new UnicodeSet(0, 0x10ffff); fSets = new UVector(status); fSets->addElement(fCRLFSet, status); fSets->addElement(fControlSet, status); fSets->addElement(fExtendSet, status); + fSets->addElement(fRegionalIndicatorSet, status); + if (!fPrependSet->isEmpty()) { + fSets->addElement(fPrependSet, status); + } + fSets->addElement(fSpacingSet, status); fSets->addElement(fHangulSet, status); fSets->addElement(fAnySet, status); if (U_FAILURE(status)) { @@ -2158,24 +1896,128 @@ RBBICharMonkey::RBBICharMonkey() { void RBBICharMonkey::setText(const UnicodeString &s) { fText = &s; - fMatcher->reset(s); } -int32_t RBBICharMonkey::next(int32_t i) { - UErrorCode status = U_ZERO_ERROR; - int32_t retVal = -1; - if (fMatcher->find(i, status)) { - retVal = fMatcher->end(status); +int32_t RBBICharMonkey::next(int32_t prevPos) { + int p0, p1, p2, p3; // Indices of the significant code points around the + // break position being tested. The candidate break + // location is before p2. + + int breakPos = -1; + + UChar32 c0, c1, c2, c3; // The code points at p0, p1, p2 & p3. + + if (U_FAILURE(deferredStatus)) { + return -1; + } + + // Previous break at end of string. return DONE. + if (prevPos >= fText->length()) { + return -1; } - if (U_FAILURE(status)){ - retVal = -1; + p0 = p1 = p2 = p3 = prevPos; + c3 = fText->char32At(prevPos); + c0 = c1 = c2 = 0; + + // Loop runs once per "significant" character position in the input text. + for (;;) { + // Move all of the positions forward in the input string. + p0 = p1; c0 = c1; + p1 = p2; c1 = c2; + p2 = p3; c2 = c3; + + // Advancd p3 by one codepoint + p3 = fText->moveIndex32(p3, 1); + c3 = fText->char32At(p3); + + if (p1 == p2) { + // Still warming up the loop. (won't work with zero length strings, but we don't care) + continue; + } + if (p2 == fText->length()) { + // Reached end of string. Always a break position. + break; + } + + // Rule GB3 CR x LF + // No Extend or Format characters may appear between the CR and LF, + // which requires the additional check for p2 immediately following p1. + // + if (c1==0x0D && c2==0x0A && p1==(p2-1)) { + continue; + } + + // Rule (GB4). ( Control | CR | LF ) + if (fControlSet->contains(c1) || + c1 == 0x0D || + c1 == 0x0A) { + break; + } + + // Rule (GB5) ( Control | CR | LF ) + // + if (fControlSet->contains(c2) || + c2 == 0x0D || + c2 == 0x0A) { + break; + } + + + // Rule (GB6) L x ( L | V | LV | LVT ) + if (fLSet->contains(c1) && + (fLSet->contains(c2) || + fVSet->contains(c2) || + fLVSet->contains(c2) || + fLVTSet->contains(c2))) { + continue; + } + + // Rule (GB7) ( LV | V ) x ( V | T ) + if ((fLVSet->contains(c1) || fVSet->contains(c1)) && + (fVSet->contains(c2) || fTSet->contains(c2))) { + continue; + } + + // Rule (GB8) ( LVT | T) x T + if ((fLVTSet->contains(c1) || fTSet->contains(c1)) && + fTSet->contains(c2)) { + continue; + } + + // Just adding extra Apple rule does here not work, behavior depends on arbitrary context + + // Rule (GB8a) Regional_Indicator x Regional_Indicator + if (fRegionalIndicatorSet->contains(c1) && fRegionalIndicatorSet->contains(c2)) { + continue; + } + + // Rule (GB9) Numeric x ALetter + if (fExtendSet->contains(c2)) { + continue; + } + + // Rule (GB9a) x SpacingMark + if (fSpacingSet->contains(c2)) { + continue; + } + + // Rule (GB9b) Prepend x + if (fPrependSet->contains(c1)) { + continue; + } + + // Rule (GB10) Any Any + break; } - return retVal; + + breakPos = p2; + return breakPos; } + UVector *RBBICharMonkey::charClasses() { return fSets; } @@ -2186,10 +2028,16 @@ RBBICharMonkey::~RBBICharMonkey() { delete fCRLFSet; delete fControlSet; delete fExtendSet; + delete fRegionalIndicatorSet; + delete fPrependSet; + delete fSpacingSet; + delete fLSet; + delete fVSet; + delete fTSet; + delete fLVSet; + delete fLVTSet; delete fHangulSet; delete fAnySet; - - delete fMatcher; } //------------------------------------------------------------------------------------------ @@ -2208,8 +2056,14 @@ public: private: UVector *fSets; + UnicodeSet *fCRSet; + UnicodeSet *fLFSet; + UnicodeSet *fNewlineSet; UnicodeSet *fKatakanaSet; UnicodeSet *fALetterSet; + // TODO(jungshik): Do we still need this change? + // UnicodeSet *fALetterSet; // matches ALetterPlus in word.txt + UnicodeSet *fMidNumLetSet; UnicodeSet *fMidLetterSet; UnicodeSet *fMidNumSet; UnicodeSet *fNumericSet; @@ -2217,6 +2071,8 @@ private: UnicodeSet *fOtherSet; UnicodeSet *fExtendSet; UnicodeSet *fExtendNumLetSet; + UnicodeSet *fRegionalIndicatorSet; + UnicodeSet *fDictionaryCjkSet; RegexMatcher *fMatcher; @@ -2224,27 +2080,40 @@ private: }; -RBBIWordMonkey::RBBIWordMonkey() +RBBIWordMonkey::RBBIWordMonkey() { UErrorCode status = U_ZERO_ERROR; - fSets = new UVector(status); + fCRSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = CR}]"), status); + fLFSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = LF}]"), status); + fNewlineSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Newline}]"), status); + fDictionaryCjkSet= new UnicodeSet("[[\\uac00-\\ud7a3][:Han:][:Hiragana:][:Katakana:]]", status); + // Exclude Hangul syllables from ALetterSet during testing. + // Leave CJK dictionary characters out from the monkey tests! +#if 0 fALetterSet = new UnicodeSet("[\\p{Word_Break = ALetter}" - "[\\p{Line_Break = Complex_Context}" - "-\\p{Grapheme_Cluster_Break = Extend}" - "-\\p{Grapheme_Cluster_Break = Control}]]", status); - //fALetterSet = new UnicodeSet("[\\p{Word_Break = ALetter}]", status); - fKatakanaSet = new UnicodeSet("[\\p{Word_Break = Katakana}-[\\uff9e\\uff9f]]", status); - fMidLetterSet = new UnicodeSet("[\\p{Word_Break = MidLetter}]", status); - fMidNumSet = new UnicodeSet("[\\p{Word_Break = MidNum}]", status); - fNumericSet = new UnicodeSet("[\\p{Word_Break = Numeric}]", status); - fFormatSet = new UnicodeSet("[\\p{Word_Break = Format}]", status); - fExtendNumLetSet = new UnicodeSet("[\\p{Word_Break = ExtendNumLet}]", status); - //fExtendSet = new UnicodeSet("[\\p{Word_Break = Extend}]", status); - fExtendSet = new UnicodeSet("[\\p{Grapheme_Cluster_Break = Extend}\\uff9e\\uff9f]", status); - + "[\\p{Line_Break = Complex_Context}" + "-\\p{Grapheme_Cluster_Break = Extend}" + "-\\p{Grapheme_Cluster_Break = Control}" + "]]", + status); +#endif + fALetterSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = ALetter}]"), status); + fALetterSet->removeAll(*fDictionaryCjkSet); + fKatakanaSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Katakana}]"), status); + fMidNumLetSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = MidNumLet}]"), status); + fMidLetterSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = MidLetter}]"), status); + fMidNumSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = MidNum}]"), status); + // TODO: this set used to contain [\\uff10-\\uff19] (fullwidth digits), but this breaks the test + // we should figure out why + fNumericSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Numeric}]"), status); + fFormatSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Format}]"), status); + fExtendNumLetSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = ExtendNumLet}]"), status); + fExtendSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Extend}]"), status); + fRegionalIndicatorSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Regional_Indicator}]"), status); + fOtherSet = new UnicodeSet(); if(U_FAILURE(status)) { deferredStatus = status; @@ -2252,6 +2121,9 @@ RBBIWordMonkey::RBBIWordMonkey() } fOtherSet->complement(); + fOtherSet->removeAll(*fCRSet); + fOtherSet->removeAll(*fLFSet); + fOtherSet->removeAll(*fNewlineSet); fOtherSet->removeAll(*fKatakanaSet); fOtherSet->removeAll(*fALetterSet); fOtherSet->removeAll(*fMidLetterSet); @@ -2260,17 +2132,25 @@ RBBIWordMonkey::RBBIWordMonkey() fOtherSet->removeAll(*fExtendNumLetSet); fOtherSet->removeAll(*fFormatSet); fOtherSet->removeAll(*fExtendSet); - + fOtherSet->removeAll(*fRegionalIndicatorSet); + // Inhibit dictionary characters from being tested at all. + fOtherSet->removeAll(*fDictionaryCjkSet); + fOtherSet->removeAll(UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{LineBreak = Complex_Context}]"), status)); + + fSets->addElement(fCRSet, status); + fSets->addElement(fLFSet, status); + fSets->addElement(fNewlineSet, status); fSets->addElement(fALetterSet, status); - fSets->addElement(fKatakanaSet, status); + //fSets->addElement(fKatakanaSet, status); //TODO: work out how to test katakana fSets->addElement(fMidLetterSet, status); + fSets->addElement(fMidNumLetSet, status); fSets->addElement(fMidNumSet, status); fSets->addElement(fNumericSet, status); fSets->addElement(fFormatSet, status); fSets->addElement(fExtendSet, status); fSets->addElement(fOtherSet, status); fSets->addElement(fExtendNumLetSet, status); - + fSets->addElement(fRegionalIndicatorSet, status); if (U_FAILURE(status)) { deferredStatus = status; @@ -2290,6 +2170,10 @@ int32_t RBBIWordMonkey::next(int32_t prevPos) { int breakPos = -1; UChar32 c0, c1, c2, c3; // The code points at p0, p1, p2 & p3. + + if (U_FAILURE(deferredStatus)) { + return -1; + } // Prev break at end of string. return DONE. if (prevPos >= fText->length()) { @@ -2307,9 +2191,13 @@ int32_t RBBIWordMonkey::next(int32_t prevPos) { p2 = p3; c2 = c3; // Advancd p3 by X(Extend | Format)* Rule 4 + // But do not advance over Extend & Format following a new line. (Unicode 5.1 change) do { p3 = fText->moveIndex32(p3, 1); c3 = fText->char32At(p3); + if (fCRSet->contains(c2) || fLFSet->contains(c2) || fNewlineSet->contains(c2)) { + break; + }; } while (fFormatSet->contains(c3) || fExtendSet->contains(c3)); @@ -2322,14 +2210,23 @@ int32_t RBBIWordMonkey::next(int32_t prevPos) { // Reached end of string. Always a break position. break; } - + // Rule (3) CR x LF // No Extend or Format characters may appear between the CR and LF, // which requires the additional check for p2 immediately following p1. // - if (c1==0x0D && c2==0x0A && p1==(p2-1)) { + if (c1==0x0D && c2==0x0A) { continue; } + + // Rule (3a) Break before and after newlines (including CR and LF) + // + if (fCRSet->contains(c1) || fLFSet->contains(c1) || fNewlineSet->contains(c1)) { + break; + }; + if (fCRSet->contains(c2) || fLFSet->contains(c2) || fNewlineSet->contains(c2)) { + break; + }; // Rule (5). ALetter x ALetter if (fALetterSet->contains(c1) && @@ -2339,10 +2236,8 @@ int32_t RBBIWordMonkey::next(int32_t prevPos) { // Rule (6) ALetter x (MidLetter | MidNumLet) ALetter // - // Also incorporates rule 7 by skipping pos ahead to position of the - // terminating ALetter. if ( fALetterSet->contains(c1) && - fMidLetterSet->contains(c2) && + (fMidLetterSet->contains(c2) || fMidNumLetSet->contains(c2)) && fALetterSet->contains(c3)) { continue; } @@ -2350,7 +2245,7 @@ int32_t RBBIWordMonkey::next(int32_t prevPos) { // Rule (7) ALetter (MidLetter | MidNumLet) x ALetter if (fALetterSet->contains(c0) && - (fMidLetterSet->contains(c1) ) && + (fMidLetterSet->contains(c1) || fMidNumLetSet->contains(c1)) && fALetterSet->contains(c2)) { continue; } @@ -2374,15 +2269,15 @@ int32_t RBBIWordMonkey::next(int32_t prevPos) { } // Rule (11) Numeric (MidNum | MidNumLet) x Numeric - if ( fNumericSet->contains(c0) && - fMidNumSet->contains(c1) && + if (fNumericSet->contains(c0) && + (fMidNumSet->contains(c1) || fMidNumLetSet->contains(c1)) && fNumericSet->contains(c2)) { continue; } // Rule (12) Numeric x (MidNum | MidNumLet) Numeric if (fNumericSet->contains(c1) && - fMidNumSet->contains(c2) && + (fMidNumSet->contains(c2) || fMidNumLetSet->contains(c2)) && fNumericSet->contains(c3)) { continue; } @@ -2398,14 +2293,19 @@ int32_t RBBIWordMonkey::next(int32_t prevPos) { fKatakanaSet->contains(c1) || fExtendNumLetSet->contains(c1)) && fExtendNumLetSet->contains(c2)) { continue; - } + } // Rule 13b if (fExtendNumLetSet->contains(c1) && (fALetterSet->contains(c2) || fNumericSet->contains(c2) || fKatakanaSet->contains(c2))) { continue; - } + } + + // Rule 13c + if (fRegionalIndicatorSet->contains(c1) && fRegionalIndicatorSet->contains(c2)) { + continue; + } // Rule 14. Break found here. break; @@ -2423,14 +2323,20 @@ UVector *RBBIWordMonkey::charClasses() { RBBIWordMonkey::~RBBIWordMonkey() { delete fSets; + delete fCRSet; + delete fLFSet; + delete fNewlineSet; delete fKatakanaSet; delete fALetterSet; + delete fMidNumLetSet; delete fMidLetterSet; delete fMidNumSet; delete fNumericSet; delete fFormatSet; delete fExtendSet; delete fExtendNumLetSet; + delete fRegionalIndicatorSet; + delete fDictionaryCjkSet; delete fOtherSet; } @@ -2465,6 +2371,7 @@ private: UnicodeSet *fOLetterSet; UnicodeSet *fNumericSet; UnicodeSet *fATermSet; + UnicodeSet *fSContinueSet; UnicodeSet *fSTermSet; UnicodeSet *fCloseSet; UnicodeSet *fOtherSet; @@ -2480,17 +2387,21 @@ RBBISentMonkey::RBBISentMonkey() fSets = new UVector(status); - fSepSet = new UnicodeSet("[\\p{Sentence_Break = Sep}]", status); - fFormatSet = new UnicodeSet("[\\p{Sentence_Break = Format}]", status); - fSpSet = new UnicodeSet("[\\p{Sentence_Break = Sp}]", status); - fLowerSet = new UnicodeSet("[\\p{Sentence_Break = Lower}]", status); - fUpperSet = new UnicodeSet("[\\p{Sentence_Break = Upper}]", status); - fOLetterSet = new UnicodeSet("[\\p{Sentence_Break = OLetter}-[\\uff9e\\uff9f]]", status); - fNumericSet = new UnicodeSet("[\\p{Sentence_Break = Numeric}]", status); - fATermSet = new UnicodeSet("[\\p{Sentence_Break = ATerm}]", status); - fSTermSet = new UnicodeSet("[\\p{Sentence_Break = STerm}]", status); - fCloseSet = new UnicodeSet("[\\p{Sentence_Break = Close}]", status); - fExtendSet = new UnicodeSet("[\\p{Grapheme_Extend}\\uff9e\\uff9f]", status); + // Separator Set Note: Beginning with Unicode 5.1, CR and LF were removed from the separator + // set and made into character classes of their own. For the monkey impl, + // they remain in SEP, since Sep always appears with CR and LF in the rules. + fSepSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Sep} \\u000a \\u000d]"), status); + fFormatSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Format}]"), status); + fSpSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Sp}]"), status); + fLowerSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Lower}]"), status); + fUpperSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Upper}]"), status); + fOLetterSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = OLetter}]"), status); + fNumericSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Numeric}]"), status); + fATermSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = ATerm}]"), status); + fSContinueSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = SContinue}]"), status); + fSTermSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = STerm}]"), status); + fCloseSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Close}]"), status); + fExtendSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Extend}]"), status); fOtherSet = new UnicodeSet(); if(U_FAILURE(status)) { @@ -2507,23 +2418,24 @@ RBBISentMonkey::RBBISentMonkey() fOtherSet->removeAll(*fOLetterSet); fOtherSet->removeAll(*fNumericSet); fOtherSet->removeAll(*fATermSet); + fOtherSet->removeAll(*fSContinueSet); fOtherSet->removeAll(*fSTermSet); fOtherSet->removeAll(*fCloseSet); fOtherSet->removeAll(*fExtendSet); - fSets->addElement(fSepSet, status); - fSets->addElement(fFormatSet, status); - - fSets->addElement(fSpSet, status); - fSets->addElement(fLowerSet, status); - fSets->addElement(fUpperSet, status); - fSets->addElement(fOLetterSet, status); - fSets->addElement(fNumericSet, status); - fSets->addElement(fATermSet, status); - fSets->addElement(fSTermSet, status); - fSets->addElement(fCloseSet, status); - fSets->addElement(fOtherSet, status); - fSets->addElement(fExtendSet, status); + fSets->addElement(fSepSet, status); + fSets->addElement(fFormatSet, status); + fSets->addElement(fSpSet, status); + fSets->addElement(fLowerSet, status); + fSets->addElement(fUpperSet, status); + fSets->addElement(fOLetterSet, status); + fSets->addElement(fNumericSet, status); + fSets->addElement(fATermSet, status); + fSets->addElement(fSContinueSet, status); + fSets->addElement(fSTermSet, status); + fSets->addElement(fCloseSet, status); + fSets->addElement(fOtherSet, status); + fSets->addElement(fExtendSet, status); if (U_FAILURE(status)) { deferredStatus = status; @@ -2543,7 +2455,7 @@ UVector *RBBISentMonkey::charClasses() { // moveBack() Find the "significant" code point preceding the index i. // Skips over ($Extend | $Format)* . -// +// int RBBISentMonkey::moveBack(int i) { if (i <= 0) { return -1; @@ -2592,6 +2504,10 @@ int32_t RBBISentMonkey::next(int32_t prevPos) { UChar32 c0, c1, c2, c3; // The code points at p0, p1, p2 & p3. UChar32 c; + if (U_FAILURE(deferredStatus)) { + return -1; + } + // Prev break at end of string. return DONE. if (prevPos >= fText->length()) { return -1; @@ -2606,7 +2522,7 @@ int32_t RBBISentMonkey::next(int32_t prevPos) { p0 = p1; c0 = c1; p1 = p2; c1 = c2; p2 = p3; c2 = c3; - + // Advancd p3 by X(Extend | Format)* Rule 4 p3 = moveForward(p3); c3 = cAt(p3); @@ -2615,7 +2531,7 @@ int32_t RBBISentMonkey::next(int32_t prevPos) { if (c1==0x0d && c2==0x0a && p2==(p1+1)) { continue; } - + // Rule (4). Sep if (fSepSet->contains(c1)) { p2 = p1+1; // Separators don't combine with Extend or Format. @@ -2631,7 +2547,7 @@ int32_t RBBISentMonkey::next(int32_t prevPos) { // Still warming up the loop. (won't work with zero length strings, but we don't care) continue; } - + // Rule (6). ATerm x Numeric if (fATermSet->contains(c1) && fNumericSet->contains(c2)) { continue; @@ -2667,9 +2583,9 @@ int32_t RBBISentMonkey::next(int32_t prevPos) { continue; } } - - // Rule 8a (STerm | ATerm) Close* Sp* x (STerm | ATerm); - if (fSTermSet->contains(c2) || fATermSet->contains(c2)) { + + // Rule 8a (STerm | ATerm) Close* Sp* x (SContinue | STerm | ATerm); + if (fSContinueSet->contains(c2) || fSTermSet->contains(c2) || fATermSet->contains(c2)) { p8 = p1; while (fSpSet->contains(cAt(p8))) { p8 = moveBack(p8); @@ -2683,7 +2599,7 @@ int32_t RBBISentMonkey::next(int32_t prevPos) { } } - // Rule (9) (STerm | ATerm) Close* x (Close | Sp | Sep) + // Rule (9) (STerm | ATerm) Close* x (Close | Sp | Sep | CR | LF) int p9 = p1; while (fCloseSet->contains(cAt(p9))) { p9 = moveBack(p9); @@ -2695,7 +2611,7 @@ int32_t RBBISentMonkey::next(int32_t prevPos) { } } - // Rule (10) (Sterm | ATerm) Close* Sp* x (Sp | Sep) + // Rule (10) (Sterm | ATerm) Close* Sp* x (Sp | Sep | CR | LF) int p10 = p1; while (fSpSet->contains(cAt(p10))) { p10 = moveBack(p10); @@ -2709,8 +2625,11 @@ int32_t RBBISentMonkey::next(int32_t prevPos) { } } - // Rule (11) (STerm | ATerm) Close* Sp* + // Rule (11) (STerm | ATerm) Close* Sp* (Sep | CR | LF)? int p11 = p1; + if (fSepSet->contains(cAt(p11))) { + p11 = moveBack(p11); + } while (fSpSet->contains(cAt(p11))) { p11 = moveBack(p11); } @@ -2738,6 +2657,7 @@ RBBISentMonkey::~RBBISentMonkey() { delete fOLetterSet; delete fNumericSet; delete fATermSet; + delete fSContinueSet; delete fSTermSet; delete fCloseSet; delete fOtherSet; @@ -2781,6 +2701,7 @@ private: UnicodeSet *fH2; UnicodeSet *fH3; UnicodeSet *fCL; + UnicodeSet *fCP; UnicodeSet *fEX; UnicodeSet *fIN; UnicodeSet *fJL; @@ -2796,7 +2717,10 @@ private: UnicodeSet *fSY; UnicodeSet *fAI; UnicodeSet *fAL; + UnicodeSet *fCJ; + UnicodeSet *fHL; UnicodeSet *fID; + UnicodeSet *fRI; UnicodeSet *fSA; UnicodeSet *fXX; @@ -2816,42 +2740,46 @@ RBBILineMonkey::RBBILineMonkey() fSets = new UVector(status); - fBK = new UnicodeSet("[\\p{Line_Break=BK}]", status); - fCR = new UnicodeSet("[\\p{Line_break=CR}]", status); - fLF = new UnicodeSet("[\\p{Line_break=LF}]", status); - fCM = new UnicodeSet("[\\p{Line_break=CM}]", status); - fNL = new UnicodeSet("[\\p{Line_break=NL}]", status); - fWJ = new UnicodeSet("[\\p{Line_break=WJ}]", status); - fZW = new UnicodeSet("[\\p{Line_break=ZW}]", status); - fGL = new UnicodeSet("[\\p{Line_break=GL}]", status); - fCB = new UnicodeSet("[\\p{Line_break=CB}]", status); - fSP = new UnicodeSet("[\\p{Line_break=SP}]", status); - fB2 = new UnicodeSet("[\\p{Line_break=B2}]", status); - fBA = new UnicodeSet("[\\p{Line_break=BA}]", status); - fBB = new UnicodeSet("[\\p{Line_break=BB}]", status); - fHY = new UnicodeSet("[\\p{Line_break=HY}]", status); - fH2 = new UnicodeSet("[\\p{Line_break=H2}]", status); - fH3 = new UnicodeSet("[\\p{Line_break=H3}]", status); - fCL = new UnicodeSet("[\\p{Line_break=CL}]", status); - fEX = new UnicodeSet("[\\p{Line_break=EX}]", status); - fIN = new UnicodeSet("[\\p{Line_break=IN}]", status); - fJL = new UnicodeSet("[\\p{Line_break=JL}]", status); - fJV = new UnicodeSet("[\\p{Line_break=JV}]", status); - fJT = new UnicodeSet("[\\p{Line_break=JT}]", status); - fNS = new UnicodeSet("[\\p{Line_break=NS}]", status); - fOP = new UnicodeSet("[\\p{Line_break=OP}]", status); - fQU = new UnicodeSet("[\\p{Line_break=QU}]", status); - fIS = new UnicodeSet("[\\p{Line_break=IS}]", status); - fNU = new UnicodeSet("[\\p{Line_break=NU}]", status); - fPO = new UnicodeSet("[\\p{Line_break=PO}]", status); - fPR = new UnicodeSet("[\\p{Line_break=PR}]", status); - fSY = new UnicodeSet("[\\p{Line_break=SY}]", status); - fAI = new UnicodeSet("[\\p{Line_break=AI}]", status); - fAL = new UnicodeSet("[\\p{Line_break=AL}]", status); - fID = new UnicodeSet("[\\p{Line_break=ID}]", status); - fSA = new UnicodeSet("[\\p{Line_break=SA}]", status); - fSG = new UnicodeSet("[\\ud800-\\udfff]", status); - fXX = new UnicodeSet("[\\p{Line_break=XX}]", status); + fBK = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_Break=BK}]"), status); + fCR = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CR}]"), status); + fLF = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=LF}]"), status); + fCM = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CM}]"), status); + fNL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=NL}]"), status); + fWJ = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=WJ}]"), status); + fZW = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=ZW}]"), status); + fGL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=GL}]"), status); + fCB = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CB}]"), status); + fSP = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=SP}]"), status); + fB2 = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=B2}]"), status); + fBA = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=BA}]"), status); + fBB = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=BB}]"), status); + fHY = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=HY}]"), status); + fH2 = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=H2}]"), status); + fH3 = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=H3}]"), status); + fCL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CL}]"), status); + fCP = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CP}]"), status); + fEX = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=EX}]"), status); + fIN = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=IN}]"), status); + fJL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=JL}]"), status); + fJV = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=JV}]"), status); + fJT = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=JT}]"), status); + fNS = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=NS}]"), status); + fOP = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=OP}]"), status); + fQU = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=QU}]"), status); + fIS = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=IS}]"), status); + fNU = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=NU}]"), status); + fPO = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=PO}]"), status); + fPR = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=PR}]"), status); + fSY = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=SY}]"), status); + fAI = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=AI}]"), status); + fAL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=AL}]"), status); + fCJ = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CJ}]"), status); + fHL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=HL}]"), status); + fID = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=ID}]"), status); + fRI = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=RI}]"), status); + fSA = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=SA}]"), status); + fSG = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\ud800-\\udfff]"), status); + fXX = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=XX}]"), status); if (U_FAILURE(status)) { deferredStatus = status; @@ -2865,6 +2793,8 @@ RBBILineMonkey::RBBILineMonkey() fAL->addAll(*fSA); // Default behavior for SA is XX, which defaults to AL fAL->addAll(*fSG); // Default behavior for SG is identical to AL. + fNS->addAll(*fCJ); // Default behavior for CJ is identical to NS. + fSets->addElement(fBK, status); fSets->addElement(fCR, status); fSets->addElement(fLF, status); @@ -2882,6 +2812,7 @@ RBBILineMonkey::RBBILineMonkey() fSets->addElement(fH2, status); fSets->addElement(fH3, status); fSets->addElement(fCL, status); + fSets->addElement(fCP, status); fSets->addElement(fEX, status); fSets->addElement(fIN, status); fSets->addElement(fJL, status); @@ -2897,19 +2828,23 @@ RBBILineMonkey::RBBILineMonkey() fSets->addElement(fSY, status); fSets->addElement(fAI, status); fSets->addElement(fAL, status); + fSets->addElement(fHL, status); fSets->addElement(fID, status); fSets->addElement(fWJ, status); + fSets->addElement(fRI, status); fSets->addElement(fSA, status); fSets->addElement(fSG, status); + const char *rules = + "((\\p{Line_Break=PR}|\\p{Line_Break=PO})\\p{Line_Break=CM}*)?" + "((\\p{Line_Break=OP}|\\p{Line_Break=HY})\\p{Line_Break=CM}*)?" + "\\p{Line_Break=NU}\\p{Line_Break=CM}*" + "((\\p{Line_Break=NU}|\\p{Line_Break=IS}|\\p{Line_Break=SY})\\p{Line_Break=CM}*)*" + "((\\p{Line_Break=CL}|\\p{Line_Break=CP})\\p{Line_Break=CM}*)?" + "((\\p{Line_Break=PR}|\\p{Line_Break=PO})\\p{Line_Break=CM}*)?"; + fNumberMatcher = new RegexMatcher( - "((\\p{Line_Break=PR}|\\p{Line_Break=PO})\\p{Line_Break=CM}*)?" - "((\\p{Line_Break=OP}|\\p{Line_Break=HY})\\p{Line_Break=CM}*)?" - "\\p{Line_Break=NU}\\p{Line_Break=CM}*" - "((\\p{Line_Break=NU}|\\p{Line_Break=IS}|\\p{Line_Break=SY})\\p{Line_Break=CM}*)*" - "(\\p{Line_Break=CL}\\p{Line_Break=CM}*)?" - "((\\p{Line_Break=PR}|\\p{Line_Break=PO})\\p{Line_Break=CM}*)?", - 0, status); + UnicodeString(rules, -1, US_INV), 0, status); fCharBI = BreakIterator::createCharacterInstance(Locale::getEnglish(), status); @@ -2986,6 +2921,9 @@ int32_t RBBILineMonkey::next(int32_t startPos) { // and thisChar may not be adjacent because combining // characters between them will be ignored. + int32_t prevPosX2; // Second previous character. Wider context for LB21a. + UChar32 prevCharX2; + int32_t nextPos; // Index of the next character following pos. // Usually skips over combining marks. int32_t nextCPPos; // Index of the code point following "pos." @@ -2993,6 +2931,10 @@ int32_t RBBILineMonkey::next(int32_t startPos) { int32_t tPos; // temp value. UChar32 c; + if (U_FAILURE(deferredStatus)) { + return -1; + } + if (startPos >= fText->length()) { return -1; } @@ -3001,14 +2943,17 @@ int32_t RBBILineMonkey::next(int32_t startPos) { // Initial values for loop. Loop will run the first time without finding breaks, // while the invalid values shift out and the "this" and // "prev" positions are filled in with good values. - pos = prevPos = -1; // Invalid value, serves as flag for initial loop iteration. - thisChar = prevChar = 0; + pos = prevPos = prevPosX2 = -1; // Invalid value, serves as flag for initial loop iteration. + thisChar = prevChar = prevCharX2 = 0; nextPos = nextCPPos = startPos; // Loop runs once per position in the test text, until a break position // is found. for (;;) { + prevPosX2 = prevPos; + prevCharX2 = prevChar; + prevPos = pos; prevChar = thisChar; @@ -3038,7 +2983,7 @@ int32_t RBBILineMonkey::next(int32_t startPos) { if (prevPos == -1) { continue; } - + // LB 4 Always break after hard line breaks, if (fBK->contains(prevChar)) { break; @@ -3088,23 +3033,30 @@ int32_t RBBILineMonkey::next(int32_t startPos) { } // LB 12 - // (!SP) x GL // GL x - if ((!fSP->contains(prevChar)) && fGL->contains(thisChar) || - fGL->contains(prevChar)) { + if (fGL->contains(prevChar)) { continue; } - + // LB 12a + // [^SP BA HY] x GL + if (!(fSP->contains(prevChar) || + fBA->contains(prevChar) || + fHY->contains(prevChar) ) && fGL->contains(thisChar)) { + continue; + } + + // LB 13 Don't break before closings. - // NU x CL and NU x IS are not matched here so that they will + // NU x CL, NU x CP and NU x IS are not matched here so that they will // fall into LB 17 and the more general number regular expression. // - if (!fNU->contains(prevChar) && fCL->contains(thisChar) || - fEX->contains(thisChar) || - !fNU->contains(prevChar) && fIS->contains(thisChar) || - !fNU->contains(prevChar) && fSY->contains(thisChar)) { + if ((!fNU->contains(prevChar) && fCL->contains(thisChar)) || + (!fNU->contains(prevChar) && fCP->contains(thisChar)) || + fEX->contains(thisChar) || + (!fNU->contains(prevChar) && fIS->contains(thisChar)) || + (!fNU->contains(prevChar) && fSY->contains(thisChar))) { continue; } @@ -3147,8 +3099,8 @@ int32_t RBBILineMonkey::next(int32_t startPos) { - // LB 16 CL SP* x NS - // Scan backwards for SP* CM* CL + // LB 16 (CL | CP) SP* x NS + // Scan backwards for SP* CM* (CL | CP) if (fNS->contains(thisChar)) { int tPos = prevPos; while (tPos>0 && fSP->contains(fText->char32At(tPos))) { @@ -3157,7 +3109,7 @@ int32_t RBBILineMonkey::next(int32_t startPos) { while (tPos>0 && fCM->contains(fText->char32At(tPos))) { tPos = fText->moveIndex32(tPos, -1); } - if (fCL->contains(fText->char32At(tPos))) { + if (fCL->contains(fText->char32At(tPos)) || fCP->contains(fText->char32At(tPos))) { continue; } } @@ -3180,7 +3132,7 @@ int32_t RBBILineMonkey::next(int32_t startPos) { } } - + // LB 18 break after space if (fSP->contains(prevChar)) { break; @@ -3206,36 +3158,52 @@ int32_t RBBILineMonkey::next(int32_t startPos) { continue; } + // LB 21a + // HL (HY | BA) x + if (fHL->contains(prevCharX2) && + (fHY->contains(prevChar) || fBA->contains(prevChar))) { + continue; + } + + // LB 21b - Added for Apple 13927604 + if (fSY->contains(prevChar) && fHL->contains(thisChar)) { + continue; + } + // LB 22 - if (fAL->contains(prevChar) && fIN->contains(thisChar) || - fID->contains(prevChar) && fIN->contains(thisChar) || - fIN->contains(prevChar) && fIN->contains(thisChar) || - fNU->contains(prevChar) && fIN->contains(thisChar) ) { + if ((fAL->contains(prevChar) && fIN->contains(thisChar)) || + (fHL->contains(prevChar) && fIN->contains(thisChar)) || + (fID->contains(prevChar) && fIN->contains(thisChar)) || + (fIN->contains(prevChar) && fIN->contains(thisChar)) || + (fNU->contains(prevChar) && fIN->contains(thisChar)) ) { continue; } // LB 23 ID x PO // AL x NU + // HL x NU // NU x AL - if (fID->contains(prevChar) && fPO->contains(thisChar) || - fAL->contains(prevChar) && fNU->contains(thisChar) || - fNU->contains(prevChar) && fAL->contains(thisChar) ) { + if ((fID->contains(prevChar) && fPO->contains(thisChar)) || + (fAL->contains(prevChar) && fNU->contains(thisChar)) || + (fHL->contains(prevChar) && fNU->contains(thisChar)) || + (fNU->contains(prevChar) && fAL->contains(thisChar)) || + (fNU->contains(prevChar) && fHL->contains(thisChar)) ) { continue; } // LB 24 Do not break between prefix and letters or ideographs. // PR x ID - // PR x AL - // PO x AL - if (fPR->contains(prevChar) && fID->contains(thisChar) || - fPR->contains(prevChar) && fAL->contains(thisChar) || - fPO->contains(prevChar) && fAL->contains(thisChar) ) { + // PR x (AL | HL) + // PO x (AL | HL) + if ((fPR->contains(prevChar) && fID->contains(thisChar)) || + (fPR->contains(prevChar) && (fAL->contains(thisChar) || fHL->contains(thisChar))) || + (fPO->contains(prevChar) && (fAL->contains(thisChar) || fHL->contains(thisChar)))) { continue; } - - - + + + // LB 25 Numbers if (fNumberMatcher->lookingAt(prevPos, status)) { if (U_FAILURE(status)) { @@ -3297,28 +3265,31 @@ int32_t RBBILineMonkey::next(int32_t startPos) { - // LB 28 Do not break between alphabetics (“at”). - if (fAL->contains(prevChar) && fAL->contains(thisChar)) { + // LB 28 Do not break between alphabetics ("at"). + if ((fAL->contains(prevChar) || fHL->contains(prevChar)) && (fAL->contains(thisChar) || fHL->contains(thisChar))) { continue; } // LB 29 Do not break between numeric punctuation and alphabetics ("e.g."). - if (fIS->contains(prevChar) && fAL->contains(thisChar)) { + if (fIS->contains(prevChar) && (fAL->contains(thisChar) || fHL->contains(thisChar))) { continue; } - //LB 30 Do not break between letters, numbers or ordinary symbols and opening or closing punctuation - // (AL | NU) x OP - // CL x (AL | NU) - if ((fAL->contains(prevChar) || fNU->contains(prevChar)) && - fOP->contains(thisChar)) { + // LB 30 Do not break between letters, numbers, or ordinary symbols and opening or closing punctuation. + // (AL | NU) x OP + // CP x (AL | NU) + if ((fAL->contains(prevChar) || fHL->contains(prevChar) || fNU->contains(prevChar)) && fOP->contains(thisChar)) { continue; } - if (fCL->contains(prevChar) && - (fAL->contains(thisChar) || fNU->contains(thisChar))) { + if (fCP->contains(prevChar) && (fAL->contains(thisChar) || fHL->contains(thisChar) || fNU->contains(thisChar))) { continue; } + // LB30a Do not break between regional indicators. + // RI x RI + if (fRI->contains(prevChar) && fRI->contains(thisChar)) { + continue; + } // LB 31 Break everywhere else break; @@ -3354,6 +3325,7 @@ RBBILineMonkey::~RBBILineMonkey() { delete fH2; delete fH3; delete fCL; + delete fCP; delete fEX; delete fIN; delete fJL; @@ -3369,7 +3341,10 @@ RBBILineMonkey::~RBBILineMonkey() { delete fSY; delete fAI; delete fAL; + delete fCJ; + delete fHL; delete fID; + delete fRI; delete fSA; delete fSG; delete fXX; @@ -3418,6 +3393,7 @@ static int32_t getIntParam(UnicodeString name, UnicodeString ¶ms, int32_t d } #endif +#if !UCONFIG_NO_REGULAR_EXPRESSIONS static void testBreakBoundPreceding(RBBITest *test, UnicodeString ustr, BreakIterator *bi, int expected[], @@ -3462,6 +3438,7 @@ static void testBreakBoundPreceding(RBBITest *test, UnicodeString ustr, for (i = bi->last(); i != BreakIterator::DONE; i = bi->previous()) { count --; if (forward[count] != i) { + printStringBreaks(ustr, expected, expectedcount); test->errln("happy break test previous() failed: expected %d but got %d", forward[count], i); break; @@ -3486,46 +3463,47 @@ static void testBreakBoundPreceding(RBBITest *test, UnicodeString ustr, } } } +#endif void RBBITest::TestWordBreaks(void) { #if !UCONFIG_NO_REGULAR_EXPRESSIONS - // <>\u1d4a\u206e\u0603\U0001d7ff<>\u2019<> Locale locale("en"); UErrorCode status = U_ZERO_ERROR; // BreakIterator *bi = BreakIterator::createCharacterInstance(locale, status); BreakIterator *bi = BreakIterator::createWordInstance(locale, status); - UChar str[300]; + // Replaced any C+J characters in a row with a random sequence of characters + // of the same length to make our C+J segmentation not get in the way. static const char *strlist[] = { "\\U000e0032\\u0097\\u0f94\\uc2d8\\u05f4\\U000e0031\\u060d", - "\\U000e0037\\u4666\\u1202\\u003a\\U000e0031\\u064d\\u0bea\\u591c\\U000e0040\\u003b", - "\\u0589\\u3e99\\U0001d7f3\\U000e0074\\u1810\\u200e\\U000e004b\\u179c\\u0027\\U000e0061\\u003a", + "\\U000e0037\\u2666\\u1202\\u003a\\U000e0031\\u064d\\u0bea\\u091c\\U000e0040\\u003b", + "\\u0589\\u3e99\\U0001d7f3\\U000e0074\\u1810\\u200e\\U000e004b\\u0027\\U000e0061\\u003a", "\\u398c\\U000104a5\\U0001d173\\u102d\\u002e\\uca3b\\u002e\\u002c\\u5622", - "\\u90ca\\u3588\\u009c\\u0953\\u194b", + "\\uac00\\u3588\\u009c\\u0953\\u194b", "\\u200e\\U000e0072\\u0a4b\\U000e003f\\ufd2b\\u2027\\u002e\\u002e", "\\u0602\\u2019\\ua191\\U000e0063\\u0a4c\\u003a\\ub4b5\\u003a\\u827f\\u002e", - "\\u7f1f\\uc634\\u65f8\\u0944\\u04f2\\uacdf\\u1f9c\\u05f4\\u002e", + "\\u2f1f\\u1634\\u05f8\\u0944\\u04f2\\u0cdf\\u1f9c\\u05f4\\u002e", "\\U000e0042\\u002e\\u0fb8\\u09ef\\u0ed1\\u2044", "\\u003b\\u024a\\u102e\\U000e0071\\u0600", "\\u2027\\U000e0067\\u0a47\\u00b7", "\\u1fcd\\u002c\\u07aa\\u0027\\u11b0", "\\u002c\\U000e003c\\U0001d7f4\\u003a\\u0c6f\\u0027", "\\u0589\\U000e006e\\u0a42\\U000104a5", - "\\u4f66\\ub523\\u003a\\uacae\\U000e0047\\u003a", + "\\u0f66\\u2523\\u003a\\u0cae\\U000e0047\\u003a", "\\u003a\\u0f21\\u0668\\u0dab\\u003a\\u0655\\u00b7", "\\u0027\\u11af\\U000e0057\\u0602", "\\U0001d7f2\\U000e007\\u0004\\u0589", "\\U000e0022\\u003a\\u10b3\\u003a\\ua21b\\u002e\\U000e0058\\u1732\\U000e002b", "\\U0001d7f2\\U000e007d\\u0004\\u0589", "\\u82ab\\u17e8\\u0736\\u2019\\U0001d64d", - "\\u0e01\\ub55c\\u0a68\\U000e0037\\u0cd6\\u002c\\ub959", + "\\ub55c\\u0a68\\U000e0037\\u0cd6\\u002c\\ub959", "\\U000e0065\\u302c\\uc986\\u09ee\\U000e0068", "\\u0be8\\u002e\\u0c68\\u066e\\u136d\\ufc99\\u59e7", "\\u0233\\U000e0020\\u0a69\\u0d6a", "\\u206f\\u0741\\ub3ab\\u2019\\ubcac\\u2019", - "\\u58f4\\U000e0049\\u20e7\\u2027", + "\\u18f4\\U000e0049\\u20e7\\u2027", "\\ub315\\U0001d7e5\\U000e0073\\u0c47\\u06f2\\u0c6a\\u0037\\u10fe", "\\ua183\\u102d\\u0bec\\u003a", "\\u17e8\\u06e7\\u002e\\u096d\\u003b", @@ -3535,19 +3513,18 @@ void RBBITest::TestWordBreaks(void) "\\U000e005d\\u2044\\u0731\\u0650\\u0061", "\\u003a\\u0664\\u00b7\\u1fba", "\\u003b\\u0027\\u00b7\\u47a3", - "\\u2027\\U000e0067\\u0a42\\u00b7\\ubddf\\uc26c\\u003a\\u4186\\u041b", + "\\u2027\\U000e0067\\u0a42\\u00b7\\u4edf\\uc26c\\u003a\\u4186\\u041b", "\\u0027\\u003a\\U0001d70f\\U0001d7df\\ubf4a\\U0001d7f5\\U0001d177\\u003a\\u0e51\\u1058\\U000e0058\\u00b7\\u0673", "\\uc30d\\u002e\\U000e002c\\u0c48\\u003a\\ub5a1\\u0661\\u002c", }; int loop; if (U_FAILURE(status)) { - errln("Creation of break iterator failed %s", u_errorName(status)); + errcheckln(status, "Creation of break iterator failed %s", u_errorName(status)); return; } for (loop = 0; loop < (int)(sizeof(strlist) / sizeof(char *)); loop ++) { // printf("looping %d\n", loop); - u_unescape(strlist[loop], str, 25); - UnicodeString ustr(str); + UnicodeString ustr = CharsToUnicodeString(strlist[loop]); // RBBICharMonkey monkey; RBBIWordMonkey monkey; @@ -3591,12 +3568,12 @@ void RBBITest::TestWordBoundary(void) "\\U0001d7f2\\U000e007d\\u0004\\u0589", "\\u82ab\\u17e8\\u0736\\u2019\\U0001d64d", "\\u0e01\\ub55c\\u0a68\\U000e0037\\u0cd6\\u002c\\ub959", - "\\U000e0065\\u302c\\uc986\\u09ee\\U000e0068", + "\\U000e0065\\u302c\\u09ee\\U000e0068", "\\u0be8\\u002e\\u0c68\\u066e\\u136d\\ufc99\\u59e7", "\\u0233\\U000e0020\\u0a69\\u0d6a", "\\u206f\\u0741\\ub3ab\\u2019\\ubcac\\u2019", "\\u58f4\\U000e0049\\u20e7\\u2027", - "\\ub315\\U0001d7e5\\U000e0073\\u0c47\\u06f2\\u0c6a\\u0037\\u10fe", + "\\U0001d7e5\\U000e0073\\u0c47\\u06f2\\u0c6a\\u0037\\u10fe", "\\ua183\\u102d\\u0bec\\u003a", "\\u17e8\\u06e7\\u002e\\u096d\\u003b", "\\u003a\\u0e57\\u0fad\\u002e", @@ -3608,7 +3585,7 @@ void RBBITest::TestWordBoundary(void) }; int loop; if (U_FAILURE(status)) { - errln("Creation of break iterator failed %s", u_errorName(status)); + errcheckln(status, "Creation of break iterator failed %s", u_errorName(status)); return; } for (loop = 0; loop < (int)(sizeof(strlist) / sizeof(char *)); loop ++) { @@ -3717,7 +3694,7 @@ void RBBITest::TestLineBreaks(void) continue; } - + UnicodeString ustr(str); RBBILineMonkey monkey; if (U_FAILURE(monkey.deferredStatus)) { @@ -3776,7 +3753,7 @@ void RBBITest::TestSentBreaks(void) }; int loop; if (U_FAILURE(status)) { - errln("Creation of break iterator failed %s", u_errorName(status)); + errcheckln(status, "Creation of break iterator failed %s", u_errorName(status)); return; } for (loop = 0; loop < (int)(sizeof(strlist) / sizeof(char *)); loop ++) { @@ -3843,7 +3820,7 @@ void RBBITest::TestMonkey(char *params) { // m.reset(p); - if (RegexMatcher("\\S", p, 0, status).find()) { + if (RegexMatcher(UNICODE_STRING_SIMPLE("\\S"), p, 0, status).find()) { // Each option is stripped out of the option string as it is processed. // All options have been checked. The option string should have been completely emptied.. char buf[100]; @@ -3866,7 +3843,7 @@ void RBBITest::TestMonkey(char *params) { } } else { - errln("Creation of character break iterator failed %s", u_errorName(status)); + errcheckln(status, "Creation of character break iterator failed %s", u_errorName(status)); } delete bi; } @@ -3879,7 +3856,7 @@ void RBBITest::TestMonkey(char *params) { RunMonkey(bi, m, "word", seed, loopCount, useUText); } else { - errln("Creation of word break iterator failed %s", u_errorName(status)); + errcheckln(status, "Creation of word break iterator failed %s", u_errorName(status)); } delete bi; } @@ -3895,12 +3872,12 @@ void RBBITest::TestMonkey(char *params) { RunMonkey(bi, m, "line", seed, loopCount, useUText); } else { - errln("Creation of line break iterator failed %s", u_errorName(status)); + errcheckln(status, "Creation of line break iterator failed %s", u_errorName(status)); } delete bi; } - if (breakType == "sent" || breakType == "all" ) { + if (breakType == "sent" || breakType == "all" ) { logln("Sentence Break Monkey Test"); RBBISentMonkey m; BreakIterator *bi = BreakIterator::createSentenceInstance(locale, status); @@ -3911,7 +3888,7 @@ void RBBITest::TestMonkey(char *params) { RunMonkey(bi, m, "sentence", seed, loopCount, useUText); } else { - errln("Creation of line break iterator failed %s", u_errorName(status)); + errcheckln(status, "Creation of line break iterator failed %s", u_errorName(status)); } delete bi; } @@ -3928,7 +3905,7 @@ void RBBITest::TestMonkey(char *params) { // seed - Seed for starting random number generator (parameter from user) // numIterations // -void RBBITest::RunMonkey(BreakIterator *bi, RBBIMonkeyKind &mk, const char *name, uint32_t seed, +void RBBITest::RunMonkey(BreakIterator *bi, RBBIMonkeyKind &mk, const char *name, uint32_t seed, int32_t numIterations, UBool useUText) { #if !UCONFIG_NO_REGULAR_EXPRESSIONS @@ -4064,11 +4041,14 @@ void RBBITest::RunMonkey(BreakIterator *bi, RBBIMonkeyKind &mk, const char *name if (breakPos <= i || breakPos < lastBreakPos || breakPos > testText.length() || - breakPos > lastBreakPos && lastBreakPos > i ) { + (breakPos > lastBreakPos && lastBreakPos > i)) { + UChar32 brkChar = testText.char32At(lastBreakPos); + if ((strcmp(name, "char") != 0 && strcmp(name, "word") != 0) || brkChar < 0x1F1E6 || brkChar > 0x1F1FF) { // Apple, skip RI char/word break monkey tests errln("%s break monkey test: " "Out of range value returned by BreakIterator::following().\n" "Random seed=%d index=%d; following returned %d; lastbreak=%d", name, seed, i, breakPos, lastBreakPos); + } break; } followingBreaks[breakPos] = 1; @@ -4076,22 +4056,29 @@ void RBBITest::RunMonkey(BreakIterator *bi, RBBIMonkeyKind &mk, const char *name } // Find the break positions using the preceding() function. - memset(precedingBreaks, 0, sizeof(followingBreaks)); + memset(precedingBreaks, 0, sizeof(precedingBreaks)); lastBreakPos = testText.length(); precedingBreaks[testText.length()] = 1; for (i=testText.length(); i>0; i--) { breakPos = bi->preceding(i); if (breakPos >= i || breakPos > lastBreakPos || - breakPos < 0 && testText.getChar32Start(i)>0 || - breakPos < lastBreakPos && lastBreakPos < testText.getChar32Start(i) ) { + (breakPos < 0 && testText.getChar32Start(i)>0) || + (breakPos < lastBreakPos && lastBreakPos < testText.getChar32Start(i)) ) { + UChar32 brkChar = testText.char32At(breakPos); + if ((strcmp(name, "char") != 0 && strcmp(name, "word") != 0) || brkChar < 0x1F1E6 || brkChar > 0x1F1FF) { // Apple, skip RI char/word break monkey tests errln("%s break monkey test: " "Out of range value returned by BreakIterator::preceding().\n" "index=%d; prev returned %d; lastBreak=%d" , name, i, breakPos, lastBreakPos); - precedingBreaks[i] = 2; // Forces an error. + if (breakPos >= 0 && breakPos < (int32_t)sizeof(precedingBreaks)) { + precedingBreaks[i] = 2; // Forces an error. + } + } } else { - precedingBreaks[breakPos] = 1; + if (breakPos >= 0) { + precedingBreaks[breakPos] = 1; + } lastBreakPos = breakPos; } } @@ -4185,9 +4172,14 @@ void RBBITest::RunMonkey(BreakIterator *bi, RBBIMonkeyKind &mk, const char *name UErrorCode status = U_ZERO_ERROR; errorText.extract(charErrorTxt, sizeof(charErrorTxt), NULL, status); charErrorTxt[sizeof(charErrorTxt)-1] = 0; - errln("%s break monkey test error. %s. Operation = %s; Random seed = %d; buf Idx = %d\n%s", - name, (expectedBreaks[i]? "break expected but not found" : "break found but not expected"), + const char *badLocale = bi->getLocaleID(ULOC_ACTUAL_LOCALE, status); + + UChar32 brkChar = testText.char32At(i); + if ((strcmp(name, "char") != 0 && strcmp(name, "word") != 0) || brkChar < 0x1F1E6 || brkChar > 0x1F1FF) { // Apple, skip RI char/word break monkey tests + errln("%s break monkey test error [%s]. %s. Operation = %s; Random seed = %d; buf Idx = %d\n%s", + name, badLocale, (expectedBreaks[i]? "break expected but not found" : "break found but not expected"), errorType, seed, i, charErrorTxt); + } break; } } @@ -4197,6 +4189,84 @@ void RBBITest::RunMonkey(BreakIterator *bi, RBBIMonkeyKind &mk, const char *name #endif } + +// Bug 5532. UTF-8 based UText fails in dictionary code. +// This test checks the initial patch, +// which is to just keep it from crashing. Correct word boundaries +// await a proper fix to the dictionary code. +// +void RBBITest::TestBug5532(void) { + // Text includes a mixture of Thai and Latin. + const unsigned char utf8Data[] = { + 0xE0u, 0xB8u, 0x82u, 0xE0u, 0xB8u, 0xB2u, 0xE0u, 0xB8u, 0xA2u, 0xE0u, + 0xB9u, 0x80u, 0xE0u, 0xB8u, 0x84u, 0xE0u, 0xB8u, 0xA3u, 0xE0u, 0xB8u, + 0xB7u, 0xE0u, 0xB9u, 0x88u, 0xE0u, 0xB8u, 0xADu, 0xE0u, 0xB8u, 0x87u, + 0xE0u, 0xB9u, 0x80u, 0xE0u, 0xB8u, 0xA5u, 0xE0u, 0xB9u, 0x88u, 0xE0u, + 0xB8u, 0x99u, 0xE0u, 0xB8u, 0x8Bu, 0xE0u, 0xB8u, 0xB5u, 0xE0u, 0xB8u, + 0x94u, 0xE0u, 0xB8u, 0xB5u, 0x20u, 0x73u, 0x69u, 0x6Du, 0x20u, 0x61u, + 0x75u, 0x64u, 0x69u, 0x6Fu, 0x2Fu, 0x20u, 0x4Du, 0x4Fu, 0x4Fu, 0x4Eu, + 0x20u, 0x65u, 0x63u, 0x6Cu, 0x69u, 0x70u, 0x73u, 0x65u, 0x20u, 0xE0u, + 0xB8u, 0xA3u, 0xE0u, 0xB8u, 0xB2u, 0xE0u, 0xB8u, 0x84u, 0xE0u, 0xB8u, + 0xB2u, 0x20u, 0x34u, 0x37u, 0x30u, 0x30u, 0x20u, 0xE0u, 0xB8u, 0xA2u, + 0xE0u, 0xB8u, 0xB9u, 0xE0u, 0xB9u, 0x82u, 0xE0u, 0xB8u, 0xA3u, 0x00}; + + UErrorCode status = U_ZERO_ERROR; + UText utext=UTEXT_INITIALIZER; + utext_openUTF8(&utext, (const char *)utf8Data, -1, &status); + TEST_ASSERT_SUCCESS(status); + + BreakIterator *bi = BreakIterator::createWordInstance(Locale("th"), status); + TEST_ASSERT_SUCCESS(status); + if (U_SUCCESS(status)) { + bi->setText(&utext, status); + TEST_ASSERT_SUCCESS(status); + + int32_t breakCount = 0; + int32_t previousBreak = -1; + for (bi->first(); bi->next() != BreakIterator::DONE; breakCount++) { + // For now, just make sure that the break iterator doesn't hang. + TEST_ASSERT(previousBreak < bi->current()); + previousBreak = bi->current(); + } + TEST_ASSERT(breakCount > 0); + } + delete bi; + utext_close(&utext); +} + + +void RBBITest::TestBug9983(void) { + UnicodeString text = UnicodeString("\\u002A" // * Other + "\\uFF65" // Other + "\\u309C" // Katakana + "\\uFF9F" // Extend + "\\uFF65" // Other + "\\u0020" // Other + "\\u0000").unescape(); + + UErrorCode status = U_ZERO_ERROR; + LocalPointer brkiter(static_cast( + BreakIterator::createWordInstance(Locale::getRoot(), status))); + TEST_ASSERT_SUCCESS(status); + if (U_FAILURE(status)) { + return; + } + brkiter->setText(text); + int32_t offset, rstatus; + brkiter->last(); + int32_t iterationCount = 0; + while ( (offset = brkiter->previous()) != UBRK_DONE ) { + iterationCount++; + rstatus = brkiter->getRuleStatus(); + // printf(" %d(%d)", offset, rstatus); + if (iterationCount >= 10) { + break; + } + } + TEST_ASSERT(iterationCount == 6); +} + + // // TestDebug - A place-holder test for debugging purposes. // For putting in fragments of other tests that can be invoked @@ -4228,4 +4298,15 @@ void RBBITest::TestDebug(void) { #endif } +void RBBITest::TestProperties() { + UErrorCode errorCode = U_ZERO_ERROR; + UnicodeSet prependSet(UNICODE_STRING_SIMPLE("[:GCB=Prepend:]"), errorCode); + if (!prependSet.isEmpty()) { + errln( + "[:GCB=Prepend:] is not empty any more. " + "Uncomment relevant lines in source/data/brkitr/char.txt and " + "change this test to the opposite condition."); + } +} + #endif /* #if !UCONFIG_NO_BREAK_ITERATION */