/********************************************************************
* COPYRIGHT:
- * Copyright (c) 1999-2006, International Business Machines Corporation and
+ * Copyright (c) 1999-2016, International Business Machines Corporation and
* others. All Rights Reserved.
********************************************************************/
/************************************************************************
************************************************************************/
#include "unicode/utypes.h"
-
#if !UCONFIG_NO_BREAK_ITERATION
-#include "unicode/utypes.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
#include "unicode/brkiter.h"
+#include "unicode/localpointer.h"
+#include "unicode/numfmt.h"
#include "unicode/rbbi.h"
+#if !UCONFIG_NO_REGULAR_EXPRESSIONS
+#include "unicode/regex.h"
+#endif
+#include "unicode/schriter.h"
#include "unicode/uchar.h"
#include "unicode/utf16.h"
#include "unicode/ucnv.h"
-#include "unicode/schriter.h"
#include "unicode/uniset.h"
-#include "unicode/regex.h" // TODO: make conditional on regexp being built.
+#include "unicode/uscript.h"
#include "unicode/ustring.h"
#include "unicode/utext.h"
+
+#include "charstr.h"
+#include "cmemory.h"
#include "intltest.h"
#include "rbbitst.h"
-#include <string.h>
+#include "utypeinfo.h" // for 'typeid' to work
#include "uvector.h"
#include "uvectr32.h"
-#include "triedict.h"
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
+
+#if !UCONFIG_NO_FILTERED_BREAK_ITERATION
+#include "unicode/filteredbrk.h"
+#endif // !UCONFIG_NO_FILTERED_BREAK_ITERATION
#define TEST_ASSERT(x) {if (!(x)) { \
errln("Failure in file %s, line %d", __FILE__, __LINE__);}}
-#define TEST_ASSERT_SUCCESS(errcode) {if (U_FAILURE(errcode)) { \
- errln("Failure in file %s, line %d, status = \"%s\"", __FILE__, __LINE__, u_errorName(errcode));}}
+#define TEST_ASSERT_SUCCESS(errcode) { if (U_FAILURE(errcode)) { \
+ errcheckln(errcode, "Failure in file %s, line %d, status = \"%s\"", __FILE__, __LINE__, u_errorName(errcode));}}
+
+
+//---------------------------------------------
+// runIndexedTest
+//---------------------------------------------
+
+
+// Note: Before adding new tests to this file, check whether the desired test data can
+// simply be added to the file testdata/rbbitest.txt. In most cases it can,
+// it's much less work than writing a new test, diagnostic output in the event of failures
+// is good, and the test data file will is shared with ICU4J, so eventually the test
+// will run there as well, without additional effort.
+
+void RBBITest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* params )
+{
+ if (exec) logln("TestSuite RuleBasedBreakIterator: ");
+
+ switch (index) {
+#if !UCONFIG_NO_FILE_IO
+ case 0: name = "TestBug4153072";
+ if(exec) TestBug4153072(); break;
+#else
+ case 0: name = "skip";
+ break;
+#endif
+
+ case 1: name = "skip";
+ break;
+ case 2: name = "TestStatusReturn";
+ if(exec) TestStatusReturn(); break;
+
+#if !UCONFIG_NO_FILE_IO
+ case 3: name = "TestUnicodeFiles";
+ if(exec) TestUnicodeFiles(); break;
+ case 4: name = "TestEmptyString";
+ if(exec) TestEmptyString(); break;
+#else
+ case 3: case 4: name = "skip";
+ break;
+#endif
+
+ case 5: name = "TestGetAvailableLocales";
+ if(exec) TestGetAvailableLocales(); break;
+
+ case 6: name = "TestGetDisplayName";
+ if(exec) TestGetDisplayName(); break;
+
+#if !UCONFIG_NO_FILE_IO
+ case 7: name = "TestEndBehaviour";
+ if(exec) TestEndBehaviour(); break;
+ case 8: case 9: case 10: name = "skip";
+ break;
+ case 11: name = "TestWordBreaks";
+ if(exec) TestWordBreaks(); break;
+ case 12: name = "TestWordBoundary";
+ if(exec) TestWordBoundary(); break;
+ case 13: name = "TestLineBreaks";
+ if(exec) TestLineBreaks(); break;
+ case 14: name = "TestSentBreaks";
+ if(exec) TestSentBreaks(); break;
+ case 15: name = "TestExtended";
+ if(exec) TestExtended(); break;
+#else
+ case 7: case 8: case 9: case 10: case 11: case 12: case 13: case 14: case 15: name = "skip";
+ break;
+#endif
+
+#if !UCONFIG_NO_REGULAR_EXPRESSIONS && !UCONFIG_NO_FILE_IO
+ case 16:
+ name = "TestMonkey"; if(exec) TestMonkey(params); break;
+#else
+ case 16:
+ name = "skip"; break;
+#endif
+
+#if !UCONFIG_NO_FILE_IO
+ case 17: name = "TestBug3818";
+ if(exec) TestBug3818(); break;
+#else
+ case 17: name = "skip";
+ break;
+#endif
+
+ case 18: name = "skip";
+ break;
+ case 19: name = "TestDebug";
+ if(exec) TestDebug(); break;
+ case 20: name = "skip";
+ break;
+
+#if !UCONFIG_NO_FILE_IO
+ case 21: name = "TestBug5775";
+ if (exec) TestBug5775(); break;
+#else
+ case 21: name = "skip";
+ break;
+#endif
+
+ case 22: name = "TestBug9983";
+ if (exec) TestBug9983(); break;
+ case 23: name = "TestDictRules";
+ if (exec) TestDictRules(); break;
+ case 24: name = "TestBug5532";
+ if (exec) TestBug5532(); break;
+ default: name = ""; break; //needed to end loop
+ }
+}
//---------------------------------------------------------------------------
o = actual - fExpectedBreakPositions.elementAti(expectedIdx-1);
}
if (actual < expected) {
- test->errln("%s unexpected break at offset %d in test item from line %d", heading, o, line);
+ test->errln("%s unexpected break at offset %d in test item from line %d. actual break: %d expected break: %d", heading, o, line, actual, expected);
} else {
- test->errln("%s Failed to find break at end of item from line %d", heading, line);
+ test->errln("%s Failed to find break at end of item from line %d. actual break: %d expected break: %d", heading, line, actual, expected);
}
}
}
-//-----------------------------------------------------------------------------------
-//
-// Cannned Test Characters
-//
-//-----------------------------------------------------------------------------------
-
-static const UChar cannedTestArray[] = {
- 0x0001, 0x0002, 0x0003, 0x0004, 0x0020, 0x0021, '\\', 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0028, 0x0029, 0x002b, 0x002d, 0x0030, 0x0031,
- 0x0032, 0x0033, 0x0034, 0x003c, 0x003d, 0x003e, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x005b, 0x005d, 0x005e, 0x005f, 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x007b,
- 0x007d, 0x007c, 0x002c, 0x00a0, 0x00a2,
- 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00ab, 0x00ad, 0x00ae, 0x00af, 0x00b0, 0x00b2, 0x00b3,
- 0x00b4, 0x00b9, 0x00bb, 0x00bc, 0x00bd, 0x02b0, 0x02b1, 0x02b2, 0x02b3, 0x02b4, 0x0300, 0x0301, 0x0302, 0x0303,
- 0x0304, 0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x0903, 0x093e, 0x093f, 0x0940, 0x0949, 0x0f3a, 0x0f3b, 0x2000,
- 0x2001, 0x2002, 0x200c, 0x200d, 0x200e, 0x200f, 0x2010, 0x2011, 0x2012, 0x2028, 0x2029, 0x202a, 0x203e, 0x203f,
- 0x2040, 0x20dd, 0x20de, 0x20df, 0x20e0, 0x2160, 0x2161, 0x2162, 0x2163, 0x2164, 0x0000
-};
-
-static UnicodeString* cannedTestChars = 0;
-
-#define halfNA "\\u0928\\u094d\\u200d"
-#define halfSA "\\u0938\\u094d\\u200d"
-#define halfCHA "\\u091a\\u094d\\u200d"
-#define halfKA "\\u0915\\u094d\\u200d"
-#define deadTA "\\u0924\\u094d"
-
//--------------------------------------------------------------------------------------
//
// RBBITest constructor and destructor
//--------------------------------------------------------------------------------------
RBBITest::RBBITest() {
- UnicodeString temp(cannedTestArray);
- cannedTestChars = new UnicodeString();
- *cannedTestChars += (UChar)0x0000;
- *cannedTestChars += temp;
}
RBBITest::~RBBITest() {
- delete cannedTestChars;
}
-
-static const int T_NUMBER = 100;
-static const int T_LETTER = 200;
-static const int T_H_OR_K = 300;
-static const int T_IDEO = 400;
-
-
-
-
-
-
-//--------------------------------------------------------------------
-//Testing the BreakIterator for devanagari script
-//--------------------------------------------------------------------
-
-#define deadRA "\\u0930\\u094d" /*deadform RA = devanagari RA + virama*/
-#define deadPHA "\\u092b\\u094d" /*deadform PHA = devanagari PHA + virama*/
-#define deadTTHA "\\u0920\\u094d"
-#define deadPA "\\u092a\\u094d"
-#define deadSA "\\u0938\\u094d"
-#define visarga "\\u0903" /*devanagari visarga looks like a english colon*/
-
-
-
-
-
-
//-----------------------------------------------------------------------------------
//
// Test for status {tag} return value from break rules.
//
//-----------------------------------------------------------------------------------
void RBBITest::TestStatusReturn() {
- UnicodeString rulesString1 = "$Letters = [:L:];\n"
+ UnicodeString rulesString1("$Letters = [:L:];\n"
"$Numbers = [:N:];\n"
"$Letters+{1};\n"
"$Numbers+{2};\n"
- "Help\\ {4}/me\\!;\n"
+ "Help\\ /me\\!{4};\n"
"[^$Letters $Numbers];\n"
- "!.*;\n";
+ "!.*;\n", -1, US_INV);
UnicodeString testString1 = "abc123..abc Help me Help me!";
// 01234567890123456789012345678
int32_t bounds1[] = {0, 3, 6, 7, 8, 11, 12, 16, 17, 19, 20, 25, 27, 28, -1};
UErrorCode status=U_ZERO_ERROR;
UParseError parseError;
- RuleBasedBreakIterator *bi = new RuleBasedBreakIterator(rulesString1, parseError, status);
+ LocalPointer <BreakIterator> bi(new RuleBasedBreakIterator(rulesString1, parseError, status));
if(U_FAILURE(status)) {
- errln("FAIL : in construction");
- } else {
- int32_t pos;
- int32_t i = 0;
- bi->setText(testString1);
- for (pos=bi->first(); pos!= BreakIterator::DONE; pos=bi->next()) {
- if (pos != bounds1[i]) {
- errln("FAIL: expected break at %d, got %d\n", bounds1[i], pos);
- break;
- }
+ dataerrln("%s:%d error in break iterator construction - %s", __FILE__, __LINE__, u_errorName(status));
+ return;
+ }
+ int32_t pos;
+ int32_t i = 0;
+ bi->setText(testString1);
+ for (pos=bi->first(); pos!= BreakIterator::DONE; pos=bi->next()) {
+ if (pos != bounds1[i]) {
+ errln("%s:%d expected break at %d, got %d\n", __FILE__, __LINE__, bounds1[i], pos);
+ break;
+ }
- int tag = bi->getRuleStatus();
- if (tag != brkStatus[i]) {
- errln("FAIL: break at %d, expected tag %d, got tag %d\n", pos, brkStatus[i], tag);
- break;
- }
- i++;
+ int tag = bi->getRuleStatus();
+ if (tag != brkStatus[i]) {
+ errln("%s:%d break at %d, expected tag %d, got tag %d\n", __FILE__, __LINE__, pos, brkStatus[i], tag);
+ break;
}
+ i++;
}
- delete bi;
}
-static void printStringBreaks(UnicodeString ustr, int expected[],
- int expectedcount)
-{
+static void printStringBreaks(UText *tstr, int expected[], int expectedCount) {
UErrorCode status = U_ZERO_ERROR;
char name[100];
printf("code alpha extend alphanum type word sent line name\n");
- int j;
- for (j = 0; j < ustr.length(); j ++) {
- if (expectedcount > 0) {
- int k;
- for (k = 0; k < expectedcount; k ++) {
- if (j == expected[k]) {
- printf("------------------------------------------------ %d\n",
- j);
- }
- }
- }
- UChar32 c = ustr.char32At(j);
- if (c > 0xffff) {
- j ++;
+ int nextExpectedIndex = 0;
+ utext_setNativeIndex(tstr, 0);
+ for (int j = 0; j < utext_nativeLength(tstr); j=utext_getNativeIndex(tstr)) {
+ if (nextExpectedIndex < expectedCount && j >= expected[nextExpectedIndex] ) {
+ printf("------------------------------------------------ %d\n", j);
+ ++nextExpectedIndex;
}
+
+ UChar32 c = utext_next32(tstr);
u_charName(c, U_UNICODE_CHAR_NAME, name, 100, &status);
printf("%7x %5d %6d %8d %4s %4s %4s %4s %s\n", (int)c,
u_isUAlphabetic(c),
}
}
-void RBBITest::TestThaiLineBreak() {
- UErrorCode status = U_ZERO_ERROR;
- BITestData thaiLineSelection(status);
-
- // \u0e2f-- the Thai paiyannoi character-- isn't a letter. It's a symbol that
- // represents elided letters at the end of a long word. It should be bound to
- // the end of the word and not treated as an independent punctuation mark.
-
-
- ADD_DATACHUNK(thaiLineSelection, NULL, 0, status); // Break at start of data
- ADD_DATACHUNK(thaiLineSelection, "\\u0e2a\\u0e16\\u0e32\\u0e19\\u0e35\\u0e2f", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e08\\u0e30", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e23\\u0e30\\u0e14\\u0e21", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e08\\u0e49\\u0e32", 0, status);
-// ADD_DATACHUNK(thaiLineSelection, "\\u0e2b\\u0e19\\u0e49\\u0e32", 0, status);
-// ADD_DATACHUNK(thaiLineSelection, "\\u0e17\\u0e35\\u0e48", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e2b\\u0e19\\u0e49\\u0e32\\u0e17\\u0e35\\u0e48", 0, status);
- // the commented-out lines (I think) are the preferred result; this line is what our current dictionary is giving us
- ADD_DATACHUNK(thaiLineSelection, "\\u0e2d\\u0e2d\\u0e01", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e21\\u0e32", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e23\\u0e48\\u0e07", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e23\\u0e30\\u0e1a\\u0e32\\u0e22", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e2d\\u0e22\\u0e48\\u0e32\\u0e07", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e15\\u0e47\\u0e21", 0, status);
-
- // the one time where the paiyannoi occurs somewhere other than at the end
- // of a word is in the Thai abbrevation for "etc.", which both begins and
- // ends with a paiyannoi
- ADD_DATACHUNK(thaiLineSelection, "\\u0e2f\\u0e25\\u0e2f", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e17\\u0e35\\u0e48", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e19\\u0e31\\u0e49\\u0e19", 0, status);
-
- RuleBasedBreakIterator* e = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(
- Locale("th"), status);
- if (U_FAILURE(status))
- {
- errln("Failed to create the BreakIterator for Thai locale in TestThaiLineBreak.\n");
- return;
- }
-
- generalIteratorTest(*e, thaiLineSelection);
- delete e;
-}
-
-
-
-void RBBITest::TestMixedThaiLineBreak()
-{
- UErrorCode status = U_ZERO_ERROR;
- BITestData thaiLineSelection(status);
- ADD_DATACHUNK(thaiLineSelection, NULL, 0, status); // Break at start of data
-
-
- // @suwit -- Test Arabic numerals, Thai numerals, Punctuation and English characters
- // start
-
- ADD_DATACHUNK(thaiLineSelection, "\\u0E1B\\u0E35", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0E1E\\u0E38\\u0E17\\u0E18\\u0E28\\u0E31\\u0E01\\u0E23\\u0E32\\u0E0A ", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "2545 ", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0E40\\u0E1B\\u0E47\\u0E19", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0E1B\\u0E35", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0E09\\u0E25\\u0E2D\\u0E07", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0E04\\u0E23\\u0E1A", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0E23\\u0E2D\\u0E1A ", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\"\\u0E52\\u0E52\\u0E50 ", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0E1b\\u0E35\" ", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0E02\\u0E2d\\u0E07", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0E01\\u0E23\\u0E38\\u0E07", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0E23\\u0E31\\u0E15\\u0E19\\u0E42\\u0E01\\u0E2A\\u0E34\\u0E19\\u0E17\\u0E23\\u0E4C ", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "(\\u0E01\\u0E23\\u0E38\\u0E07\\u0E40\\u0E17\\u0E1e\\u0E2F", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0E2B\\u0E23\\u0E37\\u0E2D ", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "Bangkok)", 0, status);
-
- // @suwit - end of changes
-
-
- RuleBasedBreakIterator* e = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(Locale("th"), status);
- if (U_FAILURE(status))
- {
- errln("Failed to create the BreakIterator for Thai locale in TestMixedThaiLineBreak.\n");
- return;
+static void printStringBreaks(const UnicodeString &ustr, int expected[], int expectedCount) {
+ UErrorCode status = U_ZERO_ERROR;
+ UText *tstr = NULL;
+ tstr = utext_openConstUnicodeString(NULL, &ustr, &status);
+ if (U_FAILURE(status)) {
+ printf("printStringBreaks, utext_openConstUnicodeString() returns %s\n", u_errorName(status));
+ return;
}
-
-
- generalIteratorTest(*e, thaiLineSelection);
- delete e;
+ printStringBreaks(tstr, expected, expectedCount);
+ utext_close(tstr);
}
-void RBBITest::TestMaiyamok()
-{
- UErrorCode status = U_ZERO_ERROR;
- BITestData thaiLineSelection(status);
- ADD_DATACHUNK(thaiLineSelection, NULL, 0, status); // Break at start of data
- // the Thai maiyamok character is a shorthand symbol that means "repeat the previous
- // word". Instead of appearing as a word unto itself, however, it's kept together
- // with the word before it
- ADD_DATACHUNK(thaiLineSelection, "\\u0e44\\u0e1b\\u0e46", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e21\\u0e32\\u0e46", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e23\\u0e30\\u0e2b\\u0e27\\u0e48\\u0e32\\u0e07", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e01\\u0e23\\u0e38\\u0e07", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e17\\u0e1e", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e41\\u0e25\\u0e30", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e40\\u0e03\\u0e35", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e22\\u0e07", 0, status);
- ADD_DATACHUNK(thaiLineSelection, "\\u0e43\\u0e2b\\u0e21\\u0e48", 0, status);
-
- RuleBasedBreakIterator* e = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(
- Locale("th"), status);
-
- if (U_FAILURE(status))
- {
- errln("Failed to create the BreakIterator for Thai locale in TestMaiyamok.\n");
- return;
- }
- generalIteratorTest(*e, thaiLineSelection);
- delete e;
-}
-
-
-
void RBBITest::TestBug3818() {
UErrorCode status = U_ZERO_ERROR;
0x0E43,0x0E2B,0x0E0D,0x0E48, 0x0E43,0x0E2B,0x0E0D,0x0E48, 0 };
UnicodeString thaiStr(thaiWordData);
- RuleBasedBreakIterator* bi =
- (RuleBasedBreakIterator *)BreakIterator::createWordInstance(Locale("th"), status);
+ BreakIterator* bi = BreakIterator::createWordInstance(Locale("th"), status);
if (U_FAILURE(status) || bi == NULL) {
- errln("Fail at file %s, line %d, status = %s", __FILE__, __LINE__, u_errorName(status));
+ errcheckln(status, "Fail at file %s, line %d, status = %s", __FILE__, __LINE__, u_errorName(status));
return;
}
bi->setText(thaiStr);
delete bi;
}
-
-void RBBITest::TestJapaneseWordBreak() {
- UErrorCode status = U_ZERO_ERROR;
- BITestData japaneseWordSelection(status);
-
- ADD_DATACHUNK(japaneseWordSelection, NULL, 0, status); // Break at start of data
- ADD_DATACHUNK(japaneseWordSelection, "\\u4ECA\\u65E5", 400, status); //2
- ADD_DATACHUNK(japaneseWordSelection, "\\u306F\\u3044\\u3044", 300, status); //5
- ADD_DATACHUNK(japaneseWordSelection, "\\u5929\\u6C17", 400, status); //7
- ADD_DATACHUNK(japaneseWordSelection, "\\u3067\\u3059\\u306D", 300, status); //10
- ADD_DATACHUNK(japaneseWordSelection, "\\u3002", 0, status); //11
- ADD_DATACHUNK(japaneseWordSelection, "\\u000D\\u000A", 0, status); //12
-
- RuleBasedBreakIterator* e = (RuleBasedBreakIterator *)BreakIterator::createWordInstance(
- Locale("ja"), status);
- if (U_FAILURE(status))
- {
- errln("Failed to create the BreakIterator for Japanese locale in TestJapaneseWordBreak.\n");
- return;
- }
-
- generalIteratorTest(*e, japaneseWordSelection);
- delete e;
-}
-
-void RBBITest::TestTrieDict() {
- UErrorCode status = U_ZERO_ERROR;
-
- //
- // Open and read the test data file.
- //
- const char *testDataDirectory = IntlTest::getSourceTestData(status);
- char testFileName[1000];
- if (testDataDirectory == NULL || strlen(testDataDirectory) + strlen("riwords.txt") + 10 >= sizeof(testFileName)) {
- errln("Can't open test data. Path too long.");
- return;
- }
- strcpy(testFileName, testDataDirectory);
- strcat(testFileName, "riwords.txt");
-
- // Items needing deleting at the end
- MutableTrieDictionary *mutableDict = NULL;
- CompactTrieDictionary *compactDict = NULL;
- UnicodeSet *breaks = NULL;
- UChar *testFile = NULL;
- StringEnumeration *enumer = NULL;
- MutableTrieDictionary *mutable2 = NULL;
- StringEnumeration *cloneEnum = NULL;
- CompactTrieDictionary *compact2 = NULL;
-
-
- const UnicodeString *originalWord = NULL;
- const UnicodeString *cloneWord = NULL;
- UChar *current;
- UChar *word;
- UChar uc;
- int32_t wordLen;
- int32_t wordCount;
- int32_t testCount;
-
- int len;
- testFile = ReadAndConvertFile(testFileName, len, status);
- if (U_FAILURE(status)) {
- goto cleanup; /* something went wrong, error already output */
- }
-
- mutableDict = new MutableTrieDictionary(0x0E1C, status);
- if (U_FAILURE(status)) {
- errln("Error creating MutableTrieDictionary: %s\n", u_errorName(status));
- goto cleanup;
- }
-
- breaks = new UnicodeSet;
- breaks->add(0x000A); // Line Feed
- breaks->add(0x000D); // Carriage Return
- breaks->add(0x2028); // Line Separator
- breaks->add(0x2029); // Paragraph Separator
-
- // Now add each non-comment line of the file as a word.
- current = testFile;
- word = current;
- uc = *current++;
- wordLen = 0;
- wordCount = 0;
-
- while (uc) {
- if (uc == 0x0023) { // #comment line, skip
- while (uc && !breaks->contains(uc)) {
- uc = *current++;
- }
- }
- else while (uc && !breaks->contains(uc)) {
- ++wordLen;
- uc = *current++;
- }
- if (wordLen > 0) {
- mutableDict->addWord(word, wordLen, status);
- if (U_FAILURE(status)) {
- errln("Could not add word to mutable dictionary; status %s\n", u_errorName(status));
- goto cleanup;
- }
- wordCount += 1;
- }
-
- // Find beginning of next line
- while (uc && breaks->contains(uc)) {
- uc = *current++;
- }
- word = current-1;
- wordLen = 0;
- }
-
- if (wordCount < 50) {
- errln("Word count (%d) unreasonably small\n", wordCount);
- goto cleanup;
- }
-
- enumer = mutableDict->openWords(status);
- if (U_FAILURE(status)) {
- errln("Could not open mutable dictionary enumerator: %s\n", u_errorName(status));
- goto cleanup;
- }
-
- testCount = 0;
- if (wordCount != (testCount = enumer->count(status))) {
- errln("MutableTrieDictionary word count (%d) differs from file word count (%d), with status %s\n",
- testCount, wordCount, u_errorName(status));
- goto cleanup;
- }
-
- delete enumer;
- enumer = NULL;
-
- // Now compact it
- compactDict = new CompactTrieDictionary(*mutableDict, status);
- if (U_FAILURE(status)) {
- errln("Failed to create CompactTrieDictionary: %s\n", u_errorName(status));
- goto cleanup;
- }
-
- enumer = compactDict->openWords(status);
- if (U_FAILURE(status)) {
- errln("Could not open compact trie dictionary enumerator: %s\n", u_errorName(status));
- goto cleanup;
- }
-
- if (wordCount != (testCount = enumer->count(status))) {
- errln("CompactTrieDictionary word count (%d) differs from file word count (%d), with status %s\n",
- testCount, wordCount, u_errorName(status));
- goto cleanup;
- }
-
- delete enumer;
- enumer = NULL;
-
- // Now un-compact it
- mutable2 = compactDict->cloneMutable(status);
- if (U_FAILURE(status)) {
- errln("Could not clone CompactTrieDictionary to MutableTrieDictionary: %s\n", u_errorName(status));
- goto cleanup;
- }
-
- cloneEnum = mutable2->openWords(status);
- if (U_FAILURE(status)) {
- errln("Could not create cloned mutable enumerator: %s\n", u_errorName(status));
- goto cleanup;
- }
-
- if (wordCount != (testCount = cloneEnum->count(status))) {
- errln("Cloned MutableTrieDictionary word count (%d) differs from file word count (%d), with status %s\n",
- testCount, wordCount, u_errorName(status));
- goto cleanup;
- }
-
- // Compact original dictionary to clone. Note that we can only compare the same kind of
- // dictionary as the order of the enumerators is not guaranteed to be the same between
- // different kinds
- enumer = mutableDict->openWords(status);
- if (U_FAILURE(status)) {
- errln("Could not re-open mutable dictionary enumerator: %s\n", u_errorName(status));
- goto cleanup;
- }
-
- originalWord = enumer->snext(status);
- cloneWord = cloneEnum->snext(status);
- while (U_SUCCESS(status) && originalWord != NULL && cloneWord != NULL) {
- if (*originalWord != *cloneWord) {
- errln("Original and cloned MutableTrieDictionary word mismatch\n");
- goto cleanup;
- }
- originalWord = enumer->snext(status);
- cloneWord = cloneEnum->snext(status);
- }
-
- if (U_FAILURE(status)) {
- errln("Enumeration failed: %s\n", u_errorName(status));
- goto cleanup;
- }
-
- if (originalWord != cloneWord) {
- errln("Original and cloned MutableTrieDictionary ended enumeration at different points\n");
- goto cleanup;
- }
-
- // Test the data copying constructor for CompactTrieDict, and the data access APIs.
- compact2 = new CompactTrieDictionary(compactDict->data(), status);
- if (U_FAILURE(status)) {
- errln("CompactTrieDictionary(const void *,...) failed\n");
- goto cleanup;
- }
-
- if (compact2->dataSize() == 0) {
- errln("CompactTrieDictionary->dataSize() == 0\n");
- goto cleanup;
- }
-
- // Now count the words via the second dictionary
- delete enumer;
- enumer = compact2->openWords(status);
- if (U_FAILURE(status)) {
- errln("Could not open compact trie dictionary 2 enumerator: %s\n", u_errorName(status));
- goto cleanup;
- }
-
- if (wordCount != (testCount = enumer->count(status))) {
- errln("CompactTrieDictionary 2 word count (%d) differs from file word count (%d), with status %s\n",
- testCount, wordCount, u_errorName(status));
- goto cleanup;
- }
-
-cleanup:
- delete compactDict;
- delete mutableDict;
- delete breaks;
- delete[] testFile;
- delete enumer;
- delete mutable2;
- delete cloneEnum;
- delete compact2;
-}
-
-//---------------------------------------------
-// runIndexedTest
-//---------------------------------------------
-
-void RBBITest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* params )
-{
- if (exec) logln("TestSuite RuleBasedBreakIterator: ");
-
- switch (index) {
- case 0: name = "TestBug4153072";
- if(exec) TestBug4153072(); break;
- case 1: name = "TestJapaneseLineBreak";
- if(exec) TestJapaneseLineBreak(); break;
- case 2: name = "TestStatusReturn";
- if(exec) TestStatusReturn(); break;
-
- case 3: name = "TestLineBreakData";
- if(exec) TestLineBreakData(); break;
- case 4: name = "TestEmptyString";
- if(exec) TestEmptyString(); break;
-
- case 5: name = "TestGetAvailableLocales";
- if(exec) TestGetAvailableLocales(); break;
-
- case 6: name = "TestGetDisplayName";
- if(exec) TestGetDisplayName(); break;
-
- case 7: name = "TestEndBehaviour";
- if(exec) TestEndBehaviour(); break;
- case 8: name = "TestMixedThaiLineBreak";
- if(exec) TestMixedThaiLineBreak(); break;
- case 9: name = "TestThaiLineBreak";
- if(exec) TestThaiLineBreak(); break;
- case 10: name = "TestMaiyamok";
- if(exec) TestMaiyamok(); break;
- case 11: name = "TestWordBreaks";
- if(exec) TestWordBreaks(); break;
- case 12: name = "TestWordBoundary";
- if(exec) TestWordBoundary(); break;
- case 13: name = "TestLineBreaks";
- if(exec) TestLineBreaks(); break;
- case 14: name = "TestSentBreaks";
- if(exec) TestSentBreaks(); break;
- case 15: name = "TestExtended";
- if(exec) TestExtended(); break;
- case 16: name = "TestMonkey";
- if(exec) {
- #if !UCONFIG_NO_REGULAR_EXPRESSIONS
- TestMonkey(params);
- #else
- logln("skipping TestMonkey (UCONFIG_NO_REGULAR_EXPRESSIONS)");
- #endif
- }
- break;
- case 17: name = "TestBug3818";
- if(exec) TestBug3818(); break;
- case 18: name = "TestJapaneseWordBreak";
- if(exec) TestJapaneseWordBreak(); break;
- case 19: name = "TestDebug";
- if(exec) TestDebug(); break;
- case 20: name = "TestTrieDict";
- if(exec) TestTrieDict(); break;
-
- default: name = ""; break; //needed to end loop
- }
-}
-
-
//----------------------------------------------------------------------------
//
// generalIteratorTest Given a break iterator and a set of test data,
int32_t lastP = 0x7ffffffe;
int32_t tag;
- logln("Test first and next");
+ logln("Test last and previous");
bi.setText(td.fDataToBreak);
td.clearResults();
RuleBasedBreakIterator* bi = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(Locale::getDefault(), status);
if (U_FAILURE(status))
{
- errln("Failed to create the BreakIterator for default locale in TestEmptyString.\n");
+ errcheckln(status, "Failed to create the BreakIterator for default locale in TestEmptyString. - %s", u_errorName(status));
return;
}
generalIteratorTest(*bi, x);
const Locale* locList = BreakIterator::getAvailableLocales(locCount);
if (locCount == 0)
- errln("getAvailableLocales() returned an empty list!");
+ dataerrln("getAvailableLocales() returned an empty list!");
// Just make sure that it's returning good memory.
int32_t i;
for (i = 0; i < locCount; ++i) {
BreakIterator::getDisplayName(Locale::getUS(), result);
if (Locale::getDefault() == Locale::getUS() && result != "English (United States)")
- errln("BreakIterator::getDisplayName() failed: expected \"English (United States)\", got \""
+ dataerrln("BreakIterator::getDisplayName() failed: expected \"English (United States)\", got \""
+ result);
BreakIterator::getDisplayName(Locale::getFrance(), Locale::getUS(), result);
if (result != "French (France)")
- errln("BreakIterator::getDisplayName() failed: expected \"French (France)\", got \""
+ dataerrln("BreakIterator::getDisplayName() failed: expected \"French (France)\", got \""
+ result);
}
/**
BreakIterator *wb = BreakIterator::createWordInstance(Locale::getDefault(), status);
if (U_FAILURE(status))
{
- errln("Failed to create the BreakIterator for default locale in TestEndBehaviour.\n");
+ errcheckln(status, "Failed to create the BreakIterator for default locale in TestEndBehaviour. - %s", u_errorName(status));
return;
}
wb->setText(testString);
BreakIterator *iter = BreakIterator::createWordInstance(Locale::getDefault(), status);
if (U_FAILURE(status))
{
- errln("Failed to create the BreakIterator for default locale in TestBug4153072\n");
+ errcheckln(status, "Failed to create the BreakIterator for default locale in TestBug4153072 - %s", u_errorName(status));
return;
}
UnicodeString str("...Hello, World!...");
}
-/**
- * Test Japanese Line Break
- * @bug 4095322
- */
-void RBBITest::TestJapaneseLineBreak()
-{
-#if 0
- // Test needs updating some more... Dump it for now.
-
-
- // Change for Unicode TR 14: Punctuation characters with categories Pi and Pf do not count
- // as opening and closing punctuation for line breaking.
- // Also, \u30fc and \u30fe are not counted as hyphens. Remove these chars
- // from these tests. 6-13-2002
- //
+//
+// Test for problem reported by Ashok Matoria on 9 July 2007
+// One.<kSoftHyphen><kSpace>Two.
+//
+// Sentence break at start (0) and then on calling next() it breaks at
+// 'T' of "Two". Now, at this point if I do next() and
+// then previous(), it breaks at <kSOftHyphen> instead of 'T' of "Two".
+//
+void RBBITest::TestBug5775() {
UErrorCode status = U_ZERO_ERROR;
- UnicodeString testString = CharsToUnicodeString("\\u4e00x\\u4e8c");
- UnicodeString precedingChars = CharsToUnicodeString(
- //"([{\\u00ab$\\u00a5\\u00a3\\u00a4\\u2018\\u201a\\u201c\\u201e\\u201b\\u201f");
- "([{$\\u00a5\\u00a3\\u00a4\\u201a\\u201e");
- UnicodeString followingChars = CharsToUnicodeString(
- // ")]}\\u00bb!%,.\\u3001\\u3002\\u3063\\u3083\\u3085\\u3087\\u30c3\\u30e3\\u30e5\\u30e7\\u30fc"
- ")]}!%,.\\u3001\\u3002\\u3063\\u3083\\u3085\\u3087\\u30c3\\u30e3\\u30e5\\u30e7"
- // ":;\\u309b\\u309c\\u3005\\u309d\\u309e\\u30fd\\u30fe\\u2019\\u201d\\u00b0\\u2032\\u2033\\u2034"
- ":;\\u309b\\u309c\\u3005\\u309d\\u309e\\u30fd\\u00b0\\u2032\\u2033\\u2034"
- "\\u2030\\u2031\\u2103\\u2109\\u00a2\\u0300\\u0301\\u0302");
- BreakIterator *iter = BreakIterator::createLineInstance(Locale::getJapan(), status);
-
- int32_t i;
- if (U_FAILURE(status))
- {
- errln("Failed to create the BreakIterator for Japanese locale in TestJapaneseLineBreak.\n");
+ BreakIterator *bi = BreakIterator::createSentenceInstance(Locale::getEnglish(), status);
+ TEST_ASSERT_SUCCESS(status);
+ if (U_FAILURE(status)) {
return;
}
-
- for (i = 0; i < precedingChars.length(); i++) {
- testString.setCharAt(1, precedingChars[i]);
- iter->setText(testString);
- int32_t j = iter->first();
- if (j != 0)
- errln("ja line break failure: failed to start at 0");
- j = iter->next();
- if (j != 1)
- errln("ja line break failure: failed to stop before '" + UCharToUnicodeString(precedingChars[i])
- + "' (" + ((int)(precedingChars[i])) + ")");
- j = iter->next();
- if (j != 3)
- errln("ja line break failure: failed to skip position after '" + UCharToUnicodeString(precedingChars[i])
- + "' (" + ((int)(precedingChars[i])) + ")");
- }
-
- for (i = 0; i < followingChars.length(); i++) {
- testString.setCharAt(1, followingChars[i]);
- iter->setText(testString);
- int j = iter->first();
- if (j != 0)
- errln("ja line break failure: failed to start at 0");
- j = iter->next();
- if (j != 2)
- errln("ja line break failure: failed to skip position before '" + UCharToUnicodeString(followingChars[i])
- + "' (" + ((int)(followingChars[i])) + ")");
- j = iter->next();
- if (j != 3)
- errln("ja line break failure: failed to stop after '" + UCharToUnicodeString(followingChars[i])
- + "' (" + ((int)(followingChars[i])) + ")");
+// Check for status first for better handling of no data errors.
+ TEST_ASSERT(bi != NULL);
+ if (bi == NULL) {
+ return;
}
- delete iter;
-#endif
+
+ UnicodeString s("One.\\u00ad Two.", -1, US_INV);
+ // 01234 56789
+ s = s.unescape();
+ bi->setText(s);
+ int pos = bi->next();
+ TEST_ASSERT(pos == 6);
+ pos = bi->next();
+ TEST_ASSERT(pos == 10);
+ pos = bi->previous();
+ TEST_ASSERT(pos == 6);
+ delete bi;
}
+
//------------------------------------------------------------------------------
//
// RBBITest::Extended Run RBBI Tests from an external test data file
//------------------------------------------------------------------------------
struct TestParams {
- BreakIterator *bi;
- UnicodeString dataToBreak;
- UVector32 *expectedBreaks;
- UVector32 *srcLine;
+ BreakIterator *bi; // Break iterator is set while parsing test source.
+ // Changed out whenever test data changes break type.
+
+ UnicodeString dataToBreak; // Data that is built up while parsing the test.
+ UVector32 *expectedBreaks; // Expected break positions, matches dataToBreak UnicodeString.
+ UVector32 *srcLine; // Positions in source file, indexed same as dataToBreak.
UVector32 *srcCol;
+
+ UText *textToBreak; // UText, could be UTF8 or UTF16.
+ UVector32 *textMap; // Map from UTF-16 dataToBreak offsets to UText offsets.
+ CharString utf8String; // UTF-8 form of text to break.
+
+ TestParams(UErrorCode &status) : dataToBreak() {
+ bi = NULL;
+ expectedBreaks = new UVector32(status);
+ srcLine = new UVector32(status);
+ srcCol = new UVector32(status);
+ textToBreak = NULL;
+ textMap = new UVector32(status);
+ }
+
+ ~TestParams() {
+ delete bi;
+ delete expectedBreaks;
+ delete srcLine;
+ delete srcCol;
+ utext_close(textToBreak);
+ delete textMap;
+ }
+
+ int32_t getSrcLine(int32_t bp);
+ int32_t getExpectedBreak(int32_t bp);
+ int32_t getSrcCol(int32_t bp);
+
+ void setUTF16(UErrorCode &status);
+ void setUTF8(UErrorCode &status);
};
-void RBBITest::executeTest(TestParams *t) {
+// Append a UnicodeString to a CharString with UTF-8 encoding.
+// Substitute any invalid chars.
+// Note: this is used with test data that includes a few unpaired surrogates in the UTF-16 that will be substituted.
+static void CharStringAppend(CharString &dest, const UnicodeString &src, UErrorCode &status) {
+ if (U_FAILURE(status)) {
+ return;
+ }
+ int32_t utf8Length;
+ u_strToUTF8WithSub(NULL, 0, &utf8Length, // Output Buffer, NULL for preflight.
+ src.getBuffer(), src.length(), // UTF-16 data
+ 0xfffd, NULL, // Substitution char, number of subs.
+ &status);
+ if (U_FAILURE(status) && status != U_BUFFER_OVERFLOW_ERROR) {
+ return;
+ }
+ status = U_ZERO_ERROR;
+ int32_t capacity;
+ char *buffer = dest.getAppendBuffer(utf8Length, utf8Length, capacity, status);
+ u_strToUTF8WithSub(buffer, utf8Length, NULL,
+ src.getBuffer(), src.length(),
+ 0xfffd, NULL, &status);
+ dest.append(buffer, utf8Length, status);
+}
+
+
+void TestParams::setUTF16(UErrorCode &status) {
+ textToBreak = utext_openUnicodeString(textToBreak, &dataToBreak, &status);
+ textMap->removeAllElements();
+ for (int32_t i=0; i<dataToBreak.length(); i++) {
+ if (i == dataToBreak.getChar32Start(i)) {
+ textMap->addElement(i, status);
+ } else {
+ textMap->addElement(-1, status);
+ }
+ }
+ textMap->addElement(dataToBreak.length(), status);
+ U_ASSERT(dataToBreak.length() + 1 == textMap->size());
+}
+
+
+void TestParams::setUTF8(UErrorCode &status) {
+ if (U_FAILURE(status)) {
+ return;
+ }
+ utf8String.clear();
+ CharStringAppend(utf8String, dataToBreak, status);
+ textToBreak = utext_openUTF8(textToBreak, utf8String.data(), utf8String.length(), &status);
+ if (U_FAILURE(status)) {
+ return;
+ }
+
+ textMap->removeAllElements();
+ int32_t utf16Index = 0;
+ for (;;) {
+ textMap->addElement(utf16Index, status);
+ UChar32 c32 = utext_current32(textToBreak);
+ if (c32 < 0) {
+ break;
+ }
+ utf16Index += U16_LENGTH(c32);
+ utext_next32(textToBreak);
+ while (textMap->size() < utext_getNativeIndex(textToBreak)) {
+ textMap->addElement(-1, status);
+ }
+ }
+ U_ASSERT(utext_nativeLength(textToBreak) + 1 == textMap->size());
+}
+
+
+int32_t TestParams::getSrcLine(int bp) {
+ if (bp >= textMap->size()) {
+ bp = textMap->size() - 1;
+ }
+ int32_t i = 0;
+ for(; bp >= 0 ; --bp) {
+ // Move to a character boundary if we are not on one already.
+ i = textMap->elementAti(bp);
+ if (i >= 0) {
+ break;
+ }
+ }
+ return srcLine->elementAti(i);
+}
+
+
+int32_t TestParams::getExpectedBreak(int bp) {
+ if (bp >= textMap->size()) {
+ return 0;
+ }
+ int32_t i = textMap->elementAti(bp);
+ int32_t retVal = 0;
+ if (i >= 0) {
+ retVal = expectedBreaks->elementAti(i);
+ }
+ return retVal;
+}
+
+
+int32_t TestParams::getSrcCol(int bp) {
+ if (bp >= textMap->size()) {
+ bp = textMap->size() - 1;
+ }
+ int32_t i = 0;
+ for(; bp >= 0; --bp) {
+ // Move bp to a character boundary if we are not on one already.
+ i = textMap->elementAti(bp);
+ if (i >= 0) {
+ break;
+ }
+ }
+ return srcCol->elementAti(i);
+}
+
+
+void RBBITest::executeTest(TestParams *t, UErrorCode &status) {
int32_t bp;
int32_t prevBP;
int32_t i;
+ TEST_ASSERT_SUCCESS(status);
+ if (U_FAILURE(status)) {
+ return;
+ }
+
if (t->bi == NULL) {
return;
}
- t->bi->setText(t->dataToBreak);
+ t->bi->setText(t->textToBreak, status);
//
// Run the iterator forward
//
if (prevBP == bp) {
// Fail for lack of forward progress.
errln("Forward Iteration, no forward progress. Break Pos=%4d File line,col=%4d,%4d",
- bp, t->srcLine->elementAti(bp), t->srcCol->elementAti(bp));
+ bp, t->getSrcLine(bp), t->getSrcCol(bp));
break;
}
- // Check that there were we didn't miss an expected break between the last one
+ // Check that there we didn't miss an expected break between the last one
// and this one.
for (i=prevBP+1; i<bp; i++) {
- if (t->expectedBreaks->elementAti(i) != 0) {
+ if (t->getExpectedBreak(i) != 0) {
int expected[] = {0, i};
printStringBreaks(t->dataToBreak, expected, 2);
errln("Forward Iteration, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
- i, t->srcLine->elementAti(i), t->srcCol->elementAti(i));
+ i, t->getSrcLine(i), t->getSrcCol(i));
}
}
// Check that the break we did find was expected
- if (t->expectedBreaks->elementAti(bp) == 0) {
+ if (t->getExpectedBreak(bp) == 0) {
int expected[] = {0, bp};
- printStringBreaks(t->dataToBreak, expected, 2);
+ printStringBreaks(t->textToBreak, expected, 2);
errln("Forward Iteration, break found, but not expected. Pos=%4d File line,col= %4d,%4d",
- bp, t->srcLine->elementAti(bp), t->srcCol->elementAti(bp));
+ bp, t->getSrcLine(bp), t->getSrcCol(bp));
} else {
// The break was expected.
// Check that the {nnn} tag value is correct.
- int32_t expectedTagVal = t->expectedBreaks->elementAti(bp);
+ int32_t expectedTagVal = t->getExpectedBreak(bp);
if (expectedTagVal == -1) {
expectedTagVal = 0;
}
- int32_t line = t->srcLine->elementAti(bp);
+ int32_t line = t->getSrcLine(bp);
int32_t rs = ((RuleBasedBreakIterator *)t->bi)->getRuleStatus();
if (rs != expectedTagVal) {
errln("Incorrect status for forward break. Pos=%4d File line,col= %4d,%4d.\n"
" Actual, Expected status = %4d, %4d",
- bp, line, t->srcCol->elementAti(bp), rs, expectedTagVal);
+ bp, line, t->getSrcCol(bp), rs, expectedTagVal);
}
}
-
prevBP = bp;
}
// Verify that there were no missed expected breaks after the last one found
- for (i=prevBP+1; i<t->expectedBreaks->size(); i++) {
- if (t->expectedBreaks->elementAti(i) != 0) {
+ for (i=prevBP+1; i<utext_nativeLength(t->textToBreak); i++) {
+ if (t->getExpectedBreak(i) != 0) {
errln("Forward Iteration, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
- i, t->srcLine->elementAti(i), t->srcCol->elementAti(i));
+ i, t->getSrcLine(i), t->getSrcCol(i));
}
}
//
// Run the iterator backwards, verify that the same breaks are found.
//
- prevBP = t->dataToBreak.length()+2; // start with a phony value for the last break pos seen.
+ prevBP = utext_nativeLength(t->textToBreak)+2; // start with a phony value for the last break pos seen.
for (bp = t->bi->last(); bp != BreakIterator::DONE; bp = t->bi->previous()) {
if (prevBP == bp) {
// Fail for lack of progress.
errln("Reverse Iteration, no progress. Break Pos=%4d File line,col=%4d,%4d",
- bp, t->srcLine->elementAti(bp), t->srcCol->elementAti(bp));
+ bp, t->getSrcLine(bp), t->getSrcCol(bp));
break;
}
- // Check that there were we didn't miss an expected break between the last one
+ // Check that we didn't miss an expected break between the last one
// and this one. (UVector returns zeros for index out of bounds.)
for (i=prevBP-1; i>bp; i--) {
- if (t->expectedBreaks->elementAti(i) != 0) {
- errln("Reverse Itertion, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
- i, t->srcLine->elementAti(i), t->srcCol->elementAti(i));
+ if (t->getExpectedBreak(i) != 0) {
+ errln("Reverse Iteration, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
+ i, t->getSrcLine(i), t->getSrcCol(i));
}
}
// Check that the break we did find was expected
- if (t->expectedBreaks->elementAti(bp) == 0) {
+ if (t->getExpectedBreak(bp) == 0) {
errln("Reverse Itertion, break found, but not expected. Pos=%4d File line,col= %4d,%4d",
- bp, t->srcLine->elementAti(bp), t->srcCol->elementAti(bp));
+ bp, t->getSrcLine(bp), t->getSrcCol(bp));
} else {
// The break was expected.
// Check that the {nnn} tag value is correct.
- int32_t expectedTagVal = t->expectedBreaks->elementAti(bp);
+ int32_t expectedTagVal = t->getExpectedBreak(bp);
if (expectedTagVal == -1) {
expectedTagVal = 0;
}
- int line = t->srcLine->elementAti(bp);
- int32_t rs = ((RuleBasedBreakIterator *)t->bi)->getRuleStatus();
+ int line = t->getSrcLine(bp);
+ int32_t rs = t->bi->getRuleStatus();
if (rs != expectedTagVal) {
errln("Incorrect status for reverse break. Pos=%4d File line,col= %4d,%4d.\n"
" Actual, Expected status = %4d, %4d",
- bp, line, t->srcCol->elementAti(bp), rs, expectedTagVal);
+ bp, line, t->getSrcCol(bp), rs, expectedTagVal);
}
}
// Verify that there were no missed breaks prior to the last one found
for (i=prevBP-1; i>=0; i--) {
- if (t->expectedBreaks->elementAti(i) != 0) {
+ if (t->getExpectedBreak(i) != 0) {
errln("Forward Itertion, break expected, but not found. Pos=%4d File line,col= %4d,%4d",
- i, t->srcLine->elementAti(i), t->srcCol->elementAti(i));
+ i, t->getSrcLine(i), t->getSrcCol(i));
+ }
+ }
+
+ // Check isBoundary()
+ for (i=0; i < utext_nativeLength(t->textToBreak); i++) {
+ UBool boundaryExpected = (t->getExpectedBreak(i) != 0);
+ UBool boundaryFound = t->bi->isBoundary(i);
+ if (boundaryExpected != boundaryFound) {
+ errln("isBoundary(%d) incorrect. File line,col= %4d,%4d\n"
+ " Expected, Actual= %s, %s",
+ i, t->getSrcLine(i), t->getSrcCol(i),
+ boundaryExpected ? "true":"false", boundaryFound? "true" : "false");
+ }
+ }
+
+ // Check following()
+ for (i=0; i < utext_nativeLength(t->textToBreak); i++) {
+ int32_t actualBreak = t->bi->following(i);
+ int32_t expectedBreak = BreakIterator::DONE;
+ for (int32_t j=i+1; j <= utext_nativeLength(t->textToBreak); j++) {
+ if (t->getExpectedBreak(j) != 0) {
+ expectedBreak = j;
+ break;
+ }
+ }
+ if (expectedBreak != actualBreak) {
+ errln("following(%d) incorrect. File line,col= %4d,%4d\n"
+ " Expected, Actual= %d, %d",
+ i, t->getSrcLine(i), t->getSrcCol(i), expectedBreak, actualBreak);
+ }
+ }
+
+ // Check preceding()
+ for (i=utext_nativeLength(t->textToBreak); i>=0; i--) {
+ int32_t actualBreak = t->bi->preceding(i);
+ int32_t expectedBreak = BreakIterator::DONE;
+
+ // For UTF-8 & UTF-16 supplementals, all code units of a character are equivalent.
+ // preceding(trailing byte) will return the index of some preceding code point,
+ // not the lead byte of the current code point, even though that has a smaller index.
+ // Therefore, start looking at the expected break data not at i-1, but at
+ // the start of code point index - 1.
+ utext_setNativeIndex(t->textToBreak, i);
+ int32_t j = utext_getNativeIndex(t->textToBreak) - 1;
+ for (; j >= 0; j--) {
+ if (t->getExpectedBreak(j) != 0) {
+ expectedBreak = j;
+ break;
+ }
+ }
+ if (expectedBreak != actualBreak) {
+ errln("preceding(%d) incorrect. File line,col= %4d,%4d\n"
+ " Expected, Actual= %d, %d",
+ i, t->getSrcLine(i), t->getSrcCol(i), expectedBreak, actualBreak);
}
}
}
Locale locale("");
UnicodeString rules;
- TestParams tp;
- tp.bi = NULL;
- tp.expectedBreaks = new UVector32(status);
- tp.srcLine = new UVector32(status);
- tp.srcCol = new UVector32(status);
+ TestParams tp(status);
- RegexMatcher localeMatcher("<locale *([\\p{L}\\p{Nd}_]*) *>", 0, status);
- TEST_ASSERT_SUCCESS(status);
+ RegexMatcher localeMatcher(UNICODE_STRING_SIMPLE("<locale *([\\p{L}\\p{Nd}_@&=-]*) *>"), 0, status);
+ if (U_FAILURE(status)) {
+ dataerrln("Failure in file %s, line %d, status = \"%s\"", __FILE__, __LINE__, u_errorName(status));
+ }
//
strcat(testFileName, "rbbitst.txt");
int len;
- UChar *testFile = ReadAndConvertFile(testFileName, len, status);
+ UChar *testFile = ReadAndConvertFile(testFileName, len, "UTF-8", status);
if (U_FAILURE(status)) {
return; /* something went wrong, error already output */
}
+ bool skipTest = false; // Skip this test?
//
// Put the test data into a UnicodeString
if (testString.compare(charIdx-1, 6, "<word>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createWordInstance(locale, status);
+ skipTest = false;
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 6, "<char>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createCharacterInstance(locale, status);
+ skipTest = false;
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 6, "<line>") == 0) {
delete tp.bi;
tp.bi = BreakIterator::createLineInstance(locale, status);
+ skipTest = false;
charIdx += 5;
break;
}
if (testString.compare(charIdx-1, 6, "<sent>") == 0) {
delete tp.bi;
- tp.bi = NULL;
- tp.bi = BreakIterator::createSentenceInstance(locale, status);
+ tp.bi = BreakIterator::createSentenceInstance(locale, status);
+ skipTest = false;
charIdx += 5;
break;
}
charIdx += 6;
break;
}
+
// <locale loc_name>
localeMatcher.reset(testString);
if (localeMatcher.lookingAt(charIdx-1, status)) {
char localeName8[100];
localeName.extract(0, localeName.length(), localeName8, sizeof(localeName8), 0);
locale = Locale::createFromName(localeName8);
- charIdx += localeMatcher.group(0, status).length();
+ charIdx += localeMatcher.group(0, status).length() - 1;
TEST_ASSERT_SUCCESS(status);
break;
}
}
errln("line %d: Tag expected in test file.", lineNum);
- goto end_test;
parseState = PARSE_COMMENT;
savedState = PARSE_DATA;
+ goto end_test; // Stop the test.
}
break;
parseState = PARSE_TAG;
charIdx += 6;
- // RUN THE TEST!
- executeTest(&tp);
+ if (!skipTest) {
+ // RUN THE TEST!
+ status = U_ZERO_ERROR;
+ tp.setUTF16(status);
+ executeTest(&tp, status);
+ TEST_ASSERT_SUCCESS(status);
+
+ // Run again, this time with UTF-8 text wrapped in a UText.
+ status = U_ZERO_ERROR;
+ tp.setUTF8(status);
+ TEST_ASSERT_SUCCESS(status);
+ executeTest(&tp, status);
+ }
break;
}
- if (testString.compare(charIdx-1, 3, "\\N{") == 0) {
+ if (testString.compare(charIdx-1, 3, UNICODE_STRING_SIMPLE("\\N{")) == 0) {
// Named character, e.g. \N{COMBINING GRAVE ACCENT}
// Get the code point from the name and insert it into the test data.
// (Damn, no API takes names in Unicode !!!
errln("Syntax Error in test file at line %d, col %d",
lineNum, column);
- goto end_test;
parseState = PARSE_COMMENT;
+ goto end_test; // Stop the test
break;
}
if (U_FAILURE(status)) {
- errln("ICU Error %s while parsing test file at line %d.",
+ dataerrln("ICU Error %s while parsing test file at line %d.",
u_errorName(status), lineNum);
- goto end_test;
status = U_ZERO_ERROR;
+ goto end_test; // Stop the test
}
}
end_test:
- delete tp.bi;
- delete tp.expectedBreaks;
- delete tp.srcLine;
- delete tp.srcCol;
delete [] testFile;
#endif
}
+//-------------------------------------------------------------------------------
+//
+// TestDictRules create a break iterator from source rules that includes a
+// dictionary range. Regression for bug #7130. Source rules
+// do not declare a break iterator type (word, line, sentence, etc.
+// but the dictionary code, without a type, would loop.
+//
+//-------------------------------------------------------------------------------
+void RBBITest::TestDictRules() {
+ const char *rules = "$dictionary = [a-z]; \n"
+ "!!forward; \n"
+ "$dictionary $dictionary; \n"
+ "!!reverse; \n"
+ "$dictionary $dictionary; \n";
+ const char *text = "aa";
+ UErrorCode status = U_ZERO_ERROR;
+ UParseError parseError;
+
+ RuleBasedBreakIterator bi(rules, parseError, status);
+ if (U_SUCCESS(status)) {
+ UnicodeString utext = text;
+ bi.setText(utext);
+ int32_t position;
+ int32_t loops;
+ for (loops = 0; loops<10; loops++) {
+ position = bi.next();
+ if (position == RuleBasedBreakIterator::DONE) {
+ break;
+ }
+ }
+ TEST_ASSERT(loops == 1);
+ } else {
+ dataerrln("Error creating RuleBasedBreakIterator: %s", u_errorName(status));
+ }
+}
+
+
+
//-------------------------------------------------------------------------------
//
// ReadAndConvertFile Read a text data file, convert it to UChars, and
-// return the datain one big UChar * buffer, which the caller must delete.
+// return the data in one big UChar * buffer, which the caller must delete.
+//
+// parameters:
+// fileName: the name of the file, with no directory part. The test data directory
+// is assumed.
+// ulen an out parameter, receives the actual length (in UChars) of the file data.
+// encoding The file encoding. If the file contains a BOM, that will override the encoding
+// specified here. The BOM, if it exists, will be stripped from the returned data.
+// Pass NULL for the system default encoding.
+// status
+// returns:
+// The file data, converted to UChar.
+// The caller must delete this when done with
+// delete [] theBuffer;
//
// TODO: This is a clone of RegexTest::ReadAndConvertFile.
// Move this function to some common place.
//
//--------------------------------------------------------------------------------
-UChar *RBBITest::ReadAndConvertFile(const char *fileName, int &ulen, UErrorCode &status) {
+UChar *RBBITest::ReadAndConvertFile(const char *fileName, int &ulen, const char *encoding, UErrorCode &status) {
UChar *retPtr = NULL;
char *fileBuf = NULL;
UConverter* conv = NULL;
//
f = fopen(fileName, "rb");
if (f == 0) {
- errln("Error opening test data file %s\n", fileName);
+ dataerrln("Error opening test data file %s\n", fileName);
status = U_FILE_ACCESS_ERROR;
return NULL;
}
//
int32_t signatureLength;
const char * fileBufC;
- const char* encoding;
+ const char* bomEncoding;
fileBufC = fileBuf;
- encoding = ucnv_detectUnicodeSignature(
+ bomEncoding = ucnv_detectUnicodeSignature(
fileBuf, fileSize, &signatureLength, &status);
- if(encoding!=NULL ){
+ if(bomEncoding!=NULL ){
fileBufC += signatureLength;
fileSize -= signatureLength;
+ encoding = bomEncoding;
}
//
ucnv_close(conv);
if (U_FAILURE(status)) {
errln("ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status));
- delete retPtr;
+ delete []retPtr;
retPtr = 0;
ulen = 0;
};
}
+
//--------------------------------------------------------------------------------------------
//
-// Exhaustive Tests, using Unicode Data Files.
+// Run tests from each of the boundary test data files distributed by the Unicode Consortium
//
-//--------------------------------------------------------------------------------------------
+//-------------------------------------------------------------------------------------------
+void RBBITest::TestUnicodeFiles() {
+ RuleBasedBreakIterator *bi;
+ UErrorCode status = U_ZERO_ERROR;
-//
-// Token level scanner for the Unicode Line Break Test Data file.
-// Return the next token, as follows:
-// >= 0: a UChar32 character, scanned from hex in the file.
-// -1: a break position, a division sign in the file.
-// -2: end of rule. A new line in the file.
-// -3: end of file. No more rules.
-// -4: Error
-//
-// The scanner
-// strips comments, ('#' to end of line)
-// Recognizes CR, CR/LF and LF as new lines.
-// Skips over spaces and Xs (don't break here) in the data.
-//
-struct ScanState {
- int32_t fPeekChar;
- UBool fPeeked;
- int32_t fLineNum;
- FILE *fFile;
- ScanState() :fPeeked(FALSE), fLineNum(0), fFile(NULL) {};
-};
+ bi = (RuleBasedBreakIterator *)BreakIterator::createCharacterInstance(Locale::getEnglish(), status);
+ TEST_ASSERT_SUCCESS(status);
+ if (U_SUCCESS(status)) {
+ runUnicodeTestData("GraphemeBreakTest.txt", bi);
+ }
+ delete bi;
-// Literal characters that are of interest. In hex to keep EBCDIC based machines happy.
-// The data itself is latin-1 on all platforms.
-static const int32_t chSpace = 0x20;
-static const int32_t chTab = 0x09;
-static const int32_t chCR = 0x0D;
-static const int32_t chLF = 0x0A;
-static const int32_t chHash = 0x23;
-static const int32_t chMult = 0xD7;
-static const int32_t chDivide = 0xF7;
-
-static int32_t nextLBDToken(ScanState *s) {
- int32_t c;
-
- // Read characters from the input file until we get something interesting
- // to return. The file is in latin-1 encoding.
- for (;;) {
- // Get the next character to look at,
- if (s->fPeeked) {
- c = s->fPeekChar;
- s->fPeeked = FALSE;
- } else {
- c = getc(s->fFile);
- }
+ bi = (RuleBasedBreakIterator *)BreakIterator::createWordInstance(Locale::getEnglish(), status);
+ TEST_ASSERT_SUCCESS(status);
+ if (U_SUCCESS(status)) {
+ runUnicodeTestData("WordBreakTest.txt", bi);
+ }
+ delete bi;
- // EOF. Return immediately.
- if (c == EOF) {
- return -3;
- }
+ bi = (RuleBasedBreakIterator *)BreakIterator::createSentenceInstance(Locale::getEnglish(), status);
+ TEST_ASSERT_SUCCESS(status);
+ if (U_SUCCESS(status)) {
+ runUnicodeTestData("SentenceBreakTest.txt", bi);
+ }
+ delete bi;
- // Spaces. Treat the multiply sign as a space - it indicates a no-break position
- // in the data, and the test program doesn't want to see them.
- // Continue the next char loop, looking for something significant.
- if (c == chSpace || c == chTab || c == chMult) {
- continue;
- }
+ bi = (RuleBasedBreakIterator *)BreakIterator::createLineInstance(Locale::getEnglish(), status);
+ TEST_ASSERT_SUCCESS(status);
+ if (U_SUCCESS(status)) {
+ runUnicodeTestData("LineBreakTest.txt", bi);
+ }
+ delete bi;
+}
- // Divide sign. Indicates an expected break position.
- if (c == chDivide) {
- return -1;
- }
- // New Line Handling. Keep track of line number in the file, which in turn
- // requires keeping track of CR/LF as a single new line.
- if (c == chCR) {
- s->fLineNum++;
- s->fPeekChar = getc(s->fFile);
- if (s->fPeekChar != chLF) {s->fPeeked = TRUE;};
- return -2;
- }
- if (c == chLF) {
- s->fLineNum++;
- return -2;
- }
+// Check for test cases from the Unicode test data files that are known to fail
+// and should be skipped because ICU is not yet able to fully implement the spec.
+// See ticket #7270.
- // Comments. Consume everything up to the next new line.
- if (c == chHash) {
- do {
- c = getc(s->fFile);
- } while (!(c == EOF || c == chCR || c == chLF));
- s->fPeekChar = c;
- s->fPeeked = TRUE;
- return nextLBDToken(s);
- }
+UBool RBBITest::testCaseIsKnownIssue(const UnicodeString &testCase, const char *fileName) {
+ static const UChar badTestCases[][4] = { // Line Numbers from Unicode 7.0.0 file.
+ {(UChar)0x200B, (UChar)0x0020, (UChar)0x007D, (UChar)0x0000}, // Line 5198
+ {(UChar)0x200B, (UChar)0x0020, (UChar)0x0029, (UChar)0x0000}, // Line 5202
+ {(UChar)0x200B, (UChar)0x0020, (UChar)0x0021, (UChar)0x0000}, // Line 5214
+ {(UChar)0x200B, (UChar)0x0020, (UChar)0x002c, (UChar)0x0000}, // Line 5246
+ {(UChar)0x200B, (UChar)0x0020, (UChar)0x002f, (UChar)0x0000}, // Line 5298
+ {(UChar)0x200B, (UChar)0x0020, (UChar)0x2060, (UChar)0x0000} // Line 5302
+ };
+ if (strcmp(fileName, "LineBreakTest.txt") != 0) {
+ return FALSE;
+ }
- // Scan a hex character (UChar32) value.
- if (u_digit(c, 16) >= 0) {
- int32_t v = u_digit(c, 16);
- for (;;) {
- c = getc(s->fFile);
- if (u_digit(c, 16) < 0) {break;};
- v <<= 4;
- v += u_digit(c, 16);
- }
- s->fPeekChar = c;
- s->fPeeked = TRUE;
- return v;
+ for (int i=0; i<UPRV_LENGTHOF(badTestCases); i++) {
+ if (testCase == UnicodeString(badTestCases[i])) {
+ return logKnownIssue("7270");
}
-
- // Error. Character was something unexpected.
- return -4;
}
+ return FALSE;
}
+//--------------------------------------------------------------------------------------------
+//
+// Run tests from one of the boundary test data files distributed by the Unicode Consortium
+//
+//-------------------------------------------------------------------------------------------
+void RBBITest::runUnicodeTestData(const char *fileName, RuleBasedBreakIterator *bi) {
+#if !UCONFIG_NO_REGULAR_EXPRESSIONS
+ UErrorCode status = U_ZERO_ERROR;
-void RBBITest::TestLineBreakData() {
-
- UErrorCode status = U_ZERO_ERROR;
- UnicodeString testString;
- UVector expectedBreaks(status);
- ScanState ss;
- int32_t tok;
-
- BreakIterator *bi = BreakIterator::createLineInstance(Locale::getDefault(), status);
- if (U_FAILURE(status)) {
- errln("Failure creating break iterator");
+ //
+ // Open and read the test data file, put it into a UnicodeString.
+ //
+ const char *testDataDirectory = IntlTest::getSourceTestData(status);
+ char testFileName[1000];
+ if (testDataDirectory == NULL || strlen(testDataDirectory) >= sizeof(testFileName)) {
+ dataerrln("Can't open test data. Path too long.");
return;
}
+ strcpy(testFileName, testDataDirectory);
+ strcat(testFileName, fileName);
- const char * lbdfName = "LBTest.txt";
+ logln("Opening data file %s\n", fileName);
- // Open the test data file.
- // TODO: a proper way to handle this data.
- ss.fFile = fopen(lbdfName, "rb");
- if (ss.fFile == NULL) {
- logln("Unable to open Line Break Test Data file. Skipping test.");
- delete bi;
+ int len;
+ UChar *testFile = ReadAndConvertFile(testFileName, len, "UTF-8", status);
+ if (status != U_FILE_ACCESS_ERROR) {
+ TEST_ASSERT_SUCCESS(status);
+ TEST_ASSERT(testFile != NULL);
+ }
+ if (U_FAILURE(status) || testFile == NULL) {
+ return; /* something went wrong, error already output */
+ }
+ UnicodeString testFileAsString(TRUE, testFile, len);
+
+ //
+ // Parse the test data file using a regular expression.
+ // Each kind of token is recognized in its own capture group; what type of item was scanned
+ // is identified by which group had a match.
+ //
+ // Caputure Group # 1 2 3 4 5
+ // Parses this item: divide x hex digits comment \n unrecognized \n
+ //
+ UnicodeString tokenExpr("[ \t]*(?:(\\u00F7)|(\\u00D7)|([0-9a-fA-F]+)|((?:#.*?)?$.)|(.*?$.))", -1, US_INV);
+ RegexMatcher tokenMatcher(tokenExpr, testFileAsString, UREGEX_MULTILINE | UREGEX_DOTALL, status);
+ UnicodeString testString;
+ UVector32 breakPositions(status);
+ int lineNumber = 1;
+ TEST_ASSERT_SUCCESS(status);
+ if (U_FAILURE(status)) {
return;
}
- // Loop once per line from the test data file.
- for (;;) {
- // Zero out test data from previous line.
- testString.truncate(0);
- expectedBreaks.removeAllElements();
-
- // Read one test's (line's) worth of data from the file.
- // Loop once per token on the input file line.
- for(;;) {
- tok = nextLBDToken(&ss);
-
- // If we scanned a character number in the file.
- // save it in the test data array.
- if (tok >= 0) {
- testString.append((UChar32)tok);
- continue;
+ //
+ // Scan through each test case, building up the string to be broken in testString,
+ // and the positions that should be boundaries in the breakPositions vector.
+ //
+ int spin = 0;
+ while (tokenMatcher.find()) {
+ if(tokenMatcher.hitEnd()) {
+ /* Shouldnt Happen(TM). This means we didn't find the symbols we were looking for.
+ This occurred when the text file was corrupt (wasn't marked as UTF-8)
+ and caused an infinite loop here on EBCDIC systems!
+ */
+ fprintf(stderr,"FAIL: hit end of file %s for the %8dth time- corrupt data file?\r", fileName, ++spin);
+ // return;
+ }
+ if (tokenMatcher.start(1, status) >= 0) {
+ // Scanned a divide sign, indicating a break position in the test data.
+ if (testString.length()>0) {
+ breakPositions.addElement(testString.length(), status);
}
-
- // If we scanned a break position in the data, record it.
- if (tok == -1) {
- expectedBreaks.addElement(testString.length(), status);
- continue;
+ }
+ else if (tokenMatcher.start(2, status) >= 0) {
+ // Scanned an 'x', meaning no break at this position in the test data
+ // Nothing to be done here.
+ }
+ else if (tokenMatcher.start(3, status) >= 0) {
+ // Scanned Hex digits. Convert them to binary, append to the character data string.
+ const UnicodeString &hexNumber = tokenMatcher.group(3, status);
+ int length = hexNumber.length();
+ if (length<=8) {
+ char buf[10];
+ hexNumber.extract (0, length, buf, sizeof(buf), US_INV);
+ UChar32 c = (UChar32)strtol(buf, NULL, 16);
+ if (c<=0x10ffff) {
+ testString.append(c);
+ } else {
+ errln("Error: Unicode Character value out of range. \'%s\', line %d.\n",
+ fileName, lineNumber);
+ }
+ } else {
+ errln("Syntax Error: Hex Unicode Character value must have no more than 8 digits at \'%s\', line %d.\n",
+ fileName, lineNumber);
+ }
+ }
+ else if (tokenMatcher.start(4, status) >= 0) {
+ // Scanned to end of a line, possibly skipping over a comment in the process.
+ // If the line from the file contained test data, run the test now.
+ if (testString.length() > 0 && !testCaseIsKnownIssue(testString, fileName)) {
+ checkUnicodeTestCase(fileName, lineNumber, testString, &breakPositions, bi);
}
- // If we scanned a new line, or EOF
- // drop out of scan loop and run the test case.
- if (tok == -2 || tok == -3) {break;};
+ // Clear out this test case.
+ // The string and breakPositions vector will be refilled as the next
+ // test case is parsed.
+ testString.remove();
+ breakPositions.removeAllElements();
+ lineNumber++;
+ } else {
+ // Scanner catchall. Something unrecognized appeared on the line.
+ char token[16];
+ UnicodeString uToken = tokenMatcher.group(0, status);
+ uToken.extract(0, uToken.length(), token, (uint32_t)sizeof(token));
+ token[sizeof(token)-1] = 0;
+ errln("Syntax error in test data file \'%s\', line %d. Scanning \"%s\"\n", fileName, lineNumber, token);
+
+ // Clean up, in preparation for continuing with the next line.
+ testString.remove();
+ breakPositions.removeAllElements();
+ lineNumber++;
+ }
+ TEST_ASSERT_SUCCESS(status);
+ if (U_FAILURE(status)) {
+ break;
+ }
+ }
- // None of above. Error.
- errln("Failure: Unrecognized data format, test file line %d", ss.fLineNum);
+ delete [] testFile;
+ #endif // !UCONFIG_NO_REGULAR_EXPRESSIONS
+}
+
+//--------------------------------------------------------------------------------------------
+//
+// checkUnicodeTestCase() Run one test case from one of the Unicode Consortium
+// test data files. Do only a simple, forward-only check -
+// this test is mostly to check that ICU and the Unicode
+// data agree with each other.
+//
+//--------------------------------------------------------------------------------------------
+void RBBITest::checkUnicodeTestCase(const char *testFileName, int lineNumber,
+ const UnicodeString &testString, // Text data to be broken
+ UVector32 *breakPositions, // Positions where breaks should be found.
+ RuleBasedBreakIterator *bi) {
+ int32_t pos; // Break Position in the test string
+ int32_t expectedI = 0; // Index of expected break position in the vector of expected results.
+ int32_t expectedPos; // Expected break position (index into test string)
+
+ bi->setText(testString);
+ pos = bi->first();
+ pos = bi->next();
+
+ while (pos != BreakIterator::DONE) {
+ if (expectedI >= breakPositions->size()) {
+ errln("Test file \"%s\", line %d, unexpected break found at position %d",
+ testFileName, lineNumber, pos);
break;
}
-
- // If this line from the test data file actually contained test data,
- // run the test.
- if (testString.length() > 0) {
- int32_t pos; // Break Position in the test string
- int32_t expectedI = 0; // Index of expected break position in vector of same.
- int32_t expectedPos; // Expected break position (index into test string)
-
- bi->setText(testString);
- pos = bi->first();
- pos = bi->next();
-
- for (; pos != BreakIterator::DONE; ) {
- expectedPos = expectedBreaks.elementAti(expectedI);
- if (pos < expectedPos) {
- errln("Failure: Test file line %d, unexpected break found at position %d",
- ss.fLineNum, pos);
- break;
- }
- if (pos > expectedPos) {
- errln("Failure: Test file line %d, failed to find break at position %d",
- ss.fLineNum, expectedPos);
- break;
- }
- pos = bi->next();
- expectedI++;
- }
+ expectedPos = breakPositions->elementAti(expectedI);
+ if (pos < expectedPos) {
+ errln("Test file \"%s\", line %d, unexpected break found at position %d",
+ testFileName, lineNumber, pos);
+ break;
}
-
- // If we've hit EOF on the input file, we're done.
- if (tok == -3) {
+ if (pos > expectedPos) {
+ errln("Test file \"%s\", line %d, failed to find expected break at position %d",
+ testFileName, lineNumber, expectedPos);
break;
}
+ pos = bi->next();
+ expectedI++;
+ }
+ if (pos==BreakIterator::DONE && expectedI<breakPositions->size()) {
+ errln("Test file \"%s\", line %d, failed to find expected break at position %d",
+ testFileName, lineNumber, breakPositions->elementAti(expectedI));
}
+}
- fclose(ss.fFile);
- delete bi;
-}
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
-
//---------------------------------------------------------------------------------------
//
// classs RBBIMonkeyKind
UnicodeSet *fCRLFSet;
UnicodeSet *fControlSet;
UnicodeSet *fExtendSet;
+ UnicodeSet *fRegionalIndicatorSet;
+ UnicodeSet *fPrependSet;
+ UnicodeSet *fSpacingSet;
+ UnicodeSet *fLSet;
+ UnicodeSet *fVSet;
+ UnicodeSet *fTSet;
+ UnicodeSet *fLVSet;
+ UnicodeSet *fLVTSet;
UnicodeSet *fHangulSet;
UnicodeSet *fAnySet;
+ UnicodeSet *fEmojiModifierSet;
+ UnicodeSet *fEmojiBaseSet;
+ UnicodeSet *fZWJSet;
+ UnicodeSet *fGAZSet;
- RegexMatcher *fMatcher;
const UnicodeString *fText;
};
UErrorCode status = U_ZERO_ERROR;
fText = NULL;
- fMatcher = new RegexMatcher("\\X", 0, status); // Pattern to match a grampheme cluster
- fCRLFSet = new UnicodeSet("[\\r\\n]", status);
- fControlSet = new UnicodeSet("[[\\p{Zl}\\p{Zp}\\p{Cc}\\p{Cf}]-[\\n]-[\\r]-\\p{Grapheme_Extend}]", status);
- fExtendSet = new UnicodeSet("[\\p{Grapheme_Extend}]", status);
- fHangulSet = new UnicodeSet(
- "[\\p{Hangul_Syllable_Type=L}\\p{Hangul_Syllable_Type=L}\\p{Hangul_Syllable_Type=T}"
- "\\p{Hangul_Syllable_Type=LV}\\p{Hangul_Syllable_Type=LVT}]", status);
- fAnySet = new UnicodeSet("[\\u0000-\\U0010ffff]", status);
+ fCRLFSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\r\\n]"), status);
+ fControlSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[[\\p{Grapheme_Cluster_Break = Control}]-[:Block=Tags:]]"), status);
+ fExtendSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[[\\p{Grapheme_Cluster_Break = Extend}][:Block=Tags:]]"), status);
+ fRegionalIndicatorSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = Regional_Indicator}]"), status);
+ fPrependSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = Prepend}]"), status);
+ fSpacingSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = SpacingMark}]"), status);
+ fLSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = L}]"), status);
+ fVSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = V}]"), status);
+ fTSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = T}]"), status);
+ fLVSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = LV}]"), status);
+ fLVTSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Grapheme_Cluster_Break = LVT}]"), status);
+ fHangulSet = new UnicodeSet();
+ fHangulSet->addAll(*fLSet);
+ fHangulSet->addAll(*fVSet);
+ fHangulSet->addAll(*fTSet);
+ fHangulSet->addAll(*fLVSet);
+ fHangulSet->addAll(*fLVTSet);
+ fAnySet = new UnicodeSet(0, 0x10ffff);
+
+
+
+ fEmojiBaseSet = new UnicodeSet(UnicodeString(
+ "[\\u261D\\u26F9\\u270A-\\u270D\\U0001F385\\U0001F3C2-\\U0001F3C4\\U0001F3C7\\U0001F3CA-\\U0001F3CC"
+ "\\U0001F442-\\U0001F443\\U0001F446-\\U0001F450\\U0001F466-\\U0001F478\\U0001F47C"
+ "\\U0001F481-\\U0001F483\\U0001F485-\\U0001F487\\U0001F4AA\\U0001F574-\\U0001F575\\U0001F57A\\U0001F590\\U0001F595-\\U0001F596"
+ "\\U0001F645-\\U0001F647\\U0001F64B-\\U0001F64F\\U0001F6A3\\U0001F6B4-\\U0001F6B6\\U0001F6C0\\U0001F6CC"
+ "\\U0001F918-\\U0001F91E\\U0001F926\\U0001F930\\U0001F933-\\U0001F939\\U0001F93C-\\U0001F93E]"), status);
+
+ fEmojiModifierSet = new UnicodeSet(0x0001F3FB, 0x0001F3FF);
+ fZWJSet = new UnicodeSet(0x200D, 0x200D);
+ fGAZSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\u2640\\u2642\\u2695-\\u2696\\u2708\\u2764"
+ "\\U0001F308\\U0001F33E\\U0001F373\\U0001F393\\U0001F3A4\\U0001F3A8\\U0001F3EB\\U0001F3ED"
+ "\\U0001F466-\\U0001F469\\U0001F48B\\U0001F4BB-\\U0001F4BC\\U0001F527\\U0001F52C\\U0001F5E8"
+ "\\U0001F680\\U0001F692]"), status);
fSets = new UVector(status);
fSets->addElement(fCRLFSet, status);
fSets->addElement(fControlSet, status);
fSets->addElement(fExtendSet, status);
+ fSets->addElement(fRegionalIndicatorSet, status);
+ if (!fPrependSet->isEmpty()) {
+ fSets->addElement(fPrependSet, status);
+ }
+ fSets->addElement(fSpacingSet, status);
fSets->addElement(fHangulSet, status);
fSets->addElement(fAnySet, status);
+ fSets->addElement(fEmojiBaseSet, status);
+ fSets->addElement(fEmojiModifierSet, status);
+ fSets->addElement(fZWJSet, status);
+ fSets->addElement(fGAZSet, status);
if (U_FAILURE(status)) {
deferredStatus = status;
}
void RBBICharMonkey::setText(const UnicodeString &s) {
fText = &s;
- fMatcher->reset(s);
}
-int32_t RBBICharMonkey::next(int32_t i) {
- UErrorCode status = U_ZERO_ERROR;
- int32_t retVal = -1;
- if (fMatcher->find(i, status)) {
- retVal = fMatcher->end(status);
+int32_t RBBICharMonkey::next(int32_t prevPos) {
+ int p0, p1, p2, p3; // Indices of the significant code points around the
+ // break position being tested. The candidate break
+ // location is before p2.
+
+ int breakPos = -1;
+
+ UChar32 c0, c1, c2, c3; // The code points at p0, p1, p2 & p3.
+ UChar32 cBase; // for (X Extend*) patterns, the X character.
+
+ if (U_FAILURE(deferredStatus)) {
+ return -1;
}
- if (U_FAILURE(status)){
- retVal = -1;
+
+ // Previous break at end of string. return DONE.
+ if (prevPos >= fText->length()) {
+ return -1;
}
- return retVal;
+ p0 = p1 = p2 = p3 = prevPos;
+ c3 = fText->char32At(prevPos);
+ c0 = c1 = c2 = cBase = 0;
+ (void)p0; // suppress set but not used warning.
+ (void)c0;
+
+ // Loop runs once per "significant" character position in the input text.
+ for (;;) {
+ // Move all of the positions forward in the input string.
+ p0 = p1; c0 = c1;
+ p1 = p2; c1 = c2;
+ p2 = p3; c2 = c3;
+
+ // Advancd p3 by one codepoint
+ p3 = fText->moveIndex32(p3, 1);
+ c3 = fText->char32At(p3);
+
+ if (p1 == p2) {
+ // Still warming up the loop. (won't work with zero length strings, but we don't care)
+ continue;
+ }
+ if (p2 == fText->length()) {
+ // Reached end of string. Always a break position.
+ break;
+ }
+
+ // Rule GB3 CR x LF
+ // No Extend or Format characters may appear between the CR and LF,
+ // which requires the additional check for p2 immediately following p1.
+ //
+ if (c1==0x0D && c2==0x0A && p1==(p2-1)) {
+ continue;
+ }
+
+ // Rule (GB4). ( Control | CR | LF ) <break>
+ if (fControlSet->contains(c1) ||
+ c1 == 0x0D ||
+ c1 == 0x0A) {
+ break;
+ }
+
+ // Rule (GB5) <break> ( Control | CR | LF )
+ //
+ if (fControlSet->contains(c2) ||
+ c2 == 0x0D ||
+ c2 == 0x0A) {
+ break;
+ }
+
+
+ // Rule (GB6) L x ( L | V | LV | LVT )
+ if (fLSet->contains(c1) &&
+ (fLSet->contains(c2) ||
+ fVSet->contains(c2) ||
+ fLVSet->contains(c2) ||
+ fLVTSet->contains(c2))) {
+ continue;
+ }
+
+ // Rule (GB7) ( LV | V ) x ( V | T )
+ if ((fLVSet->contains(c1) || fVSet->contains(c1)) &&
+ (fVSet->contains(c2) || fTSet->contains(c2))) {
+ continue;
+ }
+
+ // Rule (GB8) ( LVT | T) x T
+ if ((fLVTSet->contains(c1) || fTSet->contains(c1)) &&
+ fTSet->contains(c2)) {
+ continue;
+ }
+
+ // Rule (GB9) x (Extend | ZWJ)
+ if (fExtendSet->contains(c2) || fZWJSet->contains(c2)) {
+ if (!fExtendSet->contains(c1)) {
+ cBase = c1;
+ }
+ continue;
+ }
+
+ // Rule (GB9a) x SpacingMark
+ if (fSpacingSet->contains(c2)) {
+ continue;
+ }
+
+ // Rule (GB9b) Prepend x
+ if (fPrependSet->contains(c1)) {
+ continue;
+ }
+
+ // Rule (GB10) ($E_Base | $GAZ) $Extend* $E_Modifier;
+ if ((fEmojiBaseSet->contains(c1) || fGAZSet->contains(c1)) && fEmojiModifierSet->contains(c2)) {
+ continue;
+ }
+ if ((fEmojiBaseSet->contains(cBase) || fGAZSet->contains(cBase)) &&
+ fExtendSet->contains(c1) && fEmojiModifierSet->contains(c2)) {
+ continue;
+ }
+
+ // Rule (GB11) ZWJ x Glue_After_Zwj
+ if (fZWJSet->contains(c1) && fGAZSet->contains(c2)) {
+ continue;
+ }
+
+ // Rule (GB12-13) Regional_Indicator x Regional_Indicator
+ // Note: The first if condition is a little tricky. We only need to force
+ // a break if there are three or more contiguous RIs. If there are
+ // only two, a break following will occur via other rules, and will include
+ // any trailing extend characters, which is needed behavior.
+ if (fRegionalIndicatorSet->contains(c0) && fRegionalIndicatorSet->contains(c1)
+ && fRegionalIndicatorSet->contains(c2)) {
+ break;
+ }
+ if (fRegionalIndicatorSet->contains(c1) && fRegionalIndicatorSet->contains(c2)) {
+ continue;
+ }
+
+ // Rule (GB999) Any <break> Any
+ break;
+ }
+
+ breakPos = p2;
+ return breakPos;
}
+
UVector *RBBICharMonkey::charClasses() {
return fSets;
}
delete fCRLFSet;
delete fControlSet;
delete fExtendSet;
+ delete fRegionalIndicatorSet;
+ delete fPrependSet;
+ delete fSpacingSet;
+ delete fLSet;
+ delete fVSet;
+ delete fTSet;
+ delete fLVSet;
+ delete fLVTSet;
delete fHangulSet;
delete fAnySet;
-
- delete fMatcher;
+ delete fEmojiBaseSet;
+ delete fEmojiModifierSet;
+ delete fZWJSet;
+ delete fGAZSet;
}
//------------------------------------------------------------------------------------------
private:
UVector *fSets;
+ UnicodeSet *fCRSet;
+ UnicodeSet *fLFSet;
+ UnicodeSet *fNewlineSet;
+ UnicodeSet *fRegionalIndicatorSet;
UnicodeSet *fKatakanaSet;
+ UnicodeSet *fHebrew_LetterSet;
UnicodeSet *fALetterSet;
+ // TODO(jungshik): Do we still need this change?
+ // UnicodeSet *fALetterSet; // matches ALetterPlus in word.txt
+ UnicodeSet *fSingle_QuoteSet;
+ UnicodeSet *fDouble_QuoteSet;
+ UnicodeSet *fMidNumLetSet;
UnicodeSet *fMidLetterSet;
UnicodeSet *fMidNumSet;
UnicodeSet *fNumericSet;
UnicodeSet *fOtherSet;
UnicodeSet *fExtendSet;
UnicodeSet *fExtendNumLetSet;
-
- RegexMatcher *fMatcher;
+ UnicodeSet *fDictionaryCjkSet;
+ UnicodeSet *fEBaseSet;
+ UnicodeSet *fEModifierSet;
+ UnicodeSet *fZWSSet;
+ UnicodeSet *fGAZSet;
const UnicodeString *fText;
};
-RBBIWordMonkey::RBBIWordMonkey()
+RBBIWordMonkey::RBBIWordMonkey()
{
UErrorCode status = U_ZERO_ERROR;
-
fSets = new UVector(status);
+ fCRSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = CR}]"), status);
+ fLFSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = LF}]"), status);
+ fNewlineSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Newline}]"), status);
+ fDictionaryCjkSet= new UnicodeSet("[[\\uac00-\\ud7a3][:Han:][:Hiragana:][:Katakana:]]", status);
+ // Exclude Hangul syllables from ALetterSet during testing.
+ // Leave CJK dictionary characters out from the monkey tests!
+#if 0
fALetterSet = new UnicodeSet("[\\p{Word_Break = ALetter}"
- "[\\p{Line_Break = Complex_Context}"
- "-\\p{Grapheme_Cluster_Break = Extend}"
- "-\\p{Grapheme_Cluster_Break = Control}]]", status);
- //fALetterSet = new UnicodeSet("[\\p{Word_Break = ALetter}]", status);
- fKatakanaSet = new UnicodeSet("[\\p{Word_Break = Katakana}-[\\uff9e\\uff9f]]", status);
- fMidLetterSet = new UnicodeSet("[\\p{Word_Break = MidLetter}]", status);
- fMidNumSet = new UnicodeSet("[\\p{Word_Break = MidNum}]", status);
- fNumericSet = new UnicodeSet("[\\p{Word_Break = Numeric}]", status);
- fFormatSet = new UnicodeSet("[\\p{Word_Break = Format}]", status);
- fExtendNumLetSet = new UnicodeSet("[\\p{Word_Break = ExtendNumLet}]", status);
- //fExtendSet = new UnicodeSet("[\\p{Word_Break = Extend}]", status);
- fExtendSet = new UnicodeSet("[\\p{Grapheme_Cluster_Break = Extend}\\uff9e\\uff9f]", status);
-
+ "[\\p{Line_Break = Complex_Context}"
+ "-\\p{Grapheme_Cluster_Break = Extend}"
+ "-\\p{Grapheme_Cluster_Break = Control}"
+ "]]",
+ status);
+#endif
+ fRegionalIndicatorSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Regional_Indicator}]"), status);
+ fKatakanaSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Katakana}]"), status);
+ fHebrew_LetterSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Hebrew_Letter}]"), status);
+ fALetterSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = ALetter}]"), status);
+ fALetterSet->removeAll(*fDictionaryCjkSet);
+ fSingle_QuoteSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Single_Quote}]"), status);
+ fDouble_QuoteSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Double_Quote}]"), status);
+ fMidNumLetSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = MidNumLet}]"), status);
+ fMidLetterSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = MidLetter} - [\\:]]"), status);
+ fMidNumSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = MidNum}]"), status);
+ // TODO: this set used to contain [\\uff10-\\uff19] (fullwidth digits), but this breaks the test
+ // we should figure out why
+ fNumericSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Numeric}]"), status);
+ fFormatSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Format}]"), status);
+ fExtendNumLetSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = ExtendNumLet}]"), status);
+ fExtendSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Extend}]"), status);
+
+ fEBaseSet = new UnicodeSet(UnicodeString(
+ "[\\u261D\\u26F9\\u270A-\\u270D\\U0001F385\\U0001F3C2-\\U0001F3C4\\U0001F3C7\\U0001F3CA-\\U0001F3CC"
+ "\\U0001F442-\\U0001F443\\U0001F446-\\U0001F450\\U0001F466-\\U0001F478\\U0001F47C"
+ "\\U0001F481-\\U0001F483\\U0001F485-\\U0001F487\\U0001F4AA\\U0001F574-\\U0001F575\\U0001F57A\\U0001F590\\U0001F595-\\U0001F596"
+ "\\U0001F645-\\U0001F647\\U0001F64B-\\U0001F64F\\U0001F6A3\\U0001F6B4-\\U0001F6B6\\U0001F6C0\\U0001F6CC"
+ "\\U0001F918-\\U0001F91E\\U0001F926\\U0001F930\\U0001F933-\\U0001F939\\U0001F93C-\\U0001F93E]"), status);
+
+ fEModifierSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\U0001F3FB-\\U0001F3FF]"), status);
+ fZWSSet = new UnicodeSet((UChar32)0x200D, (UChar32)0x200D);;
+ fGAZSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\u2640\\u2642\\u2695-\\u2696\\u2708\\u2764"
+ "\\U0001F308\\U0001F33E\\U0001F373\\U0001F393\\U0001F3A4\\U0001F3A8\\U0001F3EB\\U0001F3ED"
+ "\\U0001F466-\\U0001F469\\U0001F48B\\U0001F4BB-\\U0001F4BC\\U0001F527\\U0001F52C\\U0001F5E8"
+ "\\U0001F680\\U0001F692]"), status);
+ fExtendSet->removeAll(*fZWSSet);
+
+
fOtherSet = new UnicodeSet();
if(U_FAILURE(status)) {
deferredStatus = status;
}
fOtherSet->complement();
+ fOtherSet->removeAll(*fCRSet);
+ fOtherSet->removeAll(*fLFSet);
+ fOtherSet->removeAll(*fNewlineSet);
fOtherSet->removeAll(*fKatakanaSet);
+ fOtherSet->removeAll(*fHebrew_LetterSet);
fOtherSet->removeAll(*fALetterSet);
+ fOtherSet->removeAll(*fSingle_QuoteSet);
+ fOtherSet->removeAll(*fDouble_QuoteSet);
fOtherSet->removeAll(*fMidLetterSet);
fOtherSet->removeAll(*fMidNumSet);
fOtherSet->removeAll(*fNumericSet);
fOtherSet->removeAll(*fExtendNumLetSet);
fOtherSet->removeAll(*fFormatSet);
fOtherSet->removeAll(*fExtendSet);
-
- fSets->addElement(fALetterSet, status);
- fSets->addElement(fKatakanaSet, status);
- fSets->addElement(fMidLetterSet, status);
- fSets->addElement(fMidNumSet, status);
- fSets->addElement(fNumericSet, status);
- fSets->addElement(fFormatSet, status);
- fSets->addElement(fExtendSet, status);
- fSets->addElement(fOtherSet, status);
- fSets->addElement(fExtendNumLetSet, status);
-
+ fOtherSet->removeAll(*fRegionalIndicatorSet);
+ fOtherSet->removeAll(*fEBaseSet);
+ fOtherSet->removeAll(*fEModifierSet);
+ fOtherSet->removeAll(*fZWSSet);
+ fOtherSet->removeAll(*fGAZSet);
+
+ // Inhibit dictionary characters from being tested at all.
+ fOtherSet->removeAll(*fDictionaryCjkSet);
+ fOtherSet->removeAll(UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{LineBreak = Complex_Context}]"), status));
+
+ fSets->addElement(fCRSet, status);
+ fSets->addElement(fLFSet, status);
+ fSets->addElement(fNewlineSet, status);
+ fSets->addElement(fRegionalIndicatorSet, status);
+ fSets->addElement(fHebrew_LetterSet, status);
+ fSets->addElement(fALetterSet, status);
+ fSets->addElement(fSingle_QuoteSet, status);
+ fSets->addElement(fDouble_QuoteSet, status);
+ //fSets->addElement(fKatakanaSet, status); //TODO: work out how to test katakana
+ fSets->addElement(fMidLetterSet, status);
+ fSets->addElement(fMidNumLetSet, status);
+ fSets->addElement(fMidNumSet, status);
+ fSets->addElement(fNumericSet, status);
+ fSets->addElement(fFormatSet, status);
+ fSets->addElement(fExtendSet, status);
+ fSets->addElement(fOtherSet, status);
+ fSets->addElement(fExtendNumLetSet, status);
+
+ fSets->addElement(fEBaseSet, status);
+ fSets->addElement(fEModifierSet, status);
+ fSets->addElement(fZWSSet, status);
+ fSets->addElement(fGAZSet, status);
if (U_FAILURE(status)) {
deferredStatus = status;
UChar32 c0, c1, c2, c3; // The code points at p0, p1, p2 & p3.
+ if (U_FAILURE(deferredStatus)) {
+ return -1;
+ }
+
// Prev break at end of string. return DONE.
if (prevPos >= fText->length()) {
return -1;
p0 = p1 = p2 = p3 = prevPos;
c3 = fText->char32At(prevPos);
c0 = c1 = c2 = 0;
+ (void)p0; // Suppress set but not used warning.
// Loop runs once per "significant" character position in the input text.
for (;;) {
p2 = p3; c2 = c3;
// Advancd p3 by X(Extend | Format)* Rule 4
+ // But do not advance over Extend & Format following a new line. (Unicode 5.1 change)
do {
p3 = fText->moveIndex32(p3, 1);
c3 = fText->char32At(p3);
+ if (fCRSet->contains(c2) || fLFSet->contains(c2) || fNewlineSet->contains(c2)) {
+ break;
+ };
}
- while (fFormatSet->contains(c3) || fExtendSet->contains(c3));
+ while (fFormatSet->contains(c3) || fExtendSet->contains(c3) || fZWSSet->contains(c3));
if (p1 == p2) {
// Reached end of string. Always a break position.
break;
}
-
+
// Rule (3) CR x LF
// No Extend or Format characters may appear between the CR and LF,
// which requires the additional check for p2 immediately following p1.
//
- if (c1==0x0D && c2==0x0A && p1==(p2-1)) {
+ if (c1==0x0D && c2==0x0A) {
+ continue;
+ }
+
+ // Rule (3a) Break before and after newlines (including CR and LF)
+ //
+ if (fCRSet->contains(c1) || fLFSet->contains(c1) || fNewlineSet->contains(c1)) {
+ break;
+ };
+ if (fCRSet->contains(c2) || fLFSet->contains(c2) || fNewlineSet->contains(c2)) {
+ break;
+ };
+
+ // Rule (3c) ZWJ x GAZ (Glue after ZWJ).
+ // Not ignoring extend chars, so peek into input text to
+ // get the potential ZWJ, the character immediately preceding c2.
+ // Sloppy UChar32 indexing: p2-1 may reference trail half
+ // but char32At will get the full code point.
+ if (fZWSSet->contains(fText->char32At(p2-1)) && fGAZSet->contains(c2)) {
continue;
}
- // Rule (5). ALetter x ALetter
- if (fALetterSet->contains(c1) &&
- fALetterSet->contains(c2)) {
+ // Rule (5). (ALetter | Hebrew_Letter) x (ALetter | Hebrew_Letter)
+ if ((fALetterSet->contains(c1) || fHebrew_LetterSet->contains(c1)) &&
+ (fALetterSet->contains(c2) || fHebrew_LetterSet->contains(c2))) {
continue;
}
- // Rule (6) ALetter x (MidLetter | MidNumLet) ALetter
+ // Rule (6) (ALetter | Hebrew_Letter) x (MidLetter | MidNumLet | Single_Quote) (ALetter | Hebrew_Letter)
//
- // Also incorporates rule 7 by skipping pos ahead to position of the
- // terminating ALetter.
- if ( fALetterSet->contains(c1) &&
- fMidLetterSet->contains(c2) &&
- fALetterSet->contains(c3)) {
+ if ( (fALetterSet->contains(c1) || fHebrew_LetterSet->contains(c1)) &&
+ (fMidLetterSet->contains(c2) || fMidNumLetSet->contains(c2) || fSingle_QuoteSet->contains(c2)) &&
+ (fALetterSet->contains(c3) || fHebrew_LetterSet->contains(c3))) {
+ continue;
+ }
+
+ // Rule (7) (ALetter | Hebrew_Letter) (MidLetter | MidNumLet | Single_Quote) x (ALetter | Hebrew_Letter)
+ if ((fALetterSet->contains(c0) || fHebrew_LetterSet->contains(c0)) &&
+ (fMidLetterSet->contains(c1) || fMidNumLetSet->contains(c1) || fSingle_QuoteSet->contains(c1)) &&
+ (fALetterSet->contains(c2) || fHebrew_LetterSet->contains(c2))) {
continue;
}
+ // Rule (7a) Hebrew_Letter x Single_Quote
+ if (fHebrew_LetterSet->contains(c1) && fSingle_QuoteSet->contains(c2)) {
+ continue;
+ }
+
+ // Rule (7b) Hebrew_Letter x Double_Quote Hebrew_Letter
+ if (fHebrew_LetterSet->contains(c1) && fDouble_QuoteSet->contains(c2) && fHebrew_LetterSet->contains(c3)) {
+ continue;
+ }
- // Rule (7) ALetter (MidLetter | MidNumLet) x ALetter
- if (fALetterSet->contains(c0) &&
- (fMidLetterSet->contains(c1) ) &&
- fALetterSet->contains(c2)) {
+ // Rule (7c) Hebrew_Letter Double_Quote x Hebrew_Letter
+ if (fHebrew_LetterSet->contains(c0) && fDouble_QuoteSet->contains(c1) && fHebrew_LetterSet->contains(c2)) {
continue;
}
continue;
}
- // Rule (9) ALetter x Numeric
- if (fALetterSet->contains(c1) &&
+ // Rule (9) (ALetter | Hebrew_Letter) x Numeric
+ if ((fALetterSet->contains(c1) || fHebrew_LetterSet->contains(c1)) &&
fNumericSet->contains(c2)) {
continue;
}
- // Rule (10) Numeric x ALetter
+ // Rule (10) Numeric x (ALetter | Hebrew_Letter)
if (fNumericSet->contains(c1) &&
- fALetterSet->contains(c2)) {
+ (fALetterSet->contains(c2) || fHebrew_LetterSet->contains(c2))) {
continue;
}
- // Rule (11) Numeric (MidNum | MidNumLet) x Numeric
- if ( fNumericSet->contains(c0) &&
- fMidNumSet->contains(c1) &&
+ // Rule (11) Numeric (MidNum | MidNumLet | Single_Quote) x Numeric
+ if (fNumericSet->contains(c0) &&
+ (fMidNumSet->contains(c1) || fMidNumLetSet->contains(c1) || fSingle_QuoteSet->contains(c1)) &&
fNumericSet->contains(c2)) {
continue;
}
- // Rule (12) Numeric x (MidNum | MidNumLet) Numeric
+ // Rule (12) Numeric x (MidNum | MidNumLet | SingleQuote) Numeric
if (fNumericSet->contains(c1) &&
- fMidNumSet->contains(c2) &&
+ (fMidNumSet->contains(c2) || fMidNumLetSet->contains(c2) || fSingle_QuoteSet->contains(c2)) &&
fNumericSet->contains(c3)) {
continue;
}
continue;
}
- // Rule 13a
- if ((fALetterSet->contains(c1) || fNumericSet->contains(c1) ||
+ // Rule 13a (ALetter | Hebrew_Letter | Numeric | KataKana | ExtendNumLet) x ExtendNumLet
+ if ((fALetterSet->contains(c1) || fHebrew_LetterSet->contains(c1) ||fNumericSet->contains(c1) ||
fKatakanaSet->contains(c1) || fExtendNumLetSet->contains(c1)) &&
fExtendNumLetSet->contains(c2)) {
continue;
- }
+ }
- // Rule 13b
+ // Rule 13b ExtendNumLet x (ALetter | Hebrew_Letter | Numeric | Katakana)
if (fExtendNumLetSet->contains(c1) &&
- (fALetterSet->contains(c2) || fNumericSet->contains(c2) ||
- fKatakanaSet->contains(c2))) {
- continue;
- }
+ (fALetterSet->contains(c2) || fHebrew_LetterSet->contains(c2) ||
+ fNumericSet->contains(c2) || fKatakanaSet->contains(c2))) {
+ continue;
+ }
+
+ // Rule 13c
+ if (fRegionalIndicatorSet->contains(c0) && fRegionalIndicatorSet->contains(c1)) {
+ break;
+ }
+ if (fRegionalIndicatorSet->contains(c1) && fRegionalIndicatorSet->contains(c2)) {
+ continue;
+ }
+
+ // Rule 13d
+ if ((fEBaseSet->contains(c1) || fGAZSet->contains(c1)) && fEModifierSet->contains(c2)) {
+ continue;
+ }
// Rule 14. Break found here.
break;
RBBIWordMonkey::~RBBIWordMonkey() {
delete fSets;
+ delete fCRSet;
+ delete fLFSet;
+ delete fNewlineSet;
delete fKatakanaSet;
+ delete fHebrew_LetterSet;
delete fALetterSet;
+ delete fSingle_QuoteSet;
+ delete fDouble_QuoteSet;
+ delete fMidNumLetSet;
delete fMidLetterSet;
delete fMidNumSet;
delete fNumericSet;
delete fFormatSet;
delete fExtendSet;
delete fExtendNumLetSet;
+ delete fRegionalIndicatorSet;
+ delete fDictionaryCjkSet;
delete fOtherSet;
+ delete fEBaseSet;
+ delete fEModifierSet;
+ delete fZWSSet;
+ delete fGAZSet;
}
UnicodeSet *fOLetterSet;
UnicodeSet *fNumericSet;
UnicodeSet *fATermSet;
+ UnicodeSet *fSContinueSet;
UnicodeSet *fSTermSet;
UnicodeSet *fCloseSet;
UnicodeSet *fOtherSet;
fSets = new UVector(status);
- fSepSet = new UnicodeSet("[\\p{Sentence_Break = Sep}]", status);
- fFormatSet = new UnicodeSet("[\\p{Sentence_Break = Format}]", status);
- fSpSet = new UnicodeSet("[\\p{Sentence_Break = Sp}]", status);
- fLowerSet = new UnicodeSet("[\\p{Sentence_Break = Lower}]", status);
- fUpperSet = new UnicodeSet("[\\p{Sentence_Break = Upper}]", status);
- fOLetterSet = new UnicodeSet("[\\p{Sentence_Break = OLetter}-[\\uff9e\\uff9f]]", status);
- fNumericSet = new UnicodeSet("[\\p{Sentence_Break = Numeric}]", status);
- fATermSet = new UnicodeSet("[\\p{Sentence_Break = ATerm}]", status);
- fSTermSet = new UnicodeSet("[\\p{Sentence_Break = STerm}]", status);
- fCloseSet = new UnicodeSet("[\\p{Sentence_Break = Close}]", status);
- fExtendSet = new UnicodeSet("[\\p{Grapheme_Extend}\\uff9e\\uff9f]", status);
+ // Separator Set Note: Beginning with Unicode 5.1, CR and LF were removed from the separator
+ // set and made into character classes of their own. For the monkey impl,
+ // they remain in SEP, since Sep always appears with CR and LF in the rules.
+ fSepSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Sep} \\u000a \\u000d]"), status);
+ fFormatSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Format}]"), status);
+ fSpSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Sp}]"), status);
+ fLowerSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Lower}]"), status);
+ fUpperSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Upper}]"), status);
+ fOLetterSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = OLetter}]"), status);
+ fNumericSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Numeric}]"), status);
+ fATermSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = ATerm}]"), status);
+ fSContinueSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = SContinue}]"), status);
+ fSTermSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = STerm}]"), status);
+ fCloseSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Close}]"), status);
+ fExtendSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Sentence_Break = Extend}]"), status);
fOtherSet = new UnicodeSet();
if(U_FAILURE(status)) {
fOtherSet->removeAll(*fOLetterSet);
fOtherSet->removeAll(*fNumericSet);
fOtherSet->removeAll(*fATermSet);
+ fOtherSet->removeAll(*fSContinueSet);
fOtherSet->removeAll(*fSTermSet);
fOtherSet->removeAll(*fCloseSet);
fOtherSet->removeAll(*fExtendSet);
- fSets->addElement(fSepSet, status);
- fSets->addElement(fFormatSet, status);
-
- fSets->addElement(fSpSet, status);
- fSets->addElement(fLowerSet, status);
- fSets->addElement(fUpperSet, status);
- fSets->addElement(fOLetterSet, status);
- fSets->addElement(fNumericSet, status);
- fSets->addElement(fATermSet, status);
- fSets->addElement(fSTermSet, status);
- fSets->addElement(fCloseSet, status);
- fSets->addElement(fOtherSet, status);
- fSets->addElement(fExtendSet, status);
+ fSets->addElement(fSepSet, status);
+ fSets->addElement(fFormatSet, status);
+ fSets->addElement(fSpSet, status);
+ fSets->addElement(fLowerSet, status);
+ fSets->addElement(fUpperSet, status);
+ fSets->addElement(fOLetterSet, status);
+ fSets->addElement(fNumericSet, status);
+ fSets->addElement(fATermSet, status);
+ fSets->addElement(fSContinueSet, status);
+ fSets->addElement(fSTermSet, status);
+ fSets->addElement(fCloseSet, status);
+ fSets->addElement(fOtherSet, status);
+ fSets->addElement(fExtendSet, status);
if (U_FAILURE(status)) {
deferredStatus = status;
// moveBack() Find the "significant" code point preceding the index i.
// Skips over ($Extend | $Format)* .
-//
+//
int RBBISentMonkey::moveBack(int i) {
if (i <= 0) {
return -1;
UChar32 c0, c1, c2, c3; // The code points at p0, p1, p2 & p3.
UChar32 c;
+ if (U_FAILURE(deferredStatus)) {
+ return -1;
+ }
+
// Prev break at end of string. return DONE.
if (prevPos >= fText->length()) {
return -1;
p0 = p1 = p2 = p3 = prevPos;
c3 = fText->char32At(prevPos);
c0 = c1 = c2 = 0;
+ (void)p0; // Suppress set but not used warning.
// Loop runs once per "significant" character position in the input text.
for (;;) {
p0 = p1; c0 = c1;
p1 = p2; c1 = c2;
p2 = p3; c2 = c3;
-
+
// Advancd p3 by X(Extend | Format)* Rule 4
p3 = moveForward(p3);
c3 = cAt(p3);
if (c1==0x0d && c2==0x0a && p2==(p1+1)) {
continue;
}
-
+
// Rule (4). Sep <break>
if (fSepSet->contains(c1)) {
p2 = p1+1; // Separators don't combine with Extend or Format.
// Still warming up the loop. (won't work with zero length strings, but we don't care)
continue;
}
-
+
// Rule (6). ATerm x Numeric
if (fATermSet->contains(c1) && fNumericSet->contains(c2)) {
continue;
}
- // Rule (7). Upper ATerm x Uppper
- if (fUpperSet->contains(c0) && fATermSet->contains(c1) && fUpperSet->contains(c2)) {
+ // Rule (7). (Upper | Lower) ATerm x Uppper
+ if ((fUpperSet->contains(c0) || fLowerSet->contains(c0)) &&
+ fATermSet->contains(c1) && fUpperSet->contains(c2)) {
continue;
}
continue;
}
}
-
- // Rule 8a (STerm | ATerm) Close* Sp* x (STerm | ATerm);
- if (fSTermSet->contains(c2) || fATermSet->contains(c2)) {
+
+ // Rule 8a (STerm | ATerm) Close* Sp* x (SContinue | STerm | ATerm);
+ if (fSContinueSet->contains(c2) || fSTermSet->contains(c2) || fATermSet->contains(c2)) {
p8 = p1;
while (fSpSet->contains(cAt(p8))) {
p8 = moveBack(p8);
}
}
- // Rule (9) (STerm | ATerm) Close* x (Close | Sp | Sep)
+ // Rule (9) (STerm | ATerm) Close* x (Close | Sp | Sep | CR | LF)
int p9 = p1;
while (fCloseSet->contains(cAt(p9))) {
p9 = moveBack(p9);
}
}
- // Rule (10) (Sterm | ATerm) Close* Sp* x (Sp | Sep)
+ // Rule (10) (Sterm | ATerm) Close* Sp* x (Sp | Sep | CR | LF)
int p10 = p1;
while (fSpSet->contains(cAt(p10))) {
p10 = moveBack(p10);
}
}
- // Rule (11) (STerm | ATerm) Close* Sp* <break>
+ // Rule (11) (STerm | ATerm) Close* Sp* (Sep | CR | LF)? <break>
int p11 = p1;
+ if (fSepSet->contains(cAt(p11))) {
+ p11 = moveBack(p11);
+ }
while (fSpSet->contains(cAt(p11))) {
p11 = moveBack(p11);
}
delete fOLetterSet;
delete fNumericSet;
delete fATermSet;
+ delete fSContinueSet;
delete fSTermSet;
delete fCloseSet;
delete fOtherSet;
UnicodeSet *fH2;
UnicodeSet *fH3;
UnicodeSet *fCL;
+ UnicodeSet *fCP;
UnicodeSet *fEX;
UnicodeSet *fIN;
UnicodeSet *fJL;
UnicodeSet *fSY;
UnicodeSet *fAI;
UnicodeSet *fAL;
+ UnicodeSet *fCJ;
+ UnicodeSet *fHL;
UnicodeSet *fID;
- UnicodeSet *fSA;
+ UnicodeSet *fRI;
UnicodeSet *fXX;
+ UnicodeSet *fEB;
+ UnicodeSet *fEM;
+ UnicodeSet *fZJ;
- BreakIterator *fCharBI;
-
+ BreakIterator *fCharBI;
const UnicodeString *fText;
- int32_t *fOrigPositions;
-
RegexMatcher *fNumberMatcher;
- RegexMatcher *fLB11Matcher;
};
+RBBILineMonkey::RBBILineMonkey() :
+ RBBIMonkeyKind(),
+ fSets(NULL),
+
+ fCharBI(NULL),
+ fText(NULL),
+ fNumberMatcher(NULL)
-RBBILineMonkey::RBBILineMonkey()
{
+ if (U_FAILURE(deferredStatus)) {
+ return;
+ }
+
UErrorCode status = U_ZERO_ERROR;
fSets = new UVector(status);
- fBK = new UnicodeSet("[\\p{Line_Break=BK}]", status);
- fCR = new UnicodeSet("[\\p{Line_break=CR}]", status);
- fLF = new UnicodeSet("[\\p{Line_break=LF}]", status);
- fCM = new UnicodeSet("[\\p{Line_break=CM}]", status);
- fNL = new UnicodeSet("[\\p{Line_break=NL}]", status);
- fWJ = new UnicodeSet("[\\p{Line_break=WJ}]", status);
- fZW = new UnicodeSet("[\\p{Line_break=ZW}]", status);
- fGL = new UnicodeSet("[\\p{Line_break=GL}]", status);
- fCB = new UnicodeSet("[\\p{Line_break=CB}]", status);
- fSP = new UnicodeSet("[\\p{Line_break=SP}]", status);
- fB2 = new UnicodeSet("[\\p{Line_break=B2}]", status);
- fBA = new UnicodeSet("[\\p{Line_break=BA}]", status);
- fBB = new UnicodeSet("[\\p{Line_break=BB}]", status);
- fHY = new UnicodeSet("[\\p{Line_break=HY}]", status);
- fH2 = new UnicodeSet("[\\p{Line_break=H2}]", status);
- fH3 = new UnicodeSet("[\\p{Line_break=H3}]", status);
- fCL = new UnicodeSet("[\\p{Line_break=CL}]", status);
- fEX = new UnicodeSet("[\\p{Line_break=EX}]", status);
- fIN = new UnicodeSet("[\\p{Line_break=IN}]", status);
- fJL = new UnicodeSet("[\\p{Line_break=JL}]", status);
- fJV = new UnicodeSet("[\\p{Line_break=JV}]", status);
- fJT = new UnicodeSet("[\\p{Line_break=JT}]", status);
- fNS = new UnicodeSet("[\\p{Line_break=NS}]", status);
- fOP = new UnicodeSet("[\\p{Line_break=OP}]", status);
- fQU = new UnicodeSet("[\\p{Line_break=QU}]", status);
- fIS = new UnicodeSet("[\\p{Line_break=IS}]", status);
- fNU = new UnicodeSet("[\\p{Line_break=NU}]", status);
- fPO = new UnicodeSet("[\\p{Line_break=PO}]", status);
- fPR = new UnicodeSet("[\\p{Line_break=PR}]", status);
- fSY = new UnicodeSet("[\\p{Line_break=SY}]", status);
- fAI = new UnicodeSet("[\\p{Line_break=AI}]", status);
- fAL = new UnicodeSet("[\\p{Line_break=AL}]", status);
- fID = new UnicodeSet("[\\p{Line_break=ID}]", status);
- fSA = new UnicodeSet("[\\p{Line_break=SA}]", status);
- fSG = new UnicodeSet("[\\ud800-\\udfff]", status);
- fXX = new UnicodeSet("[\\p{Line_break=XX}]", status);
+ fBK = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_Break=BK}]"), status);
+ fCR = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CR}]"), status);
+ fLF = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=LF}]"), status);
+ fCM = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CM}]"), status);
+ fNL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=NL}]"), status);
+ fWJ = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=WJ}]"), status);
+ fZW = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=ZW}]"), status);
+ fGL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=GL}]"), status);
+ fCB = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CB}]"), status);
+ fSP = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=SP}]"), status);
+ fB2 = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=B2}]"), status);
+ fBA = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=BA}]"), status);
+ fBB = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=BB}]"), status);
+ fHY = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=HY}]"), status);
+ fH2 = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=H2}]"), status);
+ fH3 = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=H3}]"), status);
+ fCL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CL}]"), status);
+ fCP = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CP}]"), status);
+ fEX = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=EX}]"), status);
+ fIN = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=IN}]"), status);
+ fJL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=JL}]"), status);
+ fJV = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=JV}]"), status);
+ fJT = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=JT}]"), status);
+ fNS = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=NS}]"), status);
+ fOP = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=OP}]"), status);
+ fQU = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=QU}]"), status);
+ fIS = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=IS}]"), status);
+ fNU = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=NU}]"), status);
+ fPO = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=PO}]"), status);
+ fPR = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=PR}]"), status);
+ fSY = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=SY}]"), status);
+ fAI = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=AI}]"), status);
+ fAL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=AL}]"), status);
+ fCJ = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=CJ}]"), status);
+ fHL = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=HL}]"), status);
+ fID = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=ID}]"), status);
+ fRI = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=RI}]"), status);
+ fSG = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\ud800-\\udfff]"), status);
+ fXX = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Line_break=XX}]"), status);
+ fEB = new UnicodeSet(UnicodeString(
+ "[\\u261D\\u26F9\\u270A-\\u270D\\U0001F385\\U0001F3C2-\\U0001F3C4\\U0001F3C7\\U0001F3CA-\\U0001F3CC"
+ "\\U0001F442-\\U0001F443\\U0001F446-\\U0001F450\\U0001F466-\\U0001F478\\U0001F47C"
+ "\\U0001F481-\\U0001F483\\U0001F485-\\U0001F487\\U0001F4AA\\U0001F574-\\U0001F575\\U0001F57A\\U0001F590\\U0001F595-\\U0001F596"
+ "\\U0001F645-\\U0001F647\\U0001F64B-\\U0001F64F\\U0001F6A3\\U0001F6B4-\\U0001F6B6\\U0001F6C0\\U0001F6CC"
+ "\\U0001F918-\\U0001F91E\\U0001F926\\U0001F930\\U0001F933-\\U0001F939\\U0001F93C-\\U0001F93E]"), status);
+ fEM = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\U0001F3FB-\\U0001F3FF]"), status);
+ fZJ = new UnicodeSet((UChar32)0x200D, (UChar32)0x200D);
if (U_FAILURE(status)) {
deferredStatus = status;
- fCharBI = NULL;
- fNumberMatcher = NULL;
return;
}
fAL->addAll(*fXX); // Default behavior for XX is identical to AL
fAL->addAll(*fAI); // Default behavior for AI is identical to AL
- fAL->addAll(*fSA); // Default behavior for SA is XX, which defaults to AL
fAL->addAll(*fSG); // Default behavior for SG is identical to AL.
+ fNS->addAll(*fCJ); // Default behavior for CJ is identical to NS.
+
+ fID->addAll(*fEB); // Emoji Base and Emoji Modifier behave as ID.
+ fID->addAll(*fEM);
+ fAL->removeAll(*fEM);
+
+
+ fAL->remove((UChar32)0x2695); // move u2695 from Al to Id
+ fAL->remove((UChar32)0x2696); // move u2696 from Al to Id
+ fAL->remove((UChar32)0x2764); // Emoji Proposal: move u2764 from Al to Id
+ fAI->remove((UChar32)0x2640); // new ZWJ seqs
+ fAI->remove((UChar32)0x2642); // new ZWJ seqs
+ fID->add((UChar32)0x2695);
+ fID->add((UChar32)0x2696);
+ fID->add((UChar32)0x2764);
+ fID->add((UChar32)0x2640);
+ fID->add((UChar32)0x2642);
+
fSets->addElement(fBK, status);
fSets->addElement(fCR, status);
fSets->addElement(fLF, status);
fSets->addElement(fH2, status);
fSets->addElement(fH3, status);
fSets->addElement(fCL, status);
+ fSets->addElement(fCP, status);
fSets->addElement(fEX, status);
fSets->addElement(fIN, status);
fSets->addElement(fJL, status);
fSets->addElement(fSY, status);
fSets->addElement(fAI, status);
fSets->addElement(fAL, status);
+ fSets->addElement(fHL, status);
fSets->addElement(fID, status);
fSets->addElement(fWJ, status);
- fSets->addElement(fSA, status);
+ fSets->addElement(fRI, status);
fSets->addElement(fSG, status);
+ fSets->addElement(fEB, status);
+ fSets->addElement(fEM, status);
+ fSets->addElement(fZJ, status);
+
+ const char *rules =
+ "((\\p{Line_Break=PR}|\\p{Line_Break=PO})\\p{Line_Break=CM}*)?"
+ "((\\p{Line_Break=OP}|\\p{Line_Break=HY})\\p{Line_Break=CM}*)?"
+ "\\p{Line_Break=NU}\\p{Line_Break=CM}*"
+ "((\\p{Line_Break=NU}|\\p{Line_Break=IS}|\\p{Line_Break=SY})\\p{Line_Break=CM}*)*"
+ "((\\p{Line_Break=CL}|\\p{Line_Break=CP})\\p{Line_Break=CM}*)?"
+ "((\\p{Line_Break=PR}|\\p{Line_Break=PO})\\p{Line_Break=CM}*)?";
fNumberMatcher = new RegexMatcher(
- "((\\p{Line_Break=PR}|\\p{Line_Break=PO})\\p{Line_Break=CM}*)?"
- "((\\p{Line_Break=OP}|\\p{Line_Break=HY})\\p{Line_Break=CM}*)?"
- "\\p{Line_Break=NU}\\p{Line_Break=CM}*"
- "((\\p{Line_Break=NU}|\\p{Line_Break=IS}|\\p{Line_Break=SY})\\p{Line_Break=CM}*)*"
- "(\\p{Line_Break=CL}\\p{Line_Break=CM}*)?"
- "((\\p{Line_Break=PR}|\\p{Line_Break=PO})\\p{Line_Break=CM}*)?",
- 0, status);
+ UnicodeString(rules, -1, US_INV), 0, status);
fCharBI = BreakIterator::createCharacterInstance(Locale::getEnglish(), status);
// and thisChar may not be adjacent because combining
// characters between them will be ignored.
+ int32_t prevPosX2; // Second previous character. Wider context for LB21a.
+ UChar32 prevCharX2;
+
int32_t nextPos; // Index of the next character following pos.
// Usually skips over combining marks.
int32_t nextCPPos; // Index of the code point following "pos."
int32_t tPos; // temp value.
UChar32 c;
+ if (U_FAILURE(deferredStatus)) {
+ return -1;
+ }
+
if (startPos >= fText->length()) {
return -1;
}
// Initial values for loop. Loop will run the first time without finding breaks,
// while the invalid values shift out and the "this" and
// "prev" positions are filled in with good values.
- pos = prevPos = -1; // Invalid value, serves as flag for initial loop iteration.
- thisChar = prevChar = 0;
+ pos = prevPos = prevPosX2 = -1; // Invalid value, serves as flag for initial loop iteration.
+ thisChar = prevChar = prevCharX2 = 0;
nextPos = nextCPPos = startPos;
// Loop runs once per position in the test text, until a break position
// is found.
for (;;) {
+ prevPosX2 = prevPos;
+ prevCharX2 = prevChar;
+
prevPos = pos;
prevChar = thisChar;
if (prevPos == -1) {
continue;
}
-
+
// LB 4 Always break after hard line breaks,
if (fBK->contains(prevChar)) {
break;
break;
}
+ // LB 8a ZJ x ID
+ // The monkey test's way of ignoring combining characters doesn't work
+ // for this rule. ZJ is also a CM. Need to get the actual character
+ // preceding "thisChar", not ignoring combining marks, possibly ZJ.
+ {
+ int32_t prevIdx = fText->moveIndex32(pos, -1);
+ UChar32 prevC = fText->char32At(prevIdx);
+ if (fZJ->contains(prevC) && fID->contains(thisChar)) {
+ continue;
+ }
+ }
+
// LB 9, 10 Already done, at top of loop.
//
}
// LB 12
- // (!SP) x GL
// GL x
- if ((!fSP->contains(prevChar)) && fGL->contains(thisChar) ||
- fGL->contains(prevChar)) {
+ if (fGL->contains(prevChar)) {
+ continue;
+ }
+
+ // LB 12a
+ // [^SP BA HY] x GL
+ if (!(fSP->contains(prevChar) ||
+ fBA->contains(prevChar) ||
+ fHY->contains(prevChar) ) && fGL->contains(thisChar)) {
continue;
}
-
-
+
+
// LB 13 Don't break before closings.
- // NU x CL and NU x IS are not matched here so that they will
+ // NU x CL, NU x CP and NU x IS are not matched here so that they will
// fall into LB 17 and the more general number regular expression.
//
- if (!fNU->contains(prevChar) && fCL->contains(thisChar) ||
- fEX->contains(thisChar) ||
- !fNU->contains(prevChar) && fIS->contains(thisChar) ||
- !fNU->contains(prevChar) && fSY->contains(thisChar)) {
+ if ((!fNU->contains(prevChar) && fCL->contains(thisChar)) ||
+ (!fNU->contains(prevChar) && fCP->contains(thisChar)) ||
+ fEX->contains(thisChar) ||
+ (!fNU->contains(prevChar) && fIS->contains(thisChar)) ||
+ (!fNU->contains(prevChar) && fSY->contains(thisChar))) {
continue;
}
- // LB 16 CL SP* x NS
- // Scan backwards for SP* CM* CL
+ // LB 16 (CL | CP) SP* x NS
+ // Scan backwards for SP* CM* (CL | CP)
if (fNS->contains(thisChar)) {
int tPos = prevPos;
while (tPos>0 && fSP->contains(fText->char32At(tPos))) {
while (tPos>0 && fCM->contains(fText->char32At(tPos))) {
tPos = fText->moveIndex32(tPos, -1);
}
- if (fCL->contains(fText->char32At(tPos))) {
+ if (fCL->contains(fText->char32At(tPos)) || fCP->contains(fText->char32At(tPos))) {
continue;
}
}
}
}
-
+
// LB 18 break after space
if (fSP->contains(prevChar)) {
break;
continue;
}
+ // LB 21a
+ // HL (HY | BA) x
+ if (fHL->contains(prevCharX2) &&
+ (fHY->contains(prevChar) || fBA->contains(prevChar))) {
+ continue;
+ }
+
+ // LB 21b
+ // SY x HL
+ if (fSY->contains(prevChar) && fHL->contains(thisChar)) {
+ continue;
+ }
+
// LB 22
- if (fAL->contains(prevChar) && fIN->contains(thisChar) ||
- fID->contains(prevChar) && fIN->contains(thisChar) ||
- fIN->contains(prevChar) && fIN->contains(thisChar) ||
- fNU->contains(prevChar) && fIN->contains(thisChar) ) {
+ if ((fAL->contains(prevChar) && fIN->contains(thisChar)) ||
+ (fEX->contains(prevChar) && fIN->contains(thisChar)) ||
+ (fHL->contains(prevChar) && fIN->contains(thisChar)) ||
+ (fID->contains(prevChar) && fIN->contains(thisChar)) ||
+ (fIN->contains(prevChar) && fIN->contains(thisChar)) ||
+ (fNU->contains(prevChar) && fIN->contains(thisChar)) ) {
continue;
}
// LB 23 ID x PO
// AL x NU
+ // HL x NU
// NU x AL
- if (fID->contains(prevChar) && fPO->contains(thisChar) ||
- fAL->contains(prevChar) && fNU->contains(thisChar) ||
- fNU->contains(prevChar) && fAL->contains(thisChar) ) {
+ if ((fID->contains(prevChar) && fPO->contains(thisChar)) ||
+ (fAL->contains(prevChar) && fNU->contains(thisChar)) ||
+ (fHL->contains(prevChar) && fNU->contains(thisChar)) ||
+ (fNU->contains(prevChar) && fAL->contains(thisChar)) ||
+ (fNU->contains(prevChar) && fHL->contains(thisChar)) ) {
continue;
}
// LB 24 Do not break between prefix and letters or ideographs.
// PR x ID
- // PR x AL
- // PO x AL
- if (fPR->contains(prevChar) && fID->contains(thisChar) ||
- fPR->contains(prevChar) && fAL->contains(thisChar) ||
- fPO->contains(prevChar) && fAL->contains(thisChar) ) {
+ // PR x (AL | HL)
+ // PO x (AL | HL)
+ // (AL | HL) x PR // Apple early addition
+ // (AL | HL) x PO // Apple early addition
+ if ((fPR->contains(prevChar) && fID->contains(thisChar)) ||
+ (fPR->contains(prevChar) && (fAL->contains(thisChar) || fHL->contains(thisChar))) ||
+ (fPO->contains(prevChar) && (fAL->contains(thisChar) || fHL->contains(thisChar))) ||
+ ((fAL->contains(prevChar) || fHL->contains(prevChar)) && fPR->contains(thisChar)) ||
+ ((fAL->contains(prevChar) || fHL->contains(prevChar)) && fPO->contains(thisChar)) ) {
continue;
}
-
-
-
+
+
+
// LB 25 Numbers
if (fNumberMatcher->lookingAt(prevPos, status)) {
if (U_FAILURE(status)) {
- // LB 28 Do not break between alphabetics (“at”).
- if (fAL->contains(prevChar) && fAL->contains(thisChar)) {
+ // LB 28 Do not break between alphabetics ("at").
+ if ((fAL->contains(prevChar) || fHL->contains(prevChar)) && (fAL->contains(thisChar) || fHL->contains(thisChar))) {
continue;
}
// LB 29 Do not break between numeric punctuation and alphabetics ("e.g.").
- if (fIS->contains(prevChar) && fAL->contains(thisChar)) {
+ if (fIS->contains(prevChar) && (fAL->contains(thisChar) || fHL->contains(thisChar))) {
continue;
}
- //LB 30 Do not break between letters, numbers or ordinary symbols and opening or closing punctuation
- // (AL | NU) x OP
- // CL x (AL | NU)
- if ((fAL->contains(prevChar) || fNU->contains(prevChar)) &&
- fOP->contains(thisChar)) {
+ // LB 30 Do not break between letters, numbers, or ordinary symbols and opening or closing punctuation.
+ // (AL | NU) x OP
+ // CP x (AL | NU)
+ if ((fAL->contains(prevChar) || fHL->contains(prevChar) || fNU->contains(prevChar)) && fOP->contains(thisChar)) {
+ continue;
+ }
+ if (fCP->contains(prevChar) && (fAL->contains(thisChar) || fHL->contains(thisChar) || fNU->contains(thisChar))) {
continue;
}
- if (fCL->contains(prevChar) &&
- (fAL->contains(thisChar) || fNU->contains(thisChar))) {
+
+ // LB30a RI RI <break> RI
+ // RI x RI
+ if (fRI->contains(prevCharX2) && fRI->contains(prevChar) && fRI->contains(thisChar)) {
+ break;
+ }
+ if (fRI->contains(prevChar) && fRI->contains(thisChar)) {
continue;
}
+ // LB30b Emoji Base x Emoji Modifier
+ if (fEB->contains(prevChar) && fEM->contains(thisChar)) {
+ continue;
+ }
// LB 31 Break everywhere else
break;
delete fH2;
delete fH3;
delete fCL;
+ delete fCP;
delete fEX;
delete fIN;
delete fJL;
delete fSY;
delete fAI;
delete fAL;
+ delete fCJ;
+ delete fHL;
delete fID;
- delete fSA;
+ delete fRI;
delete fSG;
delete fXX;
+ delete fEB;
+ delete fEM;
+ delete fZJ;
delete fCharBI;
delete fNumberMatcher;
//
// type = char | word | line | sent | title
//
+// Example:
+// intltest rbbi/RBBITest/TestMonkey@"type=line loop=-1"
+//
//-------------------------------------------------------------------------------------------
static int32_t getIntParam(UnicodeString name, UnicodeString ¶ms, int32_t defaultVal) {
}
#endif
+#if !UCONFIG_NO_REGULAR_EXPRESSIONS
static void testBreakBoundPreceding(RBBITest *test, UnicodeString ustr,
BreakIterator *bi,
int expected[],
for (i = bi->last(); i != BreakIterator::DONE; i = bi->previous()) {
count --;
if (forward[count] != i) {
+ printStringBreaks(ustr, expected, expectedcount);
test->errln("happy break test previous() failed: expected %d but got %d",
forward[count], i);
break;
}
}
}
+#endif
void RBBITest::TestWordBreaks(void)
{
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
- // <data><>\u1d4a\u206e<?>\u0603\U0001d7ff<>\u2019<></data>
Locale locale("en");
UErrorCode status = U_ZERO_ERROR;
// BreakIterator *bi = BreakIterator::createCharacterInstance(locale, status);
BreakIterator *bi = BreakIterator::createWordInstance(locale, status);
- UChar str[300];
+ // Replaced any C+J characters in a row with a random sequence of characters
+ // of the same length to make our C+J segmentation not get in the way.
static const char *strlist[] =
{
"\\U000e0032\\u0097\\u0f94\\uc2d8\\u05f4\\U000e0031\\u060d",
- "\\U000e0037\\u4666\\u1202\\u003a\\U000e0031\\u064d\\u0bea\\u591c\\U000e0040\\u003b",
- "\\u0589\\u3e99\\U0001d7f3\\U000e0074\\u1810\\u200e\\U000e004b\\u179c\\u0027\\U000e0061\\u003a",
+ "\\U000e0037\\u2666\\u1202\\u003a\\U000e0031\\u064d\\u0bea\\u091c\\U000e0040\\u003b",
+ "\\u0589\\u3e99\\U0001d7f3\\U000e0074\\u1810\\u200e\\U000e004b\\u0027\\U000e0061\\u003a",
"\\u398c\\U000104a5\\U0001d173\\u102d\\u002e\\uca3b\\u002e\\u002c\\u5622",
- "\\u90ca\\u3588\\u009c\\u0953\\u194b",
+ "\\uac00\\u3588\\u009c\\u0953\\u194b",
"\\u200e\\U000e0072\\u0a4b\\U000e003f\\ufd2b\\u2027\\u002e\\u002e",
"\\u0602\\u2019\\ua191\\U000e0063\\u0a4c\\u003a\\ub4b5\\u003a\\u827f\\u002e",
- "\\u7f1f\\uc634\\u65f8\\u0944\\u04f2\\uacdf\\u1f9c\\u05f4\\u002e",
+ "\\u2f1f\\u1634\\u05f8\\u0944\\u04f2\\u0cdf\\u1f9c\\u05f4\\u002e",
"\\U000e0042\\u002e\\u0fb8\\u09ef\\u0ed1\\u2044",
"\\u003b\\u024a\\u102e\\U000e0071\\u0600",
"\\u2027\\U000e0067\\u0a47\\u00b7",
"\\u1fcd\\u002c\\u07aa\\u0027\\u11b0",
"\\u002c\\U000e003c\\U0001d7f4\\u003a\\u0c6f\\u0027",
"\\u0589\\U000e006e\\u0a42\\U000104a5",
- "\\u4f66\\ub523\\u003a\\uacae\\U000e0047\\u003a",
+ "\\u0f66\\u2523\\u003a\\u0cae\\U000e0047\\u003a",
"\\u003a\\u0f21\\u0668\\u0dab\\u003a\\u0655\\u00b7",
"\\u0027\\u11af\\U000e0057\\u0602",
"\\U0001d7f2\\U000e007\\u0004\\u0589",
"\\U000e0022\\u003a\\u10b3\\u003a\\ua21b\\u002e\\U000e0058\\u1732\\U000e002b",
"\\U0001d7f2\\U000e007d\\u0004\\u0589",
"\\u82ab\\u17e8\\u0736\\u2019\\U0001d64d",
- "\\u0e01\\ub55c\\u0a68\\U000e0037\\u0cd6\\u002c\\ub959",
+ "\\ub55c\\u0a68\\U000e0037\\u0cd6\\u002c\\ub959",
"\\U000e0065\\u302c\\uc986\\u09ee\\U000e0068",
"\\u0be8\\u002e\\u0c68\\u066e\\u136d\\ufc99\\u59e7",
"\\u0233\\U000e0020\\u0a69\\u0d6a",
"\\u206f\\u0741\\ub3ab\\u2019\\ubcac\\u2019",
- "\\u58f4\\U000e0049\\u20e7\\u2027",
+ "\\u18f4\\U000e0049\\u20e7\\u2027",
"\\ub315\\U0001d7e5\\U000e0073\\u0c47\\u06f2\\u0c6a\\u0037\\u10fe",
"\\ua183\\u102d\\u0bec\\u003a",
"\\u17e8\\u06e7\\u002e\\u096d\\u003b",
"\\U000e005d\\u2044\\u0731\\u0650\\u0061",
"\\u003a\\u0664\\u00b7\\u1fba",
"\\u003b\\u0027\\u00b7\\u47a3",
- "\\u2027\\U000e0067\\u0a42\\u00b7\\ubddf\\uc26c\\u003a\\u4186\\u041b",
+ "\\u2027\\U000e0067\\u0a42\\u00b7\\u4edf\\uc26c\\u003a\\u4186\\u041b",
"\\u0027\\u003a\\U0001d70f\\U0001d7df\\ubf4a\\U0001d7f5\\U0001d177\\u003a\\u0e51\\u1058\\U000e0058\\u00b7\\u0673",
"\\uc30d\\u002e\\U000e002c\\u0c48\\u003a\\ub5a1\\u0661\\u002c",
};
int loop;
if (U_FAILURE(status)) {
- errln("Creation of break iterator failed %s", u_errorName(status));
+ errcheckln(status, "Creation of break iterator failed %s", u_errorName(status));
return;
}
- for (loop = 0; loop < (int)(sizeof(strlist) / sizeof(char *)); loop ++) {
+ for (loop = 0; loop < UPRV_LENGTHOF(strlist); loop ++) {
// printf("looping %d\n", loop);
- u_unescape(strlist[loop], str, 25);
- UnicodeString ustr(str);
+ UnicodeString ustr = CharsToUnicodeString(strlist[loop]);
// RBBICharMonkey monkey;
RBBIWordMonkey monkey;
"\\U0001d7f2\\U000e007d\\u0004\\u0589",
"\\u82ab\\u17e8\\u0736\\u2019\\U0001d64d",
"\\u0e01\\ub55c\\u0a68\\U000e0037\\u0cd6\\u002c\\ub959",
- "\\U000e0065\\u302c\\uc986\\u09ee\\U000e0068",
+ "\\U000e0065\\u302c\\u09ee\\U000e0068",
"\\u0be8\\u002e\\u0c68\\u066e\\u136d\\ufc99\\u59e7",
"\\u0233\\U000e0020\\u0a69\\u0d6a",
"\\u206f\\u0741\\ub3ab\\u2019\\ubcac\\u2019",
"\\u58f4\\U000e0049\\u20e7\\u2027",
- "\\ub315\\U0001d7e5\\U000e0073\\u0c47\\u06f2\\u0c6a\\u0037\\u10fe",
+ "\\U0001d7e5\\U000e0073\\u0c47\\u06f2\\u0c6a\\u0037\\u10fe",
"\\ua183\\u102d\\u0bec\\u003a",
"\\u17e8\\u06e7\\u002e\\u096d\\u003b",
"\\u003a\\u0e57\\u0fad\\u002e",
};
int loop;
if (U_FAILURE(status)) {
- errln("Creation of break iterator failed %s", u_errorName(status));
+ errcheckln(status, "Creation of break iterator failed %s", u_errorName(status));
return;
}
- for (loop = 0; loop < (int)(sizeof(strlist) / sizeof(char *)); loop ++) {
+ for (loop = 0; loop < UPRV_LENGTHOF(strlist); loop ++) {
// printf("looping %d\n", loop);
u_unescape(strlist[loop], str, 20);
UnicodeString ustr(str);
"\\ufe3c\\u201c\\u000d\\u2025\\u2007\\u201c\\u002d\\u20a0\\u002d\\u30a7\\u17a4",
"\\u2772\\u0020\\U000e010a\\u0020\\u2025\\u000a\\U000e0123",
"\\u002d\\uff1b\\u02c8\\u2029\\ufeff\\u0f22\\u2044\\ufe09\\u003a\\u096d\\u2009\\u000a\\u06f7\\u02cc\\u1019\\u2060",
- "\\u1781\\u0b68\\u0f0c\\u3010\\u0085\\U00011f7a\\u0020\\u0dd6\\u200b\\U000e007a\\u000a\\u2060\\u2026\\u002f\\u2026\\u24dc\\u101e\\u2014\\u2007\\u30a5",
"\\u2770\\u0020\\U000e010f\\u0020\\u2060\\u000a\\u02cc\\u0bcc\\u060d\\u30e7\\u0f3b\\u002f",
"\\ufeff\\u0028\\u003b\\U00012fec\\u2010\\u0020\\u0004\\u200b\\u0020\\u275c\\u002f\\u17b1",
"\\u20a9\\u2014\\u00a2\\u31f1\\u002f\\u0020\\u05b8\\u200b\\u0cc2\\u003b\\u060d\\u02c8\\ua4e8\\u002f\\u17d5",
"\\u002f\\uf22e\\u1944\\ufe3d\\u0020\\u206f\\u31b3\\u2014\\u002d\\u2025\\u0f0c\\u0085\\u2763",
"\\u002f\\u2563\\u202f\\u0085\\u17d5\\u200b\\u0020\\U000e0043\\u2014\\u058a\\u3d0a\\ufe57\\u2035\\u2028\\u2029",
"\\u20ae\\U0001d169\\u9293\\uff1f\\uff1f\\u0021\\u2012\\u2039\\u0085\\u02cc\\u00a2\\u0020\\U000e01ab\\u3085\\u0f3a\\u1806\\u0f0c\\u1945\\u000a\\U0001d7e7",
- "\\uffe6\\u00a0\\u200b\\u0085\\u2116\\u255b\\U0001d7f7\\u178c\\ufffc",
"\\u02cc\\ufe6a\\u00a0\\u0021\\u002d\\u7490\\uec2e\\u200b\\u000a",
"\\uec2e\\u200b\\u000a\\u0020\\u2028\\u2014\\u8945",
"\\u7490\\uec2e\\u200b\\u000a\\u0020\\u2028\\u2014",
"\\u0020\\u2028\\u2014\\u8945\\u002c\\u005b",
"\\u000a\\ufe3c\\u201c\\u000d\\u2025\\u2007\\u201c\\u002d\\u20a0",
- "\\u2473\\u0e9d\\u0020\\u0085\\u000a\\ufe3c\\u201c\\u000d\\u2025",
"\\U0001d16e\\ufffc\\u2025\\u0021\\u002d",
"\\ufffc\\u301b\\u0fa5\\U000e0103\\u2060\\u208e\\u17d5\\u034f\\u1009\\u003a\\u180e\\u2009\\u3111",
- "\\u2014\\u0020\\u000a\\u17c5\\u24fc",
"\\ufffc\\u0020\\u2116\\uff6c\\u200b\\u0ac3\\U0001028f",
"\\uaeb0\\u0344\\u0085\\ufffc\\u073b\\u2010",
"\\ufeff\\u0589\\u0085\\u0eb8\\u30fd\\u002f\\u003a\\u2014\\ufe43",
"\\u09cc\\u256a\\u276d\\u002d\\u3085\\u000d\\u0e05\\u2028\\u0fbb",
"\\u2034\\u00bb\\u0ae6\\u300c\\u0020\\u31f8\\ufffc",
"\\u2116\\u0ed2\\uff64\\u02cd\\u2001\\u2060",
- "\\u809d\\u2e02\\u0f0a\\uc48f\\u2540\\u000d\\u0cef\\u003a\\u0e4d"
- "\\U000e0172\\U000e005c\\u17cf\\U00010ca6\\ufeff\\uf621\\u06f3\\uffe5"
- "\\u0ea2\\ufeff\\udcea\\u3085\\ua874\\u000a\\u0020\\u000b\\u200b",
"\\ufe10\\u2060\\u1a5a\\u2060\\u17e4\\ufffc\\ubbe1\\ufe15\\u0020\\u00a0",
"\\u2060\\u2213\\u200b\\u2019\\uc2dc\\uff6a\\u1736\\u0085\\udb07",
};
if (U_FAILURE(status)) {
return;
}
- for (loop = 0; loop < (int)(sizeof(strlist) / sizeof(char *)); loop ++) {
+ for (loop = 0; loop < UPRV_LENGTHOF(strlist); loop ++) {
// printf("looping %d\n", loop);
int32_t t = u_unescape(strlist[loop], str, STRSIZE);
if (t >= STRSIZE) {
continue;
}
-
+
UnicodeString ustr(str);
RBBILineMonkey monkey;
if (U_FAILURE(monkey.deferredStatus)) {
};
int loop;
if (U_FAILURE(status)) {
- errln("Creation of break iterator failed %s", u_errorName(status));
+ errcheckln(status, "Creation of break iterator failed %s", u_errorName(status));
return;
}
- for (loop = 0; loop < (int)(sizeof(strlist) / sizeof(char *)); loop ++) {
- u_unescape(strlist[loop], str, (int32_t)(sizeof(str) / sizeof(str[0])));
+ for (loop = 0; loop < UPRV_LENGTHOF(strlist); loop ++) {
+ u_unescape(strlist[loop], str, UPRV_LENGTHOF(str));
UnicodeString ustr(str);
RBBISentMonkey monkey;
// m.reset(p);
- if (RegexMatcher("\\S", p, 0, status).find()) {
+ if (RegexMatcher(UNICODE_STRING_SIMPLE("\\S"), p, 0, status).find()) {
// Each option is stripped out of the option string as it is processed.
// All options have been checked. The option string should have been completely emptied..
char buf[100];
}
}
else {
- errln("Creation of character break iterator failed %s", u_errorName(status));
+ errcheckln(status, "Creation of character break iterator failed %s", u_errorName(status));
}
delete bi;
}
RunMonkey(bi, m, "word", seed, loopCount, useUText);
}
else {
- errln("Creation of word break iterator failed %s", u_errorName(status));
+ errcheckln(status, "Creation of word break iterator failed %s", u_errorName(status));
}
delete bi;
}
RunMonkey(bi, m, "line", seed, loopCount, useUText);
}
else {
- errln("Creation of line break iterator failed %s", u_errorName(status));
+ errcheckln(status, "Creation of line break iterator failed %s", u_errorName(status));
}
delete bi;
}
- if (breakType == "sent" || breakType == "all" ) {
+ if (breakType == "sent" || breakType == "all" ) {
logln("Sentence Break Monkey Test");
RBBISentMonkey m;
BreakIterator *bi = BreakIterator::createSentenceInstance(locale, status);
RunMonkey(bi, m, "sentence", seed, loopCount, useUText);
}
else {
- errln("Creation of line break iterator failed %s", u_errorName(status));
+ errcheckln(status, "Creation of line break iterator failed %s", u_errorName(status));
}
delete bi;
}
// seed - Seed for starting random number generator (parameter from user)
// numIterations
//
-void RBBITest::RunMonkey(BreakIterator *bi, RBBIMonkeyKind &mk, const char *name, uint32_t seed,
+void RBBITest::RunMonkey(BreakIterator *bi, RBBIMonkeyKind &mk, const char *name, uint32_t seed,
int32_t numIterations, UBool useUText) {
#if !UCONFIG_NO_REGULAR_EXPRESSIONS
int32_t charIdx = m_rand() % classSet->size();
UChar32 c = classSet->charAt(charIdx);
if (c < 0) { // TODO: deal with sets containing strings.
- errln("c < 0");
+ errln("%s:%d c < 0", __FILE__, __LINE__);
break;
}
+ // Do not assemble a supplementary character from randomly generated separate surrogates.
+ // (It could be a dictionary character)
+ if (U16_IS_TRAIL(c) && testText.length() > 0 && U16_IS_LEAD(testText.charAt(testText.length()-1))) {
+ continue;
+ }
+
testText.append(c);
}
expectedBreaks[breakPos] = 1;
U_ASSERT(expectedCount<testText.length());
expected[expectedCount ++] = breakPos;
+ (void)expected; // Set but not used warning.
+ // TODO (andy): check it out.
}
// Find the break positions using forward iteration
if (breakPos <= i ||
breakPos < lastBreakPos ||
breakPos > testText.length() ||
- breakPos > lastBreakPos && lastBreakPos > i ) {
+ (breakPos > lastBreakPos && lastBreakPos > i)) {
errln("%s break monkey test: "
"Out of range value returned by BreakIterator::following().\n"
"Random seed=%d index=%d; following returned %d; lastbreak=%d",
}
// Find the break positions using the preceding() function.
- memset(precedingBreaks, 0, sizeof(followingBreaks));
+ memset(precedingBreaks, 0, sizeof(precedingBreaks));
lastBreakPos = testText.length();
precedingBreaks[testText.length()] = 1;
for (i=testText.length(); i>0; i--) {
breakPos = bi->preceding(i);
if (breakPos >= i ||
breakPos > lastBreakPos ||
- breakPos < 0 && testText.getChar32Start(i)>0 ||
- breakPos < lastBreakPos && lastBreakPos < testText.getChar32Start(i) ) {
+ (breakPos < 0 && testText.getChar32Start(i)>0) ||
+ (breakPos < lastBreakPos && lastBreakPos < testText.getChar32Start(i)) ) {
errln("%s break monkey test: "
"Out of range value returned by BreakIterator::preceding().\n"
"index=%d; prev returned %d; lastBreak=%d" ,
name, i, breakPos, lastBreakPos);
- precedingBreaks[i] = 2; // Forces an error.
+ if (breakPos >= 0 && breakPos < (int32_t)sizeof(precedingBreaks)) {
+ precedingBreaks[i] = 2; // Forces an error.
+ }
} else {
- precedingBreaks[breakPos] = 1;
+ if (breakPos >= 0) {
+ precedingBreaks[breakPos] = 1;
+ }
lastBreakPos = breakPos;
}
}
UErrorCode status = U_ZERO_ERROR;
errorText.extract(charErrorTxt, sizeof(charErrorTxt), NULL, status);
charErrorTxt[sizeof(charErrorTxt)-1] = 0;
- errln("%s break monkey test error. %s. Operation = %s; Random seed = %d; buf Idx = %d\n%s",
- name, (expectedBreaks[i]? "break expected but not found" : "break found but not expected"),
+ const char *badLocale = bi->getLocaleID(ULOC_ACTUAL_LOCALE, status);
+
+ errln("%s break monkey test error [%s]. %s. Operation = %s; Random seed = %d; buf Idx = %d\n%s",
+ name, badLocale, (expectedBreaks[i]? "break expected but not found" : "break found but not expected"),
errorType, seed, i, charErrorTxt);
break;
}
#endif
}
+
+// Bug 5532. UTF-8 based UText fails in dictionary code.
+// This test checks the initial patch,
+// which is to just keep it from crashing. Correct word boundaries
+// await a proper fix to the dictionary code.
+//
+void RBBITest::TestBug5532(void) {
+ // Text includes a mixture of Thai and Latin.
+ const unsigned char utf8Data[] = {
+ 0xE0u, 0xB8u, 0x82u, 0xE0u, 0xB8u, 0xB2u, 0xE0u, 0xB8u, 0xA2u, 0xE0u,
+ 0xB9u, 0x80u, 0xE0u, 0xB8u, 0x84u, 0xE0u, 0xB8u, 0xA3u, 0xE0u, 0xB8u,
+ 0xB7u, 0xE0u, 0xB9u, 0x88u, 0xE0u, 0xB8u, 0xADu, 0xE0u, 0xB8u, 0x87u,
+ 0xE0u, 0xB9u, 0x80u, 0xE0u, 0xB8u, 0xA5u, 0xE0u, 0xB9u, 0x88u, 0xE0u,
+ 0xB8u, 0x99u, 0xE0u, 0xB8u, 0x8Bu, 0xE0u, 0xB8u, 0xB5u, 0xE0u, 0xB8u,
+ 0x94u, 0xE0u, 0xB8u, 0xB5u, 0x20u, 0x73u, 0x69u, 0x6Du, 0x20u, 0x61u,
+ 0x75u, 0x64u, 0x69u, 0x6Fu, 0x2Fu, 0x20u, 0x4Du, 0x4Fu, 0x4Fu, 0x4Eu,
+ 0x20u, 0x65u, 0x63u, 0x6Cu, 0x69u, 0x70u, 0x73u, 0x65u, 0x20u, 0xE0u,
+ 0xB8u, 0xA3u, 0xE0u, 0xB8u, 0xB2u, 0xE0u, 0xB8u, 0x84u, 0xE0u, 0xB8u,
+ 0xB2u, 0x20u, 0x34u, 0x37u, 0x30u, 0x30u, 0x20u, 0xE0u, 0xB8u, 0xA2u,
+ 0xE0u, 0xB8u, 0xB9u, 0xE0u, 0xB9u, 0x82u, 0xE0u, 0xB8u, 0xA3u, 0x00};
+
+ UErrorCode status = U_ZERO_ERROR;
+ UText utext=UTEXT_INITIALIZER;
+ utext_openUTF8(&utext, (const char *)utf8Data, -1, &status);
+ TEST_ASSERT_SUCCESS(status);
+
+ BreakIterator *bi = BreakIterator::createWordInstance(Locale("th"), status);
+ TEST_ASSERT_SUCCESS(status);
+ if (U_SUCCESS(status)) {
+ bi->setText(&utext, status);
+ TEST_ASSERT_SUCCESS(status);
+
+ int32_t breakCount = 0;
+ int32_t previousBreak = -1;
+ for (bi->first(); bi->next() != BreakIterator::DONE; breakCount++) {
+ // For now, just make sure that the break iterator doesn't hang.
+ TEST_ASSERT(previousBreak < bi->current());
+ previousBreak = bi->current();
+ }
+ TEST_ASSERT(breakCount > 0);
+ }
+ delete bi;
+ utext_close(&utext);
+}
+
+
+void RBBITest::TestBug9983(void) {
+ UnicodeString text = UnicodeString("\\u002A" // * Other
+ "\\uFF65" // Other
+ "\\u309C" // Katakana
+ "\\uFF9F" // Extend
+ "\\uFF65" // Other
+ "\\u0020" // Other
+ "\\u0000").unescape();
+
+ UErrorCode status = U_ZERO_ERROR;
+ LocalPointer<RuleBasedBreakIterator> brkiter(static_cast<RuleBasedBreakIterator *>(
+ BreakIterator::createWordInstance(Locale::getRoot(), status)));
+ TEST_ASSERT_SUCCESS(status);
+ LocalPointer<RuleBasedBreakIterator> brkiterPOSIX(static_cast<RuleBasedBreakIterator *>(
+ BreakIterator::createWordInstance(Locale::createFromName("en_US_POSIX"), status)));
+ TEST_ASSERT_SUCCESS(status);
+ if (U_FAILURE(status)) {
+ return;
+ }
+ int32_t offset, rstatus, iterationCount;
+
+ brkiter->setText(text);
+ brkiter->last();
+ iterationCount = 0;
+ while ( (offset = brkiter->previous()) != UBRK_DONE ) {
+ iterationCount++;
+ rstatus = brkiter->getRuleStatus();
+ (void)rstatus; // Suppress set but not used warning.
+ if (iterationCount >= 10) {
+ break;
+ }
+ }
+ TEST_ASSERT(iterationCount == 6);
+
+ brkiterPOSIX->setText(text);
+ brkiterPOSIX->last();
+ iterationCount = 0;
+ while ( (offset = brkiterPOSIX->previous()) != UBRK_DONE ) {
+ iterationCount++;
+ rstatus = brkiterPOSIX->getRuleStatus();
+ (void)rstatus; // Suppress set but not used warning.
+ if (iterationCount >= 10) {
+ break;
+ }
+ }
+ TEST_ASSERT(iterationCount == 6);
+}
+
+
//
// TestDebug - A place-holder test for debugging purposes.
// For putting in fragments of other tests that can be invoked
#endif
}
+void RBBITest::TestProperties() {
+ UErrorCode errorCode = U_ZERO_ERROR;
+ UnicodeSet prependSet(UNICODE_STRING_SIMPLE("[:GCB=Prepend:]"), errorCode);
+ if (!prependSet.isEmpty()) {
+ errln(
+ "[:GCB=Prepend:] is not empty any more. "
+ "Uncomment relevant lines in source/data/brkitr/char.txt and "
+ "change this test to the opposite condition.");
+ }
+}
+
#endif /* #if !UCONFIG_NO_BREAK_ITERATION */
projects / apple / icu.git / blobdiff