1 /********************************************************************
3 * Copyright (c) 1997-2013,2015, International Business Machines Corporation and
4 * others. All Rights Reserved.
5 ********************************************************************/
6 /********************************************************************************
10 * Modification History:
12 * Madhu Katragadda Creation
13 *********************************************************************************/
14 /*C API TEST FOR BREAKITERATOR */
16 * This is an API test. It doesn't test very many cases, and doesn't
17 * try to test the full functionality. It just calls each function in the class and
18 * verifies that it works on a basic level.
21 #include "unicode/utypes.h"
23 #if !UCONFIG_NO_BREAK_ITERATION
27 #include "unicode/uloc.h"
28 #include "unicode/ubrk.h"
29 #include "unicode/ustring.h"
30 #include "unicode/ucnv.h"
31 #include "unicode/utext.h"
35 #define TEST_ASSERT_SUCCESS(status) {if (U_FAILURE(status)) { \
36 log_data_err("Failure at file %s, line %d, error = %s (Are you missing data?)\n", __FILE__, __LINE__, u_errorName(status));}}
38 #define TEST_ASSERT(expr) {if ((expr)==FALSE) { \
39 log_data_err("Test Failure at file %s, line %d (Are you missing data?)\n", __FILE__, __LINE__);}}
41 #if !UCONFIG_NO_FILE_IO
42 static void TestBreakIteratorSafeClone(void);
44 static void TestBreakIteratorRules(void);
45 static void TestBreakIteratorRuleError(void);
46 static void TestBreakIteratorStatusVec(void);
47 static void TestBreakIteratorUText(void);
48 static void TestBreakIteratorTailoring(void);
49 static void TestBreakIteratorRefresh(void);
50 static void TestBreakIteratorSuppressions(void); /* Apple-specific for now */
52 void addBrkIterAPITest(TestNode
** root
);
54 void addBrkIterAPITest(TestNode
** root
)
56 #if !UCONFIG_NO_FILE_IO
57 addTest(root
, &TestBreakIteratorCAPI
, "tstxtbd/cbiapts/TestBreakIteratorCAPI");
58 addTest(root
, &TestBreakIteratorSafeClone
, "tstxtbd/cbiapts/TestBreakIteratorSafeClone");
59 addTest(root
, &TestBreakIteratorUText
, "tstxtbd/cbiapts/TestBreakIteratorUText");
61 addTest(root
, &TestBreakIteratorRules
, "tstxtbd/cbiapts/TestBreakIteratorRules");
62 addTest(root
, &TestBreakIteratorRuleError
, "tstxtbd/cbiapts/TestBreakIteratorRuleError");
63 addTest(root
, &TestBreakIteratorStatusVec
, "tstxtbd/cbiapts/TestBreakIteratorStatusVec");
64 addTest(root
, &TestBreakIteratorTailoring
, "tstxtbd/cbiapts/TestBreakIteratorTailoring");
65 addTest(root
, &TestBreakIteratorRefresh
, "tstxtbd/cbiapts/TestBreakIteratorRefresh");
66 addTest(root
, &TestBreakIteratorSuppressions
, "tstxtbd/cbiapts/TestBreakIteratorSuppressions");
69 #define CLONETEST_ITERATOR_COUNT 2
72 * Utility function for converting char * to UChar * strings, to
73 * simplify the test code. Converted strings are put in heap allocated
74 * storage. A hook (probably a local in the caller's code) allows all
75 * strings converted with that hook to be freed with a single call.
77 typedef struct StringStruct
{
78 struct StringStruct
*link
;
83 static UChar
* toUChar(const char *src
, void **freeHook
) {
84 /* Structure of the memory that we allocate on the heap */
88 UChar stackBuf
[2000 + sizeof(void *)/sizeof(UChar
)];
92 UErrorCode status
= U_ZERO_ERROR
;
97 cnv
= ucnv_open(NULL
, &status
);
98 if(U_FAILURE(status
) || cnv
== NULL
) {
102 numUChars
= ucnv_toUChars(cnv
,
108 destSize
= (numUChars
+1) * sizeof(UChar
) + sizeof(struct StringStruct
);
109 dest
= (StringStruct
*)malloc(destSize
);
111 if (status
== U_BUFFER_OVERFLOW_ERROR
|| status
== U_STRING_NOT_TERMINATED_WARNING
) {
112 ucnv_toUChars(cnv
, dest
->str
, numUChars
+1, src
, -1, &status
);
113 } else if (status
== U_ZERO_ERROR
) {
114 u_strcpy(dest
->str
, stackBuf
);
121 ucnv_reset(cnv
); /* be good citizens */
127 dest
->link
= (StringStruct
*)(*freeHook
);
132 static void freeToUCharStrings(void **hook
) {
133 StringStruct
*s
= *(StringStruct
**)hook
;
135 StringStruct
*next
= s
->link
;
142 #if !UCONFIG_NO_FILE_IO
143 static void TestBreakIteratorCAPI()
145 UErrorCode status
= U_ZERO_ERROR
;
146 UBreakIterator
*word
, *sentence
, *line
, *character
, *b
, *bogus
;
147 int32_t start
,pos
,end
,to
;
153 /* Note: the adjacent "" are concatenating strings, not adding a \" to the
154 string, which is probably what whoever wrote this intended. Don't fix,
155 because it would throw off the hard coded break positions in the following
157 u_uastrcpy(text
, "He's from Africa. ""Mr. Livingston, I presume?"" Yeah");
161 log_verbose("\nTesting BreakIterator open functions\n");
163 /* Use french for fun */
164 word
= ubrk_open(UBRK_WORD
, "en_US", text
, u_strlen(text
), &status
);
165 if(status
== U_FILE_ACCESS_ERROR
) {
166 log_data_err("Check your data - it doesn't seem to be around\n");
168 } else if(U_FAILURE(status
)){
169 log_err_status(status
, "FAIL: Error in ubrk_open() for word breakiterator: %s\n", myErrorName(status
));
172 log_verbose("PASS: Successfully opened word breakiterator\n");
175 sentence
= ubrk_open(UBRK_SENTENCE
, "en_US", text
, u_strlen(text
), &status
);
176 if(U_FAILURE(status
)){
177 log_err_status(status
, "FAIL: Error in ubrk_open() for sentence breakiterator: %s\n", myErrorName(status
));
181 log_verbose("PASS: Successfully opened sentence breakiterator\n");
184 line
= ubrk_open(UBRK_LINE
, "en_US", text
, u_strlen(text
), &status
);
185 if(U_FAILURE(status
)){
186 log_err("FAIL: Error in ubrk_open() for line breakiterator: %s\n", myErrorName(status
));
190 log_verbose("PASS: Successfully opened line breakiterator\n");
193 character
= ubrk_open(UBRK_CHARACTER
, "en_US", text
, u_strlen(text
), &status
);
194 if(U_FAILURE(status
)){
195 log_err("FAIL: Error in ubrk_open() for character breakiterator: %s\n", myErrorName(status
));
199 log_verbose("PASS: Successfully opened character breakiterator\n");
201 /*trying to open an illegal iterator*/
202 bogus
= ubrk_open((UBreakIteratorType
)5, "en_US", text
, u_strlen(text
), &status
);
204 log_err("FAIL: expected NULL from opening an invalid break iterator.\n");
206 if(U_SUCCESS(status
)){
207 log_err("FAIL: Error in ubrk_open() for BOGUS breakiterator. Expected U_ILLEGAL_ARGUMENT_ERROR\n");
209 if(U_FAILURE(status
)){
210 if(status
!= U_ILLEGAL_ARGUMENT_ERROR
){
211 log_err("FAIL: Error in ubrk_open() for BOGUS breakiterator. Expected U_ILLEGAL_ARGUMENT_ERROR\n Got %s\n", myErrorName(status
));
217 /* ======= Test ubrk_countAvialable() and ubrk_getAvialable() */
219 log_verbose("\nTesting ubrk_countAvailable() and ubrk_getAvailable()\n");
220 count
=ubrk_countAvailable();
221 /* use something sensible w/o hardcoding the count */
223 log_err("FAIL: Error in ubrk_countAvialable() returned %d\n", count
);
226 log_verbose("PASS: ubrk_countAvialable() successful returned %d\n", count
);
230 log_verbose("%s\n", ubrk_getAvailable(i
));
231 if (ubrk_getAvailable(i
) == 0)
232 log_err("No locale for which breakiterator is applicable\n");
234 log_verbose("A locale %s for which breakiterator is applicable\n",ubrk_getAvailable(i
));
237 /*========Test ubrk_first(), ubrk_last()...... and other functions*/
239 log_verbose("\nTesting the functions for word\n");
240 start
= ubrk_first(word
);
242 log_err("error ubrk_start(word) did not return 0\n");
243 log_verbose("first (word = %d\n", (int32_t)start
);
246 log_err("error ubrk_next(word) did not return 4\n");
247 log_verbose("next (word = %d\n", (int32_t)pos
);
248 pos
=ubrk_following(word
, 4);
250 log_err("error ubrl_following(word,4) did not return 6\n");
251 log_verbose("next (word = %d\n", (int32_t)pos
);
254 log_err("error ubrk_last(word) did not return 49\n");
255 log_verbose("last (word = %d\n", (int32_t)end
);
257 pos
=ubrk_previous(word
);
258 log_verbose("%d %d\n", end
, pos
);
260 pos
=ubrk_previous(word
);
261 log_verbose("%d \n", pos
);
263 if (ubrk_isBoundary(word
, 2) != FALSE
) {
264 log_err("error ubrk_isBoundary(word, 2) did not return FALSE\n");
266 pos
=ubrk_current(word
);
268 log_err("error ubrk_current() != 4 after ubrk_isBoundary(word, 2)\n");
270 if (ubrk_isBoundary(word
, 4) != TRUE
) {
271 log_err("error ubrk_isBoundary(word, 4) did not return TRUE\n");
276 log_verbose("\nTesting the functions for character\n");
277 ubrk_first(character
);
278 pos
= ubrk_following(character
, 5);
280 log_err("error ubrk_following(character,5) did not return 6\n");
281 log_verbose("Following (character,5) = %d\n", (int32_t)pos
);
282 pos
=ubrk_following(character
, 18);
284 log_err("error ubrk_following(character,18) did not return 19\n");
285 log_verbose("Followingcharacter,18) = %d\n", (int32_t)pos
);
286 pos
=ubrk_preceding(character
, 22);
288 log_err("error ubrk_preceding(character,22) did not return 21\n");
289 log_verbose("preceding(character,22) = %d\n", (int32_t)pos
);
292 log_verbose("\nTesting the functions for line\n");
293 pos
=ubrk_first(line
);
295 log_err("error ubrk_first(line) returned %d, expected 0\n", (int32_t)pos
);
296 pos
= ubrk_next(line
);
297 pos
=ubrk_following(line
, 18);
299 log_err("error ubrk_following(line) did not return 22\n");
300 log_verbose("following (line) = %d\n", (int32_t)pos
);
303 log_verbose("\nTesting the functions for sentence\n");
304 ubrk_first(sentence
);
305 pos
= ubrk_current(sentence
);
306 log_verbose("Current(sentence) = %d\n", (int32_t)pos
);
307 pos
= ubrk_last(sentence
);
309 log_err("error ubrk_last for sentence did not return 49\n");
310 log_verbose("Last (sentence) = %d\n", (int32_t)pos
);
311 ubrk_first(sentence
);
312 to
= ubrk_following( sentence
, 0 );
313 if (to
== 0) log_err("ubrk_following returned 0\n");
314 to
= ubrk_preceding( sentence
, to
);
315 if (to
!= 0) log_err("ubrk_preceding didn't return 0\n");
316 if (ubrk_first(sentence
)!=ubrk_current(sentence
)) {
317 log_err("error in ubrk_first() or ubrk_current()\n");
322 /*Testing ubrk_open and ubrk_close()*/
323 log_verbose("\nTesting open and close for us locale\n");
324 b
= ubrk_open(UBRK_WORD
, "fr_FR", text
, u_strlen(text
), &status
);
325 if (U_FAILURE(status
)) {
326 log_err("ubrk_open for word returned NULL: %s\n", myErrorName(status
));
330 /* Test setText and setUText */
332 UChar s1
[] = {0x41, 0x42, 0x20, 0};
333 UChar s2
[] = {0x41, 0x42, 0x43, 0x44, 0x45, 0};
338 log_verbose("\nTesting ubrk_setText() and ubrk_setUText()\n");
339 status
= U_ZERO_ERROR
;
340 bb
= ubrk_open(UBRK_WORD
, "en_US", NULL
, 0, &status
);
341 TEST_ASSERT_SUCCESS(status
);
342 ubrk_setText(bb
, s1
, -1, &status
);
343 TEST_ASSERT_SUCCESS(status
);
347 ut
= utext_openUChars(ut
, s2
, -1, &status
);
348 ubrk_setUText(bb
, ut
, &status
);
349 TEST_ASSERT_SUCCESS(status
);
358 ubrk_close(sentence
);
360 ubrk_close(character
);
363 static void TestBreakIteratorSafeClone(void)
365 UChar text
[51]; /* Keep this odd to test for 64-bit memory alignment */
366 /* NOTE: This doesn't reliably force mis-alignment of following items. */
367 uint8_t buffer
[CLONETEST_ITERATOR_COUNT
] [U_BRK_SAFECLONE_BUFFERSIZE
];
368 int32_t bufferSize
= U_BRK_SAFECLONE_BUFFERSIZE
;
370 UBreakIterator
* someIterators
[CLONETEST_ITERATOR_COUNT
];
371 UBreakIterator
* someClonedIterators
[CLONETEST_ITERATOR_COUNT
];
373 UBreakIterator
* brk
;
374 UErrorCode status
= U_ZERO_ERROR
;
378 /*Testing ubrk_safeClone */
380 /* Note: the adjacent "" are concatenating strings, not adding a \" to the
381 string, which is probably what whoever wrote this intended. Don't fix,
382 because it would throw off the hard coded break positions in the following
384 u_uastrcpy(text
, "He's from Africa. ""Mr. Livingston, I presume?"" Yeah");
386 /* US & Thai - rule-based & dictionary based */
387 someIterators
[0] = ubrk_open(UBRK_WORD
, "en_US", text
, u_strlen(text
), &status
);
388 if(!someIterators
[0] || U_FAILURE(status
)) {
389 log_data_err("Couldn't open en_US word break iterator - %s\n", u_errorName(status
));
393 someIterators
[1] = ubrk_open(UBRK_WORD
, "th_TH", text
, u_strlen(text
), &status
);
394 if(!someIterators
[1] || U_FAILURE(status
)) {
395 log_data_err("Couldn't open th_TH word break iterator - %s\n", u_errorName(status
));
399 /* test each type of iterator */
400 for (i
= 0; i
< CLONETEST_ITERATOR_COUNT
; i
++)
403 /* Check the various error & informational states */
405 /* Null status - just returns NULL */
406 if (NULL
!= ubrk_safeClone(someIterators
[i
], buffer
[i
], &bufferSize
, NULL
))
408 log_err("FAIL: Cloned Iterator failed to deal correctly with null status\n");
410 /* error status - should return 0 & keep error the same */
411 status
= U_MEMORY_ALLOCATION_ERROR
;
412 if (NULL
!= ubrk_safeClone(someIterators
[i
], buffer
[i
], &bufferSize
, &status
) || status
!= U_MEMORY_ALLOCATION_ERROR
)
414 log_err("FAIL: Cloned Iterator failed to deal correctly with incoming error status\n");
416 status
= U_ZERO_ERROR
;
418 /* Null buffer size pointer is ok */
419 if (NULL
== (brk
= ubrk_safeClone(someIterators
[i
], buffer
[i
], NULL
, &status
)) || U_FAILURE(status
))
421 log_err("FAIL: Cloned Iterator failed to deal correctly with null bufferSize pointer\n");
424 status
= U_ZERO_ERROR
;
426 /* buffer size pointer is 0 - fill in pbufferSize with a size */
428 if (NULL
!= ubrk_safeClone(someIterators
[i
], buffer
[i
], &bufferSize
, &status
) ||
429 U_FAILURE(status
) || bufferSize
<= 0)
431 log_err("FAIL: Cloned Iterator failed a sizing request ('preflighting')\n");
433 /* Verify our define is large enough */
434 if (U_BRK_SAFECLONE_BUFFERSIZE
< bufferSize
)
436 log_err("FAIL: Pre-calculated buffer size is too small - %d but needed %d\n", U_BRK_SAFECLONE_BUFFERSIZE
, bufferSize
);
438 /* Verify we can use this run-time calculated size */
439 if (NULL
== (brk
= ubrk_safeClone(someIterators
[i
], buffer
[i
], &bufferSize
, &status
)) || U_FAILURE(status
))
441 log_err("FAIL: Iterator can't be cloned with run-time size\n");
445 /* size one byte too small - should allocate & let us know */
446 if (bufferSize
> 1) {
449 if (NULL
== (brk
= ubrk_safeClone(someIterators
[i
], NULL
, &bufferSize
, &status
)) || status
!= U_SAFECLONE_ALLOCATED_WARNING
)
451 log_err("FAIL: Cloned Iterator failed to deal correctly with too-small buffer size\n");
455 status
= U_ZERO_ERROR
;
456 bufferSize
= U_BRK_SAFECLONE_BUFFERSIZE
;
458 /* Null buffer pointer - return Iterator & set error to U_SAFECLONE_ALLOCATED_ERROR */
459 if (NULL
== (brk
= ubrk_safeClone(someIterators
[i
], NULL
, &bufferSize
, &status
)) || status
!= U_SAFECLONE_ALLOCATED_WARNING
)
461 log_err("FAIL: Cloned Iterator failed to deal correctly with null buffer pointer\n");
465 status
= U_ZERO_ERROR
;
467 /* Mis-aligned buffer pointer. */
469 char stackBuf
[U_BRK_SAFECLONE_BUFFERSIZE
+sizeof(void *)];
471 brk
= ubrk_safeClone(someIterators
[i
], &stackBuf
[1], &bufferSize
, &status
);
472 if (U_FAILURE(status
) || brk
== NULL
) {
473 log_err("FAIL: Cloned Iterator failed with misaligned buffer pointer\n");
475 if (status
== U_SAFECLONE_ALLOCATED_WARNING
) {
476 log_verbose("Cloned Iterator allocated when using a mis-aligned buffer.\n");
483 /* Null Iterator - return NULL & set U_ILLEGAL_ARGUMENT_ERROR */
484 if (NULL
!= ubrk_safeClone(NULL
, buffer
[i
], &bufferSize
, &status
) || status
!= U_ILLEGAL_ARGUMENT_ERROR
)
486 log_err("FAIL: Cloned Iterator failed to deal correctly with null Iterator pointer\n");
488 status
= U_ZERO_ERROR
;
490 /* Do these cloned Iterators work at all - make a first & next call */
491 bufferSize
= U_BRK_SAFECLONE_BUFFERSIZE
;
492 someClonedIterators
[i
] = ubrk_safeClone(someIterators
[i
], buffer
[i
], &bufferSize
, &status
);
494 start
= ubrk_first(someClonedIterators
[i
]);
496 log_err("error ubrk_start(clone) did not return 0\n");
497 pos
=ubrk_next(someClonedIterators
[i
]);
499 log_err("error ubrk_next(clone) did not return 4\n");
501 ubrk_close(someClonedIterators
[i
]);
502 ubrk_close(someIterators
[i
]);
509 // Open a break iterator from char * rules. Take care of conversion
510 // of the rules and error checking.
512 static UBreakIterator
* testOpenRules(char *rules
) {
513 UErrorCode status
= U_ZERO_ERROR
;
514 UChar
*ruleSourceU
= NULL
;
515 void *strCleanUp
= NULL
;
516 UParseError parseErr
;
519 ruleSourceU
= toUChar(rules
, &strCleanUp
);
521 bi
= ubrk_openRules(ruleSourceU
, -1, /* The rules */
522 NULL
, -1, /* The text to be iterated over. */
525 if (U_FAILURE(status
)) {
526 log_data_err("FAIL: ubrk_openRules: ICU Error \"%s\" (Are you missing data?)\n", u_errorName(status
));
529 freeToUCharStrings(&strCleanUp
);
535 * TestBreakIteratorRules - Verify that a break iterator can be created from
536 * a set of source rules.
538 static void TestBreakIteratorRules() {
539 /* Rules will keep together any run of letters not including 'a', OR
540 * keep together 'abc', but only when followed by 'def', OTHERWISE
541 * just return one char at a time.
543 char rules
[] = "abc{666}/def;\n [\\p{L} - [a]]* {2}; . {1};";
544 /* 0123456789012345678 */
545 char data
[] = "abcdex abcdefgh-def"; /* the test data string */
546 char breaks
[] = "** ** * ** *"; /* * the expected break positions */
547 char tags
[] = "01 21 6 21 2"; /* expected tag values at break positions */
548 int32_t tagMap
[] = {0, 1, 2, 3, 4, 5, 666};
551 void *freeHook
= NULL
;
552 UErrorCode status
= U_ZERO_ERROR
;
556 UBreakIterator
*bi
= testOpenRules(rules
);
557 if (bi
== NULL
) {return;}
558 uData
= toUChar(data
, &freeHook
);
559 ubrk_setText(bi
, uData
, -1, &status
);
561 pos
= ubrk_first(bi
);
562 for (i
=0; i
<sizeof(breaks
); i
++) {
563 if (pos
== i
&& breaks
[i
] != '*') {
564 log_err("FAIL: unexpected break at position %d found\n", pos
);
567 if (pos
!= i
&& breaks
[i
] == '*') {
568 log_err("FAIL: expected break at position %d not found.\n", i
);
572 int32_t tag
, expectedTag
;
573 tag
= ubrk_getRuleStatus(bi
);
574 expectedTag
= tagMap
[tags
[i
]&0xf];
575 if (tag
!= expectedTag
) {
576 log_err("FAIL: incorrect tag value. Position = %d; expected tag %d, got %d",
577 pos
, expectedTag
, tag
);
584 freeToUCharStrings(&freeHook
);
588 static void TestBreakIteratorRuleError() {
590 * TestBreakIteratorRuleError - Try to create a BI from rules with syntax errors,
591 * check that the error is reported correctly.
593 char rules
[] = " # This is a rule comment on line 1\n"
594 "[:L:]; # this rule is OK.\n"
595 "abcdefg); # Error, mismatched parens\n";
597 void *freeHook
= NULL
;
598 UErrorCode status
= U_ZERO_ERROR
;
599 UParseError parseErr
;
602 uRules
= toUChar(rules
, &freeHook
);
603 bi
= ubrk_openRules(uRules
, -1, /* The rules */
604 NULL
, -1, /* The text to be iterated over. */
606 if (U_SUCCESS(status
)) {
607 log_err("FAIL: construction of break iterator succeeded when it should have failed.\n");
610 if (parseErr
.line
!= 3 || parseErr
.offset
!= 8) {
611 log_data_err("FAIL: incorrect error position reported. Got line %d, char %d, expected line 3, char 7 (Are you missing data?)\n",
612 parseErr
.line
, parseErr
.offset
);
615 freeToUCharStrings(&freeHook
);
620 * TestsBreakIteratorStatusVals() Test the ubrk_getRuleStatusVec() funciton
622 static void TestBreakIteratorStatusVec() {
623 #define RULE_STRING_LENGTH 200
624 UChar rules
[RULE_STRING_LENGTH
];
626 #define TEST_STRING_LENGTH 25
627 UChar testString
[TEST_STRING_LENGTH
];
628 UBreakIterator
*bi
= NULL
;
632 UErrorCode status
= U_ZERO_ERROR
;
634 u_uastrncpy(rules
, "[A-N]{100}; \n"
639 "!.*;\n", RULE_STRING_LENGTH
);
640 u_uastrncpy(testString
, "ABC", TEST_STRING_LENGTH
);
643 bi
= ubrk_openRules(rules
, -1, testString
, -1, NULL
, &status
);
644 TEST_ASSERT_SUCCESS(status
);
645 TEST_ASSERT(bi
!= NULL
);
647 /* The TEST_ASSERT above should change too... */
650 TEST_ASSERT(pos
== 1);
652 memset(vals
, -1, sizeof(vals
));
653 numVals
= ubrk_getRuleStatusVec(bi
, vals
, 10, &status
);
654 TEST_ASSERT_SUCCESS(status
);
655 TEST_ASSERT(numVals
== 2);
656 TEST_ASSERT(vals
[0] == 100);
657 TEST_ASSERT(vals
[1] == 300);
658 TEST_ASSERT(vals
[2] == -1);
660 numVals
= ubrk_getRuleStatusVec(bi
, vals
, 0, &status
);
661 TEST_ASSERT(status
== U_BUFFER_OVERFLOW_ERROR
);
662 TEST_ASSERT(numVals
== 2);
670 * static void TestBreakIteratorUText(void);
672 * Test that ubrk_setUText() is present and works for a simple case.
674 static void TestBreakIteratorUText(void) {
675 const char *UTF8Str
= "\x41\xc3\x85\x5A\x20\x41\x52\x69\x6E\x67"; /* c3 85 is utf-8 for A with a ring on top */
678 UErrorCode status
= U_ZERO_ERROR
;
679 UBreakIterator
*bi
= NULL
;
683 UText
*ut
= utext_openUTF8(NULL
, UTF8Str
, -1, &status
);
684 TEST_ASSERT_SUCCESS(status
);
686 bi
= ubrk_open(UBRK_WORD
, "en_US", NULL
, 0, &status
);
687 if (U_FAILURE(status
)) {
688 log_err_status(status
, "Failure at file %s, line %d, error = %s\n", __FILE__
, __LINE__
, u_errorName(status
));
692 ubrk_setUText(bi
, ut
, &status
);
693 if (U_FAILURE(status
)) {
694 log_err("Failure at file %s, line %d, error = %s\n", __FILE__
, __LINE__
, u_errorName(status
));
698 pos
= ubrk_first(bi
);
699 TEST_ASSERT(pos
== 0);
702 TEST_ASSERT(pos
== 4);
705 TEST_ASSERT(pos
== 5);
708 TEST_ASSERT(pos
== 10);
711 TEST_ASSERT(pos
== UBRK_DONE
);
717 * static void TestBreakIteratorTailoring(void);
719 * Test break iterator tailorings from CLDR data.
722 /* Thai/Lao grapheme break tailoring */
723 static const UChar thTest
[] = { 0x0020, 0x0E40, 0x0E01, 0x0020,
724 0x0E01, 0x0E30, 0x0020, 0x0E01, 0x0E33, 0x0020, 0 };
725 /*in Unicode 6.1 en should behave just like th for this*/
726 /*static const int32_t thTestOffs_enFwd[] = { 1, 3, 4, 6, 7, 9, 10 };*/
727 static const int32_t thTestOffs_thFwd
[] = { 1, 2, 3, 4, 5, 6, 7, 9, 10 };
728 /*static const int32_t thTestOffs_enRev[] = { 9, 7, 6, 4, 3, 1, 0 };*/
729 static const int32_t thTestOffs_thRev
[] = { 9, 7, 6, 5, 4, 3, 2, 1, 0 };
731 /* Hebrew line break tailoring, for cldrbug 3028 */
732 static const UChar heTest
[] = { 0x0020, 0x002D, 0x0031, 0x0032, 0x0020,
733 0x0061, 0x002D, 0x006B, 0x0020,
734 0x0061, 0x0300, 0x2010, 0x006B, 0x0020,
735 0x05DE, 0x05D4, 0x002D, 0x0069, 0x0020,
736 0x05D1, 0x05BC, 0x2010, 0x0047, 0x0020, 0 };
737 /*in Unicode 6.1 en should behave just like he for this*/
738 /*static const int32_t heTestOffs_enFwd[] = { 1, 5, 7, 9, 12, 14, 17, 19, 22, 24 };*/
739 static const int32_t heTestOffs_heFwd
[] = { 1, 5, 7, 9, 12, 14, 19, 24 };
740 /*static const int32_t heTestOffs_enRev[] = { 22, 19, 17, 14, 12, 9, 7, 5, 1, 0 };*/
741 static const int32_t heTestOffs_heRev
[] = { 19, 14, 12, 9, 7, 5, 1, 0 };
743 /* Finnish line break tailoring, for cldrbug 3029 */
744 static const UChar fiTest
[] = { /* 00 */ 0x0020, 0x002D, 0x0031, 0x0032, 0x0020,
745 /* 05 */ 0x0061, 0x002D, 0x006B, 0x0020,
746 /* 09 */ 0x0061, 0x0300, 0x2010, 0x006B, 0x0020,
747 /* 14 */ 0x0061, 0x0020, 0x002D, 0x006B, 0x0020,
748 /* 19 */ 0x0061, 0x0300, 0x0020, 0x2010, 0x006B, 0x0020, 0 };
749 static const int32_t fiTestOffs_enFwd
[] = { 1, 5, 7, 9, 12, 14, 16, 17, 19, 22, 23, 25 };
750 static const int32_t fiTestOffs_fiFwd
[] = { 1, 5, 7, 9, 12, 14, 16, 19, 22, 25 };
751 static const int32_t fiTestOffs_enRev
[] = { 23, 22, 19, 17, 16, 14, 12, 9, 7, 5, 1, 0 };
752 static const int32_t fiTestOffs_fiRev
[] = { 22, 19, 16, 14, 12, 9, 7, 5, 1, 0 };
754 /* Khmer dictionary-based work break, for ICU ticket #8329 */
755 static const UChar kmTest
[] = { /* 00 */ 0x179F, 0x17BC, 0x1798, 0x1785, 0x17C6, 0x178E, 0x17B6, 0x1799, 0x1796, 0x17C1,
756 /* 10 */ 0x179B, 0x1794, 0x1793, 0x17D2, 0x178F, 0x17B7, 0x1785, 0x178A, 0x17BE, 0x1798,
757 /* 20 */ 0x17D2, 0x1794, 0x17B8, 0x17A2, 0x1792, 0x17B7, 0x179F, 0x17D2, 0x178B, 0x17B6,
758 /* 30 */ 0x1793, 0x17A2, 0x179A, 0x1796, 0x17D2, 0x179A, 0x17C7, 0x1782, 0x17BB, 0x178E,
759 /* 40 */ 0x178A, 0x179B, 0x17CB, 0x1796, 0x17D2, 0x179A, 0x17C7, 0x17A2, 0x1784, 0x17D2,
760 /* 50 */ 0x1782, 0 };
761 static const int32_t kmTestOffs_kmFwd
[] = { 3, /*8,*/ 11, 17, 23, 31, /*33,*/ 40, 43, 51 }; /* TODO: Investigate failure to break at offset 8 */
762 static const int32_t kmTestOffs_kmRev
[] = { 43, 40, /*33,*/ 31, 23, 17, 11, /*8,*/ 3, 0 };
766 UBreakIteratorType type
;
768 const int32_t * offsFwd
;
769 const int32_t * offsRev
;
773 static const RBBITailoringTest tailoringTests
[] = {
774 { "en", UBRK_CHARACTER
, thTest
, thTestOffs_thFwd
, thTestOffs_thRev
, sizeof(thTestOffs_thFwd
)/sizeof(thTestOffs_thFwd
[0]) },
775 { "en_US_POSIX", UBRK_CHARACTER
, thTest
, thTestOffs_thFwd
, thTestOffs_thRev
, sizeof(thTestOffs_thFwd
)/sizeof(thTestOffs_thFwd
[0]) },
776 { "en", UBRK_LINE
, heTest
, heTestOffs_heFwd
, heTestOffs_heRev
, sizeof(heTestOffs_heFwd
)/sizeof(heTestOffs_heFwd
[0]) },
777 { "he", UBRK_LINE
, heTest
, heTestOffs_heFwd
, heTestOffs_heRev
, sizeof(heTestOffs_heFwd
)/sizeof(heTestOffs_heFwd
[0]) },
778 { "en", UBRK_LINE
, fiTest
, fiTestOffs_enFwd
, fiTestOffs_enRev
, sizeof(fiTestOffs_enFwd
)/sizeof(fiTestOffs_enFwd
[0]) },
779 { "fi", UBRK_LINE
, fiTest
, fiTestOffs_fiFwd
, fiTestOffs_fiRev
, sizeof(fiTestOffs_fiFwd
)/sizeof(fiTestOffs_fiFwd
[0]) },
780 { "km", UBRK_WORD
, kmTest
, kmTestOffs_kmFwd
, kmTestOffs_kmRev
, sizeof(kmTestOffs_kmFwd
)/sizeof(kmTestOffs_kmFwd
[0]) },
781 { NULL
, 0, NULL
, NULL
, NULL
, 0 },
784 static void TestBreakIteratorTailoring(void) {
785 const RBBITailoringTest
* testPtr
;
786 for (testPtr
= tailoringTests
; testPtr
->locale
!= NULL
; ++testPtr
) {
787 UErrorCode status
= U_ZERO_ERROR
;
788 UBreakIterator
* ubrkiter
= ubrk_open(testPtr
->type
, testPtr
->locale
, testPtr
->test
, -1, &status
);
789 if ( U_SUCCESS(status
) ) {
790 int32_t offset
, offsindx
;
794 for (offsindx
= 0; (offset
= ubrk_next(ubrkiter
)) != UBRK_DONE
; ++offsindx
) {
795 if (!foundError
&& offsindx
>= testPtr
->numOffsets
) {
796 log_err("FAIL: locale %s, break type %d, ubrk_next expected UBRK_DONE, got %d\n",
797 testPtr
->locale
, testPtr
->type
, offset
);
799 } else if (!foundError
&& offset
!= testPtr
->offsFwd
[offsindx
]) {
800 log_err("FAIL: locale %s, break type %d, ubrk_next expected %d, got %d\n",
801 testPtr
->locale
, testPtr
->type
, testPtr
->offsFwd
[offsindx
], offset
);
805 if (!foundError
&& offsindx
< testPtr
->numOffsets
) {
806 log_err("FAIL: locale %s, break type %d, ubrk_next expected %d, got UBRK_DONE\n",
807 testPtr
->locale
, testPtr
->type
, testPtr
->offsFwd
[offsindx
]);
811 for (offsindx
= 0; (offset
= ubrk_previous(ubrkiter
)) != UBRK_DONE
; ++offsindx
) {
812 if (!foundError
&& offsindx
>= testPtr
->numOffsets
) {
813 log_err("FAIL: locale %s, break type %d, ubrk_previous expected UBRK_DONE, got %d\n",
814 testPtr
->locale
, testPtr
->type
, offset
);
816 } else if (!foundError
&& offset
!= testPtr
->offsRev
[offsindx
]) {
817 log_err("FAIL: locale %s, break type %d, ubrk_previous expected %d, got %d\n",
818 testPtr
->locale
, testPtr
->type
, testPtr
->offsRev
[offsindx
], offset
);
822 if (!foundError
&& offsindx
< testPtr
->numOffsets
) {
823 log_err("FAIL: locale %s, break type %d, ubrk_previous expected %d, got UBRK_DONE\n",
824 testPtr
->locale
, testPtr
->type
, testPtr
->offsRev
[offsindx
]);
827 ubrk_close(ubrkiter
);
829 log_err_status(status
, "FAIL: locale %s, break type %d, ubrk_open status: %s\n", testPtr
->locale
, testPtr
->type
, u_errorName(status
));
835 static void TestBreakIteratorRefresh(void) {
837 * RefreshInput changes out the input of a Break Iterator without
838 * changing anything else in the iterator's state. Used with Java JNI,
839 * when Java moves the underlying string storage. This test
840 * runs a ubrk_next() repeatedly, moving the text in the middle of the sequence.
841 * The right set of boundaries should still be found.
843 UChar testStr
[] = {0x20, 0x41, 0x20, 0x42, 0x20, 0x43, 0x20, 0x44, 0x0}; /* = " A B C D" */
844 UChar movedStr
[] = {0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0};
845 UErrorCode status
= U_ZERO_ERROR
;
847 UText ut1
= UTEXT_INITIALIZER
;
848 UText ut2
= UTEXT_INITIALIZER
;
850 bi
= ubrk_open(UBRK_LINE
, "en_US", NULL
, 0, &status
);
851 TEST_ASSERT_SUCCESS(status
);
853 utext_openUChars(&ut1
, testStr
, -1, &status
);
854 TEST_ASSERT_SUCCESS(status
);
855 ubrk_setUText(bi
, &ut1
, &status
);
856 TEST_ASSERT_SUCCESS(status
);
858 if (U_SUCCESS(status
)) {
859 /* Line boundaries will occur before each letter in the original string */
860 TEST_ASSERT(1 == ubrk_next(bi
));
861 TEST_ASSERT(3 == ubrk_next(bi
));
863 /* Move the string, kill the original string. */
864 u_strcpy(movedStr
, testStr
);
865 u_memset(testStr
, 0x20, u_strlen(testStr
));
866 utext_openUChars(&ut2
, movedStr
, -1, &status
);
867 TEST_ASSERT_SUCCESS(status
);
868 ubrk_refreshUText(bi
, &ut2
, &status
);
869 TEST_ASSERT_SUCCESS(status
);
871 /* Find the following matches, now working in the moved string. */
872 TEST_ASSERT(5 == ubrk_next(bi
));
873 TEST_ASSERT(7 == ubrk_next(bi
));
874 TEST_ASSERT(8 == ubrk_next(bi
));
875 TEST_ASSERT(UBRK_DONE
== ubrk_next(bi
));
876 TEST_ASSERT_SUCCESS(status
);
884 static const char testSentenceSuppressionsEn
[] = "Mr. Jones comes home. Dr. Smith Ph.D. is out. In the U.S.A. it is hot.";
885 static const int32_t testSentSuppFwdOffsetsEn
[] = { 22, 46, 70, -1 }; /* With suppressions */
886 static const int32_t testSentFwdOffsetsEn
[] = { 4, 22, 26, 35, 46, 70, -1 }; /* Without suppressions */
887 static const int32_t testSentSuppRevOffsetsEn
[] = { 46, 22, 0, -1 }; /* With suppressions */
888 static const int32_t testSentRevOffsetsEn
[] = { 46, 35, 26, 22, 4, 0, -1 }; /* Without suppressions */
890 static const char testSentenceSuppressionsDe
[] = "Wenn ich schon h\\u00F6re zu Guttenberg kommt evtl. zur\\u00FCck.";
891 // "Wenn ich schon höre zu Guttenberg kommt evtl. zurück."
892 static const int32_t testSentSuppFwdOffsetsDe
[] = { 53, -1 }; /* With suppressions */
893 static const int32_t testSentFwdOffsetsDe
[] = { 53, -1 }; /* Without suppressions; no break in evtl. zur due to casing */
894 static const int32_t testSentSuppRevOffsetsDe
[] = { 0, -1 }; /* With suppressions */
895 static const int32_t testSentRevOffsetsDe
[] = { 0, -1 }; /* Without suppressions */
897 static const char testSentenceSuppressionsEs
[] = "Te esperamos todos los miercoles en Bravo 416, Col. El Pueblo a las 7 PM.";
898 static const int32_t testSentSuppFwdOffsetsEs
[] = { 73, -1 }; /* With suppressions */
899 static const int32_t testSentFwdOffsetsEs
[] = { 52, 73, -1 }; /* Without suppressions */
900 static const int32_t testSentSuppRevOffsetsEs
[] = { 0, -1 }; /* With suppressions */
901 static const int32_t testSentRevOffsetsEs
[] = { 52, 0, -1 }; /* Without suppressions */
903 enum { kTextULenMax
= 128 };
908 const int32_t * expFwdOffsets
;
909 const int32_t * expRevOffsets
;
910 } TestBISuppressionsItem
;
912 static const TestBISuppressionsItem testBISuppressionsItems
[] = {
913 { "en@ss=standard", testSentenceSuppressionsEn
, testSentSuppFwdOffsetsEn
, testSentSuppRevOffsetsEn
},
914 { "en", testSentenceSuppressionsEn
, testSentFwdOffsetsEn
, testSentRevOffsetsEn
},
915 { "fr@ss=standard", testSentenceSuppressionsEn
, testSentFwdOffsetsEn
, testSentRevOffsetsEn
},
916 { "af@ss=standard", testSentenceSuppressionsEn
, testSentSuppFwdOffsetsEn
, testSentSuppRevOffsetsEn
}, /* no brkiter data => en suppressions? */
917 { "zh@ss=standard", testSentenceSuppressionsEn
, testSentFwdOffsetsEn
, testSentRevOffsetsEn
}, /* brkiter data, no suppressions data => no suppressions */
918 { "zh_Hant@ss=standard", testSentenceSuppressionsEn
, testSentFwdOffsetsEn
, testSentRevOffsetsEn
}, /* brkiter data, no suppressions data => no suppressions */
919 { "fi@ss=standard", testSentenceSuppressionsEn
, testSentFwdOffsetsEn
, testSentRevOffsetsEn
}, /* brkiter data, no suppressions data => no suppressions */
920 { "ja@ss=standard", testSentenceSuppressionsEn
, testSentFwdOffsetsEn
, testSentRevOffsetsEn
}, /* brkiter data, no suppressions data => no suppressions */
921 { "de@ss=standard", testSentenceSuppressionsDe
, testSentSuppFwdOffsetsDe
, testSentSuppRevOffsetsDe
},
922 { "de", testSentenceSuppressionsDe
, testSentFwdOffsetsDe
, testSentRevOffsetsDe
},
923 { "es@ss=standard", testSentenceSuppressionsEs
, testSentSuppFwdOffsetsEs
, testSentSuppRevOffsetsEs
},
924 { "es", testSentenceSuppressionsEs
, testSentFwdOffsetsEs
, testSentRevOffsetsEs
},
928 static void TestBreakIteratorSuppressions(void) {
929 const TestBISuppressionsItem
* itemPtr
;
931 for (itemPtr
= testBISuppressionsItems
; itemPtr
->locale
!= NULL
; itemPtr
++) {
932 UChar textU
[kTextULenMax
];
933 int32_t textULen
= u_unescape(itemPtr
->text
, textU
, kTextULenMax
);
934 UErrorCode status
= U_ZERO_ERROR
;
935 UBreakIterator
*bi
= ubrk_open(UBRK_SENTENCE
, itemPtr
->locale
, textU
, textULen
, &status
);
936 if (U_SUCCESS(status
)) {
937 int32_t offset
, start
;
938 const int32_t * expOffsetPtr
;
940 expOffsetPtr
= itemPtr
->expFwdOffsets
;
942 for (; (offset
= ubrk_next(bi
)) != UBRK_DONE
&& *expOffsetPtr
>= 0; expOffsetPtr
++) {
943 if (offset
!= *expOffsetPtr
) {
944 log_err("FAIL: ubrk_next loc \"%s\", expected %d, got %d\n", itemPtr
->locale
, *expOffsetPtr
, offset
);
947 if (offset
!= UBRK_DONE
|| *expOffsetPtr
>= 0) {
948 log_err("FAIL: ubrk_next loc \"%s\", expected UBRK_DONE & expOffset -1, got %d and %d\n", itemPtr
->locale
, offset
, *expOffsetPtr
);
951 expOffsetPtr
= itemPtr
->expFwdOffsets
;
952 start
= ubrk_first(bi
) + 1;
953 for (; (offset
= ubrk_following(bi
, start
)) != UBRK_DONE
&& *expOffsetPtr
>= 0; expOffsetPtr
++) {
954 if (offset
!= *expOffsetPtr
) {
955 log_err("FAIL: ubrk_following(%d) loc \"%s\", expected %d, got %d\n", start
, itemPtr
->locale
, *expOffsetPtr
, offset
);
957 start
= *expOffsetPtr
+ 1;
959 if (offset
!= UBRK_DONE
|| *expOffsetPtr
>= 0) {
960 log_err("FAIL: ubrk_following(%d) loc \"%s\", expected UBRK_DONE & expOffset -1, got %d and %d\n", start
, itemPtr
->locale
, offset
, *expOffsetPtr
);
963 expOffsetPtr
= itemPtr
->expRevOffsets
;
965 for (; (offset
= ubrk_previous(bi
)) != UBRK_DONE
&& *expOffsetPtr
>= 0; expOffsetPtr
++) {
966 if (offset
!= *expOffsetPtr
) {
967 log_err("FAIL: ubrk_previous loc \"%s\", expected %d, got %d\n", itemPtr
->locale
, *expOffsetPtr
, offset
);
970 if (offset
!= UBRK_DONE
|| *expOffsetPtr
>= 0) {
971 log_err("FAIL: ubrk_previous loc \"%s\", expected UBRK_DONE & expOffset -1, got %d and %d\n", itemPtr
->locale
, offset
, *expOffsetPtr
);
974 expOffsetPtr
= itemPtr
->expRevOffsets
;
975 start
= ubrk_last(bi
) - 1;
976 for (; (offset
= ubrk_preceding(bi
, start
)) != UBRK_DONE
&& *expOffsetPtr
>= 0; expOffsetPtr
++) {
977 if (offset
!= *expOffsetPtr
) {
978 log_err("FAIL: ubrk_preceding(%d) loc \"%s\", expected %d, got %d\n", start
, itemPtr
->locale
, *expOffsetPtr
, offset
);
980 start
= *expOffsetPtr
- 1;
982 if (start
>=0 && (offset
!= UBRK_DONE
|| *expOffsetPtr
>= 0)) {
983 log_err("FAIL: ubrk_preceding loc(%d) \"%s\", expected UBRK_DONE & expOffset -1, got %d and %d\n", start
, itemPtr
->locale
, offset
, *expOffsetPtr
);
988 log_data_err("FAIL: ubrk_open(UBRK_SENTENCE, \"%s\", ...) status %s (Are you missing data?)\n", itemPtr
->locale
, u_errorName(status
));
993 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */