2 *******************************************************************************
3 * Copyright (C) 1997-2009, International Business Machines
4 * Corporation and others. All Rights Reserved.
5 *******************************************************************************
6 * Date Name Description
7 * 06/23/00 aliu Creation.
8 *******************************************************************************
11 #include "unicode/utypes.h"
13 #if !UCONFIG_NO_TRANSLITERATION
17 #include "unicode/utrans.h"
18 #include "unicode/ustring.h"
21 #define TEST(x) addTest(root, &x, "utrans/" # x)
23 static void TestAPI(void);
24 static void TestSimpleRules(void);
25 static void TestFilter(void);
26 static void TestOpenInverse(void);
27 static void TestClone(void);
28 static void TestRegisterUnregister(void);
29 static void TestExtractBetween(void);
30 static void TestUnicodeIDs(void);
32 static void _expectRules(const char*, const char*, const char*);
33 static void _expect(const UTransliterator
* trans
, const char* cfrom
, const char* cto
);
35 void addUTransTest(TestNode
** root
);
39 addUTransTest(TestNode
** root
) {
41 TEST(TestSimpleRules
);
43 TEST(TestOpenInverse
);
45 TEST(TestRegisterUnregister
);
46 TEST(TestExtractBetween
);
50 /*------------------------------------------------------------------
53 * To test the Replaceable glue we have to dummy up a C-based
54 * Replaceable callback. This code is for testing purposes only.
55 *------------------------------------------------------------------*/
57 typedef struct XReplaceable
{
58 UChar
* text
; /* MUST BE null-terminated */
61 static void InitXReplaceable(XReplaceable
* rep
, const char* cstring
) {
62 rep
->text
= malloc(sizeof(UChar
) * (strlen(cstring
)+1));
63 u_uastrcpy(rep
->text
, cstring
);
66 static void FreeXReplaceable(XReplaceable
* rep
) {
67 if (rep
->text
!= NULL
) {
73 /* UReplaceableCallbacks callback */
74 static int32_t Xlength(const UReplaceable
* rep
) {
75 const XReplaceable
* x
= (const XReplaceable
*)rep
;
76 return u_strlen(x
->text
);
79 /* UReplaceableCallbacks callback */
80 static UChar
XcharAt(const UReplaceable
* rep
, int32_t offset
) {
81 const XReplaceable
* x
= (const XReplaceable
*)rep
;
82 return x
->text
[offset
];
85 /* UReplaceableCallbacks callback */
86 static UChar32
Xchar32At(const UReplaceable
* rep
, int32_t offset
) {
87 const XReplaceable
* x
= (const XReplaceable
*)rep
;
88 return x
->text
[offset
];
91 /* UReplaceableCallbacks callback */
92 static void Xreplace(UReplaceable
* rep
, int32_t start
, int32_t limit
,
93 const UChar
* text
, int32_t textLength
) {
94 XReplaceable
* x
= (XReplaceable
*)rep
;
95 int32_t newLen
= Xlength(rep
) + limit
- start
+ textLength
;
96 UChar
* newText
= (UChar
*) malloc(sizeof(UChar
) * (newLen
+1));
97 u_strncpy(newText
, x
->text
, start
);
98 u_strncpy(newText
+ start
, text
, textLength
);
99 u_strcpy(newText
+ start
+ textLength
, x
->text
+ limit
);
104 /* UReplaceableCallbacks callback */
105 static void Xcopy(UReplaceable
* rep
, int32_t start
, int32_t limit
, int32_t dest
) {
106 XReplaceable
* x
= (XReplaceable
*)rep
;
107 int32_t newLen
= Xlength(rep
) + limit
- start
;
108 UChar
* newText
= (UChar
*) malloc(sizeof(UChar
) * (newLen
+1));
109 u_strncpy(newText
, x
->text
, dest
);
110 u_strncpy(newText
+ dest
, x
->text
+ start
, limit
- start
);
111 u_strcpy(newText
+ dest
+ limit
- start
, x
->text
+ dest
);
116 /* UReplaceableCallbacks callback */
117 static void Xextract(UReplaceable
* rep
, int32_t start
, int32_t limit
, UChar
* dst
) {
118 XReplaceable
* x
= (XReplaceable
*)rep
;
119 int32_t len
= limit
- start
;
120 u_strncpy(dst
, x
->text
, len
);
123 static void InitXReplaceableCallbacks(UReplaceableCallbacks
* callbacks
) {
124 callbacks
->length
= Xlength
;
125 callbacks
->charAt
= XcharAt
;
126 callbacks
->char32At
= Xchar32At
;
127 callbacks
->replace
= Xreplace
;
128 callbacks
->extract
= Xextract
;
129 callbacks
->copy
= Xcopy
;
132 /*------------------------------------------------------------------
134 *------------------------------------------------------------------*/
136 static void TestAPI() {
137 enum { BUF_CAP
= 128 };
138 char buf
[BUF_CAP
], buf2
[BUF_CAP
];
139 UErrorCode status
= U_ZERO_ERROR
;
140 UTransliterator
* trans
= NULL
;
143 /* Test getAvailableIDs */
144 n
= utrans_countAvailableIDs();
146 log_err("FAIL: utrans_countAvailableIDs() returned %d\n", n
);
148 log_verbose("System ID count: %d\n", n
);
150 for (i
=0; i
<n
; ++i
) {
151 utrans_getAvailableID(i
, buf
, BUF_CAP
);
153 log_err("FAIL: System transliterator %d: \"\"\n", i
);
155 log_verbose("System transliterator %d: \"%s\"\n", i
, buf
);
160 utrans_getAvailableID(0, buf
, BUF_CAP
);
161 trans
= utrans_open(buf
, UTRANS_FORWARD
,NULL
,0,NULL
, &status
);
162 if (U_FAILURE(status
)) {
163 log_err("FAIL: utrans_open(%s) failed, error=%s\n",
164 buf
, u_errorName(status
));
169 utrans_getID(trans
, buf2
, BUF_CAP
);
170 if (0 != strcmp(buf
, buf2
)) {
171 log_err("FAIL: utrans_getID(%s) returned %s\n",
178 static void TestUnicodeIDs() {
180 UTransliterator
*utrans
;
181 const UChar
*id
, *id2
;
182 int32_t idLength
, id2Length
, count
, count2
;
184 UErrorCode errorCode
;
186 errorCode
=U_ZERO_ERROR
;
187 uenum
=utrans_openIDs(&errorCode
);
188 if(U_FAILURE(errorCode
)) {
189 log_err("utrans_openIDs() failed - %s\n", u_errorName(errorCode
));
193 count
=uenum_count(uenum
, &errorCode
);
194 if(U_FAILURE(errorCode
) || count
<1) {
195 log_err("uenum_count(transliterator IDs)=%d - %s\n", count
, u_errorName(errorCode
));
200 id
=uenum_unext(uenum
, &idLength
, &errorCode
);
201 if(U_FAILURE(errorCode
)) {
202 log_err("uenum_unext(transliterator ID %d) failed - %s\n", count
, u_errorName(errorCode
));
210 /* try to actually open only a few transliterators */
214 utrans
=utrans_openU(id
, idLength
, UTRANS_FORWARD
, NULL
, 0, NULL
, &errorCode
);
215 if(U_FAILURE(errorCode
)) {
216 log_err("utrans_openU(%s) failed - %s\n", aescstrdup(id
, idLength
), u_errorName(errorCode
));
220 id2
=utrans_getUnicodeID(utrans
, &id2Length
);
221 if(idLength
!=id2Length
|| 0!=u_memcmp(id
, id2
, idLength
)) {
222 log_err("utrans_getUnicodeID(%s) does not match the original ID\n", aescstrdup(id
, idLength
));
225 utrans_close(utrans
);
228 uenum_reset(uenum
, &errorCode
);
229 if(U_FAILURE(errorCode
) || count
<1) {
230 log_err("uenum_reset(transliterator IDs) failed - %s\n", u_errorName(errorCode
));
232 count2
=uenum_count(uenum
, &errorCode
);
233 if(U_FAILURE(errorCode
) || count
<1) {
234 log_err("2nd uenum_count(transliterator IDs)=%d - %s\n", count2
, u_errorName(errorCode
));
235 } else if(count
!=count2
) {
236 log_err("uenum_unext(transliterator IDs) returned %d IDs but uenum_count() after uenum_reset() claims there are %d\n", count
, count2
);
243 static void TestOpenInverse(){
244 UErrorCode status
=U_ZERO_ERROR
;
245 UTransliterator
* t1
=NULL
;
246 UTransliterator
* inverse1
=NULL
;
247 enum { BUF_CAP
= 128 };
251 const char TransID
[][25]={
252 "Halfwidth-Fullwidth",
253 "Fullwidth-Halfwidth",
256 /*"Arabic-Latin", // Removed in 2.0*/
257 /*"Latin-Arabic", // Removed in 2.0*/
260 /*"Hebrew-Latin", // Removed in 2.0*/
261 /*"Latin-Hebrew", // Removed in 2.0*/
270 for(i
=0; i
<sizeof(TransID
)/sizeof(TransID
[0]); i
=i
+2){
271 status
= U_ZERO_ERROR
;
272 t1
=utrans_open(TransID
[i
], UTRANS_FORWARD
,NULL
,0,NULL
, &status
);
273 if(t1
== NULL
|| U_FAILURE(status
)){
274 log_data_err("FAIL: in instantiation for id=%s -> %s (Are you missing data?)\n", TransID
[i
], u_errorName(status
));
277 inverse1
=utrans_openInverse(t1
, &status
);
278 if(U_FAILURE(status
)){
279 log_err("FAIL: utrans_openInverse() failed for id=%s. Error=%s\n", TransID
[i
], myErrorName(status
));
282 utrans_getID(inverse1
, buf1
, BUF_CAP
);
283 if(strcmp(buf1
, TransID
[i
+1]) != 0){
284 log_err("FAIL :openInverse() for %s returned %s instead of %s\n", TransID
[i
], buf1
, TransID
[i
+1]);
287 utrans_close(inverse1
);
291 static void TestClone(){
292 UErrorCode status
=U_ZERO_ERROR
;
293 UTransliterator
* t1
=NULL
;
294 UTransliterator
* t2
=NULL
;
295 UTransliterator
* t3
=NULL
;
296 UTransliterator
* t4
=NULL
;
297 enum { BUF_CAP
= 128 };
298 char buf1
[BUF_CAP
], buf2
[BUF_CAP
], buf3
[BUF_CAP
];
300 t1
=utrans_open("Latin-Devanagari", UTRANS_FORWARD
, NULL
,0,NULL
,&status
);
301 if(U_FAILURE(status
)){
302 log_data_err("FAIL: construction -> %s (Are you missing data?)\n", u_errorName(status
));
305 t2
=utrans_open("Latin-Greek", UTRANS_FORWARD
, NULL
,0,NULL
,&status
);
306 if(U_FAILURE(status
)){
307 log_err("FAIL: construction\n");
312 t3
=utrans_clone(t1
, &status
);
313 t4
=utrans_clone(t2
, &status
);
315 utrans_getID(t1
, buf1
, BUF_CAP
);
316 utrans_getID(t2
, buf2
, BUF_CAP
);
317 utrans_getID(t3
, buf3
, BUF_CAP
);
319 if(strcmp(buf1
, buf3
) != 0 ||
320 strcmp(buf1
, buf2
) == 0) {
321 log_err("FAIL: utrans_clone() failed\n");
324 utrans_getID(t4
, buf3
, BUF_CAP
);
326 if(strcmp(buf2
, buf3
) != 0 ||
327 strcmp(buf1
, buf3
) == 0) {
328 log_err("FAIL: utrans_clone() failed\n");
338 static void TestRegisterUnregister(){
339 UErrorCode status
=U_ZERO_ERROR
;
340 UTransliterator
* t1
=NULL
;
341 UTransliterator
* rules
=NULL
, *rules2
;
342 UTransliterator
* inverse1
=NULL
;
343 UChar rule
[]={ 0x0061, 0x003c, 0x003e, 0x0063}; /*a<>b*/
345 U_STRING_DECL(ID
, "TestA-TestB", 11);
346 U_STRING_INIT(ID
, "TestA-TestB", 11);
348 /* Make sure it doesn't exist */
349 t1
=utrans_open("TestA-TestB", UTRANS_FORWARD
,NULL
,0,NULL
, &status
);
350 if(t1
!= NULL
|| U_SUCCESS(status
)) {
351 log_err("FAIL: TestA-TestB already registered\n");
355 /* Check inverse too */
356 inverse1
=utrans_open("TestA-TestB", UTRANS_REVERSE
, NULL
,0,NULL
,&status
);
357 if(inverse1
!= NULL
|| U_SUCCESS(status
)) {
358 log_err("FAIL: TestA-TestB already registered\n");
363 rules
=utrans_open("TestA-TestB",UTRANS_FORWARD
, rule
, 4, NULL
, &status
);
364 if(U_FAILURE(status
)){
365 log_err("FAIL: utrans_openRules(a<>B) failed with error=%s\n", myErrorName(status
));
369 /* clone it so we can register it a second time */
370 rules2
=utrans_clone(rules
, &status
);
371 if(U_FAILURE(status
)) {
372 log_err("FAIL: utrans_clone(a<>B) failed with error=%s\n", myErrorName(status
));
378 utrans_register(rules
, &status
);
379 if(U_FAILURE(status
)){
380 log_err("FAIL: utrans_register failed with error=%s\n", myErrorName(status
));
384 /* Now check again -- should exist now*/
385 t1
= utrans_open("TestA-TestB", UTRANS_FORWARD
, NULL
,0,NULL
,&status
);
386 if(U_FAILURE(status
) || t1
== NULL
){
387 log_err("FAIL: TestA-TestB not registered\n");
392 /*unregister the instance*/
394 utrans_unregister("TestA-TestB");
395 /* now Make sure it doesn't exist */
396 t1
=utrans_open("TestA-TestB", UTRANS_FORWARD
,NULL
,0,NULL
, &status
);
397 if(U_SUCCESS(status
) || t1
!= NULL
) {
398 log_err("FAIL: TestA-TestB isn't unregistered\n");
403 /* now with utrans_unregisterID(const UChar *) */
405 utrans_register(rules2
, &status
);
406 if(U_FAILURE(status
)){
407 log_err("FAIL: 2nd utrans_register failed with error=%s\n", myErrorName(status
));
411 /* Now check again -- should exist now*/
412 t1
= utrans_open("TestA-TestB", UTRANS_FORWARD
, NULL
,0,NULL
,&status
);
413 if(U_FAILURE(status
) || t1
== NULL
){
414 log_err("FAIL: 2nd TestA-TestB not registered\n");
419 /*unregister the instance*/
421 utrans_unregisterID(ID
, -1);
422 /* now Make sure it doesn't exist */
423 t1
=utrans_openU(ID
, -1, UTRANS_FORWARD
,NULL
,0,NULL
, &status
);
424 if(U_SUCCESS(status
) || t1
!= NULL
) {
425 log_err("FAIL: 2nd TestA-TestB isn't unregistered\n");
430 utrans_close(inverse1
);
433 static void TestSimpleRules() {
435 /* Example: rules 1. ab>x|y
438 * []|eabcd start - no match, copy e to tranlated buffer
439 * [e]|abcd match rule 1 - copy output & adjust cursor
440 * [ex|y]cd match rule 2 - copy output & adjust cursor
441 * [exz]|d no match, copy d to transliterated buffer
444 _expectRules("ab>x|y;"
448 /* Another set of rules:
460 _expectRules("ab>x|yzacw;"
468 _expectRules("$dummy=" "\\uE100" ";" /* careful here with E100 */
469 "$vowel=[aeiouAEIOU];"
471 "$vowel } $lu > '!';"
476 "abcdefgABCDEFGU", "&bcd&fg!^**!^*&");
478 /* Test multiple passes
480 _expectRules("abc > xy;"
483 "abc ababc aba", "xy abxy z");
486 static void TestFilter() {
487 UErrorCode status
= U_ZERO_ERROR
;
493 const char* DATA
[] = {
494 "[^c]", /* Filter out 'c' */
496 "\\u0061\\u0062c\\u0064\\u0065",
500 "\\u0061\\u0062\\u0063\\u0064\\u0065"
502 int32_t DATA_length
= sizeof(DATA
) / sizeof(DATA
[0]);
505 UTransliterator
* hex
= utrans_open("Any-Hex", UTRANS_FORWARD
, NULL
,0,NULL
,&status
);
507 if (hex
== 0 || U_FAILURE(status
)) {
508 log_err("FAIL: utrans_open(Unicode-Hex) failed, error=%s\n",
509 u_errorName(status
));
513 for (i
=0; i
<DATA_length
; i
+=3) {
514 /*u_uastrcpy(filt, DATA[i]);*/
515 u_charsToUChars(DATA
[i
], filt
, (int32_t)strlen(DATA
[i
])+1);
516 utrans_setFilter(hex
, filt
, -1, &status
);
518 if (U_FAILURE(status
)) {
519 log_err("FAIL: utrans_setFilter() failed, error=%s\n",
520 u_errorName(status
));
524 /*u_uastrcpy(buf, DATA[i+1]);*/
525 u_charsToUChars(DATA
[i
+1], buf
, (int32_t)strlen(DATA
[i
+1])+1);
527 utrans_transUChars(hex
, buf
, NULL
, 128, 0, &limit
, &status
);
529 if (U_FAILURE(status
)) {
530 log_err("FAIL: utrans_transUChars() failed, error=%s\n",
531 u_errorName(status
));
535 cbuf
=aescstrdup(buf
, -1);
536 u_charsToUChars(DATA
[i
+2], exp
, (int32_t)strlen(DATA
[i
+2])+1);
537 if (0 == u_strcmp(buf
, exp
)) {
538 log_verbose("Ok: %s | %s -> %s\n", DATA
[i
+1], DATA
[i
], cbuf
);
540 log_err("FAIL: %s | %s -> %s, expected %s\n", DATA
[i
+1], DATA
[i
], cbuf
, DATA
[i
+2]);
549 * Test the UReplaceableCallback extractBetween support. We use a
550 * transliterator known to rely on this call.
552 static void TestExtractBetween() {
554 UTransliterator
*trans
;
555 UErrorCode status
= U_ZERO_ERROR
;
556 UParseError parseErr
;
558 trans
= utrans_open("Lower", UTRANS_FORWARD
, NULL
, -1,
561 if (U_FAILURE(status
)) {
562 log_err("FAIL: utrans_open(Lower) failed, error=%s\n",
563 u_errorName(status
));
565 _expect(trans
, "ABC", "abc");
571 static void _expectRules(const char* crules
,
574 /* u_uastrcpy has no capacity param for the buffer -- so just
575 * make all buffers way too big */
578 UTransliterator
*trans
;
579 UErrorCode status
= U_ZERO_ERROR
;
580 UParseError parseErr
;
582 u_uastrcpy(rules
, crules
);
584 trans
= utrans_open(crules
/*use rules as ID*/, UTRANS_FORWARD
, rules
, -1,
586 if (U_FAILURE(status
)) {
588 log_data_err("FAIL: utrans_openRules(%s) failed, error=%s (Are you missing data?)\n",
589 crules
, u_errorName(status
));
593 _expect(trans
, cfrom
, cto
);
598 static void _expect(const UTransliterator
* trans
,
601 /* u_uastrcpy has no capacity param for the buffer -- so just
602 * make all buffers way too big */
611 UErrorCode status
= U_ZERO_ERROR
;
615 XReplaceable
*xrepPtr
= &xrep
;
616 UReplaceableCallbacks xrepVtable
;
618 u_uastrcpy(from
, cfrom
);
621 ID
= utrans_getUnicodeID(trans
, &IDLength
);
622 id
= aescstrdup(ID
, IDLength
);
624 /* utrans_transUChars() */
626 limit
= u_strlen(buf
);
627 utrans_transUChars(trans
, buf
, NULL
, CAP
, 0, &limit
, &status
);
628 if (U_FAILURE(status
)) {
629 log_err("FAIL: utrans_transUChars() failed, error=%s\n",
630 u_errorName(status
));
634 if (0 == u_strcmp(buf
, to
)) {
635 log_verbose("Ok: utrans_transUChars(%s) x %s -> %s\n",
639 u_austrcpy(actual
, buf
);
640 log_err("FAIL: utrans_transUChars(%s) x %s -> %s, expected %s\n",
641 id
, cfrom
, actual
, cto
);
644 /* utrans_transIncrementalUChars() */
646 pos
.start
= pos
.contextStart
= 0;
647 pos
.limit
= pos
.contextLimit
= u_strlen(buf
);
648 utrans_transIncrementalUChars(trans
, buf
, NULL
, CAP
, &pos
, &status
);
649 utrans_transUChars(trans
, buf
, NULL
, CAP
, pos
.start
, &pos
.limit
, &status
);
650 if (U_FAILURE(status
)) {
651 log_err("FAIL: utrans_transIncrementalUChars() failed, error=%s\n",
652 u_errorName(status
));
656 if (0 == u_strcmp(buf
, to
)) {
657 log_verbose("Ok: utrans_transIncrementalUChars(%s) x %s -> %s\n",
661 u_austrcpy(actual
, buf
);
662 log_err("FAIL: utrans_transIncrementalUChars(%s) x %s -> %s, expected %s\n",
663 id
, cfrom
, actual
, cto
);
667 InitXReplaceableCallbacks(&xrepVtable
);
668 InitXReplaceable(&xrep
, cfrom
);
669 limit
= u_strlen(from
);
670 utrans_trans(trans
, (UReplaceable
*)xrepPtr
, &xrepVtable
, 0, &limit
, &status
);
671 if (U_FAILURE(status
)) {
672 log_err("FAIL: utrans_trans() failed, error=%s\n",
673 u_errorName(status
));
674 FreeXReplaceable(&xrep
);
678 if (0 == u_strcmp(xrep
.text
, to
)) {
679 log_verbose("Ok: utrans_trans(%s) x %s -> %s\n",
683 u_austrcpy(actual
, xrep
.text
);
684 log_err("FAIL: utrans_trans(%s) x %s -> %s, expected %s\n",
685 id
, cfrom
, actual
, cto
);
687 FreeXReplaceable(&xrep
);
689 /* utrans_transIncremental() */
690 InitXReplaceable(&xrep
, cfrom
);
691 pos
.start
= pos
.contextStart
= 0;
692 pos
.limit
= pos
.contextLimit
= u_strlen(from
);
693 utrans_transIncremental(trans
, (UReplaceable
*)xrepPtr
, &xrepVtable
, &pos
, &status
);
694 utrans_trans(trans
, (UReplaceable
*)xrepPtr
, &xrepVtable
, pos
.start
, &pos
.limit
, &status
);
695 if (U_FAILURE(status
)) {
696 log_err("FAIL: utrans_transIncremental() failed, error=%s\n",
697 u_errorName(status
));
698 FreeXReplaceable(&xrep
);
702 if (0 == u_strcmp(xrep
.text
, to
)) {
703 log_verbose("Ok: utrans_transIncremental(%s) x %s -> %s\n",
707 u_austrcpy(actual
, xrep
.text
);
708 log_err("FAIL: utrans_transIncremental(%s) x %s -> %s, expected %s\n",
709 id
, cfrom
, actual
, cto
);
711 FreeXReplaceable(&xrep
);
714 #endif /* #if !UCONFIG_NO_TRANSLITERATION */