2 *******************************************************************************
3 * Copyright (C) 1997-2004, International Business Machines
4 * Corporation and others. All Rights Reserved.
5 *******************************************************************************
6 * Date Name Description
7 * 06/23/00 aliu Creation.
8 *******************************************************************************
11 #include "unicode/utypes.h"
13 #if !UCONFIG_NO_TRANSLITERATION
17 #include "unicode/utrans.h"
18 #include "unicode/ustring.h"
21 #define TEST(x) addTest(root, &x, "utrans/" # x)
23 static void TestAPI(void);
24 static void TestSimpleRules(void);
25 static void TestFilter(void);
26 static void TestOpenInverse(void);
27 static void TestClone(void);
28 static void TestRegisterUnregister(void);
29 static void TestExtractBetween(void);
30 static void TestUnicodeIDs(void);
32 static void _expectRules(const char*, const char*, const char*);
33 static void _expect(const UTransliterator
* trans
, const char* cfrom
, const char* cto
);
35 void addUTransTest(TestNode
** root
);
39 addUTransTest(TestNode
** root
) {
41 TEST(TestSimpleRules
);
43 TEST(TestOpenInverse
);
45 TEST(TestRegisterUnregister
);
46 TEST(TestExtractBetween
);
50 /*------------------------------------------------------------------
53 * To test the Replaceable glue we have to dummy up a C-based
54 * Replaceable callback. This code is for testing purposes only.
55 *------------------------------------------------------------------*/
57 typedef struct XReplaceable
{
58 UChar
* text
; /* MUST BE null-terminated */
61 static void InitXReplaceable(XReplaceable
* rep
, const char* cstring
) {
62 rep
->text
= malloc(sizeof(UChar
) * (strlen(cstring
)+1));
63 u_uastrcpy(rep
->text
, cstring
);
66 static void FreeXReplaceable(XReplaceable
* rep
) {
67 if (rep
->text
!= NULL
) {
73 /* UReplaceableCallbacks callback */
74 static int32_t Xlength(const UReplaceable
* rep
) {
75 const XReplaceable
* x
= (const XReplaceable
*)rep
;
76 return u_strlen(x
->text
);
79 /* UReplaceableCallbacks callback */
80 static UChar
XcharAt(const UReplaceable
* rep
, int32_t offset
) {
81 const XReplaceable
* x
= (const XReplaceable
*)rep
;
82 return x
->text
[offset
];
85 /* UReplaceableCallbacks callback */
86 static UChar32
Xchar32At(const UReplaceable
* rep
, int32_t offset
) {
87 const XReplaceable
* x
= (const XReplaceable
*)rep
;
88 return x
->text
[offset
];
91 /* UReplaceableCallbacks callback */
92 static void Xreplace(UReplaceable
* rep
, int32_t start
, int32_t limit
,
93 const UChar
* text
, int32_t textLength
) {
94 XReplaceable
* x
= (XReplaceable
*)rep
;
95 int32_t newLen
= Xlength(rep
) + limit
- start
+ textLength
;
96 UChar
* newText
= (UChar
*) malloc(sizeof(UChar
) * (newLen
+1));
97 u_strncpy(newText
, x
->text
, start
);
98 u_strncpy(newText
+ start
, text
, textLength
);
99 u_strcpy(newText
+ start
+ textLength
, x
->text
+ limit
);
104 /* UReplaceableCallbacks callback */
105 static void Xcopy(UReplaceable
* rep
, int32_t start
, int32_t limit
, int32_t dest
) {
106 XReplaceable
* x
= (XReplaceable
*)rep
;
107 int32_t newLen
= Xlength(rep
) + limit
- start
;
108 UChar
* newText
= (UChar
*) malloc(sizeof(UChar
) * (newLen
+1));
109 u_strncpy(newText
, x
->text
, dest
);
110 u_strncpy(newText
+ dest
, x
->text
+ start
, limit
- start
);
111 u_strcpy(newText
+ dest
+ limit
- start
, x
->text
+ dest
);
116 /* UReplaceableCallbacks callback */
117 static void Xextract(UReplaceable
* rep
, int32_t start
, int32_t limit
, UChar
* dst
) {
118 XReplaceable
* x
= (XReplaceable
*)rep
;
119 int32_t len
= limit
- start
;
120 u_strncpy(dst
, x
->text
, len
);
123 static void InitXReplaceableCallbacks(UReplaceableCallbacks
* callbacks
) {
124 callbacks
->length
= Xlength
;
125 callbacks
->charAt
= XcharAt
;
126 callbacks
->char32At
= Xchar32At
;
127 callbacks
->replace
= Xreplace
;
128 callbacks
->extract
= Xextract
;
129 callbacks
->copy
= Xcopy
;
132 /*------------------------------------------------------------------
134 *------------------------------------------------------------------*/
136 static void TestAPI() {
137 enum { BUF_CAP
= 128 };
138 char buf
[BUF_CAP
], buf2
[BUF_CAP
];
139 UErrorCode status
= U_ZERO_ERROR
;
140 UTransliterator
* trans
= NULL
;
143 /* Test getAvailableIDs */
144 n
= utrans_countAvailableIDs();
146 log_err("FAIL: utrans_countAvailableIDs() returned %d\n", n
);
148 log_verbose("System ID count: %d\n", n
);
150 for (i
=0; i
<n
; ++i
) {
151 utrans_getAvailableID(i
, buf
, BUF_CAP
);
153 log_err("FAIL: System transliterator %d: \"\"\n", i
);
155 log_verbose("System transliterator %d: \"%s\"\n", i
, buf
);
160 utrans_getAvailableID(0, buf
, BUF_CAP
);
161 trans
= utrans_open(buf
, UTRANS_FORWARD
,NULL
,0,NULL
, &status
);
162 if (U_FAILURE(status
)) {
163 log_err("FAIL: utrans_open(%s) failed, error=%s\n",
164 buf
, u_errorName(status
));
169 utrans_getID(trans
, buf2
, BUF_CAP
);
170 if (0 != strcmp(buf
, buf2
)) {
171 log_err("FAIL: utrans_getID(%s) returned %s\n",
178 static void TestUnicodeIDs() {
180 UTransliterator
*utrans
;
181 const UChar
*id
, *id2
;
182 int32_t idLength
, id2Length
, count
, count2
;
184 UErrorCode errorCode
;
186 errorCode
=U_ZERO_ERROR
;
187 uenum
=utrans_openIDs(&errorCode
);
188 if(U_FAILURE(errorCode
)) {
189 log_err("utrans_openIDs() failed - %s\n", u_errorName(errorCode
));
193 count
=uenum_count(uenum
, &errorCode
);
194 if(U_FAILURE(errorCode
) || count
<1) {
195 log_err("uenum_count(transliterator IDs)=%d - %s\n", count
, u_errorName(errorCode
));
200 id
=uenum_unext(uenum
, &idLength
, &errorCode
);
201 if(U_FAILURE(errorCode
)) {
202 log_err("uenum_unext(transliterator ID %d) failed - %s\n", count
, u_errorName(errorCode
));
210 /* try to actually open only a few transliterators */
214 utrans
=utrans_openU(id
, idLength
, UTRANS_FORWARD
, NULL
, 0, NULL
, &errorCode
);
215 if(U_FAILURE(errorCode
)) {
216 log_err("utrans_openU(%s) failed - %s\n", aescstrdup(id
, idLength
), u_errorName(errorCode
));
220 id2
=utrans_getUnicodeID(utrans
, &id2Length
);
221 if(idLength
!=id2Length
|| 0!=u_memcmp(id
, id2
, idLength
)) {
222 log_err("utrans_getUnicodeID(%s) does not match the original ID\n", aescstrdup(id
, idLength
));
225 utrans_close(utrans
);
228 uenum_reset(uenum
, &errorCode
);
229 if(U_FAILURE(errorCode
) || count
<1) {
230 log_err("uenum_reset(transliterator IDs) failed - %s\n", u_errorName(errorCode
));
232 count2
=uenum_count(uenum
, &errorCode
);
233 if(U_FAILURE(errorCode
) || count
<1) {
234 log_err("2nd uenum_count(transliterator IDs)=%d - %s\n", count2
, u_errorName(errorCode
));
235 } else if(count
!=count2
) {
236 log_err("uenum_unext(transliterator IDs) returned %d IDs but uenum_count() after uenum_reset() claims there are %d\n", count
, count2
);
243 static void TestOpenInverse(){
244 UErrorCode status
=U_ZERO_ERROR
;
245 UTransliterator
* t1
=NULL
;
246 UTransliterator
* inverse1
=NULL
;
247 enum { BUF_CAP
= 128 };
251 const char TransID
[][25]={
252 "Halfwidth-Fullwidth",
253 "Fullwidth-Halfwidth",
256 /*"Arabic-Latin", // Removed in 2.0*/
257 /*"Latin-Arabic", // Removed in 2.0*/
260 /*"Hebrew-Latin", // Removed in 2.0*/
261 /*"Latin-Hebrew", // Removed in 2.0*/
270 for(i
=0; i
<sizeof(TransID
)/sizeof(TransID
[0]); i
=i
+2){
271 status
= U_ZERO_ERROR
;
272 t1
=utrans_open(TransID
[i
], UTRANS_FORWARD
,NULL
,0,NULL
, &status
);
273 if(t1
== NULL
|| U_FAILURE(status
)){
274 log_err("FAIL: in instantiation for id=%s\n", TransID
[i
]);
277 inverse1
=utrans_openInverse(t1
, &status
);
278 if(U_FAILURE(status
)){
279 log_err("FAIL: utrans_openInverse() failed for id=%s. Error=%s\n", TransID
[i
], myErrorName(status
));
282 utrans_getID(inverse1
, buf1
, BUF_CAP
);
283 if(strcmp(buf1
, TransID
[i
+1]) != 0){
284 log_err("FAIL :openInverse() for %s returned %s instead of %s\n", TransID
[i
], buf1
, TransID
[i
+1]);
287 utrans_close(inverse1
);
291 static void TestClone(){
292 UErrorCode status
=U_ZERO_ERROR
;
293 UTransliterator
* t1
=NULL
;
294 UTransliterator
* t2
=NULL
;
295 UTransliterator
* t3
=NULL
;
296 UTransliterator
* t4
=NULL
;
297 enum { BUF_CAP
= 128 };
298 char buf1
[BUF_CAP
], buf2
[BUF_CAP
], buf3
[BUF_CAP
];
300 t1
=utrans_open("Latin-Devanagari", UTRANS_FORWARD
, NULL
,0,NULL
,&status
);
301 if(U_FAILURE(status
)){
302 log_err("FAIL: construction\n");
305 t2
=utrans_open("Latin-Greek", UTRANS_FORWARD
, NULL
,0,NULL
,&status
);
306 if(U_FAILURE(status
)){
307 log_err("FAIL: construction\n");
312 t3
=utrans_clone(t1
, &status
);
313 t4
=utrans_clone(t2
, &status
);
315 utrans_getID(t1
, buf1
, BUF_CAP
);
316 utrans_getID(t2
, buf2
, BUF_CAP
);
317 utrans_getID(t3
, buf3
, BUF_CAP
);
319 if(strcmp(buf1
, buf3
) != 0 ||
320 strcmp(buf1
, buf2
) == 0) {
321 log_err("FAIL: utrans_clone() failed\n");
324 utrans_getID(t4
, buf3
, BUF_CAP
);
326 if(strcmp(buf2
, buf3
) != 0 ||
327 strcmp(buf1
, buf3
) == 0) {
328 log_err("FAIL: utrans_clone() failed\n");
338 static void TestRegisterUnregister(){
339 UErrorCode status
=U_ZERO_ERROR
;
340 UTransliterator
* t1
=NULL
;
341 UTransliterator
* rules
=NULL
, *rules2
;
342 UTransliterator
* inverse1
=NULL
;
343 UChar rule
[]={ 0x0061, 0x003c, 0x003e, 0x0063}; /*a<>b*/
345 U_STRING_DECL(ID
, "TestA-TestB", 11);
346 U_STRING_INIT(ID
, "TestA-TestB", 11);
348 /* Make sure it doesn't exist */
349 t1
=utrans_open("TestA-TestB", UTRANS_FORWARD
,NULL
,0,NULL
, &status
);
350 if(t1
!= NULL
|| U_SUCCESS(status
)) {
351 log_err("FAIL: TestA-TestB already registered\n");
355 /* Check inverse too */
356 inverse1
=utrans_open("TestA-TestB", UTRANS_REVERSE
, NULL
,0,NULL
,&status
);
357 if(inverse1
!= NULL
|| U_SUCCESS(status
)) {
358 log_err("FAIL: TestA-TestB already registered\n");
363 rules
=utrans_open("TestA-TestB",UTRANS_FORWARD
, rule
, 4, NULL
, &status
);
364 if(U_FAILURE(status
)){
365 log_err("FAIL: utrans_openRules(a<>B) failed with error=%s\n", myErrorName(status
));
369 /* clone it so we can register it a second time */
370 rules2
=utrans_clone(rules
, &status
);
371 if(U_FAILURE(status
)) {
372 log_err("FAIL: utrans_clone(a<>B) failed with error=%s\n", myErrorName(status
));
378 utrans_register(rules
, &status
);
379 if(U_FAILURE(status
)){
380 log_err("FAIL: utrans_register failed with error=%s\n", myErrorName(status
));
384 /* Now check again -- should exist now*/
385 t1
= utrans_open("TestA-TestB", UTRANS_FORWARD
, NULL
,0,NULL
,&status
);
386 if(U_FAILURE(status
) || t1
== NULL
){
387 log_err("FAIL: TestA-TestB not registered\n");
392 /*unregister the instance*/
394 utrans_unregister("TestA-TestB");
395 /* now Make sure it doesn't exist */
396 t1
=utrans_open("TestA-TestB", UTRANS_FORWARD
,NULL
,0,NULL
, &status
);
397 if(U_SUCCESS(status
) || t1
!= NULL
) {
398 log_err("FAIL: TestA-TestB isn't unregistered\n");
403 /* now with utrans_unregisterID(const UChar *) */
405 utrans_register(rules2
, &status
);
406 if(U_FAILURE(status
)){
407 log_err("FAIL: 2nd utrans_register failed with error=%s\n", myErrorName(status
));
411 /* Now check again -- should exist now*/
412 t1
= utrans_open("TestA-TestB", UTRANS_FORWARD
, NULL
,0,NULL
,&status
);
413 if(U_FAILURE(status
) || t1
== NULL
){
414 log_err("FAIL: 2nd TestA-TestB not registered\n");
419 /*unregister the instance*/
421 utrans_unregisterID(ID
, -1);
422 /* now Make sure it doesn't exist */
423 t1
=utrans_openU(ID
, -1, UTRANS_FORWARD
,NULL
,0,NULL
, &status
);
424 if(U_SUCCESS(status
) || t1
!= NULL
) {
425 log_err("FAIL: 2nd TestA-TestB isn't unregistered\n");
430 utrans_close(inverse1
);
433 static void TestSimpleRules() {
435 /* Example: rules 1. ab>x|y
438 * []|eabcd start - no match, copy e to tranlated buffer
439 * [e]|abcd match rule 1 - copy output & adjust cursor
440 * [ex|y]cd match rule 2 - copy output & adjust cursor
441 * [exz]|d no match, copy d to transliterated buffer
444 _expectRules("ab>x|y;"
448 /* Another set of rules:
460 _expectRules("ab>x|yzacw;"
468 _expectRules("$dummy=" "\\uE100" ";" /* careful here with E100 */
469 "$vowel=[aeiouAEIOU];"
471 "$vowel } $lu > '!';"
476 "abcdefgABCDEFGU", "&bcd&fg!^**!^*&");
479 static void TestFilter() {
480 UErrorCode status
= U_ZERO_ERROR
;
486 const char* DATA
[] = {
487 "[^c]", /* Filter out 'c' */
489 "\\u0061\\u0062c\\u0064\\u0065",
493 "\\u0061\\u0062\\u0063\\u0064\\u0065"
495 int32_t DATA_length
= sizeof(DATA
) / sizeof(DATA
[0]);
498 UTransliterator
* hex
= utrans_open("Any-Hex", UTRANS_FORWARD
, NULL
,0,NULL
,&status
);
500 if (hex
== 0 || U_FAILURE(status
)) {
501 log_err("FAIL: utrans_open(Unicode-Hex) failed, error=%s\n",
502 u_errorName(status
));
506 for (i
=0; i
<DATA_length
; i
+=3) {
507 /*u_uastrcpy(filt, DATA[i]);*/
508 u_charsToUChars(DATA
[i
], filt
, (int32_t)strlen(DATA
[i
])+1);
509 utrans_setFilter(hex
, filt
, -1, &status
);
511 if (U_FAILURE(status
)) {
512 log_err("FAIL: utrans_setFilter() failed, error=%s\n",
513 u_errorName(status
));
517 /*u_uastrcpy(buf, DATA[i+1]);*/
518 u_charsToUChars(DATA
[i
+1], buf
, (int32_t)strlen(DATA
[i
+1])+1);
520 utrans_transUChars(hex
, buf
, NULL
, 128, 0, &limit
, &status
);
522 if (U_FAILURE(status
)) {
523 log_err("FAIL: utrans_transUChars() failed, error=%s\n",
524 u_errorName(status
));
528 cbuf
=aescstrdup(buf
, -1);
529 u_charsToUChars(DATA
[i
+2], exp
, (int32_t)strlen(DATA
[i
+2])+1);
530 if (0 == u_strcmp(buf
, exp
)) {
531 log_verbose("Ok: %s | %s -> %s\n", DATA
[i
+1], DATA
[i
], cbuf
);
533 log_err("FAIL: %s | %s -> %s, expected %s\n", DATA
[i
+1], DATA
[i
], cbuf
, DATA
[i
+2]);
542 * Test the UReplaceableCallback extractBetween support. We use a
543 * transliterator known to rely on this call.
545 static void TestExtractBetween() {
547 UTransliterator
*trans
;
548 UErrorCode status
= U_ZERO_ERROR
;
549 UParseError parseErr
;
551 trans
= utrans_open("Lower", UTRANS_FORWARD
, NULL
, -1,
554 if (U_FAILURE(status
)) {
555 log_err("FAIL: utrans_open(Lower) failed, error=%s\n",
556 u_errorName(status
));
558 _expect(trans
, "ABC", "abc");
564 static void _expectRules(const char* crules
,
567 /* u_uastrcpy has no capacity param for the buffer -- so just
568 * make all buffers way too big */
571 UTransliterator
*trans
;
572 UErrorCode status
= U_ZERO_ERROR
;
573 UParseError parseErr
;
575 u_uastrcpy(rules
, crules
);
577 trans
= utrans_open(crules
/*use rules as ID*/, UTRANS_FORWARD
, rules
, -1,
579 if (U_FAILURE(status
)) {
581 log_err("FAIL: utrans_openRules(%s) failed, error=%s\n",
582 crules
, u_errorName(status
));
586 _expect(trans
, cfrom
, cto
);
591 static void _expect(const UTransliterator
* trans
,
594 /* u_uastrcpy has no capacity param for the buffer -- so just
595 * make all buffers way too big */
604 UErrorCode status
= U_ZERO_ERROR
;
608 UReplaceableCallbacks xrepVtable
;
610 u_uastrcpy(from
, cfrom
);
613 ID
= utrans_getUnicodeID(trans
, &IDLength
);
614 id
= aescstrdup(ID
, IDLength
);
616 /* utrans_transUChars() */
618 limit
= u_strlen(buf
);
619 utrans_transUChars(trans
, buf
, NULL
, CAP
, 0, &limit
, &status
);
620 if (U_FAILURE(status
)) {
621 log_err("FAIL: utrans_transUChars() failed, error=%s\n",
622 u_errorName(status
));
626 if (0 == u_strcmp(buf
, to
)) {
627 log_verbose("Ok: utrans_transUChars(%s) x %s -> %s\n",
631 u_austrcpy(actual
, buf
);
632 log_err("FAIL: utrans_transUChars(%s) x %s -> %s, expected %s\n",
633 id
, cfrom
, actual
, cto
);
636 /* utrans_transIncrementalUChars() */
638 pos
.start
= pos
.contextStart
= 0;
639 pos
.limit
= pos
.contextLimit
= u_strlen(buf
);
640 utrans_transIncrementalUChars(trans
, buf
, NULL
, CAP
, &pos
, &status
);
641 utrans_transUChars(trans
, buf
, NULL
, CAP
, pos
.start
, &pos
.limit
, &status
);
642 if (U_FAILURE(status
)) {
643 log_err("FAIL: utrans_transIncrementalUChars() failed, error=%s\n",
644 u_errorName(status
));
648 if (0 == u_strcmp(buf
, to
)) {
649 log_verbose("Ok: utrans_transIncrementalUChars(%s) x %s -> %s\n",
653 u_austrcpy(actual
, buf
);
654 log_err("FAIL: utrans_transIncrementalUChars(%s) x %s -> %s, expected %s\n",
655 id
, cfrom
, actual
, cto
);
659 InitXReplaceableCallbacks(&xrepVtable
);
660 InitXReplaceable(&xrep
, cfrom
);
661 limit
= u_strlen(from
);
662 utrans_trans(trans
, (UReplaceable
*)&xrep
, &xrepVtable
, 0, &limit
, &status
);
663 if (U_FAILURE(status
)) {
664 log_err("FAIL: utrans_trans() failed, error=%s\n",
665 u_errorName(status
));
666 FreeXReplaceable(&xrep
);
670 if (0 == u_strcmp(xrep
.text
, to
)) {
671 log_verbose("Ok: utrans_trans(%s) x %s -> %s\n",
675 u_austrcpy(actual
, xrep
.text
);
676 log_err("FAIL: utrans_trans(%s) x %s -> %s, expected %s\n",
677 id
, cfrom
, actual
, cto
);
679 FreeXReplaceable(&xrep
);
681 /* utrans_transIncremental() */
682 InitXReplaceable(&xrep
, cfrom
);
683 pos
.start
= pos
.contextStart
= 0;
684 pos
.limit
= pos
.contextLimit
= u_strlen(from
);
685 utrans_transIncremental(trans
, (UReplaceable
*)&xrep
, &xrepVtable
, &pos
, &status
);
686 utrans_trans(trans
, (UReplaceable
*)&xrep
, &xrepVtable
, pos
.start
, &pos
.limit
, &status
);
687 if (U_FAILURE(status
)) {
688 log_err("FAIL: utrans_transIncremental() failed, error=%s\n",
689 u_errorName(status
));
690 FreeXReplaceable(&xrep
);
694 if (0 == u_strcmp(xrep
.text
, to
)) {
695 log_verbose("Ok: utrans_transIncremental(%s) x %s -> %s\n",
699 u_austrcpy(actual
, xrep
.text
);
700 log_err("FAIL: utrans_transIncremental(%s) x %s -> %s, expected %s\n",
701 id
, cfrom
, actual
, cto
);
703 FreeXReplaceable(&xrep
);
706 #endif /* #if !UCONFIG_NO_TRANSLITERATION */