]> git.saurik.com Git - apple/icu.git/blob - icuSources/test/cintltst/nccbtst.c
projects / apple / icu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
ICU-461.12.tar.gz
[apple/icu.git] / icuSources / test / cintltst / nccbtst.c
1 /********************************************************************
2 * COPYRIGHT:
3 * Copyright (c) 1997-2010, International Business Machines Corporation and
4 * others. All Rights Reserved.
5 ********************************************************************/
6 /*
7 ********************************************************************************
8 * File NCCBTST.C
9 *
10 * Modification History:
11 * Name Description
12 * Madhu Katragadda 7/21/1999 Testing error callback routines
13 ********************************************************************************
14 */
15 #include <stdio.h>
16 #include <stdlib.h>
17 #include <string.h>
18 #include <ctype.h>
19 #include "cstring.h"
20 #include "unicode/uloc.h"
21 #include "unicode/ucnv.h"
22 #include "unicode/ucnv_err.h"
23 #include "cintltst.h"
24 #include "unicode/utypes.h"
25 #include "unicode/ustring.h"
26 #include "nccbtst.h"
27 #include "unicode/ucnv_cb.h"
28 #define NEW_MAX_BUFFER 999
29
30 #define nct_min(x,y) ((x<y) ? x : y)
31 #define ARRAY_LENGTH(array) (sizeof(array)/sizeof((array)[0]))
32
33 static int32_t gInBufferSize = 0;
34 static int32_t gOutBufferSize = 0;
35 static char gNuConvTestName[1024];
36
37 static void printSeq(const uint8_t* a, int len)
38 {
39 int i=0;
40 log_verbose("\n{");
41 while (i<len)
42 log_verbose("0x%02X, ", a[i++]);
43 log_verbose("}\n");
44 }
45
46 static void printUSeq(const UChar* a, int len)
47 {
48 int i=0;
49 log_verbose("{");
50 while (i<len)
51 log_verbose(" 0x%04x, ", a[i++]);
52 log_verbose("}\n");
53 }
54
55 static void printSeqErr(const uint8_t* a, int len)
56 {
57 int i=0;
58 fprintf(stderr, "{");
59 while (i<len)
60 fprintf(stderr, " 0x%02x, ", a[i++]);
61 fprintf(stderr, "}\n");
62 }
63
64 static void printUSeqErr(const UChar* a, int len)
65 {
66 int i=0;
67 fprintf(stderr, "{");
68 while (i<len)
69 fprintf(stderr, "0x%04x, ", a[i++]);
70 fprintf(stderr,"}\n");
71 }
72
73 static void setNuConvTestName(const char *codepage, const char *direction)
74 {
75 sprintf(gNuConvTestName, "[testing %s %s Unicode, InputBufSiz=%d, OutputBufSiz=%d]",
76 codepage,
77 direction,
78 (int)gInBufferSize,
79 (int)gOutBufferSize);
80 }
81
82
83 static void TestCallBackFailure(void);
84
85 void addTestConvertErrorCallBack(TestNode** root);
86
87 void addTestConvertErrorCallBack(TestNode** root)
88 {
89 addTest(root, &TestSkipCallBack, "tsconv/nccbtst/TestSkipCallBack");
90 addTest(root, &TestStopCallBack, "tsconv/nccbtst/TestStopCallBack");
91 addTest(root, &TestSubCallBack, "tsconv/nccbtst/TestSubCallBack");
92 addTest(root, &TestSubWithValueCallBack, "tsconv/nccbtst/TestSubWithValueCallBack");
93
94 #if !UCONFIG_NO_LEGACY_CONVERSION
95 addTest(root, &TestLegalAndOtherCallBack, "tsconv/nccbtst/TestLegalAndOtherCallBack");
96 addTest(root, &TestSingleByteCallBack, "tsconv/nccbtst/TestSingleByteCallBack");
97 #endif
98
99 addTest(root, &TestCallBackFailure, "tsconv/nccbtst/TestCallBackFailure");
100 }
101
102 static void TestSkipCallBack()
103 {
104 TestSkip(NEW_MAX_BUFFER, NEW_MAX_BUFFER);
105 TestSkip(1,NEW_MAX_BUFFER);
106 TestSkip(1,1);
107 TestSkip(NEW_MAX_BUFFER, 1);
108 }
109
110 static void TestStopCallBack()
111 {
112 TestStop(NEW_MAX_BUFFER, NEW_MAX_BUFFER);
113 TestStop(1,NEW_MAX_BUFFER);
114 TestStop(1,1);
115 TestStop(NEW_MAX_BUFFER, 1);
116 }
117
118 static void TestSubCallBack()
119 {
120 TestSub(NEW_MAX_BUFFER, NEW_MAX_BUFFER);
121 TestSub(1,NEW_MAX_BUFFER);
122 TestSub(1,1);
123 TestSub(NEW_MAX_BUFFER, 1);
124
125 #if !UCONFIG_NO_LEGACY_CONVERSION
126 TestEBCDIC_STATEFUL_Sub(1, 1);
127 TestEBCDIC_STATEFUL_Sub(1, NEW_MAX_BUFFER);
128 TestEBCDIC_STATEFUL_Sub(NEW_MAX_BUFFER, 1);
129 TestEBCDIC_STATEFUL_Sub(NEW_MAX_BUFFER, NEW_MAX_BUFFER);
130 #endif
131 }
132
133 static void TestSubWithValueCallBack()
134 {
135 TestSubWithValue(NEW_MAX_BUFFER, NEW_MAX_BUFFER);
136 TestSubWithValue(1,NEW_MAX_BUFFER);
137 TestSubWithValue(1,1);
138 TestSubWithValue(NEW_MAX_BUFFER, 1);
139 }
140
141 #if !UCONFIG_NO_LEGACY_CONVERSION
142 static void TestLegalAndOtherCallBack()
143 {
144 TestLegalAndOthers(NEW_MAX_BUFFER, NEW_MAX_BUFFER);
145 TestLegalAndOthers(1,NEW_MAX_BUFFER);
146 TestLegalAndOthers(1,1);
147 TestLegalAndOthers(NEW_MAX_BUFFER, 1);
148 }
149
150 static void TestSingleByteCallBack()
151 {
152 TestSingleByte(NEW_MAX_BUFFER, NEW_MAX_BUFFER);
153 TestSingleByte(1,NEW_MAX_BUFFER);
154 TestSingleByte(1,1);
155 TestSingleByte(NEW_MAX_BUFFER, 1);
156 }
157 #endif
158
159 static void TestSkip(int32_t inputsize, int32_t outputsize)
160 {
161 static const uint8_t expskipIBM_949[]= {
162 0x00, 0xb0, 0xa1, 0xb0, 0xa2, 0xc8, 0xd3 };
163
164 static const uint8_t expskipIBM_943[] = {
165 0x9f, 0xaf, 0x9f, 0xb1, 0x89, 0x59 };
166
167 static const uint8_t expskipIBM_930[] = {
168 0x0e, 0x5d, 0x5f, 0x5d, 0x63, 0x46, 0x6b, 0x0f };
169
170 gInBufferSize = inputsize;
171 gOutBufferSize = outputsize;
172
173 /*From Unicode*/
174 log_verbose("Testing fromUnicode with UCNV_FROM_U_CALLBACK_SKIP \n");
175
176 #if !UCONFIG_NO_LEGACY_CONVERSION
177 {
178 static const UChar sampleText[] = { 0x0000, 0xAC00, 0xAC01, 0xEF67, 0xD700 };
179 static const UChar sampleText2[] = { 0x6D63, 0x6D64, 0x6D65, 0x6D66 };
180
181 static const int32_t toIBM949Offsskip [] = { 0, 1, 1, 2, 2, 4, 4 };
182 static const int32_t toIBM943Offsskip [] = { 0, 0, 1, 1, 3, 3 };
183
184 if(!testConvertFromUnicode(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
185 expskipIBM_949, sizeof(expskipIBM_949), "ibm-949",
186 UCNV_FROM_U_CALLBACK_SKIP, toIBM949Offsskip, NULL, 0 ))
187 log_err("u-> ibm-949 with skip did not match.\n");
188 if(!testConvertFromUnicode(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
189 expskipIBM_943, sizeof(expskipIBM_943), "ibm-943",
190 UCNV_FROM_U_CALLBACK_SKIP, toIBM943Offsskip, NULL, 0 ))
191 log_err("u-> ibm-943 with skip did not match.\n");
192 }
193
194 {
195 static const UChar fromU[] = { 0x61, 0xff5e, 0x62, 0x6d63, 0xff5e, 0x6d64, 0x63, 0xff5e, 0x6d66 };
196 static const uint8_t fromUBytes[] = { 0x62, 0x63, 0x0e, 0x5d, 0x5f, 0x5d, 0x63, 0x0f, 0x64, 0x0e, 0x46, 0x6b, 0x0f };
197 static const int32_t fromUOffsets[] = { 0, 2, 3, 3, 3, 5, 5, 6, 6, 8, 8, 8, 8 };
198
199 /* test ibm-930 (EBCDIC_STATEFUL) with fallbacks that are not taken to check correct state transitions */
200 if(!testConvertFromUnicode(fromU, sizeof(fromU)/U_SIZEOF_UCHAR,
201 fromUBytes, sizeof(fromUBytes),
202 "ibm-930",
203 UCNV_FROM_U_CALLBACK_SKIP, fromUOffsets,
204 NULL, 0)
205 ) {
206 log_err("u->ibm-930 with skip with untaken fallbacks did not match.\n");
207 }
208 }
209 #endif
210
211 {
212 static const UChar usasciiFromU[] = { 0x61, 0x80, 0x4e00, 0x31, 0xd800, 0xdfff, 0x39 };
213 static const uint8_t usasciiFromUBytes[] = { 0x61, 0x31, 0x39 };
214 static const int32_t usasciiFromUOffsets[] = { 0, 3, 6 };
215
216 static const UChar latin1FromU[] = { 0x61, 0xa0, 0x4e00, 0x31, 0xd800, 0xdfff, 0x39 };
217 static const uint8_t latin1FromUBytes[] = { 0x61, 0xa0, 0x31, 0x39 };
218 static const int32_t latin1FromUOffsets[] = { 0, 1, 3, 6 };
219
220 /* US-ASCII */
221 if(!testConvertFromUnicode(usasciiFromU, sizeof(usasciiFromU)/U_SIZEOF_UCHAR,
222 usasciiFromUBytes, sizeof(usasciiFromUBytes),
223 "US-ASCII",
224 UCNV_FROM_U_CALLBACK_SKIP, usasciiFromUOffsets,
225 NULL, 0)
226 ) {
227 log_err("u->US-ASCII with skip did not match.\n");
228 }
229
230 #if !UCONFIG_NO_LEGACY_CONVERSION
231 /* SBCS NLTC codepage 367 for US-ASCII */
232 if(!testConvertFromUnicode(usasciiFromU, sizeof(usasciiFromU)/U_SIZEOF_UCHAR,
233 usasciiFromUBytes, sizeof(usasciiFromUBytes),
234 "ibm-367",
235 UCNV_FROM_U_CALLBACK_SKIP, usasciiFromUOffsets,
236 NULL, 0)
237 ) {
238 log_err("u->ibm-367 with skip did not match.\n");
239 }
240 #endif
241
242 /* ISO-Latin-1 */
243 if(!testConvertFromUnicode(latin1FromU, sizeof(latin1FromU)/U_SIZEOF_UCHAR,
244 latin1FromUBytes, sizeof(latin1FromUBytes),
245 "LATIN_1",
246 UCNV_FROM_U_CALLBACK_SKIP, latin1FromUOffsets,
247 NULL, 0)
248 ) {
249 log_err("u->LATIN_1 with skip did not match.\n");
250 }
251
252 #if !UCONFIG_NO_LEGACY_CONVERSION
253 /* windows-1252 */
254 if(!testConvertFromUnicode(latin1FromU, sizeof(latin1FromU)/U_SIZEOF_UCHAR,
255 latin1FromUBytes, sizeof(latin1FromUBytes),
256 "windows-1252",
257 UCNV_FROM_U_CALLBACK_SKIP, latin1FromUOffsets,
258 NULL, 0)
259 ) {
260 log_err("u->windows-1252 with skip did not match.\n");
261 }
262 }
263
264 {
265 static const UChar inputTest[] = { 0x0061, 0xd801, 0xdc01, 0xd801, 0x0061 };
266 static const uint8_t toIBM943[]= { 0x61, 0x61 };
267 static const int32_t offset[]= {0, 4};
268
269 /* EUC_JP*/
270 static const UChar euc_jp_inputText[]={ 0x0061, 0x4edd, 0x5bec, 0xd801, 0xdc01, 0xd801, 0x0061, 0x00a2 };
271 static const uint8_t to_euc_jp[]={ 0x61, 0xa1, 0xb8, 0x8f, 0xf4, 0xae,
272 0x61, 0x8e, 0xe0,
273 };
274 static const int32_t fromEUC_JPOffs [] ={ 0, 1, 1, 2, 2, 2, 6, 7, 7};
275
276 /*EUC_TW*/
277 static const UChar euc_tw_inputText[]={ 0x0061, 0x2295, 0x5BF2, 0xd801, 0xdc01, 0xd801, 0x0061, 0x8706, 0x8a, };
278 static const uint8_t to_euc_tw[]={
279 0x61, 0xa2, 0xd3, 0x8e, 0xa2, 0xdc, 0xe5,
280 0x61, 0xe6, 0xca, 0x8a,
281 };
282 static const int32_t from_euc_twOffs [] ={ 0, 1, 1, 2, 2, 2, 2, 6, 7, 7, 8,};
283
284 /*ISO-2022-JP*/
285 static const UChar iso_2022_jp_inputText[]={0x0041, 0x00E9/*unassigned*/,0x0042, };
286 static const uint8_t to_iso_2022_jp[]={
287 0x41,
288 0x42,
289
290 };
291 static const int32_t from_iso_2022_jpOffs [] ={0,2};
292
293 /*ISO-2022-JP*/
294 UChar const iso_2022_jp_inputText2[]={0x0041, 0x00E9/*unassigned*/,0x43,0xd800/*illegal*/,0x0042, };
295 static const uint8_t to_iso_2022_jp2[]={
296 0x41,
297 0x43,
298
299 };
300 static const int32_t from_iso_2022_jpOffs2 [] ={0,2};
301
302 /*ISO-2022-cn*/
303 static const UChar iso_2022_cn_inputText[]={ 0x0041, 0x3712/*unassigned*/, 0x0042, };
304 static const uint8_t to_iso_2022_cn[]={
305 0x41, 0x42
306 };
307 static const int32_t from_iso_2022_cnOffs [] ={
308 0, 2
309 };
310
311 /*ISO-2022-CN*/
312 static const UChar iso_2022_cn_inputText1[]={0x0041, 0x3712/*unassigned*/,0x43,0xd800/*illegal*/,0x0042, };
313 static const uint8_t to_iso_2022_cn1[]={
314 0x41, 0x43
315
316 };
317 static const int32_t from_iso_2022_cnOffs1 [] ={ 0, 2 };
318
319 /*ISO-2022-kr*/
320 static const UChar iso_2022_kr_inputText[]={ 0x0041, 0x03A0,0x3712/*unassigned*/,0x03A0, 0x0042, };
321 static const uint8_t to_iso_2022_kr[]={
322 0x1b, 0x24, 0x29, 0x43,
323 0x41,
324 0x0e, 0x25, 0x50,
325 0x25, 0x50,
326 0x0f, 0x42,
327 };
328 static const int32_t from_iso_2022_krOffs [] ={
329 -1,-1,-1,-1,
330 0,
331 1,1,1,
332 3,3,
333 4,4
334 };
335
336 /*ISO-2022-kr*/
337 static const UChar iso_2022_kr_inputText1[]={ 0x0041, 0x03A0,0x3712/*unassigned*/,0x03A0,0xd801/*illegal*/, 0x0042, };
338 static const uint8_t to_iso_2022_kr1[]={
339 0x1b, 0x24, 0x29, 0x43,
340 0x41,
341 0x0e, 0x25, 0x50,
342 0x25, 0x50,
343
344 };
345 static const int32_t from_iso_2022_krOffs1 [] ={
346 -1,-1,-1,-1,
347 0,
348 1,1,1,
349 3,3,
350
351 };
352 /* HZ encoding */
353 static const UChar hz_inputText[]={ 0x0041, 0x03A0,0x0662/*unassigned*/,0x03A0, 0x0042, };
354
355 static const uint8_t to_hz[]={
356 0x7e, 0x7d, 0x41,
357 0x7e, 0x7b, 0x26, 0x30,
358 0x26, 0x30,
359 0x7e, 0x7d, 0x42,
360
361 };
362 static const int32_t from_hzOffs [] ={
363 0,0,0,
364 1,1,1,1,
365 3,3,
366 4,4,4,4
367 };
368
369 static const UChar hz_inputText1[]={ 0x0041, 0x03A0,0x0662/*unassigned*/,0x03A0,0xd801/*illegal*/, 0x0042, };
370
371 static const uint8_t to_hz1[]={
372 0x7e, 0x7d, 0x41,
373 0x7e, 0x7b, 0x26, 0x30,
374 0x26, 0x30,
375
376
377 };
378 static const int32_t from_hzOffs1 [] ={
379 0,0,0,
380 1,1,1,1,
381 3,3,
382
383 };
384
385 #endif
386
387 static const UChar SCSU_inputText[]={ 0x0041, 0xd801/*illegal*/, 0x0042, };
388
389 static const uint8_t to_SCSU[]={
390 0x41,
391 0x42
392
393
394 };
395 static const int32_t from_SCSUOffs [] ={
396 0,
397 2,
398
399 };
400
401 #if !UCONFIG_NO_LEGACY_CONVERSION
402 /* ISCII */
403 static const UChar iscii_inputText[]={ 0x0041, 0x3712/*unassigned*/, 0x0042, };
404 static const uint8_t to_iscii[]={
405 0x41,
406 0x42,
407 };
408 static const int32_t from_isciiOffs [] ={
409 0,2,
410
411 };
412 /*ISCII*/
413 static const UChar iscii_inputText1[]={0x0044, 0x3712/*unassigned*/,0x43,0xd800/*illegal*/,0x0042, };
414 static const uint8_t to_iscii1[]={
415 0x44,
416 0x43,
417
418 };
419 static const int32_t from_isciiOffs1 [] ={0,2};
420
421 if(!testConvertFromUnicode(inputTest, sizeof(inputTest)/sizeof(inputTest[0]),
422 toIBM943, sizeof(toIBM943), "ibm-943",
423 UCNV_FROM_U_CALLBACK_SKIP, offset, NULL, 0 ))
424 log_err("u-> ibm-943 with skip did not match.\n");
425
426 if(!testConvertFromUnicode(euc_jp_inputText, sizeof(euc_jp_inputText)/sizeof(euc_jp_inputText[0]),
427 to_euc_jp, sizeof(to_euc_jp), "euc-jp",
428 UCNV_FROM_U_CALLBACK_SKIP, fromEUC_JPOffs, NULL, 0 ))
429 log_err("u-> euc-jp with skip did not match.\n");
430
431 if(!testConvertFromUnicode(euc_tw_inputText, sizeof(euc_tw_inputText)/sizeof(euc_tw_inputText[0]),
432 to_euc_tw, sizeof(to_euc_tw), "euc-tw",
433 UCNV_FROM_U_CALLBACK_SKIP, from_euc_twOffs, NULL, 0 ))
434 log_err("u-> euc-tw with skip did not match.\n");
435
436 /*iso_2022_jp*/
437 if(!testConvertFromUnicode(iso_2022_jp_inputText, sizeof(iso_2022_jp_inputText)/sizeof(iso_2022_jp_inputText[0]),
438 to_iso_2022_jp, sizeof(to_iso_2022_jp), "iso-2022-jp",
439 UCNV_FROM_U_CALLBACK_SKIP, from_iso_2022_jpOffs, NULL, 0 ))
440 log_err("u-> iso-2022-jp with skip did not match.\n");
441
442 /* with context */
443 if(!testConvertFromUnicodeWithContext(iso_2022_jp_inputText2, sizeof(iso_2022_jp_inputText2)/sizeof(iso_2022_jp_inputText2[0]),
444 to_iso_2022_jp2, sizeof(to_iso_2022_jp2), "iso-2022-jp",
445 UCNV_FROM_U_CALLBACK_SKIP, from_iso_2022_jpOffs2, NULL, 0,UCNV_SKIP_STOP_ON_ILLEGAL,U_ILLEGAL_CHAR_FOUND ))
446 log_err("u-> iso-2022-jp with skip & UCNV_SKIP_STOP_ON_ILLEGAL did not match.\n");
447
448 /*iso_2022_cn*/
449 if(!testConvertFromUnicode(iso_2022_cn_inputText, sizeof(iso_2022_cn_inputText)/sizeof(iso_2022_cn_inputText[0]),
450 to_iso_2022_cn, sizeof(to_iso_2022_cn), "iso-2022-cn",
451 UCNV_FROM_U_CALLBACK_SKIP, from_iso_2022_cnOffs, NULL, 0 ))
452 log_err("u-> iso-2022-cn with skip did not match.\n");
453 /*with context*/
454 if(!testConvertFromUnicodeWithContext(iso_2022_cn_inputText1, sizeof(iso_2022_cn_inputText1)/sizeof(iso_2022_cn_inputText1[0]),
455 to_iso_2022_cn1, sizeof(to_iso_2022_cn1), "iso-2022-cn",
456 UCNV_FROM_U_CALLBACK_SKIP, from_iso_2022_cnOffs1, NULL, 0,UCNV_SKIP_STOP_ON_ILLEGAL,U_ILLEGAL_CHAR_FOUND ))
457 log_err("u-> iso-2022-cn with skip & UCNV_SKIP_STOP_ON_ILLEGAL did not match.\n");
458
459 /*iso_2022_kr*/
460 if(!testConvertFromUnicode(iso_2022_kr_inputText, sizeof(iso_2022_kr_inputText)/sizeof(iso_2022_kr_inputText[0]),
461 to_iso_2022_kr, sizeof(to_iso_2022_kr), "iso-2022-kr",
462 UCNV_FROM_U_CALLBACK_SKIP, from_iso_2022_krOffs, NULL, 0 ))
463 log_err("u-> iso-2022-kr with skip did not match.\n");
464 /*with context*/
465 if(!testConvertFromUnicodeWithContext(iso_2022_kr_inputText1, sizeof(iso_2022_kr_inputText1)/sizeof(iso_2022_kr_inputText1[0]),
466 to_iso_2022_kr1, sizeof(to_iso_2022_kr1), "iso-2022-kr",
467 UCNV_FROM_U_CALLBACK_SKIP, from_iso_2022_krOffs1, NULL, 0,UCNV_SKIP_STOP_ON_ILLEGAL,U_ILLEGAL_CHAR_FOUND ))
468 log_err("u-> iso-2022-kr with skip & UCNV_SKIP_STOP_ON_ILLEGAL did not match.\n");
469
470 /*hz*/
471 if(!testConvertFromUnicode(hz_inputText, sizeof(hz_inputText)/sizeof(hz_inputText[0]),
472 to_hz, sizeof(to_hz), "HZ",
473 UCNV_FROM_U_CALLBACK_SKIP, from_hzOffs, NULL, 0 ))
474 log_err("u-> HZ with skip did not match.\n");
475 /*with context*/
476 if(!testConvertFromUnicodeWithContext(hz_inputText1, sizeof(hz_inputText1)/sizeof(hz_inputText1[0]),
477 to_hz1, sizeof(to_hz1), "hz",
478 UCNV_FROM_U_CALLBACK_SKIP, from_hzOffs1, NULL, 0,UCNV_SKIP_STOP_ON_ILLEGAL,U_ILLEGAL_CHAR_FOUND ))
479 log_err("u-> hz with skip & UCNV_SKIP_STOP_ON_ILLEGAL did not match.\n");
480 #endif
481
482 /*SCSU*/
483 if(!testConvertFromUnicode(SCSU_inputText, sizeof(SCSU_inputText)/sizeof(SCSU_inputText[0]),
484 to_SCSU, sizeof(to_SCSU), "SCSU",
485 UCNV_FROM_U_CALLBACK_SKIP, from_SCSUOffs, NULL, 0 ))
486 log_err("u-> SCSU with skip did not match.\n");
487
488 #if !UCONFIG_NO_LEGACY_CONVERSION
489 /*ISCII*/
490 if(!testConvertFromUnicode(iscii_inputText, sizeof(iscii_inputText)/sizeof(iscii_inputText[0]),
491 to_iscii, sizeof(to_iscii), "ISCII,version=0",
492 UCNV_FROM_U_CALLBACK_SKIP, from_isciiOffs, NULL, 0 ))
493 log_err("u-> iscii with skip did not match.\n");
494 /*with context*/
495 if(!testConvertFromUnicodeWithContext(iscii_inputText1, sizeof(iscii_inputText1)/sizeof(iscii_inputText1[0]),
496 to_iscii1, sizeof(to_iscii1), "ISCII,version=0",
497 UCNV_FROM_U_CALLBACK_SKIP, from_isciiOffs1, NULL, 0,UCNV_SKIP_STOP_ON_ILLEGAL,U_ILLEGAL_CHAR_FOUND ))
498 log_err("u-> iscii with skip & UCNV_SKIP_STOP_ON_ILLEGAL did not match.\n");
499 #endif
500 }
501
502 log_verbose("Testing fromUnicode for BOCU-1 with UCNV_TO_U_CALLBACK_SKIP\n");
503 {
504 static const uint8_t sampleText[]={ /* from cintltst/bocu1tst.c/TestBOCU1 text 1 */
505 0xFB, 0xEE, 0x28, /* from source offset 0 */
506 0x24, 0x1E, 0x52,
507 0xB2,
508 0x20,
509 0xB3,
510 0xB1,
511 0x0D,
512 0x0A,
513
514 0x20, /* from 8 */
515 0x00,
516 0xD0, 0x6C,
517 0xB6,
518 0xD8, 0xA5,
519 0x20,
520 0x68,
521 0x59,
522
523 0xF9, 0x28, /* from 16 */
524 0x6D,
525 0x20,
526 0x73,
527 0xE0, 0x2D,
528 0xDE, 0x43,
529 0xD0, 0x33,
530 0x20,
531
532 0xFA, 0x83, /* from 24 */
533 0x25, 0x01,
534 0xFB, 0x16, 0x87,
535 0x4B, 0x16,
536 0x20,
537 0xE6, 0xBD,
538 0xEB, 0x5B,
539 0x4B, 0xCC,
540
541 0xF9, 0xA2, /* from 32 */
542 0xFC, 0x10, 0x3E,
543 0xFE, 0x16, 0x3A, 0x8C,
544 0x20,
545 0xFC, 0x03, 0xAC,
546
547 0x01, /* from 41 */
548 0xDE, 0x83,
549 0x20,
550 0x09
551 };
552 static const UChar expected[]={
553 0xFEFF, 0x0061, 0x0062, 0x0020, /* 0 */
554 0x0063, 0x0061, 0x000D, 0x000A,
555
556 0x0020, 0x0000, 0x00DF, 0x00E6, /* 8 */
557 0x0930, 0x0020, 0x0918, 0x0909,
558
559 0x3086, 0x304D, 0x0020, 0x3053, /* 16 */
560 0x4000, 0x4E00, 0x7777, 0x0020,
561
562 0x9FA5, 0x4E00, 0xAC00, 0xBCDE, /* 24 */
563 0x0020, 0xD7A3, 0xDC00, 0xD800,
564
565 0xD800, 0xDC00, 0xD845, 0xDDDD, /* 32 */
566 0xDBBB, 0xDDEE, 0x0020, 0xDBFF,
567
568 0xDFFF, 0x0001, 0x0E40, 0x0020, /* 40 */
569 0x0009
570 };
571 static const int32_t offsets[]={
572 0, 0, 0, 1, 1, 1, 2, 3, 4, 5, 6, 7,
573 8, 9, 10, 10, 11, 12, 12, 13, 14, 15,
574 16, 16, 17, 18, 19, 20, 20, 21, 21, 22, 22, 23,
575 24, 24, 25, 25, 26, 26, 26, 27, 27, 28, 29, 29, 30, 30, 31, 31,
576 32, 32, 34, 34, 34, 36, 36, 36, 36, 38, 39, 39, 39,
577 41, 42, 42, 43, 44
578 };
579
580 /* BOCU-1 fromUnicode never calls callbacks, so this only tests single-byte and offsets behavior */
581 if(!testConvertFromUnicode(expected, ARRAY_LENGTH(expected),
582 sampleText, sizeof(sampleText),
583 "BOCU-1",
584 UCNV_FROM_U_CALLBACK_SKIP, offsets, NULL, 0)
585 ) {
586 log_err("u->BOCU-1 with skip did not match.\n");
587 }
588 }
589
590 log_verbose("Testing fromUnicode for CESU-8 with UCNV_TO_U_CALLBACK_SKIP\n");
591 {
592 const uint8_t sampleText[]={
593 0x61, /* 'a' */
594 0xc4, 0xb5, /* U+0135 */
595 0xed, 0x80, 0xa0, /* Hangul U+d020 */
596 0xed, 0xa0, 0x81, 0xed, 0xb0, 0x81, /* surrogate pair for U+10401 */
597 0xee, 0x80, 0x80, /* PUA U+e000 */
598 0xed, 0xb0, 0x81, /* unpaired trail surrogate U+dc01 */
599 0x62, /* 'b' */
600 0xed, 0xa0, 0x81, /* unpaired lead surrogate U+d801 */
601 0xd0, 0x80 /* U+0400 */
602 };
603 UChar expected[]={
604 0x0061,
605 0x0135,
606 0xd020,
607 0xd801, 0xdc01,
608 0xe000,
609 0xdc01,
610 0x0062,
611 0xd801,
612 0x0400
613 };
614 int32_t offsets[]={
615 0,
616 1, 1,
617 2, 2, 2,
618 3, 3, 3, 4, 4, 4,
619 5, 5, 5,
620 6, 6, 6,
621 7,
622 8, 8, 8,
623 9, 9
624 };
625
626 /* CESU-8 fromUnicode never calls callbacks, so this only tests conversion and offsets behavior */
627
628 /* without offsets */
629 if(!testConvertFromUnicode(expected, ARRAY_LENGTH(expected),
630 sampleText, sizeof(sampleText),
631 "CESU-8",
632 UCNV_FROM_U_CALLBACK_SKIP, NULL, NULL, 0)
633 ) {
634 log_err("u->CESU-8 with skip did not match.\n");
635 }
636
637 /* with offsets */
638 if(!testConvertFromUnicode(expected, ARRAY_LENGTH(expected),
639 sampleText, sizeof(sampleText),
640 "CESU-8",
641 UCNV_FROM_U_CALLBACK_SKIP, offsets, NULL, 0)
642 ) {
643 log_err("u->CESU-8 with skip did not match.\n");
644 }
645 }
646
647 /*to Unicode*/
648 log_verbose("Testing toUnicode with UCNV_TO_U_CALLBACK_SKIP \n");
649
650 #if !UCONFIG_NO_LEGACY_CONVERSION
651 {
652
653 static const UChar IBM_949skiptoUnicode[]= {0x0000, 0xAC00, 0xAC01, 0xD700 };
654 static const UChar IBM_943skiptoUnicode[]= { 0x6D63, 0x6D64, 0x6D66 };
655 static const UChar IBM_930skiptoUnicode[]= { 0x6D63, 0x6D64, 0x6D66 };
656
657 static const int32_t fromIBM949Offs [] = { 0, 1, 3, 5};
658 static const int32_t fromIBM943Offs [] = { 0, 2, 4};
659 static const int32_t fromIBM930Offs [] = { 1, 3, 5};
660
661 if(!testConvertToUnicode(expskipIBM_949, sizeof(expskipIBM_949),
662 IBM_949skiptoUnicode, sizeof(IBM_949skiptoUnicode)/sizeof(IBM_949skiptoUnicode),"ibm-949",
663 UCNV_TO_U_CALLBACK_SKIP, fromIBM949Offs, NULL, 0 ))
664 log_err("ibm-949->u with skip did not match.\n");
665 if(!testConvertToUnicode(expskipIBM_943, sizeof(expskipIBM_943),
666 IBM_943skiptoUnicode, sizeof(IBM_943skiptoUnicode)/sizeof(IBM_943skiptoUnicode[0]),"ibm-943",
667 UCNV_TO_U_CALLBACK_SKIP, fromIBM943Offs, NULL, 0 ))
668 log_err("ibm-943->u with skip did not match.\n");
669
670
671 if(!testConvertToUnicode(expskipIBM_930, sizeof(expskipIBM_930),
672 IBM_930skiptoUnicode, sizeof(IBM_930skiptoUnicode)/sizeof(IBM_930skiptoUnicode[0]),"ibm-930",
673 UCNV_TO_U_CALLBACK_SKIP, fromIBM930Offs, NULL, 0 ))
674 log_err("ibm-930->u with skip did not match.\n");
675
676
677 if(!testConvertToUnicodeWithContext(expskipIBM_930, sizeof(expskipIBM_930),
678 IBM_930skiptoUnicode, sizeof(IBM_930skiptoUnicode)/sizeof(IBM_930skiptoUnicode[0]),"ibm-930",
679 UCNV_TO_U_CALLBACK_SKIP, fromIBM930Offs, NULL, 0,"i",U_ILLEGAL_CHAR_FOUND ))
680 log_err("ibm-930->u with skip did not match.\n");
681 }
682 #endif
683
684 {
685 static const uint8_t usasciiToUBytes[] = { 0x61, 0x80, 0x31 };
686 static const UChar usasciiToU[] = { 0x61, 0x31 };
687 static const int32_t usasciiToUOffsets[] = { 0, 2 };
688
689 static const uint8_t latin1ToUBytes[] = { 0x61, 0xa0, 0x31 };
690 static const UChar latin1ToU[] = { 0x61, 0xa0, 0x31 };
691 static const int32_t latin1ToUOffsets[] = { 0, 1, 2 };
692
693 /* US-ASCII */
694 if(!testConvertToUnicode(usasciiToUBytes, sizeof(usasciiToUBytes),
695 usasciiToU, sizeof(usasciiToU)/U_SIZEOF_UCHAR,
696 "US-ASCII",
697 UCNV_TO_U_CALLBACK_SKIP, usasciiToUOffsets,
698 NULL, 0)
699 ) {
700 log_err("US-ASCII->u with skip did not match.\n");
701 }
702
703 #if !UCONFIG_NO_LEGACY_CONVERSION
704 /* SBCS NLTC codepage 367 for US-ASCII */
705 if(!testConvertToUnicode(usasciiToUBytes, sizeof(usasciiToUBytes),
706 usasciiToU, sizeof(usasciiToU)/U_SIZEOF_UCHAR,
707 "ibm-367",
708 UCNV_TO_U_CALLBACK_SKIP, usasciiToUOffsets,
709 NULL, 0)
710 ) {
711 log_err("ibm-367->u with skip did not match.\n");
712 }
713 #endif
714
715 /* ISO-Latin-1 */
716 if(!testConvertToUnicode(latin1ToUBytes, sizeof(latin1ToUBytes),
717 latin1ToU, sizeof(latin1ToU)/U_SIZEOF_UCHAR,
718 "LATIN_1",
719 UCNV_TO_U_CALLBACK_SKIP, latin1ToUOffsets,
720 NULL, 0)
721 ) {
722 log_err("LATIN_1->u with skip did not match.\n");
723 }
724
725 #if !UCONFIG_NO_LEGACY_CONVERSION
726 /* windows-1252 */
727 if(!testConvertToUnicode(latin1ToUBytes, sizeof(latin1ToUBytes),
728 latin1ToU, sizeof(latin1ToU)/U_SIZEOF_UCHAR,
729 "windows-1252",
730 UCNV_TO_U_CALLBACK_SKIP, latin1ToUOffsets,
731 NULL, 0)
732 ) {
733 log_err("windows-1252->u with skip did not match.\n");
734 }
735 #endif
736 }
737
738 #if !UCONFIG_NO_LEGACY_CONVERSION
739 {
740 static const uint8_t sampleTxtEBCIDIC_STATEFUL [] ={
741 0x0e, 0x5d, 0x5f , 0x41, 0x79, 0x41, 0x44
742 };
743 static const UChar EBCIDIC_STATEFUL_toUnicode[] ={ 0x6d63, 0x03b4
744 };
745 static const int32_t from_EBCIDIC_STATEFULOffsets []={ 1, 5};
746
747
748 /* euc-jp*/
749 static const uint8_t sampleTxt_euc_jp[]={ 0x61, 0xa1, 0xb8, 0x8f, 0xf4, 0xae,
750 0x8f, 0xda, 0xa1, /*unassigned*/
751 0x8e, 0xe0,
752 };
753 static const UChar euc_jptoUnicode[]={ 0x0061, 0x4edd, 0x5bec, 0x00a2};
754 static const int32_t from_euc_jpOffs [] ={ 0, 1, 3, 9};
755
756 /*EUC_TW*/
757 static const uint8_t sampleTxt_euc_tw[]={ 0x61, 0xa2, 0xd3, 0x8e, 0xa2, 0xdc, 0xe5,
758 0x8e, 0xaa, 0xbb, 0xcc,/*unassigned*/
759 0xe6, 0xca, 0x8a,
760 };
761 static const UChar euc_twtoUnicode[]={ 0x0061, 0x2295, 0x5BF2, 0x8706, 0x8a, };
762 static const int32_t from_euc_twOffs [] ={ 0, 1, 3, 11, 13};
763 /*iso-2022-jp*/
764 static const uint8_t sampleTxt_iso_2022_jp[]={
765 0x41,
766 0x1b, 0x24, 0x42, 0x2A, 0x44, /*unassigned*/
767 0x1b, 0x28, 0x42, 0x42,
768
769 };
770 static const UChar iso_2022_jptoUnicode[]={ 0x41,0x42 };
771 static const int32_t from_iso_2022_jpOffs [] ={ 0,9 };
772
773 /*iso-2022-cn*/
774 static const uint8_t sampleTxt_iso_2022_cn[]={
775 0x0f, 0x41, 0x44,
776 0x1B, 0x24, 0x29, 0x47,
777 0x0E, 0x40, 0x6f, /*unassigned*/
778 0x0f, 0x42,
779
780 };
781
782 static const UChar iso_2022_cntoUnicode[]={ 0x41, 0x44,0x42 };
783 static const int32_t from_iso_2022_cnOffs [] ={ 1, 2, 11 };
784
785 /*iso-2022-kr*/
786 static const uint8_t sampleTxt_iso_2022_kr[]={
787 0x1b, 0x24, 0x29, 0x43,
788 0x41,
789 0x0E, 0x7f, 0x1E,
790 0x0e, 0x25, 0x50,
791 0x0f, 0x51,
792 0x42, 0x43,
793
794 };
795 static const UChar iso_2022_krtoUnicode[]={ 0x41,0x03A0,0x51, 0x42,0x43};
796 static const int32_t from_iso_2022_krOffs [] ={ 4, 9, 12, 13 , 14 };
797
798 /*hz*/
799 static const uint8_t sampleTxt_hz[]={
800 0x41,
801 0x7e, 0x7b, 0x26, 0x30,
802 0x7f, 0x1E, /*unassigned*/
803 0x26, 0x30,
804 0x7e, 0x7d, 0x42,
805 0x7e, 0x7b, 0x7f, 0x1E,/*unassigned*/
806 0x7e, 0x7d, 0x42,
807 };
808 static const UChar hztoUnicode[]={
809 0x41,
810 0x03a0,
811 0x03A0,
812 0x42,
813 0x42,};
814
815 static const int32_t from_hzOffs [] ={0,3,7,11,18, };
816
817 /*ISCII*/
818 static const uint8_t sampleTxt_iscii[]={
819 0x41,
820 0xa1,
821 0xEB, /*unassigned*/
822 0x26,
823 0x30,
824 0xa2,
825 0xEC, /*unassigned*/
826 0x42,
827 };
828 static const UChar isciitoUnicode[]={
829 0x41,
830 0x0901,
831 0x26,
832 0x30,
833 0x0902,
834 0x42,
835 };
836
837 static const int32_t from_isciiOffs [] ={0,1,3,4,5,7 };
838
839 /*LMBCS*/
840 static const uint8_t sampleTxtLMBCS[]={ 0x12, 0xc9, 0x50,
841 0x12, 0x92, 0xa0, /*unassigned*/
842 0x12, 0x92, 0xA1,
843 };
844 static const UChar LMBCSToUnicode[]={ 0x4e2e, 0xe5c4};
845 static const int32_t fromLMBCS[] = {0, 6};
846
847 if(!testConvertToUnicode(sampleTxtEBCIDIC_STATEFUL, sizeof(sampleTxtEBCIDIC_STATEFUL),
848 EBCIDIC_STATEFUL_toUnicode, sizeof(EBCIDIC_STATEFUL_toUnicode)/sizeof(EBCIDIC_STATEFUL_toUnicode[0]),"ibm-930",
849 UCNV_TO_U_CALLBACK_SKIP, from_EBCIDIC_STATEFULOffsets, NULL, 0 ))
850 log_err("EBCIDIC_STATEFUL->u with skip did not match.\n");
851
852 if(!testConvertToUnicodeWithContext(sampleTxtEBCIDIC_STATEFUL, sizeof(sampleTxtEBCIDIC_STATEFUL),
853 EBCIDIC_STATEFUL_toUnicode, sizeof(EBCIDIC_STATEFUL_toUnicode)/sizeof(EBCIDIC_STATEFUL_toUnicode[0]),"ibm-930",
854 UCNV_TO_U_CALLBACK_SKIP, from_EBCIDIC_STATEFULOffsets, NULL, 0,"i",U_ILLEGAL_CHAR_FOUND ))
855 log_err("EBCIDIC_STATEFUL->u with skip did not match.\n");
856
857 if(!testConvertToUnicode(sampleTxt_euc_jp, sizeof(sampleTxt_euc_jp),
858 euc_jptoUnicode, sizeof(euc_jptoUnicode)/sizeof(euc_jptoUnicode[0]),"euc-jp",
859 UCNV_TO_U_CALLBACK_SKIP, from_euc_jpOffs , NULL, 0))
860 log_err("euc-jp->u with skip did not match.\n");
861
862
863
864 if(!testConvertToUnicode(sampleTxt_euc_tw, sizeof(sampleTxt_euc_tw),
865 euc_twtoUnicode, sizeof(euc_twtoUnicode)/sizeof(euc_twtoUnicode[0]),"euc-tw",
866 UCNV_TO_U_CALLBACK_SKIP, from_euc_twOffs , NULL, 0))
867 log_err("euc-tw->u with skip did not match.\n");
868
869
870 if(!testConvertToUnicode(sampleTxt_iso_2022_jp, sizeof(sampleTxt_iso_2022_jp),
871 iso_2022_jptoUnicode, sizeof(iso_2022_jptoUnicode)/sizeof(iso_2022_jptoUnicode[0]),"iso-2022-jp",
872 UCNV_TO_U_CALLBACK_SKIP, from_iso_2022_jpOffs , NULL, 0))
873 log_err("iso-2022-jp->u with skip did not match.\n");
874
875 if(!testConvertToUnicode(sampleTxt_iso_2022_cn, sizeof(sampleTxt_iso_2022_cn),
876 iso_2022_cntoUnicode, sizeof(iso_2022_cntoUnicode)/sizeof(iso_2022_cntoUnicode[0]),"iso-2022-cn",
877 UCNV_TO_U_CALLBACK_SKIP, from_iso_2022_cnOffs , NULL, 0))
878 log_err("iso-2022-cn->u with skip did not match.\n");
879
880 if(!testConvertToUnicode(sampleTxt_iso_2022_kr, sizeof(sampleTxt_iso_2022_kr),
881 iso_2022_krtoUnicode, sizeof(iso_2022_krtoUnicode)/sizeof(iso_2022_krtoUnicode[0]),"iso-2022-kr",
882 UCNV_TO_U_CALLBACK_SKIP, from_iso_2022_krOffs , NULL, 0))
883 log_err("iso-2022-kr->u with skip did not match.\n");
884
885 if(!testConvertToUnicode(sampleTxt_hz, sizeof(sampleTxt_hz),
886 hztoUnicode, sizeof(hztoUnicode)/sizeof(hztoUnicode[0]),"HZ",
887 UCNV_TO_U_CALLBACK_SKIP, from_hzOffs , NULL, 0))
888 log_err("HZ->u with skip did not match.\n");
889
890 if(!testConvertToUnicode(sampleTxt_iscii, sizeof(sampleTxt_iscii),
891 isciitoUnicode, sizeof(isciitoUnicode)/sizeof(isciitoUnicode[0]),"ISCII,version=0",
892 UCNV_TO_U_CALLBACK_SKIP, from_isciiOffs , NULL, 0))
893 log_err("iscii->u with skip did not match.\n");
894
895 if(!testConvertToUnicode(sampleTxtLMBCS, sizeof(sampleTxtLMBCS),
896 LMBCSToUnicode, sizeof(LMBCSToUnicode)/sizeof(LMBCSToUnicode[0]),"LMBCS-1",
897 UCNV_TO_U_CALLBACK_SKIP, fromLMBCS , NULL, 0))
898 log_err("LMBCS->u with skip did not match.\n");
899
900 }
901 #endif
902
903 log_verbose("Testing to Unicode for UTF-8 with UCNV_TO_U_CALLBACK_SKIP \n");
904 {
905 const uint8_t sampleText1[] = { 0x31, 0xe4, 0xba, 0x8c,
906 0xe0, 0x80, 0x61,};
907 UChar expected1[] = { 0x0031, 0x4e8c, 0x0061};
908 int32_t offsets1[] = { 0x0000, 0x0001, 0x0006};
909
910 if(!testConvertToUnicode(sampleText1, sizeof(sampleText1),
911 expected1, sizeof(expected1)/sizeof(expected1[0]),"utf8",
912 UCNV_TO_U_CALLBACK_SKIP, offsets1, NULL, 0 ))
913 log_err("utf8->u with skip did not match.\n");;
914 }
915
916 log_verbose("Testing toUnicode for SCSU with UCNV_TO_U_CALLBACK_SKIP \n");
917 {
918 const uint8_t sampleText1[] = { 0xba, 0x8c,0xF8, 0x61,0x0c, 0x0c,};
919 UChar expected1[] = { 0x00ba, 0x008c, 0x00f8, 0x0061,0xfffe,0xfffe};
920 int32_t offsets1[] = { 0x0000, 0x0001,0x0002,0x0003,4,5};
921
922 if(!testConvertToUnicode(sampleText1, sizeof(sampleText1),
923 expected1, sizeof(expected1)/sizeof(expected1[0]),"SCSU",
924 UCNV_TO_U_CALLBACK_SKIP, offsets1, NULL, 0 ))
925 log_err("scsu->u with skip did not match.\n");
926 }
927
928 log_verbose("Testing toUnicode for BOCU-1 with UCNV_TO_U_CALLBACK_SKIP\n");
929 {
930 const uint8_t sampleText[]={ /* modified from cintltst/bocu1tst.c/TestBOCU1 text 1 */
931 0xFB, 0xEE, 0x28, /* single-code point sequence at offset 0 */
932 0x24, 0x1E, 0x52, /* 3 */
933 0xB2, /* 6 */
934 0x20, /* 7 */
935 0x40, 0x07, /* 8 - wrong trail byte */
936 0xB3, /* 10 */
937 0xB1, /* 11 */
938 0xD0, 0x20, /* 12 - wrong trail byte */
939 0x0D, /* 14 */
940 0x0A, /* 15 */
941 0x20, /* 16 */
942 0x00, /* 17 */
943 0xD0, 0x6C, /* 18 */
944 0xB6, /* 20 */
945 0xD8, 0xA5, /* 21 */
946 0x20, /* 23 */
947 0x68, /* 24 */
948 0x59, /* 25 */
949 0xF9, 0x28, /* 26 */
950 0x6D, /* 28 */
951 0x20, /* 29 */
952 0x73, /* 30 */
953 0xE0, 0x2D, /* 31 */
954 0xDE, 0x43, /* 33 */
955 0xD0, 0x33, /* 35 */
956 0x20, /* 37 */
957 0xFA, 0x83, /* 38 */
958 0x25, 0x01, /* 40 */
959 0xFB, 0x16, 0x87, /* 42 */
960 0x4B, 0x16, /* 45 */
961 0x20, /* 47 */
962 0xE6, 0xBD, /* 48 */
963 0xEB, 0x5B, /* 50 */
964 0x4B, 0xCC, /* 52 */
965 0xF9, 0xA2, /* 54 */
966 0xFC, 0x10, 0x3E, /* 56 */
967 0xFE, 0x16, 0x3A, 0x8C, /* 59 */
968 0x20, /* 63 */
969 0xFC, 0x03, 0xAC, /* 64 */
970 0xFF, /* 67 - FF just resets the state without encoding anything */
971 0x01, /* 68 */
972 0xDE, 0x83, /* 69 */
973 0x20, /* 71 */
974 0x09 /* 72 */
975 };
976 UChar expected[]={
977 0xFEFF, 0x0061, 0x0062, 0x0020,
978 0x0063, 0x0061, 0x000D, 0x000A,
979 0x0020, 0x0000, 0x00DF, 0x00E6,
980 0x0930, 0x0020, 0x0918, 0x0909,
981 0x3086, 0x304D, 0x0020, 0x3053,
982 0x4000, 0x4E00, 0x7777, 0x0020,
983 0x9FA5, 0x4E00, 0xAC00, 0xBCDE,
984 0x0020, 0xD7A3, 0xDC00, 0xD800,
985 0xD800, 0xDC00, 0xD845, 0xDDDD,
986 0xDBBB, 0xDDEE, 0x0020, 0xDBFF,
987 0xDFFF, 0x0001, 0x0E40, 0x0020,
988 0x0009
989 };
990 int32_t offsets[]={
991 0, 3, 6, 7, /* skip 8, */
992 10, 11, /* skip 12, */
993 14, 15, 16, 17, 18,
994 20, 21, 23, 24, 25, 26, 28, 29,
995 30, 31, 33, 35, 37, 38,
996 40, 42, 45, 47, 48,
997 50, 52, 54, /* trail */ 54, 56, /* trail */ 56, 59, /* trail */ 59,
998 63, 64, /* trail */ 64, /* reset only 67, */
999 68, 69,
1000 71, 72
1001 };
1002
1003 if(!testConvertToUnicode(sampleText, sizeof(sampleText),
1004 expected, ARRAY_LENGTH(expected), "BOCU-1",
1005 UCNV_TO_U_CALLBACK_SKIP, offsets, NULL, 0)
1006 ) {
1007 log_err("BOCU-1->u with skip did not match.\n");
1008 }
1009 }
1010
1011 log_verbose("Testing toUnicode for CESU-8 with UCNV_TO_U_CALLBACK_SKIP\n");
1012 {
1013 const uint8_t sampleText[]={
1014 0x61, /* 0 'a' */
1015 0xc0, 0x80, /* 1 non-shortest form */
1016 0xc4, 0xb5, /* 3 U+0135 */
1017 0xed, 0x80, 0xa0, /* 5 Hangul U+d020 */
1018 0xed, 0xa0, 0x81, 0xed, 0xb0, 0x81, /* 8 surrogate pair for U+10401 */
1019 0xee, 0x80, 0x80, /* 14 PUA U+e000 */
1020 0xed, 0xb0, 0x81, /* 17 unpaired trail surrogate U+dc01 */
1021 0xf0, 0x90, 0x80, 0x80, /* 20 illegal 4-byte form for U+10000 */
1022 0x62, /* 24 'b' */
1023 0xed, 0xa0, 0x81, /* 25 unpaired lead surrogate U+d801 */
1024 0xed, 0xa0, /* 28 incomplete sequence */
1025 0xd0, 0x80 /* 30 U+0400 */
1026 };
1027 UChar expected[]={
1028 0x0061,
1029 /* skip */
1030 0x0135,
1031 0xd020,
1032 0xd801, 0xdc01,
1033 0xe000,
1034 0xdc01,
1035 /* skip */
1036 0x0062,
1037 0xd801,
1038 0x0400
1039 };
1040 int32_t offsets[]={
1041 0,
1042 /* skip 1, */
1043 3,
1044 5,
1045 8, 11,
1046 14,
1047 17,
1048 /* skip 20, 20, */
1049 24,
1050 25,
1051 /* skip 28 */
1052 30
1053 };
1054
1055 /* without offsets */
1056 if(!testConvertToUnicode(sampleText, sizeof(sampleText),
1057 expected, ARRAY_LENGTH(expected), "CESU-8",
1058 UCNV_TO_U_CALLBACK_SKIP, NULL, NULL, 0)
1059 ) {
1060 log_err("CESU-8->u with skip did not match.\n");
1061 }
1062
1063 /* with offsets */
1064 if(!testConvertToUnicode(sampleText, sizeof(sampleText),
1065 expected, ARRAY_LENGTH(expected), "CESU-8",
1066 UCNV_TO_U_CALLBACK_SKIP, offsets, NULL, 0)
1067 ) {
1068 log_err("CESU-8->u with skip did not match.\n");
1069 }
1070 }
1071 }
1072
1073 static void TestStop(int32_t inputsize, int32_t outputsize)
1074 {
1075 static const UChar sampleText[] = { 0x0000, 0xAC00, 0xAC01, 0xEF67, 0xD700 };
1076 static const UChar sampleText2[] = { 0x6D63, 0x6D64, 0x6D65, 0x6D66 };
1077
1078 static const uint8_t expstopIBM_949[]= {
1079 0x00, 0xb0, 0xa1, 0xb0, 0xa2};
1080
1081 static const uint8_t expstopIBM_943[] = {
1082 0x9f, 0xaf, 0x9f, 0xb1};
1083
1084 static const uint8_t expstopIBM_930[] = {
1085 0x0e, 0x5d, 0x5f, 0x5d, 0x63};
1086
1087 static const UChar IBM_949stoptoUnicode[]= {0x0000, 0xAC00, 0xAC01};
1088 static const UChar IBM_943stoptoUnicode[]= { 0x6D63, 0x6D64};
1089 static const UChar IBM_930stoptoUnicode[]= { 0x6D63, 0x6D64};
1090
1091
1092 static const int32_t toIBM949Offsstop [] = { 0, 1, 1, 2, 2};
1093 static const int32_t toIBM943Offsstop [] = { 0, 0, 1, 1};
1094 static const int32_t toIBM930Offsstop [] = { 0, 0, 0, 1, 1};
1095
1096 static const int32_t fromIBM949Offs [] = { 0, 1, 3};
1097 static const int32_t fromIBM943Offs [] = { 0, 2};
1098 static const int32_t fromIBM930Offs [] = { 1, 3};
1099
1100 gInBufferSize = inputsize;
1101 gOutBufferSize = outputsize;
1102
1103 /*From Unicode*/
1104
1105 #if !UCONFIG_NO_LEGACY_CONVERSION
1106 if(!testConvertFromUnicode(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
1107 expstopIBM_949, sizeof(expstopIBM_949), "ibm-949",
1108 UCNV_FROM_U_CALLBACK_STOP, toIBM949Offsstop, NULL, 0 ))
1109 log_err("u-> ibm-949 with stop did not match.\n");
1110 if(!testConvertFromUnicode(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
1111 expstopIBM_943, sizeof(expstopIBM_943), "ibm-943",
1112 UCNV_FROM_U_CALLBACK_STOP, toIBM943Offsstop , NULL, 0))
1113 log_err("u-> ibm-943 with stop did not match.\n");
1114 if(!testConvertFromUnicode(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
1115 expstopIBM_930, sizeof(expstopIBM_930), "ibm-930",
1116 UCNV_FROM_U_CALLBACK_STOP, toIBM930Offsstop, NULL, 0 ))
1117 log_err("u-> ibm-930 with stop did not match.\n");
1118
1119 log_verbose("Testing fromUnicode with UCNV_FROM_U_CALLBACK_STOP \n");
1120 {
1121 static const UChar inputTest[] = { 0x0061, 0xd801, 0xdc01, 0xd801, 0x0061 };
1122 static const uint8_t toIBM943[]= { 0x61,};
1123 static const int32_t offset[]= {0,} ;
1124
1125 /*EUC_JP*/
1126 static const UChar euc_jp_inputText[]={ 0x0061, 0x4edd, 0x5bec, 0xd801, 0xdc01, 0xd801, 0x0061, 0x00a2 };
1127 static const uint8_t to_euc_jp[]={ 0x61, 0xa1, 0xb8, 0x8f, 0xf4, 0xae,};
1128 static const int32_t fromEUC_JPOffs [] ={ 0, 1, 1, 2, 2, 2,};
1129
1130 /*EUC_TW*/
1131 static const UChar euc_tw_inputText[]={ 0x0061, 0x2295, 0x5BF2, 0xd801, 0xdc01, 0xd801, 0x0061, 0x8706, 0x8a, };
1132 static const uint8_t to_euc_tw[]={
1133 0x61, 0xa2, 0xd3, 0x8e, 0xa2, 0xdc, 0xe5,};
1134 static const int32_t from_euc_twOffs [] ={ 0, 1, 1, 2, 2, 2, 2,};
1135
1136 /*ISO-2022-JP*/
1137 static const UChar iso_2022_jp_inputText[]={0x0041, 0x00E9, 0x0042, };
1138 static const uint8_t to_iso_2022_jp[]={
1139 0x41,
1140
1141 };
1142 static const int32_t from_iso_2022_jpOffs [] ={0,};
1143
1144 /*ISO-2022-cn*/
1145 static const UChar iso_2022_cn_inputText[]={ 0x0041, 0x3712, 0x0042, };
1146 static const uint8_t to_iso_2022_cn[]={
1147 0x41,
1148
1149 };
1150 static const int32_t from_iso_2022_cnOffs [] ={
1151 0,0,
1152 2,2,
1153 };
1154
1155 /*ISO-2022-kr*/
1156 static const UChar iso_2022_kr_inputText[]={ 0x0041, 0x03A0,0x3712/*unassigned*/,0x03A0, 0x0042, };
1157 static const uint8_t to_iso_2022_kr[]={
1158 0x1b, 0x24, 0x29, 0x43,
1159 0x41,
1160 0x0e, 0x25, 0x50,
1161 };
1162 static const int32_t from_iso_2022_krOffs [] ={
1163 -1,-1,-1,-1,
1164 0,
1165 1,1,1,
1166 };
1167
1168 /* HZ encoding */
1169 static const UChar hz_inputText[]={ 0x0041, 0x03A0,0x0662/*unassigned*/,0x03A0, 0x0042, };
1170
1171 static const uint8_t to_hz[]={
1172 0x7e, 0x7d, 0x41,
1173 0x7e, 0x7b, 0x26, 0x30,
1174
1175 };
1176 static const int32_t from_hzOffs [] ={
1177 0, 0,0,
1178 1,1,1,1,
1179 };
1180
1181 /*ISCII*/
1182 static const UChar iscii_inputText[]={ 0x0041, 0x3712, 0x0042, };
1183 static const uint8_t to_iscii[]={
1184 0x41,
1185 };
1186 static const int32_t from_isciiOffs [] ={
1187 0,
1188 };
1189
1190 if(!testConvertFromUnicode(inputTest, sizeof(inputTest)/sizeof(inputTest[0]),
1191 toIBM943, sizeof(toIBM943), "ibm-943",
1192 UCNV_FROM_U_CALLBACK_STOP, offset, NULL, 0 ))
1193 log_err("u-> ibm-943 with stop did not match.\n");
1194
1195 if(!testConvertFromUnicode(euc_jp_inputText, sizeof(euc_jp_inputText)/sizeof(euc_jp_inputText[0]),
1196 to_euc_jp, sizeof(to_euc_jp), "euc-jp",
1197 UCNV_FROM_U_CALLBACK_STOP, fromEUC_JPOffs, NULL, 0 ))
1198 log_err("u-> euc-jp with stop did not match.\n");
1199
1200 if(!testConvertFromUnicode(euc_tw_inputText, sizeof(euc_tw_inputText)/sizeof(euc_tw_inputText[0]),
1201 to_euc_tw, sizeof(to_euc_tw), "euc-tw",
1202 UCNV_FROM_U_CALLBACK_STOP, from_euc_twOffs, NULL, 0 ))
1203 log_err("u-> euc-tw with stop did not match.\n");
1204
1205 if(!testConvertFromUnicode(iso_2022_jp_inputText, sizeof(iso_2022_jp_inputText)/sizeof(iso_2022_jp_inputText[0]),
1206 to_iso_2022_jp, sizeof(to_iso_2022_jp), "iso-2022-jp",
1207 UCNV_FROM_U_CALLBACK_STOP, from_iso_2022_jpOffs, NULL, 0 ))
1208 log_err("u-> iso-2022-jp with stop did not match.\n");
1209
1210 if(!testConvertFromUnicode(iso_2022_jp_inputText, sizeof(iso_2022_jp_inputText)/sizeof(iso_2022_jp_inputText[0]),
1211 to_iso_2022_jp, sizeof(to_iso_2022_jp), "iso-2022-jp",
1212 UCNV_FROM_U_CALLBACK_STOP, from_iso_2022_jpOffs, NULL, 0 ))
1213 log_err("u-> iso-2022-jp with stop did not match.\n");
1214
1215 if(!testConvertFromUnicode(iso_2022_cn_inputText, sizeof(iso_2022_cn_inputText)/sizeof(iso_2022_cn_inputText[0]),
1216 to_iso_2022_cn, sizeof(to_iso_2022_cn), "iso-2022-cn",
1217 UCNV_FROM_U_CALLBACK_STOP, from_iso_2022_cnOffs, NULL, 0 ))
1218 log_err("u-> iso-2022-cn with stop did not match.\n");
1219
1220 if(!testConvertFromUnicode(iso_2022_kr_inputText, sizeof(iso_2022_kr_inputText)/sizeof(iso_2022_kr_inputText[0]),
1221 to_iso_2022_kr, sizeof(to_iso_2022_kr), "iso-2022-kr",
1222 UCNV_FROM_U_CALLBACK_STOP, from_iso_2022_krOffs, NULL, 0 ))
1223 log_err("u-> iso-2022-kr with stop did not match.\n");
1224
1225 if(!testConvertFromUnicode(hz_inputText, sizeof(hz_inputText)/sizeof(hz_inputText[0]),
1226 to_hz, sizeof(to_hz), "HZ",
1227 UCNV_FROM_U_CALLBACK_STOP, from_hzOffs, NULL, 0 ))
1228 log_err("u-> HZ with stop did not match.\n");\
1229
1230 if(!testConvertFromUnicode(iscii_inputText, sizeof(iscii_inputText)/sizeof(iscii_inputText[0]),
1231 to_iscii, sizeof(to_iscii), "ISCII,version=0",
1232 UCNV_FROM_U_CALLBACK_STOP, from_isciiOffs, NULL, 0 ))
1233 log_err("u-> iscii with stop did not match.\n");
1234
1235
1236 }
1237 #endif
1238
1239 log_verbose("Testing fromUnicode for SCSU with UCNV_FROM_U_CALLBACK_STOP \n");
1240 {
1241 static const UChar SCSU_inputText[]={ 0x0041, 0xd801/*illegal*/, 0x0042, };
1242
1243 static const uint8_t to_SCSU[]={
1244 0x41,
1245
1246 };
1247 int32_t from_SCSUOffs [] ={
1248 0,
1249
1250 };
1251 if(!testConvertFromUnicode(SCSU_inputText, sizeof(SCSU_inputText)/sizeof(SCSU_inputText[0]),
1252 to_SCSU, sizeof(to_SCSU), "SCSU",
1253 UCNV_FROM_U_CALLBACK_STOP, from_SCSUOffs, NULL, 0 ))
1254 log_err("u-> SCSU with skip did not match.\n");
1255
1256 }
1257
1258 /*to Unicode*/
1259
1260 #if !UCONFIG_NO_LEGACY_CONVERSION
1261 if(!testConvertToUnicode(expstopIBM_949, sizeof(expstopIBM_949),
1262 IBM_949stoptoUnicode, sizeof(IBM_949stoptoUnicode)/sizeof(IBM_949stoptoUnicode[0]),"ibm-949",
1263 UCNV_TO_U_CALLBACK_STOP, fromIBM949Offs, NULL, 0 ))
1264 log_err("ibm-949->u with stop did not match.\n");
1265 if(!testConvertToUnicode(expstopIBM_943, sizeof(expstopIBM_943),
1266 IBM_943stoptoUnicode, sizeof(IBM_943stoptoUnicode)/sizeof(IBM_943stoptoUnicode[0]),"ibm-943",
1267 UCNV_TO_U_CALLBACK_STOP, fromIBM943Offs, NULL, 0 ))
1268 log_err("ibm-943->u with stop did not match.\n");
1269 if(!testConvertToUnicode(expstopIBM_930, sizeof(expstopIBM_930),
1270 IBM_930stoptoUnicode, sizeof(IBM_930stoptoUnicode)/sizeof(IBM_930stoptoUnicode[0]),"ibm-930",
1271 UCNV_TO_U_CALLBACK_STOP, fromIBM930Offs, NULL, 0 ))
1272 log_err("ibm-930->u with stop did not match.\n");
1273
1274 log_verbose("Testing toUnicode with UCNV_TO_U_CALLBACK_STOP \n");
1275 {
1276
1277 static const uint8_t sampleTxtEBCIDIC_STATEFUL [] ={
1278 0x0e, 0x5d, 0x5f , 0x41, 0x79, 0x41, 0x44
1279 };
1280 static const UChar EBCIDIC_STATEFUL_toUnicode[] ={ 0x6d63 };
1281 static const int32_t from_EBCIDIC_STATEFULOffsets []={ 1};
1282
1283
1284 /*EUC-JP*/
1285 static const uint8_t sampleTxt_euc_jp[]={ 0x61, 0xa1, 0xb8, 0x8f, 0xf4, 0xae,
1286 0x8f, 0xda, 0xa1, /*unassigned*/
1287 0x8e, 0xe0,
1288 };
1289 static const UChar euc_jptoUnicode[]={ 0x0061, 0x4edd, 0x5bec};
1290 static const int32_t from_euc_jpOffs [] ={ 0, 1, 3};
1291
1292 /*EUC_TW*/
1293 static const uint8_t sampleTxt_euc_tw[]={ 0x61, 0xa2, 0xd3, 0x8e, 0xa2, 0xdc, 0xe5,
1294 0x8e, 0xaa, 0xbb, 0xcc,/*unassigned*/
1295 0xe6, 0xca, 0x8a,
1296 };
1297 UChar euc_twtoUnicode[]={ 0x0061, 0x2295, 0x5BF2};
1298 int32_t from_euc_twOffs [] ={ 0, 1, 3};
1299
1300
1301
1302 if(!testConvertToUnicode(sampleTxtEBCIDIC_STATEFUL, sizeof(sampleTxtEBCIDIC_STATEFUL),
1303 EBCIDIC_STATEFUL_toUnicode, sizeof(EBCIDIC_STATEFUL_toUnicode)/sizeof(EBCIDIC_STATEFUL_toUnicode[0]),"ibm-930",
1304 UCNV_TO_U_CALLBACK_STOP, from_EBCIDIC_STATEFULOffsets, NULL, 0 ))
1305 log_err("EBCIDIC_STATEFUL->u with stop did not match.\n");
1306
1307 if(!testConvertToUnicode(sampleTxt_euc_jp, sizeof(sampleTxt_euc_jp),
1308 euc_jptoUnicode, sizeof(euc_jptoUnicode)/sizeof(euc_jptoUnicode[0]),"euc-jp",
1309 UCNV_TO_U_CALLBACK_STOP, from_euc_jpOffs , NULL, 0))
1310 log_err("euc-jp->u with stop did not match.\n");
1311
1312 if(!testConvertToUnicode(sampleTxt_euc_tw, sizeof(sampleTxt_euc_tw),
1313 euc_twtoUnicode, sizeof(euc_twtoUnicode)/sizeof(euc_twtoUnicode[0]),"euc-tw",
1314 UCNV_TO_U_CALLBACK_STOP, from_euc_twOffs, NULL, 0 ))
1315 log_err("euc-tw->u with stop did not match.\n");
1316 }
1317 #endif
1318
1319 log_verbose("Testing toUnicode for UTF-8 with UCNV_TO_U_CALLBACK_STOP \n");
1320 {
1321 static const uint8_t sampleText1[] = { 0x31, 0xe4, 0xba, 0x8c,
1322 0xe0, 0x80, 0x61,};
1323 static const UChar expected1[] = { 0x0031, 0x4e8c,};
1324 static const int32_t offsets1[] = { 0x0000, 0x0001};
1325
1326 if(!testConvertToUnicode(sampleText1, sizeof(sampleText1),
1327 expected1, sizeof(expected1)/sizeof(expected1[0]),"utf8",
1328 UCNV_TO_U_CALLBACK_STOP, offsets1, NULL, 0 ))
1329 log_err("utf8->u with stop did not match.\n");;
1330 }
1331 log_verbose("Testing toUnicode for SCSU with UCNV_TO_U_CALLBACK_STOP \n");
1332 {
1333 static const uint8_t sampleText1[] = { 0xba, 0x8c,0xF8, 0x61,0x0c, 0x0c,0x04};
1334 static const UChar expected1[] = { 0x00ba, 0x008c, 0x00f8, 0x0061};
1335 static const int32_t offsets1[] = { 0x0000, 0x0001,0x0002,0x0003};
1336
1337 if(!testConvertToUnicode(sampleText1, sizeof(sampleText1),
1338 expected1, sizeof(expected1)/sizeof(expected1[0]),"SCSU",
1339 UCNV_TO_U_CALLBACK_STOP, offsets1, NULL, 0 ))
1340 log_err("scsu->u with stop did not match.\n");;
1341 }
1342
1343 }
1344
1345 static void TestSub(int32_t inputsize, int32_t outputsize)
1346 {
1347 static const UChar sampleText[] = { 0x0000, 0xAC00, 0xAC01, 0xEF67, 0xD700 };
1348 static const UChar sampleText2[]= { 0x6D63, 0x6D64, 0x6D65, 0x6D66 };
1349
1350 static const uint8_t expsubIBM_949[] =
1351 { 0x00, 0xb0, 0xa1, 0xb0, 0xa2, 0xaf, 0xfe, 0xc8, 0xd3 };
1352
1353 static const uint8_t expsubIBM_943[] = {
1354 0x9f, 0xaf, 0x9f, 0xb1, 0xfc, 0xfc, 0x89, 0x59 };
1355
1356 static const uint8_t expsubIBM_930[] = {
1357 0x0e, 0x5d, 0x5f, 0x5d, 0x63, 0xfe, 0xfe, 0x46, 0x6b, 0x0f };
1358
1359 static const UChar IBM_949subtoUnicode[]= {0x0000, 0xAC00, 0xAC01, 0xfffd, 0xD700 };
1360 static const UChar IBM_943subtoUnicode[]= {0x6D63, 0x6D64, 0xfffd, 0x6D66 };
1361 static const UChar IBM_930subtoUnicode[]= {0x6D63, 0x6D64, 0xfffd, 0x6D66 };
1362
1363 static const int32_t toIBM949Offssub [] ={ 0, 1, 1, 2, 2, 3, 3, 4, 4 };
1364 static const int32_t toIBM943Offssub [] ={ 0, 0, 1, 1, 2, 2, 3, 3 };
1365 static const int32_t toIBM930Offssub [] ={ 0, 0, 0, 1, 1, 2, 2, 3, 3, 3 };
1366
1367 static const int32_t fromIBM949Offs [] = { 0, 1, 3, 5, 7 };
1368 static const int32_t fromIBM943Offs [] = { 0, 2, 4, 6 };
1369 static const int32_t fromIBM930Offs [] = { 1, 3, 5, 7 };
1370
1371 gInBufferSize = inputsize;
1372 gOutBufferSize = outputsize;
1373
1374 /*from unicode*/
1375
1376 #if !UCONFIG_NO_LEGACY_CONVERSION
1377 if(!testConvertFromUnicode(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
1378 expsubIBM_949, sizeof(expsubIBM_949), "ibm-949",
1379 UCNV_FROM_U_CALLBACK_SUBSTITUTE, toIBM949Offssub, NULL, 0 ))
1380 log_err("u-> ibm-949 with subst did not match.\n");
1381 if(!testConvertFromUnicode(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
1382 expsubIBM_943, sizeof(expsubIBM_943), "ibm-943",
1383 UCNV_FROM_U_CALLBACK_SUBSTITUTE, toIBM943Offssub , NULL, 0))
1384 log_err("u-> ibm-943 with subst did not match.\n");
1385 if(!testConvertFromUnicode(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
1386 expsubIBM_930, sizeof(expsubIBM_930), "ibm-930",
1387 UCNV_FROM_U_CALLBACK_SUBSTITUTE, toIBM930Offssub, NULL, 0 ))
1388 log_err("u-> ibm-930 with subst did not match.\n");
1389
1390 log_verbose("Testing fromUnicode with UCNV_FROM_U_CALLBACK_SUBSTITUTE \n");
1391 {
1392 static const UChar inputTest[] = { 0x0061, 0xd801, 0xdc01, 0xd801, 0x0061 };
1393 static const uint8_t toIBM943[]= { 0x61, 0xfc, 0xfc, 0xfc, 0xfc, 0x61 };
1394 static const int32_t offset[]= {0, 1, 1, 3, 3, 4};
1395
1396
1397 /* EUC_JP*/
1398 static const UChar euc_jp_inputText[]={ 0x0061, 0x4edd, 0x5bec, 0xd801, 0xdc01, 0xd801, 0x0061, 0x00a2 };
1399 static const uint8_t to_euc_jp[]={ 0x61, 0xa1, 0xb8, 0x8f, 0xf4, 0xae,
1400 0xf4, 0xfe, 0xf4, 0xfe,
1401 0x61, 0x8e, 0xe0,
1402 };
1403 static const int32_t fromEUC_JPOffs [] ={ 0, 1, 1, 2, 2, 2, 3, 3, 5, 5, 6, 7, 7};
1404
1405 /*EUC_TW*/
1406 static const UChar euc_tw_inputText[]={ 0x0061, 0x2295, 0x5BF2, 0xd801, 0xdc01, 0xd801, 0x0061, 0x8706, 0x8a, };
1407 static const uint8_t to_euc_tw[]={
1408 0x61, 0xa2, 0xd3, 0x8e, 0xa2, 0xdc, 0xe5,
1409 0xfd, 0xfe, 0xfd, 0xfe,
1410 0x61, 0xe6, 0xca, 0x8a,
1411 };
1412
1413 static const int32_t from_euc_twOffs [] ={ 0, 1, 1, 2, 2, 2, 2, 3, 3, 5, 5, 6, 7, 7, 8,};
1414
1415 if(!testConvertFromUnicode(inputTest, sizeof(inputTest)/sizeof(inputTest[0]),
1416 toIBM943, sizeof(toIBM943), "ibm-943",
1417 UCNV_FROM_U_CALLBACK_SUBSTITUTE, offset, NULL, 0 ))
1418 log_err("u-> ibm-943 with substitute did not match.\n");
1419
1420 if(!testConvertFromUnicode(euc_jp_inputText, sizeof(euc_jp_inputText)/sizeof(euc_jp_inputText[0]),
1421 to_euc_jp, sizeof(to_euc_jp), "euc-jp",
1422 UCNV_FROM_U_CALLBACK_SUBSTITUTE, fromEUC_JPOffs, NULL, 0 ))
1423 log_err("u-> euc-jp with substitute did not match.\n");
1424
1425 if(!testConvertFromUnicode(euc_tw_inputText, sizeof(euc_tw_inputText)/sizeof(euc_tw_inputText[0]),
1426 to_euc_tw, sizeof(to_euc_tw), "euc-tw",
1427 UCNV_FROM_U_CALLBACK_SUBSTITUTE, from_euc_twOffs, NULL, 0 ))
1428 log_err("u-> euc-tw with substitute did not match.\n");
1429 }
1430 #endif
1431
1432 log_verbose("Testing fromUnicode for SCSU with UCNV_FROM_U_CALLBACK_SUBSTITUTE \n");
1433 {
1434 UChar SCSU_inputText[]={ 0x0041, 0xd801/*illegal*/, 0x0042, };
1435
1436 const uint8_t to_SCSU[]={
1437 0x41,
1438 0x0e, 0xff,0xfd,
1439 0x42
1440
1441
1442 };
1443 int32_t from_SCSUOffs [] ={
1444 0,
1445 1,1,1,
1446 2,
1447
1448 };
1449 const uint8_t to_SCSU_1[]={
1450 0x41,
1451
1452 };
1453 int32_t from_SCSUOffs_1 [] ={
1454 0,
1455
1456 };
1457 if(!testConvertFromUnicode(SCSU_inputText, sizeof(SCSU_inputText)/sizeof(SCSU_inputText[0]),
1458 to_SCSU, sizeof(to_SCSU), "SCSU",
1459 UCNV_FROM_U_CALLBACK_SUBSTITUTE, from_SCSUOffs, NULL, 0 ))
1460 log_err("u-> SCSU with substitute did not match.\n");
1461
1462 if(!testConvertFromUnicodeWithContext(SCSU_inputText, sizeof(SCSU_inputText)/sizeof(SCSU_inputText[0]),
1463 to_SCSU_1, sizeof(to_SCSU_1), "SCSU",
1464 UCNV_FROM_U_CALLBACK_SUBSTITUTE, from_SCSUOffs_1, NULL, 0,"i",U_ILLEGAL_CHAR_FOUND ))
1465 log_err("u-> SCSU with substitute did not match.\n");
1466 }
1467
1468 log_verbose("Testing fromUnicode for UTF-8 with UCNV_FROM_U_CALLBACK_SUBSTITUTE\n");
1469 {
1470 static const UChar testinput[]={ 0x20ac, 0xd801, 0xdc01, 0xdc01, 0xd801, 0xffff, 0x0061,};
1471 static const uint8_t expectedUTF8[]= { 0xe2, 0x82, 0xac,
1472 0xf0, 0x90, 0x90, 0x81,
1473 0xef, 0xbf, 0xbd, 0xef, 0xbf, 0xbd,
1474 0xef, 0xbf, 0xbf, 0x61,
1475
1476 };
1477 static const int32_t offsets[]={ 0, 0, 0, 1, 1, 1, 1, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6 };
1478 if(!testConvertFromUnicode(testinput, sizeof(testinput)/sizeof(testinput[0]),
1479 expectedUTF8, sizeof(expectedUTF8), "utf8",
1480 UCNV_FROM_U_CALLBACK_SUBSTITUTE, offsets, NULL, 0 )) {
1481 log_err("u-> utf8 with stop did not match.\n");
1482 }
1483 }
1484
1485 log_verbose("Testing fromUnicode for UTF-16 with UCNV_FROM_U_CALLBACK_SUBSTITUTE\n");
1486 {
1487 static const UChar in[]={ 0x0041, 0xfeff };
1488
1489 static const uint8_t out[]={
1490 #if U_IS_BIG_ENDIAN
1491 0xfe, 0xff,
1492 0x00, 0x41,
1493 0xfe, 0xff
1494 #else
1495 0xff, 0xfe,
1496 0x41, 0x00,
1497 0xff, 0xfe
1498 #endif
1499 };
1500 static const int32_t offsets[]={
1501 -1, -1, 0, 0, 1, 1
1502 };
1503
1504 if(!testConvertFromUnicode(in, ARRAY_LENGTH(in),
1505 out, sizeof(out), "UTF-16",
1506 UCNV_FROM_U_CALLBACK_SUBSTITUTE, offsets, NULL, 0)
1507 ) {
1508 log_err("u->UTF-16 with substitute did not match.\n");
1509 }
1510 }
1511
1512 log_verbose("Testing fromUnicode for UTF-32 with UCNV_FROM_U_CALLBACK_SUBSTITUTE\n");
1513 {
1514 static const UChar in[]={ 0x0041, 0xfeff };
1515
1516 static const uint8_t out[]={
1517 #if U_IS_BIG_ENDIAN
1518 0x00, 0x00, 0xfe, 0xff,
1519 0x00, 0x00, 0x00, 0x41,
1520 0x00, 0x00, 0xfe, 0xff
1521 #else
1522 0xff, 0xfe, 0x00, 0x00,
1523 0x41, 0x00, 0x00, 0x00,
1524 0xff, 0xfe, 0x00, 0x00
1525 #endif
1526 };
1527 static const int32_t offsets[]={
1528 -1, -1, -1, -1, 0, 0, 0, 0, 1, 1, 1, 1
1529 };
1530
1531 if(!testConvertFromUnicode(in, ARRAY_LENGTH(in),
1532 out, sizeof(out), "UTF-32",
1533 UCNV_FROM_U_CALLBACK_SUBSTITUTE, offsets, NULL, 0)
1534 ) {
1535 log_err("u->UTF-32 with substitute did not match.\n");
1536 }
1537 }
1538
1539 /*to unicode*/
1540
1541 #if !UCONFIG_NO_LEGACY_CONVERSION
1542 if(!testConvertToUnicode(expsubIBM_949, sizeof(expsubIBM_949),
1543 IBM_949subtoUnicode, sizeof(IBM_949subtoUnicode)/sizeof(IBM_949subtoUnicode[0]),"ibm-949",
1544 UCNV_TO_U_CALLBACK_SUBSTITUTE, fromIBM949Offs, NULL, 0 ))
1545 log_err("ibm-949->u with substitute did not match.\n");
1546 if(!testConvertToUnicode(expsubIBM_943, sizeof(expsubIBM_943),
1547 IBM_943subtoUnicode, sizeof(IBM_943subtoUnicode)/sizeof(IBM_943subtoUnicode[0]),"ibm-943",
1548 UCNV_TO_U_CALLBACK_SUBSTITUTE, fromIBM943Offs, NULL, 0 ))
1549 log_err("ibm-943->u with substitute did not match.\n");
1550 if(!testConvertToUnicode(expsubIBM_930, sizeof(expsubIBM_930),
1551 IBM_930subtoUnicode, sizeof(IBM_930subtoUnicode)/sizeof(IBM_930subtoUnicode[0]),"ibm-930",
1552 UCNV_TO_U_CALLBACK_SUBSTITUTE, fromIBM930Offs, NULL, 0 ))
1553 log_err("ibm-930->u with substitute did not match.\n");
1554
1555 log_verbose("Testing toUnicode with UCNV_TO_U_CALLBACK_SUBSTITUTE \n");
1556 {
1557
1558 const uint8_t sampleTxtEBCIDIC_STATEFUL [] ={
1559 0x0e, 0x5d, 0x5f , 0x41, 0x79, 0x41, 0x44
1560 };
1561 UChar EBCIDIC_STATEFUL_toUnicode[] ={ 0x6d63, 0xfffd, 0x03b4
1562 };
1563 int32_t from_EBCIDIC_STATEFULOffsets []={ 1, 3, 5};
1564
1565
1566 /* EUC_JP*/
1567 const uint8_t sampleTxt_euc_jp[]={ 0x61, 0xa1, 0xb8, 0x8f, 0xf4, 0xae,
1568 0x8f, 0xda, 0xa1, /*unassigned*/
1569 0x8e, 0xe0, 0x8a
1570 };
1571 UChar euc_jptoUnicode[]={ 0x0061, 0x4edd, 0x5bec, 0xfffd, 0x00a2, 0x008a };
1572 int32_t from_euc_jpOffs [] ={ 0, 1, 3, 6, 9, 11 };
1573
1574 /*EUC_TW*/
1575 const uint8_t sampleTxt_euc_tw[]={
1576 0x61, 0xa2, 0xd3, 0x8e, 0xa2, 0xdc, 0xe5,
1577 0x8e, 0xaa, 0xbb, 0xcc,/*unassigned*/
1578 0xe6, 0xca, 0x8a,
1579 };
1580 UChar euc_twtoUnicode[]={ 0x0061, 0x2295, 0x5BF2, 0xfffd, 0x8706, 0x8a, };
1581 int32_t from_euc_twOffs [] ={ 0, 1, 3, 7, 11, 13};
1582
1583
1584 if(!testConvertToUnicode(sampleTxtEBCIDIC_STATEFUL, sizeof(sampleTxtEBCIDIC_STATEFUL),
1585 EBCIDIC_STATEFUL_toUnicode, sizeof(EBCIDIC_STATEFUL_toUnicode)/sizeof(EBCIDIC_STATEFUL_toUnicode[0]),"ibm-930",
1586 UCNV_TO_U_CALLBACK_SUBSTITUTE, from_EBCIDIC_STATEFULOffsets, NULL, 0 ))
1587 log_err("EBCIDIC_STATEFUL->u with substitute did not match.\n");
1588
1589
1590 if(!testConvertToUnicode(sampleTxt_euc_jp, sizeof(sampleTxt_euc_jp),
1591 euc_jptoUnicode, sizeof(euc_jptoUnicode)/sizeof(euc_jptoUnicode[0]),"euc-jp",
1592 UCNV_TO_U_CALLBACK_SUBSTITUTE, from_euc_jpOffs, NULL, 0 ))
1593 log_err("euc-jp->u with substitute did not match.\n");
1594
1595
1596 if(!testConvertToUnicode(sampleTxt_euc_tw, sizeof(sampleTxt_euc_tw),
1597 euc_twtoUnicode, sizeof(euc_twtoUnicode)/sizeof(euc_twtoUnicode[0]),"euc-tw",
1598 UCNV_TO_U_CALLBACK_SUBSTITUTE, from_euc_twOffs, NULL, 0 ))
1599 log_err("euc-tw->u with substitute did not match.\n");
1600
1601
1602 if(!testConvertToUnicodeWithContext(sampleTxt_euc_jp, sizeof(sampleTxt_euc_jp),
1603 euc_jptoUnicode, sizeof(euc_jptoUnicode)/sizeof(euc_jptoUnicode[0]),"euc-jp",
1604 UCNV_TO_U_CALLBACK_SUBSTITUTE, from_euc_jpOffs, NULL, 0 ,"i", U_ILLEGAL_CHAR_FOUND))
1605 log_err("euc-jp->u with substitute did not match.\n");
1606 }
1607 #endif
1608
1609 log_verbose("Testing toUnicode for UTF-8 with UCNV_TO_U_CALLBACK_SUBSTITUTE \n");
1610 {
1611 const uint8_t sampleText1[] = { 0x31, 0xe4, 0xba, 0x8c,
1612 0xe0, 0x80, 0x61,};
1613 UChar expected1[] = { 0x0031, 0x4e8c, 0xfffd, 0x0061};
1614 int32_t offsets1[] = { 0x0000, 0x0001, 0x0004, 0x0006};
1615
1616 if(!testConvertToUnicode(sampleText1, sizeof(sampleText1),
1617 expected1, sizeof(expected1)/sizeof(expected1[0]),"utf8",
1618 UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets1, NULL, 0 ))
1619 log_err("utf8->u with substitute did not match.\n");;
1620 }
1621 log_verbose("Testing toUnicode for SCSU with UCNV_TO_U_CALLBACK_SUBSTITUTE \n");
1622 {
1623 const uint8_t sampleText1[] = { 0xba, 0x8c,0xF8, 0x61,0x0c, 0x0c,};
1624 UChar expected1[] = { 0x00ba, 0x008c, 0x00f8, 0x0061,0xfffd,0xfffd};
1625 int32_t offsets1[] = { 0x0000, 0x0001,0x0002,0x0003,4,5};
1626
1627 if(!testConvertToUnicode(sampleText1, sizeof(sampleText1),
1628 expected1, sizeof(expected1)/sizeof(expected1[0]),"SCSU",
1629 UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets1, NULL, 0 ))
1630 log_err("scsu->u with stop did not match.\n");;
1631 }
1632
1633 #if !UCONFIG_NO_LEGACY_CONVERSION
1634 log_verbose("Testing ibm-930 subchar/subchar1\n");
1635 {
1636 static const UChar u1[]={ 0x6d63, 0x6d64, 0x6d65, 0x6d66, 0xdf };
1637 static const uint8_t s1[]={ 0x0e, 0x5d, 0x5f, 0x5d, 0x63, 0xfe, 0xfe, 0x46, 0x6b, 0x0f, 0x3f };
1638 static const int32_t offsets1[]={ 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4 };
1639
1640 static const UChar u2[]={ 0x6d63, 0x6d64, 0xfffd, 0x6d66, 0x1a };
1641 static const uint8_t s2[]={ 0x0e, 0x5d, 0x5f, 0x5d, 0x63, 0xfc, 0xfc, 0x46, 0x6b, 0x0f, 0x57 };
1642 static const int32_t offsets2[]={ 1, 3, 5, 7, 10 };
1643
1644 if(!testConvertFromUnicode(u1, ARRAY_LENGTH(u1), s1, ARRAY_LENGTH(s1), "ibm-930",
1645 UCNV_FROM_U_CALLBACK_SUBSTITUTE, offsets1, NULL, 0)
1646 ) {
1647 log_err("u->ibm-930 subchar/subchar1 did not match.\n");
1648 }
1649
1650 if(!testConvertToUnicode(s2, ARRAY_LENGTH(s2), u2, ARRAY_LENGTH(u2), "ibm-930",
1651 UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets2, NULL, 0)
1652 ) {
1653 log_err("ibm-930->u subchar/subchar1 did not match.\n");
1654 }
1655 }
1656
1657 log_verbose("Testing GB 18030 with substitute callbacks\n");
1658 {
1659 static const UChar u2[]={
1660 0x24, 0x7f, 0x80, 0x1f9, 0x20ac, 0x4e00, 0x9fa6, 0xffff, 0xd800, 0xdc00, 0xfffd, 0xdbff, 0xdfff };
1661 static const uint8_t gb2[]={
1662 0x24, 0x7f, 0x81, 0x30, 0x81, 0x30, 0xa8, 0xbf, 0xa2, 0xe3, 0xd2, 0xbb, 0x82, 0x35, 0x8f, 0x33, 0x84, 0x31, 0xa4, 0x39, 0x90, 0x30, 0x81, 0x30, 0xe3, 0x32, 0x9a, 0x36, 0xe3, 0x32, 0x9a, 0x35 };
1663 static const int32_t offsets2[]={
1664 0, 1, 2, 6, 8, 10, 12, 16, 20, 20, 24, 28, 28 };
1665
1666 if(!testConvertToUnicode(gb2, ARRAY_LENGTH(gb2), u2, ARRAY_LENGTH(u2), "gb18030",
1667 UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets2, NULL, 0)
1668 ) {
1669 log_err("gb18030->u with substitute did not match.\n");
1670 }
1671 }
1672 #endif
1673
1674 log_verbose("Testing UTF-7 toUnicode with substitute callbacks\n");
1675 {
1676 static const uint8_t utf7[]={
1677 /* a~ a+AB~ a+AB\x0c a+AB- a+AB. a+. */
1678 0x61, 0x7e, 0x61, 0x2b, 0x41, 0x42, 0x7e, 0x61, 0x2b, 0x41, 0x42, 0x0c, 0x61, 0x2b, 0x41, 0x42, 0x2d, 0x61, 0x2b, 0x41, 0x42, 0x2e, 0x61, 0x2b, 0x2e
1679 };
1680 static const UChar unicode[]={
1681 0x61, 0xfffd, 0x61, 0xfffd, 0xfffd, 0x61, 0xfffd, 0xfffd, 0x61, 0xfffd, 0x61, 0xfffd, 0x2e, 0x61, 0xfffd, 0x2e
1682 };
1683 static const int32_t offsets[]={
1684 0, 1, 2, 4, 6, 7, 9, 11, 12, 14, 17, 19, 21, 22, 23, 24
1685 };
1686
1687 if(!testConvertToUnicode(utf7, ARRAY_LENGTH(utf7), unicode, ARRAY_LENGTH(unicode), "UTF-7",
1688 UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets, NULL, 0)
1689 ) {
1690 log_err("UTF-7->u with substitute did not match.\n");
1691 }
1692 }
1693
1694 log_verbose("Testing UTF-16 toUnicode with substitute callbacks\n");
1695 {
1696 static const uint8_t
1697 in1[]={ 0xfe, 0xff, 0x4e, 0x00, 0xfe, 0xff },
1698 in2[]={ 0xff, 0xfe, 0x4e, 0x00, 0xfe, 0xff },
1699 in3[]={ 0xfe, 0xfd, 0x4e, 0x00, 0xfe, 0xff };
1700
1701 static const UChar
1702 out1[]={ 0x4e00, 0xfeff },
1703 out2[]={ 0x004e, 0xfffe },
1704 out3[]={ 0xfefd, 0x4e00, 0xfeff };
1705
1706 static const int32_t
1707 offsets1[]={ 2, 4 },
1708 offsets2[]={ 2, 4 },
1709 offsets3[]={ 0, 2, 4 };
1710
1711 if(!testConvertToUnicode(in1, ARRAY_LENGTH(in1), out1, ARRAY_LENGTH(out1), "UTF-16",
1712 UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets1, NULL, 0)
1713 ) {
1714 log_err("UTF-16 (BE BOM)->u with substitute did not match.\n");
1715 }
1716
1717 if(!testConvertToUnicode(in2, ARRAY_LENGTH(in2), out2, ARRAY_LENGTH(out2), "UTF-16",
1718 UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets2, NULL, 0)
1719 ) {
1720 log_err("UTF-16 (LE BOM)->u with substitute did not match.\n");
1721 }
1722
1723 if(!testConvertToUnicode(in3, ARRAY_LENGTH(in3), out3, ARRAY_LENGTH(out3), "UTF-16",
1724 UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets3, NULL, 0)
1725 ) {
1726 log_err("UTF-16 (no BOM)->u with substitute did not match.\n");
1727 }
1728 }
1729
1730 log_verbose("Testing UTF-32 toUnicode with substitute callbacks\n");
1731 {
1732 static const uint8_t
1733 in1[]={ 0x00, 0x00, 0xfe, 0xff, 0x00, 0x10, 0x0f, 0x00, 0x00, 0x00, 0xfe, 0xff },
1734 in2[]={ 0xff, 0xfe, 0x00, 0x00, 0x00, 0x10, 0x0f, 0x00, 0xfe, 0xff, 0x00, 0x00 },
1735 in3[]={ 0x00, 0x00, 0xfe, 0xfe, 0x00, 0x10, 0x0f, 0x00, 0x00, 0x00, 0xd8, 0x40, 0x00, 0x00, 0xdc, 0x01 },
1736 in4[]={ 0x00, 0x01, 0x02, 0x03, 0x00, 0x11, 0x12, 0x00, 0x00, 0x00, 0x4e, 0x00 };
1737
1738 static const UChar
1739 out1[]={ UTF16_LEAD(0x100f00), UTF16_TRAIL(0x100f00), 0xfeff },
1740 out2[]={ UTF16_LEAD(0x0f1000), UTF16_TRAIL(0x0f1000), 0xfffe },
1741 out3[]={ 0xfefe, UTF16_LEAD(0x100f00), UTF16_TRAIL(0x100f00), 0xfffd, 0xfffd },
1742 out4[]={ UTF16_LEAD(0x10203), UTF16_TRAIL(0x10203), 0xfffd, 0x4e00 };
1743
1744 static const int32_t
1745 offsets1[]={ 4, 4, 8 },
1746 offsets2[]={ 4, 4, 8 },
1747 offsets3[]={ 0, 4, 4, 8, 12 },
1748 offsets4[]={ 0, 0, 4, 8 };
1749
1750 if(!testConvertToUnicode(in1, ARRAY_LENGTH(in1), out1, ARRAY_LENGTH(out1), "UTF-32",
1751 UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets1, NULL, 0)
1752 ) {
1753 log_err("UTF-32 (BE BOM)->u with substitute did not match.\n");
1754 }
1755
1756 if(!testConvertToUnicode(in2, ARRAY_LENGTH(in2), out2, ARRAY_LENGTH(out2), "UTF-32",
1757 UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets2, NULL, 0)
1758 ) {
1759 log_err("UTF-32 (LE BOM)->u with substitute did not match.\n");
1760 }
1761
1762 if(!testConvertToUnicode(in3, ARRAY_LENGTH(in3), out3, ARRAY_LENGTH(out3), "UTF-32",
1763 UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets3, NULL, 0)
1764 ) {
1765 log_err("UTF-32 (no BOM)->u with substitute did not match.\n");
1766 }
1767
1768 if(!testConvertToUnicode(in4, ARRAY_LENGTH(in4), out4, ARRAY_LENGTH(out4), "UTF-32",
1769 UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets4, NULL, 0)
1770 ) {
1771 log_err("UTF-32 (no BOM, with error)->u with substitute did not match.\n");
1772 }
1773 }
1774 }
1775
1776 static void TestSubWithValue(int32_t inputsize, int32_t outputsize)
1777 {
1778 UChar sampleText[] = { 0x0000, 0xAC00, 0xAC01, 0xEF67, 0xD700 };
1779 UChar sampleText2[] = { 0x6D63, 0x6D64, 0x6D65, 0x6D66 };
1780
1781 const uint8_t expsubwvalIBM_949[]= {
1782 0x00, 0xb0, 0xa1, 0xb0, 0xa2,
1783 0x25, 0x55, 0x45, 0x46, 0x36, 0x37, 0xc8, 0xd3 };
1784
1785 const uint8_t expsubwvalIBM_943[]= {
1786 0x9f, 0xaf, 0x9f, 0xb1,
1787 0x25, 0x55, 0x36, 0x44, 0x36, 0x35, 0x89, 0x59 };
1788
1789 const uint8_t expsubwvalIBM_930[] = {
1790 0x0e, 0x5d, 0x5f, 0x5d, 0x63, 0x0f, 0x6c, 0xe4, 0xf6, 0xc4, 0xf6, 0xf5, 0x0e, 0x46, 0x6b, 0x0f };
1791
1792 int32_t toIBM949Offs [] ={ 0, 1, 1, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4 };
1793 int32_t toIBM943Offs [] = { 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3 };
1794 int32_t toIBM930Offs [] = { 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3 }; /* last item: 3,3,3,3 because there's SO+DBCS+SI */
1795
1796 gInBufferSize = inputsize;
1797 gOutBufferSize = outputsize;
1798
1799 /*from Unicode*/
1800
1801 #if !UCONFIG_NO_LEGACY_CONVERSION
1802 if(!testConvertFromUnicode(sampleText, sizeof(sampleText)/sizeof(sampleText[0]),
1803 expsubwvalIBM_949, sizeof(expsubwvalIBM_949), "ibm-949",
1804 UCNV_FROM_U_CALLBACK_ESCAPE, toIBM949Offs, NULL, 0 ))
1805 log_err("u-> ibm-949 with subst with value did not match.\n");
1806
1807 if(!testConvertFromUnicode(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
1808 expsubwvalIBM_943, sizeof(expsubwvalIBM_943), "ibm-943",
1809 UCNV_FROM_U_CALLBACK_ESCAPE, toIBM943Offs, NULL, 0 ))
1810 log_err("u-> ibm-943 with sub with value did not match.\n");
1811
1812 if(!testConvertFromUnicode(sampleText2, sizeof(sampleText2)/sizeof(sampleText2[0]),
1813 expsubwvalIBM_930, sizeof(expsubwvalIBM_930), "ibm-930",
1814 UCNV_FROM_U_CALLBACK_ESCAPE, toIBM930Offs, NULL, 0 ))
1815 log_err("u-> ibm-930 with subst with value did not match.\n");
1816
1817
1818 log_verbose("Testing fromUnicode with UCNV_FROM_U_CALLBACK_ESCAPE \n");
1819 {
1820 static const UChar inputTest[] = { 0x0061, 0xd801, 0xdc01, 0xd801, 0x0061 };
1821 static const uint8_t toIBM943[]= { 0x61,
1822 0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
1823 0x25, 0x55, 0x44, 0x43, 0x30, 0x31,
1824 0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
1825 0x61 };
1826 static const int32_t offset[]= {0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 4};
1827
1828
1829 /* EUC_JP*/
1830 static const UChar euc_jp_inputText[]={ 0x0061, 0x4edd, 0x5bec, 0xd801, 0xdc01, 0xd801, 0x0061, 0x00a2, };
1831 static const uint8_t to_euc_jp[]={ 0x61, 0xa1, 0xb8, 0x8f, 0xf4, 0xae,
1832 0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
1833 0x25, 0x55, 0x44, 0x43, 0x30, 0x31,
1834 0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
1835 0x61, 0x8e, 0xe0,
1836 };
1837 static const int32_t fromEUC_JPOffs [] ={ 0, 1, 1, 2, 2, 2,
1838 3, 3, 3, 3, 3, 3,
1839 3, 3, 3, 3, 3, 3,
1840 5, 5, 5, 5, 5, 5,
1841 6, 7, 7,
1842 };
1843
1844 /*EUC_TW*/
1845 static const UChar euc_tw_inputText[]={ 0x0061, 0x2295, 0x5BF2, 0xd801, 0xdc01, 0xd801, 0x0061, 0x8706, 0x8a, };
1846 static const uint8_t to_euc_tw[]={
1847 0x61, 0xa2, 0xd3, 0x8e, 0xa2, 0xdc, 0xe5,
1848 0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
1849 0x25, 0x55, 0x44, 0x43, 0x30, 0x31,
1850 0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
1851 0x61, 0xe6, 0xca, 0x8a,
1852 };
1853 static const int32_t from_euc_twOffs [] ={ 0, 1, 1, 2, 2, 2, 2,
1854 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5,
1855 6, 7, 7, 8,
1856 };
1857 /*ISO-2022-JP*/
1858 static const UChar iso_2022_jp_inputText1[]={ 0x3000, 0x00E9, 0x3001,0x00E9, 0x0042} ;
1859 static const uint8_t to_iso_2022_jp1[]={
1860 0x1b, 0x24, 0x42, 0x21, 0x21,
1861 0x1b, 0x28, 0x42, 0x25, 0x55, 0x30, 0x30, 0x45, 0x39,
1862 0x1b, 0x24, 0x42, 0x21, 0x22,
1863 0x1b, 0x28, 0x42, 0x25, 0x55, 0x30, 0x30, 0x45, 0x39,
1864 0x42,
1865 };
1866
1867 static const int32_t from_iso_2022_jpOffs1 [] ={
1868 0,0,0,0,0,
1869 1,1,1,1,1,1,1,1,1,
1870 2,2,2,2,2,
1871 3,3,3,3,3,3,3,3,3,
1872 4,
1873 };
1874 /* surrogate pair*/
1875 static const UChar iso_2022_jp_inputText2[]={ 0x3000, 0xD84D, 0xDC56, 0x3001,0xD84D,0xDC56, 0x0042} ;
1876 static const uint8_t to_iso_2022_jp2[]={
1877 0x1b, 0x24, 0x42, 0x21, 0x21,
1878 0x1b, 0x28, 0x42, 0x25, 0x55, 0x44, 0x38, 0x34, 0x44,
1879 0x25, 0x55, 0x44, 0x43, 0x35, 0x36,
1880 0x1b, 0x24, 0x42, 0x21, 0x22,
1881 0x1b, 0x28, 0x42, 0x25, 0x55, 0x44, 0x38, 0x34, 0x44,
1882 0x25, 0x55, 0x44, 0x43, 0x35, 0x36,
1883 0x42,
1884 };
1885 static const int32_t from_iso_2022_jpOffs2 [] ={
1886 0,0,0,0,0,
1887 1,1,1,1,1,1,1,1,1,
1888 1,1,1,1,1,1,
1889 3,3,3,3,3,
1890 4,4,4,4,4,4,4,4,4,
1891 4,4,4,4,4,4,
1892 6,
1893 };
1894
1895 /*ISO-2022-cn*/
1896 static const UChar iso_2022_cn_inputText[]={ 0x0041, 0x3712, 0x0042, };
1897 static const uint8_t to_iso_2022_cn[]={
1898 0x41,
1899 0x25, 0x55, 0x33, 0x37, 0x31, 0x32,
1900 0x42,
1901 };
1902 static const int32_t from_iso_2022_cnOffs [] ={
1903 0,
1904 1,1,1,1,1,1,
1905 2,
1906 };
1907
1908 static const UChar iso_2022_cn_inputText4[]={ 0x3000, 0xD84D, 0xDC56, 0x3001,0xD84D,0xDC56, 0x0042};
1909
1910 static const uint8_t to_iso_2022_cn4[]={
1911 0x1b, 0x24, 0x29, 0x41, 0x0e, 0x21, 0x21,
1912 0x0f, 0x25, 0x55, 0x44, 0x38, 0x34, 0x44,
1913 0x25, 0x55, 0x44, 0x43, 0x35, 0x36,
1914 0x0e, 0x21, 0x22,
1915 0x0f, 0x25, 0x55, 0x44, 0x38, 0x34, 0x44,
1916 0x25, 0x55, 0x44, 0x43, 0x35, 0x36,
1917 0x42,
1918 };
1919 static const int32_t from_iso_2022_cnOffs4 [] ={
1920 0,0,0,0,0,0,0,
1921 1,1,1,1,1,1,1,
1922 1,1,1,1,1,1,
1923 3,3,3,
1924 4,4,4,4,4,4,4,
1925 4,4,4,4,4,4,
1926 6
1927
1928 };
1929
1930 /*ISO-2022-kr*/
1931 static const UChar iso_2022_kr_inputText2[]={ 0x0041, 0x03A0,0xD84D, 0xDC56/*unassigned*/,0x03A0, 0x0042,0xD84D, 0xDC56/*unassigned*/,0x43 };
1932 static const uint8_t to_iso_2022_kr2[]={
1933 0x1b, 0x24, 0x29, 0x43,
1934 0x41,
1935 0x0e, 0x25, 0x50,
1936 0x0f, 0x25, 0x55, 0x44, 0x38, 0x34, 0x44,
1937 0x25, 0x55, 0x44, 0x43, 0x35, 0x36,
1938 0x0e, 0x25, 0x50,
1939 0x0f, 0x42,
1940 0x25, 0x55, 0x44, 0x38, 0x34, 0x44,
1941 0x25, 0x55, 0x44, 0x43, 0x35, 0x36,
1942 0x43
1943 };
1944 static const int32_t from_iso_2022_krOffs2 [] ={
1945 -1,-1,-1,-1,
1946 0,
1947 1,1,1,
1948 2,2,2,2,2,2,2,
1949 2,2,2,2,2,2,
1950 4,4,4,
1951 5,5,
1952 6,6,6,6,6,6,
1953 6,6,6,6,6,6,
1954 8,
1955 };
1956
1957 static const UChar iso_2022_kr_inputText[]={ 0x0041, 0x03A0,0x3712/*unassigned*/,0x03A0, 0x0042,0x3712/*unassigned*/,0x43 };
1958 static const uint8_t to_iso_2022_kr[]={
1959 0x1b, 0x24, 0x29, 0x43,
1960 0x41,
1961 0x0e, 0x25, 0x50,
1962 0x0f, 0x25, 0x55, 0x33, 0x37, 0x31, 0x32, /*unassigned*/
1963 0x0e, 0x25, 0x50,
1964 0x0f, 0x42,
1965 0x25, 0x55, 0x33, 0x37, 0x31, 0x32, /*unassigned*/
1966 0x43
1967 };
1968
1969
1970 static const int32_t from_iso_2022_krOffs [] ={
1971 -1,-1,-1,-1,
1972 0,
1973 1,1,1,
1974 2,2,2,2,2,2,2,
1975 3,3,3,
1976 4,4,
1977 5,5,5,5,5,5,
1978 6,
1979 };
1980 /* HZ encoding */
1981 static const UChar hz_inputText[]={ 0x0041, 0x03A0,0x0662/*unassigned*/,0x03A0, 0x0042, };
1982
1983 static const uint8_t to_hz[]={
1984 0x7e, 0x7d, 0x41,
1985 0x7e, 0x7b, 0x26, 0x30,
1986 0x7e, 0x7d, 0x25, 0x55, 0x30, 0x36, 0x36, 0x32, /*unassigned*/
1987 0x7e, 0x7b, 0x26, 0x30,
1988 0x7e, 0x7d, 0x42,
1989
1990 };
1991 static const int32_t from_hzOffs [] ={
1992 0,0,0,
1993 1,1,1,1,
1994 2,2,2,2,2,2,2,2,
1995 3,3,3,3,
1996 4,4,4
1997 };
1998
1999 static const UChar hz_inputText2[]={ 0x0041, 0x03A0,0xD84D, 0xDC56/*unassigned*/,0x03A0, 0x0042,0xD84D, 0xDC56/*unassigned*/,0x43 };
2000 static const uint8_t to_hz2[]={
2001 0x7e, 0x7d, 0x41,
2002 0x7e, 0x7b, 0x26, 0x30,
2003 0x7e, 0x7d, 0x25, 0x55, 0x44, 0x38, 0x34, 0x44,
2004 0x25, 0x55, 0x44, 0x43, 0x35, 0x36,
2005 0x7e, 0x7b, 0x26, 0x30,
2006 0x7e, 0x7d, 0x42,
2007 0x25, 0x55, 0x44, 0x38, 0x34, 0x44,
2008 0x25, 0x55, 0x44, 0x43, 0x35, 0x36,
2009 0x43
2010 };
2011 static const int32_t from_hzOffs2 [] ={
2012 0,0,0,
2013 1,1,1,1,
2014 2,2,2,2,2,2,2,2,
2015 2,2,2,2,2,2,
2016 4,4,4,4,
2017 5,5,5,
2018 6,6,6,6,6,6,
2019 6,6,6,6,6,6,
2020 8,
2021 };
2022
2023 /*ISCII*/
2024 static const UChar iscii_inputText[]={ 0x0041, 0x0901,0x3712/*unassigned*/,0x0902, 0x0042,0x3712/*unassigned*/,0x43 };
2025 static const uint8_t to_iscii[]={
2026 0x41,
2027 0xef, 0x42, 0xa1,
2028 0x25, 0x55, 0x33, 0x37, 0x31, 0x32, /*unassigned*/
2029 0xa2,
2030 0x42,
2031 0x25, 0x55, 0x33, 0x37, 0x31, 0x32, /*unassigned*/
2032 0x43
2033 };
2034
2035
2036 static const int32_t from_isciiOffs [] ={
2037 0,
2038 1,1,1,
2039 2,2,2,2,2,2,
2040 3,
2041 4,
2042 5,5,5,5,5,5,
2043 6,
2044 };
2045
2046 if(!testConvertFromUnicode(inputTest, sizeof(inputTest)/sizeof(inputTest[0]),
2047 toIBM943, sizeof(toIBM943), "ibm-943",
2048 UCNV_FROM_U_CALLBACK_ESCAPE, offset, NULL, 0 ))
2049 log_err("u-> ibm-943 with subst with value did not match.\n");
2050
2051 if(!testConvertFromUnicode(euc_jp_inputText, sizeof(euc_jp_inputText)/sizeof(euc_jp_inputText[0]),
2052 to_euc_jp, sizeof(to_euc_jp), "euc-jp",
2053 UCNV_FROM_U_CALLBACK_ESCAPE, fromEUC_JPOffs, NULL, 0 ))
2054 log_err("u-> euc-jp with subst with value did not match.\n");
2055
2056 if(!testConvertFromUnicode(euc_tw_inputText, sizeof(euc_tw_inputText)/sizeof(euc_tw_inputText[0]),
2057 to_euc_tw, sizeof(to_euc_tw), "euc-tw",
2058 UCNV_FROM_U_CALLBACK_ESCAPE, from_euc_twOffs, NULL, 0 ))
2059 log_err("u-> euc-tw with subst with value did not match.\n");
2060
2061 if(!testConvertFromUnicode(iso_2022_jp_inputText1, sizeof(iso_2022_jp_inputText1)/sizeof(iso_2022_jp_inputText1[0]),
2062 to_iso_2022_jp1, sizeof(to_iso_2022_jp1), "iso-2022-jp",
2063 UCNV_FROM_U_CALLBACK_ESCAPE, from_iso_2022_jpOffs1, NULL, 0 ))
2064 log_err("u-> iso_2022_jp with subst with value did not match.\n");
2065
2066 if(!testConvertFromUnicode(iso_2022_jp_inputText1, sizeof(iso_2022_jp_inputText1)/sizeof(iso_2022_jp_inputText1[0]),
2067 to_iso_2022_jp1, sizeof(to_iso_2022_jp1), "iso-2022-jp",
2068 UCNV_FROM_U_CALLBACK_ESCAPE, from_iso_2022_jpOffs1, NULL, 0 ))
2069 log_err("u-> iso_2022_jp with subst with value did not match.\n");
2070
2071 if(!testConvertFromUnicode(iso_2022_jp_inputText2, sizeof(iso_2022_jp_inputText2)/sizeof(iso_2022_jp_inputText2[0]),
2072 to_iso_2022_jp2, sizeof(to_iso_2022_jp2), "iso-2022-jp",
2073 UCNV_FROM_U_CALLBACK_ESCAPE, from_iso_2022_jpOffs2, NULL, 0 ))
2074 log_err("u-> iso_2022_jp with subst with value did not match.\n");
2075 /*ESCAPE OPTIONS*/
2076 {
2077 /* surrogate pair*/
2078 static const UChar iso_2022_jp_inputText3[]={ 0x3000, 0xD84D, 0xDC56, 0x3001,0xD84D,0xDC56, 0x0042,0x0901c } ;
2079 static const uint8_t to_iso_2022_jp3_v2[]={
2080 0x1b, 0x24, 0x42, 0x21, 0x21,
2081 0x1b, 0x28, 0x42, 0x26, 0x23, 0x31, 0x34, 0x34, 0x34, 0x37, 0x30, 0x3b,
2082
2083 0x1b, 0x24, 0x42, 0x21, 0x22,
2084 0x1b, 0x28, 0x42, 0x26, 0x23, 0x31, 0x34, 0x34, 0x34, 0x37, 0x30, 0x3b,
2085
2086 0x42,
2087 0x26, 0x23, 0x33, 0x36, 0x38, 0x39, 0x32, 0x3b,
2088 };
2089
2090 static const int32_t from_iso_2022_jpOffs3_v2 [] ={
2091 0,0,0,0,0,
2092 1,1,1,1,1,1,1,1,1,1,1,1,
2093
2094 3,3,3,3,3,
2095 4,4,4,4,4,4,4,4,4,4,4,4,
2096
2097 6,
2098 7,7,7,7,7,7,7,7,7
2099 };
2100
2101 if(!testConvertFromUnicodeWithContext(iso_2022_jp_inputText3, sizeof(iso_2022_jp_inputText3)/sizeof(iso_2022_jp_inputText3[0]),
2102 to_iso_2022_jp3_v2, sizeof(to_iso_2022_jp3_v2), "iso-2022-jp",
2103 UCNV_FROM_U_CALLBACK_ESCAPE, from_iso_2022_jpOffs3_v2, NULL, 0,UCNV_ESCAPE_XML_DEC,U_ZERO_ERROR ))
2104 log_err("u-> iso-2022-jp with sub & UCNV_ESCAPE_XML_DEC did not match.\n");
2105 }
2106 {
2107 static const UChar iso_2022_cn_inputText5[]={ 0x3000, 0xD84D, 0xDC56, 0x3001,0xD84D,0xDC56, 0x0042,0x0902};
2108 static const uint8_t to_iso_2022_cn5_v2[]={
2109 0x1b, 0x24, 0x29, 0x41, 0x0e, 0x21, 0x21,
2110 0x0f, 0x5c, 0x75, 0x44, 0x38, 0x34, 0x44,
2111 0x5c, 0x75, 0x44, 0x43, 0x35, 0x36,
2112 0x0e, 0x21, 0x22,
2113 0x0f, 0x5c, 0x75, 0x44, 0x38, 0x34, 0x44,
2114 0x5c, 0x75, 0x44, 0x43, 0x35, 0x36,
2115 0x42,
2116 0x5c, 0x75, 0x30, 0x39, 0x30, 0x32,
2117 };
2118 static const int32_t from_iso_2022_cnOffs5_v2 [] ={
2119 0,0,0,0,0,0,0,
2120 1,1,1,1,1,1,1,
2121 1,1,1,1,1,1,
2122 3,3,3,
2123 4,4,4,4,4,4,4,
2124 4,4,4,4,4,4,
2125 6,
2126 7,7,7,7,7,7
2127 };
2128 if(!testConvertFromUnicodeWithContext(iso_2022_cn_inputText5, sizeof(iso_2022_cn_inputText5)/sizeof(iso_2022_cn_inputText5[0]),
2129 to_iso_2022_cn5_v2, sizeof(to_iso_2022_cn5_v2), "iso-2022-cn",
2130 UCNV_FROM_U_CALLBACK_ESCAPE, from_iso_2022_cnOffs5_v2, NULL, 0,UCNV_ESCAPE_JAVA,U_ZERO_ERROR ))
2131 log_err("u-> iso-2022-cn with sub & UCNV_ESCAPE_JAVA did not match.\n");
2132
2133 }
2134 {
2135 static const UChar iso_2022_cn_inputText6[]={ 0x3000, 0xD84D, 0xDC56, 0x3001,0xD84D,0xDC56, 0x0042,0x0902};
2136 static const uint8_t to_iso_2022_cn6_v2[]={
2137 0x1b, 0x24, 0x29, 0x41, 0x0e, 0x21, 0x21,
2138 0x0f, 0x7b, 0x55, 0x2b, 0x32, 0x33, 0x34, 0x35, 0x36, 0x7d,
2139 0x0e, 0x21, 0x22,
2140 0x0f, 0x7b, 0x55, 0x2b, 0x32, 0x33, 0x34, 0x35, 0x36, 0x7d,
2141 0x42,
2142 0x7b, 0x55, 0x2b, 0x30, 0x39, 0x30, 0x32, 0x7d
2143 };
2144 static const int32_t from_iso_2022_cnOffs6_v2 [] ={
2145 0, 0, 0, 0, 0, 0, 0,
2146 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
2147 3, 3, 3,
2148 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
2149 6,
2150 7, 7, 7, 7, 7, 7, 7, 7,
2151 };
2152 if(!testConvertFromUnicodeWithContext(iso_2022_cn_inputText6, sizeof(iso_2022_cn_inputText6)/sizeof(iso_2022_cn_inputText6[0]),
2153 to_iso_2022_cn6_v2, sizeof(to_iso_2022_cn6_v2), "iso-2022-cn",
2154 UCNV_FROM_U_CALLBACK_ESCAPE, from_iso_2022_cnOffs6_v2, NULL, 0,UCNV_ESCAPE_UNICODE,U_ZERO_ERROR ))
2155 log_err("u-> iso-2022-cn with sub & UCNV_ESCAPE_UNICODE did not match.\n");
2156
2157 }
2158 {
2159 static const UChar iso_2022_cn_inputText7[]={ 0x3000, 0xD84D, 0xDC56, 0x3001,0xD84D,0xDC56, 0x0042,0x0902};
2160 static const uint8_t to_iso_2022_cn7_v2[]={
2161 0x1b, 0x24, 0x29, 0x41, 0x0e, 0x21, 0x21,
2162 0x0f, 0x25, 0x55, 0x44, 0x38, 0x34, 0x44, 0x25, 0x55, 0x44, 0x43, 0x35, 0x36,
2163 0x0e, 0x21, 0x22,
2164 0x0f, 0x25, 0x55, 0x44, 0x38, 0x34, 0x44, 0x25, 0x55, 0x44, 0x43, 0x35, 0x36,
2165 0x42, 0x25, 0x55, 0x30, 0x39, 0x30, 0x32,
2166 };
2167 static const int32_t from_iso_2022_cnOffs7_v2 [] ={
2168 0, 0, 0, 0, 0, 0, 0,
2169 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
2170 3, 3, 3,
2171 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
2172 6,
2173 7, 7, 7, 7, 7, 7,
2174 };
2175 if(!testConvertFromUnicodeWithContext(iso_2022_cn_inputText7, sizeof(iso_2022_cn_inputText7)/sizeof(iso_2022_cn_inputText7[0]),
2176 to_iso_2022_cn7_v2, sizeof(to_iso_2022_cn7_v2), "iso-2022-cn",
2177 UCNV_FROM_U_CALLBACK_ESCAPE, from_iso_2022_cnOffs7_v2, NULL, 0,"K" ,U_ZERO_ERROR ))
2178 log_err("u-> iso-2022-cn with sub & K did not match.\n");
2179
2180 }
2181 {
2182 static const UChar iso_2022_cn_inputText8[]={
2183 0x3000,
2184 0xD84D, 0xDC56,
2185 0x3001,
2186 0xD84D, 0xDC56,
2187 0xDBFF, 0xDFFF,
2188 0x0042,
2189 0x0902};
2190 static const uint8_t to_iso_2022_cn8_v2[]={
2191 0x1b, 0x24, 0x29, 0x41, 0x0e, 0x21, 0x21,
2192 0x0f, 0x5c, 0x32, 0x33, 0x34, 0x35, 0x36, 0x20,
2193 0x0e, 0x21, 0x22,
2194 0x0f, 0x5c, 0x32, 0x33, 0x34, 0x35, 0x36, 0x20,
2195 0x5c, 0x31, 0x30, 0x46, 0x46, 0x46, 0x46, 0x20,
2196 0x42,
2197 0x5c, 0x39, 0x30, 0x32, 0x20
2198 };
2199 static const int32_t from_iso_2022_cnOffs8_v2 [] ={
2200 0, 0, 0, 0, 0, 0, 0,
2201 1, 1, 1, 1, 1, 1, 1, 1,
2202 3, 3, 3,
2203 4, 4, 4, 4, 4, 4, 4, 4,
2204 6, 6, 6, 6, 6, 6, 6, 6,
2205 8,
2206 9, 9, 9, 9, 9
2207 };
2208 if(!testConvertFromUnicodeWithContext(iso_2022_cn_inputText8, sizeof(iso_2022_cn_inputText8)/sizeof(iso_2022_cn_inputText8[0]),
2209 to_iso_2022_cn8_v2, sizeof(to_iso_2022_cn8_v2), "iso-2022-cn",
2210 UCNV_FROM_U_CALLBACK_ESCAPE, from_iso_2022_cnOffs8_v2, NULL, 0,UCNV_ESCAPE_CSS2,U_ZERO_ERROR ))
2211 log_err("u-> iso-2022-cn with sub & UCNV_ESCAPE_CSS2 did not match.\n");
2212
2213 }
2214 {
2215 static const uint8_t to_iso_2022_cn4_v3[]={
2216 0x1b, 0x24, 0x29, 0x41, 0x0e, 0x21, 0x21,
2217 0x0f, 0x5c, 0x55, 0x30, 0x30, 0x30, 0x32, 0x33, 0x34, 0x35, 0x36,
2218 0x0e, 0x21, 0x22,
2219 0x0f, 0x5c, 0x55, 0x30, 0x30, 0x30, 0x32, 0x33, 0x34, 0x35, 0x36,
2220 0x42
2221 };
2222
2223
2224 static const int32_t from_iso_2022_cnOffs4_v3 [] ={
2225 0,0,0,0,0,0,0,
2226 1,1,1,1,1,1,1,1,1,1,1,
2227
2228 3,3,3,
2229 4,4,4,4,4,4,4,4,4,4,4,
2230
2231 6
2232
2233 };
2234 if(!testConvertFromUnicodeWithContext(iso_2022_cn_inputText4, sizeof(iso_2022_cn_inputText4)/sizeof(iso_2022_cn_inputText4[0]),
2235 to_iso_2022_cn4_v3, sizeof(to_iso_2022_cn4_v3), "iso-2022-cn",
2236 UCNV_FROM_U_CALLBACK_ESCAPE, from_iso_2022_cnOffs4_v3, NULL, 0,UCNV_ESCAPE_C,U_ZERO_ERROR ))
2237 {
2238 log_err("u-> iso-2022-cn with skip & UCNV_ESCAPE_C did not match.\n");
2239 }
2240 }
2241 if(!testConvertFromUnicode(iso_2022_cn_inputText, sizeof(iso_2022_cn_inputText)/sizeof(iso_2022_cn_inputText[0]),
2242 to_iso_2022_cn, sizeof(to_iso_2022_cn), "iso-2022-cn",
2243 UCNV_FROM_U_CALLBACK_ESCAPE, from_iso_2022_cnOffs, NULL, 0 ))
2244 log_err("u-> iso_2022_cn with subst with value did not match.\n");
2245
2246 if(!testConvertFromUnicode(iso_2022_cn_inputText4, sizeof(iso_2022_cn_inputText4)/sizeof(iso_2022_cn_inputText4[0]),
2247 to_iso_2022_cn4, sizeof(to_iso_2022_cn4), "iso-2022-cn",
2248 UCNV_FROM_U_CALLBACK_ESCAPE, from_iso_2022_cnOffs4, NULL, 0 ))
2249 log_err("u-> iso_2022_cn with subst with value did not match.\n");
2250 if(!testConvertFromUnicode(iso_2022_kr_inputText, sizeof(iso_2022_kr_inputText)/sizeof(iso_2022_kr_inputText[0]),
2251 to_iso_2022_kr, sizeof(to_iso_2022_kr), "iso-2022-kr",
2252 UCNV_FROM_U_CALLBACK_ESCAPE, from_iso_2022_krOffs, NULL, 0 ))
2253 log_err("u-> iso_2022_kr with subst with value did not match.\n");
2254 if(!testConvertFromUnicode(iso_2022_kr_inputText2, sizeof(iso_2022_kr_inputText2)/sizeof(iso_2022_kr_inputText2[0]),
2255 to_iso_2022_kr2, sizeof(to_iso_2022_kr2), "iso-2022-kr",
2256 UCNV_FROM_U_CALLBACK_ESCAPE, from_iso_2022_krOffs2, NULL, 0 ))
2257 log_err("u-> iso_2022_kr2 with subst with value did not match.\n");
2258 if(!testConvertFromUnicode(hz_inputText, sizeof(hz_inputText)/sizeof(hz_inputText[0]),
2259 to_hz, sizeof(to_hz), "HZ",
2260 UCNV_FROM_U_CALLBACK_ESCAPE, from_hzOffs, NULL, 0 ))
2261 log_err("u-> hz with subst with value did not match.\n");
2262 if(!testConvertFromUnicode(hz_inputText2, sizeof(hz_inputText2)/sizeof(hz_inputText2[0]),
2263 to_hz2, sizeof(to_hz2), "HZ",
2264 UCNV_FROM_U_CALLBACK_ESCAPE, from_hzOffs2, NULL, 0 ))
2265 log_err("u-> hz with subst with value did not match.\n");
2266
2267 if(!testConvertFromUnicode(iscii_inputText, sizeof(iscii_inputText)/sizeof(iscii_inputText[0]),
2268 to_iscii, sizeof(to_iscii), "ISCII,version=0",
2269 UCNV_FROM_U_CALLBACK_ESCAPE, from_isciiOffs, NULL, 0 ))
2270 log_err("u-> iscii with subst with value did not match.\n");
2271 }
2272 #endif
2273
2274 log_verbose("Testing toUnicode with UCNV_TO_U_CALLBACK_ESCAPE \n");
2275 /*to Unicode*/
2276 {
2277 #if !UCONFIG_NO_LEGACY_CONVERSION
2278 static const uint8_t sampleTxtToU[]= { 0x00, 0x9f, 0xaf,
2279 0x81, 0xad, /*unassigned*/
2280 0x89, 0xd3 };
2281 static const UChar IBM_943toUnicode[] = { 0x0000, 0x6D63,
2282 0x25, 0x58, 0x38, 0x31, 0x25, 0x58, 0x41, 0x44,
2283 0x7B87};
2284 static const int32_t fromIBM943Offs [] = { 0, 1, 3, 3, 3, 3, 3, 3, 3, 3, 5};
2285
2286 /* EUC_JP*/
2287 static const uint8_t sampleTxt_EUC_JP[]={ 0x61, 0xa1, 0xb8, 0x8f, 0xf4, 0xae,
2288 0x8f, 0xda, 0xa1, /*unassigned*/
2289 0x8e, 0xe0,
2290 };
2291 static const UChar EUC_JPtoUnicode[]={ 0x0061, 0x4edd, 0x5bec,
2292 0x25, 0x58, 0x38, 0x46, 0x25, 0x58, 0x44, 0x41, 0x25, 0x58, 0x41, 0x31,
2293 0x00a2 };
2294 static const int32_t fromEUC_JPOffs [] ={ 0, 1, 3,
2295 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
2296 9,
2297 };
2298
2299 /*EUC_TW*/
2300 static const uint8_t sampleTxt_euc_tw[]={
2301 0x61, 0xa2, 0xd3, 0x8e, 0xa2, 0xdc, 0xe5,
2302 0x8e, 0xaa, 0xbb, 0xcc,/*unassigned*/
2303 0xe6, 0xca, 0x8a,
2304 };
2305 static const UChar euc_twtoUnicode[]={ 0x0061, 0x2295, 0x5BF2,
2306 0x25, 0x58, 0x38, 0x45, 0x25, 0x58, 0x41, 0x41, 0x25, 0x58, 0x42, 0x42, 0x25, 0x58, 0x43, 0x43,
2307 0x8706, 0x8a, };
2308 static const int32_t from_euc_twOffs [] ={ 0, 1, 3,
2309 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
2310 11, 13};
2311
2312 /*iso-2022-jp*/
2313 static const uint8_t sampleTxt_iso_2022_jp[]={
2314 0x1b, 0x28, 0x42, 0x41,
2315 0x1b, 0x24, 0x42, 0x2A, 0x44, /*unassigned*/
2316 0x1b, 0x28, 0x42, 0x42,
2317
2318 };
2319 static const UChar iso_2022_jptoUnicode[]={ 0x41,0x25,0x58,0x32,0x41,0x25,0x58,0x34,0x34, 0x42 };
2320 static const int32_t from_iso_2022_jpOffs [] ={ 3, 7, 7, 7, 7, 7, 7, 7, 7, 12 };
2321
2322 /*iso-2022-cn*/
2323 static const uint8_t sampleTxt_iso_2022_cn[]={
2324 0x0f, 0x41, 0x44,
2325 0x1B, 0x24, 0x29, 0x47,
2326 0x0E, 0x40, 0x6c, /*unassigned*/
2327 0x0f, 0x42,
2328
2329 };
2330 static const UChar iso_2022_cntoUnicode[]={ 0x41, 0x44,0x25,0x58,0x34,0x30,0x25,0x58,0x36,0x43,0x42 };
2331 static const int32_t from_iso_2022_cnOffs [] ={ 1, 2, 8, 8, 8, 8, 8, 8, 8, 8, 11 };
2332
2333 /*iso-2022-kr*/
2334 static const uint8_t sampleTxt_iso_2022_kr[]={
2335 0x1b, 0x24, 0x29, 0x43,
2336 0x41,
2337 0x0E, 0x7f, 0x1E,
2338 0x0e, 0x25, 0x50,
2339 0x0f, 0x51,
2340 0x42, 0x43,
2341
2342 };
2343 static const UChar iso_2022_krtoUnicode[]={ 0x41,0x25,0x58,0x37,0x46,0x25,0x58,0x31,0x45,0x03A0,0x51, 0x42,0x43};
2344 static const int32_t from_iso_2022_krOffs [] ={ 4, 6, 6, 6, 6, 6, 6, 6, 6, 9, 12, 13 , 14 };
2345
2346 /*hz*/
2347 static const uint8_t sampleTxt_hz[]={
2348 0x41,
2349 0x7e, 0x7b, 0x26, 0x30,
2350 0x7f, 0x1E, /*unassigned*/
2351 0x26, 0x30,
2352 0x7e, 0x7d, 0x42,
2353 0x7e, 0x7b, 0x7f, 0x1E,/*unassigned*/
2354 0x7e, 0x7d, 0x42,
2355 };
2356 static const UChar hztoUnicode[]={
2357 0x41,
2358 0x03a0,
2359 0x25,0x58,0x37,0x46,0x25,0x58,0x31,0x45,
2360 0x03A0,
2361 0x42,
2362 0x25,0x58,0x37,0x46,0x25,0x58,0x31,0x45,
2363 0x42,};
2364
2365 static const int32_t from_hzOffs [] ={0,3,5,5,5,5,5,5,5,5,7,11,14,14,14,14,14,14,14,14,18, };
2366
2367
2368 /*iscii*/
2369 static const uint8_t sampleTxt_iscii[]={
2370 0x41,
2371 0x30,
2372 0xEB, /*unassigned*/
2373 0xa3,
2374 0x42,
2375 0xEC, /*unassigned*/
2376 0x42,
2377 };
2378 static const UChar isciitoUnicode[]={
2379 0x41,
2380 0x30,
2381 0x25, 0x58, 0x45, 0x42,
2382 0x0903,
2383 0x42,
2384 0x25, 0x58, 0x45, 0x43,
2385 0x42,};
2386
2387 static const int32_t from_isciiOffs [] ={0,1,2,2,2,2,3,4,5,5,5,5,6 };
2388 #endif
2389
2390 /*UTF8*/
2391 static const uint8_t sampleTxtUTF8[]={
2392 0x20, 0x64, 0x50,
2393 0xC2, 0x7E, /* truncated char */
2394 0x20,
2395 0xE0, 0xB5, 0x7E, /* truncated char */
2396 0x40,
2397 };
2398 static const UChar UTF8ToUnicode[]={
2399 0x0020, 0x0064, 0x0050,
2400 0x0025, 0x0058, 0x0043, 0x0032, 0x007E, /* \xC2~ */
2401 0x0020,
2402 0x0025, 0x0058, 0x0045, 0x0030, 0x0025, 0x0058, 0x0042, 0x0035, 0x007E,
2403 0x0040
2404 };
2405 static const int32_t fromUTF8[] = {
2406 0, 1, 2,
2407 3, 3, 3, 3, 4,
2408 5,
2409 6, 6, 6, 6, 6, 6, 6, 6, 8,
2410 9
2411 };
2412 static const UChar UTF8ToUnicodeXML_DEC[]={
2413 0x0020, 0x0064, 0x0050,
2414 0x0026, 0x0023, 0x0031, 0x0039, 0x0034, 0x003B, 0x007E, /* &#194;~ */
2415 0x0020,
2416 0x0026, 0x0023, 0x0032, 0x0032, 0x0034, 0x003B, 0x0026, 0x0023, 0x0031, 0x0038, 0x0031, 0x003B, 0x007E,
2417 0x0040
2418 };
2419 static const int32_t fromUTF8XML_DEC[] = {
2420 0, 1, 2,
2421 3, 3, 3, 3, 3, 3, 4,
2422 5,
2423 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 8,
2424 9
2425 };
2426
2427
2428 #if !UCONFIG_NO_LEGACY_CONVERSION
2429 if(!testConvertToUnicode(sampleTxtToU, sizeof(sampleTxtToU),
2430 IBM_943toUnicode, sizeof(IBM_943toUnicode)/sizeof(IBM_943toUnicode[0]),"ibm-943",
2431 UCNV_TO_U_CALLBACK_ESCAPE, fromIBM943Offs, NULL, 0 ))
2432 log_err("ibm-943->u with substitute with value did not match.\n");
2433
2434 if(!testConvertToUnicode(sampleTxt_EUC_JP, sizeof(sampleTxt_EUC_JP),
2435 EUC_JPtoUnicode, sizeof(EUC_JPtoUnicode)/sizeof(EUC_JPtoUnicode[0]),"euc-jp",
2436 UCNV_TO_U_CALLBACK_ESCAPE, fromEUC_JPOffs, NULL, 0))
2437 log_err("euc-jp->u with substitute with value did not match.\n");
2438
2439 if(!testConvertToUnicode(sampleTxt_euc_tw, sizeof(sampleTxt_euc_tw),
2440 euc_twtoUnicode, sizeof(euc_twtoUnicode)/sizeof(euc_twtoUnicode[0]),"euc-tw",
2441 UCNV_TO_U_CALLBACK_ESCAPE, from_euc_twOffs, NULL, 0))
2442 log_err("euc-tw->u with substitute with value did not match.\n");
2443
2444 if(!testConvertToUnicode(sampleTxt_iso_2022_jp, sizeof(sampleTxt_iso_2022_jp),
2445 iso_2022_jptoUnicode, sizeof(iso_2022_jptoUnicode)/sizeof(iso_2022_jptoUnicode[0]),"iso-2022-jp",
2446 UCNV_TO_U_CALLBACK_ESCAPE, from_iso_2022_jpOffs, NULL, 0))
2447 log_err("iso-2022-jp->u with substitute with value did not match.\n");
2448
2449 if(!testConvertToUnicodeWithContext(sampleTxt_iso_2022_jp, sizeof(sampleTxt_iso_2022_jp),
2450 iso_2022_jptoUnicode, sizeof(iso_2022_jptoUnicode)/sizeof(iso_2022_jptoUnicode[0]),"iso-2022-jp",
2451 UCNV_TO_U_CALLBACK_ESCAPE, from_iso_2022_jpOffs, NULL, 0,"K",U_ZERO_ERROR))
2452 log_err("iso-2022-jp->u with substitute with value did not match.\n");
2453
2454 {/* test UCNV_TO_U_CALLBACK_ESCAPE with options */
2455 {
2456 static const UChar iso_2022_jptoUnicodeDec[]={
2457 0x0041,
2458 0x0026, 0x0023, 0x0034, 0x0032, 0x003b,
2459 0x0026, 0x0023, 0x0036, 0x0038, 0x003b,
2460 0x0042 };
2461 static const int32_t from_iso_2022_jpOffsDec [] ={ 3,7,7,7,7,7,7,7,7,7,7,12, };
2462 if(!testConvertToUnicodeWithContext(sampleTxt_iso_2022_jp, sizeof(sampleTxt_iso_2022_jp),
2463 iso_2022_jptoUnicodeDec, sizeof(iso_2022_jptoUnicodeDec)/sizeof(iso_2022_jptoUnicode[0]),"iso-2022-jp",
2464 UCNV_TO_U_CALLBACK_ESCAPE, from_iso_2022_jpOffsDec, NULL, 0,UCNV_ESCAPE_XML_DEC,U_ZERO_ERROR ))
2465 log_err("iso-2022-jp->u with substitute with value and UCNV_ESCAPE_XML_DEC did not match.\n");
2466 }
2467 {
2468 static const UChar iso_2022_jptoUnicodeHex[]={
2469 0x0041,
2470 0x0026, 0x0023, 0x0078, 0x0032, 0x0041, 0x003b,
2471 0x0026, 0x0023, 0x0078, 0x0034, 0x0034, 0x003b,
2472 0x0042 };
2473 static const int32_t from_iso_2022_jpOffsHex [] ={ 3,7,7,7,7,7,7,7,7,7,7,7,7,12 };
2474 if(!testConvertToUnicodeWithContext(sampleTxt_iso_2022_jp, sizeof(sampleTxt_iso_2022_jp),
2475 iso_2022_jptoUnicodeHex, sizeof(iso_2022_jptoUnicodeHex)/sizeof(iso_2022_jptoUnicode[0]),"iso-2022-jp",
2476 UCNV_TO_U_CALLBACK_ESCAPE, from_iso_2022_jpOffsHex, NULL, 0,UCNV_ESCAPE_XML_HEX,U_ZERO_ERROR ))
2477 log_err("iso-2022-jp->u with substitute with value and UCNV_ESCAPE_XML_HEX did not match.\n");
2478 }
2479 {
2480 static const UChar iso_2022_jptoUnicodeC[]={
2481 0x0041,
2482 0x005C, 0x0078, 0x0032, 0x0041,
2483 0x005C, 0x0078, 0x0034, 0x0034,
2484 0x0042 };
2485 int32_t from_iso_2022_jpOffsC [] ={ 3,7,7,7,7,7,7,7,7,12 };
2486 if(!testConvertToUnicodeWithContext(sampleTxt_iso_2022_jp, sizeof(sampleTxt_iso_2022_jp),
2487 iso_2022_jptoUnicodeC, sizeof(iso_2022_jptoUnicodeC)/sizeof(iso_2022_jptoUnicode[0]),"iso-2022-jp",
2488 UCNV_TO_U_CALLBACK_ESCAPE, from_iso_2022_jpOffsC, NULL, 0,UCNV_ESCAPE_C,U_ZERO_ERROR ))
2489 log_err("iso-2022-jp->u with substitute with value and UCNV_ESCAPE_C did not match.\n");
2490 }
2491 }
2492 if(!testConvertToUnicode(sampleTxt_iso_2022_cn, sizeof(sampleTxt_iso_2022_cn),
2493 iso_2022_cntoUnicode, sizeof(iso_2022_cntoUnicode)/sizeof(iso_2022_cntoUnicode[0]),"iso-2022-cn",
2494 UCNV_TO_U_CALLBACK_ESCAPE, from_iso_2022_cnOffs, NULL, 0))
2495 log_err("iso-2022-cn->u with substitute with value did not match.\n");
2496
2497 if(!testConvertToUnicode(sampleTxt_iso_2022_kr, sizeof(sampleTxt_iso_2022_kr),
2498 iso_2022_krtoUnicode, sizeof(iso_2022_krtoUnicode)/sizeof(iso_2022_krtoUnicode[0]),"iso-2022-kr",
2499 UCNV_TO_U_CALLBACK_ESCAPE, from_iso_2022_krOffs, NULL, 0))
2500 log_err("iso-2022-kr->u with substitute with value did not match.\n");
2501
2502 if(!testConvertToUnicode(sampleTxt_hz, sizeof(sampleTxt_hz),
2503 hztoUnicode, sizeof(hztoUnicode)/sizeof(hztoUnicode[0]),"HZ",
2504 UCNV_TO_U_CALLBACK_ESCAPE, from_hzOffs, NULL, 0))
2505 log_err("hz->u with substitute with value did not match.\n");
2506
2507 if(!testConvertToUnicode(sampleTxt_iscii, sizeof(sampleTxt_iscii),
2508 isciitoUnicode, sizeof(isciitoUnicode)/sizeof(isciitoUnicode[0]),"ISCII,version=0",
2509 UCNV_TO_U_CALLBACK_ESCAPE, from_isciiOffs, NULL, 0))
2510 log_err("ISCII ->u with substitute with value did not match.\n");
2511 #endif
2512
2513 if(!testConvertToUnicode(sampleTxtUTF8, sizeof(sampleTxtUTF8),
2514 UTF8ToUnicode, sizeof(UTF8ToUnicode)/sizeof(UTF8ToUnicode[0]),"UTF-8",
2515 UCNV_TO_U_CALLBACK_ESCAPE, fromUTF8, NULL, 0))
2516 log_err("UTF8->u with UCNV_TO_U_CALLBACK_ESCAPE with value did not match.\n");
2517 if(!testConvertToUnicodeWithContext(sampleTxtUTF8, sizeof(sampleTxtUTF8),
2518 UTF8ToUnicodeXML_DEC, sizeof(UTF8ToUnicodeXML_DEC)/sizeof(UTF8ToUnicodeXML_DEC[0]),"UTF-8",
2519 UCNV_TO_U_CALLBACK_ESCAPE, fromUTF8XML_DEC, NULL, 0, UCNV_ESCAPE_XML_DEC, U_ZERO_ERROR))
2520 log_err("UTF8->u with UCNV_TO_U_CALLBACK_ESCAPE with value did not match.\n");
2521 }
2522 }
2523
2524 #if !UCONFIG_NO_LEGACY_CONVERSION
2525 static void TestLegalAndOthers(int32_t inputsize, int32_t outputsize)
2526 {
2527 static const UChar legalText[] = { 0x0000, 0xAC00, 0xAC01, 0xD700 };
2528 static const uint8_t templegal949[] ={ 0x00, 0xb0, 0xa1, 0xb0, 0xa2, 0xc8, 0xd3 };
2529 static const int32_t to949legal[] = {0, 1, 1, 2, 2, 3, 3};
2530
2531
2532 static const uint8_t text943[] = {
2533 0x82, 0xa9, 0x82, 0x20, 0x61, 0x8a, 0xbf, 0x8e, 0x9a };
2534 static const UChar toUnicode943sub[] = { 0x304b, 0x1a, 0x20, 0x0061, 0x6f22, 0x5b57 };
2535 static const UChar toUnicode943skip[]= { 0x304b, 0x20, 0x0061, 0x6f22, 0x5b57 };
2536 static const UChar toUnicode943stop[]= { 0x304b};
2537
2538 static const int32_t fromIBM943Offssub[] = { 0, 2, 3, 4, 5, 7 };
2539 static const int32_t fromIBM943Offsskip[] = { 0, 3, 4, 5, 7 };
2540 static const int32_t fromIBM943Offsstop[] = { 0};
2541
2542 gInBufferSize = inputsize;
2543 gOutBufferSize = outputsize;
2544 /*checking with a legal value*/
2545 if(!testConvertFromUnicode(legalText, sizeof(legalText)/sizeof(legalText[0]),
2546 templegal949, sizeof(templegal949), "ibm-949",
2547 UCNV_FROM_U_CALLBACK_SKIP, to949legal, NULL, 0 ))
2548 log_err("u-> ibm-949 with skip did not match.\n");
2549
2550 /*checking illegal value for ibm-943 with substitute*/
2551 if(!testConvertToUnicode(text943, sizeof(text943),
2552 toUnicode943sub, sizeof(toUnicode943sub)/sizeof(toUnicode943sub[0]),"ibm-943",
2553 UCNV_TO_U_CALLBACK_SUBSTITUTE, fromIBM943Offssub, NULL, 0 ))
2554 log_err("ibm-943->u with subst did not match.\n");
2555 /*checking illegal value for ibm-943 with skip */
2556 if(!testConvertToUnicode(text943, sizeof(text943),
2557 toUnicode943skip, sizeof(toUnicode943skip)/sizeof(toUnicode943skip[0]),"ibm-943",
2558 UCNV_TO_U_CALLBACK_SKIP, fromIBM943Offsskip, NULL, 0 ))
2559 log_err("ibm-943->u with skip did not match.\n");
2560
2561 /*checking illegal value for ibm-943 with stop */
2562 if(!testConvertToUnicode(text943, sizeof(text943),
2563 toUnicode943stop, sizeof(toUnicode943stop)/sizeof(toUnicode943stop[0]),"ibm-943",
2564 UCNV_TO_U_CALLBACK_STOP, fromIBM943Offsstop, NULL, 0 ))
2565 log_err("ibm-943->u with stop did not match.\n");
2566
2567 }
2568
2569 static void TestSingleByte(int32_t inputsize, int32_t outputsize)
2570 {
2571 static const uint8_t sampleText[] = {
2572 0x82, 0xa9, 0x61, 0x62, 0x63 , 0x82,
2573 0xff, 0x32, 0x33};
2574 static const UChar toUnicode943sub[] = { 0x304b, 0x0061, 0x0062, 0x0063, 0x1a, 0x1a, 0x0032, 0x0033 };
2575 static const int32_t fromIBM943Offssub[] = { 0, 2, 3, 4, 5, 6, 7, 8 };
2576 /*checking illegal value for ibm-943 with substitute*/
2577 gInBufferSize = inputsize;
2578 gOutBufferSize = outputsize;
2579
2580 if(!testConvertToUnicode(sampleText, sizeof(sampleText),
2581 toUnicode943sub, sizeof(toUnicode943sub)/sizeof(toUnicode943sub[0]),"ibm-943",
2582 UCNV_TO_U_CALLBACK_SUBSTITUTE, fromIBM943Offssub, NULL, 0 ))
2583 log_err("ibm-943->u with subst did not match.\n");
2584 }
2585
2586 static void TestEBCDIC_STATEFUL_Sub(int32_t inputsize, int32_t outputsize)
2587 {
2588 /*EBCDIC_STATEFUL*/
2589 static const UChar ebcdic_inputTest[] = { 0x0061, 0x6d64, 0x0061, 0x00A2, 0x6d65, 0x0061 };
2590 static const uint8_t toIBM930[]= { 0x62, 0x0e, 0x5d, 0x63, 0x0f, 0x62, 0xb1, 0x0e, 0xfe, 0xfe, 0x0f, 0x62 };
2591 static const int32_t offset_930[]= { 0, 1, 1, 1, 2, 2, 3, 4, 4, 4, 5, 5 };
2592 /* s SO doubl SI sng s SO fe fe SI s */
2593
2594 /*EBCDIC_STATEFUL with subChar=3f*/
2595 static const uint8_t toIBM930_subvaried[]= { 0x62, 0x0e, 0x5d, 0x63, 0x0f, 0x62, 0xb1, 0x3f, 0x62 };
2596 static const int32_t offset_930_subvaried[]= { 0, 1, 1, 1, 2, 2, 3, 4, 5 };
2597 static const char mySubChar[]={ 0x3f};
2598
2599 gInBufferSize = inputsize;
2600 gOutBufferSize = outputsize;
2601
2602 if(!testConvertFromUnicode(ebcdic_inputTest, sizeof(ebcdic_inputTest)/sizeof(ebcdic_inputTest[0]),
2603 toIBM930, sizeof(toIBM930), "ibm-930",
2604 UCNV_FROM_U_CALLBACK_SUBSTITUTE, offset_930, NULL, 0 ))
2605 log_err("u-> ibm-930(EBCDIC_STATEFUL) with subst did not match.\n");
2606
2607 if(!testConvertFromUnicode(ebcdic_inputTest, sizeof(ebcdic_inputTest)/sizeof(ebcdic_inputTest[0]),
2608 toIBM930_subvaried, sizeof(toIBM930_subvaried), "ibm-930",
2609 UCNV_FROM_U_CALLBACK_SUBSTITUTE, offset_930_subvaried, mySubChar, 1 ))
2610 log_err("u-> ibm-930(EBCDIC_STATEFUL) with subst(setSubChar=0x3f) did not match.\n");
2611 }
2612 #endif
2613
2614 UBool testConvertFromUnicode(const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
2615 const char *codepage, UConverterFromUCallback callback , const int32_t *expectOffsets,
2616 const char *mySubChar, int8_t len)
2617 {
2618
2619
2620 UErrorCode status = U_ZERO_ERROR;
2621 UConverter *conv = 0;
2622 char junkout[NEW_MAX_BUFFER]; /* FIX */
2623 int32_t junokout[NEW_MAX_BUFFER]; /* FIX */
2624 const UChar *src;
2625 char *end;
2626 char *targ;
2627 int32_t *offs;
2628 int i;
2629 int32_t realBufferSize;
2630 char *realBufferEnd;
2631 const UChar *realSourceEnd;
2632 const UChar *sourceLimit;
2633 UBool checkOffsets = TRUE;
2634 UBool doFlush;
2635 char junk[9999];
2636 char offset_str[9999];
2637 char *p;
2638 UConverterFromUCallback oldAction = NULL;
2639 const void* oldContext = NULL;
2640
2641
2642 for(i=0;i<NEW_MAX_BUFFER;i++)
2643 junkout[i] = (char)0xF0;
2644 for(i=0;i<NEW_MAX_BUFFER;i++)
2645 junokout[i] = 0xFF;
2646 setNuConvTestName(codepage, "FROM");
2647
2648 log_verbose("\nTesting========= %s FROM \n inputbuffer= %d outputbuffer= %d\n", codepage, gInBufferSize,
2649 gOutBufferSize);
2650
2651 conv = ucnv_open(codepage, &status);
2652 if(U_FAILURE(status))
2653 {
2654 log_data_err("Couldn't open converter %s\n",codepage);
2655 return TRUE;
2656 }
2657
2658 log_verbose("Converter opened..\n");
2659
2660 /*----setting the callback routine----*/
2661 ucnv_setFromUCallBack (conv, callback, NULL, &oldAction, &oldContext, &status);
2662 if (U_FAILURE(status))
2663 {
2664 log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));
2665 }
2666 /*------------------------*/
2667 /*setting the subChar*/
2668 if(mySubChar != NULL){
2669 ucnv_setSubstChars(conv, mySubChar, len, &status);
2670 if (U_FAILURE(status)) {
2671 log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));
2672 }
2673 }
2674 /*------------*/
2675
2676 src = source;
2677 targ = junkout;
2678 offs = junokout;
2679
2680 realBufferSize = (sizeof(junkout)/sizeof(junkout[0]));
2681 realBufferEnd = junkout + realBufferSize;
2682 realSourceEnd = source + sourceLen;
2683
2684 if ( gOutBufferSize != realBufferSize )
2685 checkOffsets = FALSE;
2686
2687 if( gInBufferSize != NEW_MAX_BUFFER )
2688 checkOffsets = FALSE;
2689
2690 do
2691 {
2692 end = nct_min(targ + gOutBufferSize, realBufferEnd);
2693 sourceLimit = nct_min(src + gInBufferSize, realSourceEnd);
2694
2695 doFlush = (UBool)(sourceLimit == realSourceEnd);
2696
2697 if(targ == realBufferEnd)
2698 {
2699 log_err("Error, overflowed the real buffer while about to call fromUnicode! targ=%08lx %s", targ, gNuConvTestName);
2700 return FALSE;
2701 }
2702 log_verbose("calling fromUnicode @ SOURCE:%08lx to %08lx TARGET: %08lx to %08lx, flush=%s\n", src,sourceLimit, targ,end, doFlush?"TRUE":"FALSE");
2703
2704
2705 status = U_ZERO_ERROR;
2706
2707 ucnv_fromUnicode (conv,
2708 (char **)&targ,
2709 (const char *)end,
2710 &src,
2711 sourceLimit,
2712 checkOffsets ? offs : NULL,
2713 doFlush, /* flush if we're at the end of the input data */
2714 &status);
2715 } while ( (status == U_BUFFER_OVERFLOW_ERROR) || (U_SUCCESS(status) && (sourceLimit < realSourceEnd)) );
2716
2717
2718 if(status==U_INVALID_CHAR_FOUND || status == U_ILLEGAL_CHAR_FOUND){
2719 UChar errChars[50]; /* should be sufficient */
2720 int8_t errLen = 50;
2721 UErrorCode err = U_ZERO_ERROR;
2722 const UChar* limit= NULL;
2723 const UChar* start= NULL;
2724 ucnv_getInvalidUChars(conv,errChars, &errLen, &err);
2725 if(U_FAILURE(err)){
2726 log_err("ucnv_getInvalidUChars failed with error : %s\n",u_errorName(err));
2727 }
2728 /* src points to limit of invalid chars */
2729 limit = src;
2730 /* length of in invalid chars should be equal to returned length*/
2731 start = src - errLen;
2732 if(u_strncmp(errChars,start,errLen)!=0){
2733 log_err("ucnv_getInvalidUChars did not return the correct invalid chars for encoding %s \n", ucnv_getName(conv,&err));
2734 }
2735 }
2736 /* allow failure codes for the stop callback */
2737 if(U_FAILURE(status) &&
2738 (callback != UCNV_FROM_U_CALLBACK_STOP || (status != U_INVALID_CHAR_FOUND && status != U_ILLEGAL_CHAR_FOUND)))
2739 {
2740 log_err("Problem in fromUnicode, errcode %s %s\n", myErrorName(status), gNuConvTestName);
2741 return FALSE;
2742 }
2743
2744 log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
2745 sourceLen, targ-junkout);
2746 if(getTestOption(VERBOSITY_OPTION))
2747 {
2748
2749 junk[0] = 0;
2750 offset_str[0] = 0;
2751 for(p = junkout;p<targ;p++)
2752 {
2753 sprintf(junk + strlen(junk), "0x%02x, ", (0xFF) & (unsigned int)*p);
2754 sprintf(offset_str + strlen(offset_str), "0x%02x, ", (0xFF) & (unsigned int)junokout[p-junkout]);
2755 }
2756
2757 log_verbose(junk);
2758 printSeq(expect, expectLen);
2759 if ( checkOffsets )
2760 {
2761 log_verbose("\nOffsets:");
2762 log_verbose(offset_str);
2763 }
2764 log_verbose("\n");
2765 }
2766 ucnv_close(conv);
2767
2768
2769 if(expectLen != targ-junkout)
2770 {
2771 log_err("Expected %d chars out, got %d %s\n", expectLen, targ-junkout, gNuConvTestName);
2772 log_verbose("Expected %d chars out, got %d %s\n", expectLen, targ-junkout, gNuConvTestName);
2773 printSeqErr((const uint8_t *)junkout, (int32_t)(targ-junkout));
2774 printSeqErr(expect, expectLen);
2775 return FALSE;
2776 }
2777
2778 if (checkOffsets && (expectOffsets != 0) )
2779 {
2780 log_verbose("comparing %d offsets..\n", targ-junkout);
2781 if(memcmp(junokout,expectOffsets,(targ-junkout) * sizeof(int32_t) )){
2782 log_err("did not get the expected offsets while %s \n", gNuConvTestName);
2783 log_err("Got Output : ");
2784 printSeqErr((const uint8_t *)junkout, (int32_t)(targ-junkout));
2785 log_err("Got Offsets: ");
2786 for(p=junkout;p<targ;p++)
2787 log_err("%d,", junokout[p-junkout]);
2788 log_err("\n");
2789 log_err("Expected Offsets: ");
2790 for(i=0; i<(targ-junkout); i++)
2791 log_err("%d,", expectOffsets[i]);
2792 log_err("\n");
2793 return FALSE;
2794 }
2795 }
2796
2797 if(!memcmp(junkout, expect, expectLen))
2798 {
2799 log_verbose("String matches! %s\n", gNuConvTestName);
2800 return TRUE;
2801 }
2802 else
2803 {
2804 log_err("String does not match. %s\n", gNuConvTestName);
2805 log_err("source: ");
2806 printUSeqErr(source, sourceLen);
2807 log_err("Got: ");
2808 printSeqErr((const uint8_t *)junkout, expectLen);
2809 log_err("Expected: ");
2810 printSeqErr(expect, expectLen);
2811 return FALSE;
2812 }
2813 }
2814
2815 UBool testConvertToUnicode( const uint8_t *source, int sourcelen, const UChar *expect, int expectlen,
2816 const char *codepage, UConverterToUCallback callback, const int32_t *expectOffsets,
2817 const char *mySubChar, int8_t len)
2818 {
2819 UErrorCode status = U_ZERO_ERROR;
2820 UConverter *conv = 0;
2821 UChar junkout[NEW_MAX_BUFFER]; /* FIX */
2822 int32_t junokout[NEW_MAX_BUFFER]; /* FIX */
2823 const char *src;
2824 const char *realSourceEnd;
2825 const char *srcLimit;
2826 UChar *targ;
2827 UChar *end;
2828 int32_t *offs;
2829 int i;
2830 UBool checkOffsets = TRUE;
2831 char junk[9999];
2832 char offset_str[9999];
2833 UChar *p;
2834 UConverterToUCallback oldAction = NULL;
2835 const void* oldContext = NULL;
2836
2837 int32_t realBufferSize;
2838 UChar *realBufferEnd;
2839
2840
2841 for(i=0;i<NEW_MAX_BUFFER;i++)
2842 junkout[i] = 0xFFFE;
2843
2844 for(i=0;i<NEW_MAX_BUFFER;i++)
2845 junokout[i] = -1;
2846
2847 setNuConvTestName(codepage, "TO");
2848
2849 log_verbose("\n========= %s\n", gNuConvTestName);
2850
2851 conv = ucnv_open(codepage, &status);
2852 if(U_FAILURE(status))
2853 {
2854 log_data_err("Couldn't open converter %s\n",gNuConvTestName);
2855 return TRUE;
2856 }
2857
2858 log_verbose("Converter opened..\n");
2859
2860 src = (const char *)source;
2861 targ = junkout;
2862 offs = junokout;
2863
2864 realBufferSize = (sizeof(junkout)/sizeof(junkout[0]));
2865 realBufferEnd = junkout + realBufferSize;
2866 realSourceEnd = src + sourcelen;
2867 /*----setting the callback routine----*/
2868 ucnv_setToUCallBack (conv, callback, NULL, &oldAction, &oldContext, &status);
2869 if (U_FAILURE(status))
2870 {
2871 log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));
2872 }
2873 /*-------------------------------------*/
2874 /*setting the subChar*/
2875 if(mySubChar != NULL){
2876 ucnv_setSubstChars(conv, mySubChar, len, &status);
2877 if (U_FAILURE(status)) {
2878 log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));
2879 }
2880 }
2881 /*------------*/
2882
2883
2884 if ( gOutBufferSize != realBufferSize )
2885 checkOffsets = FALSE;
2886
2887 if( gInBufferSize != NEW_MAX_BUFFER )
2888 checkOffsets = FALSE;
2889
2890 do
2891 {
2892 end = nct_min( targ + gOutBufferSize, realBufferEnd);
2893 srcLimit = nct_min(realSourceEnd, src + gInBufferSize);
2894
2895 if(targ == realBufferEnd)
2896 {
2897 log_err("Error, the end would overflow the real output buffer while about to call toUnicode! tarjey=%08lx %s",targ,gNuConvTestName);
2898 return FALSE;
2899 }
2900 log_verbose("calling toUnicode @ %08lx to %08lx\n", targ,end);
2901
2902
2903
2904 status = U_ZERO_ERROR;
2905
2906 ucnv_toUnicode (conv,
2907 &targ,
2908 end,
2909 (const char **)&src,
2910 (const char *)srcLimit,
2911 checkOffsets ? offs : NULL,
2912 (UBool)(srcLimit == realSourceEnd), /* flush if we're at the end of the source data */
2913 &status);
2914 } while ( (status == U_BUFFER_OVERFLOW_ERROR) || (U_SUCCESS(status) && (srcLimit < realSourceEnd)) ); /* while we just need another buffer */
2915
2916 if(status==U_INVALID_CHAR_FOUND || status == U_ILLEGAL_CHAR_FOUND){
2917 char errChars[50]; /* should be sufficient */
2918 int8_t errLen = 50;
2919 UErrorCode err = U_ZERO_ERROR;
2920 const char* limit= NULL;
2921 const char* start= NULL;
2922 ucnv_getInvalidChars(conv,errChars, &errLen, &err);
2923 if(U_FAILURE(err)){
2924 log_err("ucnv_getInvalidChars failed with error : %s\n",u_errorName(err));
2925 }
2926 /* src points to limit of invalid chars */
2927 limit = src;
2928 /* length of in invalid chars should be equal to returned length*/
2929 start = src - errLen;
2930 if(uprv_strncmp(errChars,start,errLen)!=0){
2931 log_err("ucnv_getInvalidChars did not return the correct invalid chars for encoding %s \n", ucnv_getName(conv,&err));
2932 }
2933 }
2934 /* allow failure codes for the stop callback */
2935 if(U_FAILURE(status) &&
2936 (callback != UCNV_TO_U_CALLBACK_STOP || (status != U_INVALID_CHAR_FOUND && status != U_ILLEGAL_CHAR_FOUND && status != U_TRUNCATED_CHAR_FOUND)))
2937 {
2938 log_err("Problem doing toUnicode, errcode %s %s\n", myErrorName(status), gNuConvTestName);
2939 return FALSE;
2940 }
2941
2942 log_verbose("\nConversion done. %d bytes -> %d chars.\nResult :",
2943 sourcelen, targ-junkout);
2944 if(getTestOption(VERBOSITY_OPTION))
2945 {
2946
2947 junk[0] = 0;
2948 offset_str[0] = 0;
2949
2950 for(p = junkout;p<targ;p++)
2951 {
2952 sprintf(junk + strlen(junk), "0x%04x, ", (0xFFFF) & (unsigned int)*p);
2953 sprintf(offset_str + strlen(offset_str), "0x%04x, ", (0xFFFF) & (unsigned int)junokout[p-junkout]);
2954 }
2955
2956 log_verbose(junk);
2957 printUSeq(expect, expectlen);
2958 if ( checkOffsets )
2959 {
2960 log_verbose("\nOffsets:");
2961 log_verbose(offset_str);
2962 }
2963 log_verbose("\n");
2964 }
2965 ucnv_close(conv);
2966
2967 log_verbose("comparing %d uchars (%d bytes)..\n",expectlen,expectlen*2);
2968
2969 if (checkOffsets && (expectOffsets != 0))
2970 {
2971 if(memcmp(junokout,expectOffsets,(targ-junkout) * sizeof(int32_t)))
2972 {
2973 log_err("did not get the expected offsets while %s \n", gNuConvTestName);
2974 log_err("Got offsets: ");
2975 for(p=junkout;p<targ;p++)
2976 log_err(" %2d,", junokout[p-junkout]);
2977 log_err("\n");
2978 log_err("Expected offsets: ");
2979 for(i=0; i<(targ-junkout); i++)
2980 log_err(" %2d,", expectOffsets[i]);
2981 log_err("\n");
2982 log_err("Got output: ");
2983 for(i=0; i<(targ-junkout); i++)
2984 log_err("0x%04x,", junkout[i]);
2985 log_err("\n");
2986 log_err("From source: ");
2987 for(i=0; i<(src-(const char *)source); i++)
2988 log_err(" 0x%02x,", (unsigned char)source[i]);
2989 log_err("\n");
2990 }
2991 }
2992
2993 if(!memcmp(junkout, expect, expectlen*2))
2994 {
2995 log_verbose("Matches!\n");
2996 return TRUE;
2997 }
2998 else
2999 {
3000 log_err("String does not match. %s\n", gNuConvTestName);
3001 log_verbose("String does not match. %s\n", gNuConvTestName);
3002 log_err("Got: ");
3003 printUSeqErr(junkout, expectlen);
3004 log_err("Expected: ");
3005 printUSeqErr(expect, expectlen);
3006 log_err("\n");
3007 return FALSE;
3008 }
3009 }
3010
3011 UBool testConvertFromUnicodeWithContext(const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
3012 const char *codepage, UConverterFromUCallback callback , const int32_t *expectOffsets,
3013 const char *mySubChar, int8_t len, const void* context, UErrorCode expectedError)
3014 {
3015
3016
3017 UErrorCode status = U_ZERO_ERROR;
3018 UConverter *conv = 0;
3019 char junkout[NEW_MAX_BUFFER]; /* FIX */
3020 int32_t junokout[NEW_MAX_BUFFER]; /* FIX */
3021 const UChar *src;
3022 char *end;
3023 char *targ;
3024 int32_t *offs;
3025 int i;
3026 int32_t realBufferSize;
3027 char *realBufferEnd;
3028 const UChar *realSourceEnd;
3029 const UChar *sourceLimit;
3030 UBool checkOffsets = TRUE;
3031 UBool doFlush;
3032 char junk[9999];
3033 char offset_str[9999];
3034 char *p;
3035 UConverterFromUCallback oldAction = NULL;
3036 const void* oldContext = NULL;
3037
3038
3039 for(i=0;i<NEW_MAX_BUFFER;i++)
3040 junkout[i] = (char)0xF0;
3041 for(i=0;i<NEW_MAX_BUFFER;i++)
3042 junokout[i] = 0xFF;
3043 setNuConvTestName(codepage, "FROM");
3044
3045 log_verbose("\nTesting========= %s FROM \n inputbuffer= %d outputbuffer= %d\n", codepage, gInBufferSize,
3046 gOutBufferSize);
3047
3048 conv = ucnv_open(codepage, &status);
3049 if(U_FAILURE(status))
3050 {
3051 log_data_err("Couldn't open converter %s\n",codepage);
3052 return TRUE; /* Because the err has already been logged. */
3053 }
3054
3055 log_verbose("Converter opened..\n");
3056
3057 /*----setting the callback routine----*/
3058 ucnv_setFromUCallBack (conv, callback, context, &oldAction, &oldContext, &status);
3059 if (U_FAILURE(status))
3060 {
3061 log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));
3062 }
3063 /*------------------------*/
3064 /*setting the subChar*/
3065 if(mySubChar != NULL){
3066 ucnv_setSubstChars(conv, mySubChar, len, &status);
3067 if (U_FAILURE(status)) {
3068 log_err("FAILURE in setting substitution chars! %s\n", myErrorName(status));
3069 }
3070 }
3071 /*------------*/
3072
3073 src = source;
3074 targ = junkout;
3075 offs = junokout;
3076
3077 realBufferSize = (sizeof(junkout)/sizeof(junkout[0]));
3078 realBufferEnd = junkout + realBufferSize;
3079 realSourceEnd = source + sourceLen;
3080
3081 if ( gOutBufferSize != realBufferSize )
3082 checkOffsets = FALSE;
3083
3084 if( gInBufferSize != NEW_MAX_BUFFER )
3085 checkOffsets = FALSE;
3086
3087 do
3088 {
3089 end = nct_min(targ + gOutBufferSize, realBufferEnd);
3090 sourceLimit = nct_min(src + gInBufferSize, realSourceEnd);
3091
3092 doFlush = (UBool)(sourceLimit == realSourceEnd);
3093
3094 if(targ == realBufferEnd)
3095 {
3096 log_err("Error, overflowed the real buffer while about to call fromUnicode! targ=%08lx %s", targ, gNuConvTestName);
3097 return FALSE;
3098 }
3099 log_verbose("calling fromUnicode @ SOURCE:%08lx to %08lx TARGET: %08lx to %08lx, flush=%s\n", src,sourceLimit, targ,end, doFlush?"TRUE":"FALSE");
3100
3101
3102 status = U_ZERO_ERROR;
3103
3104 ucnv_fromUnicode (conv,
3105 (char **)&targ,
3106 (const char *)end,
3107 &src,
3108 sourceLimit,
3109 checkOffsets ? offs : NULL,
3110 doFlush, /* flush if we're at the end of the input data */
3111 &status);
3112 } while ( (status == U_BUFFER_OVERFLOW_ERROR) || (U_SUCCESS(status) && (sourceLimit < realSourceEnd)) );
3113
3114 /* allow failure codes for the stop callback */
3115 if(U_FAILURE(status) && status != expectedError)
3116 {
3117 log_err("Problem in fromUnicode, errcode %s %s\n", myErrorName(status), gNuConvTestName);
3118 return FALSE;
3119 }
3120
3121 log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
3122 sourceLen, targ-junkout);
3123 if(getTestOption(VERBOSITY_OPTION))
3124 {
3125
3126 junk[0] = 0;
3127 offset_str[0] = 0;
3128 for(p = junkout;p<targ;p++)
3129 {
3130 sprintf(junk + strlen(junk), "0x%02x, ", (0xFF) & (unsigned int)*p);
3131 sprintf(offset_str + strlen(offset_str), "0x%02x, ", (0xFF) & (unsigned int)junokout[p-junkout]);
3132 }
3133
3134 log_verbose(junk);
3135 printSeq(expect, expectLen);
3136 if ( checkOffsets )
3137 {
3138 log_verbose("\nOffsets:");
3139 log_verbose(offset_str);
3140 }
3141 log_verbose("\n");
3142 }
3143 ucnv_close(conv);
3144
3145
3146 if(expectLen != targ-junkout)
3147 {
3148 log_err("Expected %d chars out, got %d %s\n", expectLen, targ-junkout, gNuConvTestName);
3149 log_verbose("Expected %d chars out, got %d %s\n", expectLen, targ-junkout, gNuConvTestName);
3150 printSeqErr((const uint8_t *)junkout, (int32_t)(targ-junkout));
3151 printSeqErr(expect, expectLen);
3152 return FALSE;
3153 }
3154
3155 if (checkOffsets && (expectOffsets != 0) )
3156 {
3157 log_verbose("comparing %d offsets..\n", targ-junkout);
3158 if(memcmp(junokout,expectOffsets,(targ-junkout) * sizeof(int32_t) )){
3159 log_err("did not get the expected offsets while %s \n", gNuConvTestName);
3160 log_err("Got Output : ");
3161 printSeqErr((const uint8_t *)junkout, (int32_t)(targ-junkout));
3162 log_err("Got Offsets: ");
3163 for(p=junkout;p<targ;p++)
3164 log_err("%d,", junokout[p-junkout]);
3165 log_err("\n");
3166 log_err("Expected Offsets: ");
3167 for(i=0; i<(targ-junkout); i++)
3168 log_err("%d,", expectOffsets[i]);
3169 log_err("\n");
3170 return FALSE;
3171 }
3172 }
3173
3174 if(!memcmp(junkout, expect, expectLen))
3175 {
3176 log_verbose("String matches! %s\n", gNuConvTestName);
3177 return TRUE;
3178 }
3179 else
3180 {
3181 log_err("String does not match. %s\n", gNuConvTestName);
3182 log_err("source: ");
3183 printUSeqErr(source, sourceLen);
3184 log_err("Got: ");
3185 printSeqErr((const uint8_t *)junkout, expectLen);
3186 log_err("Expected: ");
3187 printSeqErr(expect, expectLen);
3188 return FALSE;
3189 }
3190 }
3191 UBool testConvertToUnicodeWithContext( const uint8_t *source, int sourcelen, const UChar *expect, int expectlen,
3192 const char *codepage, UConverterToUCallback callback, const int32_t *expectOffsets,
3193 const char *mySubChar, int8_t len, const void* context, UErrorCode expectedError)
3194 {
3195 UErrorCode status = U_ZERO_ERROR;
3196 UConverter *conv = 0;
3197 UChar junkout[NEW_MAX_BUFFER]; /* FIX */
3198 int32_t junokout[NEW_MAX_BUFFER]; /* FIX */
3199 const char *src;
3200 const char *realSourceEnd;
3201 const char *srcLimit;
3202 UChar *targ;
3203 UChar *end;
3204 int32_t *offs;
3205 int i;
3206 UBool checkOffsets = TRUE;
3207 char junk[9999];
3208 char offset_str[9999];
3209 UChar *p;
3210 UConverterToUCallback oldAction = NULL;
3211 const void* oldContext = NULL;
3212
3213 int32_t realBufferSize;
3214 UChar *realBufferEnd;
3215
3216
3217 for(i=0;i<NEW_MAX_BUFFER;i++)
3218 junkout[i] = 0xFFFE;
3219
3220 for(i=0;i<NEW_MAX_BUFFER;i++)
3221 junokout[i] = -1;
3222
3223 setNuConvTestName(codepage, "TO");
3224
3225 log_verbose("\n========= %s\n", gNuConvTestName);
3226
3227 conv = ucnv_open(codepage, &status);
3228 if(U_FAILURE(status))
3229 {
3230 log_data_err("Couldn't open converter %s\n",gNuConvTestName);
3231 return TRUE;
3232 }
3233
3234 log_verbose("Converter opened..\n");
3235
3236 src = (const char *)source;
3237 targ = junkout;
3238 offs = junokout;
3239
3240 realBufferSize = (sizeof(junkout)/sizeof(junkout[0]));
3241 realBufferEnd = junkout + realBufferSize;
3242 realSourceEnd = src + sourcelen;
3243 /*----setting the callback routine----*/
3244 ucnv_setToUCallBack (conv, callback, context, &oldAction, &oldContext, &status);
3245 if (U_FAILURE(status))
3246 {
3247 log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));
3248 }
3249 /*-------------------------------------*/
3250 /*setting the subChar*/
3251 if(mySubChar != NULL){
3252 ucnv_setSubstChars(conv, mySubChar, len, &status);
3253 if (U_FAILURE(status)) {
3254 log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));
3255 }
3256 }
3257 /*------------*/
3258
3259
3260 if ( gOutBufferSize != realBufferSize )
3261 checkOffsets = FALSE;
3262
3263 if( gInBufferSize != NEW_MAX_BUFFER )
3264 checkOffsets = FALSE;
3265
3266 do
3267 {
3268 end = nct_min( targ + gOutBufferSize, realBufferEnd);
3269 srcLimit = nct_min(realSourceEnd, src + gInBufferSize);
3270
3271 if(targ == realBufferEnd)
3272 {
3273 log_err("Error, the end would overflow the real output buffer while about to call toUnicode! tarjey=%08lx %s",targ,gNuConvTestName);
3274 return FALSE;
3275 }
3276 log_verbose("calling toUnicode @ %08lx to %08lx\n", targ,end);
3277
3278
3279
3280 status = U_ZERO_ERROR;
3281
3282 ucnv_toUnicode (conv,
3283 &targ,
3284 end,
3285 (const char **)&src,
3286 (const char *)srcLimit,
3287 checkOffsets ? offs : NULL,
3288 (UBool)(srcLimit == realSourceEnd), /* flush if we're at the end of the source data */
3289 &status);
3290 } while ( (status == U_BUFFER_OVERFLOW_ERROR) || (U_SUCCESS(status) && (srcLimit < realSourceEnd)) ); /* while we just need another buffer */
3291
3292 /* allow failure codes for the stop callback */
3293 if(U_FAILURE(status) && status!=expectedError)
3294 {
3295 log_err("Problem doing toUnicode, errcode %s %s\n", myErrorName(status), gNuConvTestName);
3296 return FALSE;
3297 }
3298
3299 log_verbose("\nConversion done. %d bytes -> %d chars.\nResult :",
3300 sourcelen, targ-junkout);
3301 if(getTestOption(VERBOSITY_OPTION))
3302 {
3303
3304 junk[0] = 0;
3305 offset_str[0] = 0;
3306
3307 for(p = junkout;p<targ;p++)
3308 {
3309 sprintf(junk + strlen(junk), "0x%04x, ", (0xFFFF) & (unsigned int)*p);
3310 sprintf(offset_str + strlen(offset_str), "0x%04x, ", (0xFFFF) & (unsigned int)junokout[p-junkout]);
3311 }
3312
3313 log_verbose(junk);
3314 printUSeq(expect, expectlen);
3315 if ( checkOffsets )
3316 {
3317 log_verbose("\nOffsets:");
3318 log_verbose(offset_str);
3319 }
3320 log_verbose("\n");
3321 }
3322 ucnv_close(conv);
3323
3324 log_verbose("comparing %d uchars (%d bytes)..\n",expectlen,expectlen*2);
3325
3326 if (checkOffsets && (expectOffsets != 0))
3327 {
3328 if(memcmp(junokout,expectOffsets,(targ-junkout) * sizeof(int32_t)))
3329 {
3330 log_err("did not get the expected offsets while %s \n", gNuConvTestName);
3331 log_err("Got offsets: ");
3332 for(p=junkout;p<targ;p++)
3333 log_err(" %2d,", junokout[p-junkout]);
3334 log_err("\n");
3335 log_err("Expected offsets: ");
3336 for(i=0; i<(targ-junkout); i++)
3337 log_err(" %2d,", expectOffsets[i]);
3338 log_err("\n");
3339 log_err("Got output: ");
3340 for(i=0; i<(targ-junkout); i++)
3341 log_err("0x%04x,", junkout[i]);
3342 log_err("\n");
3343 log_err("From source: ");
3344 for(i=0; i<(src-(const char *)source); i++)
3345 log_err(" 0x%02x,", (unsigned char)source[i]);
3346 log_err("\n");
3347 }
3348 }
3349
3350 if(!memcmp(junkout, expect, expectlen*2))
3351 {
3352 log_verbose("Matches!\n");
3353 return TRUE;
3354 }
3355 else
3356 {
3357 log_err("String does not match. %s\n", gNuConvTestName);
3358 log_verbose("String does not match. %s\n", gNuConvTestName);
3359 log_err("Got: ");
3360 printUSeqErr(junkout, expectlen);
3361 log_err("Expected: ");
3362 printUSeqErr(expect, expectlen);
3363 log_err("\n");
3364 return FALSE;
3365 }
3366 }
3367
3368 static void TestCallBackFailure(void) {
3369 UErrorCode status = U_USELESS_COLLATOR_ERROR;
3370 ucnv_cbFromUWriteBytes(NULL, NULL, -1, -1, &status);
3371 if (status != U_USELESS_COLLATOR_ERROR) {
3372 log_err("Error: ucnv_cbFromUWriteBytes did not react correctly to a bad UErrorCode\n");
3373 }
3374 ucnv_cbFromUWriteUChars(NULL, NULL, NULL, -1, &status);
3375 if (status != U_USELESS_COLLATOR_ERROR) {
3376 log_err("Error: ucnv_cbFromUWriteUChars did not react correctly to a bad UErrorCode\n");
3377 }
3378 ucnv_cbFromUWriteSub(NULL, -1, &status);
3379 if (status != U_USELESS_COLLATOR_ERROR) {
3380 log_err("Error: ucnv_cbFromUWriteSub did not react correctly to a bad UErrorCode\n");
3381 }
3382 ucnv_cbToUWriteUChars(NULL, NULL, -1, -1, &status);
3383 if (status != U_USELESS_COLLATOR_ERROR) {
3384 log_err("Error: ucnv_cbToUWriteUChars did not react correctly to a bad UErrorCode\n");
3385 }
3386 }
mirror of ICU