]> git.saurik.com Git - apple/icu.git/blame_incremental - icuSources/test/cintltst/ncnvtst.c
projects / apple / icu.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
ICU-66108.tar.gz
[apple/icu.git] / icuSources / test / cintltst / ncnvtst.c
... / ...
CommitLineData
1// © 2016 and later: Unicode, Inc. and others.
2// License & terms of use: http://www.unicode.org/copyright.html
3/********************************************************************
4 * COPYRIGHT:
5 * Copyright (c) 1997-2016, International Business Machines Corporation and
6 * others. All Rights Reserved.
7 ********************************************************************/
8/*****************************************************************************
9*
10* File ncnvtst.c
11*
12* Modification History:
13* Name Description
14* Madhu Katragadda 7/7/2000 Converter Tests for extended code coverage
15******************************************************************************
16*/
17#include <stdio.h>
18#include <stdlib.h>
19#include <string.h>
20#include "unicode/uloc.h"
21#include "unicode/ucnv.h"
22#include "unicode/utypes.h"
23#include "unicode/ustring.h"
24#include "unicode/uset.h"
25#include "unicode/utf8.h"
26#include "unicode/utf16.h"
27#include "cintltst.h"
28#include "cmemory.h"
29
30#define MAX_LENGTH 999
31
32#define UNICODE_LIMIT 0x10FFFF
33#define SURROGATE_HIGH_START 0xD800
34#define SURROGATE_LOW_END 0xDFFF
35
36static int32_t gInBufferSize = 0;
37static int32_t gOutBufferSize = 0;
38static char gNuConvTestName[1024];
39
40#define nct_min(x,y) ((x<y) ? x : y)
41
42static void printSeq(const unsigned char* a, int len);
43static void printSeqErr(const unsigned char* a, int len);
44static void printUSeq(const UChar* a, int len);
45static void printUSeqErr(const UChar* a, int len);
46static UBool convertFromU( const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
47 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus);
48static UBool convertToU( const uint8_t *source, int sourceLen, const UChar *expect, int expectLen,
49 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus);
50
51static UBool testConvertFromU( const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
52 const char *codepage, UConverterFromUCallback callback, const int32_t *expectOffsets, UBool testReset);
53static UBool testConvertToU( const uint8_t *source, int sourcelen, const UChar *expect, int expectlen,
54 const char *codepage, UConverterToUCallback callback, const int32_t *expectOffsets, UBool testReset);
55
56static void setNuConvTestName(const char *codepage, const char *direction)
57{
58 sprintf(gNuConvTestName, "[Testing %s %s Unicode, InputBufSiz=%d, OutputBufSiz=%d]",
59 codepage,
60 direction,
61 (int)gInBufferSize,
62 (int)gOutBufferSize);
63}
64
65
66static void TestSurrogateBehaviour(void);
67static void TestErrorBehaviour(void);
68
69#if !UCONFIG_NO_LEGACY_CONVERSION
70static void TestToUnicodeErrorBehaviour(void);
71static void TestGetNextErrorBehaviour(void);
72#endif
73
74static void TestRegressionUTF8(void);
75static void TestRegressionUTF32(void);
76static void TestAvailableConverters(void);
77static void TestFlushInternalBuffer(void); /*for improved code coverage in ucnv_cnv.c*/
78static void TestResetBehaviour(void);
79static void TestTruncated(void);
80static void TestUnicodeSet(void);
81
82static void TestWithBufferSize(int32_t osize, int32_t isize);
83
84
85static void printSeq(const unsigned char* a, int len)
86{
87 int i=0;
88 log_verbose("\n{");
89 while (i<len)
90 log_verbose("0x%02X ", a[i++]);
91 log_verbose("}\n");
92}
93
94static void printUSeq(const UChar* a, int len)
95{
96 int i=0;
97 log_verbose("\n{");
98 while (i<len)
99 log_verbose("%0x04X ", a[i++]);
100 log_verbose("}\n");
101}
102
103static void printSeqErr(const unsigned char* a, int len)
104{
105 int i=0;
106 fprintf(stderr, "\n{");
107 while (i<len) fprintf(stderr, "0x%02X ", a[i++]);
108 fprintf(stderr, "}\n");
109}
110
111static void printUSeqErr(const UChar* a, int len)
112{
113 int i=0;
114 fprintf(stderr, "\n{");
115 while (i<len)
116 fprintf(stderr, "0x%04X ", a[i++]);
117 fprintf(stderr,"}\n");
118}
119
120void addExtraTests(TestNode** root);
121
122void addExtraTests(TestNode** root)
123{
124 addTest(root, &TestSurrogateBehaviour, "tsconv/ncnvtst/TestSurrogateBehaviour");
125 addTest(root, &TestErrorBehaviour, "tsconv/ncnvtst/TestErrorBehaviour");
126
127#if !UCONFIG_NO_LEGACY_CONVERSION
128 addTest(root, &TestToUnicodeErrorBehaviour, "tsconv/ncnvtst/ToUnicodeErrorBehaviour");
129 addTest(root, &TestGetNextErrorBehaviour, "tsconv/ncnvtst/TestGetNextErrorBehaviour");
130#endif
131
132 addTest(root, &TestAvailableConverters, "tsconv/ncnvtst/TestAvailableConverters");
133 addTest(root, &TestFlushInternalBuffer, "tsconv/ncnvtst/TestFlushInternalBuffer");
134 addTest(root, &TestResetBehaviour, "tsconv/ncnvtst/TestResetBehaviour");
135 addTest(root, &TestRegressionUTF8, "tsconv/ncnvtst/TestRegressionUTF8");
136 addTest(root, &TestRegressionUTF32, "tsconv/ncnvtst/TestRegressionUTF32");
137 addTest(root, &TestTruncated, "tsconv/ncnvtst/TestTruncated");
138 addTest(root, &TestUnicodeSet, "tsconv/ncnvtst/TestUnicodeSet");
139}
140
141/*test surrogate behaviour*/
142static void TestSurrogateBehaviour(){
143 log_verbose("Testing for SBCS and LATIN_1\n");
144 {
145 UChar sampleText[] = {0x0031, 0xd801, 0xdc01, 0x0032};
146 const uint8_t expected[] = {0x31, 0x1a, 0x32};
147
148#if !UCONFIG_NO_LEGACY_CONVERSION
149 /*SBCS*/
150 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
151 expected, sizeof(expected), "ibm-920", 0 , TRUE, U_ZERO_ERROR))
152 log_err("u-> ibm-920 [UCNV_SBCS] not match.\n");
153#endif
154
155 /*LATIN_1*/
156 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
157 expected, sizeof(expected), "LATIN_1", 0, TRUE, U_ZERO_ERROR ))
158 log_err("u-> LATIN_1 not match.\n");
159
160 }
161
162#if !UCONFIG_NO_LEGACY_CONVERSION
163 log_verbose("Testing for DBCS and MBCS\n");
164 {
165 UChar sampleText[] = {0x00a1, 0xd801, 0xdc01, 0x00a4};
166 const uint8_t expected[] = {0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
167 int32_t offsets[] = {0x00, 0x00, 0x01, 0x01, 0x03, 0x03 };
168
169 /*DBCS*/
170 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
171 expected, sizeof(expected), "ibm-1363", 0 , TRUE, U_ZERO_ERROR))
172 log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
173 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
174 expected, sizeof(expected), "ibm-1363", offsets , TRUE, U_ZERO_ERROR))
175 log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
176 /*MBCS*/
177 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
178 expected, sizeof(expected), "ibm-1363", 0 , TRUE, U_ZERO_ERROR))
179 log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
180 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
181 expected, sizeof(expected), "ibm-1363", offsets, TRUE, U_ZERO_ERROR))
182 log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
183 }
184
185 log_verbose("Testing for ISO-2022-jp\n");
186 {
187 UChar sampleText[] = { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
188
189 const uint8_t expected[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
190 0x31,0x1A, 0x32};
191
192
193 int32_t offsets[] = {0,0,0,0,0,1,1,2,2,2,2,3,5 };
194
195 /*iso-2022-jp*/
196 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
197 expected, sizeof(expected), "iso-2022-jp", 0 , TRUE, U_ZERO_ERROR))
198 log_err("u-> not match.\n");
199 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
200 expected, sizeof(expected), "iso-2022-jp", offsets , TRUE, U_ZERO_ERROR))
201 log_err("u-> not match.\n");
202 }
203
204 log_verbose("Testing for ISO-2022-cn\n");
205 {
206 static const UChar sampleText[] = { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
207
208 static const uint8_t expected[] = {
209 0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B,
210 0x36, 0x21,
211 0x0F, 0x31,
212 0x1A,
213 0x32
214 };
215
216
217
218 static const int32_t offsets[] = {
219 0, 0, 0, 0, 0, 0, 0,
220 1, 1,
221 2, 2,
222 3,
223 5, };
224
225 /*iso-2022-CN*/
226 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
227 expected, sizeof(expected), "iso-2022-cn", 0 , TRUE, U_ZERO_ERROR))
228 log_err("u-> not match.\n");
229 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
230 expected, sizeof(expected), "iso-2022-cn", offsets , TRUE, U_ZERO_ERROR))
231 log_err("u-> not match.\n");
232 }
233
234 log_verbose("Testing for ISO-2022-kr\n");
235 {
236 static const UChar sampleText[] = { 0x4e00,0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
237
238 static const uint8_t expected[] = {0x1B, 0x24, 0x29, 0x43,
239 0x0E, 0x6C, 0x69,
240 0x0f, 0x1A,
241 0x0e, 0x6F, 0x4B,
242 0x0F, 0x31,
243 0x1A,
244 0x32 };
245
246 static const int32_t offsets[] = {-1, -1, -1, -1,
247 0, 0, 0,
248 1, 1,
249 3, 3, 3,
250 4, 4,
251 5,
252 7,
253 };
254
255 /*iso-2022-kr*/
256 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
257 expected, sizeof(expected), "iso-2022-kr", 0 , TRUE, U_ZERO_ERROR))
258 log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
259 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
260 expected, sizeof(expected), "iso-2022-kr", offsets , TRUE, U_ZERO_ERROR))
261 log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
262 }
263
264 log_verbose("Testing for HZ\n");
265 {
266 static const UChar sampleText[] = { 0x4e00, 0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
267
268 static const uint8_t expected[] = {0x7E, 0x7B, 0x52, 0x3B,
269 0x7E, 0x7D, 0x1A,
270 0x7E, 0x7B, 0x36, 0x21,
271 0x7E, 0x7D, 0x31,
272 0x1A,
273 0x32 };
274
275
276 static const int32_t offsets[] = {0,0,0,0,
277 1,1,1,
278 3,3,3,3,
279 4,4,4,
280 5,
281 7,};
282
283 /*hz*/
284 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
285 expected, sizeof(expected), "HZ", 0 , TRUE, U_ZERO_ERROR))
286 log_err("u-> HZ not match.\n");
287 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
288 expected, sizeof(expected), "HZ", offsets , TRUE, U_ZERO_ERROR))
289 log_err("u-> HZ not match.\n");
290 }
291#endif
292
293 /*UTF-8*/
294 log_verbose("Testing for UTF8\n");
295 {
296 static const UChar sampleText[] = { 0x4e00, 0x0701, 0x0031, 0xbfc1, 0xd801, 0xdc01, 0x0032};
297 static const int32_t offsets[]={0x00, 0x00, 0x00, 0x01, 0x01, 0x02,
298 0x03, 0x03, 0x03, 0x04, 0x04, 0x04,
299 0x04, 0x06 };
300 static const uint8_t expected[] = {0xe4, 0xb8, 0x80, 0xdc, 0x81, 0x31,
301 0xeb, 0xbf, 0x81, 0xF0, 0x90, 0x90, 0x81, 0x32};
302
303
304 static const int32_t fromOffsets[] = { 0x0000, 0x0003, 0x0005, 0x0006, 0x0009, 0x0009, 0x000D };
305 /*UTF-8*/
306 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
307 expected, sizeof(expected), "UTF8", offsets, TRUE, U_ZERO_ERROR ))
308 log_err("u-> UTF8 with offsets and flush true did not match.\n");
309 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
310 expected, sizeof(expected), "UTF8", 0, TRUE, U_ZERO_ERROR ))
311 log_err("u-> UTF8 with offsets and flush true did not match.\n");
312 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
313 expected, sizeof(expected), "UTF8", offsets, FALSE, U_ZERO_ERROR ))
314 log_err("u-> UTF8 with offsets and flush true did not match.\n");
315 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
316 expected, sizeof(expected), "UTF8", 0, FALSE, U_ZERO_ERROR ))
317 log_err("u-> UTF8 with offsets and flush true did not match.\n");
318
319 if(!convertToU(expected, sizeof(expected),
320 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", 0, TRUE, U_ZERO_ERROR ))
321 log_err("UTF8 -> u did not match.\n");
322 if(!convertToU(expected, sizeof(expected),
323 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", 0, FALSE, U_ZERO_ERROR ))
324 log_err("UTF8 -> u did not match.\n");
325 if(!convertToU(expected, sizeof(expected),
326 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", fromOffsets, TRUE, U_ZERO_ERROR ))
327 log_err("UTF8 ->u did not match.\n");
328 if(!convertToU(expected, sizeof(expected),
329 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", fromOffsets, FALSE, U_ZERO_ERROR ))
330 log_err("UTF8 -> u did not match.\n");
331
332 }
333}
334
335/*test various error behaviours*/
336static void TestErrorBehaviour(){
337 log_verbose("Testing for SBCS and LATIN_1\n");
338 {
339 static const UChar sampleText[] = { 0x0031, 0xd801};
340 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
341 static const uint8_t expected0[] = { 0x31};
342 static const uint8_t expected[] = { 0x31, 0x1a};
343 static const uint8_t expected2[] = { 0x31, 0x1a, 0x32};
344
345#if !UCONFIG_NO_LEGACY_CONVERSION
346 /*SBCS*/
347 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
348 expected, sizeof(expected), "ibm-920", 0, TRUE, U_ZERO_ERROR))
349 log_err("u-> ibm-920 [UCNV_SBCS] \n");
350 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
351 expected0, sizeof(expected0), "ibm-920", 0, FALSE, U_ZERO_ERROR))
352 log_err("u-> ibm-920 [UCNV_SBCS] \n");
353 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
354 expected2, sizeof(expected2), "ibm-920", 0, TRUE, U_ZERO_ERROR))
355 log_err("u-> ibm-920 [UCNV_SBCS] did not match\n");
356#endif
357
358 /*LATIN_1*/
359 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
360 expected, sizeof(expected), "LATIN_1", 0, TRUE, U_ZERO_ERROR))
361 log_err("u-> LATIN_1 is supposed to fail\n");
362 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
363 expected0, sizeof(expected0), "LATIN_1", 0, FALSE, U_ZERO_ERROR))
364 log_err("u-> LATIN_1 is supposed to fail\n");
365
366 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
367 expected2, sizeof(expected2), "LATIN_1", 0, TRUE, U_ZERO_ERROR))
368 log_err("u-> LATIN_1 did not match\n");
369 }
370
371#if !UCONFIG_NO_LEGACY_CONVERSION
372 log_verbose("Testing for DBCS and MBCS\n");
373 {
374 static const UChar sampleText[] = { 0x00a1, 0xd801};
375 static const uint8_t expected[] = { 0xa2, 0xae};
376 static const int32_t offsets[] = { 0x00, 0x00};
377 static const uint8_t expectedSUB[] = { 0xa2, 0xae, 0xa1, 0xe0};
378 static const int32_t offsetsSUB[] = { 0x00, 0x00, 0x01, 0x01};
379
380 static const UChar sampleText2[] = { 0x00a1, 0xd801, 0x00a4};
381 static const uint8_t expected2[] = { 0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
382 static const int32_t offsets2[] = { 0x00, 0x00, 0x01, 0x01, 0x02, 0x02};
383
384 static const UChar sampleText3MBCS[] = { 0x0001, 0x00a4, 0xdc01};
385 static const uint8_t expected3MBCS[] = { 0x01, 0xa2, 0xb4, 0xa1, 0xe0};
386 static const int32_t offsets3MBCS[] = { 0x00, 0x01, 0x01, 0x02, 0x02};
387
388 static const UChar sampleText4MBCS[] = { 0x0061, 0xFFE4, 0xdc01};
389 static const uint8_t expected4MBCS[] = { 0x61, 0x8f, 0xa2, 0xc3, 0xf4, 0xfe};
390 static const int32_t offsets4MBCS[] = { 0x00, 0x01, 0x01, 0x01, 0x02, 0x02 };
391
392 /*DBCS*/
393 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
394 expectedSUB, sizeof(expectedSUB), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
395 log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
396 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
397 expected, sizeof(expected), "ibm-1363", 0, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
398 log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
399
400 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
401 expectedSUB, sizeof(expectedSUB), "ibm-1363", offsetsSUB, TRUE, U_ZERO_ERROR))
402 log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
403 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
404 expected, sizeof(expected), "ibm-1363", offsets, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
405 log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
406
407
408 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
409 expected2, sizeof(expected2), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
410 log_err("u-> ibm-1363 [UCNV_DBCS portion] did not match \n");
411 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
412 expected2, sizeof(expected2), "ibm-1363", offsets2, TRUE, U_ZERO_ERROR))
413 log_err("u-> ibm-1363 [UCNV_DBCS portion] did not match \n");
414
415 /*MBCS*/
416 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
417 expectedSUB, sizeof(expectedSUB), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
418 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
419 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
420 expected, sizeof(expected), "ibm-1363", 0, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
421 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
422
423 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
424 expected2, sizeof(expected2), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
425 log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");
426 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
427 expected2, sizeof(expected2), "ibm-1363", 0, FALSE, U_ZERO_ERROR))
428 log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");
429 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
430 expected2, sizeof(expected2), "ibm-1363", offsets2, FALSE, U_ZERO_ERROR))
431 log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");
432
433 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
434 expected3MBCS, sizeof(expected3MBCS), "ibm-1363", offsets3MBCS, TRUE, U_ZERO_ERROR))
435 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
436 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
437 expected3MBCS, sizeof(expected3MBCS), "ibm-1363", offsets3MBCS, FALSE, U_ZERO_ERROR))
438 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
439
440 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
441 expected4MBCS, sizeof(expected4MBCS), "IBM-eucJP", offsets4MBCS, TRUE, U_ZERO_ERROR))
442 log_err("u-> euc-jp [UCNV_MBCS] \n");
443 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
444 expected4MBCS, sizeof(expected4MBCS), "IBM-eucJP", offsets4MBCS, FALSE, U_ZERO_ERROR))
445 log_err("u-> euc-jp [UCNV_MBCS] \n");
446 }
447
448 /*iso-2022-jp*/
449 log_verbose("Testing for iso-2022-jp\n");
450 {
451 static const UChar sampleText[] = { 0x0031, 0xd801};
452 static const uint8_t expected[] = { 0x31};
453 static const uint8_t expectedSUB[] = { 0x31, 0x1a};
454 static const int32_t offsets[] = { 0x00, 1};
455
456 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
457 static const uint8_t expected2[] = { 0x31,0x1A,0x32};
458 static const int32_t offsets2[] = { 0x00,0x01,0x02};
459
460 static const UChar sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
461 static const uint8_t expected4MBCS[] = { 0x61, 0x1b, 0x24, 0x42, 0x30, 0x6c,0x1b,0x28,0x42,0x1a};
462 static const int32_t offsets4MBCS[] = { 0x00, 0x01, 0x01 ,0x01, 0x01, 0x01,0x02,0x02,0x02,0x02 };
463 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
464 expectedSUB, sizeof(expectedSUB), "iso-2022-jp", offsets, TRUE, U_ZERO_ERROR))
465 log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
466 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
467 expected, sizeof(expected), "iso-2022-jp", offsets, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
468 log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
469
470 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
471 expected2, sizeof(expected2), "iso-2022-jp", offsets2, TRUE, U_ZERO_ERROR))
472 log_err("u->iso-2022-jp[UCNV_DBCS] did not match\n");
473 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
474 expected2, sizeof(expected2), "iso-2022-jp", offsets2, FALSE, U_ZERO_ERROR))
475 log_err("u-> iso-2022-jp [UCNV_DBCS] did not match\n");
476 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
477 expected2, sizeof(expected2), "iso-2022-jp", offsets2, FALSE, U_ZERO_ERROR))
478 log_err("u-> iso-2022-jp [UCNV_DBCS] did not match\n");
479
480 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
481 expected4MBCS, sizeof(expected4MBCS), "iso-2022-jp", offsets4MBCS, TRUE, U_ZERO_ERROR))
482 log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
483 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
484 expected4MBCS, sizeof(expected4MBCS), "iso-2022-jp", offsets4MBCS, FALSE, U_ZERO_ERROR))
485 log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
486 }
487
488 /*iso-2022-cn*/
489 log_verbose("Testing for iso-2022-cn\n");
490 {
491 static const UChar sampleText[] = { 0x0031, 0xd801};
492 static const uint8_t expected[] = { 0x31};
493 static const uint8_t expectedSUB[] = { 0x31, 0x1A};
494 static const int32_t offsets[] = { 0x00, 1};
495
496 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
497 static const uint8_t expected2[] = { 0x31, 0x1A,0x32};
498 static const int32_t offsets2[] = { 0x00, 0x01,0x02};
499
500 static const UChar sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
501 static const uint8_t expected3MBCS[] = {0x51, 0x50, 0x1A};
502 static const int32_t offsets3MBCS[] = { 0x00, 0x01, 0x02 };
503
504 static const UChar sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
505 static const uint8_t expected4MBCS[] = { 0x61, 0x1b, 0x24, 0x29, 0x41, 0x0e, 0x52, 0x3b, 0x0f, 0x1a };
506 static const int32_t offsets4MBCS[] = { 0x00, 0x01, 0x01 ,0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02 };
507 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
508 expectedSUB, sizeof(expectedSUB), "iso-2022-cn", offsets, TRUE, U_ZERO_ERROR))
509 log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
510 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
511 expected, sizeof(expected), "iso-2022-cn", offsets, FALSE, U_ZERO_ERROR))
512 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
513
514 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
515 expected2, sizeof(expected2), "iso-2022-cn", offsets2, TRUE, U_ZERO_ERROR))
516 log_err("u->iso-2022-cn[UCNV_DBCS] did not match\n");
517 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
518 expected2, sizeof(expected2), "iso-2022-cn", offsets2, FALSE, U_ZERO_ERROR))
519 log_err("u-> iso-2022-cn [UCNV_DBCS] did not match\n");
520 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
521 expected2, sizeof(expected2), "iso-2022-cn", offsets2, FALSE, U_ZERO_ERROR))
522 log_err("u-> iso-2022-cn [UCNV_DBCS] did not match\n");
523
524 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
525 expected3MBCS, sizeof(expected3MBCS), "iso-2022-cn", offsets3MBCS, TRUE, U_ZERO_ERROR))
526 log_err("u->iso-2022-cn [UCNV_MBCS] \n");
527 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
528 expected3MBCS, sizeof(expected3MBCS), "iso-2022-cn", offsets3MBCS, FALSE, U_ZERO_ERROR))
529 log_err("u-> iso-2022-cn[UCNV_MBCS] \n");
530
531 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
532 expected4MBCS, sizeof(expected4MBCS), "iso-2022-cn", offsets4MBCS, TRUE, U_ZERO_ERROR))
533 log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
534 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
535 expected4MBCS, sizeof(expected4MBCS), "iso-2022-cn", offsets4MBCS, FALSE, U_ZERO_ERROR))
536 log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
537 }
538
539 /*iso-2022-kr*/
540 log_verbose("Testing for iso-2022-kr\n");
541 {
542 static const UChar sampleText[] = { 0x0031, 0xd801};
543 static const uint8_t expected[] = { 0x1b, 0x24, 0x29, 0x43, 0x31};
544 static const uint8_t expectedSUB[] = { 0x1b, 0x24, 0x29, 0x43, 0x31, 0x1A};
545 static const int32_t offsets[] = { -1, -1, -1, -1, 0x00, 1};
546
547 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
548 static const uint8_t expected2[] = { 0x1b, 0x24, 0x29, 0x43, 0x31, 0x1A, 0x32};
549 static const int32_t offsets2[] = { -1, -1, -1, -1, 0x00, 0x01, 0x02};
550
551 static const UChar sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
552 static const uint8_t expected3MBCS[] = { 0x1b, 0x24, 0x29, 0x43, 0x51, 0x50, 0x1A };
553 static const int32_t offsets3MBCS[] = { -1, -1, -1, -1, 0x00, 0x01, 0x02, 0x02 };
554
555 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
556 expectedSUB, sizeof(expectedSUB), "iso-2022-kr", offsets, TRUE, U_ZERO_ERROR))
557 log_err("u-> iso-2022-kr [UCNV_MBCS] \n");
558 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
559 expected, sizeof(expected), "iso-2022-kr", offsets, FALSE, U_ZERO_ERROR))
560 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
561
562 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
563 expected2, sizeof(expected2), "iso-2022-kr", offsets2, TRUE, U_ZERO_ERROR))
564 log_err("u->iso-2022-kr[UCNV_DBCS] did not match\n");
565 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
566 expected2, sizeof(expected2), "iso-2022-kr", offsets2, FALSE, U_ZERO_ERROR))
567 log_err("u-> iso-2022-kr [UCNV_DBCS] did not match\n");
568 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
569 expected2, sizeof(expected2), "iso-2022-kr", offsets2, FALSE, U_ZERO_ERROR))
570 log_err("u-> iso-2022-kr [UCNV_DBCS] did not match\n");
571
572 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
573 expected3MBCS, sizeof(expected3MBCS), "iso-2022-kr", offsets3MBCS, TRUE, U_ZERO_ERROR))
574 log_err("u->iso-2022-kr [UCNV_MBCS] \n");
575 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
576 expected3MBCS, sizeof(expected3MBCS), "iso-2022-kr", offsets3MBCS, FALSE, U_ZERO_ERROR))
577 log_err("u-> iso-2022-kr[UCNV_MBCS] \n");
578 }
579
580 /*HZ*/
581 log_verbose("Testing for HZ\n");
582 {
583 static const UChar sampleText[] = { 0x0031, 0xd801};
584 static const uint8_t expected[] = { 0x7e, 0x7d, 0x31};
585 static const uint8_t expectedSUB[] = { 0x7e, 0x7d, 0x31, 0x1A};
586 static const int32_t offsets[] = { 0x00, 0x00, 0x00, 1};
587
588 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
589 static const uint8_t expected2[] = { 0x7e, 0x7d, 0x31, 0x1A, 0x32 };
590 static const int32_t offsets2[] = { 0x00, 0x00, 0x00, 0x01, 0x02 };
591
592 static const UChar sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
593 static const uint8_t expected3MBCS[] = { 0x7e, 0x7d, 0x51, 0x50, 0x1A };
594 static const int32_t offsets3MBCS[] = { 0x00, 0x00, 0x00, 0x01, 0x02};
595
596 static const UChar sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
597 static const uint8_t expected4MBCS[] = { 0x7e, 0x7d, 0x61, 0x7e, 0x7b, 0x52, 0x3b, 0x7e, 0x7d, 0x1a };
598 static const int32_t offsets4MBCS[] = { 0x00, 0x00, 0x00, 0x01, 0x01, 0x01 ,0x01, 0x02, 0x02, 0x02 };
599 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
600 expectedSUB, sizeof(expectedSUB), "HZ", offsets, TRUE, U_ZERO_ERROR))
601 log_err("u-> HZ [UCNV_MBCS] \n");
602 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
603 expected, sizeof(expected), "HZ", offsets, FALSE, U_ZERO_ERROR))
604 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
605
606 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
607 expected2, sizeof(expected2), "HZ", offsets2, TRUE, U_ZERO_ERROR))
608 log_err("u->HZ[UCNV_DBCS] did not match\n");
609 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
610 expected2, sizeof(expected2), "HZ", offsets2, FALSE, U_ZERO_ERROR))
611 log_err("u-> HZ [UCNV_DBCS] did not match\n");
612 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
613 expected2, sizeof(expected2), "HZ", offsets2, FALSE, U_ZERO_ERROR))
614 log_err("u-> HZ [UCNV_DBCS] did not match\n");
615
616 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
617 expected3MBCS, sizeof(expected3MBCS), "HZ", offsets3MBCS, TRUE, U_ZERO_ERROR))
618 log_err("u->HZ [UCNV_MBCS] \n");
619 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
620 expected3MBCS, sizeof(expected3MBCS), "HZ", offsets3MBCS, FALSE, U_ZERO_ERROR))
621 log_err("u-> HZ[UCNV_MBCS] \n");
622
623 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
624 expected4MBCS, sizeof(expected4MBCS), "HZ", offsets4MBCS, TRUE, U_ZERO_ERROR))
625 log_err("u-> HZ [UCNV_MBCS] \n");
626 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
627 expected4MBCS, sizeof(expected4MBCS), "HZ", offsets4MBCS, FALSE, U_ZERO_ERROR))
628 log_err("u-> HZ [UCNV_MBCS] \n");
629 }
630#endif
631}
632
633#if !UCONFIG_NO_LEGACY_CONVERSION
634/*test different convertToUnicode error behaviours*/
635static void TestToUnicodeErrorBehaviour()
636{
637 log_verbose("Testing error conditions for DBCS\n");
638 {
639 uint8_t sampleText[] = { 0xa2, 0xae, 0x03, 0x04};
640 const UChar expected[] = { 0x00a1 };
641
642 if(!convertToU(sampleText, sizeof(sampleText),
643 expected, UPRV_LENGTHOF(expected), "ibm-1363", 0, TRUE, U_AMBIGUOUS_ALIAS_WARNING ))
644 log_err("DBCS (ibm-1363)->Unicode did not match.\n");
645 if(!convertToU(sampleText, sizeof(sampleText),
646 expected, UPRV_LENGTHOF(expected), "ibm-1363", 0, FALSE, U_AMBIGUOUS_ALIAS_WARNING ))
647 log_err("DBCS (ibm-1363)->Unicode with flush = false did not match.\n");
648 }
649 log_verbose("Testing error conditions for SBCS\n");
650 {
651 uint8_t sampleText[] = { 0xa2, 0xFF};
652 const UChar expected[] = { 0x00c2 };
653
654 /* uint8_t sampleText2[] = { 0xa2, 0x70 };
655 const UChar expected2[] = { 0x0073 };*/
656
657 if(!convertToU(sampleText, sizeof(sampleText),
658 expected, UPRV_LENGTHOF(expected), "ibm-1051", 0, TRUE, U_ZERO_ERROR ))
659 log_err("SBCS (ibm-1051)->Unicode did not match.\n");
660 if(!convertToU(sampleText, sizeof(sampleText),
661 expected, UPRV_LENGTHOF(expected), "ibm-1051", 0, FALSE, U_ZERO_ERROR ))
662 log_err("SBCS (ibm-1051)->Unicode with flush = false did not match.\n");
663
664 }
665}
666
667static void TestGetNextErrorBehaviour(){
668 /*Test for unassigned character*/
669#define INPUT_SIZE 1
670 static const char input1[INPUT_SIZE]={ 0x70 };
671 const char* source=(const char*)input1;
672 UErrorCode err=U_ZERO_ERROR;
673 UChar32 c=0;
674 UConverter *cnv=ucnv_open("ibm-424", &err);
675 if(U_FAILURE(err)) {
676 log_data_err("Unable to open a SBCS(ibm-424) converter: %s\n", u_errorName(err));
677 return;
678 }
679 c=ucnv_getNextUChar(cnv, &source, source + INPUT_SIZE, &err);
680 if(err != U_INVALID_CHAR_FOUND && c!=0xfffd){
681 log_err("FAIL in TestGetNextErrorBehaviour(unassigned): Expected: U_INVALID_CHAR_ERROR or 0xfffd ----Got:%s and 0x%lx\n", myErrorName(err), c);
682 }
683 ucnv_close(cnv);
684}
685#endif
686
687#define MAX_UTF16_LEN 2
688#define MAX_UTF8_LEN 4
689
690/*Regression test for utf8 converter*/
691static void TestRegressionUTF8(){
692 UChar32 currCh = 0;
693 int32_t offset8;
694 int32_t offset16;
695 UChar *standardForm = (UChar*)malloc(MAX_LENGTH*sizeof(UChar));
696 uint8_t *utf8 = (uint8_t*)malloc(MAX_LENGTH);
697
698 while (currCh <= UNICODE_LIMIT) {
699 offset16 = 0;
700 offset8 = 0;
701 while(currCh <= UNICODE_LIMIT
702 && offset16 < ((int32_t)(MAX_LENGTH/sizeof(UChar) - MAX_UTF16_LEN))
703 && offset8 < (MAX_LENGTH - MAX_UTF8_LEN))
704 {
705 if (currCh == SURROGATE_HIGH_START) {
706 currCh = SURROGATE_LOW_END + 1; /* Skip surrogate range */
707 }
708 U16_APPEND_UNSAFE(standardForm, offset16, currCh);
709 U8_APPEND_UNSAFE(utf8, offset8, currCh);
710 currCh++;
711 }
712 if(!convertFromU(standardForm, offset16,
713 utf8, offset8, "UTF8", 0, TRUE, U_ZERO_ERROR )) {
714 log_err("Unicode->UTF8 did not match.\n");
715 }
716 if(!convertToU(utf8, offset8,
717 standardForm, offset16, "UTF8", 0, TRUE, U_ZERO_ERROR )) {
718 log_err("UTF8->Unicode did not match.\n");
719 }
720 }
721
722 free(standardForm);
723 free(utf8);
724
725 {
726 static const char src8[] = { (char)0xCC, (char)0x81, (char)0xCC, (char)0x80 };
727 static const UChar expected[] = { 0x0301, 0x0300 };
728 UConverter *conv8;
729 UErrorCode err = U_ZERO_ERROR;
730 UChar pivotBuffer[100];
731 const UChar* const pivEnd = pivotBuffer + 100;
732 const char* srcBeg;
733 const char* srcEnd;
734 UChar* pivBeg;
735
736 conv8 = ucnv_open("UTF-8", &err);
737
738 srcBeg = src8;
739 pivBeg = pivotBuffer;
740 srcEnd = src8 + 3;
741 ucnv_toUnicode(conv8, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, FALSE, &err);
742 if (srcBeg != srcEnd) {
743 log_err("Did not consume whole buffer on first call.\n");
744 }
745
746 srcEnd = src8 + 4;
747 ucnv_toUnicode(conv8, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, TRUE, &err);
748 if (srcBeg != srcEnd) {
749 log_err("Did not consume whole buffer on second call.\n");
750 }
751
752 if (U_FAILURE(err) || (int32_t)(pivBeg - pivotBuffer) != 2 || u_strncmp(pivotBuffer, expected, 2) != 0) {
753 log_err("Did not get expected results for UTF-8.\n");
754 }
755 ucnv_close(conv8);
756 }
757}
758
759#define MAX_UTF32_LEN 1
760
761static void TestRegressionUTF32(){
762#if !UCONFIG_ONLY_HTML_CONVERSION
763 UChar32 currCh = 0;
764 int32_t offset32;
765 int32_t offset16;
766 UChar *standardForm = (UChar*)malloc(MAX_LENGTH*sizeof(UChar));
767 UChar32 *utf32 = (UChar32*)malloc(MAX_LENGTH*sizeof(UChar32));
768
769 while (currCh <= UNICODE_LIMIT) {
770 offset16 = 0;
771 offset32 = 0;
772 while(currCh <= UNICODE_LIMIT
773 && offset16 < ((int32_t)(MAX_LENGTH/sizeof(UChar) - MAX_UTF16_LEN))
774 && offset32 < ((int32_t)(MAX_LENGTH/sizeof(UChar32) - MAX_UTF32_LEN)))
775 {
776 if (currCh == SURROGATE_HIGH_START) {
777 currCh = SURROGATE_LOW_END + 1; /* Skip surrogate range */
778 }
779 U16_APPEND_UNSAFE(standardForm, offset16, currCh);
780 utf32[offset32++] = currCh;
781 currCh++;
782 }
783 if(!convertFromU(standardForm, offset16,
784 (const uint8_t *)utf32, offset32*sizeof(UChar32), "UTF32_PlatformEndian", 0, TRUE, U_ZERO_ERROR )) {
785 log_err("Unicode->UTF32 did not match.\n");
786 }
787 if(!convertToU((const uint8_t *)utf32, offset32*sizeof(UChar32),
788 standardForm, offset16, "UTF32_PlatformEndian", 0, TRUE, U_ZERO_ERROR )) {
789 log_err("UTF32->Unicode did not match.\n");
790 }
791 }
792 free(standardForm);
793 free(utf32);
794
795 {
796 /* Check for lone surrogate error handling. */
797 static const UChar sampleBadStartSurrogate[] = { 0x0031, 0xD800, 0x0032 };
798 static const UChar sampleBadEndSurrogate[] = { 0x0031, 0xDC00, 0x0032 };
799 static const uint8_t expectedUTF32BE[] = {
800 0x00, 0x00, 0x00, 0x31,
801 0x00, 0x00, 0xff, 0xfd,
802 0x00, 0x00, 0x00, 0x32
803 };
804 static const uint8_t expectedUTF32LE[] = {
805 0x31, 0x00, 0x00, 0x00,
806 0xfd, 0xff, 0x00, 0x00,
807 0x32, 0x00, 0x00, 0x00
808 };
809 static const int32_t offsetsUTF32[] = {
810 0x00, 0x00, 0x00, 0x00,
811 0x01, 0x01, 0x01, 0x01,
812 0x02, 0x02, 0x02, 0x02
813 };
814
815 if(!convertFromU(sampleBadStartSurrogate, UPRV_LENGTHOF(sampleBadStartSurrogate),
816 expectedUTF32BE, sizeof(expectedUTF32BE), "UTF-32BE", offsetsUTF32, TRUE, U_ZERO_ERROR))
817 log_err("u->UTF-32BE\n");
818 if(!convertFromU(sampleBadEndSurrogate, UPRV_LENGTHOF(sampleBadEndSurrogate),
819 expectedUTF32BE, sizeof(expectedUTF32BE), "UTF-32BE", offsetsUTF32, TRUE, U_ZERO_ERROR))
820 log_err("u->UTF-32BE\n");
821
822 if(!convertFromU(sampleBadStartSurrogate, UPRV_LENGTHOF(sampleBadStartSurrogate),
823 expectedUTF32LE, sizeof(expectedUTF32LE), "UTF-32LE", offsetsUTF32, TRUE, U_ZERO_ERROR))
824 log_err("u->UTF-32LE\n");
825 if(!convertFromU(sampleBadEndSurrogate, UPRV_LENGTHOF(sampleBadEndSurrogate),
826 expectedUTF32LE, sizeof(expectedUTF32LE), "UTF-32LE", offsetsUTF32, TRUE, U_ZERO_ERROR))
827 log_err("u->UTF-32LE\n");
828 }
829
830 {
831 static const char srcBE[] = { 0, 0, 0, 0x31, 0, 0, 0, 0x30 };
832 static const UChar expected[] = { 0x0031, 0x0030 };
833 UConverter *convBE;
834 UErrorCode err = U_ZERO_ERROR;
835 UChar pivotBuffer[100];
836 const UChar* const pivEnd = pivotBuffer + 100;
837 const char* srcBeg;
838 const char* srcEnd;
839 UChar* pivBeg;
840
841 convBE = ucnv_open("UTF-32BE", &err);
842
843 srcBeg = srcBE;
844 pivBeg = pivotBuffer;
845 srcEnd = srcBE + 5;
846 ucnv_toUnicode(convBE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, FALSE, &err);
847 if (srcBeg != srcEnd) {
848 log_err("Did not consume whole buffer on first call.\n");
849 }
850
851 srcEnd = srcBE + 8;
852 ucnv_toUnicode(convBE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, TRUE, &err);
853 if (srcBeg != srcEnd) {
854 log_err("Did not consume whole buffer on second call.\n");
855 }
856
857 if (U_FAILURE(err) || (int32_t)(pivBeg - pivotBuffer) != 2 || u_strncmp(pivotBuffer, expected, 2) != 0) {
858 log_err("Did not get expected results for UTF-32BE.\n");
859 }
860 ucnv_close(convBE);
861 }
862 {
863 static const char srcLE[] = { 0x31, 0, 0, 0, 0x30, 0, 0, 0 };
864 static const UChar expected[] = { 0x0031, 0x0030 };
865 UConverter *convLE;
866 UErrorCode err = U_ZERO_ERROR;
867 UChar pivotBuffer[100];
868 const UChar* const pivEnd = pivotBuffer + 100;
869 const char* srcBeg;
870 const char* srcEnd;
871 UChar* pivBeg;
872
873 convLE = ucnv_open("UTF-32LE", &err);
874
875 srcBeg = srcLE;
876 pivBeg = pivotBuffer;
877 srcEnd = srcLE + 5;
878 ucnv_toUnicode(convLE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, FALSE, &err);
879 if (srcBeg != srcEnd) {
880 log_err("Did not consume whole buffer on first call.\n");
881 }
882
883 srcEnd = srcLE + 8;
884 ucnv_toUnicode(convLE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, TRUE, &err);
885 if (srcBeg != srcEnd) {
886 log_err("Did not consume whole buffer on second call.\n");
887 }
888
889 if (U_FAILURE(err) || (int32_t)(pivBeg - pivotBuffer) != 2 || u_strncmp(pivotBuffer, expected, 2) != 0) {
890 log_err("Did not get expected results for UTF-32LE.\n");
891 }
892 ucnv_close(convLE);
893 }
894#endif
895}
896
897/*Walk through the available converters*/
898static void TestAvailableConverters(){
899 UErrorCode status=U_ZERO_ERROR;
900 UConverter *conv=NULL;
901 int32_t i=0;
902 for(i=0; i < ucnv_countAvailable(); i++){
903 status=U_ZERO_ERROR;
904 conv=ucnv_open(ucnv_getAvailableName(i), &status);
905 if(U_FAILURE(status)){
906 log_err("ERROR: converter creation failed. Failure in alias table or the data table for \n converter=%s. Error=%s\n",
907 ucnv_getAvailableName(i), myErrorName(status));
908 continue;
909 }
910 ucnv_close(conv);
911 }
912
913}
914
915static void TestFlushInternalBuffer(){
916 TestWithBufferSize(MAX_LENGTH, 1);
917 TestWithBufferSize(1, 1);
918 TestWithBufferSize(1, MAX_LENGTH);
919 TestWithBufferSize(MAX_LENGTH, MAX_LENGTH);
920}
921
922static void TestWithBufferSize(int32_t insize, int32_t outsize){
923
924 gInBufferSize =insize;
925 gOutBufferSize = outsize;
926
927 log_verbose("Testing fromUnicode for UTF-8 with UCNV_TO_U_CALLBACK_SUBSTITUTE \n");
928 {
929 UChar sampleText[] =
930 { 0x0031, 0x0032, 0x0033, 0x0000, 0x4e00, 0x4e8c, 0x4e09, 0x002E };
931 const uint8_t expectedUTF8[] =
932 { 0x31, 0x32, 0x33, 0x00, 0xe4, 0xb8, 0x80, 0xe4, 0xba, 0x8c, 0xe4, 0xb8, 0x89, 0x2E };
933 int32_t toUTF8Offs[] =
934 { 0x00, 0x01, 0x02, 0x03, 0x04, 0x04, 0x04, 0x05, 0x05, 0x05, 0x06, 0x06, 0x06, 0x07};
935 /* int32_t fmUTF8Offs[] =
936 { 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0007, 0x000a, 0x000d };*/
937
938 /*UTF-8*/
939 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
940 expectedUTF8, sizeof(expectedUTF8), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE, toUTF8Offs ,FALSE))
941 log_err("u-> UTF8 did not match.\n");
942 }
943
944#if !UCONFIG_NO_LEGACY_CONVERSION
945 log_verbose("Testing fromUnicode with UCNV_FROM_U_CALLBACK_ESCAPE \n");
946 {
947 UChar inputTest[] = { 0x0061, 0xd801, 0xdc01, 0xd801, 0x0061 };
948 const uint8_t toIBM943[]= { 0x61,
949 0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
950 0x25, 0x55, 0x44, 0x43, 0x30, 0x31,
951 0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
952 0x61 };
953 int32_t offset[]= {0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 4};
954
955 if(!testConvertFromU(inputTest, UPRV_LENGTHOF(inputTest),
956 toIBM943, sizeof(toIBM943), "ibm-943",
957 (UConverterFromUCallback)UCNV_FROM_U_CALLBACK_ESCAPE, offset,FALSE))
958 log_err("u-> ibm-943 with subst with value did not match.\n");
959 }
960#endif
961
962 log_verbose("Testing fromUnicode for UTF-8 with UCNV_TO_U_CALLBACK_SUBSTITUTE \n");
963 {
964 const uint8_t sampleText1[] = { 0x31, 0xe4, 0xba, 0x8c,
965 0xe0, 0x80, 0x61};
966 UChar expected1[] = { 0x0031, 0x4e8c, 0xfffd, 0xfffd, 0x0061};
967 int32_t offsets1[] = { 0x0000, 0x0001, 0x0004, 0x0005, 0x0006};
968
969 if(!testConvertToU(sampleText1, sizeof(sampleText1),
970 expected1, UPRV_LENGTHOF(expected1),"utf8", UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets1,FALSE))
971 log_err("utf8->u with substitute did not match.\n");
972 }
973
974#if !UCONFIG_NO_LEGACY_CONVERSION
975 log_verbose("Testing toUnicode with UCNV_TO_U_CALLBACK_ESCAPE \n");
976 /*to Unicode*/
977 {
978 const uint8_t sampleTxtToU[]= { 0x00, 0x9f, 0xaf,
979 0x81, 0xad, /*unassigned*/
980 0x89, 0xd3 };
981 UChar IBM_943toUnicode[] = { 0x0000, 0x6D63,
982 0x25, 0x58, 0x38, 0x31, 0x25, 0x58, 0x41, 0x44,
983 0x7B87};
984 int32_t fromIBM943Offs [] = { 0, 1, 3, 3, 3, 3, 3, 3, 3, 3, 5};
985
986 if(!testConvertToU(sampleTxtToU, sizeof(sampleTxtToU),
987 IBM_943toUnicode, UPRV_LENGTHOF(IBM_943toUnicode),"ibm-943",
988 (UConverterToUCallback)UCNV_TO_U_CALLBACK_ESCAPE, fromIBM943Offs,FALSE))
989 log_err("ibm-943->u with substitute with value did not match.\n");
990
991 }
992#endif
993}
994
995static UBool convertFromU( const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
996 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus)
997{
998
999 int32_t i=0;
1000 char *p=0;
1001 const UChar *src;
1002 char buffer[MAX_LENGTH];
1003 int32_t offsetBuffer[MAX_LENGTH];
1004 int32_t *offs=0;
1005 char *targ;
1006 char *targetLimit;
1007 UChar *sourceLimit=0;
1008 UErrorCode status = U_ZERO_ERROR;
1009 UConverter *conv = 0;
1010 conv = ucnv_open(codepage, &status);
1011 if(U_FAILURE(status))
1012 {
1013 log_data_err("Couldn't open converter %s\n",codepage);
1014 return TRUE;
1015 }
1016 log_verbose("Converter %s opened..\n", ucnv_getName(conv, &status));
1017
1018 for(i=0; i<MAX_LENGTH; i++){
1019 buffer[i]=(char)0xF0;
1020 offsetBuffer[i]=0xFF;
1021 }
1022
1023 src=source;
1024 sourceLimit=(UChar*)src+(sourceLen);
1025 targ=buffer;
1026 targetLimit=targ+MAX_LENGTH;
1027 offs=offsetBuffer;
1028 ucnv_fromUnicode (conv,
1029 (char **)&targ,
1030 (const char *)targetLimit,
1031 &src,
1032 sourceLimit,
1033 expectOffsets ? offs : NULL,
1034 doFlush,
1035 &status);
1036 ucnv_close(conv);
1037 if(status != expectedStatus){
1038 log_err("ucnv_fromUnicode() failed for codepage=%s. Error =%s Expected=%s\n", codepage, myErrorName(status), myErrorName(expectedStatus));
1039 return FALSE;
1040 }
1041
1042 log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
1043 sourceLen, targ-buffer);
1044
1045 if(expectLen != targ-buffer)
1046 {
1047 log_err("Expected %d chars out, got %d FROM Unicode to %s\n", expectLen, targ-buffer, codepage);
1048 log_verbose("Expected %d chars out, got %d FROM Unicode to %s\n", expectLen, targ-buffer, codepage);
1049 printSeqErr((const unsigned char *)buffer, (int32_t)(targ-buffer));
1050 printSeqErr((const unsigned char*)expect, expectLen);
1051 return FALSE;
1052 }
1053
1054 if(memcmp(buffer, expect, expectLen)){
1055 log_err("String does not match. FROM Unicode to codePage%s\n", codepage);
1056 log_info("\nGot:");
1057 printSeqErr((const unsigned char *)buffer, expectLen);
1058 log_info("\nExpected:");
1059 printSeqErr((const unsigned char *)expect, expectLen);
1060 return FALSE;
1061 }
1062 else {
1063 log_verbose("Matches!\n");
1064 }
1065
1066 if (expectOffsets != 0){
1067 log_verbose("comparing %d offsets..\n", targ-buffer);
1068 if(memcmp(offsetBuffer,expectOffsets,(targ-buffer) * sizeof(int32_t) )){
1069 log_err("did not get the expected offsets. for FROM Unicode to %s\n", codepage);
1070 log_info("\nGot : ");
1071 printSeqErr((const unsigned char*)buffer, (int32_t)(targ-buffer));
1072 for(p=buffer;p<targ;p++)
1073 log_info("%d, ", offsetBuffer[p-buffer]);
1074 log_info("\nExpected: ");
1075 for(i=0; i< (targ-buffer); i++)
1076 log_info("%d,", expectOffsets[i]);
1077 }
1078 }
1079
1080 return TRUE;
1081}
1082
1083
1084static UBool convertToU( const uint8_t *source, int sourceLen, const UChar *expect, int expectLen,
1085 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus)
1086{
1087 UErrorCode status = U_ZERO_ERROR;
1088 UConverter *conv = 0;
1089 int32_t i=0;
1090 UChar *p=0;
1091 const char* src;
1092 UChar buffer[MAX_LENGTH];
1093 int32_t offsetBuffer[MAX_LENGTH];
1094 int32_t *offs=0;
1095 UChar *targ;
1096 UChar *targetLimit;
1097 uint8_t *sourceLimit=0;
1098
1099
1100
1101 conv = ucnv_open(codepage, &status);
1102 if(U_FAILURE(status))
1103 {
1104 log_data_err("Couldn't open converter %s\n",codepage);
1105 return TRUE;
1106 }
1107 log_verbose("Converter %s opened..\n", ucnv_getName(conv, &status));
1108
1109
1110
1111 for(i=0; i<MAX_LENGTH; i++){
1112 buffer[i]=0xFFFE;
1113 offsetBuffer[i]=-1;
1114 }
1115
1116 src=(const char *)source;
1117 sourceLimit=(uint8_t*)(src+(sourceLen));
1118 targ=buffer;
1119 targetLimit=targ+MAX_LENGTH;
1120 offs=offsetBuffer;
1121
1122
1123
1124 ucnv_toUnicode (conv,
1125 &targ,
1126 targetLimit,
1127 (const char **)&src,
1128 (const char *)sourceLimit,
1129 expectOffsets ? offs : NULL,
1130 doFlush,
1131 &status);
1132
1133 ucnv_close(conv);
1134 if(status != expectedStatus){
1135 log_err("ucnv_fromUnicode() failed for codepage=%s. Error =%s Expected=%s\n", codepage, myErrorName(status), myErrorName(expectedStatus));
1136 return FALSE;
1137 }
1138 log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
1139 sourceLen, targ-buffer);
1140
1141
1142
1143
1144 log_verbose("comparing %d uchars (%d bytes)..\n",expectLen,expectLen*2);
1145
1146 if (expectOffsets != 0) {
1147 if(memcmp(offsetBuffer, expectOffsets, (targ-buffer) * sizeof(int32_t))){
1148
1149 log_err("did not get the expected offsets from %s To UNICODE\n", codepage);
1150 log_info("\nGot : ");
1151 for(p=buffer;p<targ;p++)
1152 log_info("%d, ", offsetBuffer[p-buffer]);
1153 log_info("\nExpected: ");
1154 for(i=0; i<(targ-buffer); i++)
1155 log_info("%d, ", expectOffsets[i]);
1156 log_info("\nGot result:");
1157 for(i=0; i<(targ-buffer); i++)
1158 log_info("0x%04X,", buffer[i]);
1159 log_info("\nFrom Input:");
1160 for(i=0; i<(src-(const char *)source); i++)
1161 log_info("0x%02X,", (unsigned char)source[i]);
1162 log_info("\n");
1163 }
1164 }
1165 if(memcmp(buffer, expect, expectLen*2)){
1166 log_err("String does not match. from codePage %s TO Unicode\n", codepage);
1167 log_info("\nGot:");
1168 printUSeqErr(buffer, expectLen);
1169 log_info("\nExpected:");
1170 printUSeqErr(expect, expectLen);
1171 return FALSE;
1172 }
1173 else {
1174 log_verbose("Matches!\n");
1175 }
1176
1177 return TRUE;
1178}
1179
1180
1181static UBool testConvertFromU( const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
1182 const char *codepage, UConverterFromUCallback callback , const int32_t *expectOffsets, UBool testReset)
1183{
1184 UErrorCode status = U_ZERO_ERROR;
1185 UConverter *conv = 0;
1186 char junkout[MAX_LENGTH]; /* FIX */
1187 int32_t junokout[MAX_LENGTH]; /* FIX */
1188 char *p;
1189 const UChar *src;
1190 char *end;
1191 char *targ;
1192 int32_t *offs;
1193 int i;
1194 int32_t realBufferSize;
1195 char *realBufferEnd;
1196 const UChar *realSourceEnd;
1197 const UChar *sourceLimit;
1198 UBool checkOffsets = TRUE;
1199 UBool doFlush;
1200
1201 UConverterFromUCallback oldAction = NULL;
1202 const void* oldContext = NULL;
1203
1204 for(i=0;i<MAX_LENGTH;i++)
1205 junkout[i] = (char)0xF0;
1206 for(i=0;i<MAX_LENGTH;i++)
1207 junokout[i] = 0xFF;
1208
1209 setNuConvTestName(codepage, "FROM");
1210
1211 log_verbose("\n========= %s\n", gNuConvTestName);
1212
1213 conv = ucnv_open(codepage, &status);
1214 if(U_FAILURE(status))
1215 {
1216 log_data_err("Couldn't open converter %s\n",codepage);
1217 return TRUE;
1218 }
1219
1220 log_verbose("Converter opened..\n");
1221 /*----setting the callback routine----*/
1222 ucnv_setFromUCallBack (conv, callback, NULL, &oldAction, &oldContext, &status);
1223 if (U_FAILURE(status)) {
1224 log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));
1225 }
1226 /*------------------------*/
1227
1228 src = source;
1229 targ = junkout;
1230 offs = junokout;
1231
1232 realBufferSize = UPRV_LENGTHOF(junkout);
1233 realBufferEnd = junkout + realBufferSize;
1234 realSourceEnd = source + sourceLen;
1235
1236 if ( gOutBufferSize != realBufferSize )
1237 checkOffsets = FALSE;
1238
1239 if( gInBufferSize != MAX_LENGTH )
1240 checkOffsets = FALSE;
1241
1242 do
1243 {
1244 end = nct_min(targ + gOutBufferSize, realBufferEnd);
1245 sourceLimit = nct_min(src + gInBufferSize, realSourceEnd);
1246
1247 doFlush = (UBool)(sourceLimit == realSourceEnd);
1248
1249 if(targ == realBufferEnd)
1250 {
1251 log_err("Error, overflowed the real buffer while about to call fromUnicode! targ=%08lx %s", targ, gNuConvTestName);
1252 return FALSE;
1253 }
1254 log_verbose("calling fromUnicode @ SOURCE:%08lx to %08lx TARGET: %08lx to %08lx, flush=%s\n", src,sourceLimit, targ,end, doFlush?"TRUE":"FALSE");
1255
1256
1257 status = U_ZERO_ERROR;
1258 if(gInBufferSize ==999 && gOutBufferSize==999)
1259 doFlush = FALSE;
1260 ucnv_fromUnicode (conv,
1261 (char **)&targ,
1262 (const char *)end,
1263 &src,
1264 sourceLimit,
1265 offs,
1266 doFlush, /* flush if we're at the end of the input data */
1267 &status);
1268 if(testReset)
1269 ucnv_resetToUnicode(conv);
1270 if(gInBufferSize ==999 && gOutBufferSize==999)
1271 ucnv_resetToUnicode(conv);
1272
1273 } while ( (status == U_BUFFER_OVERFLOW_ERROR) || (U_SUCCESS(status) && sourceLimit < realSourceEnd) );
1274
1275 if(U_FAILURE(status)) {
1276 log_err("Problem doing fromUnicode to %s, errcode %s %s\n", codepage, myErrorName(status), gNuConvTestName);
1277 return FALSE;
1278 }
1279
1280 log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
1281 sourceLen, targ-junkout);
1282 if(getTestOption(VERBOSITY_OPTION))
1283 {
1284 char junk[999];
1285 char offset_str[999];
1286 char *ptr;
1287
1288 junk[0] = 0;
1289 offset_str[0] = 0;
1290 for(ptr = junkout;ptr<targ;ptr++)
1291 {
1292 sprintf(junk + strlen(junk), "0x%02x, ", (0xFF) & (unsigned int)*ptr);
1293 sprintf(offset_str + strlen(offset_str), "0x%02x, ", (0xFF) & (unsigned int)junokout[ptr-junkout]);
1294 }
1295
1296 log_verbose(junk);
1297 printSeq((const unsigned char *)expect, expectLen);
1298 if ( checkOffsets )
1299 {
1300 log_verbose("\nOffsets:");
1301 log_verbose(offset_str);
1302 }
1303 log_verbose("\n");
1304 }
1305 ucnv_close(conv);
1306
1307
1308 if(expectLen != targ-junkout)
1309 {
1310 log_err("Expected %d chars out, got %d %s\n", expectLen, targ-junkout, gNuConvTestName);
1311 log_verbose("Expected %d chars out, got %d %s\n", expectLen, targ-junkout, gNuConvTestName);
1312 log_info("\nGot:");
1313 printSeqErr((const unsigned char*)junkout, (int32_t)(targ-junkout));
1314 log_info("\nExpected:");
1315 printSeqErr((const unsigned char*)expect, expectLen);
1316 return FALSE;
1317 }
1318
1319 if (checkOffsets && (expectOffsets != 0) )
1320 {
1321 log_verbose("comparing %d offsets..\n", targ-junkout);
1322 if(memcmp(junokout,expectOffsets,(targ-junkout) * sizeof(int32_t) )){
1323 log_err("did not get the expected offsets. %s", gNuConvTestName);
1324 log_err("Got : ");
1325 printSeqErr((const unsigned char*)junkout, (int32_t)(targ-junkout));
1326 for(p=junkout;p<targ;p++)
1327 log_err("%d, ", junokout[p-junkout]);
1328 log_err("\nExpected: ");
1329 for(i=0; i<(targ-junkout); i++)
1330 log_err("%d,", expectOffsets[i]);
1331 }
1332 }
1333
1334 log_verbose("comparing..\n");
1335 if(!memcmp(junkout, expect, expectLen))
1336 {
1337 log_verbose("Matches!\n");
1338 return TRUE;
1339 }
1340 else
1341 {
1342 log_err("String does not match. %s\n", gNuConvTestName);
1343 printUSeqErr(source, sourceLen);
1344 log_info("\nGot:");
1345 printSeqErr((const unsigned char *)junkout, expectLen);
1346 log_info("\nExpected:");
1347 printSeqErr((const unsigned char *)expect, expectLen);
1348
1349 return FALSE;
1350 }
1351}
1352
1353static UBool testConvertToU( const uint8_t *source, int sourcelen, const UChar *expect, int expectlen,
1354 const char *codepage, UConverterToUCallback callback, const int32_t *expectOffsets, UBool testReset)
1355{
1356 UErrorCode status = U_ZERO_ERROR;
1357 UConverter *conv = 0;
1358 UChar junkout[MAX_LENGTH]; /* FIX */
1359 int32_t junokout[MAX_LENGTH]; /* FIX */
1360 const char *src;
1361 const char *realSourceEnd;
1362 const char *srcLimit;
1363 UChar *p;
1364 UChar *targ;
1365 UChar *end;
1366 int32_t *offs;
1367 int i;
1368 UBool checkOffsets = TRUE;
1369 int32_t realBufferSize;
1370 UChar *realBufferEnd;
1371 UBool doFlush;
1372
1373 UConverterToUCallback oldAction = NULL;
1374 const void* oldContext = NULL;
1375
1376
1377 for(i=0;i<MAX_LENGTH;i++)
1378 junkout[i] = 0xFFFE;
1379
1380 for(i=0;i<MAX_LENGTH;i++)
1381 junokout[i] = -1;
1382
1383 setNuConvTestName(codepage, "TO");
1384
1385 log_verbose("\n========= %s\n", gNuConvTestName);
1386
1387 conv = ucnv_open(codepage, &status);
1388 if(U_FAILURE(status))
1389 {
1390 log_data_err("Couldn't open converter %s\n",gNuConvTestName);
1391 return TRUE;
1392 }
1393
1394 log_verbose("Converter opened..\n");
1395 /*----setting the callback routine----*/
1396 ucnv_setToUCallBack (conv, callback, NULL, &oldAction, &oldContext, &status);
1397 if (U_FAILURE(status)) {
1398 log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));
1399 }
1400 /*-------------------------------------*/
1401
1402 src = (const char *)source;
1403 targ = junkout;
1404 offs = junokout;
1405
1406 realBufferSize = UPRV_LENGTHOF(junkout);
1407 realBufferEnd = junkout + realBufferSize;
1408 realSourceEnd = src + sourcelen;
1409
1410 if ( gOutBufferSize != realBufferSize )
1411 checkOffsets = FALSE;
1412
1413 if( gInBufferSize != MAX_LENGTH )
1414 checkOffsets = FALSE;
1415
1416 do
1417 {
1418 end = nct_min( targ + gOutBufferSize, realBufferEnd);
1419 srcLimit = nct_min(realSourceEnd, src + gInBufferSize);
1420
1421 if(targ == realBufferEnd)
1422 {
1423 log_err("Error, the end would overflow the real output buffer while about to call toUnicode! tarjey=%08lx %s",targ,gNuConvTestName);
1424 return FALSE;
1425 }
1426 log_verbose("calling toUnicode @ %08lx to %08lx\n", targ,end);
1427
1428 /* oldTarg = targ; */
1429
1430 status = U_ZERO_ERROR;
1431 doFlush=(UBool)((gInBufferSize ==999 && gOutBufferSize==999)?(srcLimit == realSourceEnd) : FALSE);
1432
1433 ucnv_toUnicode (conv,
1434 &targ,
1435 end,
1436 (const char **)&src,
1437 (const char *)srcLimit,
1438 offs,
1439 doFlush, /* flush if we're at the end of hte source data */
1440 &status);
1441 if(testReset)
1442 ucnv_resetFromUnicode(conv);
1443 if(gInBufferSize ==999 && gOutBufferSize==999)
1444 ucnv_resetToUnicode(conv);
1445 /* offs += (targ-oldTarg); */
1446
1447 } while ( (status == U_BUFFER_OVERFLOW_ERROR) || (U_SUCCESS(status) && (srcLimit < realSourceEnd)) ); /* while we just need another buffer */
1448
1449 if(U_FAILURE(status))
1450 {
1451 log_err("Problem doing %s toUnicode, errcode %s %s\n", codepage, myErrorName(status), gNuConvTestName);
1452 return FALSE;
1453 }
1454
1455 log_verbose("\nConversion done. %d bytes -> %d chars.\nResult :",
1456 sourcelen, targ-junkout);
1457 if(getTestOption(VERBOSITY_OPTION))
1458 {
1459 char junk[999];
1460 char offset_str[999];
1461
1462 UChar *ptr;
1463
1464 junk[0] = 0;
1465 offset_str[0] = 0;
1466
1467 for(ptr = junkout;ptr<targ;ptr++)
1468 {
1469 sprintf(junk + strlen(junk), "0x%04x, ", (0xFFFF) & (unsigned int)*ptr);
1470 sprintf(offset_str + strlen(offset_str), "0x%04x, ", (0xFFFF) & (unsigned int)junokout[ptr-junkout]);
1471 }
1472
1473 log_verbose(junk);
1474
1475 if ( checkOffsets )
1476 {
1477 log_verbose("\nOffsets:");
1478 log_verbose(offset_str);
1479 }
1480 log_verbose("\n");
1481 }
1482 ucnv_close(conv);
1483
1484 log_verbose("comparing %d uchars (%d bytes)..\n",expectlen,expectlen*2);
1485
1486 if (checkOffsets && (expectOffsets != 0))
1487 {
1488 if(memcmp(junokout,expectOffsets,(targ-junkout) * sizeof(int32_t))){
1489
1490 log_err("did not get the expected offsets. %s",gNuConvTestName);
1491 for(p=junkout;p<targ;p++)
1492 log_err("%d, ", junokout[p-junkout]);
1493 log_err("\nExpected: ");
1494 for(i=0; i<(targ-junkout); i++)
1495 log_err("%d,", expectOffsets[i]);
1496 log_err("");
1497 for(i=0; i<(targ-junkout); i++)
1498 log_err("%X,", junkout[i]);
1499 log_err("");
1500 for(i=0; i<(src-(const char *)source); i++)
1501 log_err("%X,", (unsigned char)source[i]);
1502 }
1503 }
1504
1505 if(!memcmp(junkout, expect, expectlen*2))
1506 {
1507 log_verbose("Matches!\n");
1508 return TRUE;
1509 }
1510 else
1511 {
1512 log_err("String does not match. %s\n", gNuConvTestName);
1513 log_verbose("String does not match. %s\n", gNuConvTestName);
1514 log_info("\nGot:");
1515 printUSeq(junkout, expectlen);
1516 log_info("\nExpected:");
1517 printUSeq(expect, expectlen);
1518 return FALSE;
1519 }
1520}
1521
1522
1523static void TestResetBehaviour(void){
1524#if !UCONFIG_NO_LEGACY_CONVERSION
1525 log_verbose("Testing Reset for DBCS and MBCS\n");
1526 {
1527 static const UChar sampleText[] = {0x00a1, 0xd801, 0xdc01, 0x00a4};
1528 static const uint8_t expected[] = {0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
1529 static const int32_t offsets[] = {0x00, 0x00, 0x01, 0x01, 0x03, 0x03 };
1530
1531
1532 static const UChar sampleText1[] = {0x00a1, 0x00a4, 0x00a7, 0x00a8};
1533 static const uint8_t expected1[] = {0xa2, 0xae,0xA2,0xB4,0xA1,0xD7,0xA1,0xA7};
1534 static const int32_t offsets1[] = { 0,2,4,6};
1535
1536 /*DBCS*/
1537 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1538 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1539 log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
1540 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1541 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1542 log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
1543
1544 if(!testConvertToU(expected1, sizeof(expected1),
1545 sampleText1, UPRV_LENGTHOF(sampleText1), "ibm-1363",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1546 offsets1, TRUE))
1547 log_err("ibm-1363 -> did not match.\n");
1548 /*MBCS*/
1549 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1550 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1551 log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
1552 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1553 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1554 log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
1555
1556 if(!testConvertToU(expected1, sizeof(expected1),
1557 sampleText1, UPRV_LENGTHOF(sampleText1), "ibm-1363",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1558 offsets1, TRUE))
1559 log_err("ibm-1363 -> did not match.\n");
1560
1561 }
1562
1563 log_verbose("Testing Reset for ISO-2022-jp\n");
1564 {
1565 static const UChar sampleText[] = { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1566
1567 static const uint8_t expected[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
1568 0x31,0x1A, 0x32};
1569
1570
1571 static const int32_t offsets[] = {0,0,0,0,0,1,1,2,2,2,2,3,5 };
1572
1573
1574 static const UChar sampleText1[] = {0x4e00, 0x04e01, 0x0031,0x001A, 0x0032};
1575 static const uint8_t expected1[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
1576 0x31,0x1A, 0x32};
1577 static const int32_t offsets1[] = { 3,5,10,11,12};
1578
1579 /*iso-2022-jp*/
1580 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1581 expected, sizeof(expected), "iso-2022-jp", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1582 log_err("u-> not match.\n");
1583 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1584 expected, sizeof(expected), "iso-2022-jp", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1585 log_err("u-> not match.\n");
1586
1587 if(!testConvertToU(expected1, sizeof(expected1),
1588 sampleText1, UPRV_LENGTHOF(sampleText1), "iso-2022-jp",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1589 offsets1, TRUE))
1590 log_err("iso-2022-jp -> did not match.\n");
1591
1592 }
1593
1594 log_verbose("Testing Reset for ISO-2022-cn\n");
1595 {
1596 static const UChar sampleText[] = { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1597
1598 static const uint8_t expected[] = {
1599 0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B,
1600 0x36, 0x21,
1601 0x0f, 0x31,
1602 0x1A,
1603 0x32
1604 };
1605
1606
1607 static const int32_t offsets[] = {
1608 0, 0, 0, 0, 0, 0, 0,
1609 1, 1,
1610 2, 2,
1611 3,
1612 5, };
1613
1614 UChar sampleText1[] = {0x4e00, 0x04e01, 0x0031,0x001A, 0x0032};
1615 static const uint8_t expected1[] = {
1616 0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B,
1617 0x36, 0x21,
1618 0x1B, 0x24, 0x29, 0x47, 0x24, 0x22,
1619 0x0f, 0x1A,
1620 0x32
1621 };
1622 static const int32_t offsets1[] = { 5,7,13,16,17};
1623
1624 /*iso-2022-CN*/
1625 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1626 expected, sizeof(expected), "iso-2022-cn", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1627 log_err("u-> not match.\n");
1628 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1629 expected, sizeof(expected), "iso-2022-cn", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1630 log_err("u-> not match.\n");
1631
1632 if(!testConvertToU(expected1, sizeof(expected1),
1633 sampleText1, UPRV_LENGTHOF(sampleText1), "iso-2022-cn",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1634 offsets1, TRUE))
1635 log_err("iso-2022-cn -> did not match.\n");
1636 }
1637
1638 log_verbose("Testing Reset for ISO-2022-kr\n");
1639 {
1640 UChar sampleText[] = { 0x4e00,0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1641
1642 static const uint8_t expected[] = {0x1B, 0x24, 0x29, 0x43,
1643 0x0E, 0x6C, 0x69,
1644 0x0f, 0x1A,
1645 0x0e, 0x6F, 0x4B,
1646 0x0F, 0x31,
1647 0x1A,
1648 0x32 };
1649
1650 static const int32_t offsets[] = {-1, -1, -1, -1,
1651 0, 0, 0,
1652 1, 1,
1653 3, 3, 3,
1654 4, 4,
1655 5,
1656 7,
1657 };
1658 static const UChar sampleText1[] = { 0x4e00,0x0041, 0x04e01, 0x0031, 0x0042, 0x0032};
1659
1660 static const uint8_t expected1[] = {0x1B, 0x24, 0x29, 0x43,
1661 0x0E, 0x6C, 0x69,
1662 0x0f, 0x41,
1663 0x0e, 0x6F, 0x4B,
1664 0x0F, 0x31,
1665 0x42,
1666 0x32 };
1667
1668 static const int32_t offsets1[] = {
1669 5, 8, 10,
1670 13, 14, 15
1671
1672 };
1673 /*iso-2022-kr*/
1674 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1675 expected, sizeof(expected), "iso-2022-kr", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1676 log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
1677 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1678 expected, sizeof(expected), "iso-2022-kr", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1679 log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
1680 if(!testConvertToU(expected1, sizeof(expected1),
1681 sampleText1, UPRV_LENGTHOF(sampleText1), "iso-2022-kr",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1682 offsets1, TRUE))
1683 log_err("iso-2022-kr -> did not match.\n");
1684 }
1685
1686 log_verbose("Testing Reset for HZ\n");
1687 {
1688 static const UChar sampleText[] = { 0x4e00, 0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1689
1690 static const uint8_t expected[] = {0x7E, 0x7B, 0x52, 0x3B,
1691 0x7E, 0x7D, 0x1A,
1692 0x7E, 0x7B, 0x36, 0x21,
1693 0x7E, 0x7D, 0x31,
1694 0x1A,
1695 0x32 };
1696
1697
1698 static const int32_t offsets[] = {0,0,0,0,
1699 1,1,1,
1700 3,3,3,3,
1701 4,4,4,
1702 5,
1703 7,};
1704 static const UChar sampleText1[] = { 0x4e00, 0x0035, 0x04e01, 0x0031, 0x0041, 0x0032};
1705
1706 static const uint8_t expected1[] = {0x7E, 0x7B, 0x52, 0x3B,
1707 0x7E, 0x7D, 0x35,
1708 0x7E, 0x7B, 0x36, 0x21,
1709 0x7E, 0x7D, 0x31,
1710 0x41,
1711 0x32 };
1712
1713
1714 static const int32_t offsets1[] = {2,6,9,13,14,15
1715 };
1716
1717 /*hz*/
1718 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1719 expected, sizeof(expected), "HZ", UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
1720 log_err("u-> not match.\n");
1721 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1722 expected, sizeof(expected), "HZ", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1723 log_err("u-> not match.\n");
1724 if(!testConvertToU(expected1, sizeof(expected1),
1725 sampleText1, UPRV_LENGTHOF(sampleText1), "hz",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1726 offsets1, TRUE))
1727 log_err("hz -> did not match.\n");
1728 }
1729#endif
1730
1731 /*UTF-8*/
1732 log_verbose("Testing for UTF8\n");
1733 {
1734 static const UChar sampleText[] = { 0x4e00, 0x0701, 0x0031, 0xbfc1, 0xd801, 0xdc01, 0x0032};
1735 int32_t offsets[]={0x00, 0x00, 0x00, 0x01, 0x01, 0x02,
1736 0x03, 0x03, 0x03, 0x04, 0x04, 0x04,
1737 0x04, 0x06 };
1738 static const uint8_t expected[] = {0xe4, 0xb8, 0x80, 0xdc, 0x81, 0x31,
1739 0xeb, 0xbf, 0x81, 0xF0, 0x90, 0x90, 0x81, 0x32};
1740
1741
1742 static const int32_t fromOffsets[] = { 0x0000, 0x0003, 0x0005, 0x0006, 0x0009, 0x0009, 0x000D };
1743 /*UTF-8*/
1744 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1745 expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1746 log_err("u-> UTF8 with offsets and flush true did not match.\n");
1747 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1748 expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
1749 log_err("u-> UTF8 with offsets and flush true did not match.\n");
1750 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1751 expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1752 log_err("u-> UTF8 with offsets and flush true did not match.\n");
1753 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1754 expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
1755 log_err("u-> UTF8 with offsets and flush true did not match.\n");
1756 if(!testConvertToU(expected, sizeof(expected),
1757 sampleText, UPRV_LENGTHOF(sampleText), "UTF8",UCNV_TO_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1758 log_err("UTF8 -> did not match.\n");
1759 if(!testConvertToU(expected, sizeof(expected),
1760 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", UCNV_TO_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1761 log_err("UTF8 -> did not match.\n");
1762 if(!testConvertToU(expected, sizeof(expected),
1763 sampleText, UPRV_LENGTHOF(sampleText), "UTF8",UCNV_TO_U_CALLBACK_SUBSTITUTE , fromOffsets, TRUE))
1764 log_err("UTF8 -> did not match.\n");
1765 if(!testConvertToU(expected, sizeof(expected),
1766 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", UCNV_TO_U_CALLBACK_SUBSTITUTE , fromOffsets, TRUE))
1767 log_err("UTF8 -> did not match.\n");
1768
1769 }
1770
1771}
1772
1773/* Test that U_TRUNCATED_CHAR_FOUND is set. */
1774static void
1775doTestTruncated(const char *cnvName, const uint8_t *bytes, int32_t length) {
1776 UConverter *cnv;
1777
1778 UChar buffer[2];
1779 UChar *target, *targetLimit;
1780 const char *source, *sourceLimit;
1781
1782 UErrorCode errorCode;
1783
1784 errorCode=U_ZERO_ERROR;
1785 cnv=ucnv_open(cnvName, &errorCode);
1786 if(U_FAILURE(errorCode)) {
1787 log_data_err("error TestTruncated: unable to open \"%s\" - %s\n", cnvName, u_errorName(errorCode));
1788 return;
1789 }
1790 ucnv_setToUCallBack(cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
1791 if(U_FAILURE(errorCode)) {
1792 log_data_err("error TestTruncated: unable to set the stop callback on \"%s\" - %s\n",
1793 cnvName, u_errorName(errorCode));
1794 ucnv_close(cnv);
1795 return;
1796 }
1797
1798 source=(const char *)bytes;
1799 sourceLimit=source+length;
1800 target=buffer;
1801 targetLimit=buffer+UPRV_LENGTHOF(buffer);
1802
1803 /* 1. input bytes with flush=FALSE, then input nothing with flush=TRUE */
1804 ucnv_toUnicode(cnv, &target, targetLimit, &source, sourceLimit, NULL, FALSE, &errorCode);
1805 if(U_FAILURE(errorCode) || source!=sourceLimit || target!=buffer) {
1806 log_err("error TestTruncated(%s, 1a): input bytes[%d], flush=FALSE: %s, input left %d, output %d\n",
1807 cnvName, length, u_errorName(errorCode), (int)(sourceLimit-source), (int)(target-buffer));
1808 }
1809
1810 errorCode=U_ZERO_ERROR;
1811 source=sourceLimit;
1812 target=buffer;
1813 ucnv_toUnicode(cnv, &target, targetLimit, &source, sourceLimit, NULL, TRUE, &errorCode);
1814 if(errorCode!=U_TRUNCATED_CHAR_FOUND || target!=buffer) {
1815 log_err("error TestTruncated(%s, 1b): no input (previously %d), flush=TRUE: %s (should be U_TRUNCATED_CHAR_FOUND), output %d\n",
1816 cnvName, (int)length, u_errorName(errorCode), (int)(target-buffer));
1817 }
1818
1819 /* 2. input bytes with flush=TRUE */
1820 ucnv_resetToUnicode(cnv);
1821
1822 errorCode=U_ZERO_ERROR;
1823 source=(const char *)bytes;
1824 target=buffer;
1825 ucnv_toUnicode(cnv, &target, targetLimit, &source, sourceLimit, NULL, TRUE, &errorCode);
1826 if(errorCode!=U_TRUNCATED_CHAR_FOUND || source!=sourceLimit || target!=buffer) {
1827 log_err("error TestTruncated(%s, 2): input bytes[%d], flush=TRUE: %s (should be U_TRUNCATED_CHAR_FOUND), input left %d, output %d\n",
1828 cnvName, length, u_errorName(errorCode), (int)(sourceLimit-source), (int)(target-buffer));
1829 }
1830
1831
1832 ucnv_close(cnv);
1833}
1834
1835static void
1836TestTruncated() {
1837 static const struct {
1838 const char *cnvName;
1839 uint8_t bytes[8]; /* partial input bytes resulting in no output */
1840 int32_t length;
1841 } testCases[]={
1842 { "IMAP-mailbox-name", { 0x26 }, 1 }, /* & */
1843 { "IMAP-mailbox-name", { 0x26, 0x42 }, 2 }, /* &B */
1844 { "IMAP-mailbox-name", { 0x26, 0x42, 0x42 }, 3 }, /* &BB */
1845 { "IMAP-mailbox-name", { 0x26, 0x41, 0x41 }, 3 }, /* &AA */
1846
1847 { "UTF-7", { 0x2b, 0x42 }, 2 }, /* +B */
1848 { "UTF-8", { 0xd1 }, 1 },
1849
1850 { "UTF-16BE", { 0x4e }, 1 },
1851 { "UTF-16LE", { 0x4e }, 1 },
1852 { "UTF-16", { 0x4e }, 1 },
1853 { "UTF-16", { 0xff }, 1 },
1854 { "UTF-16", { 0xfe, 0xff, 0x4e }, 3 },
1855
1856 { "UTF-32BE", { 0, 0, 0x4e }, 3 },
1857 { "UTF-32LE", { 0x4e }, 1 },
1858 { "UTF-32", { 0, 0, 0x4e }, 3 },
1859 { "UTF-32", { 0xff }, 1 },
1860 { "UTF-32", { 0, 0, 0xfe, 0xff, 0 }, 5 },
1861 { "SCSU", { 0x0e, 0x4e }, 2 }, /* SQU 0x4e */
1862
1863#if !UCONFIG_NO_LEGACY_CONVERSION
1864 { "BOCU-1", { 0xd5 }, 1 },
1865
1866 { "Shift-JIS", { 0xe0 }, 1 },
1867
1868 { "ibm-939", { 0x0e, 0x41 }, 2 } /* SO 0x41 */
1869#else
1870 { "BOCU-1", { 0xd5 }, 1 ,}
1871#endif
1872 };
1873 int32_t i;
1874
1875 for(i=0; i<UPRV_LENGTHOF(testCases); ++i) {
1876 doTestTruncated(testCases[i].cnvName, testCases[i].bytes, testCases[i].length);
1877 }
1878}
1879
1880typedef struct NameRange {
1881 const char *name;
1882 UChar32 start, end, start2, end2, notStart, notEnd;
1883} NameRange;
1884
1885static void
1886TestUnicodeSet() {
1887 UErrorCode errorCode;
1888 UConverter *cnv;
1889 USet *set;
1890 const char *name;
1891 int32_t i, count;
1892
1893 static const char *const completeSetNames[]={
1894 "UTF-7",
1895 "UTF-8",
1896 "UTF-16",
1897 "UTF-16BE",
1898 "UTF-16LE",
1899 "UTF-32",
1900 "UTF-32BE",
1901 "UTF-32LE",
1902 "SCSU",
1903 "BOCU-1",
1904 "CESU-8",
1905#if !UCONFIG_NO_LEGACY_CONVERSION
1906 "gb18030",
1907#endif
1908 "IMAP-mailbox-name"
1909 };
1910#if !UCONFIG_NO_LEGACY_CONVERSION
1911 static const char *const lmbcsNames[]={
1912 "LMBCS-1",
1913 "LMBCS-2",
1914 "LMBCS-3",
1915 "LMBCS-4",
1916 "LMBCS-5",
1917 "LMBCS-6",
1918 "LMBCS-8",
1919 "LMBCS-11",
1920 "LMBCS-16",
1921 "LMBCS-17",
1922 "LMBCS-18",
1923 "LMBCS-19"
1924 };
1925#endif
1926
1927 static const NameRange nameRanges[]={
1928 { "US-ASCII", 0, 0x7f, -1, -1, 0x80, 0x10ffff },
1929#if !UCONFIG_NO_LEGACY_CONVERSION
1930 { "ibm-367", 0, 0x7f, -1, -1, 0x80, 0x10ffff },
1931#endif
1932 { "ISO-8859-1", 0, 0x7f, -1, -1, 0x100, 0x10ffff },
1933#if !UCONFIG_NO_LEGACY_CONVERSION
1934 { "UTF-8", 0, 0xd7ff, 0xe000, 0x10ffff, 0xd800, 0xdfff },
1935 { "windows-1251", 0, 0x7f, 0x410, 0x44f, 0x3000, 0xd7ff },
1936 /* HZ test case fixed and moved to intltest's conversion.txt, ticket #6002 */
1937 { "shift-jis", 0x3041, 0x3093, 0x30a1, 0x30f3, 0x900, 0x1cff }
1938#else
1939 { "UTF-8", 0, 0xd7ff, 0xe000, 0x10ffff, 0xd800, 0xdfff }
1940#endif
1941 };
1942
1943 /* open an empty set */
1944 set=uset_open(1, 0);
1945
1946 count=ucnv_countAvailable();
1947 for(i=0; i<count; ++i) {
1948 errorCode=U_ZERO_ERROR;
1949 name=ucnv_getAvailableName(i);
1950 cnv=ucnv_open(name, &errorCode);
1951 if(U_FAILURE(errorCode)) {
1952 log_data_err("error: unable to open converter %s - %s\n",
1953 name, u_errorName(errorCode));
1954 continue;
1955 }
1956
1957 uset_clear(set);
1958 ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
1959 if(U_FAILURE(errorCode)) {
1960 log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
1961 name, u_errorName(errorCode));
1962 } else if(uset_size(set)==0) {
1963 log_err("error: ucnv_getUnicodeSet(%s) returns an empty set\n", name);
1964 }
1965
1966 ucnv_close(cnv);
1967 }
1968
1969 /* test converters that are known to convert all of Unicode (except maybe for surrogates) */
1970 for(i=0; i<UPRV_LENGTHOF(completeSetNames); ++i) {
1971 errorCode=U_ZERO_ERROR;
1972 name=completeSetNames[i];
1973 cnv=ucnv_open(name, &errorCode);
1974 if(U_FAILURE(errorCode)) {
1975 log_data_err("error: unable to open converter %s - %s\n",
1976 name, u_errorName(errorCode));
1977 continue;
1978 }
1979
1980 uset_clear(set);
1981 ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
1982 if(U_FAILURE(errorCode)) {
1983 log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
1984 name, u_errorName(errorCode));
1985 } else if(!uset_containsRange(set, 0, 0xd7ff) || !uset_containsRange(set, 0xe000, 0x10ffff)) {
1986 log_err("error: ucnv_getUnicodeSet(%s) does not return an all-Unicode set\n", name);
1987 }
1988
1989 ucnv_close(cnv);
1990 }
1991
1992#if !UCONFIG_NO_LEGACY_CONVERSION
1993 /* test LMBCS variants which convert all of Unicode except for U+F6xx */
1994 for(i=0; i<UPRV_LENGTHOF(lmbcsNames); ++i) {
1995 errorCode=U_ZERO_ERROR;
1996 name=lmbcsNames[i];
1997 cnv=ucnv_open(name, &errorCode);
1998 if(U_FAILURE(errorCode)) {
1999 log_data_err("error: unable to open converter %s - %s\n",
2000 name, u_errorName(errorCode));
2001 continue;
2002 }
2003
2004 uset_clear(set);
2005 ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
2006 if(U_FAILURE(errorCode)) {
2007 log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
2008 name, u_errorName(errorCode));
2009 } else if(!uset_containsRange(set, 0, 0xf5ff) || !uset_containsRange(set, 0xf700, 0x10ffff)) {
2010 log_err("error: ucnv_getUnicodeSet(%s) does not return an all-Unicode set (minus U+F6xx)\n", name);
2011 }
2012
2013 ucnv_close(cnv);
2014 }
2015#endif
2016
2017 /* test specific sets */
2018 for(i=0; i<UPRV_LENGTHOF(nameRanges); ++i) {
2019 errorCode=U_ZERO_ERROR;
2020 name=nameRanges[i].name;
2021 cnv=ucnv_open(name, &errorCode);
2022 if(U_FAILURE(errorCode)) {
2023 log_data_err("error: unable to open converter %s - %s\n",
2024 name, u_errorName(errorCode));
2025 continue;
2026 }
2027
2028 uset_clear(set);
2029 ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
2030 if(U_FAILURE(errorCode)) {
2031 log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
2032 name, u_errorName(errorCode));
2033 } else if(
2034 !uset_containsRange(set, nameRanges[i].start, nameRanges[i].end) ||
2035 (nameRanges[i].start2>=0 && !uset_containsRange(set, nameRanges[i].start2, nameRanges[i].end2))
2036 ) {
2037 log_err("error: ucnv_getUnicodeSet(%s) does not contain the expected ranges\n", name);
2038 } else if(nameRanges[i].notStart>=0) {
2039 /* simulate containsAny() with the C API */
2040 uset_complement(set);
2041 if(!uset_containsRange(set, nameRanges[i].notStart, nameRanges[i].notEnd)) {
2042 log_err("error: ucnv_getUnicodeSet(%s) contains part of the unexpected range\n", name);
2043 }
2044 }
2045
2046 ucnv_close(cnv);
2047 }
2048
2049 errorCode = U_ZERO_ERROR;
2050 ucnv_getUnicodeSet(NULL, set, UCNV_ROUNDTRIP_SET, &errorCode);
2051 if (errorCode != U_ILLEGAL_ARGUMENT_ERROR) {
2052 log_err("error: ucnv_getUnicodeSet(NULL) returned wrong status code %s\n", u_errorName(errorCode));
2053 }
2054 errorCode = U_PARSE_ERROR;
2055 /* Make sure that it does nothing if an error is passed in. Difficult to proper test for. */
2056 ucnv_getUnicodeSet(NULL, NULL, UCNV_ROUNDTRIP_SET, &errorCode);
2057 if (errorCode != U_PARSE_ERROR) {
2058 log_err("error: ucnv_getUnicodeSet(NULL) returned wrong status code %s\n", u_errorName(errorCode));
2059 }
2060
2061 uset_close(set);
2062}
mirror of ICU