]> git.saurik.com Git - apple/icu.git/blame - icuSources/test/cintltst/ncnvtst.c
projects / apple / icu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ICU-59173.0.1.tar.gz
[apple/icu.git] / icuSources / test / cintltst / ncnvtst.c
CommitLineData
f3c0d7a5
A		 1// © 2016 and later: Unicode, Inc. and others.
2// License & terms of use: http://www.unicode.org/copyright.html
b75a7d8f
A		 3/********************************************************************
4 * COPYRIGHT:
2ca993e8 5 * Copyright (c) 1997-2016, International Business Machines Corporation and
b75a7d8f
A		 6 * others. All Rights Reserved.
7 ********************************************************************/
46f4442e 8/*****************************************************************************
b75a7d8f 9*
b331163b 10* File ncnvtst.c
b75a7d8f
A		 11*
12* Modification History:
13* Name Description
14* Madhu Katragadda 7/7/2000 Converter Tests for extended code coverage
46f4442e 15******************************************************************************
b75a7d8f
A		 16*/
17#include <stdio.h>
374ca955
A		 18#include <stdlib.h>
19#include <string.h>
b75a7d8f
A		 20#include "unicode/uloc.h"
21#include "unicode/ucnv.h"
22#include "unicode/utypes.h"
23#include "unicode/ustring.h"
24#include "unicode/uset.h"
25#include "cintltst.h"
b331163b 26#include "cmemory.h"
b75a7d8f
A		 27
28#define MAX_LENGTH 999
29
30#define UNICODE_LIMIT 0x10FFFF
31#define SURROGATE_HIGH_START 0xD800
32#define SURROGATE_LOW_END 0xDFFF
33
34static int32_t gInBufferSize = 0;
35static int32_t gOutBufferSize = 0;
36static char gNuConvTestName[1024];
37
38#define nct_min(x,y) ((x<y) ? x : y)
b75a7d8f
A		 39
40static void printSeq(const unsigned char* a, int len);
41static void printSeqErr(const unsigned char* a, int len);
42static void printUSeq(const UChar* a, int len);
43static void printUSeqErr(const UChar* a, int len);
44static UBool convertFromU( const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
45 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus);
46static UBool convertToU( const uint8_t *source, int sourceLen, const UChar *expect, int expectLen,
47 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus);
48
49static UBool testConvertFromU( const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
50 const char *codepage, UConverterFromUCallback callback, const int32_t *expectOffsets, UBool testReset);
51static UBool testConvertToU( const uint8_t *source, int sourcelen, const UChar *expect, int expectlen,
52 const char *codepage, UConverterToUCallback callback, const int32_t *expectOffsets, UBool testReset);
53
54static void setNuConvTestName(const char *codepage, const char *direction)
55{
374ca955
A		 56 sprintf(gNuConvTestName, "[Testing %s %s Unicode, InputBufSiz=%d, OutputBufSiz=%d]",
57 codepage,
58 direction,
59 (int)gInBufferSize,
60 (int)gOutBufferSize);
b75a7d8f
A		 61}
62
63
64static void TestSurrogateBehaviour(void);
65static void TestErrorBehaviour(void);
73c04bcf
A		 66
67#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 68static void TestToUnicodeErrorBehaviour(void);
69static void TestGetNextErrorBehaviour(void);
73c04bcf
A		 70#endif
71
b75a7d8f
A		 72static void TestRegressionUTF8(void);
73static void TestRegressionUTF32(void);
74static void TestAvailableConverters(void);
75static void TestFlushInternalBuffer(void); /*for improved code coverage in ucnv_cnv.c*/
76static void TestResetBehaviour(void);
77static void TestTruncated(void);
78static void TestUnicodeSet(void);
79
80static void TestWithBufferSize(int32_t osize, int32_t isize);
81
82
83static void printSeq(const unsigned char* a, int len)
84{
85 int i=0;
86 log_verbose("\n{");
87 while (i<len)
88 log_verbose("0x%02X ", a[i++]);
89 log_verbose("}\n");
90}
91
92static void printUSeq(const UChar* a, int len)
93{
94 int i=0;
95 log_verbose("\n{");
96 while (i<len)
97 log_verbose("%0x04X ", a[i++]);
98 log_verbose("}\n");
99}
100
101static void printSeqErr(const unsigned char* a, int len)
102{
103 int i=0;
104 fprintf(stderr, "\n{");
105 while (i<len) fprintf(stderr, "0x%02X ", a[i++]);
106 fprintf(stderr, "}\n");
107}
108
109static void printUSeqErr(const UChar* a, int len)
110{
111 int i=0;
112 fprintf(stderr, "\n{");
113 while (i<len)
114 fprintf(stderr, "0x%04X ", a[i++]);
115 fprintf(stderr,"}\n");
116}
117
118void addExtraTests(TestNode** root);
119
120void addExtraTests(TestNode** root)
121{
122 addTest(root, &TestSurrogateBehaviour, "tsconv/ncnvtst/TestSurrogateBehaviour");
123 addTest(root, &TestErrorBehaviour, "tsconv/ncnvtst/TestErrorBehaviour");
73c04bcf
A		 124
125#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 126 addTest(root, &TestToUnicodeErrorBehaviour, "tsconv/ncnvtst/ToUnicodeErrorBehaviour");
127 addTest(root, &TestGetNextErrorBehaviour, "tsconv/ncnvtst/TestGetNextErrorBehaviour");
73c04bcf
A		 128#endif
129
b75a7d8f
A		 130 addTest(root, &TestAvailableConverters, "tsconv/ncnvtst/TestAvailableConverters");
131 addTest(root, &TestFlushInternalBuffer, "tsconv/ncnvtst/TestFlushInternalBuffer");
132 addTest(root, &TestResetBehaviour, "tsconv/ncnvtst/TestResetBehaviour");
133 addTest(root, &TestRegressionUTF8, "tsconv/ncnvtst/TestRegressionUTF8");
134 addTest(root, &TestRegressionUTF32, "tsconv/ncnvtst/TestRegressionUTF32");
135 addTest(root, &TestTruncated, "tsconv/ncnvtst/TestTruncated");
136 addTest(root, &TestUnicodeSet, "tsconv/ncnvtst/TestUnicodeSet");
137}
138
139/*test surrogate behaviour*/
140static void TestSurrogateBehaviour(){
141 log_verbose("Testing for SBCS and LATIN_1\n");
142 {
143 UChar sampleText[] = {0x0031, 0xd801, 0xdc01, 0x0032};
144 const uint8_t expected[] = {0x31, 0x1a, 0x32};
73c04bcf
A		 145
146#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f 147 /*SBCS*/
2ca993e8 148 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 149 expected, sizeof(expected), "ibm-920", 0 , TRUE, U_ZERO_ERROR))
150 log_err("u-> ibm-920 [UCNV_SBCS] not match.\n");
73c04bcf 151#endif
b75a7d8f
A		 152
153 /*LATIN_1*/
2ca993e8 154 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 155 expected, sizeof(expected), "LATIN_1", 0, TRUE, U_ZERO_ERROR ))
156 log_err("u-> LATIN_1 not match.\n");
157
158 }
73c04bcf
A		 159
160#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 161 log_verbose("Testing for DBCS and MBCS\n");
162 {
163 UChar sampleText[] = {0x00a1, 0xd801, 0xdc01, 0x00a4};
164 const uint8_t expected[] = {0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
165 int32_t offsets[] = {0x00, 0x00, 0x01, 0x01, 0x03, 0x03 };
166
167 /*DBCS*/
2ca993e8 168 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 169 expected, sizeof(expected), "ibm-1363", 0 , TRUE, U_ZERO_ERROR))
170 log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
2ca993e8 171 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 172 expected, sizeof(expected), "ibm-1363", offsets , TRUE, U_ZERO_ERROR))
173 log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
174 /*MBCS*/
2ca993e8 175 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 176 expected, sizeof(expected), "ibm-1363", 0 , TRUE, U_ZERO_ERROR))
177 log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
2ca993e8 178 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 179 expected, sizeof(expected), "ibm-1363", offsets, TRUE, U_ZERO_ERROR))
180 log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
181 }
73c04bcf 182
b75a7d8f
A		 183 log_verbose("Testing for ISO-2022-jp\n");
184 {
185 UChar sampleText[] = { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
186
187 const uint8_t expected[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
188 0x31,0x1A, 0x32};
189
190
191 int32_t offsets[] = {0,0,0,0,0,1,1,2,2,2,2,3,5 };
192
193 /*iso-2022-jp*/
2ca993e8 194 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 195 expected, sizeof(expected), "iso-2022-jp", 0 , TRUE, U_ZERO_ERROR))
196 log_err("u-> not match.\n");
2ca993e8 197 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 198 expected, sizeof(expected), "iso-2022-jp", offsets , TRUE, U_ZERO_ERROR))
199 log_err("u-> not match.\n");
200 }
73c04bcf 201
b75a7d8f
A		 202 log_verbose("Testing for ISO-2022-cn\n");
203 {
204 static const UChar sampleText[] = { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
205
206 static const uint8_t expected[] = {
207 0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B,
208 0x36, 0x21,
209 0x0F, 0x31,
210 0x1A,
374ca955 211 0x32
b75a7d8f
A		 212 };
213
214
215
216 static const int32_t offsets[] = {
217 0, 0, 0, 0, 0, 0, 0,
218 1, 1,
219 2, 2,
220 3,
374ca955 221 5, };
b75a7d8f
A		 222
223 /*iso-2022-CN*/
2ca993e8 224 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 225 expected, sizeof(expected), "iso-2022-cn", 0 , TRUE, U_ZERO_ERROR))
226 log_err("u-> not match.\n");
2ca993e8 227 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 228 expected, sizeof(expected), "iso-2022-cn", offsets , TRUE, U_ZERO_ERROR))
229 log_err("u-> not match.\n");
230 }
73c04bcf 231
b75a7d8f
A		 232 log_verbose("Testing for ISO-2022-kr\n");
233 {
234 static const UChar sampleText[] = { 0x4e00,0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
235
236 static const uint8_t expected[] = {0x1B, 0x24, 0x29, 0x43,
237 0x0E, 0x6C, 0x69,
238 0x0f, 0x1A,
239 0x0e, 0x6F, 0x4B,
240 0x0F, 0x31,
241 0x1A,
242 0x32 };
243
244 static const int32_t offsets[] = {-1, -1, -1, -1,
245 0, 0, 0,
246 1, 1,
247 3, 3, 3,
248 4, 4,
249 5,
250 7,
251 };
252
253 /*iso-2022-kr*/
2ca993e8 254 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 255 expected, sizeof(expected), "iso-2022-kr", 0 , TRUE, U_ZERO_ERROR))
256 log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
2ca993e8 257 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 258 expected, sizeof(expected), "iso-2022-kr", offsets , TRUE, U_ZERO_ERROR))
259 log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
260 }
73c04bcf 261
b75a7d8f
A		 262 log_verbose("Testing for HZ\n");
263 {
264 static const UChar sampleText[] = { 0x4e00, 0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
265
266 static const uint8_t expected[] = {0x7E, 0x7B, 0x52, 0x3B,
267 0x7E, 0x7D, 0x1A,
268 0x7E, 0x7B, 0x36, 0x21,
269 0x7E, 0x7D, 0x31,
270 0x1A,
271 0x32 };
272
273
274 static const int32_t offsets[] = {0,0,0,0,
275 1,1,1,
276 3,3,3,3,
277 4,4,4,
278 5,
279 7,};
280
281 /*hz*/
2ca993e8 282 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f 283 expected, sizeof(expected), "HZ", 0 , TRUE, U_ZERO_ERROR))
73c04bcf 284 log_err("u-> HZ not match.\n");
2ca993e8 285 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f 286 expected, sizeof(expected), "HZ", offsets , TRUE, U_ZERO_ERROR))
73c04bcf 287 log_err("u-> HZ not match.\n");
b75a7d8f 288 }
73c04bcf
A		 289#endif
290
b75a7d8f
A		 291 /*UTF-8*/
292 log_verbose("Testing for UTF8\n");
293 {
294 static const UChar sampleText[] = { 0x4e00, 0x0701, 0x0031, 0xbfc1, 0xd801, 0xdc01, 0x0032};
295 static const int32_t offsets[]={0x00, 0x00, 0x00, 0x01, 0x01, 0x02,
296 0x03, 0x03, 0x03, 0x04, 0x04, 0x04,
297 0x04, 0x06 };
298 static const uint8_t expected[] = {0xe4, 0xb8, 0x80, 0xdc, 0x81, 0x31,
299 0xeb, 0xbf, 0x81, 0xF0, 0x90, 0x90, 0x81, 0x32};
300
301
302 static const int32_t fromOffsets[] = { 0x0000, 0x0003, 0x0005, 0x0006, 0x0009, 0x0009, 0x000D };
303 /*UTF-8*/
2ca993e8 304 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 305 expected, sizeof(expected), "UTF8", offsets, TRUE, U_ZERO_ERROR ))
306 log_err("u-> UTF8 with offsets and flush true did not match.\n");
2ca993e8 307 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 308 expected, sizeof(expected), "UTF8", 0, TRUE, U_ZERO_ERROR ))
309 log_err("u-> UTF8 with offsets and flush true did not match.\n");
2ca993e8 310 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 311 expected, sizeof(expected), "UTF8", offsets, FALSE, U_ZERO_ERROR ))
312 log_err("u-> UTF8 with offsets and flush true did not match.\n");
2ca993e8 313 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 314 expected, sizeof(expected), "UTF8", 0, FALSE, U_ZERO_ERROR ))
315 log_err("u-> UTF8 with offsets and flush true did not match.\n");
316
317 if(!convertToU(expected, sizeof(expected),
2ca993e8 318 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", 0, TRUE, U_ZERO_ERROR ))
73c04bcf 319 log_err("UTF8 -> u did not match.\n");
b75a7d8f 320 if(!convertToU(expected, sizeof(expected),
2ca993e8 321 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", 0, FALSE, U_ZERO_ERROR ))
73c04bcf 322 log_err("UTF8 -> u did not match.\n");
b75a7d8f 323 if(!convertToU(expected, sizeof(expected),
2ca993e8 324 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", fromOffsets, TRUE, U_ZERO_ERROR ))
73c04bcf 325 log_err("UTF8 ->u did not match.\n");
b75a7d8f 326 if(!convertToU(expected, sizeof(expected),
2ca993e8 327 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", fromOffsets, FALSE, U_ZERO_ERROR ))
73c04bcf 328 log_err("UTF8 -> u did not match.\n");
b75a7d8f
A		 329
330 }
b75a7d8f
A		 331}
332
333/*test various error behaviours*/
334static void TestErrorBehaviour(){
335 log_verbose("Testing for SBCS and LATIN_1\n");
336 {
337 static const UChar sampleText[] = { 0x0031, 0xd801};
338 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
374ca955
A		 339 static const uint8_t expected0[] = { 0x31};
340 static const uint8_t expected[] = { 0x31, 0x1a};
b75a7d8f
A		 341 static const uint8_t expected2[] = { 0x31, 0x1a, 0x32};
342
73c04bcf 343#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f 344 /*SBCS*/
2ca993e8 345 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 346 expected, sizeof(expected), "ibm-920", 0, TRUE, U_ZERO_ERROR))
b75a7d8f 347 log_err("u-> ibm-920 [UCNV_SBCS] \n");
2ca993e8 348 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 349 expected0, sizeof(expected0), "ibm-920", 0, FALSE, U_ZERO_ERROR))
b75a7d8f 350 log_err("u-> ibm-920 [UCNV_SBCS] \n");
2ca993e8 351 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 352 expected2, sizeof(expected2), "ibm-920", 0, TRUE, U_ZERO_ERROR))
353 log_err("u-> ibm-920 [UCNV_SBCS] did not match\n");
73c04bcf 354#endif
b75a7d8f
A		 355
356 /*LATIN_1*/
2ca993e8 357 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 358 expected, sizeof(expected), "LATIN_1", 0, TRUE, U_ZERO_ERROR))
b75a7d8f 359 log_err("u-> LATIN_1 is supposed to fail\n");
2ca993e8 360 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 361 expected0, sizeof(expected0), "LATIN_1", 0, FALSE, U_ZERO_ERROR))
b75a7d8f
A		 362 log_err("u-> LATIN_1 is supposed to fail\n");
363
2ca993e8 364 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 365 expected2, sizeof(expected2), "LATIN_1", 0, TRUE, U_ZERO_ERROR))
366 log_err("u-> LATIN_1 did not match\n");
367 }
368
73c04bcf 369#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 370 log_verbose("Testing for DBCS and MBCS\n");
371 {
372 static const UChar sampleText[] = { 0x00a1, 0xd801};
373 static const uint8_t expected[] = { 0xa2, 0xae};
374ca955
A		 374 static const int32_t offsets[] = { 0x00, 0x00};
375 static const uint8_t expectedSUB[] = { 0xa2, 0xae, 0xa1, 0xe0};
376 static const int32_t offsetsSUB[] = { 0x00, 0x00, 0x01, 0x01};
b75a7d8f
A		 377
378 static const UChar sampleText2[] = { 0x00a1, 0xd801, 0x00a4};
379 static const uint8_t expected2[] = { 0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
380 static const int32_t offsets2[] = { 0x00, 0x00, 0x01, 0x01, 0x02, 0x02};
381
382 static const UChar sampleText3MBCS[] = { 0x0001, 0x00a4, 0xdc01};
383 static const uint8_t expected3MBCS[] = { 0x01, 0xa2, 0xb4, 0xa1, 0xe0};
384 static const int32_t offsets3MBCS[] = { 0x00, 0x01, 0x01, 0x02, 0x02};
385
46f4442e 386 static const UChar sampleText4MBCS[] = { 0x0061, 0xFFE4, 0xdc01};
b75a7d8f
A		 387 static const uint8_t expected4MBCS[] = { 0x61, 0x8f, 0xa2, 0xc3, 0xf4, 0xfe};
388 static const int32_t offsets4MBCS[] = { 0x00, 0x01, 0x01, 0x01, 0x02, 0x02 };
389
b75a7d8f 390 /*DBCS*/
2ca993e8 391 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 392 expectedSUB, sizeof(expectedSUB), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
b75a7d8f 393 log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
2ca993e8 394 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
51004dcb 395 expected, sizeof(expected), "ibm-1363", 0, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
b75a7d8f
A		 396 log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
397
2ca993e8 398 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 399 expectedSUB, sizeof(expectedSUB), "ibm-1363", offsetsSUB, TRUE, U_ZERO_ERROR))
b75a7d8f 400 log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
2ca993e8 401 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
51004dcb 402 expected, sizeof(expected), "ibm-1363", offsets, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
b75a7d8f
A		 403 log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
404
405
2ca993e8 406 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 407 expected2, sizeof(expected2), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
408 log_err("u-> ibm-1363 [UCNV_DBCS portion] did not match \n");
2ca993e8 409 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 410 expected2, sizeof(expected2), "ibm-1363", offsets2, TRUE, U_ZERO_ERROR))
411 log_err("u-> ibm-1363 [UCNV_DBCS portion] did not match \n");
412
413 /*MBCS*/
2ca993e8 414 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 415 expectedSUB, sizeof(expectedSUB), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
b75a7d8f 416 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
2ca993e8 417 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
51004dcb 418 expected, sizeof(expected), "ibm-1363", 0, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
b75a7d8f
A		 419 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
420
2ca993e8 421 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 422 expected2, sizeof(expected2), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
423 log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");
2ca993e8 424 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 425 expected2, sizeof(expected2), "ibm-1363", 0, FALSE, U_ZERO_ERROR))
426 log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");
2ca993e8 427 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 428 expected2, sizeof(expected2), "ibm-1363", offsets2, FALSE, U_ZERO_ERROR))
429 log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");
430
2ca993e8 431 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
b75a7d8f
A		 432 expected3MBCS, sizeof(expected3MBCS), "ibm-1363", offsets3MBCS, TRUE, U_ZERO_ERROR))
433 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
2ca993e8 434 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
b75a7d8f
A		 435 expected3MBCS, sizeof(expected3MBCS), "ibm-1363", offsets3MBCS, FALSE, U_ZERO_ERROR))
436 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
437
2ca993e8 438 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
51004dcb 439 expected4MBCS, sizeof(expected4MBCS), "IBM-eucJP", offsets4MBCS, TRUE, U_ZERO_ERROR))
b75a7d8f 440 log_err("u-> euc-jp [UCNV_MBCS] \n");
2ca993e8 441 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
51004dcb 442 expected4MBCS, sizeof(expected4MBCS), "IBM-eucJP", offsets4MBCS, FALSE, U_ZERO_ERROR))
b75a7d8f
A		 443 log_err("u-> euc-jp [UCNV_MBCS] \n");
444 }
73c04bcf 445
b75a7d8f
A		 446 /*iso-2022-jp*/
447 log_verbose("Testing for iso-2022-jp\n");
448 {
449 static const UChar sampleText[] = { 0x0031, 0xd801};
450 static const uint8_t expected[] = { 0x31};
374ca955
A		 451 static const uint8_t expectedSUB[] = { 0x31, 0x1a};
452 static const int32_t offsets[] = { 0x00, 1};
b75a7d8f
A		 453
454 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
455 static const uint8_t expected2[] = { 0x31,0x1A,0x32};
456 static const int32_t offsets2[] = { 0x00,0x01,0x02};
457
b75a7d8f
A		 458 static const UChar sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
459 static const uint8_t expected4MBCS[] = { 0x61, 0x1b, 0x24, 0x42, 0x30, 0x6c,0x1b,0x28,0x42,0x1a};
460 static const int32_t offsets4MBCS[] = { 0x00, 0x01, 0x01 ,0x01, 0x01, 0x01,0x02,0x02,0x02,0x02 };
2ca993e8 461 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 462 expectedSUB, sizeof(expectedSUB), "iso-2022-jp", offsets, TRUE, U_ZERO_ERROR))
b75a7d8f 463 log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
2ca993e8 464 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
51004dcb
A		 465 expected, sizeof(expected), "iso-2022-jp", offsets, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
466 log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
b75a7d8f 467
2ca993e8 468 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 469 expected2, sizeof(expected2), "iso-2022-jp", offsets2, TRUE, U_ZERO_ERROR))
470 log_err("u->iso-2022-jp[UCNV_DBCS] did not match\n");
2ca993e8 471 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 472 expected2, sizeof(expected2), "iso-2022-jp", offsets2, FALSE, U_ZERO_ERROR))
473 log_err("u-> iso-2022-jp [UCNV_DBCS] did not match\n");
2ca993e8 474 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 475 expected2, sizeof(expected2), "iso-2022-jp", offsets2, FALSE, U_ZERO_ERROR))
476 log_err("u-> iso-2022-jp [UCNV_DBCS] did not match\n");
477
2ca993e8 478 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
b75a7d8f
A		 479 expected4MBCS, sizeof(expected4MBCS), "iso-2022-jp", offsets4MBCS, TRUE, U_ZERO_ERROR))
480 log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
2ca993e8 481 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
b75a7d8f
A		 482 expected4MBCS, sizeof(expected4MBCS), "iso-2022-jp", offsets4MBCS, FALSE, U_ZERO_ERROR))
483 log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
484 }
73c04bcf 485
b75a7d8f
A		 486 /*iso-2022-cn*/
487 log_verbose("Testing for iso-2022-cn\n");
488 {
489 static const UChar sampleText[] = { 0x0031, 0xd801};
374ca955
A		 490 static const uint8_t expected[] = { 0x31};
491 static const uint8_t expectedSUB[] = { 0x31, 0x1A};
492 static const int32_t offsets[] = { 0x00, 1};
b75a7d8f
A		 493
494 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
374ca955
A		 495 static const uint8_t expected2[] = { 0x31, 0x1A,0x32};
496 static const int32_t offsets2[] = { 0x00, 0x01,0x02};
b75a7d8f
A		 497
498 static const UChar sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
374ca955
A		 499 static const uint8_t expected3MBCS[] = {0x51, 0x50, 0x1A};
500 static const int32_t offsets3MBCS[] = { 0x00, 0x01, 0x02 };
b75a7d8f
A		 501
502 static const UChar sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
374ca955
A		 503 static const uint8_t expected4MBCS[] = { 0x61, 0x1b, 0x24, 0x29, 0x41, 0x0e, 0x52, 0x3b, 0x0f, 0x1a };
504 static const int32_t offsets4MBCS[] = { 0x00, 0x01, 0x01 ,0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02 };
2ca993e8 505 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 506 expectedSUB, sizeof(expectedSUB), "iso-2022-cn", offsets, TRUE, U_ZERO_ERROR))
b75a7d8f 507 log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
2ca993e8 508 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 509 expected, sizeof(expected), "iso-2022-cn", offsets, FALSE, U_ZERO_ERROR))
510 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
511
2ca993e8 512 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 513 expected2, sizeof(expected2), "iso-2022-cn", offsets2, TRUE, U_ZERO_ERROR))
514 log_err("u->iso-2022-cn[UCNV_DBCS] did not match\n");
2ca993e8 515 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 516 expected2, sizeof(expected2), "iso-2022-cn", offsets2, FALSE, U_ZERO_ERROR))
517 log_err("u-> iso-2022-cn [UCNV_DBCS] did not match\n");
2ca993e8 518 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 519 expected2, sizeof(expected2), "iso-2022-cn", offsets2, FALSE, U_ZERO_ERROR))
520 log_err("u-> iso-2022-cn [UCNV_DBCS] did not match\n");
521
2ca993e8 522 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
b75a7d8f
A		 523 expected3MBCS, sizeof(expected3MBCS), "iso-2022-cn", offsets3MBCS, TRUE, U_ZERO_ERROR))
524 log_err("u->iso-2022-cn [UCNV_MBCS] \n");
2ca993e8 525 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
b75a7d8f
A		 526 expected3MBCS, sizeof(expected3MBCS), "iso-2022-cn", offsets3MBCS, FALSE, U_ZERO_ERROR))
527 log_err("u-> iso-2022-cn[UCNV_MBCS] \n");
528
2ca993e8 529 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
b75a7d8f
A		 530 expected4MBCS, sizeof(expected4MBCS), "iso-2022-cn", offsets4MBCS, TRUE, U_ZERO_ERROR))
531 log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
2ca993e8 532 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
b75a7d8f
A		 533 expected4MBCS, sizeof(expected4MBCS), "iso-2022-cn", offsets4MBCS, FALSE, U_ZERO_ERROR))
534 log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
535 }
73c04bcf 536
b75a7d8f
A		 537 /*iso-2022-kr*/
538 log_verbose("Testing for iso-2022-kr\n");
539 {
540 static const UChar sampleText[] = { 0x0031, 0xd801};
541 static const uint8_t expected[] = { 0x1b, 0x24, 0x29, 0x43, 0x31};
374ca955
A		 542 static const uint8_t expectedSUB[] = { 0x1b, 0x24, 0x29, 0x43, 0x31, 0x1A};
543 static const int32_t offsets[] = { -1, -1, -1, -1, 0x00, 1};
b75a7d8f
A		 544
545 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
546 static const uint8_t expected2[] = { 0x1b, 0x24, 0x29, 0x43, 0x31, 0x1A, 0x32};
547 static const int32_t offsets2[] = { -1, -1, -1, -1, 0x00, 0x01, 0x02};
548
549 static const UChar sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
550 static const uint8_t expected3MBCS[] = { 0x1b, 0x24, 0x29, 0x43, 0x51, 0x50, 0x1A };
551 static const int32_t offsets3MBCS[] = { -1, -1, -1, -1, 0x00, 0x01, 0x02, 0x02 };
552
2ca993e8 553 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 554 expectedSUB, sizeof(expectedSUB), "iso-2022-kr", offsets, TRUE, U_ZERO_ERROR))
b75a7d8f 555 log_err("u-> iso-2022-kr [UCNV_MBCS] \n");
2ca993e8 556 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 557 expected, sizeof(expected), "iso-2022-kr", offsets, FALSE, U_ZERO_ERROR))
558 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
559
2ca993e8 560 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 561 expected2, sizeof(expected2), "iso-2022-kr", offsets2, TRUE, U_ZERO_ERROR))
562 log_err("u->iso-2022-kr[UCNV_DBCS] did not match\n");
2ca993e8 563 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 564 expected2, sizeof(expected2), "iso-2022-kr", offsets2, FALSE, U_ZERO_ERROR))
565 log_err("u-> iso-2022-kr [UCNV_DBCS] did not match\n");
2ca993e8 566 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 567 expected2, sizeof(expected2), "iso-2022-kr", offsets2, FALSE, U_ZERO_ERROR))
568 log_err("u-> iso-2022-kr [UCNV_DBCS] did not match\n");
569
2ca993e8 570 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
b75a7d8f
A		 571 expected3MBCS, sizeof(expected3MBCS), "iso-2022-kr", offsets3MBCS, TRUE, U_ZERO_ERROR))
572 log_err("u->iso-2022-kr [UCNV_MBCS] \n");
2ca993e8 573 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
b75a7d8f
A		 574 expected3MBCS, sizeof(expected3MBCS), "iso-2022-kr", offsets3MBCS, FALSE, U_ZERO_ERROR))
575 log_err("u-> iso-2022-kr[UCNV_MBCS] \n");
b75a7d8f
A		 576 }
577
578 /*HZ*/
579 log_verbose("Testing for HZ\n");
580 {
581 static const UChar sampleText[] = { 0x0031, 0xd801};
582 static const uint8_t expected[] = { 0x7e, 0x7d, 0x31};
374ca955
A		 583 static const uint8_t expectedSUB[] = { 0x7e, 0x7d, 0x31, 0x1A};
584 static const int32_t offsets[] = { 0x00, 0x00, 0x00, 1};
b75a7d8f
A		 585
586 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
587 static const uint8_t expected2[] = { 0x7e, 0x7d, 0x31, 0x1A, 0x32 };
588 static const int32_t offsets2[] = { 0x00, 0x00, 0x00, 0x01, 0x02 };
589
590 static const UChar sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
591 static const uint8_t expected3MBCS[] = { 0x7e, 0x7d, 0x51, 0x50, 0x1A };
592 static const int32_t offsets3MBCS[] = { 0x00, 0x00, 0x00, 0x01, 0x02};
593
594 static const UChar sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
595 static const uint8_t expected4MBCS[] = { 0x7e, 0x7d, 0x61, 0x7e, 0x7b, 0x52, 0x3b, 0x7e, 0x7d, 0x1a };
596 static const int32_t offsets4MBCS[] = { 0x00, 0x00, 0x00, 0x01, 0x01, 0x01 ,0x01, 0x02, 0x02, 0x02 };
2ca993e8 597 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 598 expectedSUB, sizeof(expectedSUB), "HZ", offsets, TRUE, U_ZERO_ERROR))
b75a7d8f 599 log_err("u-> HZ [UCNV_MBCS] \n");
2ca993e8 600 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 601 expected, sizeof(expected), "HZ", offsets, FALSE, U_ZERO_ERROR))
602 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
603
2ca993e8 604 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 605 expected2, sizeof(expected2), "HZ", offsets2, TRUE, U_ZERO_ERROR))
606 log_err("u->HZ[UCNV_DBCS] did not match\n");
2ca993e8 607 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 608 expected2, sizeof(expected2), "HZ", offsets2, FALSE, U_ZERO_ERROR))
609 log_err("u-> HZ [UCNV_DBCS] did not match\n");
2ca993e8 610 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 611 expected2, sizeof(expected2), "HZ", offsets2, FALSE, U_ZERO_ERROR))
612 log_err("u-> HZ [UCNV_DBCS] did not match\n");
613
2ca993e8 614 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
b75a7d8f
A		 615 expected3MBCS, sizeof(expected3MBCS), "HZ", offsets3MBCS, TRUE, U_ZERO_ERROR))
616 log_err("u->HZ [UCNV_MBCS] \n");
2ca993e8 617 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
b75a7d8f
A		 618 expected3MBCS, sizeof(expected3MBCS), "HZ", offsets3MBCS, FALSE, U_ZERO_ERROR))
619 log_err("u-> HZ[UCNV_MBCS] \n");
620
2ca993e8 621 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
b75a7d8f
A		 622 expected4MBCS, sizeof(expected4MBCS), "HZ", offsets4MBCS, TRUE, U_ZERO_ERROR))
623 log_err("u-> HZ [UCNV_MBCS] \n");
2ca993e8 624 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
b75a7d8f
A		 625 expected4MBCS, sizeof(expected4MBCS), "HZ", offsets4MBCS, FALSE, U_ZERO_ERROR))
626 log_err("u-> HZ [UCNV_MBCS] \n");
627 }
73c04bcf 628#endif
b75a7d8f
A		 629}
630
73c04bcf 631#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 632/*test different convertToUnicode error behaviours*/
633static void TestToUnicodeErrorBehaviour()
634{
635 log_verbose("Testing error conditions for DBCS\n");
636 {
637 uint8_t sampleText[] = { 0xa2, 0xae, 0x03, 0x04};
638 const UChar expected[] = { 0x00a1 };
639
b75a7d8f 640 if(!convertToU(sampleText, sizeof(sampleText),
2ca993e8 641 expected, UPRV_LENGTHOF(expected), "ibm-1363", 0, TRUE, U_AMBIGUOUS_ALIAS_WARNING ))
b75a7d8f
A		 642 log_err("DBCS (ibm-1363)->Unicode did not match.\n");
643 if(!convertToU(sampleText, sizeof(sampleText),
2ca993e8 644 expected, UPRV_LENGTHOF(expected), "ibm-1363", 0, FALSE, U_AMBIGUOUS_ALIAS_WARNING ))
b75a7d8f 645 log_err("DBCS (ibm-1363)->Unicode with flush = false did not match.\n");
b75a7d8f
A		 646 }
647 log_verbose("Testing error conditions for SBCS\n");
648 {
649 uint8_t sampleText[] = { 0xa2, 0xFF};
650 const UChar expected[] = { 0x00c2 };
651
652 /* uint8_t sampleText2[] = { 0xa2, 0x70 };
653 const UChar expected2[] = { 0x0073 };*/
654
655 if(!convertToU(sampleText, sizeof(sampleText),
2ca993e8 656 expected, UPRV_LENGTHOF(expected), "ibm-1051", 0, TRUE, U_ZERO_ERROR ))
b75a7d8f
A		 657 log_err("SBCS (ibm-1051)->Unicode did not match.\n");
658 if(!convertToU(sampleText, sizeof(sampleText),
2ca993e8 659 expected, UPRV_LENGTHOF(expected), "ibm-1051", 0, FALSE, U_ZERO_ERROR ))
b75a7d8f
A		 660 log_err("SBCS (ibm-1051)->Unicode with flush = false did not match.\n");
661
662 }
b75a7d8f
A		 663}
664
665static void TestGetNextErrorBehaviour(){
666 /*Test for unassigned character*/
667#define INPUT_SIZE 1
668 static const char input1[INPUT_SIZE]={ 0x70 };
669 const char* source=(const char*)input1;
670 UErrorCode err=U_ZERO_ERROR;
671 UChar32 c=0;
672 UConverter *cnv=ucnv_open("ibm-424", &err);
673 if(U_FAILURE(err)) {
674 log_data_err("Unable to open a SBCS(ibm-424) converter: %s\n", u_errorName(err));
675 return;
676 }
677 c=ucnv_getNextUChar(cnv, &source, source + INPUT_SIZE, &err);
678 if(err != U_INVALID_CHAR_FOUND && c!=0xfffd){
679 log_err("FAIL in TestGetNextErrorBehaviour(unassigned): Expected: U_INVALID_CHAR_ERROR or 0xfffd ----Got:%s and 0x%lx\n", myErrorName(err), c);
680 }
681 ucnv_close(cnv);
682}
73c04bcf 683#endif
b75a7d8f
A		 684
685#define MAX_UTF16_LEN 2
686#define MAX_UTF8_LEN 4
687
688/*Regression test for utf8 converter*/
689static void TestRegressionUTF8(){
690 UChar32 currCh = 0;
691 int32_t offset8;
692 int32_t offset16;
693 UChar *standardForm = (UChar*)malloc(MAX_LENGTH*sizeof(UChar));
694 uint8_t *utf8 = (uint8_t*)malloc(MAX_LENGTH);
695
696 while (currCh <= UNICODE_LIMIT) {
697 offset16 = 0;
698 offset8 = 0;
699 while(currCh <= UNICODE_LIMIT
700 && offset16 < (MAX_LENGTH/sizeof(UChar) - MAX_UTF16_LEN)
701 && offset8 < (MAX_LENGTH - MAX_UTF8_LEN))
702 {
703 if (currCh == SURROGATE_HIGH_START) {
704 currCh = SURROGATE_LOW_END + 1; /* Skip surrogate range */
705 }
706 UTF16_APPEND_CHAR_SAFE(standardForm, offset16, MAX_LENGTH, currCh);
707 UTF8_APPEND_CHAR_SAFE(utf8, offset8, MAX_LENGTH, currCh);
708 currCh++;
709 }
710 if(!convertFromU(standardForm, offset16,
711 utf8, offset8, "UTF8", 0, TRUE, U_ZERO_ERROR )) {
712 log_err("Unicode->UTF8 did not match.\n");
713 }
714 if(!convertToU(utf8, offset8,
715 standardForm, offset16, "UTF8", 0, TRUE, U_ZERO_ERROR )) {
716 log_err("UTF8->Unicode did not match.\n");
717 }
718 }
73c04bcf 719
b75a7d8f
A		 720 free(standardForm);
721 free(utf8);
73c04bcf
A		 722
723 {
724 static const char src8[] = { (char)0xCC, (char)0x81, (char)0xCC, (char)0x80 };
725 static const UChar expected[] = { 0x0301, 0x0300 };
726 UConverter *conv8;
727 UErrorCode err = U_ZERO_ERROR;
728 UChar pivotBuffer[100];
729 const UChar* const pivEnd = pivotBuffer + 100;
730 const char* srcBeg;
731 const char* srcEnd;
732 UChar* pivBeg;
733
734 conv8 = ucnv_open("UTF-8", &err);
735
736 srcBeg = src8;
737 pivBeg = pivotBuffer;
738 srcEnd = src8 + 3;
739 ucnv_toUnicode(conv8, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, FALSE, &err);
740 if (srcBeg != srcEnd) {
741 log_err("Did not consume whole buffer on first call.\n");
742 }
743
744 srcEnd = src8 + 4;
745 ucnv_toUnicode(conv8, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, TRUE, &err);
746 if (srcBeg != srcEnd) {
747 log_err("Did not consume whole buffer on second call.\n");
748 }
749
750 if (U_FAILURE(err) || (int32_t)(pivBeg - pivotBuffer) != 2 || u_strncmp(pivotBuffer, expected, 2) != 0) {
751 log_err("Did not get expected results for UTF-8.\n");
752 }
753 ucnv_close(conv8);
754 }
b75a7d8f
A		 755}
756
757#define MAX_UTF32_LEN 1
758
759static void TestRegressionUTF32(){
b331163b 760#if !UCONFIG_ONLY_HTML_CONVERSION
b75a7d8f
A		 761 UChar32 currCh = 0;
762 int32_t offset32;
763 int32_t offset16;
764 UChar *standardForm = (UChar*)malloc(MAX_LENGTH*sizeof(UChar));
765 UChar32 *utf32 = (UChar32*)malloc(MAX_LENGTH*sizeof(UChar32));
766
767 while (currCh <= UNICODE_LIMIT) {
768 offset16 = 0;
769 offset32 = 0;
770 while(currCh <= UNICODE_LIMIT
771 && offset16 < (MAX_LENGTH/sizeof(UChar) - MAX_UTF16_LEN)
772 && offset32 < (MAX_LENGTH/sizeof(UChar32) - MAX_UTF32_LEN))
773 {
774 if (currCh == SURROGATE_HIGH_START) {
775 currCh = SURROGATE_LOW_END + 1; /* Skip surrogate range */
776 }
777 UTF16_APPEND_CHAR_SAFE(standardForm, offset16, MAX_LENGTH, currCh);
778 UTF32_APPEND_CHAR_SAFE(utf32, offset32, MAX_LENGTH, currCh);
779 currCh++;
780 }
781 if(!convertFromU(standardForm, offset16,
782 (const uint8_t *)utf32, offset32*sizeof(UChar32), "UTF32_PlatformEndian", 0, TRUE, U_ZERO_ERROR )) {
783 log_err("Unicode->UTF32 did not match.\n");
784 }
785 if(!convertToU((const uint8_t *)utf32, offset32*sizeof(UChar32),
786 standardForm, offset16, "UTF32_PlatformEndian", 0, TRUE, U_ZERO_ERROR )) {
787 log_err("UTF32->Unicode did not match.\n");
788 }
789 }
790 free(standardForm);
791 free(utf32);
73c04bcf
A		 792
793 {
794 /* Check for lone surrogate error handling. */
795 static const UChar sampleBadStartSurrogate[] = { 0x0031, 0xD800, 0x0032 };
796 static const UChar sampleBadEndSurrogate[] = { 0x0031, 0xDC00, 0x0032 };
797 static const uint8_t expectedUTF32BE[] = {
798 0x00, 0x00, 0x00, 0x31,
799 0x00, 0x00, 0xff, 0xfd,
800 0x00, 0x00, 0x00, 0x32
801 };
802 static const uint8_t expectedUTF32LE[] = {
803 0x31, 0x00, 0x00, 0x00,
804 0xfd, 0xff, 0x00, 0x00,
805 0x32, 0x00, 0x00, 0x00
806 };
807 static const int32_t offsetsUTF32[] = {
808 0x00, 0x00, 0x00, 0x00,
809 0x01, 0x01, 0x01, 0x01,
810 0x02, 0x02, 0x02, 0x02
811 };
812
2ca993e8 813 if(!convertFromU(sampleBadStartSurrogate, UPRV_LENGTHOF(sampleBadStartSurrogate),
73c04bcf
A		 814 expectedUTF32BE, sizeof(expectedUTF32BE), "UTF-32BE", offsetsUTF32, TRUE, U_ZERO_ERROR))
815 log_err("u->UTF-32BE\n");
2ca993e8 816 if(!convertFromU(sampleBadEndSurrogate, UPRV_LENGTHOF(sampleBadEndSurrogate),
73c04bcf
A		 817 expectedUTF32BE, sizeof(expectedUTF32BE), "UTF-32BE", offsetsUTF32, TRUE, U_ZERO_ERROR))
818 log_err("u->UTF-32BE\n");
819
2ca993e8 820 if(!convertFromU(sampleBadStartSurrogate, UPRV_LENGTHOF(sampleBadStartSurrogate),
73c04bcf
A		 821 expectedUTF32LE, sizeof(expectedUTF32LE), "UTF-32LE", offsetsUTF32, TRUE, U_ZERO_ERROR))
822 log_err("u->UTF-32LE\n");
2ca993e8 823 if(!convertFromU(sampleBadEndSurrogate, UPRV_LENGTHOF(sampleBadEndSurrogate),
73c04bcf
A		 824 expectedUTF32LE, sizeof(expectedUTF32LE), "UTF-32LE", offsetsUTF32, TRUE, U_ZERO_ERROR))
825 log_err("u->UTF-32LE\n");
826 }
827
828 {
829 static const char srcBE[] = { 0, 0, 0, 0x31, 0, 0, 0, 0x30 };
830 static const UChar expected[] = { 0x0031, 0x0030 };
831 UConverter *convBE;
832 UErrorCode err = U_ZERO_ERROR;
833 UChar pivotBuffer[100];
834 const UChar* const pivEnd = pivotBuffer + 100;
835 const char* srcBeg;
836 const char* srcEnd;
837 UChar* pivBeg;
838
839 convBE = ucnv_open("UTF-32BE", &err);
840
841 srcBeg = srcBE;
842 pivBeg = pivotBuffer;
843 srcEnd = srcBE + 5;
844 ucnv_toUnicode(convBE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, FALSE, &err);
845 if (srcBeg != srcEnd) {
846 log_err("Did not consume whole buffer on first call.\n");
847 }
848
849 srcEnd = srcBE + 8;
850 ucnv_toUnicode(convBE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, TRUE, &err);
851 if (srcBeg != srcEnd) {
852 log_err("Did not consume whole buffer on second call.\n");
853 }
854
855 if (U_FAILURE(err) || (int32_t)(pivBeg - pivotBuffer) != 2 || u_strncmp(pivotBuffer, expected, 2) != 0) {
856 log_err("Did not get expected results for UTF-32BE.\n");
857 }
858 ucnv_close(convBE);
859 }
860 {
861 static const char srcLE[] = { 0x31, 0, 0, 0, 0x30, 0, 0, 0 };
862 static const UChar expected[] = { 0x0031, 0x0030 };
863 UConverter *convLE;
864 UErrorCode err = U_ZERO_ERROR;
865 UChar pivotBuffer[100];
866 const UChar* const pivEnd = pivotBuffer + 100;
867 const char* srcBeg;
868 const char* srcEnd;
869 UChar* pivBeg;
870
871 convLE = ucnv_open("UTF-32LE", &err);
872
873 srcBeg = srcLE;
874 pivBeg = pivotBuffer;
875 srcEnd = srcLE + 5;
876 ucnv_toUnicode(convLE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, FALSE, &err);
877 if (srcBeg != srcEnd) {
878 log_err("Did not consume whole buffer on first call.\n");
879 }
880
881 srcEnd = srcLE + 8;
882 ucnv_toUnicode(convLE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, TRUE, &err);
883 if (srcBeg != srcEnd) {
884 log_err("Did not consume whole buffer on second call.\n");
885 }
886
887 if (U_FAILURE(err) || (int32_t)(pivBeg - pivotBuffer) != 2 || u_strncmp(pivotBuffer, expected, 2) != 0) {
888 log_err("Did not get expected results for UTF-32LE.\n");
889 }
890 ucnv_close(convLE);
891 }
b331163b 892#endif
b75a7d8f
A		 893}
894
895/*Walk through the available converters*/
896static void TestAvailableConverters(){
897 UErrorCode status=U_ZERO_ERROR;
898 UConverter *conv=NULL;
899 int32_t i=0;
900 for(i=0; i < ucnv_countAvailable(); i++){
901 status=U_ZERO_ERROR;
902 conv=ucnv_open(ucnv_getAvailableName(i), &status);
903 if(U_FAILURE(status)){
904 log_err("ERROR: converter creation failed. Failure in alias table or the data table for \n converter=%s. Error=%s\n",
905 ucnv_getAvailableName(i), myErrorName(status));
906 continue;
907 }
908 ucnv_close(conv);
909 }
910
911}
912
913static void TestFlushInternalBuffer(){
914 TestWithBufferSize(MAX_LENGTH, 1);
915 TestWithBufferSize(1, 1);
916 TestWithBufferSize(1, MAX_LENGTH);
917 TestWithBufferSize(MAX_LENGTH, MAX_LENGTH);
918}
919
920static void TestWithBufferSize(int32_t insize, int32_t outsize){
921
922 gInBufferSize =insize;
923 gOutBufferSize = outsize;
924
925 log_verbose("Testing fromUnicode for UTF-8 with UCNV_TO_U_CALLBACK_SUBSTITUTE \n");
926 {
927 UChar sampleText[] =
928 { 0x0031, 0x0032, 0x0033, 0x0000, 0x4e00, 0x4e8c, 0x4e09, 0x002E };
929 const uint8_t expectedUTF8[] =
930 { 0x31, 0x32, 0x33, 0x00, 0xe4, 0xb8, 0x80, 0xe4, 0xba, 0x8c, 0xe4, 0xb8, 0x89, 0x2E };
931 int32_t toUTF8Offs[] =
932 { 0x00, 0x01, 0x02, 0x03, 0x04, 0x04, 0x04, 0x05, 0x05, 0x05, 0x06, 0x06, 0x06, 0x07};
933 /* int32_t fmUTF8Offs[] =
934 { 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0007, 0x000a, 0x000d };*/
935
936 /*UTF-8*/
2ca993e8 937 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 938 expectedUTF8, sizeof(expectedUTF8), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE, toUTF8Offs ,FALSE))
939 log_err("u-> UTF8 did not match.\n");
940 }
941
73c04bcf 942#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 943 log_verbose("Testing fromUnicode with UCNV_FROM_U_CALLBACK_ESCAPE \n");
944 {
945 UChar inputTest[] = { 0x0061, 0xd801, 0xdc01, 0xd801, 0x0061 };
946 const uint8_t toIBM943[]= { 0x61,
947 0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
948 0x25, 0x55, 0x44, 0x43, 0x30, 0x31,
949 0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
950 0x61 };
951 int32_t offset[]= {0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 4};
952
2ca993e8 953 if(!testConvertFromU(inputTest, UPRV_LENGTHOF(inputTest),
b75a7d8f
A		 954 toIBM943, sizeof(toIBM943), "ibm-943",
955 (UConverterFromUCallback)UCNV_FROM_U_CALLBACK_ESCAPE, offset,FALSE))
956 log_err("u-> ibm-943 with subst with value did not match.\n");
957 }
73c04bcf 958#endif
b75a7d8f
A		 959
960 log_verbose("Testing fromUnicode for UTF-8 with UCNV_TO_U_CALLBACK_SUBSTITUTE \n");
961 {
962 const uint8_t sampleText1[] = { 0x31, 0xe4, 0xba, 0x8c,
963 0xe0, 0x80, 0x61};
964 UChar expected1[] = { 0x0031, 0x4e8c, 0xfffd, 0x0061};
965 int32_t offsets1[] = { 0x0000, 0x0001, 0x0004, 0x0006};
966
967 if(!testConvertToU(sampleText1, sizeof(sampleText1),
2ca993e8 968 expected1, UPRV_LENGTHOF(expected1),"utf8", UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets1,FALSE))
b75a7d8f
A		 969 log_err("utf8->u with substitute did not match.\n");;
970 }
971
73c04bcf 972#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 973 log_verbose("Testing toUnicode with UCNV_TO_U_CALLBACK_ESCAPE \n");
974 /*to Unicode*/
975 {
976 const uint8_t sampleTxtToU[]= { 0x00, 0x9f, 0xaf,
977 0x81, 0xad, /*unassigned*/
978 0x89, 0xd3 };
979 UChar IBM_943toUnicode[] = { 0x0000, 0x6D63,
980 0x25, 0x58, 0x38, 0x31, 0x25, 0x58, 0x41, 0x44,
981 0x7B87};
982 int32_t fromIBM943Offs [] = { 0, 1, 3, 3, 3, 3, 3, 3, 3, 3, 5};
983
984 if(!testConvertToU(sampleTxtToU, sizeof(sampleTxtToU),
2ca993e8 985 IBM_943toUnicode, UPRV_LENGTHOF(IBM_943toUnicode),"ibm-943",
b75a7d8f
A		 986 (UConverterToUCallback)UCNV_TO_U_CALLBACK_ESCAPE, fromIBM943Offs,FALSE))
987 log_err("ibm-943->u with substitute with value did not match.\n");
988
989 }
73c04bcf 990#endif
b75a7d8f
A		 991}
992
993static UBool convertFromU( const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
994 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus)
995{
996
997 int32_t i=0;
73c04bcf 998 char *p=0;
b75a7d8f 999 const UChar *src;
73c04bcf 1000 char buffer[MAX_LENGTH];
b75a7d8f
A		 1001 int32_t offsetBuffer[MAX_LENGTH];
1002 int32_t *offs=0;
73c04bcf
A		 1003 char *targ;
1004 char *targetLimit;
b75a7d8f
A		 1005 UChar *sourceLimit=0;
1006 UErrorCode status = U_ZERO_ERROR;
1007 UConverter *conv = 0;
1008 conv = ucnv_open(codepage, &status);
1009 if(U_FAILURE(status))
1010 {
1011 log_data_err("Couldn't open converter %s\n",codepage);
1012 return TRUE;
1013 }
1014 log_verbose("Converter %s opened..\n", ucnv_getName(conv, &status));
1015
1016 for(i=0; i<MAX_LENGTH; i++){
73c04bcf 1017 buffer[i]=(char)0xF0;
b75a7d8f
A		 1018 offsetBuffer[i]=0xFF;
1019 }
1020
1021 src=source;
1022 sourceLimit=(UChar*)src+(sourceLen);
1023 targ=buffer;
1024 targetLimit=targ+MAX_LENGTH;
1025 offs=offsetBuffer;
1026 ucnv_fromUnicode (conv,
1027 (char **)&targ,
1028 (const char *)targetLimit,
1029 &src,
1030 sourceLimit,
1031 expectOffsets ? offs : NULL,
1032 doFlush,
1033 &status);
1034 ucnv_close(conv);
1035 if(status != expectedStatus){
1036 log_err("ucnv_fromUnicode() failed for codepage=%s. Error =%s Expected=%s\n", codepage, myErrorName(status), myErrorName(expectedStatus));
1037 return FALSE;
1038 }
1039
1040 log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
1041 sourceLen, targ-buffer);
1042
1043 if(expectLen != targ-buffer)
1044 {
1045 log_err("Expected %d chars out, got %d FROM Unicode to %s\n", expectLen, targ-buffer, codepage);
1046 log_verbose("Expected %d chars out, got %d FROM Unicode to %s\n", expectLen, targ-buffer, codepage);
73c04bcf 1047 printSeqErr((const unsigned char *)buffer, (int32_t)(targ-buffer));
b75a7d8f
A		 1048 printSeqErr((const unsigned char*)expect, expectLen);
1049 return FALSE;
1050 }
1051
1052 if(memcmp(buffer, expect, expectLen)){
1053 log_err("String does not match. FROM Unicode to codePage%s\n", codepage);
374ca955 1054 log_info("\nGot:");
b75a7d8f 1055 printSeqErr((const unsigned char *)buffer, expectLen);
374ca955 1056 log_info("\nExpected:");
b75a7d8f
A		 1057 printSeqErr((const unsigned char *)expect, expectLen);
1058 return FALSE;
1059 }
1060 else {
1061 log_verbose("Matches!\n");
1062 }
1063
1064 if (expectOffsets != 0){
1065 log_verbose("comparing %d offsets..\n", targ-buffer);
1066 if(memcmp(offsetBuffer,expectOffsets,(targ-buffer) * sizeof(int32_t) )){
1067 log_err("did not get the expected offsets. for FROM Unicode to %s\n", codepage);
374ca955 1068 log_info("\nGot : ");
73c04bcf 1069 printSeqErr((const unsigned char*)buffer, (int32_t)(targ-buffer));
b75a7d8f 1070 for(p=buffer;p<targ;p++)
374ca955
A		 1071 log_info("%d, ", offsetBuffer[p-buffer]);
1072 log_info("\nExpected: ");
b75a7d8f 1073 for(i=0; i< (targ-buffer); i++)
374ca955 1074 log_info("%d,", expectOffsets[i]);
b75a7d8f
A		 1075 }
1076 }
1077
1078 return TRUE;
1079}
1080
1081
1082static UBool convertToU( const uint8_t *source, int sourceLen, const UChar *expect, int expectLen,
1083 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus)
1084{
1085 UErrorCode status = U_ZERO_ERROR;
1086 UConverter *conv = 0;
1087 int32_t i=0;
1088 UChar *p=0;
73c04bcf 1089 const char* src;
b75a7d8f
A		 1090 UChar buffer[MAX_LENGTH];
1091 int32_t offsetBuffer[MAX_LENGTH];
1092 int32_t *offs=0;
1093 UChar *targ;
1094 UChar *targetLimit;
1095 uint8_t *sourceLimit=0;
1096
1097
1098
1099 conv = ucnv_open(codepage, &status);
1100 if(U_FAILURE(status))
1101 {
1102 log_data_err("Couldn't open converter %s\n",codepage);
1103 return TRUE;
1104 }
1105 log_verbose("Converter %s opened..\n", ucnv_getName(conv, &status));
1106
1107
1108
1109 for(i=0; i<MAX_LENGTH; i++){
1110 buffer[i]=0xFFFE;
1111 offsetBuffer[i]=-1;
1112 }
1113
73c04bcf 1114 src=(const char *)source;
b75a7d8f
A		 1115 sourceLimit=(uint8_t*)(src+(sourceLen));
1116 targ=buffer;
1117 targetLimit=targ+MAX_LENGTH;
1118 offs=offsetBuffer;
1119
1120
1121
1122 ucnv_toUnicode (conv,
1123 &targ,
1124 targetLimit,
1125 (const char **)&src,
1126 (const char *)sourceLimit,
1127 expectOffsets ? offs : NULL,
1128 doFlush,
1129 &status);
1130
1131 ucnv_close(conv);
1132 if(status != expectedStatus){
1133 log_err("ucnv_fromUnicode() failed for codepage=%s. Error =%s Expected=%s\n", codepage, myErrorName(status), myErrorName(expectedStatus));
1134 return FALSE;
1135 }
1136 log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
1137 sourceLen, targ-buffer);
1138
1139
1140
1141
1142 log_verbose("comparing %d uchars (%d bytes)..\n",expectLen,expectLen*2);
1143
1144 if (expectOffsets != 0) {
1145 if(memcmp(offsetBuffer, expectOffsets, (targ-buffer) * sizeof(int32_t))){
1146
1147 log_err("did not get the expected offsets from %s To UNICODE\n", codepage);
374ca955 1148 log_info("\nGot : ");
b75a7d8f 1149 for(p=buffer;p<targ;p++)
374ca955
A		 1150 log_info("%d, ", offsetBuffer[p-buffer]);
1151 log_info("\nExpected: ");
b75a7d8f 1152 for(i=0; i<(targ-buffer); i++)
374ca955
A		 1153 log_info("%d, ", expectOffsets[i]);
1154 log_info("\nGot result:");
b75a7d8f 1155 for(i=0; i<(targ-buffer); i++)
374ca955
A		 1156 log_info("0x%04X,", buffer[i]);
1157 log_info("\nFrom Input:");
73c04bcf 1158 for(i=0; i<(src-(const char *)source); i++)
374ca955
A		 1159 log_info("0x%02X,", (unsigned char)source[i]);
1160 log_info("\n");
b75a7d8f
A		 1161 }
1162 }
1163 if(memcmp(buffer, expect, expectLen*2)){
1164 log_err("String does not match. from codePage %s TO Unicode\n", codepage);
374ca955 1165 log_info("\nGot:");
b75a7d8f 1166 printUSeqErr(buffer, expectLen);
374ca955 1167 log_info("\nExpected:");
b75a7d8f
A		 1168 printUSeqErr(expect, expectLen);
1169 return FALSE;
1170 }
1171 else {
1172 log_verbose("Matches!\n");
1173 }
1174
1175 return TRUE;
1176}
1177
1178
1179static UBool testConvertFromU( const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
1180 const char *codepage, UConverterFromUCallback callback , const int32_t *expectOffsets, UBool testReset)
1181{
1182 UErrorCode status = U_ZERO_ERROR;
1183 UConverter *conv = 0;
73c04bcf 1184 char junkout[MAX_LENGTH]; /* FIX */
b75a7d8f 1185 int32_t junokout[MAX_LENGTH]; /* FIX */
73c04bcf 1186 char *p;
b75a7d8f 1187 const UChar *src;
73c04bcf
A		 1188 char *end;
1189 char *targ;
b75a7d8f
A		 1190 int32_t *offs;
1191 int i;
1192 int32_t realBufferSize;
73c04bcf 1193 char *realBufferEnd;
b75a7d8f
A		 1194 const UChar *realSourceEnd;
1195 const UChar *sourceLimit;
1196 UBool checkOffsets = TRUE;
1197 UBool doFlush;
1198
1199 UConverterFromUCallback oldAction = NULL;
1200 const void* oldContext = NULL;
1201
1202 for(i=0;i<MAX_LENGTH;i++)
73c04bcf 1203 junkout[i] = (char)0xF0;
b75a7d8f
A		 1204 for(i=0;i<MAX_LENGTH;i++)
1205 junokout[i] = 0xFF;
1206
1207 setNuConvTestName(codepage, "FROM");
1208
1209 log_verbose("\n========= %s\n", gNuConvTestName);
1210
1211 conv = ucnv_open(codepage, &status);
1212 if(U_FAILURE(status))
1213 {
1214 log_data_err("Couldn't open converter %s\n",codepage);
1215 return TRUE;
1216 }
1217
1218 log_verbose("Converter opened..\n");
1219 /*----setting the callback routine----*/
1220 ucnv_setFromUCallBack (conv, callback, NULL, &oldAction, &oldContext, &status);
1221 if (U_FAILURE(status)) {
1222 log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));
1223 }
1224 /*------------------------*/
1225
1226 src = source;
1227 targ = junkout;
1228 offs = junokout;
1229
2ca993e8 1230 realBufferSize = UPRV_LENGTHOF(junkout);
b75a7d8f
A		 1231 realBufferEnd = junkout + realBufferSize;
1232 realSourceEnd = source + sourceLen;
1233
1234 if ( gOutBufferSize != realBufferSize )
1235 checkOffsets = FALSE;
1236
1237 if( gInBufferSize != MAX_LENGTH )
1238 checkOffsets = FALSE;
1239
1240 do
73c04bcf 1241 {
b75a7d8f
A		 1242 end = nct_min(targ + gOutBufferSize, realBufferEnd);
1243 sourceLimit = nct_min(src + gInBufferSize, realSourceEnd);
1244
1245 doFlush = (UBool)(sourceLimit == realSourceEnd);
1246
1247 if(targ == realBufferEnd)
1248 {
1249 log_err("Error, overflowed the real buffer while about to call fromUnicode! targ=%08lx %s", targ, gNuConvTestName);
1250 return FALSE;
1251 }
1252 log_verbose("calling fromUnicode @ SOURCE:%08lx to %08lx TARGET: %08lx to %08lx, flush=%s\n", src,sourceLimit, targ,end, doFlush?"TRUE":"FALSE");
1253
1254
1255 status = U_ZERO_ERROR;
1256 if(gInBufferSize ==999 && gOutBufferSize==999)
1257 doFlush = FALSE;
1258 ucnv_fromUnicode (conv,
1259 (char **)&targ,
1260 (const char *)end,
1261 &src,
1262 sourceLimit,
1263 offs,
1264 doFlush, /* flush if we're at the end of the input data */
1265 &status);
1266 if(testReset)
1267 ucnv_resetToUnicode(conv);
1268 if(gInBufferSize ==999 && gOutBufferSize==999)
1269 ucnv_resetToUnicode(conv);
1270
1271 } while ( (status == U_BUFFER_OVERFLOW_ERROR) || (U_SUCCESS(status) && sourceLimit < realSourceEnd) );
1272
1273 if(U_FAILURE(status)) {
1274 log_err("Problem doing fromUnicode to %s, errcode %s %s\n", codepage, myErrorName(status), gNuConvTestName);
1275 return FALSE;
1276 }
1277
1278 log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
1279 sourceLen, targ-junkout);
729e4ab9 1280 if(getTestOption(VERBOSITY_OPTION))
b75a7d8f
A		 1281 {
1282 char junk[999];
1283 char offset_str[999];
73c04bcf 1284 char *ptr;
b75a7d8f
A		 1285
1286 junk[0] = 0;
1287 offset_str[0] = 0;
1288 for(ptr = junkout;ptr<targ;ptr++)
1289 {
1290 sprintf(junk + strlen(junk), "0x%02x, ", (0xFF) & (unsigned int)*ptr);
1291 sprintf(offset_str + strlen(offset_str), "0x%02x, ", (0xFF) & (unsigned int)junokout[ptr-junkout]);
1292 }
1293
1294 log_verbose(junk);
1295 printSeq((const unsigned char *)expect, expectLen);
1296 if ( checkOffsets )
1297 {
1298 log_verbose("\nOffsets:");
1299 log_verbose(offset_str);
1300 }
1301 log_verbose("\n");
1302 }
1303 ucnv_close(conv);
1304
1305
1306 if(expectLen != targ-junkout)
1307 {
1308 log_err("Expected %d chars out, got %d %s\n", expectLen, targ-junkout, gNuConvTestName);
1309 log_verbose("Expected %d chars out, got %d %s\n", expectLen, targ-junkout, gNuConvTestName);
374ca955 1310 log_info("\nGot:");
73c04bcf 1311 printSeqErr((const unsigned char*)junkout, (int32_t)(targ-junkout));
374ca955 1312 log_info("\nExpected:");
b75a7d8f
A		 1313 printSeqErr((const unsigned char*)expect, expectLen);
1314 return FALSE;
1315 }
1316
1317 if (checkOffsets && (expectOffsets != 0) )
1318 {
1319 log_verbose("comparing %d offsets..\n", targ-junkout);
1320 if(memcmp(junokout,expectOffsets,(targ-junkout) * sizeof(int32_t) )){
1321 log_err("did not get the expected offsets. %s", gNuConvTestName);
1322 log_err("Got : ");
73c04bcf 1323 printSeqErr((const unsigned char*)junkout, (int32_t)(targ-junkout));
b75a7d8f
A		 1324 for(p=junkout;p<targ;p++)
1325 log_err("%d, ", junokout[p-junkout]);
1326 log_err("\nExpected: ");
1327 for(i=0; i<(targ-junkout); i++)
1328 log_err("%d,", expectOffsets[i]);
1329 }
1330 }
1331
1332 log_verbose("comparing..\n");
1333 if(!memcmp(junkout, expect, expectLen))
1334 {
1335 log_verbose("Matches!\n");
1336 return TRUE;
1337 }
1338 else
1339 {
1340 log_err("String does not match. %s\n", gNuConvTestName);
1341 printUSeqErr(source, sourceLen);
374ca955 1342 log_info("\nGot:");
b75a7d8f 1343 printSeqErr((const unsigned char *)junkout, expectLen);
374ca955 1344 log_info("\nExpected:");
b75a7d8f
A		 1345 printSeqErr((const unsigned char *)expect, expectLen);
1346
1347 return FALSE;
1348 }
1349}
1350
1351static UBool testConvertToU( const uint8_t *source, int sourcelen, const UChar *expect, int expectlen,
1352 const char *codepage, UConverterToUCallback callback, const int32_t *expectOffsets, UBool testReset)
1353{
1354 UErrorCode status = U_ZERO_ERROR;
1355 UConverter *conv = 0;
1356 UChar junkout[MAX_LENGTH]; /* FIX */
1357 int32_t junokout[MAX_LENGTH]; /* FIX */
73c04bcf
A		 1358 const char *src;
1359 const char *realSourceEnd;
1360 const char *srcLimit;
b75a7d8f
A		 1361 UChar *p;
1362 UChar *targ;
1363 UChar *end;
1364 int32_t *offs;
1365 int i;
1366 UBool checkOffsets = TRUE;
1367 int32_t realBufferSize;
1368 UChar *realBufferEnd;
1369 UBool doFlush;
1370
1371 UConverterToUCallback oldAction = NULL;
1372 const void* oldContext = NULL;
1373
1374
1375 for(i=0;i<MAX_LENGTH;i++)
1376 junkout[i] = 0xFFFE;
1377
1378 for(i=0;i<MAX_LENGTH;i++)
1379 junokout[i] = -1;
1380
1381 setNuConvTestName(codepage, "TO");
1382
1383 log_verbose("\n========= %s\n", gNuConvTestName);
1384
1385 conv = ucnv_open(codepage, &status);
1386 if(U_FAILURE(status))
1387 {
1388 log_data_err("Couldn't open converter %s\n",gNuConvTestName);
1389 return TRUE;
1390 }
1391
1392 log_verbose("Converter opened..\n");
1393 /*----setting the callback routine----*/
1394 ucnv_setToUCallBack (conv, callback, NULL, &oldAction, &oldContext, &status);
1395 if (U_FAILURE(status)) {
1396 log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));
1397 }
1398 /*-------------------------------------*/
1399
73c04bcf 1400 src = (const char *)source;
b75a7d8f
A		 1401 targ = junkout;
1402 offs = junokout;
1403
2ca993e8 1404 realBufferSize = UPRV_LENGTHOF(junkout);
b75a7d8f
A		 1405 realBufferEnd = junkout + realBufferSize;
1406 realSourceEnd = src + sourcelen;
1407
1408 if ( gOutBufferSize != realBufferSize )
1409 checkOffsets = FALSE;
1410
1411 if( gInBufferSize != MAX_LENGTH )
1412 checkOffsets = FALSE;
1413
1414 do
1415 {
1416 end = nct_min( targ + gOutBufferSize, realBufferEnd);
1417 srcLimit = nct_min(realSourceEnd, src + gInBufferSize);
1418
1419 if(targ == realBufferEnd)
1420 {
1421 log_err("Error, the end would overflow the real output buffer while about to call toUnicode! tarjey=%08lx %s",targ,gNuConvTestName);
1422 return FALSE;
1423 }
1424 log_verbose("calling toUnicode @ %08lx to %08lx\n", targ,end);
1425
1426 /* oldTarg = targ; */
1427
1428 status = U_ZERO_ERROR;
1429 doFlush=(UBool)((gInBufferSize ==999 && gOutBufferSize==999)?(srcLimit == realSourceEnd) : FALSE);
1430
1431 ucnv_toUnicode (conv,
1432 &targ,
1433 end,
1434 (const char **)&src,
1435 (const char *)srcLimit,
1436 offs,
1437 doFlush, /* flush if we're at the end of hte source data */
1438 &status);
1439 if(testReset)
1440 ucnv_resetFromUnicode(conv);
1441 if(gInBufferSize ==999 && gOutBufferSize==999)
1442 ucnv_resetToUnicode(conv);
1443 /* offs += (targ-oldTarg); */
1444
1445 } while ( (status == U_BUFFER_OVERFLOW_ERROR) || (U_SUCCESS(status) && (srcLimit < realSourceEnd)) ); /* while we just need another buffer */
1446
1447 if(U_FAILURE(status))
1448 {
1449 log_err("Problem doing %s toUnicode, errcode %s %s\n", codepage, myErrorName(status), gNuConvTestName);
1450 return FALSE;
1451 }
1452
1453 log_verbose("\nConversion done. %d bytes -> %d chars.\nResult :",
1454 sourcelen, targ-junkout);
729e4ab9 1455 if(getTestOption(VERBOSITY_OPTION))
b75a7d8f
A		 1456 {
1457 char junk[999];
1458 char offset_str[999];
1459
1460 UChar *ptr;
1461
1462 junk[0] = 0;
1463 offset_str[0] = 0;
1464
1465 for(ptr = junkout;ptr<targ;ptr++)
1466 {
1467 sprintf(junk + strlen(junk), "0x%04x, ", (0xFFFF) & (unsigned int)*ptr);
1468 sprintf(offset_str + strlen(offset_str), "0x%04x, ", (0xFFFF) & (unsigned int)junokout[ptr-junkout]);
1469 }
1470
1471 log_verbose(junk);
1472
1473 if ( checkOffsets )
1474 {
1475 log_verbose("\nOffsets:");
1476 log_verbose(offset_str);
1477 }
1478 log_verbose("\n");
1479 }
1480 ucnv_close(conv);
1481
1482 log_verbose("comparing %d uchars (%d bytes)..\n",expectlen,expectlen*2);
1483
1484 if (checkOffsets && (expectOffsets != 0))
1485 {
1486 if(memcmp(junokout,expectOffsets,(targ-junkout) * sizeof(int32_t))){
1487
1488 log_err("did not get the expected offsets. %s",gNuConvTestName);
1489 for(p=junkout;p<targ;p++)
1490 log_err("%d, ", junokout[p-junkout]);
1491 log_err("\nExpected: ");
1492 for(i=0; i<(targ-junkout); i++)
1493 log_err("%d,", expectOffsets[i]);
1494 log_err("");
1495 for(i=0; i<(targ-junkout); i++)
1496 log_err("%X,", junkout[i]);
1497 log_err("");
73c04bcf 1498 for(i=0; i<(src-(const char *)source); i++)
b75a7d8f
A		 1499 log_err("%X,", (unsigned char)source[i]);
1500 }
1501 }
1502
1503 if(!memcmp(junkout, expect, expectlen*2))
1504 {
1505 log_verbose("Matches!\n");
1506 return TRUE;
1507 }
1508 else
1509 {
1510 log_err("String does not match. %s\n", gNuConvTestName);
1511 log_verbose("String does not match. %s\n", gNuConvTestName);
374ca955 1512 log_info("\nGot:");
b75a7d8f 1513 printUSeq(junkout, expectlen);
374ca955 1514 log_info("\nExpected:");
b75a7d8f
A		 1515 printUSeq(expect, expectlen);
1516 return FALSE;
1517 }
1518}
1519
1520
1521static void TestResetBehaviour(void){
73c04bcf 1522#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 1523 log_verbose("Testing Reset for DBCS and MBCS\n");
1524 {
1525 static const UChar sampleText[] = {0x00a1, 0xd801, 0xdc01, 0x00a4};
1526 static const uint8_t expected[] = {0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
1527 static const int32_t offsets[] = {0x00, 0x00, 0x01, 0x01, 0x03, 0x03 };
1528
1529
1530 static const UChar sampleText1[] = {0x00a1, 0x00a4, 0x00a7, 0x00a8};
1531 static const uint8_t expected1[] = {0xa2, 0xae,0xA2,0xB4,0xA1,0xD7,0xA1,0xA7};
1532 static const int32_t offsets1[] = { 0,2,4,6};
1533
1534 /*DBCS*/
2ca993e8 1535 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1536 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1537 log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
2ca993e8 1538 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1539 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1540 log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
1541
1542 if(!testConvertToU(expected1, sizeof(expected1),
2ca993e8 1543 sampleText1, UPRV_LENGTHOF(sampleText1), "ibm-1363",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
b75a7d8f
A		 1544 offsets1, TRUE))
1545 log_err("ibm-1363 -> did not match.\n");
1546 /*MBCS*/
2ca993e8 1547 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1548 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1549 log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
2ca993e8 1550 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1551 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1552 log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
1553
1554 if(!testConvertToU(expected1, sizeof(expected1),
2ca993e8 1555 sampleText1, UPRV_LENGTHOF(sampleText1), "ibm-1363",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
b75a7d8f
A		 1556 offsets1, TRUE))
1557 log_err("ibm-1363 -> did not match.\n");
1558
1559 }
73c04bcf 1560
b75a7d8f
A		 1561 log_verbose("Testing Reset for ISO-2022-jp\n");
1562 {
1563 static const UChar sampleText[] = { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1564
1565 static const uint8_t expected[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
1566 0x31,0x1A, 0x32};
1567
1568
1569 static const int32_t offsets[] = {0,0,0,0,0,1,1,2,2,2,2,3,5 };
1570
1571
1572 static const UChar sampleText1[] = {0x4e00, 0x04e01, 0x0031,0x001A, 0x0032};
1573 static const uint8_t expected1[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
1574 0x31,0x1A, 0x32};
1575 static const int32_t offsets1[] = { 3,5,10,11,12};
1576
1577 /*iso-2022-jp*/
2ca993e8 1578 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1579 expected, sizeof(expected), "iso-2022-jp", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1580 log_err("u-> not match.\n");
2ca993e8 1581 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1582 expected, sizeof(expected), "iso-2022-jp", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1583 log_err("u-> not match.\n");
1584
1585 if(!testConvertToU(expected1, sizeof(expected1),
2ca993e8 1586 sampleText1, UPRV_LENGTHOF(sampleText1), "iso-2022-jp",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
b75a7d8f
A		 1587 offsets1, TRUE))
1588 log_err("iso-2022-jp -> did not match.\n");
1589
1590 }
73c04bcf 1591
b75a7d8f
A		 1592 log_verbose("Testing Reset for ISO-2022-cn\n");
1593 {
1594 static const UChar sampleText[] = { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1595
1596 static const uint8_t expected[] = {
1597 0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B,
1598 0x36, 0x21,
1599 0x0f, 0x31,
1600 0x1A,
374ca955 1601 0x32
b75a7d8f
A		 1602 };
1603
1604
1605 static const int32_t offsets[] = {
1606 0, 0, 0, 0, 0, 0, 0,
1607 1, 1,
1608 2, 2,
1609 3,
374ca955 1610 5, };
b75a7d8f
A		 1611
1612 UChar sampleText1[] = {0x4e00, 0x04e01, 0x0031,0x001A, 0x0032};
1613 static const uint8_t expected1[] = {
1614 0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B,
1615 0x36, 0x21,
374ca955 1616 0x1B, 0x24, 0x29, 0x47, 0x24, 0x22,
b75a7d8f
A		 1617 0x0f, 0x1A,
1618 0x32
1619 };
374ca955 1620 static const int32_t offsets1[] = { 5,7,13,16,17};
b75a7d8f
A		 1621
1622 /*iso-2022-CN*/
2ca993e8 1623 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1624 expected, sizeof(expected), "iso-2022-cn", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1625 log_err("u-> not match.\n");
2ca993e8 1626 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1627 expected, sizeof(expected), "iso-2022-cn", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1628 log_err("u-> not match.\n");
1629
1630 if(!testConvertToU(expected1, sizeof(expected1),
2ca993e8 1631 sampleText1, UPRV_LENGTHOF(sampleText1), "iso-2022-cn",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
b75a7d8f
A		 1632 offsets1, TRUE))
1633 log_err("iso-2022-cn -> did not match.\n");
1634 }
73c04bcf 1635
b75a7d8f
A		 1636 log_verbose("Testing Reset for ISO-2022-kr\n");
1637 {
1638 UChar sampleText[] = { 0x4e00,0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1639
1640 static const uint8_t expected[] = {0x1B, 0x24, 0x29, 0x43,
1641 0x0E, 0x6C, 0x69,
1642 0x0f, 0x1A,
1643 0x0e, 0x6F, 0x4B,
1644 0x0F, 0x31,
1645 0x1A,
1646 0x32 };
1647
1648 static const int32_t offsets[] = {-1, -1, -1, -1,
1649 0, 0, 0,
1650 1, 1,
1651 3, 3, 3,
1652 4, 4,
1653 5,
1654 7,
1655 };
1656 static const UChar sampleText1[] = { 0x4e00,0x0041, 0x04e01, 0x0031, 0x0042, 0x0032};
1657
1658 static const uint8_t expected1[] = {0x1B, 0x24, 0x29, 0x43,
1659 0x0E, 0x6C, 0x69,
1660 0x0f, 0x41,
1661 0x0e, 0x6F, 0x4B,
1662 0x0F, 0x31,
1663 0x42,
1664 0x32 };
1665
1666 static const int32_t offsets1[] = {
1667 5, 8, 10,
1668 13, 14, 15
1669
1670 };
1671 /*iso-2022-kr*/
2ca993e8 1672 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1673 expected, sizeof(expected), "iso-2022-kr", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1674 log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
2ca993e8 1675 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1676 expected, sizeof(expected), "iso-2022-kr", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1677 log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
1678 if(!testConvertToU(expected1, sizeof(expected1),
2ca993e8 1679 sampleText1, UPRV_LENGTHOF(sampleText1), "iso-2022-kr",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
b75a7d8f
A		 1680 offsets1, TRUE))
1681 log_err("iso-2022-kr -> did not match.\n");
1682 }
73c04bcf 1683
b75a7d8f
A		 1684 log_verbose("Testing Reset for HZ\n");
1685 {
1686 static const UChar sampleText[] = { 0x4e00, 0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1687
1688 static const uint8_t expected[] = {0x7E, 0x7B, 0x52, 0x3B,
1689 0x7E, 0x7D, 0x1A,
1690 0x7E, 0x7B, 0x36, 0x21,
1691 0x7E, 0x7D, 0x31,
1692 0x1A,
1693 0x32 };
1694
1695
1696 static const int32_t offsets[] = {0,0,0,0,
1697 1,1,1,
1698 3,3,3,3,
1699 4,4,4,
1700 5,
1701 7,};
1702 static const UChar sampleText1[] = { 0x4e00, 0x0035, 0x04e01, 0x0031, 0x0041, 0x0032};
1703
1704 static const uint8_t expected1[] = {0x7E, 0x7B, 0x52, 0x3B,
1705 0x7E, 0x7D, 0x35,
1706 0x7E, 0x7B, 0x36, 0x21,
1707 0x7E, 0x7D, 0x31,
1708 0x41,
1709 0x32 };
1710
1711
1712 static const int32_t offsets1[] = {2,6,9,13,14,15
1713 };
1714
1715 /*hz*/
2ca993e8 1716 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1717 expected, sizeof(expected), "HZ", UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
1718 log_err("u-> not match.\n");
2ca993e8 1719 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1720 expected, sizeof(expected), "HZ", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1721 log_err("u-> not match.\n");
1722 if(!testConvertToU(expected1, sizeof(expected1),
2ca993e8 1723 sampleText1, UPRV_LENGTHOF(sampleText1), "hz",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
b75a7d8f
A		 1724 offsets1, TRUE))
1725 log_err("hz -> did not match.\n");
1726 }
73c04bcf
A		 1727#endif
1728
b75a7d8f
A		 1729 /*UTF-8*/
1730 log_verbose("Testing for UTF8\n");
1731 {
1732 static const UChar sampleText[] = { 0x4e00, 0x0701, 0x0031, 0xbfc1, 0xd801, 0xdc01, 0x0032};
1733 int32_t offsets[]={0x00, 0x00, 0x00, 0x01, 0x01, 0x02,
1734 0x03, 0x03, 0x03, 0x04, 0x04, 0x04,
1735 0x04, 0x06 };
1736 static const uint8_t expected[] = {0xe4, 0xb8, 0x80, 0xdc, 0x81, 0x31,
1737 0xeb, 0xbf, 0x81, 0xF0, 0x90, 0x90, 0x81, 0x32};
1738
1739
1740 static const int32_t fromOffsets[] = { 0x0000, 0x0003, 0x0005, 0x0006, 0x0009, 0x0009, 0x000D };
1741 /*UTF-8*/
2ca993e8 1742 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1743 expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1744 log_err("u-> UTF8 with offsets and flush true did not match.\n");
2ca993e8 1745 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1746 expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
1747 log_err("u-> UTF8 with offsets and flush true did not match.\n");
2ca993e8 1748 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1749 expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1750 log_err("u-> UTF8 with offsets and flush true did not match.\n");
2ca993e8 1751 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1752 expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
1753 log_err("u-> UTF8 with offsets and flush true did not match.\n");
1754 if(!testConvertToU(expected, sizeof(expected),
2ca993e8 1755 sampleText, UPRV_LENGTHOF(sampleText), "UTF8",UCNV_TO_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
b75a7d8f
A		 1756 log_err("UTF8 -> did not match.\n");
1757 if(!testConvertToU(expected, sizeof(expected),
2ca993e8 1758 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", UCNV_TO_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
b75a7d8f
A		 1759 log_err("UTF8 -> did not match.\n");
1760 if(!testConvertToU(expected, sizeof(expected),
2ca993e8 1761 sampleText, UPRV_LENGTHOF(sampleText), "UTF8",UCNV_TO_U_CALLBACK_SUBSTITUTE , fromOffsets, TRUE))
b75a7d8f
A		 1762 log_err("UTF8 -> did not match.\n");
1763 if(!testConvertToU(expected, sizeof(expected),
2ca993e8 1764 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", UCNV_TO_U_CALLBACK_SUBSTITUTE , fromOffsets, TRUE))
b75a7d8f
A		 1765 log_err("UTF8 -> did not match.\n");
1766
1767 }
1768
1769}
1770
1771/* Test that U_TRUNCATED_CHAR_FOUND is set. */
1772static void
1773doTestTruncated(const char *cnvName, const uint8_t *bytes, int32_t length) {
1774 UConverter *cnv;
1775
1776 UChar buffer[2];
1777 UChar *target, *targetLimit;
1778 const char *source, *sourceLimit;
1779
1780 UErrorCode errorCode;
1781
1782 errorCode=U_ZERO_ERROR;
1783 cnv=ucnv_open(cnvName, &errorCode);
1784 if(U_FAILURE(errorCode)) {
1785 log_data_err("error TestTruncated: unable to open \"%s\" - %s\n", cnvName, u_errorName(errorCode));
1786 return;
1787 }
374ca955
A		 1788 ucnv_setToUCallBack(cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
1789 if(U_FAILURE(errorCode)) {
1790 log_data_err("error TestTruncated: unable to set the stop callback on \"%s\" - %s\n",
1791 cnvName, u_errorName(errorCode));
1792 ucnv_close(cnv);
1793 return;
1794 }
b75a7d8f
A		 1795
1796 source=(const char *)bytes;
1797 sourceLimit=source+length;
1798 target=buffer;
b331163b 1799 targetLimit=buffer+UPRV_LENGTHOF(buffer);
b75a7d8f
A		 1800
1801 /* 1. input bytes with flush=FALSE, then input nothing with flush=TRUE */
1802 ucnv_toUnicode(cnv, &target, targetLimit, &source, sourceLimit, NULL, FALSE, &errorCode);
1803 if(U_FAILURE(errorCode) || source!=sourceLimit || target!=buffer) {
1804 log_err("error TestTruncated(%s, 1a): input bytes[%d], flush=FALSE: %s, input left %d, output %d\n",
1805 cnvName, length, u_errorName(errorCode), (int)(sourceLimit-source), (int)(target-buffer));
1806 }
1807
1808 errorCode=U_ZERO_ERROR;
1809 source=sourceLimit;
1810 target=buffer;
1811 ucnv_toUnicode(cnv, &target, targetLimit, &source, sourceLimit, NULL, TRUE, &errorCode);
1812 if(errorCode!=U_TRUNCATED_CHAR_FOUND || target!=buffer) {
374ca955
A		 1813 log_err("error TestTruncated(%s, 1b): no input (previously %d), flush=TRUE: %s (should be U_TRUNCATED_CHAR_FOUND), output %d\n",
1814 cnvName, (int)length, u_errorName(errorCode), (int)(target-buffer));
b75a7d8f
A		 1815 }
1816
b75a7d8f
A		 1817 /* 2. input bytes with flush=TRUE */
1818 ucnv_resetToUnicode(cnv);
1819
1820 errorCode=U_ZERO_ERROR;
1821 source=(const char *)bytes;
1822 target=buffer;
1823 ucnv_toUnicode(cnv, &target, targetLimit, &source, sourceLimit, NULL, TRUE, &errorCode);
374ca955 1824 if(errorCode!=U_TRUNCATED_CHAR_FOUND || source!=sourceLimit || target!=buffer) {
b75a7d8f
A		 1825 log_err("error TestTruncated(%s, 2): input bytes[%d], flush=TRUE: %s (should be U_TRUNCATED_CHAR_FOUND), input left %d, output %d\n",
1826 cnvName, length, u_errorName(errorCode), (int)(sourceLimit-source), (int)(target-buffer));
1827 }
1828
1829
1830 ucnv_close(cnv);
1831}
1832
1833static void
1834TestTruncated() {
1835 static const struct {
1836 const char *cnvName;
1837 uint8_t bytes[8]; /* partial input bytes resulting in no output */
1838 int32_t length;
1839 } testCases[]={
1840 { "IMAP-mailbox-name", { 0x26 }, 1 }, /* & */
1841 { "IMAP-mailbox-name", { 0x26, 0x42 }, 2 }, /* &B */
1842 { "IMAP-mailbox-name", { 0x26, 0x42, 0x42 }, 3 }, /* &BB */
1843 { "IMAP-mailbox-name", { 0x26, 0x41, 0x41 }, 3 }, /* &AA */
1844
1845 { "UTF-7", { 0x2b, 0x42 }, 2 }, /* +B */
1846 { "UTF-8", { 0xd1 }, 1 },
1847
1848 { "UTF-16BE", { 0x4e }, 1 },
1849 { "UTF-16LE", { 0x4e }, 1 },
1850 { "UTF-16", { 0x4e }, 1 },
1851 { "UTF-16", { 0xff }, 1 },
1852 { "UTF-16", { 0xfe, 0xff, 0x4e }, 3 },
1853
1854 { "UTF-32BE", { 0, 0, 0x4e }, 3 },
1855 { "UTF-32LE", { 0x4e }, 1 },
1856 { "UTF-32", { 0, 0, 0x4e }, 3 },
1857 { "UTF-32", { 0xff }, 1 },
1858 { "UTF-32", { 0, 0, 0xfe, 0xff, 0 }, 5 },
b75a7d8f 1859 { "SCSU", { 0x0e, 0x4e }, 2 }, /* SQU 0x4e */
73c04bcf
A		 1860
1861#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 1862 { "BOCU-1", { 0xd5 }, 1 },
1863
1864 { "Shift-JIS", { 0xe0 }, 1 },
1865
1866 { "ibm-939", { 0x0e, 0x41 }, 2 } /* SO 0x41 */
73c04bcf
A		 1867#else
1868 { "BOCU-1", { 0xd5 }, 1 ,}
1869#endif
b75a7d8f
A		 1870 };
1871 int32_t i;
1872
b331163b 1873 for(i=0; i<UPRV_LENGTHOF(testCases); ++i) {
b75a7d8f
A		 1874 doTestTruncated(testCases[i].cnvName, testCases[i].bytes, testCases[i].length);
1875 }
1876}
1877
1878typedef struct NameRange {
1879 const char *name;
1880 UChar32 start, end, start2, end2, notStart, notEnd;
1881} NameRange;
1882
1883static void
1884TestUnicodeSet() {
1885 UErrorCode errorCode;
1886 UConverter *cnv;
1887 USet *set;
1888 const char *name;
1889 int32_t i, count;
1890
1891 static const char *const completeSetNames[]={
1892 "UTF-7",
1893 "UTF-8",
1894 "UTF-16",
1895 "UTF-16BE",
1896 "UTF-16LE",
1897 "UTF-32",
1898 "UTF-32BE",
1899 "UTF-32LE",
1900 "SCSU",
1901 "BOCU-1",
1902 "CESU-8",
73c04bcf 1903#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f 1904 "gb18030",
73c04bcf 1905#endif
374ca955
A		 1906 "IMAP-mailbox-name"
1907 };
73c04bcf 1908#if !UCONFIG_NO_LEGACY_CONVERSION
729e4ab9 1909 static const char *const lmbcsNames[]={
b75a7d8f
A		 1910 "LMBCS-1",
1911 "LMBCS-2",
1912 "LMBCS-3",
1913 "LMBCS-4",
1914 "LMBCS-5",
1915 "LMBCS-6",
1916 "LMBCS-8",
1917 "LMBCS-11",
1918 "LMBCS-16",
1919 "LMBCS-17",
1920 "LMBCS-18",
1921 "LMBCS-19"
1922 };
729e4ab9 1923#endif
b75a7d8f
A		 1924
1925 static const NameRange nameRanges[]={
1926 { "US-ASCII", 0, 0x7f, -1, -1, 0x80, 0x10ffff },
73c04bcf 1927#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f 1928 { "ibm-367", 0, 0x7f, -1, -1, 0x80, 0x10ffff },
73c04bcf 1929#endif
b75a7d8f 1930 { "ISO-8859-1", 0, 0x7f, -1, -1, 0x100, 0x10ffff },
73c04bcf 1931#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 1932 { "UTF-8", 0, 0xd7ff, 0xe000, 0x10ffff, 0xd800, 0xdfff },
1933 { "windows-1251", 0, 0x7f, 0x410, 0x44f, 0x3000, 0xd7ff },
46f4442e 1934 /* HZ test case fixed and moved to intltest's conversion.txt, ticket #6002 */
b75a7d8f 1935 { "shift-jis", 0x3041, 0x3093, 0x30a1, 0x30f3, 0x900, 0x1cff }
73c04bcf
A		 1936#else
1937 { "UTF-8", 0, 0xd7ff, 0xe000, 0x10ffff, 0xd800, 0xdfff }
1938#endif
b75a7d8f
A		 1939 };
1940
1941 /* open an empty set */
1942 set=uset_open(1, 0);
1943
1944 count=ucnv_countAvailable();
1945 for(i=0; i<count; ++i) {
1946 errorCode=U_ZERO_ERROR;
1947 name=ucnv_getAvailableName(i);
1948 cnv=ucnv_open(name, &errorCode);
1949 if(U_FAILURE(errorCode)) {
729e4ab9 1950 log_data_err("error: unable to open converter %s - %s\n",
b75a7d8f
A		 1951 name, u_errorName(errorCode));
1952 continue;
1953 }
1954
1955 uset_clear(set);
1956 ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
1957 if(U_FAILURE(errorCode)) {
1958 log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
1959 name, u_errorName(errorCode));
1960 } else if(uset_size(set)==0) {
1961 log_err("error: ucnv_getUnicodeSet(%s) returns an empty set\n", name);
1962 }
1963
1964 ucnv_close(cnv);
1965 }
1966
1967 /* test converters that are known to convert all of Unicode (except maybe for surrogates) */
b331163b 1968 for(i=0; i<UPRV_LENGTHOF(completeSetNames); ++i) {
b75a7d8f
A		 1969 errorCode=U_ZERO_ERROR;
1970 name=completeSetNames[i];
1971 cnv=ucnv_open(name, &errorCode);
1972 if(U_FAILURE(errorCode)) {
729e4ab9 1973 log_data_err("error: unable to open converter %s - %s\n",
b75a7d8f
A		 1974 name, u_errorName(errorCode));
1975 continue;
1976 }
1977
1978 uset_clear(set);
1979 ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
1980 if(U_FAILURE(errorCode)) {
1981 log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
1982 name, u_errorName(errorCode));
1983 } else if(!uset_containsRange(set, 0, 0xd7ff) || !uset_containsRange(set, 0xe000, 0x10ffff)) {
1984 log_err("error: ucnv_getUnicodeSet(%s) does not return an all-Unicode set\n", name);
1985 }
1986
1987 ucnv_close(cnv);
1988 }
1989
729e4ab9 1990#if !UCONFIG_NO_LEGACY_CONVERSION
374ca955 1991 /* test LMBCS variants which convert all of Unicode except for U+F6xx */
b331163b 1992 for(i=0; i<UPRV_LENGTHOF(lmbcsNames); ++i) {
374ca955
A		 1993 errorCode=U_ZERO_ERROR;
1994 name=lmbcsNames[i];
1995 cnv=ucnv_open(name, &errorCode);
1996 if(U_FAILURE(errorCode)) {
729e4ab9 1997 log_data_err("error: unable to open converter %s - %s\n",
374ca955
A		 1998 name, u_errorName(errorCode));
1999 continue;
2000 }
2001
2002 uset_clear(set);
2003 ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
2004 if(U_FAILURE(errorCode)) {
2005 log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
2006 name, u_errorName(errorCode));
2007 } else if(!uset_containsRange(set, 0, 0xf5ff) || !uset_containsRange(set, 0xf700, 0x10ffff)) {
2008 log_err("error: ucnv_getUnicodeSet(%s) does not return an all-Unicode set (minus U+F6xx)\n", name);
2009 }
2010
2011 ucnv_close(cnv);
2012 }
729e4ab9 2013#endif
374ca955 2014
b75a7d8f 2015 /* test specific sets */
b331163b 2016 for(i=0; i<UPRV_LENGTHOF(nameRanges); ++i) {
b75a7d8f
A		 2017 errorCode=U_ZERO_ERROR;
2018 name=nameRanges[i].name;
2019 cnv=ucnv_open(name, &errorCode);
2020 if(U_FAILURE(errorCode)) {
2021 log_data_err("error: unable to open converter %s - %s\n",
2022 name, u_errorName(errorCode));
2023 continue;
2024 }
2025
2026 uset_clear(set);
2027 ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
2028 if(U_FAILURE(errorCode)) {
2029 log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
2030 name, u_errorName(errorCode));
2031 } else if(
2032 !uset_containsRange(set, nameRanges[i].start, nameRanges[i].end) ||
2033 (nameRanges[i].start2>=0 && !uset_containsRange(set, nameRanges[i].start2, nameRanges[i].end2))
2034 ) {
2035 log_err("error: ucnv_getUnicodeSet(%s) does not contain the expected ranges\n", name);
2036 } else if(nameRanges[i].notStart>=0) {
2037 /* simulate containsAny() with the C API */
2038 uset_complement(set);
2039 if(!uset_containsRange(set, nameRanges[i].notStart, nameRanges[i].notEnd)) {
2040 log_err("error: ucnv_getUnicodeSet(%s) contains part of the unexpected range\n", name);
2041 }
2042 }
2043
2044 ucnv_close(cnv);
2045 }
2046
73c04bcf
A		 2047 errorCode = U_ZERO_ERROR;
2048 ucnv_getUnicodeSet(NULL, set, UCNV_ROUNDTRIP_SET, &errorCode);
2049 if (errorCode != U_ILLEGAL_ARGUMENT_ERROR) {
2050 log_err("error: ucnv_getUnicodeSet(NULL) returned wrong status code %s\n", u_errorName(errorCode));
2051 }
2052 errorCode = U_PARSE_ERROR;
2053 /* Make sure that it does nothing if an error is passed in. Difficult to proper test for. */
2054 ucnv_getUnicodeSet(NULL, NULL, UCNV_ROUNDTRIP_SET, &errorCode);
2055 if (errorCode != U_PARSE_ERROR) {
2056 log_err("error: ucnv_getUnicodeSet(NULL) returned wrong status code %s\n", u_errorName(errorCode));
2057 }
2058
b75a7d8f
A		 2059 uset_close(set);
2060}
mirror of ICU