]> git.saurik.com Git - apple/icu.git/blame - icuSources/test/cintltst/ncnvtst.c
projects / apple / icu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ICU-57131.0.1.tar.gz
[apple/icu.git] / icuSources / test / cintltst / ncnvtst.c
CommitLineData
b75a7d8f
A		 1/********************************************************************
2 * COPYRIGHT:
2ca993e8 3 * Copyright (c) 1997-2016, International Business Machines Corporation and
b75a7d8f
A		 4 * others. All Rights Reserved.
5 ********************************************************************/
46f4442e 6/*****************************************************************************
b75a7d8f 7*
b331163b 8* File ncnvtst.c
b75a7d8f
A		 9*
10* Modification History:
11* Name Description
12* Madhu Katragadda 7/7/2000 Converter Tests for extended code coverage
46f4442e 13******************************************************************************
b75a7d8f
A		 14*/
15#include <stdio.h>
374ca955
A		 16#include <stdlib.h>
17#include <string.h>
b75a7d8f
A		 18#include "unicode/uloc.h"
19#include "unicode/ucnv.h"
20#include "unicode/utypes.h"
21#include "unicode/ustring.h"
22#include "unicode/uset.h"
23#include "cintltst.h"
b331163b 24#include "cmemory.h"
b75a7d8f
A		 25
26#define MAX_LENGTH 999
27
28#define UNICODE_LIMIT 0x10FFFF
29#define SURROGATE_HIGH_START 0xD800
30#define SURROGATE_LOW_END 0xDFFF
31
32static int32_t gInBufferSize = 0;
33static int32_t gOutBufferSize = 0;
34static char gNuConvTestName[1024];
35
36#define nct_min(x,y) ((x<y) ? x : y)
b75a7d8f
A		 37
38static void printSeq(const unsigned char* a, int len);
39static void printSeqErr(const unsigned char* a, int len);
40static void printUSeq(const UChar* a, int len);
41static void printUSeqErr(const UChar* a, int len);
42static UBool convertFromU( const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
43 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus);
44static UBool convertToU( const uint8_t *source, int sourceLen, const UChar *expect, int expectLen,
45 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus);
46
47static UBool testConvertFromU( const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
48 const char *codepage, UConverterFromUCallback callback, const int32_t *expectOffsets, UBool testReset);
49static UBool testConvertToU( const uint8_t *source, int sourcelen, const UChar *expect, int expectlen,
50 const char *codepage, UConverterToUCallback callback, const int32_t *expectOffsets, UBool testReset);
51
52static void setNuConvTestName(const char *codepage, const char *direction)
53{
374ca955
A		 54 sprintf(gNuConvTestName, "[Testing %s %s Unicode, InputBufSiz=%d, OutputBufSiz=%d]",
55 codepage,
56 direction,
57 (int)gInBufferSize,
58 (int)gOutBufferSize);
b75a7d8f
A		 59}
60
61
62static void TestSurrogateBehaviour(void);
63static void TestErrorBehaviour(void);
73c04bcf
A		 64
65#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 66static void TestToUnicodeErrorBehaviour(void);
67static void TestGetNextErrorBehaviour(void);
73c04bcf
A		 68#endif
69
b75a7d8f
A		 70static void TestRegressionUTF8(void);
71static void TestRegressionUTF32(void);
72static void TestAvailableConverters(void);
73static void TestFlushInternalBuffer(void); /*for improved code coverage in ucnv_cnv.c*/
74static void TestResetBehaviour(void);
75static void TestTruncated(void);
76static void TestUnicodeSet(void);
77
78static void TestWithBufferSize(int32_t osize, int32_t isize);
79
80
81static void printSeq(const unsigned char* a, int len)
82{
83 int i=0;
84 log_verbose("\n{");
85 while (i<len)
86 log_verbose("0x%02X ", a[i++]);
87 log_verbose("}\n");
88}
89
90static void printUSeq(const UChar* a, int len)
91{
92 int i=0;
93 log_verbose("\n{");
94 while (i<len)
95 log_verbose("%0x04X ", a[i++]);
96 log_verbose("}\n");
97}
98
99static void printSeqErr(const unsigned char* a, int len)
100{
101 int i=0;
102 fprintf(stderr, "\n{");
103 while (i<len) fprintf(stderr, "0x%02X ", a[i++]);
104 fprintf(stderr, "}\n");
105}
106
107static void printUSeqErr(const UChar* a, int len)
108{
109 int i=0;
110 fprintf(stderr, "\n{");
111 while (i<len)
112 fprintf(stderr, "0x%04X ", a[i++]);
113 fprintf(stderr,"}\n");
114}
115
116void addExtraTests(TestNode** root);
117
118void addExtraTests(TestNode** root)
119{
120 addTest(root, &TestSurrogateBehaviour, "tsconv/ncnvtst/TestSurrogateBehaviour");
121 addTest(root, &TestErrorBehaviour, "tsconv/ncnvtst/TestErrorBehaviour");
73c04bcf
A		 122
123#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 124 addTest(root, &TestToUnicodeErrorBehaviour, "tsconv/ncnvtst/ToUnicodeErrorBehaviour");
125 addTest(root, &TestGetNextErrorBehaviour, "tsconv/ncnvtst/TestGetNextErrorBehaviour");
73c04bcf
A		 126#endif
127
b75a7d8f
A		 128 addTest(root, &TestAvailableConverters, "tsconv/ncnvtst/TestAvailableConverters");
129 addTest(root, &TestFlushInternalBuffer, "tsconv/ncnvtst/TestFlushInternalBuffer");
130 addTest(root, &TestResetBehaviour, "tsconv/ncnvtst/TestResetBehaviour");
131 addTest(root, &TestRegressionUTF8, "tsconv/ncnvtst/TestRegressionUTF8");
132 addTest(root, &TestRegressionUTF32, "tsconv/ncnvtst/TestRegressionUTF32");
133 addTest(root, &TestTruncated, "tsconv/ncnvtst/TestTruncated");
134 addTest(root, &TestUnicodeSet, "tsconv/ncnvtst/TestUnicodeSet");
135}
136
137/*test surrogate behaviour*/
138static void TestSurrogateBehaviour(){
139 log_verbose("Testing for SBCS and LATIN_1\n");
140 {
141 UChar sampleText[] = {0x0031, 0xd801, 0xdc01, 0x0032};
142 const uint8_t expected[] = {0x31, 0x1a, 0x32};
73c04bcf
A		 143
144#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f 145 /*SBCS*/
2ca993e8 146 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 147 expected, sizeof(expected), "ibm-920", 0 , TRUE, U_ZERO_ERROR))
148 log_err("u-> ibm-920 [UCNV_SBCS] not match.\n");
73c04bcf 149#endif
b75a7d8f
A		 150
151 /*LATIN_1*/
2ca993e8 152 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 153 expected, sizeof(expected), "LATIN_1", 0, TRUE, U_ZERO_ERROR ))
154 log_err("u-> LATIN_1 not match.\n");
155
156 }
73c04bcf
A		 157
158#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 159 log_verbose("Testing for DBCS and MBCS\n");
160 {
161 UChar sampleText[] = {0x00a1, 0xd801, 0xdc01, 0x00a4};
162 const uint8_t expected[] = {0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
163 int32_t offsets[] = {0x00, 0x00, 0x01, 0x01, 0x03, 0x03 };
164
165 /*DBCS*/
2ca993e8 166 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 167 expected, sizeof(expected), "ibm-1363", 0 , TRUE, U_ZERO_ERROR))
168 log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
2ca993e8 169 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 170 expected, sizeof(expected), "ibm-1363", offsets , TRUE, U_ZERO_ERROR))
171 log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
172 /*MBCS*/
2ca993e8 173 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 174 expected, sizeof(expected), "ibm-1363", 0 , TRUE, U_ZERO_ERROR))
175 log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
2ca993e8 176 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 177 expected, sizeof(expected), "ibm-1363", offsets, TRUE, U_ZERO_ERROR))
178 log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
179 }
73c04bcf 180
b75a7d8f
A		 181 log_verbose("Testing for ISO-2022-jp\n");
182 {
183 UChar sampleText[] = { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
184
185 const uint8_t expected[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
186 0x31,0x1A, 0x32};
187
188
189 int32_t offsets[] = {0,0,0,0,0,1,1,2,2,2,2,3,5 };
190
191 /*iso-2022-jp*/
2ca993e8 192 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 193 expected, sizeof(expected), "iso-2022-jp", 0 , TRUE, U_ZERO_ERROR))
194 log_err("u-> not match.\n");
2ca993e8 195 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 196 expected, sizeof(expected), "iso-2022-jp", offsets , TRUE, U_ZERO_ERROR))
197 log_err("u-> not match.\n");
198 }
73c04bcf 199
b75a7d8f
A		 200 log_verbose("Testing for ISO-2022-cn\n");
201 {
202 static const UChar sampleText[] = { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
203
204 static const uint8_t expected[] = {
205 0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B,
206 0x36, 0x21,
207 0x0F, 0x31,
208 0x1A,
374ca955 209 0x32
b75a7d8f
A		 210 };
211
212
213
214 static const int32_t offsets[] = {
215 0, 0, 0, 0, 0, 0, 0,
216 1, 1,
217 2, 2,
218 3,
374ca955 219 5, };
b75a7d8f
A		 220
221 /*iso-2022-CN*/
2ca993e8 222 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 223 expected, sizeof(expected), "iso-2022-cn", 0 , TRUE, U_ZERO_ERROR))
224 log_err("u-> not match.\n");
2ca993e8 225 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 226 expected, sizeof(expected), "iso-2022-cn", offsets , TRUE, U_ZERO_ERROR))
227 log_err("u-> not match.\n");
228 }
73c04bcf 229
b75a7d8f
A		 230 log_verbose("Testing for ISO-2022-kr\n");
231 {
232 static const UChar sampleText[] = { 0x4e00,0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
233
234 static const uint8_t expected[] = {0x1B, 0x24, 0x29, 0x43,
235 0x0E, 0x6C, 0x69,
236 0x0f, 0x1A,
237 0x0e, 0x6F, 0x4B,
238 0x0F, 0x31,
239 0x1A,
240 0x32 };
241
242 static const int32_t offsets[] = {-1, -1, -1, -1,
243 0, 0, 0,
244 1, 1,
245 3, 3, 3,
246 4, 4,
247 5,
248 7,
249 };
250
251 /*iso-2022-kr*/
2ca993e8 252 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 253 expected, sizeof(expected), "iso-2022-kr", 0 , TRUE, U_ZERO_ERROR))
254 log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
2ca993e8 255 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 256 expected, sizeof(expected), "iso-2022-kr", offsets , TRUE, U_ZERO_ERROR))
257 log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
258 }
73c04bcf 259
b75a7d8f
A		 260 log_verbose("Testing for HZ\n");
261 {
262 static const UChar sampleText[] = { 0x4e00, 0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
263
264 static const uint8_t expected[] = {0x7E, 0x7B, 0x52, 0x3B,
265 0x7E, 0x7D, 0x1A,
266 0x7E, 0x7B, 0x36, 0x21,
267 0x7E, 0x7D, 0x31,
268 0x1A,
269 0x32 };
270
271
272 static const int32_t offsets[] = {0,0,0,0,
273 1,1,1,
274 3,3,3,3,
275 4,4,4,
276 5,
277 7,};
278
279 /*hz*/
2ca993e8 280 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f 281 expected, sizeof(expected), "HZ", 0 , TRUE, U_ZERO_ERROR))
73c04bcf 282 log_err("u-> HZ not match.\n");
2ca993e8 283 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f 284 expected, sizeof(expected), "HZ", offsets , TRUE, U_ZERO_ERROR))
73c04bcf 285 log_err("u-> HZ not match.\n");
b75a7d8f 286 }
73c04bcf
A		 287#endif
288
b75a7d8f
A		 289 /*UTF-8*/
290 log_verbose("Testing for UTF8\n");
291 {
292 static const UChar sampleText[] = { 0x4e00, 0x0701, 0x0031, 0xbfc1, 0xd801, 0xdc01, 0x0032};
293 static const int32_t offsets[]={0x00, 0x00, 0x00, 0x01, 0x01, 0x02,
294 0x03, 0x03, 0x03, 0x04, 0x04, 0x04,
295 0x04, 0x06 };
296 static const uint8_t expected[] = {0xe4, 0xb8, 0x80, 0xdc, 0x81, 0x31,
297 0xeb, 0xbf, 0x81, 0xF0, 0x90, 0x90, 0x81, 0x32};
298
299
300 static const int32_t fromOffsets[] = { 0x0000, 0x0003, 0x0005, 0x0006, 0x0009, 0x0009, 0x000D };
301 /*UTF-8*/
2ca993e8 302 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 303 expected, sizeof(expected), "UTF8", offsets, TRUE, U_ZERO_ERROR ))
304 log_err("u-> UTF8 with offsets and flush true did not match.\n");
2ca993e8 305 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 306 expected, sizeof(expected), "UTF8", 0, TRUE, U_ZERO_ERROR ))
307 log_err("u-> UTF8 with offsets and flush true did not match.\n");
2ca993e8 308 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 309 expected, sizeof(expected), "UTF8", offsets, FALSE, U_ZERO_ERROR ))
310 log_err("u-> UTF8 with offsets and flush true did not match.\n");
2ca993e8 311 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 312 expected, sizeof(expected), "UTF8", 0, FALSE, U_ZERO_ERROR ))
313 log_err("u-> UTF8 with offsets and flush true did not match.\n");
314
315 if(!convertToU(expected, sizeof(expected),
2ca993e8 316 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", 0, TRUE, U_ZERO_ERROR ))
73c04bcf 317 log_err("UTF8 -> u did not match.\n");
b75a7d8f 318 if(!convertToU(expected, sizeof(expected),
2ca993e8 319 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", 0, FALSE, U_ZERO_ERROR ))
73c04bcf 320 log_err("UTF8 -> u did not match.\n");
b75a7d8f 321 if(!convertToU(expected, sizeof(expected),
2ca993e8 322 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", fromOffsets, TRUE, U_ZERO_ERROR ))
73c04bcf 323 log_err("UTF8 ->u did not match.\n");
b75a7d8f 324 if(!convertToU(expected, sizeof(expected),
2ca993e8 325 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", fromOffsets, FALSE, U_ZERO_ERROR ))
73c04bcf 326 log_err("UTF8 -> u did not match.\n");
b75a7d8f
A		 327
328 }
b75a7d8f
A		 329}
330
331/*test various error behaviours*/
332static void TestErrorBehaviour(){
333 log_verbose("Testing for SBCS and LATIN_1\n");
334 {
335 static const UChar sampleText[] = { 0x0031, 0xd801};
336 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
374ca955
A		 337 static const uint8_t expected0[] = { 0x31};
338 static const uint8_t expected[] = { 0x31, 0x1a};
b75a7d8f
A		 339 static const uint8_t expected2[] = { 0x31, 0x1a, 0x32};
340
73c04bcf 341#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f 342 /*SBCS*/
2ca993e8 343 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 344 expected, sizeof(expected), "ibm-920", 0, TRUE, U_ZERO_ERROR))
b75a7d8f 345 log_err("u-> ibm-920 [UCNV_SBCS] \n");
2ca993e8 346 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 347 expected0, sizeof(expected0), "ibm-920", 0, FALSE, U_ZERO_ERROR))
b75a7d8f 348 log_err("u-> ibm-920 [UCNV_SBCS] \n");
2ca993e8 349 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 350 expected2, sizeof(expected2), "ibm-920", 0, TRUE, U_ZERO_ERROR))
351 log_err("u-> ibm-920 [UCNV_SBCS] did not match\n");
73c04bcf 352#endif
b75a7d8f
A		 353
354 /*LATIN_1*/
2ca993e8 355 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 356 expected, sizeof(expected), "LATIN_1", 0, TRUE, U_ZERO_ERROR))
b75a7d8f 357 log_err("u-> LATIN_1 is supposed to fail\n");
2ca993e8 358 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 359 expected0, sizeof(expected0), "LATIN_1", 0, FALSE, U_ZERO_ERROR))
b75a7d8f
A		 360 log_err("u-> LATIN_1 is supposed to fail\n");
361
2ca993e8 362 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 363 expected2, sizeof(expected2), "LATIN_1", 0, TRUE, U_ZERO_ERROR))
364 log_err("u-> LATIN_1 did not match\n");
365 }
366
73c04bcf 367#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 368 log_verbose("Testing for DBCS and MBCS\n");
369 {
370 static const UChar sampleText[] = { 0x00a1, 0xd801};
371 static const uint8_t expected[] = { 0xa2, 0xae};
374ca955
A		 372 static const int32_t offsets[] = { 0x00, 0x00};
373 static const uint8_t expectedSUB[] = { 0xa2, 0xae, 0xa1, 0xe0};
374 static const int32_t offsetsSUB[] = { 0x00, 0x00, 0x01, 0x01};
b75a7d8f
A		 375
376 static const UChar sampleText2[] = { 0x00a1, 0xd801, 0x00a4};
377 static const uint8_t expected2[] = { 0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
378 static const int32_t offsets2[] = { 0x00, 0x00, 0x01, 0x01, 0x02, 0x02};
379
380 static const UChar sampleText3MBCS[] = { 0x0001, 0x00a4, 0xdc01};
381 static const uint8_t expected3MBCS[] = { 0x01, 0xa2, 0xb4, 0xa1, 0xe0};
382 static const int32_t offsets3MBCS[] = { 0x00, 0x01, 0x01, 0x02, 0x02};
383
46f4442e 384 static const UChar sampleText4MBCS[] = { 0x0061, 0xFFE4, 0xdc01};
b75a7d8f
A		 385 static const uint8_t expected4MBCS[] = { 0x61, 0x8f, 0xa2, 0xc3, 0xf4, 0xfe};
386 static const int32_t offsets4MBCS[] = { 0x00, 0x01, 0x01, 0x01, 0x02, 0x02 };
387
b75a7d8f 388 /*DBCS*/
2ca993e8 389 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 390 expectedSUB, sizeof(expectedSUB), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
b75a7d8f 391 log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
2ca993e8 392 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
51004dcb 393 expected, sizeof(expected), "ibm-1363", 0, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
b75a7d8f
A		 394 log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
395
2ca993e8 396 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 397 expectedSUB, sizeof(expectedSUB), "ibm-1363", offsetsSUB, TRUE, U_ZERO_ERROR))
b75a7d8f 398 log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
2ca993e8 399 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
51004dcb 400 expected, sizeof(expected), "ibm-1363", offsets, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
b75a7d8f
A		 401 log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
402
403
2ca993e8 404 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 405 expected2, sizeof(expected2), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
406 log_err("u-> ibm-1363 [UCNV_DBCS portion] did not match \n");
2ca993e8 407 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 408 expected2, sizeof(expected2), "ibm-1363", offsets2, TRUE, U_ZERO_ERROR))
409 log_err("u-> ibm-1363 [UCNV_DBCS portion] did not match \n");
410
411 /*MBCS*/
2ca993e8 412 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 413 expectedSUB, sizeof(expectedSUB), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
b75a7d8f 414 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
2ca993e8 415 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
51004dcb 416 expected, sizeof(expected), "ibm-1363", 0, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
b75a7d8f
A		 417 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
418
2ca993e8 419 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 420 expected2, sizeof(expected2), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
421 log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");
2ca993e8 422 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 423 expected2, sizeof(expected2), "ibm-1363", 0, FALSE, U_ZERO_ERROR))
424 log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");
2ca993e8 425 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 426 expected2, sizeof(expected2), "ibm-1363", offsets2, FALSE, U_ZERO_ERROR))
427 log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");
428
2ca993e8 429 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
b75a7d8f
A		 430 expected3MBCS, sizeof(expected3MBCS), "ibm-1363", offsets3MBCS, TRUE, U_ZERO_ERROR))
431 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
2ca993e8 432 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
b75a7d8f
A		 433 expected3MBCS, sizeof(expected3MBCS), "ibm-1363", offsets3MBCS, FALSE, U_ZERO_ERROR))
434 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
435
2ca993e8 436 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
51004dcb 437 expected4MBCS, sizeof(expected4MBCS), "IBM-eucJP", offsets4MBCS, TRUE, U_ZERO_ERROR))
b75a7d8f 438 log_err("u-> euc-jp [UCNV_MBCS] \n");
2ca993e8 439 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
51004dcb 440 expected4MBCS, sizeof(expected4MBCS), "IBM-eucJP", offsets4MBCS, FALSE, U_ZERO_ERROR))
b75a7d8f
A		 441 log_err("u-> euc-jp [UCNV_MBCS] \n");
442 }
73c04bcf 443
b75a7d8f
A		 444 /*iso-2022-jp*/
445 log_verbose("Testing for iso-2022-jp\n");
446 {
447 static const UChar sampleText[] = { 0x0031, 0xd801};
448 static const uint8_t expected[] = { 0x31};
374ca955
A		 449 static const uint8_t expectedSUB[] = { 0x31, 0x1a};
450 static const int32_t offsets[] = { 0x00, 1};
b75a7d8f
A		 451
452 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
453 static const uint8_t expected2[] = { 0x31,0x1A,0x32};
454 static const int32_t offsets2[] = { 0x00,0x01,0x02};
455
b75a7d8f
A		 456 static const UChar sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
457 static const uint8_t expected4MBCS[] = { 0x61, 0x1b, 0x24, 0x42, 0x30, 0x6c,0x1b,0x28,0x42,0x1a};
458 static const int32_t offsets4MBCS[] = { 0x00, 0x01, 0x01 ,0x01, 0x01, 0x01,0x02,0x02,0x02,0x02 };
2ca993e8 459 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 460 expectedSUB, sizeof(expectedSUB), "iso-2022-jp", offsets, TRUE, U_ZERO_ERROR))
b75a7d8f 461 log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
2ca993e8 462 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
51004dcb
A		 463 expected, sizeof(expected), "iso-2022-jp", offsets, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
464 log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
b75a7d8f 465
2ca993e8 466 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 467 expected2, sizeof(expected2), "iso-2022-jp", offsets2, TRUE, U_ZERO_ERROR))
468 log_err("u->iso-2022-jp[UCNV_DBCS] did not match\n");
2ca993e8 469 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 470 expected2, sizeof(expected2), "iso-2022-jp", offsets2, FALSE, U_ZERO_ERROR))
471 log_err("u-> iso-2022-jp [UCNV_DBCS] did not match\n");
2ca993e8 472 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 473 expected2, sizeof(expected2), "iso-2022-jp", offsets2, FALSE, U_ZERO_ERROR))
474 log_err("u-> iso-2022-jp [UCNV_DBCS] did not match\n");
475
2ca993e8 476 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
b75a7d8f
A		 477 expected4MBCS, sizeof(expected4MBCS), "iso-2022-jp", offsets4MBCS, TRUE, U_ZERO_ERROR))
478 log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
2ca993e8 479 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
b75a7d8f
A		 480 expected4MBCS, sizeof(expected4MBCS), "iso-2022-jp", offsets4MBCS, FALSE, U_ZERO_ERROR))
481 log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
482 }
73c04bcf 483
b75a7d8f
A		 484 /*iso-2022-cn*/
485 log_verbose("Testing for iso-2022-cn\n");
486 {
487 static const UChar sampleText[] = { 0x0031, 0xd801};
374ca955
A		 488 static const uint8_t expected[] = { 0x31};
489 static const uint8_t expectedSUB[] = { 0x31, 0x1A};
490 static const int32_t offsets[] = { 0x00, 1};
b75a7d8f
A		 491
492 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
374ca955
A		 493 static const uint8_t expected2[] = { 0x31, 0x1A,0x32};
494 static const int32_t offsets2[] = { 0x00, 0x01,0x02};
b75a7d8f
A		 495
496 static const UChar sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
374ca955
A		 497 static const uint8_t expected3MBCS[] = {0x51, 0x50, 0x1A};
498 static const int32_t offsets3MBCS[] = { 0x00, 0x01, 0x02 };
b75a7d8f
A		 499
500 static const UChar sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
374ca955
A		 501 static const uint8_t expected4MBCS[] = { 0x61, 0x1b, 0x24, 0x29, 0x41, 0x0e, 0x52, 0x3b, 0x0f, 0x1a };
502 static const int32_t offsets4MBCS[] = { 0x00, 0x01, 0x01 ,0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02 };
2ca993e8 503 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 504 expectedSUB, sizeof(expectedSUB), "iso-2022-cn", offsets, TRUE, U_ZERO_ERROR))
b75a7d8f 505 log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
2ca993e8 506 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 507 expected, sizeof(expected), "iso-2022-cn", offsets, FALSE, U_ZERO_ERROR))
508 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
509
2ca993e8 510 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 511 expected2, sizeof(expected2), "iso-2022-cn", offsets2, TRUE, U_ZERO_ERROR))
512 log_err("u->iso-2022-cn[UCNV_DBCS] did not match\n");
2ca993e8 513 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 514 expected2, sizeof(expected2), "iso-2022-cn", offsets2, FALSE, U_ZERO_ERROR))
515 log_err("u-> iso-2022-cn [UCNV_DBCS] did not match\n");
2ca993e8 516 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 517 expected2, sizeof(expected2), "iso-2022-cn", offsets2, FALSE, U_ZERO_ERROR))
518 log_err("u-> iso-2022-cn [UCNV_DBCS] did not match\n");
519
2ca993e8 520 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
b75a7d8f
A		 521 expected3MBCS, sizeof(expected3MBCS), "iso-2022-cn", offsets3MBCS, TRUE, U_ZERO_ERROR))
522 log_err("u->iso-2022-cn [UCNV_MBCS] \n");
2ca993e8 523 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
b75a7d8f
A		 524 expected3MBCS, sizeof(expected3MBCS), "iso-2022-cn", offsets3MBCS, FALSE, U_ZERO_ERROR))
525 log_err("u-> iso-2022-cn[UCNV_MBCS] \n");
526
2ca993e8 527 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
b75a7d8f
A		 528 expected4MBCS, sizeof(expected4MBCS), "iso-2022-cn", offsets4MBCS, TRUE, U_ZERO_ERROR))
529 log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
2ca993e8 530 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
b75a7d8f
A		 531 expected4MBCS, sizeof(expected4MBCS), "iso-2022-cn", offsets4MBCS, FALSE, U_ZERO_ERROR))
532 log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
533 }
73c04bcf 534
b75a7d8f
A		 535 /*iso-2022-kr*/
536 log_verbose("Testing for iso-2022-kr\n");
537 {
538 static const UChar sampleText[] = { 0x0031, 0xd801};
539 static const uint8_t expected[] = { 0x1b, 0x24, 0x29, 0x43, 0x31};
374ca955
A		 540 static const uint8_t expectedSUB[] = { 0x1b, 0x24, 0x29, 0x43, 0x31, 0x1A};
541 static const int32_t offsets[] = { -1, -1, -1, -1, 0x00, 1};
b75a7d8f
A		 542
543 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
544 static const uint8_t expected2[] = { 0x1b, 0x24, 0x29, 0x43, 0x31, 0x1A, 0x32};
545 static const int32_t offsets2[] = { -1, -1, -1, -1, 0x00, 0x01, 0x02};
546
547 static const UChar sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
548 static const uint8_t expected3MBCS[] = { 0x1b, 0x24, 0x29, 0x43, 0x51, 0x50, 0x1A };
549 static const int32_t offsets3MBCS[] = { -1, -1, -1, -1, 0x00, 0x01, 0x02, 0x02 };
550
2ca993e8 551 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 552 expectedSUB, sizeof(expectedSUB), "iso-2022-kr", offsets, TRUE, U_ZERO_ERROR))
b75a7d8f 553 log_err("u-> iso-2022-kr [UCNV_MBCS] \n");
2ca993e8 554 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 555 expected, sizeof(expected), "iso-2022-kr", offsets, FALSE, U_ZERO_ERROR))
556 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
557
2ca993e8 558 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 559 expected2, sizeof(expected2), "iso-2022-kr", offsets2, TRUE, U_ZERO_ERROR))
560 log_err("u->iso-2022-kr[UCNV_DBCS] did not match\n");
2ca993e8 561 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 562 expected2, sizeof(expected2), "iso-2022-kr", offsets2, FALSE, U_ZERO_ERROR))
563 log_err("u-> iso-2022-kr [UCNV_DBCS] did not match\n");
2ca993e8 564 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 565 expected2, sizeof(expected2), "iso-2022-kr", offsets2, FALSE, U_ZERO_ERROR))
566 log_err("u-> iso-2022-kr [UCNV_DBCS] did not match\n");
567
2ca993e8 568 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
b75a7d8f
A		 569 expected3MBCS, sizeof(expected3MBCS), "iso-2022-kr", offsets3MBCS, TRUE, U_ZERO_ERROR))
570 log_err("u->iso-2022-kr [UCNV_MBCS] \n");
2ca993e8 571 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
b75a7d8f
A		 572 expected3MBCS, sizeof(expected3MBCS), "iso-2022-kr", offsets3MBCS, FALSE, U_ZERO_ERROR))
573 log_err("u-> iso-2022-kr[UCNV_MBCS] \n");
b75a7d8f
A		 574 }
575
576 /*HZ*/
577 log_verbose("Testing for HZ\n");
578 {
579 static const UChar sampleText[] = { 0x0031, 0xd801};
580 static const uint8_t expected[] = { 0x7e, 0x7d, 0x31};
374ca955
A		 581 static const uint8_t expectedSUB[] = { 0x7e, 0x7d, 0x31, 0x1A};
582 static const int32_t offsets[] = { 0x00, 0x00, 0x00, 1};
b75a7d8f
A		 583
584 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
585 static const uint8_t expected2[] = { 0x7e, 0x7d, 0x31, 0x1A, 0x32 };
586 static const int32_t offsets2[] = { 0x00, 0x00, 0x00, 0x01, 0x02 };
587
588 static const UChar sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
589 static const uint8_t expected3MBCS[] = { 0x7e, 0x7d, 0x51, 0x50, 0x1A };
590 static const int32_t offsets3MBCS[] = { 0x00, 0x00, 0x00, 0x01, 0x02};
591
592 static const UChar sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
593 static const uint8_t expected4MBCS[] = { 0x7e, 0x7d, 0x61, 0x7e, 0x7b, 0x52, 0x3b, 0x7e, 0x7d, 0x1a };
594 static const int32_t offsets4MBCS[] = { 0x00, 0x00, 0x00, 0x01, 0x01, 0x01 ,0x01, 0x02, 0x02, 0x02 };
2ca993e8 595 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
374ca955 596 expectedSUB, sizeof(expectedSUB), "HZ", offsets, TRUE, U_ZERO_ERROR))
b75a7d8f 597 log_err("u-> HZ [UCNV_MBCS] \n");
2ca993e8 598 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 599 expected, sizeof(expected), "HZ", offsets, FALSE, U_ZERO_ERROR))
600 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
601
2ca993e8 602 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 603 expected2, sizeof(expected2), "HZ", offsets2, TRUE, U_ZERO_ERROR))
604 log_err("u->HZ[UCNV_DBCS] did not match\n");
2ca993e8 605 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 606 expected2, sizeof(expected2), "HZ", offsets2, FALSE, U_ZERO_ERROR))
607 log_err("u-> HZ [UCNV_DBCS] did not match\n");
2ca993e8 608 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
b75a7d8f
A		 609 expected2, sizeof(expected2), "HZ", offsets2, FALSE, U_ZERO_ERROR))
610 log_err("u-> HZ [UCNV_DBCS] did not match\n");
611
2ca993e8 612 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
b75a7d8f
A		 613 expected3MBCS, sizeof(expected3MBCS), "HZ", offsets3MBCS, TRUE, U_ZERO_ERROR))
614 log_err("u->HZ [UCNV_MBCS] \n");
2ca993e8 615 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
b75a7d8f
A		 616 expected3MBCS, sizeof(expected3MBCS), "HZ", offsets3MBCS, FALSE, U_ZERO_ERROR))
617 log_err("u-> HZ[UCNV_MBCS] \n");
618
2ca993e8 619 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
b75a7d8f
A		 620 expected4MBCS, sizeof(expected4MBCS), "HZ", offsets4MBCS, TRUE, U_ZERO_ERROR))
621 log_err("u-> HZ [UCNV_MBCS] \n");
2ca993e8 622 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
b75a7d8f
A		 623 expected4MBCS, sizeof(expected4MBCS), "HZ", offsets4MBCS, FALSE, U_ZERO_ERROR))
624 log_err("u-> HZ [UCNV_MBCS] \n");
625 }
73c04bcf 626#endif
b75a7d8f
A		 627}
628
73c04bcf 629#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 630/*test different convertToUnicode error behaviours*/
631static void TestToUnicodeErrorBehaviour()
632{
633 log_verbose("Testing error conditions for DBCS\n");
634 {
635 uint8_t sampleText[] = { 0xa2, 0xae, 0x03, 0x04};
636 const UChar expected[] = { 0x00a1 };
637
b75a7d8f 638 if(!convertToU(sampleText, sizeof(sampleText),
2ca993e8 639 expected, UPRV_LENGTHOF(expected), "ibm-1363", 0, TRUE, U_AMBIGUOUS_ALIAS_WARNING ))
b75a7d8f
A		 640 log_err("DBCS (ibm-1363)->Unicode did not match.\n");
641 if(!convertToU(sampleText, sizeof(sampleText),
2ca993e8 642 expected, UPRV_LENGTHOF(expected), "ibm-1363", 0, FALSE, U_AMBIGUOUS_ALIAS_WARNING ))
b75a7d8f 643 log_err("DBCS (ibm-1363)->Unicode with flush = false did not match.\n");
b75a7d8f
A		 644 }
645 log_verbose("Testing error conditions for SBCS\n");
646 {
647 uint8_t sampleText[] = { 0xa2, 0xFF};
648 const UChar expected[] = { 0x00c2 };
649
650 /* uint8_t sampleText2[] = { 0xa2, 0x70 };
651 const UChar expected2[] = { 0x0073 };*/
652
653 if(!convertToU(sampleText, sizeof(sampleText),
2ca993e8 654 expected, UPRV_LENGTHOF(expected), "ibm-1051", 0, TRUE, U_ZERO_ERROR ))
b75a7d8f
A		 655 log_err("SBCS (ibm-1051)->Unicode did not match.\n");
656 if(!convertToU(sampleText, sizeof(sampleText),
2ca993e8 657 expected, UPRV_LENGTHOF(expected), "ibm-1051", 0, FALSE, U_ZERO_ERROR ))
b75a7d8f
A		 658 log_err("SBCS (ibm-1051)->Unicode with flush = false did not match.\n");
659
660 }
b75a7d8f
A		 661}
662
663static void TestGetNextErrorBehaviour(){
664 /*Test for unassigned character*/
665#define INPUT_SIZE 1
666 static const char input1[INPUT_SIZE]={ 0x70 };
667 const char* source=(const char*)input1;
668 UErrorCode err=U_ZERO_ERROR;
669 UChar32 c=0;
670 UConverter *cnv=ucnv_open("ibm-424", &err);
671 if(U_FAILURE(err)) {
672 log_data_err("Unable to open a SBCS(ibm-424) converter: %s\n", u_errorName(err));
673 return;
674 }
675 c=ucnv_getNextUChar(cnv, &source, source + INPUT_SIZE, &err);
676 if(err != U_INVALID_CHAR_FOUND && c!=0xfffd){
677 log_err("FAIL in TestGetNextErrorBehaviour(unassigned): Expected: U_INVALID_CHAR_ERROR or 0xfffd ----Got:%s and 0x%lx\n", myErrorName(err), c);
678 }
679 ucnv_close(cnv);
680}
73c04bcf 681#endif
b75a7d8f
A		 682
683#define MAX_UTF16_LEN 2
684#define MAX_UTF8_LEN 4
685
686/*Regression test for utf8 converter*/
687static void TestRegressionUTF8(){
688 UChar32 currCh = 0;
689 int32_t offset8;
690 int32_t offset16;
691 UChar *standardForm = (UChar*)malloc(MAX_LENGTH*sizeof(UChar));
692 uint8_t *utf8 = (uint8_t*)malloc(MAX_LENGTH);
693
694 while (currCh <= UNICODE_LIMIT) {
695 offset16 = 0;
696 offset8 = 0;
697 while(currCh <= UNICODE_LIMIT
698 && offset16 < (MAX_LENGTH/sizeof(UChar) - MAX_UTF16_LEN)
699 && offset8 < (MAX_LENGTH - MAX_UTF8_LEN))
700 {
701 if (currCh == SURROGATE_HIGH_START) {
702 currCh = SURROGATE_LOW_END + 1; /* Skip surrogate range */
703 }
704 UTF16_APPEND_CHAR_SAFE(standardForm, offset16, MAX_LENGTH, currCh);
705 UTF8_APPEND_CHAR_SAFE(utf8, offset8, MAX_LENGTH, currCh);
706 currCh++;
707 }
708 if(!convertFromU(standardForm, offset16,
709 utf8, offset8, "UTF8", 0, TRUE, U_ZERO_ERROR )) {
710 log_err("Unicode->UTF8 did not match.\n");
711 }
712 if(!convertToU(utf8, offset8,
713 standardForm, offset16, "UTF8", 0, TRUE, U_ZERO_ERROR )) {
714 log_err("UTF8->Unicode did not match.\n");
715 }
716 }
73c04bcf 717
b75a7d8f
A		 718 free(standardForm);
719 free(utf8);
73c04bcf
A		 720
721 {
722 static const char src8[] = { (char)0xCC, (char)0x81, (char)0xCC, (char)0x80 };
723 static const UChar expected[] = { 0x0301, 0x0300 };
724 UConverter *conv8;
725 UErrorCode err = U_ZERO_ERROR;
726 UChar pivotBuffer[100];
727 const UChar* const pivEnd = pivotBuffer + 100;
728 const char* srcBeg;
729 const char* srcEnd;
730 UChar* pivBeg;
731
732 conv8 = ucnv_open("UTF-8", &err);
733
734 srcBeg = src8;
735 pivBeg = pivotBuffer;
736 srcEnd = src8 + 3;
737 ucnv_toUnicode(conv8, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, FALSE, &err);
738 if (srcBeg != srcEnd) {
739 log_err("Did not consume whole buffer on first call.\n");
740 }
741
742 srcEnd = src8 + 4;
743 ucnv_toUnicode(conv8, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, TRUE, &err);
744 if (srcBeg != srcEnd) {
745 log_err("Did not consume whole buffer on second call.\n");
746 }
747
748 if (U_FAILURE(err) || (int32_t)(pivBeg - pivotBuffer) != 2 || u_strncmp(pivotBuffer, expected, 2) != 0) {
749 log_err("Did not get expected results for UTF-8.\n");
750 }
751 ucnv_close(conv8);
752 }
b75a7d8f
A		 753}
754
755#define MAX_UTF32_LEN 1
756
757static void TestRegressionUTF32(){
b331163b 758#if !UCONFIG_ONLY_HTML_CONVERSION
b75a7d8f
A		 759 UChar32 currCh = 0;
760 int32_t offset32;
761 int32_t offset16;
762 UChar *standardForm = (UChar*)malloc(MAX_LENGTH*sizeof(UChar));
763 UChar32 *utf32 = (UChar32*)malloc(MAX_LENGTH*sizeof(UChar32));
764
765 while (currCh <= UNICODE_LIMIT) {
766 offset16 = 0;
767 offset32 = 0;
768 while(currCh <= UNICODE_LIMIT
769 && offset16 < (MAX_LENGTH/sizeof(UChar) - MAX_UTF16_LEN)
770 && offset32 < (MAX_LENGTH/sizeof(UChar32) - MAX_UTF32_LEN))
771 {
772 if (currCh == SURROGATE_HIGH_START) {
773 currCh = SURROGATE_LOW_END + 1; /* Skip surrogate range */
774 }
775 UTF16_APPEND_CHAR_SAFE(standardForm, offset16, MAX_LENGTH, currCh);
776 UTF32_APPEND_CHAR_SAFE(utf32, offset32, MAX_LENGTH, currCh);
777 currCh++;
778 }
779 if(!convertFromU(standardForm, offset16,
780 (const uint8_t *)utf32, offset32*sizeof(UChar32), "UTF32_PlatformEndian", 0, TRUE, U_ZERO_ERROR )) {
781 log_err("Unicode->UTF32 did not match.\n");
782 }
783 if(!convertToU((const uint8_t *)utf32, offset32*sizeof(UChar32),
784 standardForm, offset16, "UTF32_PlatformEndian", 0, TRUE, U_ZERO_ERROR )) {
785 log_err("UTF32->Unicode did not match.\n");
786 }
787 }
788 free(standardForm);
789 free(utf32);
73c04bcf
A		 790
791 {
792 /* Check for lone surrogate error handling. */
793 static const UChar sampleBadStartSurrogate[] = { 0x0031, 0xD800, 0x0032 };
794 static const UChar sampleBadEndSurrogate[] = { 0x0031, 0xDC00, 0x0032 };
795 static const uint8_t expectedUTF32BE[] = {
796 0x00, 0x00, 0x00, 0x31,
797 0x00, 0x00, 0xff, 0xfd,
798 0x00, 0x00, 0x00, 0x32
799 };
800 static const uint8_t expectedUTF32LE[] = {
801 0x31, 0x00, 0x00, 0x00,
802 0xfd, 0xff, 0x00, 0x00,
803 0x32, 0x00, 0x00, 0x00
804 };
805 static const int32_t offsetsUTF32[] = {
806 0x00, 0x00, 0x00, 0x00,
807 0x01, 0x01, 0x01, 0x01,
808 0x02, 0x02, 0x02, 0x02
809 };
810
2ca993e8 811 if(!convertFromU(sampleBadStartSurrogate, UPRV_LENGTHOF(sampleBadStartSurrogate),
73c04bcf
A		 812 expectedUTF32BE, sizeof(expectedUTF32BE), "UTF-32BE", offsetsUTF32, TRUE, U_ZERO_ERROR))
813 log_err("u->UTF-32BE\n");
2ca993e8 814 if(!convertFromU(sampleBadEndSurrogate, UPRV_LENGTHOF(sampleBadEndSurrogate),
73c04bcf
A		 815 expectedUTF32BE, sizeof(expectedUTF32BE), "UTF-32BE", offsetsUTF32, TRUE, U_ZERO_ERROR))
816 log_err("u->UTF-32BE\n");
817
2ca993e8 818 if(!convertFromU(sampleBadStartSurrogate, UPRV_LENGTHOF(sampleBadStartSurrogate),
73c04bcf
A		 819 expectedUTF32LE, sizeof(expectedUTF32LE), "UTF-32LE", offsetsUTF32, TRUE, U_ZERO_ERROR))
820 log_err("u->UTF-32LE\n");
2ca993e8 821 if(!convertFromU(sampleBadEndSurrogate, UPRV_LENGTHOF(sampleBadEndSurrogate),
73c04bcf
A		 822 expectedUTF32LE, sizeof(expectedUTF32LE), "UTF-32LE", offsetsUTF32, TRUE, U_ZERO_ERROR))
823 log_err("u->UTF-32LE\n");
824 }
825
826 {
827 static const char srcBE[] = { 0, 0, 0, 0x31, 0, 0, 0, 0x30 };
828 static const UChar expected[] = { 0x0031, 0x0030 };
829 UConverter *convBE;
830 UErrorCode err = U_ZERO_ERROR;
831 UChar pivotBuffer[100];
832 const UChar* const pivEnd = pivotBuffer + 100;
833 const char* srcBeg;
834 const char* srcEnd;
835 UChar* pivBeg;
836
837 convBE = ucnv_open("UTF-32BE", &err);
838
839 srcBeg = srcBE;
840 pivBeg = pivotBuffer;
841 srcEnd = srcBE + 5;
842 ucnv_toUnicode(convBE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, FALSE, &err);
843 if (srcBeg != srcEnd) {
844 log_err("Did not consume whole buffer on first call.\n");
845 }
846
847 srcEnd = srcBE + 8;
848 ucnv_toUnicode(convBE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, TRUE, &err);
849 if (srcBeg != srcEnd) {
850 log_err("Did not consume whole buffer on second call.\n");
851 }
852
853 if (U_FAILURE(err) || (int32_t)(pivBeg - pivotBuffer) != 2 || u_strncmp(pivotBuffer, expected, 2) != 0) {
854 log_err("Did not get expected results for UTF-32BE.\n");
855 }
856 ucnv_close(convBE);
857 }
858 {
859 static const char srcLE[] = { 0x31, 0, 0, 0, 0x30, 0, 0, 0 };
860 static const UChar expected[] = { 0x0031, 0x0030 };
861 UConverter *convLE;
862 UErrorCode err = U_ZERO_ERROR;
863 UChar pivotBuffer[100];
864 const UChar* const pivEnd = pivotBuffer + 100;
865 const char* srcBeg;
866 const char* srcEnd;
867 UChar* pivBeg;
868
869 convLE = ucnv_open("UTF-32LE", &err);
870
871 srcBeg = srcLE;
872 pivBeg = pivotBuffer;
873 srcEnd = srcLE + 5;
874 ucnv_toUnicode(convLE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, FALSE, &err);
875 if (srcBeg != srcEnd) {
876 log_err("Did not consume whole buffer on first call.\n");
877 }
878
879 srcEnd = srcLE + 8;
880 ucnv_toUnicode(convLE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, TRUE, &err);
881 if (srcBeg != srcEnd) {
882 log_err("Did not consume whole buffer on second call.\n");
883 }
884
885 if (U_FAILURE(err) || (int32_t)(pivBeg - pivotBuffer) != 2 || u_strncmp(pivotBuffer, expected, 2) != 0) {
886 log_err("Did not get expected results for UTF-32LE.\n");
887 }
888 ucnv_close(convLE);
889 }
b331163b 890#endif
b75a7d8f
A		 891}
892
893/*Walk through the available converters*/
894static void TestAvailableConverters(){
895 UErrorCode status=U_ZERO_ERROR;
896 UConverter *conv=NULL;
897 int32_t i=0;
898 for(i=0; i < ucnv_countAvailable(); i++){
899 status=U_ZERO_ERROR;
900 conv=ucnv_open(ucnv_getAvailableName(i), &status);
901 if(U_FAILURE(status)){
902 log_err("ERROR: converter creation failed. Failure in alias table or the data table for \n converter=%s. Error=%s\n",
903 ucnv_getAvailableName(i), myErrorName(status));
904 continue;
905 }
906 ucnv_close(conv);
907 }
908
909}
910
911static void TestFlushInternalBuffer(){
912 TestWithBufferSize(MAX_LENGTH, 1);
913 TestWithBufferSize(1, 1);
914 TestWithBufferSize(1, MAX_LENGTH);
915 TestWithBufferSize(MAX_LENGTH, MAX_LENGTH);
916}
917
918static void TestWithBufferSize(int32_t insize, int32_t outsize){
919
920 gInBufferSize =insize;
921 gOutBufferSize = outsize;
922
923 log_verbose("Testing fromUnicode for UTF-8 with UCNV_TO_U_CALLBACK_SUBSTITUTE \n");
924 {
925 UChar sampleText[] =
926 { 0x0031, 0x0032, 0x0033, 0x0000, 0x4e00, 0x4e8c, 0x4e09, 0x002E };
927 const uint8_t expectedUTF8[] =
928 { 0x31, 0x32, 0x33, 0x00, 0xe4, 0xb8, 0x80, 0xe4, 0xba, 0x8c, 0xe4, 0xb8, 0x89, 0x2E };
929 int32_t toUTF8Offs[] =
930 { 0x00, 0x01, 0x02, 0x03, 0x04, 0x04, 0x04, 0x05, 0x05, 0x05, 0x06, 0x06, 0x06, 0x07};
931 /* int32_t fmUTF8Offs[] =
932 { 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0007, 0x000a, 0x000d };*/
933
934 /*UTF-8*/
2ca993e8 935 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 936 expectedUTF8, sizeof(expectedUTF8), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE, toUTF8Offs ,FALSE))
937 log_err("u-> UTF8 did not match.\n");
938 }
939
73c04bcf 940#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 941 log_verbose("Testing fromUnicode with UCNV_FROM_U_CALLBACK_ESCAPE \n");
942 {
943 UChar inputTest[] = { 0x0061, 0xd801, 0xdc01, 0xd801, 0x0061 };
944 const uint8_t toIBM943[]= { 0x61,
945 0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
946 0x25, 0x55, 0x44, 0x43, 0x30, 0x31,
947 0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
948 0x61 };
949 int32_t offset[]= {0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 4};
950
2ca993e8 951 if(!testConvertFromU(inputTest, UPRV_LENGTHOF(inputTest),
b75a7d8f
A		 952 toIBM943, sizeof(toIBM943), "ibm-943",
953 (UConverterFromUCallback)UCNV_FROM_U_CALLBACK_ESCAPE, offset,FALSE))
954 log_err("u-> ibm-943 with subst with value did not match.\n");
955 }
73c04bcf 956#endif
b75a7d8f
A		 957
958 log_verbose("Testing fromUnicode for UTF-8 with UCNV_TO_U_CALLBACK_SUBSTITUTE \n");
959 {
960 const uint8_t sampleText1[] = { 0x31, 0xe4, 0xba, 0x8c,
961 0xe0, 0x80, 0x61};
962 UChar expected1[] = { 0x0031, 0x4e8c, 0xfffd, 0x0061};
963 int32_t offsets1[] = { 0x0000, 0x0001, 0x0004, 0x0006};
964
965 if(!testConvertToU(sampleText1, sizeof(sampleText1),
2ca993e8 966 expected1, UPRV_LENGTHOF(expected1),"utf8", UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets1,FALSE))
b75a7d8f
A		 967 log_err("utf8->u with substitute did not match.\n");;
968 }
969
73c04bcf 970#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 971 log_verbose("Testing toUnicode with UCNV_TO_U_CALLBACK_ESCAPE \n");
972 /*to Unicode*/
973 {
974 const uint8_t sampleTxtToU[]= { 0x00, 0x9f, 0xaf,
975 0x81, 0xad, /*unassigned*/
976 0x89, 0xd3 };
977 UChar IBM_943toUnicode[] = { 0x0000, 0x6D63,
978 0x25, 0x58, 0x38, 0x31, 0x25, 0x58, 0x41, 0x44,
979 0x7B87};
980 int32_t fromIBM943Offs [] = { 0, 1, 3, 3, 3, 3, 3, 3, 3, 3, 5};
981
982 if(!testConvertToU(sampleTxtToU, sizeof(sampleTxtToU),
2ca993e8 983 IBM_943toUnicode, UPRV_LENGTHOF(IBM_943toUnicode),"ibm-943",
b75a7d8f
A		 984 (UConverterToUCallback)UCNV_TO_U_CALLBACK_ESCAPE, fromIBM943Offs,FALSE))
985 log_err("ibm-943->u with substitute with value did not match.\n");
986
987 }
73c04bcf 988#endif
b75a7d8f
A		 989}
990
991static UBool convertFromU( const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
992 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus)
993{
994
995 int32_t i=0;
73c04bcf 996 char *p=0;
b75a7d8f 997 const UChar *src;
73c04bcf 998 char buffer[MAX_LENGTH];
b75a7d8f
A		 999 int32_t offsetBuffer[MAX_LENGTH];
1000 int32_t *offs=0;
73c04bcf
A		 1001 char *targ;
1002 char *targetLimit;
b75a7d8f
A		 1003 UChar *sourceLimit=0;
1004 UErrorCode status = U_ZERO_ERROR;
1005 UConverter *conv = 0;
1006 conv = ucnv_open(codepage, &status);
1007 if(U_FAILURE(status))
1008 {
1009 log_data_err("Couldn't open converter %s\n",codepage);
1010 return TRUE;
1011 }
1012 log_verbose("Converter %s opened..\n", ucnv_getName(conv, &status));
1013
1014 for(i=0; i<MAX_LENGTH; i++){
73c04bcf 1015 buffer[i]=(char)0xF0;
b75a7d8f
A		 1016 offsetBuffer[i]=0xFF;
1017 }
1018
1019 src=source;
1020 sourceLimit=(UChar*)src+(sourceLen);
1021 targ=buffer;
1022 targetLimit=targ+MAX_LENGTH;
1023 offs=offsetBuffer;
1024 ucnv_fromUnicode (conv,
1025 (char **)&targ,
1026 (const char *)targetLimit,
1027 &src,
1028 sourceLimit,
1029 expectOffsets ? offs : NULL,
1030 doFlush,
1031 &status);
1032 ucnv_close(conv);
1033 if(status != expectedStatus){
1034 log_err("ucnv_fromUnicode() failed for codepage=%s. Error =%s Expected=%s\n", codepage, myErrorName(status), myErrorName(expectedStatus));
1035 return FALSE;
1036 }
1037
1038 log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
1039 sourceLen, targ-buffer);
1040
1041 if(expectLen != targ-buffer)
1042 {
1043 log_err("Expected %d chars out, got %d FROM Unicode to %s\n", expectLen, targ-buffer, codepage);
1044 log_verbose("Expected %d chars out, got %d FROM Unicode to %s\n", expectLen, targ-buffer, codepage);
73c04bcf 1045 printSeqErr((const unsigned char *)buffer, (int32_t)(targ-buffer));
b75a7d8f
A		 1046 printSeqErr((const unsigned char*)expect, expectLen);
1047 return FALSE;
1048 }
1049
1050 if(memcmp(buffer, expect, expectLen)){
1051 log_err("String does not match. FROM Unicode to codePage%s\n", codepage);
374ca955 1052 log_info("\nGot:");
b75a7d8f 1053 printSeqErr((const unsigned char *)buffer, expectLen);
374ca955 1054 log_info("\nExpected:");
b75a7d8f
A		 1055 printSeqErr((const unsigned char *)expect, expectLen);
1056 return FALSE;
1057 }
1058 else {
1059 log_verbose("Matches!\n");
1060 }
1061
1062 if (expectOffsets != 0){
1063 log_verbose("comparing %d offsets..\n", targ-buffer);
1064 if(memcmp(offsetBuffer,expectOffsets,(targ-buffer) * sizeof(int32_t) )){
1065 log_err("did not get the expected offsets. for FROM Unicode to %s\n", codepage);
374ca955 1066 log_info("\nGot : ");
73c04bcf 1067 printSeqErr((const unsigned char*)buffer, (int32_t)(targ-buffer));
b75a7d8f 1068 for(p=buffer;p<targ;p++)
374ca955
A		 1069 log_info("%d, ", offsetBuffer[p-buffer]);
1070 log_info("\nExpected: ");
b75a7d8f 1071 for(i=0; i< (targ-buffer); i++)
374ca955 1072 log_info("%d,", expectOffsets[i]);
b75a7d8f
A		 1073 }
1074 }
1075
1076 return TRUE;
1077}
1078
1079
1080static UBool convertToU( const uint8_t *source, int sourceLen, const UChar *expect, int expectLen,
1081 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus)
1082{
1083 UErrorCode status = U_ZERO_ERROR;
1084 UConverter *conv = 0;
1085 int32_t i=0;
1086 UChar *p=0;
73c04bcf 1087 const char* src;
b75a7d8f
A		 1088 UChar buffer[MAX_LENGTH];
1089 int32_t offsetBuffer[MAX_LENGTH];
1090 int32_t *offs=0;
1091 UChar *targ;
1092 UChar *targetLimit;
1093 uint8_t *sourceLimit=0;
1094
1095
1096
1097 conv = ucnv_open(codepage, &status);
1098 if(U_FAILURE(status))
1099 {
1100 log_data_err("Couldn't open converter %s\n",codepage);
1101 return TRUE;
1102 }
1103 log_verbose("Converter %s opened..\n", ucnv_getName(conv, &status));
1104
1105
1106
1107 for(i=0; i<MAX_LENGTH; i++){
1108 buffer[i]=0xFFFE;
1109 offsetBuffer[i]=-1;
1110 }
1111
73c04bcf 1112 src=(const char *)source;
b75a7d8f
A		 1113 sourceLimit=(uint8_t*)(src+(sourceLen));
1114 targ=buffer;
1115 targetLimit=targ+MAX_LENGTH;
1116 offs=offsetBuffer;
1117
1118
1119
1120 ucnv_toUnicode (conv,
1121 &targ,
1122 targetLimit,
1123 (const char **)&src,
1124 (const char *)sourceLimit,
1125 expectOffsets ? offs : NULL,
1126 doFlush,
1127 &status);
1128
1129 ucnv_close(conv);
1130 if(status != expectedStatus){
1131 log_err("ucnv_fromUnicode() failed for codepage=%s. Error =%s Expected=%s\n", codepage, myErrorName(status), myErrorName(expectedStatus));
1132 return FALSE;
1133 }
1134 log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
1135 sourceLen, targ-buffer);
1136
1137
1138
1139
1140 log_verbose("comparing %d uchars (%d bytes)..\n",expectLen,expectLen*2);
1141
1142 if (expectOffsets != 0) {
1143 if(memcmp(offsetBuffer, expectOffsets, (targ-buffer) * sizeof(int32_t))){
1144
1145 log_err("did not get the expected offsets from %s To UNICODE\n", codepage);
374ca955 1146 log_info("\nGot : ");
b75a7d8f 1147 for(p=buffer;p<targ;p++)
374ca955
A		 1148 log_info("%d, ", offsetBuffer[p-buffer]);
1149 log_info("\nExpected: ");
b75a7d8f 1150 for(i=0; i<(targ-buffer); i++)
374ca955
A		 1151 log_info("%d, ", expectOffsets[i]);
1152 log_info("\nGot result:");
b75a7d8f 1153 for(i=0; i<(targ-buffer); i++)
374ca955
A		 1154 log_info("0x%04X,", buffer[i]);
1155 log_info("\nFrom Input:");
73c04bcf 1156 for(i=0; i<(src-(const char *)source); i++)
374ca955
A		 1157 log_info("0x%02X,", (unsigned char)source[i]);
1158 log_info("\n");
b75a7d8f
A		 1159 }
1160 }
1161 if(memcmp(buffer, expect, expectLen*2)){
1162 log_err("String does not match. from codePage %s TO Unicode\n", codepage);
374ca955 1163 log_info("\nGot:");
b75a7d8f 1164 printUSeqErr(buffer, expectLen);
374ca955 1165 log_info("\nExpected:");
b75a7d8f
A		 1166 printUSeqErr(expect, expectLen);
1167 return FALSE;
1168 }
1169 else {
1170 log_verbose("Matches!\n");
1171 }
1172
1173 return TRUE;
1174}
1175
1176
1177static UBool testConvertFromU( const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
1178 const char *codepage, UConverterFromUCallback callback , const int32_t *expectOffsets, UBool testReset)
1179{
1180 UErrorCode status = U_ZERO_ERROR;
1181 UConverter *conv = 0;
73c04bcf 1182 char junkout[MAX_LENGTH]; /* FIX */
b75a7d8f 1183 int32_t junokout[MAX_LENGTH]; /* FIX */
73c04bcf 1184 char *p;
b75a7d8f 1185 const UChar *src;
73c04bcf
A		 1186 char *end;
1187 char *targ;
b75a7d8f
A		 1188 int32_t *offs;
1189 int i;
1190 int32_t realBufferSize;
73c04bcf 1191 char *realBufferEnd;
b75a7d8f
A		 1192 const UChar *realSourceEnd;
1193 const UChar *sourceLimit;
1194 UBool checkOffsets = TRUE;
1195 UBool doFlush;
1196
1197 UConverterFromUCallback oldAction = NULL;
1198 const void* oldContext = NULL;
1199
1200 for(i=0;i<MAX_LENGTH;i++)
73c04bcf 1201 junkout[i] = (char)0xF0;
b75a7d8f
A		 1202 for(i=0;i<MAX_LENGTH;i++)
1203 junokout[i] = 0xFF;
1204
1205 setNuConvTestName(codepage, "FROM");
1206
1207 log_verbose("\n========= %s\n", gNuConvTestName);
1208
1209 conv = ucnv_open(codepage, &status);
1210 if(U_FAILURE(status))
1211 {
1212 log_data_err("Couldn't open converter %s\n",codepage);
1213 return TRUE;
1214 }
1215
1216 log_verbose("Converter opened..\n");
1217 /*----setting the callback routine----*/
1218 ucnv_setFromUCallBack (conv, callback, NULL, &oldAction, &oldContext, &status);
1219 if (U_FAILURE(status)) {
1220 log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));
1221 }
1222 /*------------------------*/
1223
1224 src = source;
1225 targ = junkout;
1226 offs = junokout;
1227
2ca993e8 1228 realBufferSize = UPRV_LENGTHOF(junkout);
b75a7d8f
A		 1229 realBufferEnd = junkout + realBufferSize;
1230 realSourceEnd = source + sourceLen;
1231
1232 if ( gOutBufferSize != realBufferSize )
1233 checkOffsets = FALSE;
1234
1235 if( gInBufferSize != MAX_LENGTH )
1236 checkOffsets = FALSE;
1237
1238 do
73c04bcf 1239 {
b75a7d8f
A		 1240 end = nct_min(targ + gOutBufferSize, realBufferEnd);
1241 sourceLimit = nct_min(src + gInBufferSize, realSourceEnd);
1242
1243 doFlush = (UBool)(sourceLimit == realSourceEnd);
1244
1245 if(targ == realBufferEnd)
1246 {
1247 log_err("Error, overflowed the real buffer while about to call fromUnicode! targ=%08lx %s", targ, gNuConvTestName);
1248 return FALSE;
1249 }
1250 log_verbose("calling fromUnicode @ SOURCE:%08lx to %08lx TARGET: %08lx to %08lx, flush=%s\n", src,sourceLimit, targ,end, doFlush?"TRUE":"FALSE");
1251
1252
1253 status = U_ZERO_ERROR;
1254 if(gInBufferSize ==999 && gOutBufferSize==999)
1255 doFlush = FALSE;
1256 ucnv_fromUnicode (conv,
1257 (char **)&targ,
1258 (const char *)end,
1259 &src,
1260 sourceLimit,
1261 offs,
1262 doFlush, /* flush if we're at the end of the input data */
1263 &status);
1264 if(testReset)
1265 ucnv_resetToUnicode(conv);
1266 if(gInBufferSize ==999 && gOutBufferSize==999)
1267 ucnv_resetToUnicode(conv);
1268
1269 } while ( (status == U_BUFFER_OVERFLOW_ERROR) || (U_SUCCESS(status) && sourceLimit < realSourceEnd) );
1270
1271 if(U_FAILURE(status)) {
1272 log_err("Problem doing fromUnicode to %s, errcode %s %s\n", codepage, myErrorName(status), gNuConvTestName);
1273 return FALSE;
1274 }
1275
1276 log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
1277 sourceLen, targ-junkout);
729e4ab9 1278 if(getTestOption(VERBOSITY_OPTION))
b75a7d8f
A		 1279 {
1280 char junk[999];
1281 char offset_str[999];
73c04bcf 1282 char *ptr;
b75a7d8f
A		 1283
1284 junk[0] = 0;
1285 offset_str[0] = 0;
1286 for(ptr = junkout;ptr<targ;ptr++)
1287 {
1288 sprintf(junk + strlen(junk), "0x%02x, ", (0xFF) & (unsigned int)*ptr);
1289 sprintf(offset_str + strlen(offset_str), "0x%02x, ", (0xFF) & (unsigned int)junokout[ptr-junkout]);
1290 }
1291
1292 log_verbose(junk);
1293 printSeq((const unsigned char *)expect, expectLen);
1294 if ( checkOffsets )
1295 {
1296 log_verbose("\nOffsets:");
1297 log_verbose(offset_str);
1298 }
1299 log_verbose("\n");
1300 }
1301 ucnv_close(conv);
1302
1303
1304 if(expectLen != targ-junkout)
1305 {
1306 log_err("Expected %d chars out, got %d %s\n", expectLen, targ-junkout, gNuConvTestName);
1307 log_verbose("Expected %d chars out, got %d %s\n", expectLen, targ-junkout, gNuConvTestName);
374ca955 1308 log_info("\nGot:");
73c04bcf 1309 printSeqErr((const unsigned char*)junkout, (int32_t)(targ-junkout));
374ca955 1310 log_info("\nExpected:");
b75a7d8f
A		 1311 printSeqErr((const unsigned char*)expect, expectLen);
1312 return FALSE;
1313 }
1314
1315 if (checkOffsets && (expectOffsets != 0) )
1316 {
1317 log_verbose("comparing %d offsets..\n", targ-junkout);
1318 if(memcmp(junokout,expectOffsets,(targ-junkout) * sizeof(int32_t) )){
1319 log_err("did not get the expected offsets. %s", gNuConvTestName);
1320 log_err("Got : ");
73c04bcf 1321 printSeqErr((const unsigned char*)junkout, (int32_t)(targ-junkout));
b75a7d8f
A		 1322 for(p=junkout;p<targ;p++)
1323 log_err("%d, ", junokout[p-junkout]);
1324 log_err("\nExpected: ");
1325 for(i=0; i<(targ-junkout); i++)
1326 log_err("%d,", expectOffsets[i]);
1327 }
1328 }
1329
1330 log_verbose("comparing..\n");
1331 if(!memcmp(junkout, expect, expectLen))
1332 {
1333 log_verbose("Matches!\n");
1334 return TRUE;
1335 }
1336 else
1337 {
1338 log_err("String does not match. %s\n", gNuConvTestName);
1339 printUSeqErr(source, sourceLen);
374ca955 1340 log_info("\nGot:");
b75a7d8f 1341 printSeqErr((const unsigned char *)junkout, expectLen);
374ca955 1342 log_info("\nExpected:");
b75a7d8f
A		 1343 printSeqErr((const unsigned char *)expect, expectLen);
1344
1345 return FALSE;
1346 }
1347}
1348
1349static UBool testConvertToU( const uint8_t *source, int sourcelen, const UChar *expect, int expectlen,
1350 const char *codepage, UConverterToUCallback callback, const int32_t *expectOffsets, UBool testReset)
1351{
1352 UErrorCode status = U_ZERO_ERROR;
1353 UConverter *conv = 0;
1354 UChar junkout[MAX_LENGTH]; /* FIX */
1355 int32_t junokout[MAX_LENGTH]; /* FIX */
73c04bcf
A		 1356 const char *src;
1357 const char *realSourceEnd;
1358 const char *srcLimit;
b75a7d8f
A		 1359 UChar *p;
1360 UChar *targ;
1361 UChar *end;
1362 int32_t *offs;
1363 int i;
1364 UBool checkOffsets = TRUE;
1365 int32_t realBufferSize;
1366 UChar *realBufferEnd;
1367 UBool doFlush;
1368
1369 UConverterToUCallback oldAction = NULL;
1370 const void* oldContext = NULL;
1371
1372
1373 for(i=0;i<MAX_LENGTH;i++)
1374 junkout[i] = 0xFFFE;
1375
1376 for(i=0;i<MAX_LENGTH;i++)
1377 junokout[i] = -1;
1378
1379 setNuConvTestName(codepage, "TO");
1380
1381 log_verbose("\n========= %s\n", gNuConvTestName);
1382
1383 conv = ucnv_open(codepage, &status);
1384 if(U_FAILURE(status))
1385 {
1386 log_data_err("Couldn't open converter %s\n",gNuConvTestName);
1387 return TRUE;
1388 }
1389
1390 log_verbose("Converter opened..\n");
1391 /*----setting the callback routine----*/
1392 ucnv_setToUCallBack (conv, callback, NULL, &oldAction, &oldContext, &status);
1393 if (U_FAILURE(status)) {
1394 log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));
1395 }
1396 /*-------------------------------------*/
1397
73c04bcf 1398 src = (const char *)source;
b75a7d8f
A		 1399 targ = junkout;
1400 offs = junokout;
1401
2ca993e8 1402 realBufferSize = UPRV_LENGTHOF(junkout);
b75a7d8f
A		 1403 realBufferEnd = junkout + realBufferSize;
1404 realSourceEnd = src + sourcelen;
1405
1406 if ( gOutBufferSize != realBufferSize )
1407 checkOffsets = FALSE;
1408
1409 if( gInBufferSize != MAX_LENGTH )
1410 checkOffsets = FALSE;
1411
1412 do
1413 {
1414 end = nct_min( targ + gOutBufferSize, realBufferEnd);
1415 srcLimit = nct_min(realSourceEnd, src + gInBufferSize);
1416
1417 if(targ == realBufferEnd)
1418 {
1419 log_err("Error, the end would overflow the real output buffer while about to call toUnicode! tarjey=%08lx %s",targ,gNuConvTestName);
1420 return FALSE;
1421 }
1422 log_verbose("calling toUnicode @ %08lx to %08lx\n", targ,end);
1423
1424 /* oldTarg = targ; */
1425
1426 status = U_ZERO_ERROR;
1427 doFlush=(UBool)((gInBufferSize ==999 && gOutBufferSize==999)?(srcLimit == realSourceEnd) : FALSE);
1428
1429 ucnv_toUnicode (conv,
1430 &targ,
1431 end,
1432 (const char **)&src,
1433 (const char *)srcLimit,
1434 offs,
1435 doFlush, /* flush if we're at the end of hte source data */
1436 &status);
1437 if(testReset)
1438 ucnv_resetFromUnicode(conv);
1439 if(gInBufferSize ==999 && gOutBufferSize==999)
1440 ucnv_resetToUnicode(conv);
1441 /* offs += (targ-oldTarg); */
1442
1443 } while ( (status == U_BUFFER_OVERFLOW_ERROR) || (U_SUCCESS(status) && (srcLimit < realSourceEnd)) ); /* while we just need another buffer */
1444
1445 if(U_FAILURE(status))
1446 {
1447 log_err("Problem doing %s toUnicode, errcode %s %s\n", codepage, myErrorName(status), gNuConvTestName);
1448 return FALSE;
1449 }
1450
1451 log_verbose("\nConversion done. %d bytes -> %d chars.\nResult :",
1452 sourcelen, targ-junkout);
729e4ab9 1453 if(getTestOption(VERBOSITY_OPTION))
b75a7d8f
A		 1454 {
1455 char junk[999];
1456 char offset_str[999];
1457
1458 UChar *ptr;
1459
1460 junk[0] = 0;
1461 offset_str[0] = 0;
1462
1463 for(ptr = junkout;ptr<targ;ptr++)
1464 {
1465 sprintf(junk + strlen(junk), "0x%04x, ", (0xFFFF) & (unsigned int)*ptr);
1466 sprintf(offset_str + strlen(offset_str), "0x%04x, ", (0xFFFF) & (unsigned int)junokout[ptr-junkout]);
1467 }
1468
1469 log_verbose(junk);
1470
1471 if ( checkOffsets )
1472 {
1473 log_verbose("\nOffsets:");
1474 log_verbose(offset_str);
1475 }
1476 log_verbose("\n");
1477 }
1478 ucnv_close(conv);
1479
1480 log_verbose("comparing %d uchars (%d bytes)..\n",expectlen,expectlen*2);
1481
1482 if (checkOffsets && (expectOffsets != 0))
1483 {
1484 if(memcmp(junokout,expectOffsets,(targ-junkout) * sizeof(int32_t))){
1485
1486 log_err("did not get the expected offsets. %s",gNuConvTestName);
1487 for(p=junkout;p<targ;p++)
1488 log_err("%d, ", junokout[p-junkout]);
1489 log_err("\nExpected: ");
1490 for(i=0; i<(targ-junkout); i++)
1491 log_err("%d,", expectOffsets[i]);
1492 log_err("");
1493 for(i=0; i<(targ-junkout); i++)
1494 log_err("%X,", junkout[i]);
1495 log_err("");
73c04bcf 1496 for(i=0; i<(src-(const char *)source); i++)
b75a7d8f
A		 1497 log_err("%X,", (unsigned char)source[i]);
1498 }
1499 }
1500
1501 if(!memcmp(junkout, expect, expectlen*2))
1502 {
1503 log_verbose("Matches!\n");
1504 return TRUE;
1505 }
1506 else
1507 {
1508 log_err("String does not match. %s\n", gNuConvTestName);
1509 log_verbose("String does not match. %s\n", gNuConvTestName);
374ca955 1510 log_info("\nGot:");
b75a7d8f 1511 printUSeq(junkout, expectlen);
374ca955 1512 log_info("\nExpected:");
b75a7d8f
A		 1513 printUSeq(expect, expectlen);
1514 return FALSE;
1515 }
1516}
1517
1518
1519static void TestResetBehaviour(void){
73c04bcf 1520#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 1521 log_verbose("Testing Reset for DBCS and MBCS\n");
1522 {
1523 static const UChar sampleText[] = {0x00a1, 0xd801, 0xdc01, 0x00a4};
1524 static const uint8_t expected[] = {0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
1525 static const int32_t offsets[] = {0x00, 0x00, 0x01, 0x01, 0x03, 0x03 };
1526
1527
1528 static const UChar sampleText1[] = {0x00a1, 0x00a4, 0x00a7, 0x00a8};
1529 static const uint8_t expected1[] = {0xa2, 0xae,0xA2,0xB4,0xA1,0xD7,0xA1,0xA7};
1530 static const int32_t offsets1[] = { 0,2,4,6};
1531
1532 /*DBCS*/
2ca993e8 1533 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1534 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1535 log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
2ca993e8 1536 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1537 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1538 log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
1539
1540 if(!testConvertToU(expected1, sizeof(expected1),
2ca993e8 1541 sampleText1, UPRV_LENGTHOF(sampleText1), "ibm-1363",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
b75a7d8f
A		 1542 offsets1, TRUE))
1543 log_err("ibm-1363 -> did not match.\n");
1544 /*MBCS*/
2ca993e8 1545 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1546 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1547 log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
2ca993e8 1548 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1549 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1550 log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
1551
1552 if(!testConvertToU(expected1, sizeof(expected1),
2ca993e8 1553 sampleText1, UPRV_LENGTHOF(sampleText1), "ibm-1363",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
b75a7d8f
A		 1554 offsets1, TRUE))
1555 log_err("ibm-1363 -> did not match.\n");
1556
1557 }
73c04bcf 1558
b75a7d8f
A		 1559 log_verbose("Testing Reset for ISO-2022-jp\n");
1560 {
1561 static const UChar sampleText[] = { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1562
1563 static const uint8_t expected[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
1564 0x31,0x1A, 0x32};
1565
1566
1567 static const int32_t offsets[] = {0,0,0,0,0,1,1,2,2,2,2,3,5 };
1568
1569
1570 static const UChar sampleText1[] = {0x4e00, 0x04e01, 0x0031,0x001A, 0x0032};
1571 static const uint8_t expected1[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
1572 0x31,0x1A, 0x32};
1573 static const int32_t offsets1[] = { 3,5,10,11,12};
1574
1575 /*iso-2022-jp*/
2ca993e8 1576 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1577 expected, sizeof(expected), "iso-2022-jp", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1578 log_err("u-> not match.\n");
2ca993e8 1579 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1580 expected, sizeof(expected), "iso-2022-jp", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1581 log_err("u-> not match.\n");
1582
1583 if(!testConvertToU(expected1, sizeof(expected1),
2ca993e8 1584 sampleText1, UPRV_LENGTHOF(sampleText1), "iso-2022-jp",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
b75a7d8f
A		 1585 offsets1, TRUE))
1586 log_err("iso-2022-jp -> did not match.\n");
1587
1588 }
73c04bcf 1589
b75a7d8f
A		 1590 log_verbose("Testing Reset for ISO-2022-cn\n");
1591 {
1592 static const UChar sampleText[] = { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1593
1594 static const uint8_t expected[] = {
1595 0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B,
1596 0x36, 0x21,
1597 0x0f, 0x31,
1598 0x1A,
374ca955 1599 0x32
b75a7d8f
A		 1600 };
1601
1602
1603 static const int32_t offsets[] = {
1604 0, 0, 0, 0, 0, 0, 0,
1605 1, 1,
1606 2, 2,
1607 3,
374ca955 1608 5, };
b75a7d8f
A		 1609
1610 UChar sampleText1[] = {0x4e00, 0x04e01, 0x0031,0x001A, 0x0032};
1611 static const uint8_t expected1[] = {
1612 0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B,
1613 0x36, 0x21,
374ca955 1614 0x1B, 0x24, 0x29, 0x47, 0x24, 0x22,
b75a7d8f
A		 1615 0x0f, 0x1A,
1616 0x32
1617 };
374ca955 1618 static const int32_t offsets1[] = { 5,7,13,16,17};
b75a7d8f
A		 1619
1620 /*iso-2022-CN*/
2ca993e8 1621 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1622 expected, sizeof(expected), "iso-2022-cn", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1623 log_err("u-> not match.\n");
2ca993e8 1624 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1625 expected, sizeof(expected), "iso-2022-cn", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1626 log_err("u-> not match.\n");
1627
1628 if(!testConvertToU(expected1, sizeof(expected1),
2ca993e8 1629 sampleText1, UPRV_LENGTHOF(sampleText1), "iso-2022-cn",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
b75a7d8f
A		 1630 offsets1, TRUE))
1631 log_err("iso-2022-cn -> did not match.\n");
1632 }
73c04bcf 1633
b75a7d8f
A		 1634 log_verbose("Testing Reset for ISO-2022-kr\n");
1635 {
1636 UChar sampleText[] = { 0x4e00,0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1637
1638 static const uint8_t expected[] = {0x1B, 0x24, 0x29, 0x43,
1639 0x0E, 0x6C, 0x69,
1640 0x0f, 0x1A,
1641 0x0e, 0x6F, 0x4B,
1642 0x0F, 0x31,
1643 0x1A,
1644 0x32 };
1645
1646 static const int32_t offsets[] = {-1, -1, -1, -1,
1647 0, 0, 0,
1648 1, 1,
1649 3, 3, 3,
1650 4, 4,
1651 5,
1652 7,
1653 };
1654 static const UChar sampleText1[] = { 0x4e00,0x0041, 0x04e01, 0x0031, 0x0042, 0x0032};
1655
1656 static const uint8_t expected1[] = {0x1B, 0x24, 0x29, 0x43,
1657 0x0E, 0x6C, 0x69,
1658 0x0f, 0x41,
1659 0x0e, 0x6F, 0x4B,
1660 0x0F, 0x31,
1661 0x42,
1662 0x32 };
1663
1664 static const int32_t offsets1[] = {
1665 5, 8, 10,
1666 13, 14, 15
1667
1668 };
1669 /*iso-2022-kr*/
2ca993e8 1670 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1671 expected, sizeof(expected), "iso-2022-kr", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1672 log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
2ca993e8 1673 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1674 expected, sizeof(expected), "iso-2022-kr", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1675 log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
1676 if(!testConvertToU(expected1, sizeof(expected1),
2ca993e8 1677 sampleText1, UPRV_LENGTHOF(sampleText1), "iso-2022-kr",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
b75a7d8f
A		 1678 offsets1, TRUE))
1679 log_err("iso-2022-kr -> did not match.\n");
1680 }
73c04bcf 1681
b75a7d8f
A		 1682 log_verbose("Testing Reset for HZ\n");
1683 {
1684 static const UChar sampleText[] = { 0x4e00, 0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1685
1686 static const uint8_t expected[] = {0x7E, 0x7B, 0x52, 0x3B,
1687 0x7E, 0x7D, 0x1A,
1688 0x7E, 0x7B, 0x36, 0x21,
1689 0x7E, 0x7D, 0x31,
1690 0x1A,
1691 0x32 };
1692
1693
1694 static const int32_t offsets[] = {0,0,0,0,
1695 1,1,1,
1696 3,3,3,3,
1697 4,4,4,
1698 5,
1699 7,};
1700 static const UChar sampleText1[] = { 0x4e00, 0x0035, 0x04e01, 0x0031, 0x0041, 0x0032};
1701
1702 static const uint8_t expected1[] = {0x7E, 0x7B, 0x52, 0x3B,
1703 0x7E, 0x7D, 0x35,
1704 0x7E, 0x7B, 0x36, 0x21,
1705 0x7E, 0x7D, 0x31,
1706 0x41,
1707 0x32 };
1708
1709
1710 static const int32_t offsets1[] = {2,6,9,13,14,15
1711 };
1712
1713 /*hz*/
2ca993e8 1714 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1715 expected, sizeof(expected), "HZ", UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
1716 log_err("u-> not match.\n");
2ca993e8 1717 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1718 expected, sizeof(expected), "HZ", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1719 log_err("u-> not match.\n");
1720 if(!testConvertToU(expected1, sizeof(expected1),
2ca993e8 1721 sampleText1, UPRV_LENGTHOF(sampleText1), "hz",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
b75a7d8f
A		 1722 offsets1, TRUE))
1723 log_err("hz -> did not match.\n");
1724 }
73c04bcf
A		 1725#endif
1726
b75a7d8f
A		 1727 /*UTF-8*/
1728 log_verbose("Testing for UTF8\n");
1729 {
1730 static const UChar sampleText[] = { 0x4e00, 0x0701, 0x0031, 0xbfc1, 0xd801, 0xdc01, 0x0032};
1731 int32_t offsets[]={0x00, 0x00, 0x00, 0x01, 0x01, 0x02,
1732 0x03, 0x03, 0x03, 0x04, 0x04, 0x04,
1733 0x04, 0x06 };
1734 static const uint8_t expected[] = {0xe4, 0xb8, 0x80, 0xdc, 0x81, 0x31,
1735 0xeb, 0xbf, 0x81, 0xF0, 0x90, 0x90, 0x81, 0x32};
1736
1737
1738 static const int32_t fromOffsets[] = { 0x0000, 0x0003, 0x0005, 0x0006, 0x0009, 0x0009, 0x000D };
1739 /*UTF-8*/
2ca993e8 1740 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1741 expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1742 log_err("u-> UTF8 with offsets and flush true did not match.\n");
2ca993e8 1743 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1744 expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
1745 log_err("u-> UTF8 with offsets and flush true did not match.\n");
2ca993e8 1746 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1747 expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1748 log_err("u-> UTF8 with offsets and flush true did not match.\n");
2ca993e8 1749 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
b75a7d8f
A		 1750 expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
1751 log_err("u-> UTF8 with offsets and flush true did not match.\n");
1752 if(!testConvertToU(expected, sizeof(expected),
2ca993e8 1753 sampleText, UPRV_LENGTHOF(sampleText), "UTF8",UCNV_TO_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
b75a7d8f
A		 1754 log_err("UTF8 -> did not match.\n");
1755 if(!testConvertToU(expected, sizeof(expected),
2ca993e8 1756 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", UCNV_TO_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
b75a7d8f
A		 1757 log_err("UTF8 -> did not match.\n");
1758 if(!testConvertToU(expected, sizeof(expected),
2ca993e8 1759 sampleText, UPRV_LENGTHOF(sampleText), "UTF8",UCNV_TO_U_CALLBACK_SUBSTITUTE , fromOffsets, TRUE))
b75a7d8f
A		 1760 log_err("UTF8 -> did not match.\n");
1761 if(!testConvertToU(expected, sizeof(expected),
2ca993e8 1762 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", UCNV_TO_U_CALLBACK_SUBSTITUTE , fromOffsets, TRUE))
b75a7d8f
A		 1763 log_err("UTF8 -> did not match.\n");
1764
1765 }
1766
1767}
1768
1769/* Test that U_TRUNCATED_CHAR_FOUND is set. */
1770static void
1771doTestTruncated(const char *cnvName, const uint8_t *bytes, int32_t length) {
1772 UConverter *cnv;
1773
1774 UChar buffer[2];
1775 UChar *target, *targetLimit;
1776 const char *source, *sourceLimit;
1777
1778 UErrorCode errorCode;
1779
1780 errorCode=U_ZERO_ERROR;
1781 cnv=ucnv_open(cnvName, &errorCode);
1782 if(U_FAILURE(errorCode)) {
1783 log_data_err("error TestTruncated: unable to open \"%s\" - %s\n", cnvName, u_errorName(errorCode));
1784 return;
1785 }
374ca955
A		 1786 ucnv_setToUCallBack(cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
1787 if(U_FAILURE(errorCode)) {
1788 log_data_err("error TestTruncated: unable to set the stop callback on \"%s\" - %s\n",
1789 cnvName, u_errorName(errorCode));
1790 ucnv_close(cnv);
1791 return;
1792 }
b75a7d8f
A		 1793
1794 source=(const char *)bytes;
1795 sourceLimit=source+length;
1796 target=buffer;
b331163b 1797 targetLimit=buffer+UPRV_LENGTHOF(buffer);
b75a7d8f
A		 1798
1799 /* 1. input bytes with flush=FALSE, then input nothing with flush=TRUE */
1800 ucnv_toUnicode(cnv, &target, targetLimit, &source, sourceLimit, NULL, FALSE, &errorCode);
1801 if(U_FAILURE(errorCode) || source!=sourceLimit || target!=buffer) {
1802 log_err("error TestTruncated(%s, 1a): input bytes[%d], flush=FALSE: %s, input left %d, output %d\n",
1803 cnvName, length, u_errorName(errorCode), (int)(sourceLimit-source), (int)(target-buffer));
1804 }
1805
1806 errorCode=U_ZERO_ERROR;
1807 source=sourceLimit;
1808 target=buffer;
1809 ucnv_toUnicode(cnv, &target, targetLimit, &source, sourceLimit, NULL, TRUE, &errorCode);
1810 if(errorCode!=U_TRUNCATED_CHAR_FOUND || target!=buffer) {
374ca955
A		 1811 log_err("error TestTruncated(%s, 1b): no input (previously %d), flush=TRUE: %s (should be U_TRUNCATED_CHAR_FOUND), output %d\n",
1812 cnvName, (int)length, u_errorName(errorCode), (int)(target-buffer));
b75a7d8f
A		 1813 }
1814
b75a7d8f
A		 1815 /* 2. input bytes with flush=TRUE */
1816 ucnv_resetToUnicode(cnv);
1817
1818 errorCode=U_ZERO_ERROR;
1819 source=(const char *)bytes;
1820 target=buffer;
1821 ucnv_toUnicode(cnv, &target, targetLimit, &source, sourceLimit, NULL, TRUE, &errorCode);
374ca955 1822 if(errorCode!=U_TRUNCATED_CHAR_FOUND || source!=sourceLimit || target!=buffer) {
b75a7d8f
A		 1823 log_err("error TestTruncated(%s, 2): input bytes[%d], flush=TRUE: %s (should be U_TRUNCATED_CHAR_FOUND), input left %d, output %d\n",
1824 cnvName, length, u_errorName(errorCode), (int)(sourceLimit-source), (int)(target-buffer));
1825 }
1826
1827
1828 ucnv_close(cnv);
1829}
1830
1831static void
1832TestTruncated() {
1833 static const struct {
1834 const char *cnvName;
1835 uint8_t bytes[8]; /* partial input bytes resulting in no output */
1836 int32_t length;
1837 } testCases[]={
1838 { "IMAP-mailbox-name", { 0x26 }, 1 }, /* & */
1839 { "IMAP-mailbox-name", { 0x26, 0x42 }, 2 }, /* &B */
1840 { "IMAP-mailbox-name", { 0x26, 0x42, 0x42 }, 3 }, /* &BB */
1841 { "IMAP-mailbox-name", { 0x26, 0x41, 0x41 }, 3 }, /* &AA */
1842
1843 { "UTF-7", { 0x2b, 0x42 }, 2 }, /* +B */
1844 { "UTF-8", { 0xd1 }, 1 },
1845
1846 { "UTF-16BE", { 0x4e }, 1 },
1847 { "UTF-16LE", { 0x4e }, 1 },
1848 { "UTF-16", { 0x4e }, 1 },
1849 { "UTF-16", { 0xff }, 1 },
1850 { "UTF-16", { 0xfe, 0xff, 0x4e }, 3 },
1851
1852 { "UTF-32BE", { 0, 0, 0x4e }, 3 },
1853 { "UTF-32LE", { 0x4e }, 1 },
1854 { "UTF-32", { 0, 0, 0x4e }, 3 },
1855 { "UTF-32", { 0xff }, 1 },
1856 { "UTF-32", { 0, 0, 0xfe, 0xff, 0 }, 5 },
b75a7d8f 1857 { "SCSU", { 0x0e, 0x4e }, 2 }, /* SQU 0x4e */
73c04bcf
A		 1858
1859#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 1860 { "BOCU-1", { 0xd5 }, 1 },
1861
1862 { "Shift-JIS", { 0xe0 }, 1 },
1863
1864 { "ibm-939", { 0x0e, 0x41 }, 2 } /* SO 0x41 */
73c04bcf
A		 1865#else
1866 { "BOCU-1", { 0xd5 }, 1 ,}
1867#endif
b75a7d8f
A		 1868 };
1869 int32_t i;
1870
b331163b 1871 for(i=0; i<UPRV_LENGTHOF(testCases); ++i) {
b75a7d8f
A		 1872 doTestTruncated(testCases[i].cnvName, testCases[i].bytes, testCases[i].length);
1873 }
1874}
1875
1876typedef struct NameRange {
1877 const char *name;
1878 UChar32 start, end, start2, end2, notStart, notEnd;
1879} NameRange;
1880
1881static void
1882TestUnicodeSet() {
1883 UErrorCode errorCode;
1884 UConverter *cnv;
1885 USet *set;
1886 const char *name;
1887 int32_t i, count;
1888
1889 static const char *const completeSetNames[]={
1890 "UTF-7",
1891 "UTF-8",
1892 "UTF-16",
1893 "UTF-16BE",
1894 "UTF-16LE",
1895 "UTF-32",
1896 "UTF-32BE",
1897 "UTF-32LE",
1898 "SCSU",
1899 "BOCU-1",
1900 "CESU-8",
73c04bcf 1901#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f 1902 "gb18030",
73c04bcf 1903#endif
374ca955
A		 1904 "IMAP-mailbox-name"
1905 };
73c04bcf 1906#if !UCONFIG_NO_LEGACY_CONVERSION
729e4ab9 1907 static const char *const lmbcsNames[]={
b75a7d8f
A		 1908 "LMBCS-1",
1909 "LMBCS-2",
1910 "LMBCS-3",
1911 "LMBCS-4",
1912 "LMBCS-5",
1913 "LMBCS-6",
1914 "LMBCS-8",
1915 "LMBCS-11",
1916 "LMBCS-16",
1917 "LMBCS-17",
1918 "LMBCS-18",
1919 "LMBCS-19"
1920 };
729e4ab9 1921#endif
b75a7d8f
A		 1922
1923 static const NameRange nameRanges[]={
1924 { "US-ASCII", 0, 0x7f, -1, -1, 0x80, 0x10ffff },
73c04bcf 1925#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f 1926 { "ibm-367", 0, 0x7f, -1, -1, 0x80, 0x10ffff },
73c04bcf 1927#endif
b75a7d8f 1928 { "ISO-8859-1", 0, 0x7f, -1, -1, 0x100, 0x10ffff },
73c04bcf 1929#if !UCONFIG_NO_LEGACY_CONVERSION
b75a7d8f
A		 1930 { "UTF-8", 0, 0xd7ff, 0xe000, 0x10ffff, 0xd800, 0xdfff },
1931 { "windows-1251", 0, 0x7f, 0x410, 0x44f, 0x3000, 0xd7ff },
46f4442e 1932 /* HZ test case fixed and moved to intltest's conversion.txt, ticket #6002 */
b75a7d8f 1933 { "shift-jis", 0x3041, 0x3093, 0x30a1, 0x30f3, 0x900, 0x1cff }
73c04bcf
A		 1934#else
1935 { "UTF-8", 0, 0xd7ff, 0xe000, 0x10ffff, 0xd800, 0xdfff }
1936#endif
b75a7d8f
A		 1937 };
1938
1939 /* open an empty set */
1940 set=uset_open(1, 0);
1941
1942 count=ucnv_countAvailable();
1943 for(i=0; i<count; ++i) {
1944 errorCode=U_ZERO_ERROR;
1945 name=ucnv_getAvailableName(i);
1946 cnv=ucnv_open(name, &errorCode);
1947 if(U_FAILURE(errorCode)) {
729e4ab9 1948 log_data_err("error: unable to open converter %s - %s\n",
b75a7d8f
A		 1949 name, u_errorName(errorCode));
1950 continue;
1951 }
1952
1953 uset_clear(set);
1954 ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
1955 if(U_FAILURE(errorCode)) {
1956 log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
1957 name, u_errorName(errorCode));
1958 } else if(uset_size(set)==0) {
1959 log_err("error: ucnv_getUnicodeSet(%s) returns an empty set\n", name);
1960 }
1961
1962 ucnv_close(cnv);
1963 }
1964
1965 /* test converters that are known to convert all of Unicode (except maybe for surrogates) */
b331163b 1966 for(i=0; i<UPRV_LENGTHOF(completeSetNames); ++i) {
b75a7d8f
A		 1967 errorCode=U_ZERO_ERROR;
1968 name=completeSetNames[i];
1969 cnv=ucnv_open(name, &errorCode);
1970 if(U_FAILURE(errorCode)) {
729e4ab9 1971 log_data_err("error: unable to open converter %s - %s\n",
b75a7d8f
A		 1972 name, u_errorName(errorCode));
1973 continue;
1974 }
1975
1976 uset_clear(set);
1977 ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
1978 if(U_FAILURE(errorCode)) {
1979 log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
1980 name, u_errorName(errorCode));
1981 } else if(!uset_containsRange(set, 0, 0xd7ff) || !uset_containsRange(set, 0xe000, 0x10ffff)) {
1982 log_err("error: ucnv_getUnicodeSet(%s) does not return an all-Unicode set\n", name);
1983 }
1984
1985 ucnv_close(cnv);
1986 }
1987
729e4ab9 1988#if !UCONFIG_NO_LEGACY_CONVERSION
374ca955 1989 /* test LMBCS variants which convert all of Unicode except for U+F6xx */
b331163b 1990 for(i=0; i<UPRV_LENGTHOF(lmbcsNames); ++i) {
374ca955
A		 1991 errorCode=U_ZERO_ERROR;
1992 name=lmbcsNames[i];
1993 cnv=ucnv_open(name, &errorCode);
1994 if(U_FAILURE(errorCode)) {
729e4ab9 1995 log_data_err("error: unable to open converter %s - %s\n",
374ca955
A		 1996 name, u_errorName(errorCode));
1997 continue;
1998 }
1999
2000 uset_clear(set);
2001 ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
2002 if(U_FAILURE(errorCode)) {
2003 log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
2004 name, u_errorName(errorCode));
2005 } else if(!uset_containsRange(set, 0, 0xf5ff) || !uset_containsRange(set, 0xf700, 0x10ffff)) {
2006 log_err("error: ucnv_getUnicodeSet(%s) does not return an all-Unicode set (minus U+F6xx)\n", name);
2007 }
2008
2009 ucnv_close(cnv);
2010 }
729e4ab9 2011#endif
374ca955 2012
b75a7d8f 2013 /* test specific sets */
b331163b 2014 for(i=0; i<UPRV_LENGTHOF(nameRanges); ++i) {
b75a7d8f
A		 2015 errorCode=U_ZERO_ERROR;
2016 name=nameRanges[i].name;
2017 cnv=ucnv_open(name, &errorCode);
2018 if(U_FAILURE(errorCode)) {
2019 log_data_err("error: unable to open converter %s - %s\n",
2020 name, u_errorName(errorCode));
2021 continue;
2022 }
2023
2024 uset_clear(set);
2025 ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
2026 if(U_FAILURE(errorCode)) {
2027 log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
2028 name, u_errorName(errorCode));
2029 } else if(
2030 !uset_containsRange(set, nameRanges[i].start, nameRanges[i].end) ||
2031 (nameRanges[i].start2>=0 && !uset_containsRange(set, nameRanges[i].start2, nameRanges[i].end2))
2032 ) {
2033 log_err("error: ucnv_getUnicodeSet(%s) does not contain the expected ranges\n", name);
2034 } else if(nameRanges[i].notStart>=0) {
2035 /* simulate containsAny() with the C API */
2036 uset_complement(set);
2037 if(!uset_containsRange(set, nameRanges[i].notStart, nameRanges[i].notEnd)) {
2038 log_err("error: ucnv_getUnicodeSet(%s) contains part of the unexpected range\n", name);
2039 }
2040 }
2041
2042 ucnv_close(cnv);
2043 }
2044
73c04bcf
A		 2045 errorCode = U_ZERO_ERROR;
2046 ucnv_getUnicodeSet(NULL, set, UCNV_ROUNDTRIP_SET, &errorCode);
2047 if (errorCode != U_ILLEGAL_ARGUMENT_ERROR) {
2048 log_err("error: ucnv_getUnicodeSet(NULL) returned wrong status code %s\n", u_errorName(errorCode));
2049 }
2050 errorCode = U_PARSE_ERROR;
2051 /* Make sure that it does nothing if an error is passed in. Difficult to proper test for. */
2052 ucnv_getUnicodeSet(NULL, NULL, UCNV_ROUNDTRIP_SET, &errorCode);
2053 if (errorCode != U_PARSE_ERROR) {
2054 log_err("error: ucnv_getUnicodeSet(NULL) returned wrong status code %s\n", u_errorName(errorCode));
2055 }
2056
b75a7d8f
A		 2057 uset_close(set);
2058}
mirror of ICU