]> git.saurik.com Git - apple/icu.git/blame - icuSources/test/testdata/conversion.txt
projects / apple / icu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ICU-6.2.4.tar.gz
[apple/icu.git] / icuSources / test / testdata / conversion.txt
CommitLineData
374ca955
A		 1//*******************************************************************************
2//
3// Copyright (C) 2003-2004, International Business Machines
4// Corporation and others. All Rights Reserved.
5//
6// file name: conversion.txt
7// encoding: US-ASCII
8// tab size: 8 (not used)
9// indentation:4
10//
11// created on: 2003jul15
12// created by: Markus W. Scherer
13//
14// ICU resource bundle source file with test data for data-driven conversion tests.
15//
16//*******************************************************************************
17
18conversion {
19 Info {
20 Description { "Test data for conversion" }
21 LongDescription {
22 "Test data for data-driven conversion tests in icu/source/test/intltest/convtest.cpp\n"
23 "Run intltest conversion\n"
24
25 "Charset names starting with '*' are for testdata names.\n"
26
27 "ICU callbacks are specified as strings with pairs of characters, each optional.\n"
28 "Callback function - '?'=Sub '0'=Skip '.'=Stop '&'=Escape\n"
29 "Callback option - a letter is passed in directly as const char * see ucnv_err.h\n"
30 "Empty string: Sub callback with NULL option\n"
31
32 "In order to specify a charset substitution character,\n"
33 "add a NUL (U+0000) to the callback string followed by the subchar bytes as Latin-1\n"
34 "characters. For example, for a Sub callback with no option and a subchar of FC FC,\n"
35 "use the string \"?\x00\xFC\xFC\"\n"
36
37 "fallbacks: per-direction boolean, currently only for fromUnicode; see Jitterbug 2401\n"
38
39 "errorCode: (empty)==zero | invalid | illegal | truncated | illesc | unsuppesc\n"
40 }
41 }
42 TestData {
43 toUnicode {
44 Headers { "charset", "bytes", "unicode", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidChars" }
45 Cases {
46 // improve coverage of unrolled loops in ucnvmbcs.c/ucnv_MBCSSingleToBMPWithOffsets()
47 {
48 "ISO-8859-3",
49 :bin{ 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f2021222324252627 },
50 "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\x22#$%&'",
51 :intvector{ 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39 },
52 :int{1}, :int{0}, "", "?", :bin{""}
53 }
54 // test that ISO-2022-JP encodes ASCII as itself
55 {
56 "ISO-2022-JP",
57 :bin{ 3f4041424344454647 },
58 "?@ABCDEFG",
59 :intvector{ 0,1,2,3,4,5,6,7,8 },
60 :int{1}, :int{1}, "", "?", :bin{""}
61 }
62 // test that ISO-2022-CN encodes ASCII as itself
63 {
64 "ISO-2022-CN",
65 :bin{ 3f4041424344454647 },
66 "?@ABCDEFG",
67 :intvector{ 0,1,2,3,4,5,6,7,8 },
68 :int{1}, :int{1}, "", "?", :bin{""}
69 }
70
71 // ISO-2022-KR
72
73 // truncated, partial escape sequence
74 {
75 "ibm-25546",
76 :bin{ 1b }, "", :intvector{},
77 :int{1}, :int{1}, "truncated", ".", :bin{ 1b }
78 }
79 {
80 "ibm-25546",
81 :bin{ 1b24 }, "", :intvector{},
82 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 }
83 }
84 {
85 "ibm-25546",
86 :bin{ 1b2429 }, "", :intvector{},
87 :int{1}, :int{1}, "truncated", ".", :bin{ 1b2429 }
88 }
89 // complete escape sequence but nothing else
90 {
91 "ibm-25546",
92 :bin{ 1b242943 }, "", :intvector{},
93 :int{1}, :int{1}, "", ".", :bin{""}
94 }
95 {
96 "ibm-25546",
97 :bin{ 1b2429430e }, "", :intvector{},
98 :int{1}, :int{1}, "", ".", :bin{""}
99 }
100 // escape plus ASCII character
101 {
102 "ibm-25546",
103 :bin{ 1b24294341 }, "A", :intvector{ 4 },
104 :int{1}, :int{1}, "", ".", :bin{""}
105 }
106 // escape plus incomplete DBCS character
107 {
108 "ibm-25546",
109 :bin{ 1b2429430e41 }, "", :intvector{},
110 :int{1}, :int{1}, "truncated", ".", :bin{ 41 }
111 }
112 // all complete with DBCS character
113 {
114 "ibm-25546",
115 :bin{ 1b2429430e4141 }, "\uc88b", :intvector{ 5 },
116 :int{1}, :int{1}, "", ".", :bin{""}
117 }
118 // more complicated example
119 {
120 "ibm-25546",
121 :bin{ 411b242943420e4141affe0f43 },
122 "AB\uc88b%XAF%XFEC",
123 :intvector{ 0, 5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 12 },
124 :int{1}, :int{1}, "", "&", :bin{""}
125 }
126
127 // truncated, partial escape sequence
128 {
129 "ISO-2022-KR",
130 :bin{ 1b }, "", :intvector{},
131 :int{1}, :int{1}, "truncated", ".", :bin{ 1b }
132 }
133 {
134 "ISO-2022-KR",
135 :bin{ 1b24 }, "", :intvector{},
136 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 }
137 }
138 {
139 "ISO-2022-KR",
140 :bin{ 1b2429 }, "", :intvector{},
141 :int{1}, :int{1}, "truncated", ".", :bin{ 1b2429 }
142 }
143 // complete escape sequence but nothing else
144 {
145 "ISO-2022-KR",
146 :bin{ 1b242943 }, "", :intvector{},
147 :int{1}, :int{1}, "", ".", :bin{""}
148 }
149 {
150 "ISO-2022-KR",
151 :bin{ 1b2429430e }, "", :intvector{},
152 :int{1}, :int{1}, "", ".", :bin{""}
153 }
154 // escape plus ASCII character
155 {
156 "ISO-2022-KR",
157 :bin{ 1b24294341 }, "A", :intvector{ 4 },
158 :int{1}, :int{1}, "", ".", :bin{""}
159 }
160 // escape plus incomplete DBCS character
161 {
162 "ISO-2022-KR",
163 :bin{ 1b2429430e41 }, "", :intvector{},
164 :int{1}, :int{1}, "truncated", ".", :bin{ 41 }
165 }
166 // all complete with DBCS character
167 {
168 "ISO-2022-KR",
169 :bin{ 1b2429430e4141 }, "\uc88b", :intvector{ 5 },
170 :int{1}, :int{1}, "", ".", :bin{""}
171 }
172 // more complicated example
173 {
174 "ISO-2022-KR",
175 :bin{ 411b242943420e4141affe0f43 },
176 "AB\uc88b%XAF%XFEC",
177 :intvector{ 0, 5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 12 },
178 :int{1}, :int{1}, "", "&", :bin{""}
179 }
180
181 // ISO-2022-JP
182
183 // truncated, partial escape sequence
184 {
185 "ISO-2022-JP",
186 :bin{ 1b }, "", :intvector{},
187 :int{1}, :int{1}, "truncated", ".", :bin{ 1b }
188 }
189 {
190 "ISO-2022-JP-2",
191 :bin{ 1b24 }, "", :intvector{},
192 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 }
193 }
194 // complete escape sequence but nothing else
195 {
196 "ISO-2022-JP-2",
197 :bin{ 1b2442 }, "", :intvector{},
198 :int{1}, :int{1}, "", ".", :bin{""}
199 }
200 // escape plus incomplete DBCS character
201 {
202 "ISO-2022-JP-2",
203 :bin{ 1b244241 }, "", :intvector{},
204 :int{1}, :int{1}, "truncated", ".", :bin{ 41 }
205 }
206 // all complete with DBCS character
207 {
208 "ISO-2022-JP-2",
209 :bin{ 1b24424141 }, "\u758f", :intvector{ 3 },
210 :int{1}, :int{1}, "", ".", :bin{""}
211 }
212 // test the G2 designator & SS2 shift
213 {
214 "ISO-2022-JP-2",
215 :bin{ 431b2e46461b244241411b4e4e353f }, "CF\u758f\u039e\u7591", :intvector{ 0, 4, 8, 12, 13 },
216 :int{1}, :int{1}, "", ".", :bin{""}
217 }
218 // JIS7 with Katakana
219 {
220 "JIS7",
221 :bin{ 41420e41420f4142 }, "AB\uff81\uff82AB", :intvector{ 0, 1, 3, 4, 6, 7 },
222 :int{1}, :int{1}, "", ".", :bin{""}
223 }
224 // JIS8 with Katakana
225 {
226 "JIS8",
227 :bin{ 41c15c1b284a5cc242 }, "A\uff81\\\xa5\uff82B", :intvector{ 0, 1, 2, 6, 7, 8 },
228 :int{1}, :int{1}, "", ".", :bin{""}
229 }
230
231 // ISO-2022-CN
232
233 // truncated, partial escape sequence
234 {
235 "ISO_2022,locale=zh,version=1",
236 :bin{ 1b }, "", :intvector{},
237 :int{1}, :int{1}, "truncated", ".", :bin{ 1b }
238 }
239 {
240 "ISO_2022,locale=zh,version=1",
241 :bin{ 1b24 }, "", :intvector{},
242 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 }
243 }
244 {
245 "ISO_2022,locale=zh,version=1",
246 :bin{ 1b2429 }, "", :intvector{},
247 :int{1}, :int{1}, "truncated", ".", :bin{ 1b2429 }
248 }
249 // complete escape sequence but nothing else
250 {
251 "ISO_2022,locale=zh,version=1",
252 :bin{ 1b242941 }, "", :intvector{},
253 :int{1}, :int{1}, "", ".", :bin{""}
254 }
255 {
256 "ISO_2022,locale=zh,version=1",
257 :bin{ 1b2429410e }, "", :intvector{},
258 :int{1}, :int{1}, "", ".", :bin{""}
259 }
260 // escape plus ASCII character
261 {
262 "ISO_2022,locale=zh,version=1",
263 :bin{ 1b24294141 }, "\x41", :intvector{ 4 },
264 :int{1}, :int{1}, "", ".", :bin{""}
265 }
266 // escape plus incomplete DBCS character
267 {
268 "ISO_2022,locale=zh,version=1",
269 :bin{ 1b2429410e41 }, "", :intvector{},
270 :int{1}, :int{1}, "truncated", ".", :bin{ 41 }
271 }
272 // all complete with DBCS character
273 {
274 "ISO_2022,locale=zh,version=1",
275 :bin{ 1b2429410e4141 }, "\u4eae", :intvector{ 5 },
276 :int{1}, :int{1}, "", ".", :bin{""}
277 }
278 // ISO-2022-CN-EXT with all subcharsets and shifts and with supplementary code points
279 {
280 "ISO-2022-CN-EXT",
281 :bin{ 1b2429411b242a480e41411b2429457e7c1b4e70341b242b4d1b2429477c341b4f664c2421 },
282 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6\x30",
283 :intvector{ 9, 15, 19, 29, 33, 33, 35 },
284 :int{1}, :int{1}, "", ".", :bin{""}
285 }
286
287 // illegal and unsupported escape sequences
288 // SS2 without designator: illegal
289 {
290 "ISO-2022-CN-EXT",
291 :bin{ 411b4e2121 }, "\x41", :intvector{ 0 },
292 :int{1}, :int{1}, "illesc", ".", :bin{ 1b4e }
293 }
294 // G3 designator: recognized, but not supported for -CN (only for -CN-EXT)
295 {
296 "ISO-2022-CN",
297 :bin{ 411b242b491b4f2121 }, "\x41", :intvector{ 0 },
298 :int{1}, :int{1}, "unsuppesc", ".", :bin{ 1b242b49 }
299 }
300
301 // ISO-2022 SBCS
302 // [U_ENABLE_GENERIC_ISO_2022]
303 // The _generic_ ISO-2022 converter is disabled starting 2003-dec-03 (ICU 2.8).
304 // For details see the icu mailing list from 2003-dec-01 and the ucnv2022.c file.
305 // Language-specific variants of ISO-2022 continue to be available as listed below.
306 //{
307 // "ISO_2022",
308 // :bin{ 0008090a0d1a1c1f203f415c7d7e7f },
309 // "\x00\x08\t\n\r\x1a\x1c\x1f ?A\\}~\x7f",
310 // :intvector{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 },
311 // :int{1}, :int{1}, "", ".", :bin{""}
312 //}
313
314 // DBCS-only extensions
315 {
316 "ibm-970",
317 :bin{ 617eece9b2eb },
318 "\x61\x7e\u4e00\ub000",
319 :intvector{ 0, 1, 2, 4 },
320 :int{1}, :int{1}, "", "?", :bin{""}
321 }
322
323 {
324 "ibm-971",
325 :bin{ 617eece9b2eb },
326 "\ufffd\u4e00\ub000",
327 :intvector{ 0, 2, 4 },
328 :int{1}, :int{1}, "", "?", :bin{""}
329 }
330
331 {
332 "ibm-16684",
333 :bin{ 430e4395ecc1404042e1 },
334 "\ufffd\u30C8\u30C8\u309A\u3000\u20ac",
335 :intvector{ 0, 2, 4, 4, 6, 8 },
336 :int{1}, :int{0}, "", "?", :bin{""}
337 }
338
339 {
340 "ibm-1399",
341 :bin{ 430e4395ecc140400fe1 },
342 "\uff62\u30C8\u30C8\u309A\u3000\u20ac",
343 :intvector{ 0, 2, 4, 4, 6, 9 },
344 :int{1}, :int{0}, "", "?", :bin{""}
345 }
346
347 // extensions
348 {
349 "ibm-1390",
350 :bin{ 430e4395ecc1 },
351 "\uff63\u30C8\u30C8\u309A",
352 :intvector{ 0, 2, 4, 4 },
353 :int{1}, :int{0}, "", "?", :bin{""}
354 }
355
356 {
357 "ibm-16684",
358 :bin{ ececec8bec8cec8d4386ecb5ecb6ecb7 },
359 "\ufffd\u31f6\u31f7\u31f8\u30ab\u304b\u309a\u304d\u309a\u304f\u309a",
360 :intvector{ 0, 2, 4, 6, 8, 10, 10, 12, 12, 14, 14 },
361 :int{1}, :int{0}, "", "?", :bin{""}
362 }
363
364 {
365 "ibm-1390",
366 :bin{ 43860eececec8bec8cec8d4386ecb5ecb6ecb7ecc10fec },
367 "\uff63\uff76\ufffd\u31f6\u31f7\u31f8\u30ab\u304b\u309a\u304d\u309a\u304f\u309a\u30C8\u309A\x1a",
368 :intvector{ 0, 1, 3, 5, 7, 9, 11, 13, 13, 15, 15, 17, 17, 19, 19, 22 },
369 :int{1}, :int{0}, "", "?", :bin{""}
370 }
371
372 {
373 "*test3",
374 :bin{ 00050601020b0701020a01020c },
375 "\u20ac\x05\x06\x0b\U00101234\U00023456\ufffd",
376 :intvector{ 0, 1, 2, 3, 6, 6, 7, 7, 10 },
377 :int{1}, :int{0}, "", "?", :bin{""}
378 }
379
380 // normal conversions
381 {
382 "UTF-16LE",
383 :bin{ 310000d801dc00d902dc320000d8330001dc3400 },
384 "1\U00010001\U000500022\ufffd3\ufffd4",
385 :intvector{ 0, 2, 2, 6, 6, 10, 12, 14, 16, 18 },
386 :int{1}, :int{0}, "", "?", :bin{""}
387 }
388 { "UTF-16LE", :bin{ 00 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00 } }
389 { "UTF-16LE", :bin{ 00d800 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00d800 } }
390
391 {
392 "UTF-16BE",
393 :bin{ 0031d800dc01d900dc020032d8000033dc010034 },
394 "1\U00010001\U000500022\ufffd3\ufffd4",
395 :intvector{ 0, 2, 2, 6, 6, 10, 12, 14, 16, 18 },
396 :int{1}, :int{0}, "", "?", :bin{""}
397 }
398 { "UTF-16BE", :bin{ 00 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00 } }
399 { "UTF-16BE", :bin{ d800dc }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ d800dc } }
400
401 // e4b8 is a partial sequence
402 { "UTF-8", :bin{ 31e4ba8ce4b8 }, "1\u4e8c", :intvector{ 0, 1 }, :int{1}, :int{0}, "truncated", ".", :bin{ e4b8 } }
403 { "UTF-8", :bin{ 31e4ba8ce4b8 }, "1\u4e8c\ufffd", :intvector{ 0, 1, 4 }, :int{1}, :int{0}, "", "?", :bin{""} }
404
405 // LMBCS with escape callback (1292a0 is unassigned)
406 {
407 "LMBCS",
408 :bin{ 12c9501292a01292a1 },
409 "\u4e2e%X12%X92%XA0\ue5c4",
410 :intvector{ 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 6 },
411 :int{1}, :int{0}, "", "&", :bin{""}
412 }
413
414 // IMAP-mailbox-name with SUB
415 // a<DEL> a&AB~ a&AB\x0c a&AB- a&AB. a&.
416 {
417 "IMAP-mailbox-name",
418 :bin{ 617f612641427e612641420c612641422d612641422e61262e },
419 "a\ufffda\ufffda\ufffda\ufffda\ufffda\ufffd",
420 :intvector{ 0, 1, 2, 4, 7, 9, 12, 14, 17, 19, 22, 23 },
421 :int{1}, :int{0}, "", "?", :bin{""}
422 }
423
424 // using testdata_test1.cnv
425 { "*test1", :bin{ 000506070809 }, "\u20ac\x05\x06\U00101234\ufffd\ufffd", :intvector{ 0, 1, 2, 3, 3, 4, 5 }, :int{1}, :int{0}, "", "", :bin{""} }
426
427 // surrogates in CESU-8
428 { "CESU-8", :bin{ eda080eda081edb081 }, "\ud800\U00010401", :intvector{ 0, 3, 6 }, :int{1}, :int{0}, "", "", :bin{""} }
429 // e080 is a partial sequence
430 { "UTF-8", :bin{ 31ffe4ba8ce08061 }, "1\ufffd\u4e8c\ufffda", :intvector{ 0, 1, 2, 5, 7 }, :int{0}, :int{0}, "", "", :bin{ e080 } }
431 // fbbfbfbfbf exceedes U+10ffff
432 { "UTF-8", :bin{ 31fbbfbfbfbf61 }, "1\ufffda", :intvector{ 0, 1, 6 }, :int{0}, :int{0}, "", "", :bin{ fbbfbfbfbf } }
433
434 // lead byte a2 without trail byte
435 { "ibm-1363", :bin{ a2aea2 }, "\u00a1", :intvector{ 0 }, :int{1}, :int{0}, "truncated", ".", :bin{ a2 } }
436 { "ibm-1363", :bin{ a2aea2 }, "\u00a1\u001a", :intvector{ 0, 2 }, :int{1}, :int{0}, "", "?", :bin{""} }
437
438 // simple sample, no error handling
439 { "UTF-8", :bin{ 61F48FBFBF }, "a\U0010FFFF", :intvector{ 0, 1, 1 }, :int{1}, :int{0}, "", "", :bin{""} }
440 }
441 }
442
443 // --------------------------------------------------------------------- ***
444
445 fromUnicode {
446 Headers { "charset", "unicode", "bytes", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidUChars" }
447 Cases {
448 // Improve ucnv_ext.c code coverage:
449 // There will be a partial match up to the lead surrogate of U+603ff
450 // which then results in one more unit in the prefetch buffer
451 // than the match length when converting one code unit at a time.
452 // See ucnv_extContinueMatchFromU() comment
453 // "the match did not use all of preFromU[] - keep the rest for replay"
454 {
455 "*test3",
456 "\U00101234\U00101234\U00050005\U000603ff",
457 :bin{ 07070001020e05ff },
458 :intvector{ 0, 2, 2, 2, 2, 2, 2, 6 },
459 :int{1}, :int{0}, "", "?", ""
460 }
461
462 // test that ISO-2022-JP encodes ASCII as itself
463 {
464 "ISO-2022-JP",
465 "?@ABCDEFG",
466 :bin{ 3f4041424344454647 },
467 :intvector{ 0,1,2,3,4,5,6,7,8 },
468 :int{1}, :int{1}, "", "?", ""
469 }
470 // test that ISO-2022-CN encodes ASCII as itself
471 {
472 "ISO-2022-CN",
473 "?@ABCDEFG",
474 :bin{ 3f4041424344454647 },
475 :intvector{ 0,1,2,3,4,5,6,7,8 },
476 :int{1}, :int{1}, "", "?", ""
477 }
478
479 // moved from cintltst /tsconv/nccbtst/TestSkipCallBack
480 {
481 "iso-2022-jp",
482 "\u3000\xe9\u3001",
483 :bin{ 1b2442212121221b2842 },
484 :intvector{ 0,0,0,0,0,2,2,2,2,2 },
485 :int{1}, :int{1}, "", "0", ""
486 }
487 // moved from cintltst /tsconv/nccbtst/TestSubCallBack
488 {
489 "iso-2022-jp",
490 "A\xe9B\xe9\u3000",
491 :bin{ 411a421a1b244221211b2842 },
492 :intvector{ 0,1,2,3,4,4,4,4,4,4,4,4 },
493 :int{1}, :int{1}, "", "?", ""
494 }
495 // moved from cintltst /tsconv/nccbtst/TestSubWithValueCallBack
496 {
497 "iso-2022-jp",
498 "A\xe9B\xe9\u3000",
499 :bin{ 41255530304539422555303045391b244221211b2842 },
500 :intvector{ 0,1,1,1,1,1,1,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4 },
501 :int{1}, :int{1}, "", "&", ""
502 }
503 {
504 "iso-2022-cn",
505 "\u4e00\u3712\u4e01",
506 :bin{ 1b2429410e523b0f2555333731320e36210f },
507 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,1,2,2,2,2 },
508 :int{1}, :int{1}, "", "&", ""
509 }
510 {
511 "iso-2022-cn",
512 "A\u3712\u4e00",
513 :bin{ 412555333731321b2429410e523b0f },
514 :intvector{ 0,1,1,1,1,1,1,2,2,2,2,2,2,2,2 },
515 :int{1}, :int{1}, "", "&", ""
516 }
517 {
518 "iso-2022-cn",
519 "\u3000\u3712\u3001",
520 :bin{ 1b2429410e21210f2555333731320e21220f },
521 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,1,2,2,2,2 },
522 :int{1}, :int{1}, "", "&", ""
523 }
524
525 // moved from cintltst /tsconv/nucnvtst/TestJIS
526 {
527 "JIS",
528 "\uFF81\uFF82\u30EC\u30ED\u30EE\u30EF\uFF93\uFF94\uFF95\uFF96\uFF97\uFF98",
529 :bin{ 1b244225412544256c256d256e256F25622564256625682569256a1b2842 },
530 :intvector{ 0,0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,11,11,11 },
531 :int{1}, :int{1}, "", "?", ""
532 }
533 {
534 "JIS7",
535 "\uFF81\uFF82\u30EC\u30ED\u30EE\u30EF\uFF93\uFF94\uFF95\uFF96\uFF97\uFF98",
536 :bin{ 0e41420f1b2442256c256d256e256F0e5354555657580f1b2842 },
537 :intvector{ 0,0,1,2,2,2,2,2,2,3,3,4,4,5,5,6,6,7,8,9,10,11,11,11,11,11 },
538 :int{1}, :int{1}, "", "?", ""
539 }
540 {
541 "JIS8",
542 "\uFF81\uFF82\u30EC\u30ED\u30EE\u30EF\uFF93\uFF94\uFF95\uFF96\uFF97\uFF98",
543 :bin{ C1C21b2442256c256d256e256F1b284AD3D4D5D6D7D81b2842 },
544 :intvector{ 0,1,2,2,2,2,2,3,3,4,4,5,5,6,6,6,6,7,8,9,10,11,11,11,11 },
545 :int{1}, :int{1}, "", "?", ""
546 }
547
548 // moved from cintltst /tsconv/ncnvtst/TestErrorBehaviour
549 {
550 "iso-2022-jp",
551 "\u3000\x50\udc01\u3001",
552 :bin{ 1B244221211B2842501A1B24422122 },
553 :intvector{ 0,0,0,0,0,1,1,1,1,2,3,3,3,3,3 },
554 :int{0}, :int{1}, "", "?", "\udc01"
555 }
556 {
557 "iso-2022-jp",
558 "\u3000\x50\udc01\u3001",
559 :bin{ 1B244221211B2842501A1B244221221b2842 },
560 :intvector{ 0,0,0,0,0,1,1,1,1,2,3,3,3,3,3,3,3,3 },
561 :int{1}, :int{1}, "", "?", ""
562 }
563 {
564 "iso-2022-kr",
565 "\x61\u4e00\udc01\u4e00",
566 :bin{ 1b242943610e6c690f1a0e6c69 },
567 :intvector{ -1,-1,-1,-1,0,1,1,1,2,2,3,3,3 },
568 :int{0}, :int{1}, "", "?", "\udc01"
569 }
570 {
571 "iso-2022-kr",
572 "\x61\u4e00\udc01\u4e00",
573 :bin{ 1b242943610e6c690f1a0e6c690f },
574 :intvector{ -1,-1,-1,-1,0,1,1,1,2,2,3,3,3,3 },
575 :int{1}, :int{1}, "", "?", ""
576 }
577
578 // ISO-2022-KR
579 {
580 "ibm-25546",
581 "AB\uc88b\U00050005\uacccC",
582 :bin{ 1b24294341420e41410f7b552b35303030357d0e306a0f43 },
583 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,3,3,3,3,3,3,3,3,5,5,5,6,6 },
584 :int{1}, :int{1}, "", "&U", ""
585 }
586 {
587 "ibm-25546",
588 "AB\uc88b\U00050005\uacccC",
589 :bin{ 1b24294341420e41410f1a0e306a0f43 },
590 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5,6,6 },
591 :int{1}, :int{1}, "", "?\x00\x1a", ""
592 }
593 {
594 "ibm-25546",
595 "AB\uc88b\U00050005\uacccC",
596 :bin{ 1b24294341420e41412f7e306a0f43 },
597 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,6,6 },
598 :int{1}, :int{1}, "", "?", ""
599 }
600 {
601 "ibm-25546",
602 "AB\uc88b\U00050005\uaccc",
603 :bin{ 1b24294341420e41412f7e306a0f },
604 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5 },
605 :int{1}, :int{1}, "", "?", ""
606 }
607 {
608 "ISO-2022-KR",
609 "AB\uc88b\U00050005\uacccC",
610 :bin{ 1b24294341420e41410f7b552b35303030357d0e306a0f43 },
611 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,3,3,3,3,3,3,3,3,5,5,5,6,6 },
612 :int{1}, :int{1}, "", "&U", ""
613 }
614 {
615 "ISO-2022-KR",
616 "AB\uc88b\U00050005\uacccC",
617 :bin{ 1b24294341420e41410f1a0e306a0f43 },
618 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5,6,6 },
619 :int{1}, :int{1}, "", "?", ""
620 }
621 {
622 "ISO-2022-KR",
623 "AB\uc88b\U00050005\uacccC",
624 :bin{ 1b24294341420e41412f7e306a0f43 },
625 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,6,6 },
626 :int{1}, :int{1}, "", "?\x00\x2f\x7e", ""
627 }
628 {
629 "ISO-2022-KR",
630 "AB\uc88b\U00050005\uaccc",
631 :bin{ 1b24294341420e41412f7e306a0f },
632 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5 },
633 :int{1}, :int{1}, "", "?\x00\x2f\x7e", ""
634 }
635
636 // ISO-2022-JP-2 with G2 designator & SS2 shift
637 {
638 "ISO-2022-JP-2",
639 "CF\u758f\u038f\u7591",
640 :bin{ 43461b244241411b2e461b4e3f353f1b2842 },
641 :intvector{ 0,1,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4 },
642 :int{1}, :int{1}, "", ".", ""
643 }
644 // JIS7 with Katakana
645 {
646 "JIS7",
647 "AB\uff81\uff82AB",
648 :bin{ 41420e41420f4142 },
649 :intvector{ 0,1,2,2,3,4,4,5 },
650 :int{1}, :int{1}, "", ".", ""
651 }
652 // JIS7 with shift to ASCII at the very end
653 {
654 "JIS7",
655 "AB\uff81\uff82",
656 :bin{ 41420e41420f },
657 :intvector{ 0,1,2,2,3,3 },
658 :int{1}, :int{1}, "", ".", ""
659 }
660 // JIS8 with Katakana
661 {
662 "JIS8",
663 "A\uff81\\\xa5\uff82B",
664 :bin{ 41c15c1b284a5cc2421b2842 },
665 :intvector{ 0,1,2,3,3,3,3,4,5,5,5,5 },
666 :int{1}, :int{1}, "", ".", ""
667 }
668
669 // ISO-2022-CN-EXT with all subcharsets and shifts and with supplementary code points
670 {
671 "ISO-2022-CN-EXT",
672 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6\x30",
673 :bin{ 1b2429410e41411b2429457e7c1b242a481b4e70341b2429477c341b242b4d1b4f664c0f30 },
674 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4,6,6 },
675 :int{1}, :int{1}, "", ".", ""
676 }
677 // ISO-2022-CN-EXT with shift to ASCII at the very end
678 {
679 "ISO-2022-CN-EXT",
680 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6",
681 :bin{ 1b2429410e41411b2429457e7c1b242a481b4e70341b2429477c341b242b4d1b4f664c0f },
682 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4 },
683 :int{1}, :int{1}, "", ".", ""
684 }
685 // ISO-2022-CN-EXT without flush so do not shift to ASCII at the very end
686 {
687 "ISO-2022-CN-EXT",
688 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6",
689 :bin{ 1b2429410e41411b2429457e7c1b242a481b4e70341b2429477c341b242b4d1b4f664c },
690 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4 },
691 :int{0}, :int{1}, "", ".", ""
692 }
693
694 // windows-936 vs. ibm-1386
695 {
696 "ibm-1386",
697 "\x1a\u20ac\u5555\x80\x81\U00055555",
698 :bin{ 7fa2e3dffb7f7fa1a1 },
699 :intvector{ 0, 1, 1, 2, 2, 3, 4, 5, 5 },
700 :int{1}, :int{1}, "", "?", ""
701 }
702 {
703 "windows-936",
704 "\x1a\u20ac\u5555\x80\x81\U00055555",
705 :bin{ 1a80dffb3f3f3f },
706 :intvector{ 0, 1, 2, 2, 3, 4, 5 },
707 :int{1}, :int{1}, "", "?", ""
708 }
709
710 // verify that if a conversion table does not have any mapping for U+0000,
711 // then there will not even be a phantom fallback to 00
712 {
713 "ibm-971",
714 "\x00",
715 :bin{ affe },
716 :intvector{ 0, 0 },
717 :int{1}, :int{1}, "", "?", ""
718 }
719
720 {
721 "*test4",
722 "\x00",
723 :bin{ ff },
724 :intvector{ 0 },
725 :int{1}, :int{1}, "", "?", ""
726 }
727
728 // extension in testdata
729 {
730 "*test4x",
731 "\u20ac\x09",
732 :bin{ 0009 },
733 :intvector{ 0, 1 },
734 :int{1}, :int{1}, "", "?", ""
735 }
736
737 // DBCS-only extensions
738 {
739 "ibm-970",
740 "\x61\uffa1\u2015\ub000",
741 :bin{ 611aa1aab2eb },
742 :intvector{ 0, 1, 2, 2, 3, 3 },
743 :int{1}, :int{1}, "", "?", ""
744 }
745
746 {
747 "ibm-971",
748 "\x61\uffa1\u2015\ub000",
749 :bin{ affeaffeaffeb2eb },
750 :intvector{ 0, 0, 1, 1, 2, 2, 3, 3 },
751 :int{1}, :int{1}, "", "?", ""
752 }
753
754 {
755 "ibm-1390,swaplfnl",
756 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a",
757 :bin{ 430e4395ecc140400fc1e115 },
758 :intvector{ 0, 1, 1, 1, 2, 2, 4, 4, 5, 5, 6, 7 },
759 :int{1}, :int{0}, "", "?", ""
760 }
761
762 {
763 "ibm-16684",
764 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a",
765 :bin{ fefe4395ecc14040fefe42e1fefe },
766 :intvector{ 0, 0, 1, 1, 2, 2, 4, 4, 5, 5, 6, 6, 7, 7 },
767 :int{1}, :int{0}, "", "?", ""
768 }
769
770 {
771 "ibm-1399",
772 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a",
773 :bin{ 440e4395ecc140400fc1e125 },
774 :intvector{ 0, 1, 1, 1, 2, 2, 4, 4, 5, 5, 6, 7 },
775 :int{1}, :int{0}, "", "?", ""
776 }
777
778 // <subchar1> from |2 mappings
779 {
780 "ibm-1390",
781 "\x0e\x0f\u0901\U00050000\uffe8\uffee",
782 :bin{ 3f3f0efefefefe0f3f3f },
783 :intvector{ 0, 1, 2, 2, 2, 3, 3, 5, 5, 6 },
784 :int{1}, :int{1}, "", "?", ""
785 }
786
787 // <subchar1> from |2 mappings, and also contains a fallback to 00
788 {
789 "*test4",
790 "\u20ac\u20ad\U00050005\U00023456\U0010ffff\x30",
791 :bin{ 0000e10102030affff },
792 :intvector{ 0, 1, 2, 4, 4, 4, 4, 6, 8 },
793 :int{1}, :int{1}, "", "?", ""
794 }
795
796 // setting a <subchar> resets the <subchar1>
797 {
798 "*test4",
799 "\u20ac\u20ad\U00050005\U00023456\U0010ffff\x30",
800 :bin{ 00000102030f0102030a0102030f0102030f },
801 :intvector{ 0, 1, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6, 8, 8, 8, 8 },
802 :int{1}, :int{1}, "", "?\x00\x01\x02\x03\x0f", ""
803 }
804
805 // fallback to 00 with old single-byte data structure
806 {
807 "*test1",
808 "\u20ac\u20ad\U00101234\U00050000",
809 :bin{ 000007ff },
810 :intvector{ 0, 1, 2, 4 },
811 :int{1}, :int{1}, "", "?", ""
812 }
813
814 // extensions
815 {
816 "ibm-1390",
817 "\u025a\u025a\u0300\u025a\u0301\u025a\u0302\uffe8\U0002a0f9",
818 :bin{ 0ed896eccaeccbd896ea530f3f0eb7c20f },
819 :intvector{ 0, 0, 0, 1, 1, 3, 3, 5, 5, 6, 6, 7, 7, 8, 8, 8, 8 },
820 :int{1}, :int{0}, "", "?", ""
821 }
822
823 {
824 "*test3",
825 "\xc4\xc4\xc4\U00101234\xc4\xc4\U00101234\x05",
826 :bin{ ffffff070501020c },
827 :intvector{ 0, 1, 2, 3, 5, 5, 5, 5 },
828 :int{1}, :int{0}, "", "?", ""
829 }
830
831 {
832 "*test3",
833 "\U00101234\U00101234\U00050005\U00101234\U00050005\U00060006",
834 :bin{ 07070001020e05070001020f09 },
835 :intvector{ 0, 2, 2, 2, 2, 2, 2, 6, 6, 6, 6, 6, 6 },
836 :int{1}, :int{0}, "", "?", ""
837 }
838
839 // normal conversions
840 {
841 "UTF-16LE",
842 "1\U00010001\U000500022\ud8003\udc014",
843 :bin{ 310000d801dc00d902dc3200fdff3300fdff3400 },
844 :intvector{ 0, 0, 1, 1, 1, 1, 3, 3, 3, 3, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9 },
845 :int{1}, :int{0}, "", "?", ""
846 }
847 { "UTF-16LE", "\ud800", :bin{""}, :intvector{}, :int{1}, :int{0}, "truncated", ".", "\ud800" }
848
849 {
850 "UTF-16BE",
851 "1\U00010001\U000500022\ud8003\udc014",
852 :bin{ 0031d800dc01d900dc020032fffd0033fffd0034 },
853 :intvector{ 0, 0, 1, 1, 1, 1, 3, 3, 3, 3, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9 },
854 :int{1}, :int{0}, "", "?", ""
855 }
856 { "UTF-16BE", "\ud800", :bin{""}, :intvector{}, :int{1}, :int{0}, "truncated", ".", "\ud800" }
857
858 // escape callback
859 {
860 "ISCII",
861 "A\u0901\U00023456\u0902B\U00023456C",
862 :bin{ 41ef42a1255544383444255544433536a24225554438344425554443353643 },
863 :intvector{
864 0,
865 1,1,1,
866 2,2,2,2,2,2,
867 2,2,2,2,2,2,
868 4,
869 5,
870 6,6,6,6,6,6,
871 6,6,6,6,6,6,
872 8
873 },
874 :int{1}, :int{0}, "", "&", ""
875 }
876
877 // escape callback (hex)
878 {
879 "iso-2022-jp",
880 "\u3000\U00023456\u3001\U00023456B\u901c",
881 :bin{ 1b244221211b284226237832333435363b1b244221221b284226237832333435363b42262378393031433b },
882 :intvector{
883 0,0,0,0,0,
884 1,1,1,1,1,1,1,1,1,1,1,1,
885 3,3,3,3,3,
886 4,4,4,4,4,4,4,4,4,4,4,4,
887 6,
888 7,7,7,7,7,7,7,7
889 },
890 :int{1}, :int{0}, "", "&X", ""
891 }
892
893 // sub callback
894 {
895 "gb18030",
896 "$\x7f\x80\u01f9\u20ac\u4e00\u9fa6\uffff\U00010000\U0010ffff",
897 :bin{ 247f81308130a8bfa2e3d2bb82358f338431a43990308130e3329a35 },
898 :intvector{ 0, 1, 2, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 10, 10, 10, 10 },
899 :int{1}, :int{0}, "", "?", ""
900 }
901
902 // skip callback
903 { "ibm-930", "\u6D63\u6D64\u6D65\u6D66", :bin{ 0e5d5f5d63466b0f }, :intvector{ 0, 0, 0, 1, 1, 3, 3, 3 }, :int{1}, :int{0}, "", "0", "" }
904 { "ibm-930", "\u6D63\u6D64\ud89a\u6D66", :bin{ 0e5d5f5d63466b0f }, :intvector{ 0, 0, 0, 1, 1, 3, 3, 3 }, :int{1}, :int{0}, "", "0", "" }
905 { "ibm-930", "\u6D63\u6D64\ud89a\u6D66", :bin{ 0e5d5f5d63 }, :intvector{ 0, 0, 0, 1, 1 }, :int{1}, :int{0}, "illegal", "0i", "\ud89a" }
906
907 // sub callback for supplementary code point
908 { "LATIN1", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "", "" }
909 { "ibm-920", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "", "" }
910
911 // sub callback with AA as subchar
912 { "ibm-920", "1\U000104012", :bin{ 31AA32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "?\x00\xAA", "" }
913
914 // same but not flushing
915 { "LATIN1", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{0}, :int{0}, "", "", "\U00010401" }
916 { "ibm-920", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{0}, :int{0}, "", "", "\U00010401" }
917
918 // simple sample, no error handling
919 { "UTF-8", "a\U0010FFFF", :bin{ 61F48FBFBF }, :intvector{ 0, 1, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" }
920 }
921 }
922
923 getUnicodeSet {
924 // charset - will be opened, and ucnv_getUnicodeSet() called on it
925 // map - set of code points and strings that must be in the returned set
926 // mapnot - set of code points and strings that must *not* be in the returned set
927 // which - numeric UConverterUnicodeSet value
928 Headers { "charset", "map", "mapnot", "which" }
929 Cases {
930 // ISO-2022-KR
931 {
932 "ISO-2022-KR",
933 "[\x00-\x7f\xa1\xa4\xfe\u0111\u4e00\u4e01\uac00-\uac02\uffe6]",
934 "[\x80-\xa0\xa3\xa5\xff-\u0110\uac03\uffe7-\U0010ffff]",
935 :int{0}
936 }
937
938 // versions of ISO-2022-JP
939 {
940 "ISO-2022-JP",
941 "[\x00-\x7f\u0391-\u03a1\uff61-\uff9f\u4e00\u4e01\uffe5]",
942 "[\u0100-\u0113\u0385-\u038a\u4e02\u4e27-\u4e29\uffe6-\U0010ffff]",
943 :int{0}
944 }
945 {
946 "ISO-2022-JP-2",
947 "[\x00-\u0113\u0385-\u038a\u0390-\u03a1\uff61-\uff9f\u4e00-\u4e05\uffe6]",
948 "[\uffe7-\U0010ffff]",
949 :int{0}
950 }
951
952 // versions of ISO-2022-CN
953 {
954 "ISO-2022-CN",
955 "[\x00-\x7f\u4e00\u4e01\u9f98\ufe6b]",
956 "[\u4e29\uffe6-\U0010ffff]",
957 :int{0}
958 }
959 {
960 "ISO-2022-CN-EXT",
961 "[\x00-\x7f\u4e00-\u4e05\u9f98\ufe6b\u4e28-\u4e2b\U00020000\U00020003-\U00020005\U00029664]",
962 "[\U00020001\U00020002\U0002a6d7-\U0010ffff]",
963 :int{0}
964 }
965
966 // DBCS-only
967 {
968 "ibm-971",
969 "[\xa1\xa4\uac01\ub000]",
970 "[\x00-\x9f\u2015]",
971 :int{0}
972 }
973
974 {
975 "ibm-16684",
976 "[\xa0\xa1\xa4\xa6-\xab\xad-\u017f\u0254\u309b-\u30ff\u4e00-\u4e05\U00023d00\U000243bc\U0002a6b2"
977 "{\u0254\u0300}{\u0254\u0301}{\u304b\u309a}{\u30ad\u309a}{\u30af\u309a}]",
978 "[\x00-0x9f\xa2\xa3\xa5\xac\u0200-\u024f\U00010000-\U0001ffff\U0002a61b-\U0002a6b1]",
979 :int{0}
980 }
981
982 // extensions
983 {
984 "ibm-1390",
985 "[\x00-\x0d\x10-\u017f\u0254\u309b-\u30ff\u4e00-\u4e05\U00023d00\U000243bc\U0002a6b2"
986 "{\u0254\u0300}{\u0254\u0301}{\u304b\u309a}{\u30ad\u309a}{\u30af\u309a}]",
987 "[\x0e\x0f\u0200-\u024f\U00010000-\U0001ffff\U0002a61b-\U0002a6b1]",
988 :int{0}
989 }
990
991 {
992 "*test3",
993 "[\x05\x0b\xc0\u20ac\U00023456\U00101234"
994 "{\U00101234\U00050005\U00060006}{\U00101234\U00050005}{\U00101234\U00060006}{\xc4\xc4\U00101234\x05}]",
995 "[\x06\x0e\U00034567\U000febcd{\U00101234\U00070007}]",
996 :int{0}
997 }
998 }
999 }
1000 }
1001}
mirror of ICU