]> git.saurik.com Git - apple/icu.git/blame - icuSources/test/testdata/conversion.txt
projects / apple / icu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ICU-8.11.1.tar.gz
[apple/icu.git] / icuSources / test / testdata / conversion.txt
CommitLineData
374ca955
A		 1//*******************************************************************************
2//
73c04bcf 3// Copyright (C) 2003-2006, International Business Machines
374ca955
A		 4// Corporation and others. All Rights Reserved.
5//
6// file name: conversion.txt
7// encoding: US-ASCII
8// tab size: 8 (not used)
9// indentation:4
10//
11// created on: 2003jul15
12// created by: Markus W. Scherer
13//
14// ICU resource bundle source file with test data for data-driven conversion tests.
15//
16//*******************************************************************************
17
73c04bcf 18conversion:table(nofallback) {
374ca955
A		 19 Info {
20 Description { "Test data for conversion" }
21 LongDescription {
22 "Test data for data-driven conversion tests in icu/source/test/intltest/convtest.cpp\n"
23 "Run intltest conversion\n"
24
25 "Charset names starting with '*' are for testdata names.\n"
26
27 "ICU callbacks are specified as strings with pairs of characters, each optional.\n"
28 "Callback function - '?'=Sub '0'=Skip '.'=Stop '&'=Escape\n"
29 "Callback option - a letter is passed in directly as const char * see ucnv_err.h\n"
30 "Empty string: Sub callback with NULL option\n"
31
73c04bcf 32 "In order to specify a charset substitution character (for ucnv_setSubstChars()),\n"
374ca955
A		 33 "add a NUL (U+0000) to the callback string followed by the subchar bytes as Latin-1\n"
34 "characters. For example, for a Sub callback with no option and a subchar of FC FC,\n"
35 "use the string \"?\x00\xFC\xFC\"\n"
36
73c04bcf
A		 37 "In order to specify a substitution string (for ucnv_setSubstString()),\n"
38 "add an '=' to the callback string followed by the substitution string.\n"
39 "For example, for a Sub callback with no option and a substitution string\n"
40 "of \"ab\", use the string \"?=ab\"\n"
41
374ca955
A		 42 "fallbacks: per-direction boolean, currently only for fromUnicode; see Jitterbug 2401\n"
43
44 "errorCode: (empty)==zero | invalid | illegal | truncated | illesc | unsuppesc\n"
45 }
46 }
47 TestData {
48 toUnicode {
49 Headers { "charset", "bytes", "unicode", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidChars" }
50 Cases {
51 // improve coverage of unrolled loops in ucnvmbcs.c/ucnv_MBCSSingleToBMPWithOffsets()
52 {
53 "ISO-8859-3",
54 :bin{ 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f2021222324252627 },
55 "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\x22#$%&'",
56 :intvector{ 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39 },
57 :int{1}, :int{0}, "", "?", :bin{""}
58 }
59 // test that ISO-2022-JP encodes ASCII as itself
60 {
61 "ISO-2022-JP",
62 :bin{ 3f4041424344454647 },
63 "?@ABCDEFG",
64 :intvector{ 0,1,2,3,4,5,6,7,8 },
65 :int{1}, :int{1}, "", "?", :bin{""}
66 }
67 // test that ISO-2022-CN encodes ASCII as itself
68 {
69 "ISO-2022-CN",
70 :bin{ 3f4041424344454647 },
71 "?@ABCDEFG",
72 :intvector{ 0,1,2,3,4,5,6,7,8 },
73 :int{1}, :int{1}, "", "?", :bin{""}
74 }
75
76 // ISO-2022-KR
77
78 // truncated, partial escape sequence
79 {
80 "ibm-25546",
81 :bin{ 1b }, "", :intvector{},
82 :int{1}, :int{1}, "truncated", ".", :bin{ 1b }
83 }
84 {
85 "ibm-25546",
86 :bin{ 1b24 }, "", :intvector{},
87 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 }
88 }
89 {
90 "ibm-25546",
91 :bin{ 1b2429 }, "", :intvector{},
92 :int{1}, :int{1}, "truncated", ".", :bin{ 1b2429 }
93 }
94 // complete escape sequence but nothing else
95 {
96 "ibm-25546",
97 :bin{ 1b242943 }, "", :intvector{},
98 :int{1}, :int{1}, "", ".", :bin{""}
99 }
100 {
101 "ibm-25546",
102 :bin{ 1b2429430e }, "", :intvector{},
103 :int{1}, :int{1}, "", ".", :bin{""}
104 }
105 // escape plus ASCII character
106 {
107 "ibm-25546",
108 :bin{ 1b24294341 }, "A", :intvector{ 4 },
109 :int{1}, :int{1}, "", ".", :bin{""}
110 }
111 // escape plus incomplete DBCS character
112 {
113 "ibm-25546",
114 :bin{ 1b2429430e41 }, "", :intvector{},
115 :int{1}, :int{1}, "truncated", ".", :bin{ 41 }
116 }
117 // all complete with DBCS character
118 {
119 "ibm-25546",
120 :bin{ 1b2429430e4141 }, "\uc88b", :intvector{ 5 },
121 :int{1}, :int{1}, "", ".", :bin{""}
122 }
123 // more complicated example
124 {
125 "ibm-25546",
126 :bin{ 411b242943420e4141affe0f43 },
127 "AB\uc88b%XAF%XFEC",
128 :intvector{ 0, 5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 12 },
129 :int{1}, :int{1}, "", "&", :bin{""}
130 }
131
132 // truncated, partial escape sequence
133 {
134 "ISO-2022-KR",
135 :bin{ 1b }, "", :intvector{},
136 :int{1}, :int{1}, "truncated", ".", :bin{ 1b }
137 }
138 {
139 "ISO-2022-KR",
140 :bin{ 1b24 }, "", :intvector{},
141 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 }
142 }
143 {
144 "ISO-2022-KR",
145 :bin{ 1b2429 }, "", :intvector{},
146 :int{1}, :int{1}, "truncated", ".", :bin{ 1b2429 }
147 }
148 // complete escape sequence but nothing else
149 {
150 "ISO-2022-KR",
151 :bin{ 1b242943 }, "", :intvector{},
152 :int{1}, :int{1}, "", ".", :bin{""}
153 }
154 {
155 "ISO-2022-KR",
156 :bin{ 1b2429430e }, "", :intvector{},
157 :int{1}, :int{1}, "", ".", :bin{""}
158 }
159 // escape plus ASCII character
160 {
161 "ISO-2022-KR",
162 :bin{ 1b24294341 }, "A", :intvector{ 4 },
163 :int{1}, :int{1}, "", ".", :bin{""}
164 }
165 // escape plus incomplete DBCS character
166 {
167 "ISO-2022-KR",
168 :bin{ 1b2429430e41 }, "", :intvector{},
169 :int{1}, :int{1}, "truncated", ".", :bin{ 41 }
170 }
171 // all complete with DBCS character
172 {
173 "ISO-2022-KR",
174 :bin{ 1b2429430e4141 }, "\uc88b", :intvector{ 5 },
175 :int{1}, :int{1}, "", ".", :bin{""}
176 }
177 // more complicated example
178 {
179 "ISO-2022-KR",
180 :bin{ 411b242943420e4141affe0f43 },
181 "AB\uc88b%XAF%XFEC",
182 :intvector{ 0, 5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 12 },
183 :int{1}, :int{1}, "", "&", :bin{""}
184 }
d5d484b0
A		 185 // empty segment (using substitution and stop)
186 {
187 "ISO-2022-KR",
188 :bin{ 1b242943610e0f620d0a },
189 "a\uFFFDb\u000D\u000A",
190 :intvector{ 4, 6, 7, 8, 9 },
191 :int{1}, :int{1}, "", "?", :bin{""}
192 }
193 {
194 "ISO-2022-KR",
195 :bin{ 1b242943610e0f620d0a },
196 "a",
197 :intvector{ 4 },
198 :int{1}, :int{1}, "illesc", ".", :bin{"0f"}
199 }
374ca955
A		 200
201 // ISO-2022-JP
202
203 // truncated, partial escape sequence
204 {
205 "ISO-2022-JP",
206 :bin{ 1b }, "", :intvector{},
207 :int{1}, :int{1}, "truncated", ".", :bin{ 1b }
208 }
209 {
210 "ISO-2022-JP-2",
211 :bin{ 1b24 }, "", :intvector{},
212 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 }
213 }
214 // complete escape sequence but nothing else
215 {
216 "ISO-2022-JP-2",
217 :bin{ 1b2442 }, "", :intvector{},
218 :int{1}, :int{1}, "", ".", :bin{""}
219 }
220 // escape plus incomplete DBCS character
221 {
222 "ISO-2022-JP-2",
223 :bin{ 1b244241 }, "", :intvector{},
224 :int{1}, :int{1}, "truncated", ".", :bin{ 41 }
225 }
226 // all complete with DBCS character
227 {
228 "ISO-2022-JP-2",
229 :bin{ 1b24424141 }, "\u758f", :intvector{ 3 },
230 :int{1}, :int{1}, "", ".", :bin{""}
231 }
232 // test the G2 designator & SS2 shift
233 {
234 "ISO-2022-JP-2",
235 :bin{ 431b2e46461b244241411b4e4e353f }, "CF\u758f\u039e\u7591", :intvector{ 0, 4, 8, 12, 13 },
236 :int{1}, :int{1}, "", ".", :bin{""}
237 }
238 // JIS7 with Katakana
239 {
240 "JIS7",
241 :bin{ 41420e41420f4142 }, "AB\uff81\uff82AB", :intvector{ 0, 1, 3, 4, 6, 7 },
242 :int{1}, :int{1}, "", ".", :bin{""}
243 }
244 // JIS8 with Katakana
245 {
246 "JIS8",
247 :bin{ 41c15c1b284a5cc242 }, "A\uff81\\\xa5\uff82B", :intvector{ 0, 1, 2, 6, 7, 8 },
248 :int{1}, :int{1}, "", ".", :bin{""}
249 }
d5d484b0
A		 250 // empty segment (using substitution and stop)
251 {
252 "ISO-2022-JP",
253 :bin{ 61621b24421b284263640d0a },
254 "ab\uFFFDcd\u000D\u000A",
255 :intvector{ 0, 1, 5, 8, 9, 10, 11 },
256 :int{1}, :int{1}, "", "?", :bin{""}
257 }
258 {
259 "ISO-2022-JP",
260 :bin{ 61621b24421b284263640d0a },
261 "ab",
262 :intvector{ 0, 1 },
263 :int{1}, :int{1}, "illesc", ".", :bin{"1b2842"}
264 }
374ca955
A		 265
266 // ISO-2022-CN
267
268 // truncated, partial escape sequence
269 {
270 "ISO_2022,locale=zh,version=1",
271 :bin{ 1b }, "", :intvector{},
272 :int{1}, :int{1}, "truncated", ".", :bin{ 1b }
273 }
274 {
275 "ISO_2022,locale=zh,version=1",
276 :bin{ 1b24 }, "", :intvector{},
277 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 }
278 }
279 {
280 "ISO_2022,locale=zh,version=1",
281 :bin{ 1b2429 }, "", :intvector{},
282 :int{1}, :int{1}, "truncated", ".", :bin{ 1b2429 }
283 }
284 // complete escape sequence but nothing else
285 {
286 "ISO_2022,locale=zh,version=1",
287 :bin{ 1b242941 }, "", :intvector{},
288 :int{1}, :int{1}, "", ".", :bin{""}
289 }
290 {
291 "ISO_2022,locale=zh,version=1",
292 :bin{ 1b2429410e }, "", :intvector{},
293 :int{1}, :int{1}, "", ".", :bin{""}
294 }
295 // escape plus ASCII character
296 {
297 "ISO_2022,locale=zh,version=1",
298 :bin{ 1b24294141 }, "\x41", :intvector{ 4 },
299 :int{1}, :int{1}, "", ".", :bin{""}
300 }
301 // escape plus incomplete DBCS character
302 {
303 "ISO_2022,locale=zh,version=1",
304 :bin{ 1b2429410e41 }, "", :intvector{},
305 :int{1}, :int{1}, "truncated", ".", :bin{ 41 }
306 }
307 // all complete with DBCS character
308 {
309 "ISO_2022,locale=zh,version=1",
310 :bin{ 1b2429410e4141 }, "\u4eae", :intvector{ 5 },
311 :int{1}, :int{1}, "", ".", :bin{""}
312 }
313 // ISO-2022-CN-EXT with all subcharsets and shifts and with supplementary code points
314 {
315 "ISO-2022-CN-EXT",
316 :bin{ 1b2429411b242a480e41411b2429457e7c1b4e70341b242b4d1b2429477c341b4f664c2421 },
317 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6\x30",
318 :intvector{ 9, 15, 19, 29, 33, 33, 35 },
319 :int{1}, :int{1}, "", ".", :bin{""}
320 }
321
322 // illegal and unsupported escape sequences
323 // SS2 without designator: illegal
324 {
325 "ISO-2022-CN-EXT",
326 :bin{ 411b4e2121 }, "\x41", :intvector{ 0 },
327 :int{1}, :int{1}, "illesc", ".", :bin{ 1b4e }
328 }
329 // G3 designator: recognized, but not supported for -CN (only for -CN-EXT)
330 {
331 "ISO-2022-CN",
332 :bin{ 411b242b491b4f2121 }, "\x41", :intvector{ 0 },
333 :int{1}, :int{1}, "unsuppesc", ".", :bin{ 1b242b49 }
334 }
d5d484b0
A		 335 // empty segment 1 (using substitution and stop)
336 {
337 "ISO-2022-CN",
338 :bin{ 611b242941620e0f1b242a481b4e6a65630d0a },
339 "ab\uFFFD\u994Cc\u000D\u000A",
340 :intvector{ 0, 5, 7, 14, 16, 17, 18 },
341 :int{1}, :int{1}, "", "?", :bin{""}
342 }
343 {
344 "ISO-2022-CN",
345 :bin{ 611b242941620e0f1b242a481b4e6a65630d0a },
346 "ab",
347 :intvector{ 0, 5 },
348 :int{1}, :int{1}, "illesc", ".", :bin{"0f"}
349 }
350 // empty segment 2 (using substitution and stop)
351 {
352 "ISO-2022-CN",
353 :bin{ 611b242941620e1b24294768640f630d0a },
354 "ab\uFFFD\u5F70c\u000D\u000A",
355 :intvector{ 0, 5, 7, 11, 14, 15, 16 },
356 :int{1}, :int{1}, "", "?", :bin{""}
357 }
358 {
359 "ISO-2022-CN",
360 :bin{ 611b242941620e1b24294768640f630d0a },
361 "ab",
362 :intvector{ 0, 5 },
363 :int{1}, :int{1}, "illesc", ".", :bin{"1b242947"}
364 }
374ca955
A		 365
366 // ISO-2022 SBCS
367 // [U_ENABLE_GENERIC_ISO_2022]
368 // The _generic_ ISO-2022 converter is disabled starting 2003-dec-03 (ICU 2.8).
369 // For details see the icu mailing list from 2003-dec-01 and the ucnv2022.c file.
370 // Language-specific variants of ISO-2022 continue to be available as listed below.
371 //{
372 // "ISO_2022",
373 // :bin{ 0008090a0d1a1c1f203f415c7d7e7f },
374 // "\x00\x08\t\n\r\x1a\x1c\x1f ?A\\}~\x7f",
375 // :intvector{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 },
376 // :int{1}, :int{1}, "", ".", :bin{""}
377 //}
378
d5d484b0
A		 379 // HZ-GB-2312
380
381 // empty segment (using substitution and stop)
382 {
383 "HZ-GB-2312",
384 :bin{ 61627e7b7e7d6364 },
385 "ab\uFFFDcd",
386 :intvector{ 0, 1, 4, 6, 7 },
387 :int{1}, :int{1}, "", "?", :bin{""}
388 }
389 {
390 "HZ-GB-2312",
391 :bin{ 61627e7b7e7d63640d0a },
392 "ab",
393 :intvector{ 0, 1 },
394 :int{1}, :int{1}, "illesc", ".", :bin{"7e7d"}
395 }
396
374ca955
A		 397 // DBCS-only extensions
398 {
399 "ibm-970",
400 :bin{ 617eece9b2eb },
401 "\x61\x7e\u4e00\ub000",
402 :intvector{ 0, 1, 2, 4 },
403 :int{1}, :int{1}, "", "?", :bin{""}
404 }
405
406 {
407 "ibm-971",
408 :bin{ 617eece9b2eb },
409 "\ufffd\u4e00\ub000",
410 :intvector{ 0, 2, 4 },
411 :int{1}, :int{1}, "", "?", :bin{""}
412 }
413
414 {
415 "ibm-16684",
416 :bin{ 430e4395ecc1404042e1 },
417 "\ufffd\u30C8\u30C8\u309A\u3000\u20ac",
418 :intvector{ 0, 2, 4, 4, 6, 8 },
419 :int{1}, :int{0}, "", "?", :bin{""}
420 }
421
422 {
423 "ibm-1399",
424 :bin{ 430e4395ecc140400fe1 },
425 "\uff62\u30C8\u30C8\u309A\u3000\u20ac",
426 :intvector{ 0, 2, 4, 4, 6, 9 },
427 :int{1}, :int{0}, "", "?", :bin{""}
428 }
429
430 // extensions
431 {
432 "ibm-1390",
433 :bin{ 430e4395ecc1 },
434 "\uff63\u30C8\u30C8\u309A",
435 :intvector{ 0, 2, 4, 4 },
436 :int{1}, :int{0}, "", "?", :bin{""}
437 }
438
439 {
440 "ibm-16684",
441 :bin{ ececec8bec8cec8d4386ecb5ecb6ecb7 },
442 "\ufffd\u31f6\u31f7\u31f8\u30ab\u304b\u309a\u304d\u309a\u304f\u309a",
443 :intvector{ 0, 2, 4, 6, 8, 10, 10, 12, 12, 14, 14 },
444 :int{1}, :int{0}, "", "?", :bin{""}
445 }
446
447 {
448 "ibm-1390",
449 :bin{ 43860eececec8bec8cec8d4386ecb5ecb6ecb7ecc10fec },
450 "\uff63\uff76\ufffd\u31f6\u31f7\u31f8\u30ab\u304b\u309a\u304d\u309a\u304f\u309a\u30C8\u309A\x1a",
451 :intvector{ 0, 1, 3, 5, 7, 9, 11, 13, 13, 15, 15, 17, 17, 19, 19, 22 },
452 :int{1}, :int{0}, "", "?", :bin{""}
453 }
454
455 {
456 "*test3",
457 :bin{ 00050601020b0701020a01020c },
458 "\u20ac\x05\x06\x0b\U00101234\U00023456\ufffd",
459 :intvector{ 0, 1, 2, 3, 6, 6, 7, 7, 10 },
460 :int{1}, :int{0}, "", "?", :bin{""}
461 }
462
463 // normal conversions
464 {
465 "UTF-16LE",
466 :bin{ 310000d801dc00d902dc320000d8330001dc3400 },
467 "1\U00010001\U000500022\ufffd3\ufffd4",
468 :intvector{ 0, 2, 2, 6, 6, 10, 12, 14, 16, 18 },
469 :int{1}, :int{0}, "", "?", :bin{""}
470 }
471 { "UTF-16LE", :bin{ 00 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00 } }
472 { "UTF-16LE", :bin{ 00d800 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00d800 } }
473
474 {
475 "UTF-16BE",
476 :bin{ 0031d800dc01d900dc020032d8000033dc010034 },
477 "1\U00010001\U000500022\ufffd3\ufffd4",
478 :intvector{ 0, 2, 2, 6, 6, 10, 12, 14, 16, 18 },
479 :int{1}, :int{0}, "", "?", :bin{""}
480 }
481 { "UTF-16BE", :bin{ 00 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00 } }
482 { "UTF-16BE", :bin{ d800dc }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ d800dc } }
483
484 // e4b8 is a partial sequence
485 { "UTF-8", :bin{ 31e4ba8ce4b8 }, "1\u4e8c", :intvector{ 0, 1 }, :int{1}, :int{0}, "truncated", ".", :bin{ e4b8 } }
486 { "UTF-8", :bin{ 31e4ba8ce4b8 }, "1\u4e8c\ufffd", :intvector{ 0, 1, 4 }, :int{1}, :int{0}, "", "?", :bin{""} }
487
488 // LMBCS with escape callback (1292a0 is unassigned)
489 {
490 "LMBCS",
491 :bin{ 12c9501292a01292a1 },
492 "\u4e2e%X12%X92%XA0\ue5c4",
493 :intvector{ 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 6 },
494 :int{1}, :int{0}, "", "&", :bin{""}
495 }
496
497 // IMAP-mailbox-name with SUB
498 // a<DEL> a&AB~ a&AB\x0c a&AB- a&AB. a&.
499 {
500 "IMAP-mailbox-name",
501 :bin{ 617f612641427e612641420c612641422d612641422e61262e },
502 "a\ufffda\ufffda\ufffda\ufffda\ufffda\ufffd",
503 :intvector{ 0, 1, 2, 4, 7, 9, 12, 14, 17, 19, 22, 23 },
504 :int{1}, :int{0}, "", "?", :bin{""}
505 }
506
507 // using testdata_test1.cnv
508 { "*test1", :bin{ 000506070809 }, "\u20ac\x05\x06\U00101234\ufffd\ufffd", :intvector{ 0, 1, 2, 3, 3, 4, 5 }, :int{1}, :int{0}, "", "", :bin{""} }
509
510 // surrogates in CESU-8
511 { "CESU-8", :bin{ eda080eda081edb081 }, "\ud800\U00010401", :intvector{ 0, 3, 6 }, :int{1}, :int{0}, "", "", :bin{""} }
512 // e080 is a partial sequence
513 { "UTF-8", :bin{ 31ffe4ba8ce08061 }, "1\ufffd\u4e8c\ufffda", :intvector{ 0, 1, 2, 5, 7 }, :int{0}, :int{0}, "", "", :bin{ e080 } }
514 // fbbfbfbfbf exceedes U+10ffff
515 { "UTF-8", :bin{ 31fbbfbfbfbf61 }, "1\ufffda", :intvector{ 0, 1, 6 }, :int{0}, :int{0}, "", "", :bin{ fbbfbfbfbf } }
516
517 // lead byte a2 without trail byte
518 { "ibm-1363", :bin{ a2aea2 }, "\u00a1", :intvector{ 0 }, :int{1}, :int{0}, "truncated", ".", :bin{ a2 } }
519 { "ibm-1363", :bin{ a2aea2 }, "\u00a1\u001a", :intvector{ 0, 2 }, :int{1}, :int{0}, "", "?", :bin{""} }
520
521 // simple sample, no error handling
522 { "UTF-8", :bin{ 61F48FBFBF }, "a\U0010FFFF", :intvector{ 0, 1, 1 }, :int{1}, :int{0}, "", "", :bin{""} }
523 }
524 }
525
526 // --------------------------------------------------------------------- ***
527
528 fromUnicode {
529 Headers { "charset", "unicode", "bytes", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidUChars" }
530 Cases {
73c04bcf
A		 531 {
532 "SCSU",
533 "1\U00010001\u0085\U000500022\ud8003\udc014\ue001",
534 :bin{ 310be0008102050fd900dc02e7320efffd330efffd34186881 },
535 :intvector{ 0,1,1,1,1,3,3,4,4,4,4,4,6,6,7,7,7,8,9,9,9,10,11,11,11 },
536 :int{1}, :int{0}, "", "?", ""
537 }
538 // Test substitution strings.
539 {
540 "windows-1252", // stateless MBCS with WriteSub(), should internally set char *
541 "a\ufdd0c",
542 :bin{ 61402421402463 },
543 :intvector{ 0,1,1,1,1,1,2 },
544 :int{1}, :int{0}, "", "?=@$!@$", ""
545 }
546 {
547 "ibm-930", // stateful MBCS
548 "a\ufdd0\u4e00\ufdd0e",
549 :bin{ 620e4bce0f400e45414bce0f4066 },
550 :intvector{ 0,1,1,1,1,1,2,2,2,3,3,3,3,4 },
551 :int{1}, :int{0}, "", "?=\u4e01 ", ""
552 }
553 {
554 "iso-2022-jp",
555 "a\x1bc", // Unicode ESC must not occur as a character
556 :bin{ 6163 },
557 :intvector{ 0,2 },
558 :int{1}, :int{0}, "", "?=", "" // empty substitution string
559 }
560 {
561 "iso-2022-cn",
562 "a\x1bc", // Unicode ESC must not occur as a character
563 :bin{ 61202063 },
564 :intvector{ 0,1,1,2 },
565 :int{1}, :int{0}, "", "?= ", ""
566 }
567 {
568 "iso-2022-cn",
569 "a\x1bc", // Unicode ESC must not occur as a character
570 :bin{ 611b2429410e523b0f2063 },
571 :intvector{ 0,1,1,1,1,1,1,1,1,1,2 },
572 :int{1}, :int{0}, "", "?=\u4e00 ", ""
573 }
574 {
575 "us-ascii",
576 "a\x85c",
577 :bin{ 61402421402463 },
578 :intvector{ 0,1,1,1,1,1,2 },
579 :int{1}, :int{0}, "", "?=@$!@$", ""
580 }
581 // ISO 2022-CN: test a single-byte subchar, j5171
582 {
583 "iso-2022-cn",
584 "a\x1bc", // Unicode ESC must not occur as a character
585 :bin{ 612163 },
586 :intvector{ 0,1,2 },
587 :int{1}, :int{0}, "", "?\x00\x21", ""
588 }
589 // UTF-16/32: do not output a BOM if there is no data at all
590 {
591 "UTF-16",
592 "",
593 :bin{ "" },
594 :intvector{ },
595 :int{1}, :int{1}, "", "?", ""
596 }
597 {
598 "UTF-32",
599 "",
600 :bin{ "" },
601 :intvector{ },
602 :int{1}, :int{1}, "", "?", ""
603 }
604
605 // do not convert SO/SI/ESC
606 {
607 "iso-2022-jp",
608 "A\x0eB\x0f\x09\x1bC",
609 :bin{ 411a421a091a43 },
610 :intvector{ 0,1,2,3,4,5,6 },
611 :int{1}, :int{1}, "", "?", ""
612 }
613 {
614 "iso-2022-cn",
615 "A\x0eB\x0f\x09\x1bC",
616 :bin{ 411a421a091a43 },
617 :intvector{ 0,1,2,3,4,5,6 },
618 :int{1}, :int{1}, "", "?", ""
619 }
620 {
621 "iso-2022-kr",
622 "A\x0eB\x0f\x09\x1bC",
623 :bin{ 1b242943411a421a091a43 },
624 :intvector{ -1,-1,-1,-1,0,1,2,3,4,5,6 },
625 :int{1}, :int{1}, "", "?", ""
626 }
627 {
628 "ibm-25546",
629 "A\x0eB\x0f\x09\x1bC",
630 :bin{ 1b242943411a421a091a43 },
631 :intvector{ -1,-1,-1,-1,0,1,2,3,4,5,6 },
632 :int{1}, :int{1}, "", "?", ""
633 }
634
635 // test ISO 8859-1/7 vs. JIS X 0201
636 {
637 "ISO-2022-JP-2",
638 "?@A\u00e4\u03b1\u203EB",
639 :bin{ 3f40411B2E411B4E641B2E461B4E611B284A7E421B2842 },
640 :intvector{ 0,1,2,3,3,3,3,3,3,4,4,4,4,4,4,5,5,5,5,6,6,6,6 },
641 :int{1}, :int{1}, "", "?", ""
642 }
643
374ca955
A		 644 // Improve ucnv_ext.c code coverage:
645 // There will be a partial match up to the lead surrogate of U+603ff
646 // which then results in one more unit in the prefetch buffer
647 // than the match length when converting one code unit at a time.
648 // See ucnv_extContinueMatchFromU() comment
649 // "the match did not use all of preFromU[] - keep the rest for replay"
650 {
651 "*test3",
652 "\U00101234\U00101234\U00050005\U000603ff",
653 :bin{ 07070001020e05ff },
654 :intvector{ 0, 2, 2, 2, 2, 2, 2, 6 },
655 :int{1}, :int{0}, "", "?", ""
656 }
657
658 // test that ISO-2022-JP encodes ASCII as itself
659 {
660 "ISO-2022-JP",
661 "?@ABCDEFG",
662 :bin{ 3f4041424344454647 },
663 :intvector{ 0,1,2,3,4,5,6,7,8 },
664 :int{1}, :int{1}, "", "?", ""
665 }
666 // test that ISO-2022-CN encodes ASCII as itself
667 {
668 "ISO-2022-CN",
669 "?@ABCDEFG",
670 :bin{ 3f4041424344454647 },
671 :intvector{ 0,1,2,3,4,5,6,7,8 },
672 :int{1}, :int{1}, "", "?", ""
673 }
674
675 // moved from cintltst /tsconv/nccbtst/TestSkipCallBack
676 {
677 "iso-2022-jp",
678 "\u3000\xe9\u3001",
679 :bin{ 1b2442212121221b2842 },
680 :intvector{ 0,0,0,0,0,2,2,2,2,2 },
681 :int{1}, :int{1}, "", "0", ""
682 }
683 // moved from cintltst /tsconv/nccbtst/TestSubCallBack
684 {
685 "iso-2022-jp",
686 "A\xe9B\xe9\u3000",
687 :bin{ 411a421a1b244221211b2842 },
688 :intvector{ 0,1,2,3,4,4,4,4,4,4,4,4 },
689 :int{1}, :int{1}, "", "?", ""
690 }
691 // moved from cintltst /tsconv/nccbtst/TestSubWithValueCallBack
692 {
693 "iso-2022-jp",
694 "A\xe9B\xe9\u3000",
695 :bin{ 41255530304539422555303045391b244221211b2842 },
696 :intvector{ 0,1,1,1,1,1,1,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4 },
697 :int{1}, :int{1}, "", "&", ""
698 }
699 {
700 "iso-2022-cn",
701 "\u4e00\u3712\u4e01",
702 :bin{ 1b2429410e523b0f2555333731320e36210f },
703 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,1,2,2,2,2 },
704 :int{1}, :int{1}, "", "&", ""
705 }
706 {
707 "iso-2022-cn",
708 "A\u3712\u4e00",
709 :bin{ 412555333731321b2429410e523b0f },
710 :intvector{ 0,1,1,1,1,1,1,2,2,2,2,2,2,2,2 },
711 :int{1}, :int{1}, "", "&", ""
712 }
713 {
714 "iso-2022-cn",
715 "\u3000\u3712\u3001",
716 :bin{ 1b2429410e21210f2555333731320e21220f },
717 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,1,2,2,2,2 },
718 :int{1}, :int{1}, "", "&", ""
719 }
720
721 // moved from cintltst /tsconv/nucnvtst/TestJIS
722 {
723 "JIS",
724 "\uFF81\uFF82\u30EC\u30ED\u30EE\u30EF\uFF93\uFF94\uFF95\uFF96\uFF97\uFF98",
725 :bin{ 1b244225412544256c256d256e256F25622564256625682569256a1b2842 },
726 :intvector{ 0,0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,11,11,11 },
727 :int{1}, :int{1}, "", "?", ""
728 }
729 {
730 "JIS7",
731 "\uFF81\uFF82\u30EC\u30ED\u30EE\u30EF\uFF93\uFF94\uFF95\uFF96\uFF97\uFF98",
732 :bin{ 0e41420f1b2442256c256d256e256F0e5354555657580f1b2842 },
733 :intvector{ 0,0,1,2,2,2,2,2,2,3,3,4,4,5,5,6,6,7,8,9,10,11,11,11,11,11 },
734 :int{1}, :int{1}, "", "?", ""
735 }
736 {
737 "JIS8",
738 "\uFF81\uFF82\u30EC\u30ED\u30EE\u30EF\uFF93\uFF94\uFF95\uFF96\uFF97\uFF98",
739 :bin{ C1C21b2442256c256d256e256F1b284AD3D4D5D6D7D81b2842 },
740 :intvector{ 0,1,2,2,2,2,2,3,3,4,4,5,5,6,6,6,6,7,8,9,10,11,11,11,11 },
741 :int{1}, :int{1}, "", "?", ""
742 }
743
744 // moved from cintltst /tsconv/ncnvtst/TestErrorBehaviour
745 {
746 "iso-2022-jp",
747 "\u3000\x50\udc01\u3001",
748 :bin{ 1B244221211B2842501A1B24422122 },
749 :intvector{ 0,0,0,0,0,1,1,1,1,2,3,3,3,3,3 },
750 :int{0}, :int{1}, "", "?", "\udc01"
751 }
752 {
753 "iso-2022-jp",
754 "\u3000\x50\udc01\u3001",
755 :bin{ 1B244221211B2842501A1B244221221b2842 },
756 :intvector{ 0,0,0,0,0,1,1,1,1,2,3,3,3,3,3,3,3,3 },
757 :int{1}, :int{1}, "", "?", ""
758 }
759 {
760 "iso-2022-kr",
761 "\x61\u4e00\udc01\u4e00",
762 :bin{ 1b242943610e6c690f1a0e6c69 },
763 :intvector{ -1,-1,-1,-1,0,1,1,1,2,2,3,3,3 },
764 :int{0}, :int{1}, "", "?", "\udc01"
765 }
766 {
767 "iso-2022-kr",
768 "\x61\u4e00\udc01\u4e00",
769 :bin{ 1b242943610e6c690f1a0e6c690f },
770 :intvector{ -1,-1,-1,-1,0,1,1,1,2,2,3,3,3,3 },
771 :int{1}, :int{1}, "", "?", ""
772 }
773
774 // ISO-2022-KR
775 {
776 "ibm-25546",
777 "AB\uc88b\U00050005\uacccC",
778 :bin{ 1b24294341420e41410f7b552b35303030357d0e306a0f43 },
779 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,3,3,3,3,3,3,3,3,5,5,5,6,6 },
780 :int{1}, :int{1}, "", "&U", ""
781 }
782 {
783 "ibm-25546",
784 "AB\uc88b\U00050005\uacccC",
785 :bin{ 1b24294341420e41410f1a0e306a0f43 },
786 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5,6,6 },
787 :int{1}, :int{1}, "", "?\x00\x1a", ""
788 }
789 {
790 "ibm-25546",
791 "AB\uc88b\U00050005\uacccC",
792 :bin{ 1b24294341420e41412f7e306a0f43 },
793 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,6,6 },
794 :int{1}, :int{1}, "", "?", ""
795 }
796 {
797 "ibm-25546",
798 "AB\uc88b\U00050005\uaccc",
799 :bin{ 1b24294341420e41412f7e306a0f },
800 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5 },
801 :int{1}, :int{1}, "", "?", ""
802 }
803 {
804 "ISO-2022-KR",
805 "AB\uc88b\U00050005\uacccC",
806 :bin{ 1b24294341420e41410f7b552b35303030357d0e306a0f43 },
807 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,3,3,3,3,3,3,3,3,5,5,5,6,6 },
808 :int{1}, :int{1}, "", "&U", ""
809 }
810 {
811 "ISO-2022-KR",
812 "AB\uc88b\U00050005\uacccC",
813 :bin{ 1b24294341420e41410f1a0e306a0f43 },
814 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5,6,6 },
815 :int{1}, :int{1}, "", "?", ""
816 }
817 {
818 "ISO-2022-KR",
819 "AB\uc88b\U00050005\uacccC",
820 :bin{ 1b24294341420e41412f7e306a0f43 },
821 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,6,6 },
822 :int{1}, :int{1}, "", "?\x00\x2f\x7e", ""
823 }
824 {
825 "ISO-2022-KR",
826 "AB\uc88b\U00050005\uaccc",
827 :bin{ 1b24294341420e41412f7e306a0f },
828 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5 },
829 :int{1}, :int{1}, "", "?\x00\x2f\x7e", ""
830 }
831
832 // ISO-2022-JP-2 with G2 designator & SS2 shift
833 {
834 "ISO-2022-JP-2",
835 "CF\u758f\u038f\u7591",
836 :bin{ 43461b244241411b2e461b4e3f353f1b2842 },
837 :intvector{ 0,1,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4 },
838 :int{1}, :int{1}, "", ".", ""
839 }
840 // JIS7 with Katakana
841 {
842 "JIS7",
843 "AB\uff81\uff82AB",
844 :bin{ 41420e41420f4142 },
845 :intvector{ 0,1,2,2,3,4,4,5 },
846 :int{1}, :int{1}, "", ".", ""
847 }
848 // JIS7 with shift to ASCII at the very end
849 {
850 "JIS7",
851 "AB\uff81\uff82",
852 :bin{ 41420e41420f },
853 :intvector{ 0,1,2,2,3,3 },
854 :int{1}, :int{1}, "", ".", ""
855 }
856 // JIS8 with Katakana
857 {
858 "JIS8",
859 "A\uff81\\\xa5\uff82B",
860 :bin{ 41c15c1b284a5cc2421b2842 },
861 :intvector{ 0,1,2,3,3,3,3,4,5,5,5,5 },
862 :int{1}, :int{1}, "", ".", ""
863 }
864
865 // ISO-2022-CN-EXT with all subcharsets and shifts and with supplementary code points
866 {
867 "ISO-2022-CN-EXT",
868 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6\x30",
869 :bin{ 1b2429410e41411b2429457e7c1b242a481b4e70341b2429477c341b242b4d1b4f664c0f30 },
870 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4,6,6 },
871 :int{1}, :int{1}, "", ".", ""
872 }
873 // ISO-2022-CN-EXT with shift to ASCII at the very end
874 {
875 "ISO-2022-CN-EXT",
876 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6",
877 :bin{ 1b2429410e41411b2429457e7c1b242a481b4e70341b2429477c341b242b4d1b4f664c0f },
878 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4 },
879 :int{1}, :int{1}, "", ".", ""
880 }
881 // ISO-2022-CN-EXT without flush so do not shift to ASCII at the very end
882 {
883 "ISO-2022-CN-EXT",
884 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6",
885 :bin{ 1b2429410e41411b2429457e7c1b242a481b4e70341b2429477c341b242b4d1b4f664c },
886 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4 },
887 :int{0}, :int{1}, "", ".", ""
888 }
889
890 // windows-936 vs. ibm-1386
891 {
892 "ibm-1386",
893 "\x1a\u20ac\u5555\x80\x81\U00055555",
894 :bin{ 7fa2e3dffb7f7fa1a1 },
895 :intvector{ 0, 1, 1, 2, 2, 3, 4, 5, 5 },
896 :int{1}, :int{1}, "", "?", ""
897 }
898 {
899 "windows-936",
900 "\x1a\u20ac\u5555\x80\x81\U00055555",
901 :bin{ 1a80dffb3f3f3f },
902 :intvector{ 0, 1, 2, 2, 3, 4, 5 },
903 :int{1}, :int{1}, "", "?", ""
904 }
905
906 // verify that if a conversion table does not have any mapping for U+0000,
907 // then there will not even be a phantom fallback to 00
908 {
909 "ibm-971",
910 "\x00",
911 :bin{ affe },
912 :intvector{ 0, 0 },
913 :int{1}, :int{1}, "", "?", ""
914 }
915
916 {
917 "*test4",
918 "\x00",
919 :bin{ ff },
920 :intvector{ 0 },
921 :int{1}, :int{1}, "", "?", ""
922 }
923
924 // extension in testdata
925 {
926 "*test4x",
927 "\u20ac\x09",
928 :bin{ 0009 },
929 :intvector{ 0, 1 },
930 :int{1}, :int{1}, "", "?", ""
931 }
932
933 // DBCS-only extensions
934 {
935 "ibm-970",
936 "\x61\uffa1\u2015\ub000",
937 :bin{ 611aa1aab2eb },
938 :intvector{ 0, 1, 2, 2, 3, 3 },
939 :int{1}, :int{1}, "", "?", ""
940 }
941
942 {
943 "ibm-971",
944 "\x61\uffa1\u2015\ub000",
945 :bin{ affeaffeaffeb2eb },
946 :intvector{ 0, 0, 1, 1, 2, 2, 3, 3 },
947 :int{1}, :int{1}, "", "?", ""
948 }
949
950 {
951 "ibm-1390,swaplfnl",
952 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a",
953 :bin{ 430e4395ecc140400fc1e115 },
954 :intvector{ 0, 1, 1, 1, 2, 2, 4, 4, 5, 5, 6, 7 },
955 :int{1}, :int{0}, "", "?", ""
956 }
957
958 {
959 "ibm-16684",
960 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a",
961 :bin{ fefe4395ecc14040fefe42e1fefe },
962 :intvector{ 0, 0, 1, 1, 2, 2, 4, 4, 5, 5, 6, 6, 7, 7 },
963 :int{1}, :int{0}, "", "?", ""
964 }
965
966 {
967 "ibm-1399",
968 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a",
969 :bin{ 440e4395ecc140400fc1e125 },
970 :intvector{ 0, 1, 1, 1, 2, 2, 4, 4, 5, 5, 6, 7 },
971 :int{1}, :int{0}, "", "?", ""
972 }
973
974 // <subchar1> from |2 mappings
975 {
976 "ibm-1390",
977 "\x0e\x0f\u0901\U00050000\uffe8\uffee",
978 :bin{ 3f3f0efefefefe0f3f3f },
979 :intvector{ 0, 1, 2, 2, 2, 3, 3, 5, 5, 6 },
980 :int{1}, :int{1}, "", "?", ""
981 }
982
983 // <subchar1> from |2 mappings, and also contains a fallback to 00
984 {
985 "*test4",
986 "\u20ac\u20ad\U00050005\U00023456\U0010ffff\x30",
987 :bin{ 0000e10102030affff },
988 :intvector{ 0, 1, 2, 4, 4, 4, 4, 6, 8 },
989 :int{1}, :int{1}, "", "?", ""
990 }
991
992 // setting a <subchar> resets the <subchar1>
993 {
994 "*test4",
995 "\u20ac\u20ad\U00050005\U00023456\U0010ffff\x30",
996 :bin{ 00000102030f0102030a0102030f0102030f },
997 :intvector{ 0, 1, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6, 8, 8, 8, 8 },
998 :int{1}, :int{1}, "", "?\x00\x01\x02\x03\x0f", ""
999 }
1000
1001 // fallback to 00 with old single-byte data structure
1002 {
1003 "*test1",
1004 "\u20ac\u20ad\U00101234\U00050000",
1005 :bin{ 000007ff },
1006 :intvector{ 0, 1, 2, 4 },
1007 :int{1}, :int{1}, "", "?", ""
1008 }
1009
1010 // extensions
1011 {
1012 "ibm-1390",
1013 "\u025a\u025a\u0300\u025a\u0301\u025a\u0302\uffe8\U0002a0f9",
1014 :bin{ 0ed896eccaeccbd896ea530f3f0eb7c20f },
1015 :intvector{ 0, 0, 0, 1, 1, 3, 3, 5, 5, 6, 6, 7, 7, 8, 8, 8, 8 },
1016 :int{1}, :int{0}, "", "?", ""
1017 }
1018
1019 {
1020 "*test3",
1021 "\xc4\xc4\xc4\U00101234\xc4\xc4\U00101234\x05",
1022 :bin{ ffffff070501020c },
1023 :intvector{ 0, 1, 2, 3, 5, 5, 5, 5 },
1024 :int{1}, :int{0}, "", "?", ""
1025 }
1026
1027 {
1028 "*test3",
1029 "\U00101234\U00101234\U00050005\U00101234\U00050005\U00060006",
1030 :bin{ 07070001020e05070001020f09 },
1031 :intvector{ 0, 2, 2, 2, 2, 2, 2, 6, 6, 6, 6, 6, 6 },
1032 :int{1}, :int{0}, "", "?", ""
1033 }
1034
1035 // normal conversions
1036 {
1037 "UTF-16LE",
1038 "1\U00010001\U000500022\ud8003\udc014",
1039 :bin{ 310000d801dc00d902dc3200fdff3300fdff3400 },
1040 :intvector{ 0, 0, 1, 1, 1, 1, 3, 3, 3, 3, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9 },
1041 :int{1}, :int{0}, "", "?", ""
1042 }
1043 { "UTF-16LE", "\ud800", :bin{""}, :intvector{}, :int{1}, :int{0}, "truncated", ".", "\ud800" }
1044
1045 {
1046 "UTF-16BE",
1047 "1\U00010001\U000500022\ud8003\udc014",
1048 :bin{ 0031d800dc01d900dc020032fffd0033fffd0034 },
1049 :intvector{ 0, 0, 1, 1, 1, 1, 3, 3, 3, 3, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9 },
1050 :int{1}, :int{0}, "", "?", ""
1051 }
1052 { "UTF-16BE", "\ud800", :bin{""}, :intvector{}, :int{1}, :int{0}, "truncated", ".", "\ud800" }
1053
73c04bcf
A		 1054 {
1055 "SCSU",
1056 "1\U00010001\u00082\ud8003\udc014\ue001",
1057 :bin{ 310be000810108320efffd330efffd34186881 },
1058 :intvector{ 0, 1, 1, 1, 1, 3, 3, 4, 5, 5, 5, 6, 7, 7, 7, 8, 9, 9, 9 },
1059 :int{1}, :int{0}, "", "?", ""
1060 }
1061
374ca955
A		 1062 // escape callback
1063 {
1064 "ISCII",
1065 "A\u0901\U00023456\u0902B\U00023456C",
1066 :bin{ 41ef42a1255544383444255544433536a24225554438344425554443353643 },
1067 :intvector{
1068 0,
1069 1,1,1,
1070 2,2,2,2,2,2,
1071 2,2,2,2,2,2,
1072 4,
1073 5,
1074 6,6,6,6,6,6,
1075 6,6,6,6,6,6,
1076 8
1077 },
1078 :int{1}, :int{0}, "", "&", ""
1079 }
1080
1081 // escape callback (hex)
1082 {
1083 "iso-2022-jp",
1084 "\u3000\U00023456\u3001\U00023456B\u901c",
1085 :bin{ 1b244221211b284226237832333435363b1b244221221b284226237832333435363b42262378393031433b },
1086 :intvector{
1087 0,0,0,0,0,
1088 1,1,1,1,1,1,1,1,1,1,1,1,
1089 3,3,3,3,3,
1090 4,4,4,4,4,4,4,4,4,4,4,4,
1091 6,
1092 7,7,7,7,7,7,7,7
1093 },
1094 :int{1}, :int{0}, "", "&X", ""
1095 }
1096
1097 // sub callback
1098 {
1099 "gb18030",
1100 "$\x7f\x80\u01f9\u20ac\u4e00\u9fa6\uffff\U00010000\U0010ffff",
1101 :bin{ 247f81308130a8bfa2e3d2bb82358f338431a43990308130e3329a35 },
1102 :intvector{ 0, 1, 2, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 10, 10, 10, 10 },
1103 :int{1}, :int{0}, "", "?", ""
1104 }
1105
1106 // skip callback
1107 { "ibm-930", "\u6D63\u6D64\u6D65\u6D66", :bin{ 0e5d5f5d63466b0f }, :intvector{ 0, 0, 0, 1, 1, 3, 3, 3 }, :int{1}, :int{0}, "", "0", "" }
1108 { "ibm-930", "\u6D63\u6D64\ud89a\u6D66", :bin{ 0e5d5f5d63466b0f }, :intvector{ 0, 0, 0, 1, 1, 3, 3, 3 }, :int{1}, :int{0}, "", "0", "" }
1109 { "ibm-930", "\u6D63\u6D64\ud89a\u6D66", :bin{ 0e5d5f5d63 }, :intvector{ 0, 0, 0, 1, 1 }, :int{1}, :int{0}, "illegal", "0i", "\ud89a" }
1110
1111 // sub callback for supplementary code point
1112 { "LATIN1", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "", "" }
1113 { "ibm-920", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "", "" }
1114
1115 // sub callback with AA as subchar
1116 { "ibm-920", "1\U000104012", :bin{ 31AA32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "?\x00\xAA", "" }
1117
1118 // same but not flushing
1119 { "LATIN1", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{0}, :int{0}, "", "", "\U00010401" }
1120 { "ibm-920", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{0}, :int{0}, "", "", "\U00010401" }
1121
1122 // simple sample, no error handling
1123 { "UTF-8", "a\U0010FFFF", :bin{ 61F48FBFBF }, :intvector{ 0, 1, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" }
1124 }
1125 }
1126
1127 getUnicodeSet {
1128 // charset - will be opened, and ucnv_getUnicodeSet() called on it
1129 // map - set of code points and strings that must be in the returned set
1130 // mapnot - set of code points and strings that must *not* be in the returned set
1131 // which - numeric UConverterUnicodeSet value
1132 Headers { "charset", "map", "mapnot", "which" }
1133 Cases {
73c04bcf
A		 1134 // Unicode charsets that do not map surrogate code points
1135 {
1136 "UTF-8",
1137 "[\x00-\ud7ff\ue000-\U0010ffff]",
1138 "[\ud800-\udfff]",
1139 :int{0}
1140 }
1141 {
1142 "UTF-16",
1143 "[\x00-\ud7ff\ue000-\U0010ffff]",
1144 "[\ud800-\udfff]",
1145 :int{0}
1146 }
1147 {
1148 "UTF-16BE",
1149 "[\x00-\ud7ff\ue000-\U0010ffff]",
1150 "[\ud800-\udfff]",
1151 :int{0}
1152 }
1153 {
1154 "UTF-16LE",
1155 "[\x00-\ud7ff\ue000-\U0010ffff]",
1156 "[\ud800-\udfff]",
1157 :int{0}
1158 }
1159 {
1160 "UTF-32",
1161 "[\x00-\ud7ff\ue000-\U0010ffff]",
1162 "[\ud800-\udfff]",
1163 :int{0}
1164 }
1165 {
1166 "UTF-32BE",
1167 "[\x00-\ud7ff\ue000-\U0010ffff]",
1168 "[\ud800-\udfff]",
1169 :int{0}
1170 }
1171 {
1172 "UTF-32LE",
1173 "[\x00-\ud7ff\ue000-\U0010ffff]",
1174 "[\ud800-\udfff]",
1175 :int{0}
1176 }
1177
1178 // Unicode charsets that do map surrogate code points
1179 {
1180 "UTF-7",
1181 "[\x00-\U0010ffff]",
1182 "[]",
1183 :int{0}
1184 }
1185 {
1186 "SCSU",
1187 "[\x00-\U0010ffff]",
1188 "[]",
1189 :int{0}
1190 }
1191 {
1192 "BOCU-1",
1193 "[\x00-\U0010ffff]",
1194 "[]",
1195 :int{0}
1196 }
1197 {
1198 "CESU-8",
1199 "[\x00-\U0010ffff]",
1200 "[]",
1201 :int{0}
1202 }
1203
1204 // versions of ISO-2022-KR
374ca955
A		 1205 {
1206 "ISO-2022-KR",
73c04bcf
A		 1207 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa1\xa4\xfe\u0111\u4e00\u4e01\uac00-\uac02\uffe6]",
1208 "[\x0e\x0f\x1b\x80-\xa0\xa3\xa5\xff-\u0110\uac03\uffe7-\U0010ffff]",
1209 :int{0}
1210 }
1211 {
1212 "ibm-25546",
1213 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa1\xa4\xfe\u0111\u4e00\u4e01\uac00-\uac01\uffe6]",
1214 "[\x0e\x0f\x1b\x80-\xa0\xa3\xa5\xff-\u0110\uac02\uffe7-\U0010ffff]",
374ca955
A		 1215 :int{0}
1216 }
1217
1218 // versions of ISO-2022-JP
1219 {
1220 "ISO-2022-JP",
73c04bcf
A		 1221 "[\x00-\x0d\x10-\x1a\x1c-\x7f\u0391-\u03a1\uff61-\uff9f\u4e00\u4e01\uffe5]",
1222 "[\x0e\x0f\x1b\u0100-\u0113\u0385-\u038a\u4e02\u4e27-\u4e29\uffe6-\U0010ffff]",
374ca955
A		 1223 :int{0}
1224 }
1225 {
1226 "ISO-2022-JP-2",
73c04bcf
A		 1227 "[\x00-\x0d\x10-\x1a\x1c-\u0113\u0385-\u038a\u0390-\u03a1\uff61-\uff9f\u4e00-\u4e05\uffe6]",
1228 "[\x0e\x0f\x1b\uffe7-\U0010ffff]",
374ca955
A		 1229 :int{0}
1230 }
1231
1232 // versions of ISO-2022-CN
1233 {
1234 "ISO-2022-CN",
73c04bcf
A		 1235 "[\x00-\x0d\x10-\x1a\x1c-\x7f\u4e00\u4e01\u9f98\ufe6b]",
1236 "[\x0e\x0f\x1b\u4e29\uffe6-\U0010ffff]",
374ca955
A		 1237 :int{0}
1238 }
1239 {
1240 "ISO-2022-CN-EXT",
73c04bcf
A		 1241 "[\x00-\x0d\x10-\x1a\x1c-\x7f\u4e00-\u4e05\u9f98\ufe6b\u4e28-\u4e2b\U00020000\U00020003-\U00020005\U00029664]",
1242 "[\x0e\x0f\x1b\U00020001\U00020002\U0002a6d7-\U0010ffff]",
374ca955
A		 1243 :int{0}
1244 }
1245
1246 // DBCS-only
1247 {
1248 "ibm-971",
1249 "[\xa1\xa4\uac01\ub000]",
1250 "[\x00-\x9f\u2015]",
1251 :int{0}
1252 }
1253
1254 {
1255 "ibm-16684",
1256 "[\xa0\xa1\xa4\xa6-\xab\xad-\u017f\u0254\u309b-\u30ff\u4e00-\u4e05\U00023d00\U000243bc\U0002a6b2"
1257 "{\u0254\u0300}{\u0254\u0301}{\u304b\u309a}{\u30ad\u309a}{\u30af\u309a}]",
1258 "[\x00-0x9f\xa2\xa3\xa5\xac\u0200-\u024f\U00010000-\U0001ffff\U0002a61b-\U0002a6b1]",
1259 :int{0}
1260 }
1261
1262 // extensions
1263 {
1264 "ibm-1390",
1265 "[\x00-\x0d\x10-\u017f\u0254\u309b-\u30ff\u4e00-\u4e05\U00023d00\U000243bc\U0002a6b2"
1266 "{\u0254\u0300}{\u0254\u0301}{\u304b\u309a}{\u30ad\u309a}{\u30af\u309a}]",
1267 "[\x0e\x0f\u0200-\u024f\U00010000-\U0001ffff\U0002a61b-\U0002a6b1]",
1268 :int{0}
1269 }
1270
1271 {
1272 "*test3",
1273 "[\x05\x0b\xc0\u20ac\U00023456\U00101234"
1274 "{\U00101234\U00050005\U00060006}{\U00101234\U00050005}{\U00101234\U00060006}{\xc4\xc4\U00101234\x05}]",
1275 "[\x06\x0e\U00034567\U000febcd{\U00101234\U00070007}]",
1276 :int{0}
1277 }
1278 }
1279 }
1280 }
1281}
mirror of ICU