X-Git-Url: https://git.saurik.com/apple/icu.git/blobdiff_plain/fd0068a84e9996f225edba706498f6ed413d0673..4d9eefca008a7bc544144ef830fa144ce89deaa0:/icuSources/test/testdata/conversion.txt diff --git a/icuSources/test/testdata/conversion.txt b/icuSources/test/testdata/conversion.txt index 45ae1e69..4e434c66 100644 --- a/icuSources/test/testdata/conversion.txt +++ b/icuSources/test/testdata/conversion.txt @@ -1,6 +1,6 @@ //******************************************************************************* // -// Copyright (C) 2003-2006,2008 International Business Machines +// Copyright (C) 2003-2013, International Business Machines // Corporation and others. All Rights Reserved. // // file name: conversion.txt @@ -23,6 +23,7 @@ conversion:table(nofallback) { "Run intltest conversion\n" "Charset names starting with '*' are for testdata names.\n" + "Charset names starting with '+' are for charsets currently not supported in ICU4J.\n" "ICU callbacks are specified as strings with pairs of characters, each optional.\n" "Callback function - '?'=Sub '0'=Skip '.'=Stop '&'=Escape\n" @@ -48,6 +49,39 @@ conversion:table(nofallback) { toUnicode { Headers { "charset", "bytes", "unicode", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidChars" } Cases { + // Test ticket 6789: implement Java-compatible Unicode, UnicodeBig and UnicodeLittle converters + // For details about these encodings see convrtrs.txt. + // Standard UTF-16 + { "UTF-16", :bin{ 00610062 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + { "UTF-16", :bin{ feff0061 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + { "UTF-16", :bin{ fffe0061 }, "\u6100", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + // Java "Unicode" requires a BOM + { "+UTF-16,version=1", :bin{ 00610062 }, "\\x00\\x61b", :intvector{ 0,0,0,0,0,0,0,0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + { "+UTF-16,version=1", :bin{ feff0061 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + { "+UTF-16,version=1", :bin{ fffe0061 }, "\u6100", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + // Standard UTF-16BE + { "UTF-16BE", :bin{ 00610062 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + { "UTF-16BE", :bin{ feff0061 }, "\ufeffa", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + { "UTF-16BE", :bin{ fffe0061 }, "\ufffea", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + // Java "UnicodeBig" requires a BE BOM or no BOM; it consumes the BE BOM + { "UTF-16BE,version=1",:bin{ 00610062 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + { "UnicodeBig", :bin{ feff0061 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + { "UnicodeBig", :bin{ fffe0061 }, "\\xFF\\xFEa", :intvector{ 0,0,0,0,0,0,0,0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + // Standard UTF-16LE + { "UTF-16LE", :bin{ 61006200 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + { "UTF-16LE", :bin{ fffe6100 }, "\ufeffa", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + { "UTF-16LE", :bin{ feff6100 }, "\ufffea", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + // Java "UnicodeLittle" requires an LE BOM or no BOM; it consumes the LE BOM + { "UTF-16LE,version=1",:bin{ 61006200 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + { "UnicodeLittle", :bin{ fffe6100 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + { "x-UTF-16LE-BOM", :bin{ feff6100 }, "\\xFE\\xFFa", :intvector{ 0,0,0,0,0,0,0,0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + + // Test ticket 7704: implement Java-compatible "UTF-16" converter. + // Same as standard UTF-16 but fromUnicode always writes big-endian byte stream. + { "+UTF-16,version=2", :bin{ 00610062 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + { "+UTF-16,version=2", :bin{ feff0061 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + { "+UTF-16,version=2", :bin{ fffe0061 }, "\u6100", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } + // Test ticket 5691: consistent illegal sequences // The following test cases are for illegal character byte sequences. // @@ -74,9 +108,9 @@ conversion:table(nofallback) { } { "gb18030", - :bin{ 618130fc318130fc8181303c3e813cfc817a }, - "a\u05ed\\x810\u9f07\\x810<>\\x81<\u9f07z", - :intvector{ 0,1,5,5,5,5,6,7,9,9,9,9,10,11,12,13,13,13,13,14,15,17 }, + :bin{ 618130fc318130fc8181303c3e813cfc817afe90a8bc }, + "a\u05ed\\x810\u9f07\\x810<>\\x81<\u9f07z\ue854\u1e3f", + :intvector{ 0,1,5,5,5,5,6,7,9,9,9,9,10,11,12,13,13,13,13,14,15,17,18,20 }, :int{1}, :int{0}, "", "&C", :bin{""} } { @@ -114,6 +148,13 @@ conversion:table(nofallback) { :intvector{ 0,5,7,9,9,9,9,10,12,12,12,12,12,12,12,12,14,17 }, :int{1}, :int{0}, "", "&C", :bin{""} } + { + "ISO-2022-CN-CNS", + :bin{ 411b2429470e21702541256f0f }, + "A\u00a7\u03c4\u02c7", + :intvector{ 0,6,8,10 }, + :int{1}, :int{0}, "", "&C", :bin{""} + } { "HZ", :bin{ 417e7b4141af4142affe41437e7d5a }, @@ -155,6 +196,8 @@ conversion:table(nofallback) { :intvector{ 0,1,1,1,1,2,3,8,11,13,14,14,14,14,15,16 }, :int{1}, :int{0}, "", "&C", :bin{""} } + /* + * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 { "ISO-2022-CN-EXT", :bin{ 4f1b4f4f1b242b494f1b4f4f4f4f1b204f }, @@ -162,6 +205,7 @@ conversion:table(nofallback) { :intvector{ 0,1,1,1,1,2,3,8,11,13,14,14,14,14,15,16 }, :int{1}, :int{0}, "", "&C", :bin{""} } + */ // Test ticket 5691: HZ with illegal tilde sequences. { "HZ", @@ -180,6 +224,14 @@ conversion:table(nofallback) { :intvector{ 3,5,7,9,14,15,16,17,18,22,23,24 }, :int{1}, :int{0}, "", "?", :bin{""} } + // Test bug 6071 (2:1 Unicode:charset SBCS mapping). + { + "*test1bmp", + :bin{ 050008 }, + "e@uv", + :intvector{ 0,1,2,2 }, + :int{1}, :int{1}, "", "?", :bin{""} + } // test that HZ limits its byte values to lead bytes 21..7d and trail bytes 21..7e { "HZ", @@ -197,6 +249,22 @@ conversion:table(nofallback) { :intvector{ 3,4,5,9,11,12,14,16,17,19,21,23,25,27 }, :int{1}, :int{1}, "", "?", :bin{""} } + // improve coverage of ISO-2022-JP converter by simulating erroneous input + { + "ISO-2022-JP-2", + :bin{ 0f0ed11b2e41461b244141411b4e411b2e4147451b4ed31b2e4641411b4ed2 }, + "\u0046\u4eae\u00c1\u6865\u4eae", + :intvector{ 6, 10, 14, 18, 26 }, + :int{1}, :int{0}, "", "0", :bin{""} + } + // improve coverage of JIS7 converter by simulating incomplete shifted input + { + "JIS7", + :bin{ 0e11 }, + "", + :intvector{}, + :int{1}, :int{0}, "", "0", :bin{""} + } // improve coverage of unrolled loops in ucnvmbcs.c/ucnv_MBCSSingleToBMPWithOffsets() { "ISO-8859-3", @@ -459,6 +527,8 @@ conversion:table(nofallback) { :bin{ 1b2429410e4141 }, "\u4eae", :intvector{ 5 }, :int{1}, :int{1}, "", ".", :bin{""} } + /* + * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 // ISO-2022-CN-EXT with all subcharsets and shifts and with supplementary code points { "ISO-2022-CN-EXT", @@ -467,6 +537,7 @@ conversion:table(nofallback) { :intvector{ 9, 15, 19, 29, 33, 33, 35 }, :int{1}, :int{1}, "", ".", :bin{""} } + */ // illegal and unsupported escape sequences // SS2 without designator: illegal @@ -527,7 +598,7 @@ conversion:table(nofallback) { // HZ-GB-2312 - // empty segment (using substitution and stop) + // empty segment 1 (using substitution and stop) { "HZ-GB-2312", :bin{ 61627e7b7e7d6364 }, @@ -542,6 +613,21 @@ conversion:table(nofallback) { :intvector{ 0, 1 }, :int{1}, :int{1}, "illesc", ".", :bin{"7e7d"} } + // empty segment 2 & legal redundant switches (using substitution and stop) + { + "HZ-GB-2312", + :bin{ 61627e7b323b3f557e7b7e7b523b7e7d63647e7d65667e7d7e7d }, + "ab\u4E0D\u7A7A\uFFFD\u4E00cdef\uFFFD", + :intvector{ 0, 1, 4, 6, 10, 12, 16, 17, 20, 21, 24 }, + :int{1}, :int{1}, "", "?", :bin{""} + } + { + "HZ-GB-2312", + :bin{ 61627e7b323b3f557e7b7e7b523b7e7d63647e7d65667e7d7e7d }, + "ab\u4E0D\u7A7A", + :intvector{ 0, 1, 4, 6 }, + :int{1}, :int{1}, "illesc", ".", :bin{"7e7b"} + } // DBCS-only extensions { @@ -609,6 +695,15 @@ conversion:table(nofallback) { :int{1}, :int{0}, "", "?", :bin{""} } + // test mapping to sequence of multiple Unicode characters which includes nonBMP (ticket #9235) + { + "*test3", + :bin{ 05070001020e050501020c06 }, + "\x05\U00101234\U00050005\u00c4\u00c4\U00101234\x05\x06", + :intvector{ 0, 1, 1, 1, 1, 7, 7, 7, 7, 7, 11 }, + :int{1}, :int{0}, "", "?", :bin{""} + } + // normal conversions { "UTF-16LE", @@ -642,6 +737,13 @@ conversion:table(nofallback) { :intvector{ 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 6 }, :int{1}, :int{0}, "", "&", :bin{""} } + { + "LMBCS", + :bin{ 61012981a00f270f91140a7414f60214d84d14dc561088a0 }, + "\u0061\u2013\u00fc\u00e1\u0007\u0091\u0a74\u0200\ud84d\udc56\u5516", + :intvector{ 0, 1, 3, 4, 5, 7, 9, 12, 15, 18, 21 }, + :int{1}, :int{0}, "", ".", :bin{""} + } // IMAP-mailbox-name with SUB // a a&AB~ a&AB\x0c a&AB- a&AB. a&. @@ -669,6 +771,231 @@ conversion:table(nofallback) { // simple sample, no error handling { "UTF-8", :bin{ 61F48FBFBF }, "a\U0010FFFF", :intvector{ 0, 1, 1 }, :int{1}, :int{0}, "", "", :bin{""} } + // Headers { "charset", "bytes", "unicode", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidChars" } + { + "iscii-dev", + :bin{ EF4BC0E9BFE9E8D80AEF4AC0D4BFD4E8D80AEF4838B30AEF4939B30AEF4A3AB30AEF4B3BB30A}, + "\u0A5C\u0A4D\u0A39\u0A5C\u0A4D\u0A39\u000A" /* Gurmukhi test */ + "\u0AA2\u0AB5\u0AA1\u0AB5\u0ACD\u0AB9\u000A" /* Gujarati test */ + "\u0038\u0C95\u000A" /* Kannada test */ + "\u0039\u0D15\u000A" /* Malayalam test */ + "\u003A\u0A95\u000A" /* Gujarati test */ + "\u003B\u0A15\u000A" /* Punjabi test */, + :intvector { 2, 2, 2, 4, 6, 7, 8, 11, 12, 13, 14, 15, 16, 17, 20, 21, 22, 25, 26, 27, 30, 31, 32, 35, 36, 37 }, + :int{1}, :int{0}, "", "", :bin{ "" } + } + { + "iscii-gur", + :bin{3BB30AC0E9BFE9E8D80AEF43C0E9BFE9E8D80A3BB30AEF403BB30A}, + "\u003b\u0a15\u000a" /* Easy characters */ + "\u0a5c\u0a4d\u0a39\u0a5c\u0a4d\u0a39\u000a" /* Gurmukhi test */ + "\u09dd\u09dc\u09cd\u09b9\u000a" /* Switch script: to Bengali*/ + "\u003b\u0a15\u000a" /* Easy characters - new line" so should default!*/ + "\u003b\u0a15\u000a", /* Back to Gurmukhi*/ + :intvector { 0, 1, 2, 3, 3, 3, 5, 7, 8, 9, 13, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26 }, + :int{1}, :int{0}, "", "", :bin{ "" } + } + // Test Gurmukhi (Bindi Tippi and Consonant Cluster) + { + "iscii-gur", + :bin{EF4BB3A2A2B3E8B3C0E9BFE9E8D8}, + "\u0a15\u0a70\u0a02\u0a71\u0a15\u0a5c\u0a4d\u0a39\u0a5c\u0a4d\u0a39", + :intvector { 2, 3, 4, 5, 5, 8, 8, 8, 10, 12, 13 }, + :int{1}, :int{0}, "", ".", :bin{ "" } + } + { // Verify Supplementary support + "Big5-HKSCS", + :bin{ fcfcfcfd }, + "\U000233E6\U00026DA0", + :intvector{ 0, 0, 2, 2 }, + :int{1}, :int{0}, "", "?", :bin{""} + } + { // Verify non-supplementary support + "big5-hkscs:unicode3.0", + :bin{ fcfcfcfd }, + "\uE1D4\uE1D5", + :intvector{ 0, 2 }, + :int{1}, :int{0}, "", "?", :bin{""} + } + { // Verify gb18030 enumeration + "gb18030", + :bin{ 8130D2398130D3308136A5318136A532 }, + "\u0450\u0452\u200F\u2011", + :intvector{ 0, 4, 8, 12 }, + :int{1}, :int{0}, "", "?", :bin{""} + } + { + "x11-compound-text", + :bin{ 1b242944b5ac1b2d41a5e31b2d43d5f51b2d4dd01b2d41411b2d43bc1b2d42ff1b2d54df1b2d44c0b31b2d46b41b2d47b01b2d48e01b2d4ca1 }, + "\u54A1\u00A5\u00E3\u0120\u0121\u011E\u0041\u0135\u02D9\u0E3F\u0100\u0157\u0384\u0660\u05D0\u0401", + :intvector{ }, + :int{1}, :int{0}, "", "?", :bin{""} + } + // Improve coverage of ISCII + { + "iscii-bng", + :bin{ f0a0f0b0b8 }, + "", + :intvector{}, + :int{1}, :int{0}, "", "0", :bin{""} + } + // Test iso-2022-jp-2 miscellaneous symbols + { + "iso-2022-jp-2", + :bin{ 1b242843224f224e1b2842 }, + "\u260E\u260F", + :intvector{ 4, 6 }, + :int{1}, :int{0}, "", ".", :bin{""} + } + + // Improve Code Coverage for BOCU-1 + { + "BOCU-1", + :bin{ 91fbc555fd6349 }, + "\u0041\ud841\ud888\udc81", + :intvector{ 0, 1, 4, 4 }, + :int{1}, :int{0}, "", ".", :bin{""} + } + { + "BOCU-1", + :bin{ fbeda44ff0fe189bb821f05926 }, + "\ufe88\ufe70\udbff\udfff\u0061", + :intvector{}, + :int{1}, :int{0}, "", ".", :bin{""} + } + { + "BOCU-1", + :bin{ 5b4bccf9 }, + "\u000b", + :intvector{}, + :int{1}, :int{0}, "", "0", :bin{""} + } + { + "BOCU-1", + :bin{ fe0053c6 }, + "\u0003\u0076", + :intvector{}, + :int{1}, :int{0}, "", "0", :bin{""} + } + + //Improve code coverage for SCSU + { + "SCSU", + :bin{ 0fd899dc7fd888dc99e041424361 }, + "\ud899\udc7f\ud888\udc99\u0041\u0042\u0043\u0061", + :intvector{}, + :int{1}, :int{0}, "", "0", :bin{""} + } + { + "SCSU", + :bin{ 41df1281035f10df1b03df1c88800bbfffff }, + "\u0041\u00df\u0401\u015f\u00df\u01df\uf000\udbff\udfff", + :intvector{}, + :int{1}, :int{0}, "", ".", :bin{""} + } + { + "SCSU", + :bin{ 1b9a1b541bb2411bfd1b0041 }, + "", + :intvector{}, + :int{1}, :int{0}, "", ".", :bin{""} + } + { + "SCSU", + :bin{ 0f6441b413a733f2 }, + "\u6441\ub413\ua733", + :intvector{}, + :int{1}, :int{0}, "illegal", ".", :bin{ f2 } + } + + //Improve code coverage for MBCS + { + "*test5", + :bin{ 0506 }, + "\ufffd\x06", + :intvector{}, + :int{1}, :int{0}, "", "?", :bin{""} + } + { + "ibm-1390,swaplfnl", + :bin{ 430e4395ecc140400fc1e115 }, + "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a", + :intvector{ 0, 2, 4, 4, 6, 9, 10, 11 }, + :int{1}, :int{0}, "", "?", :bin{""} + } + /* + * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 + { + "ISO-2022-CN-EXT", + :bin{ 1b242b4d1b4f66791b242b4d1b4f216a }, + "\u3667", + :intvector{ 14 }, + :int{1}, :int{0}, "", "0", :bin{""} + } + */ + { + "*test1bmp", + :bin{ 060a05 }, + "\u0066\ufffd\u0065", + :intvector{ 0, 2, 2 }, + :int{0}, :int{1}, "", "?", :bin{""} + } + { + "*test1bmp", + :bin{ 060708 }, + "\u0066", + :intvector{}, + :int{1}, :int{0}, "invalid", ".", :bin{ 07 } + } + { + "*test5", + :bin{ 010304 }, + "\u0034\ufffd", + :intvector{}, + :int{1}, :int{0}, "", "?", :bin{""} + } + { + "*test1", + :bin{ 0a0b }, + "", + :intvector{}, + :int{1}, :int{0}, "", "0", :bin{""} + } + { + "*test1bmp", + :bin{ 0c06 }, + "\u0066", + :intvector{}, + :int{0}, :int{0}, "", ".", :bin{""} + } + { + "*test5", + :bin{ 0906 }, + "\udbc8\udf45\u0006", + :intvector{}, + :int{0}, :int{0}, "", ".", :bin{""} + } + { + "ibm-16684", + :bin{ 0e }, + "", + :intvector{}, + :int{0}, :int{0}, "illegal", ".", :bin{ 0e } + } + { + "UTF-7", + :bin{ 2b414b4d2d492b414b4d4170412d }, + "\u00a3I\u00a3\u00a4", + :intvector{ 1,5,7,9 }, + :int{0}, :int{0}, "", ".", :bin{""} + } + { + "x11-compound-text", + :bin{ 1e6ddc9b26bc10801bbcad50a040fc }, + "\u001e\u006d\u00dc\u009b\u0026\u00bc\u0010\u0080\ufffd\u00bc\u00ad\u0050\u00a0\u0040\u00fc", + :intvector{ }, + :int{1}, :int{0}, "", "?", :bin{""} + } } } @@ -677,6 +1004,120 @@ conversion:table(nofallback) { fromUnicode { Headers { "charset", "unicode", "bytes", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidUChars" } Cases { + // Test ticket 9602: Add "good one-way" mapping type (|4). + // Such mappings are used regardless of the fallback flag. + { + "+*test3", "##\uFE0E#\uFE0F", + :bin{ 010204010204010204 }, :intvector{ 0,0,0,1,1,1,3,3,3 }, + :int{1}, :int{0}, // no fallbacks + "", "?", "" + } + { + "+*test3", "##\uFE0E#\uFE0F", + :bin{ 010204010204010204 }, :intvector{ 0,0,0,1,1,1,3,3,3 }, + :int{1}, :int{1}, // with fallbacks + "", "?", "" + } + // Test ticket 6789: implement Java-compatible Unicode, UnicodeBig and UnicodeLittle converters + // For details about these encodings see convrtrs.txt. + // Standard UTF-16BE + { "UTF-16BE", "a", :bin{ 0061 }, :intvector{ 0,0 }, :int{1}, :int{0}, "", "?", "" } + // Java "UnicodeBig" writes a BOM + { "UnicodeBig", "a", :bin{ feff0061 }, :intvector{ -1,-1,0,0 }, :int{1}, :int{0}, "", "?", "" } + // Standard UTF-16LE + { "UTF-16LE", "a", :bin{ 6100 }, :intvector{ 0,0 }, :int{1}, :int{0}, "", "?", "" } + // Java "UnicodeLittle" writes a BOM + { "UnicodeLittle", "a", :bin{ fffe6100 }, :intvector{ -1,-1,0,0 }, :int{1}, :int{0}, "", "?", "" } + + // Test ticket 7704: implement Java-compatible "UTF-16" converter. + // Same as standard UTF-16 but fromUnicode always writes big-endian byte stream. + { "+UTF-16,version=2", "a", :bin{ feff0061 }, :intvector{ -1,-1,0,0 }, :int{1}, :int{0}, "", "?", "" } + + // Test bug 6071 (1:2 Unicode:charset SBCS mapping). + { + "*test1bmp", + "e@t", + :bin{ 05000709 }, + :intvector{ 0,1,2,2 }, + :int{1}, :int{0}, "", "?", "" + } + // improve coverage of ISO-2022-JP converter with hardcoded JIS X 0201 and + // using the Shift-JIS table for JIS X 0208 (ticket #5797) + { + "ISO-2022-JP", + "\u203e\xa5\u4e00\ufa10\u6f3e\u0391", + :bin{ 1b284a7e5c1b2442306c222e5f2126211b2842 }, + :intvector{ 0,0,0,0,1,2,2,2,2,2,3,3,4,4,5,5,5,5,5 }, + :int{1}, :int{0}, "", "?=\u3013", "" // U+3013 Geta Mark converts to 222e + } + // Verify that mappings that would result in byte values outside 20..7F (for SBCS) + // or 21..7E (for DBCS) are not used. + // ibm-9005_X110-2007.ucm (ISO 8859-7, .F=1b2e46): + // \x9F |0 (also in ISO 8859-1) + // \xB7 |1 + // windows-949-2000 (KSC_5601, $(C=1b242843): + // \xA0\xA1 |0 + // \xC0\x41 |0 + // \xC8\xFE |0 + { + "JIS8", // =ISO_2022,locale=ja,version=4 + "\u009f\u0387\uc829\ud4fe\ud79d", + :bin{ 1a1b2e461b4e371a1a1b242843487e1b2842 }, + :intvector{ 0,1,1,1,1,1,1,2,3,4,4,4,4,4,4,4,4,4 }, + :int{1}, :int{1}, "", "?", "" + } + // Ticket 5483: ISO 2022 converter incorrectly using fallback mapping + // Verify that a roundtrip mapping is used even when a fallback mapping is + // available in the current state. + // U+FF61 is handled in code + // jisx-208.ucm ($B=1b2442): + // \x21\x34 |0 + // \x21\x51 |0 and + // ibm-897_P100-1995.ucm (JIS X 0201, (J=1b284a): + // \x7D |1 + // ibm-9005_X110-2007.ucm (ISO 8859-7, .F=1b2e46): + // \xF6 |1 + // \xAF |0 + // \x7D |1 (not legal for ISO 2022) + // windows-949-2000 (KSC_5601, $(C=1b242843): + // \xB0\xA1 |0 + // \xA3\xFD |0 + // \xA1\xAD |0 (in extension table) + { + "JIS8", // =ISO_2022,locale=ja,version=4 + "a\uff61\u03d5\uff5d\uac00\u223c\uff5d\u30fe\uff5d", // Make it switch to ISO-8859-7, KSC 5601 and JIS X 0208. + :bin{ 61a11b2e461b4e761b244221511b2428433021212d237d1b2442213421511b2842 }, + :intvector{ 0,1,2,2,2,2,2,2,3,3,3,3,3,4,4,4,4,4,4,5,5,6,6,7,7,7,7,7,8,8,8,8,8 }, + :int{1}, :int{1}, "", "?", "" + } + // Code coverage for UTF-8->SBCS conversion (ucnv_convertEx()). + // Test code path for non-roundtripping ASCII characters + // (try EBCDIC SBCS, and IBM PC SBCS with control code rotation). + { + "ibm-37", + "a\x85c", + :bin{ 811583 }, + :intvector{ 0,1,2 }, + :int{1}, :int{0}, "", "?", "" + } + { + "ibm-850", + "a\x1ac", + :bin{ 617f63 }, + :intvector{ 0,1,2 }, + :int{1}, :int{0}, "", "?", "" + } + // Code coverage for UTF-8->DBCS conversion (ucnv_convertEx()). + // Test code path for non-roundtripping ASCII characters + // (try IBM PC DBCS with control code rotation). + { + "ibm-943", + "a\x1ac\u30a1\x7ff", + :bin{ 617f6383401c66 }, + :intvector{ 0,1,2,3,3,4,5 }, + :int{1}, :int{0}, "", "?", "" + } + // SCSU regression test. { "SCSU", "1\U00010001\u0085\U000500022\ud8003\udc014\ue001", @@ -684,6 +1125,14 @@ conversion:table(nofallback) { :intvector{ 0,1,1,1,1,3,3,4,4,4,4,4,6,6,7,7,7,8,9,9,9,10,11,11,11 }, :int{1}, :int{0}, "", "?", "" } + // LMBCS + { + "lmbcs", + "\u0061\u2013\u00fc\u00e1\u0007\u0091\u0a74\u0200\ud84d\udc56\u5516", + :bin { 61039681a00f270f91140a7414f60214d84d14dc561088a0 }, + :intvector{ 0, 1, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10, 10, 10 }, + :int{1}, :int{0}, "", ".", "" + } // Test substitution strings. { "windows-1252", // stateless MBCS with WriteSub(), should internally set char * @@ -692,6 +1141,34 @@ conversion:table(nofallback) { :intvector{ 0,1,1,1,1,1,2 }, :int{1}, :int{0}, "", "?=@$!@$", "" } + { + "windows-1252", + "1\U00010001\u0085\U000500022\ud8003\udc014\ue001", + :bin{ 311a1a1a321a331a341a }, + :intvector{ 0,1,3,4,6,7,8,9,10,11 }, + :int{1}, :int{0}, "", "?", "" + } + { + "windows-1252", + "\uD87E", // lone surrogate can cause an offset overflow + :bin{ 1a }, + :intvector{ 0 }, + :int{1}, :int{0}, "", "?", "" + } + { + "windows-1252", + "\uD87E", // lone surrogate can cause an offset overflow + :bin{ 6875683f }, + :intvector{ 0,0,0,0 }, + :int{1}, :int{0}, "", "?=huh?", "" // Use a long substitution character + } + { + "*test4", + "\u30ab", // An incomplete multi-codepoint character + :bin{ ff }, + :intvector{ 0 }, + :int{1}, :int{0}, "", "?", "" + } { "ibm-930", // stateful MBCS "a\ufdd0\u4e00\ufdd0e", @@ -977,6 +1454,14 @@ conversion:table(nofallback) { :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5 }, :int{1}, :int{1}, "", "?\x00\x2f\x7e", "" } + // ISO-2022-KR + { + "ibm-25546", + "AB\uc88b\U00050005\uacccC", + :bin{ 1b24294341420e41410f5c3530303035200e306a0f43 }, + :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,3,3,3,3,3,3,5,5,5,6,6 }, + :int{1}, :int{1}, "", "&S", "" + } // ISO-2022-JP-2 with G2 designator & SS2 shift { @@ -1011,6 +1496,8 @@ conversion:table(nofallback) { :int{1}, :int{1}, "", ".", "" } + /* + * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 // ISO-2022-CN-EXT with all subcharsets and shifts and with supplementary code points { "ISO-2022-CN-EXT", @@ -1035,6 +1522,7 @@ conversion:table(nofallback) { :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4 }, :int{0}, :int{1}, "", ".", "" } + */ // windows-936 vs. ibm-1386 { @@ -1207,10 +1695,24 @@ conversion:table(nofallback) { :intvector{ 0, 1, 1, 1, 1, 3, 3, 4, 5, 5, 5, 6, 7, 7, 7, 8, 9, 9, 9 }, :int{1}, :int{0}, "", "?", "" } - + { + "x11-compound-text", + "\u54A1\u00A5\u00E3\u0120\u0121\u011E\u0041\u0135\u02D9\u0E3F\u0100\u0157\u0384\u0660\u05D0\u0401", + :bin{ 1b242944b5ac1b2d41a5e31b2d43d5f51b2d4dd01b2d41411b2d43bc1b2d42ff1b2d54df1b2d44c0b31b2d46b41b2d47b01b2d48e01b2d4ca1 }, + :intvector{ }, + :int{1}, :int{0}, "", "?", "" + } + // Test Gurmukhi (Bindi Tippi and Consonant clusters) + { + "iscii-gur", + "\u0a15\u0a70\u0a02\u0a71\u0a15\u0a5c\u0a4d\u0a39\u000a\u0043\u0041\u000a", + :bin { ef4bb3a2a2b3e8b3bfe9e8d80aef4b43410a }, + :intvector{ 0, 0, 0, 1, 2, 3, 3, 3, 5, 5, 6, 7, 8, 8, 8, 9, 10, 11 }, + :int{1}, :int{0}, "", ".", "" + } // escape callback { - "ISCII", + "iscii-dev", "A\u0901\U00023456\u0902B\U00023456C", :bin{ 41ef42a1255544383444255544433536a24225554438344425554443353643 }, :intvector{ @@ -1270,6 +1772,71 @@ conversion:table(nofallback) { // simple sample, no error handling { "UTF-8", "a\U0010FFFF", :bin{ 61F48FBFBF }, :intvector{ 0, 1, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" } + + // Verify that incomplete surrogates are handled as an error + { "UTF-8", "a\udc00", :bin{ 61efbfbd }, :intvector{ 0, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" } + { "UTF-8", "a\ud800", :bin{ 61efbfbd }, :intvector{ 0, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" } + { "UTF-8", "a\udc00b", :bin{ 61efbfbd62 }, :intvector{ 0, 1, 1, 1, 2 }, :int{1}, :int{0}, "", "", "" } + { "UTF-8", "a\ud800b", :bin{ 61efbfbd62 }, :intvector{ 0, 1, 1, 1, 2 }, :int{1}, :int{0}, "", "", "" } + + // Code coverage for the EUC variants. + { "IBM-eucJP", "\u0061\u4edd\u5bec\ud801\udc01\ud801\u0061\u00a2", :bin{ 61a1b88ff4ae618ee0 }, :intvector{ 0, 1, 1, 2, 2, 2, 6, 7, 7 }, :int{1}, :int{0}, "", "0", "" } + { "IBM-eucJP", "\u0061\u4edd\u5bec\ud801\udc01\ud801\u0061\u00a2", :bin{ 61a1b88ff4aef4fef4fe618ee0 }, :intvector{ 0, 1, 1, 2, 2, 2, 3, 3, 5, 5, 6, 7, 7 }, :int{1}, :int{0}, "", "", "" } + { "EUC-TW", "\u0061\u2295\u5BF2\ud801\udc01\ud801\u0061\u8706\u008a", :bin{ 61a2d38ea2dce561e6ca8a }, :intvector{ 0, 1, 1, 2, 2, 2, 2, 6, 7, 7, 8 }, :int{1}, :int{0}, "", "0", "" } + { "EUC-TW", "\u0061\u2295\u5BF2\ud801\udc01\ud801\u0061\u8706\u008a", :bin{ 61a2d38ea2dce5fdfefdfe61e6ca8a }, :intvector{ 0, 1, 1, 2, 2, 2, 2, 3, 3, 5, 5, 6, 7, 7, 8 }, :int{1}, :int{0}, "", "", "" } + + // Code Coverage for BOCU-1 + { + "BOCU-1", + "\u0041\ud841\ud888\udc81", + :bin{ 91fbc555fd6349 }, + :intvector{ 0, 1, 1, 1, 2, 2, 2 }, + :int{1}, :int{0}, "", ".", "" + } + { + "BOCU-1", + "\ufe88\ufe70\udbff\udfff\u0061", + :bin{ fbeda44ff0fe189bb821f05926 }, + :intvector{}, + :int{1}, :int{0}, "", ".", "" + } + // Improve code coverage for SCSU + { + "SCSU", + "\ud899\udc7f\ud977\ud888\udc99\ud888\u0041", + :bin{ 0fd899dc7fd888dc99e041 }, + :intvector{}, + :int{1}, :int{0}, "", "0", "" + } + { + "gb18030", + "\U00020087\ue790\ue78f\u1e3f", + :bin{ 95329031a6dca6dba8bc }, + :intvector{ 0,0,0,0,2,2,3,3,4,4 }, + :int{1}, :int{0}, "", "0", "" + } + { + "UTF-7", + "\u00a3I\u00a3\u00a4", + :bin{ 2b414b4d2d492b414b4d4170412d }, + :intvector{ 0,0,0,0,0,1,2,2,2,3,3,3,3,3 }, + :int{1}, :int{0}, "", "0", "" + } + // Bug #9601 direct-from-UTF-8 m:n Unicode:charset conversion. + { + "*test1bmp", + "uv", + :bin{ 08 }, + :intvector{ 0 }, + :int{1}, :int{0}, "", "?", "" + } + { + "*test2", + "\U00101234\U00050005", + :bin{ 0700010e05 }, + :intvector{ 0,0,0,0,0 }, + :int{1}, :int{0}, "", "?", "" + } } } @@ -1280,6 +1847,20 @@ conversion:table(nofallback) { // which - numeric UConverterUnicodeSet value Headers { "charset", "map", "mapnot", "which" } Cases { + // Test ticket 9602: Add "good one-way" mapping type (|4). + // Excluded from roundtrip set, included in the set with fallbacks. + { + "+*test3", + "[{#\uFE0F}]", + "[#{#\uFE0E}]", + :int{0} + } + { + "+*test3", + "[#{#\uFE0E}{#\uFE0F}]", + "[]", + :int{1} + } // Unicode charsets that do not map surrogate code points { "UTF-8", @@ -1367,16 +1948,29 @@ conversion:table(nofallback) { // versions of ISO-2022-JP { "ISO-2022-JP", - "[\x00-\x0d\x10-\x1a\x1c-\x7f\u0391-\u03a1\uff61-\uff9f\u4e00\u4e01\uffe5]", - "[\x0e\x0f\x1b\u0100-\u0113\u0385-\u038a\u4e02\u4e27-\u4e29\uffe6-\U0010ffff]", + "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa5\u0391-\u03a1\u2015\u203e\u4e00\u4e01\uffe5]", + "[\x0e\x0f\x1b\u0100-\u0113\u0385-\u038a\u2014\u301c\u4e02\u4e27-\u4e29\u4fe0\u663b\u9eb5\ufa0e-\ufa2d\uff61-\uff9f\uffe4\uffe6-\U0010ffff]", :int{0} } { "ISO-2022-JP-2", - "[\x00-\x0d\x10-\x1a\x1c-\u0113\u0385-\u038a\u0390-\u03a1\uff61-\uff9f\u4e00-\u4e05\uffe6]", - "[\x0e\x0f\x1b\uffe7-\U0010ffff]", + "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa0-\u0113\u0384-\u0386\u0388-\u038a\u0390-\u03a1\u203e\u4e00-\u4e05\u4fe0\u663b\uffe6]", + "[\x0e\x0f\x1b\uff61-\uff9f\uffe4\uffe7-\U0010ffff]", :int{0} } + { + "JIS7", + "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa0-\u0113\u0384-\u0386\u0388-\u038a\u0390-\u03a1\u203e\u4e00-\u4e05\u4fe0\u663b\uff61-\uff9f\uffe6]", + "[\x0e\x0f\x1b\uffe4\uffe7-\U0010ffff]", + :int{0} + } + // with fallbacks + { + "ISO-2022-JP", + "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa5\u0391-\u03a1\u2014\u2015\u203e\u301c\u4e00\u4e01\u4fe0\u9eb5\uff61-\uff9f\uffe5]", + "[\x0e\x0f\x1b\xa6\u0100-\u0113\u0385-\u038a\u4e02\u4e27-\u4e29\u663b\ufa0e-\ufa2d\uffe4\uffe6-\U0010ffff]", + :int{1} + } // versions of ISO-2022-CN { @@ -1385,13 +1979,53 @@ conversion:table(nofallback) { "[\x0e\x0f\x1b\u4e29\uffe6-\U0010ffff]", :int{0} } + /* + * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 { "ISO-2022-CN-EXT", "[\x00-\x0d\x10-\x1a\x1c-\x7f\u4e00-\u4e05\u9f98\ufe6b\u4e28-\u4e2b\U00020000\U00020003-\U00020005\U00029664]", "[\x0e\x0f\x1b\U00020001\U00020002\U0002a6d7-\U0010ffff]", :int{0} } + */ + // HZ + { + "HZ", + "[\u0410-\u044f\u4e00\u4e01\u4e03]", + "[\u4e02\u4e04-\u4e06\uac00-\ud7ff]", + :int{0} + } + + // LMBCS + { + "LMBCS", + "[\x00-\U0010ffff]", + "[]", + :int{0} + } + + // ISCII,version=0 (note: all versions of ISCII generates the same Unicodeset) + { + "iscii-dev", + "[\x00-\xa0\u0909-\u0939\u0993-\u09a8\u0a13-\u0a28\u0a93-\u0aa8\u0ae6-\u0aef\u0b05-\u0b0c\u0b13-\u0b28\u0bae-\u0bb5\u0c12-\u0c28\u0c92-\u0ca8\u0d12-\u0d28]", + "[\u0971-\u0975\u09e4\u0a4e-\u0a58\u0a80\u0b72-\u0b7a\u0bfb-\u0bff\u0c70-\u0c7f\u0c80\u0d00]", + :int{0} + } + + { + "iso-8859-1", + "[\x00-\xff]", + "[\u0100-\u01ff]", + :int{0} + } + + { + "us-ascii", + "[\x00-\x7f]", + "[\u0100-\u01ff]", + :int{0} + } // DBCS-only { "ibm-971",