ICU-400.37.tar.gz

[apple/icu.git] / icuSources / test / testdata / conversion.txt

diff --git a/icuSources/test/testdata/conversion.txt b/icuSources/test/testdata/conversion.txt
index 45ae1e693b42fc6c18f1635e10d3f3a39928cea6..312b76e75f38a90d2ec8c93366eceee0943ec82c 100644 (file)
--- a/icuSources/test/testdata/conversion.txt
+++ b/icuSources/test/testdata/conversion.txt
@@ -1,6 +1,6 @@
 //*******************************************************************************
 //
 //*******************************************************************************
 //

-//   Copyright (C) 2003-2006,2008 International Business Machines
+//   Copyright (C) 2003-2008, International Business Machines

 //   Corporation and others.  All Rights Reserved.
 //
 //   file name:  conversion.txt
 //   Corporation and others.  All Rights Reserved.
 //
 //   file name:  conversion.txt


@@ -180,6 +180,14 @@ conversion:table(nofallback) {
           :intvector{ 3,5,7,9,14,15,16,17,18,22,23,24 },
           :int{1}, :int{0}, "", "?", :bin{""}
         }
           :intvector{ 3,5,7,9,14,15,16,17,18,22,23,24 },
           :int{1}, :int{0}, "", "?", :bin{""}
         }

+        // Test bug 6071 (2:1 Unicode:charset SBCS mapping).
+        {
+          "*test1bmp",
+          :bin{ 050008 },
+          "e@uv",
+          :intvector{ 0,1,2,2 },
+          :int{1}, :int{1}, "", "?", :bin{""}
+        }

         // test that HZ limits its byte values to lead bytes 21..7d and trail bytes 21..7e
         {
           "HZ",
         // test that HZ limits its byte values to lead bytes 21..7d and trail bytes 21..7e
         {
           "HZ",


@@ -197,6 +205,22 @@ conversion:table(nofallback) {
           :intvector{ 3,4,5,9,11,12,14,16,17,19,21,23,25,27 },
           :int{1}, :int{1}, "", "?", :bin{""}
         }
           :intvector{ 3,4,5,9,11,12,14,16,17,19,21,23,25,27 },
           :int{1}, :int{1}, "", "?", :bin{""}
         }

+        // improve coverage of ISO-2022-JP converter by simulating erroneous input
+        {
+          "ISO-2022-JP-2",
+          :bin{ 0f0ed11b2e41461b244141411b4e411b2e4147451b4ed31b2e4641411b4ed2 },
+          "\u0046\u4eae\u00c1\u6865\u4eae",
+          :intvector{ 6, 10, 14, 18, 26 },
+          :int{1}, :int{0}, "", "0", :bin{""}
+        }
+        // improve coverage of JIS7 converter by simulating incomplete shifted input
+        {
+          "JIS7",
+          :bin{ 0e11 },
+          "",
+          :intvector{},
+          :int{1}, :int{0}, "", "0", :bin{""}
+        }

         // improve coverage of unrolled loops in ucnvmbcs.c/ucnv_MBCSSingleToBMPWithOffsets()
         {
           "ISO-8859-3",
         // improve coverage of unrolled loops in ucnvmbcs.c/ucnv_MBCSSingleToBMPWithOffsets()
         {
           "ISO-8859-3",


@@ -527,7 +551,7 @@ conversion:table(nofallback) {
 
         // HZ-GB-2312
 
 
         // HZ-GB-2312
 

-        // empty segment (using substitution and stop)
+        // empty segment 1 (using substitution and stop)

         {
           "HZ-GB-2312",
           :bin{ 61627e7b7e7d6364 },
         {
           "HZ-GB-2312",
           :bin{ 61627e7b7e7d6364 },


@@ -542,6 +566,21 @@ conversion:table(nofallback) {
           :intvector{ 0, 1 },
           :int{1}, :int{1}, "illesc", ".", :bin{"7e7d"}
         }
           :intvector{ 0, 1 },
           :int{1}, :int{1}, "illesc", ".", :bin{"7e7d"}
         }

+        // empty segment 2 & legal redundant switches (using substitution and stop)
+        {
+          "HZ-GB-2312",
+          :bin{ 61627e7b323b3f557e7b7e7b523b7e7d63647e7d65667e7d7e7d },
+          "ab\u4E0D\u7A7A\uFFFD\u4E00cdef\uFFFD",
+          :intvector{ 0, 1, 4, 6, 10, 12, 16, 17, 20, 21, 24 },
+          :int{1}, :int{1}, "", "?", :bin{""}
+        }
+        {
+          "HZ-GB-2312",
+          :bin{ 61627e7b323b3f557e7b7e7b523b7e7d63647e7d65667e7d7e7d },
+          "ab\u4E0D\u7A7A",
+          :intvector{ 0, 1, 4, 6 },
+          :int{1}, :int{1}, "illesc", ".", :bin{"7e7b"}
+        }

 
         // DBCS-only extensions
         {
 
         // DBCS-only extensions
         {


@@ -669,6 +708,67 @@ conversion:table(nofallback) {
 
         // simple sample, no error handling
         { "UTF-8", :bin{ 61F48FBFBF }, "a\U0010FFFF", :intvector{ 0, 1, 1 }, :int{1}, :int{0}, "", "", :bin{""} }
 
         // simple sample, no error handling
         { "UTF-8", :bin{ 61F48FBFBF }, "a\U0010FFFF", :intvector{ 0, 1, 1 }, :int{1}, :int{0}, "", "", :bin{""} }

+        //      Headers { "charset", "bytes", "unicode", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidChars" }
+        {
+            "iscii",
+            :bin{ EF4BC0E9BFE9E8D80AEF4AC0D4BFD4E8D80AEF4838B30AEF4939B30AEF4A3AB30AEF4B3BB30A},  
+            "\u0A22\u0A3C\u0A5C\u0A4D\u0A39\u000A" /* Gurmukhi test */
+                "\u0AA2\u0AB5\u0AA1\u0AB5\u0ACD\u0AB9\u000A"     /* Gujarati test */
+                "\u0038\u0C95\u000A" /* Kannada test */
+                "\u0039\u0D15\u000A" /* Malayalam test */
+                "\u003A\u0A95\u000A" /* Gujarati test */
+                "\u003B\u0A15\u000A" /* Punjabi test */,
+                :intvector  { 2,  3,  5,  6,  7,  8,  11,  12,  13,  14,  15,  16,  17,  20,  21,  22,  25,  26,  27,  30,  31, 32,  35,  36,  37  },
+                :int{1}, :int{0}, "", "", :bin{ "" }
+        }
+        {
+            "iscii-gur",
+            :bin{3BB30AC0E9BFE9E8D80AEF43C0E9BFE9E8D80A3BB30AEF403BB30A},
+            "\u003b\u0a15\u000a" /* Easy characters */
+            "\u0a22\u0a3c\u0a5c\u0a4d\u0a39\u000a" /* Gurmukhi test */
+            "\u09dd\u09dc\u09cd\u09b9\u000a" /* Switch script: to Bengali*/ 
+            "\u003b\u0a15\u000a" /* Easy characters - new line" so should default!*/
+            "\u003b\u0a15\u000a", /* Back to Gurmukhi*/
+            :intvector { 0,  1,  2,  3,  4,  6,  7,  8,  9,  13,  15,  16,  17,  18,  19,  20,  21,  24,  25,  26  }, 
+            :int{1}, :int{0}, "", "", :bin{ "" }
+        }
+        { // Verify Supplementary support
+          "Big5-HKSCS",
+          :bin{ fcfcfcfd },
+          "\U000233E6\U00026DA0",
+          :intvector{ 0, 0, 2, 2 },
+          :int{1}, :int{0}, "", "?", :bin{""}
+        }
+        { // Verify non-supplementary support
+          "big5-hkscs:unicode3.0",
+          :bin{ fcfcfcfd },
+          "\uE1D4\uE1D5",
+          :intvector{ 0, 2 },
+          :int{1}, :int{0}, "", "?", :bin{""}
+        }
+        { // Verify gb18030 enumeration
+          "gb18030",
+          :bin{ 8130D2398130D3308136A5318136A532 },
+          "\u0450\u0452\u200F\u2011",
+          :intvector{ 0, 4, 8, 12 },
+          :int{1}, :int{0}, "", "?", :bin{""}
+        }
+        // Improve coverage of ISCII
+        {
+          "iscii-bng",
+          :bin{ f0a0f0b0b8 },
+          "",
+          :intvector{},
+          :int{1}, :int{0}, "", "0", :bin{""} 
+        }
+        // Test iso-2022-jp-2 miscellaneous symbols
+        {
+          "iso-2022-jp-2",
+          :bin{ 1b242843224f224e1b2842 },
+          "\u260E\u260F",
+          :intvector{ 4, 6 },
+          :int{1}, :int{0}, "", ".", :bin{""}
+        }

       }
     }
 
       }
     }
 


@@ -677,6 +777,91 @@ conversion:table(nofallback) {
     fromUnicode {
       Headers { "charset", "unicode", "bytes", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidUChars" }
       Cases {
     fromUnicode {
       Headers { "charset", "unicode", "bytes", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidUChars" }
       Cases {

+        // Test bug 6071 (1:2 Unicode:charset SBCS mapping).
+        {
+          "*test1bmp",
+          "e@t",
+          :bin{       05000709 },
+          :intvector{ 0,1,2,2 },
+          :int{1}, :int{0}, "", "?", ""
+        }
+        // improve coverage of ISO-2022-JP converter with hardcoded JIS X 0201 and
+        // using the Shift-JIS table for JIS X 0208 (ticket #5797)
+        {
+          "ISO-2022-JP",
+          "\u203e\xa5\u4e00\ufa10\u6f3e\u0391",
+          :bin{       1b284a7e5c1b2442306c222e5f2126211b2842 },
+          :intvector{ 0,0,0,0,1,2,2,2,2,2,3,3,4,4,5,5,5,5,5 },
+          :int{1}, :int{0}, "", "?=\u3013", ""  // U+3013 Geta Mark converts to 222e
+        }
+        // Verify that mappings that would result in byte values outside 20..7F (for SBCS)
+        // or 21..7E (for DBCS) are not used.
+        // ibm-9005_X110-2007.ucm (ISO 8859-7, <ESC>.F=1b2e46):
+        //   <U009F> \x9F |0 (also in ISO 8859-1)
+        //   <U0387> \xB7 |1
+        // windows-949-2000 (KSC_5601, <ESC>$(C=1b242843):
+        //   <UC829> \xA0\xA1 |0
+        //   <UD4FE> \xC0\x41 |0
+        //   <UD79D> \xC8\xFE |0
+        {
+          "JIS8",  // =ISO_2022,locale=ja,version=4
+          "\u009f\u0387\uc829\ud4fe\ud79d",
+          :bin{       1a1b2e461b4e371a1a1b242843487e1b2842 },
+          :intvector{ 0,1,1,1,1,1,1,2,3,4,4,4,4,4,4,4,4,4 },
+          :int{1}, :int{1}, "", "?", ""
+        }
+        // Ticket 5483: ISO 2022 converter incorrectly using fallback mapping
+        // Verify that a roundtrip mapping is used even when a fallback mapping is
+        // available in the current state.
+        //   U+FF61 is handled in code
+        // jisx-208.ucm (<ESC>$B=1b2442):
+        //   <U30FE> \x21\x34 |0
+        //   <UFF5D> \x21\x51 |0  and
+        // ibm-897_P100-1995.ucm (JIS X 0201, <ESC>(J=1b284a):
+        //   <UFF5D> \x7D |1
+        // ibm-9005_X110-2007.ucm (ISO 8859-7, <ESC>.F=1b2e46):
+        //   <U03D5> \xF6 |1
+        //   <U2015> \xAF |0
+        //   <UFF5D> \x7D |1 (not legal for ISO 2022)
+        // windows-949-2000 (KSC_5601, <ESC>$(C=1b242843):
+        //   <UAC00> \xB0\xA1 |0
+        //   <UFF5D> \xA3\xFD |0
+        //   <U223C> \xA1\xAD |0 (in extension table)
+        {
+          "JIS8",  // =ISO_2022,locale=ja,version=4
+          "a\uff61\u03d5\uff5d\uac00\u223c\uff5d\u30fe\uff5d",  // Make it switch to ISO-8859-7, KSC 5601 and JIS X 0208.
+          :bin{       61a11b2e461b4e761b244221511b2428433021212d237d1b2442213421511b2842 },
+          :intvector{ 0,1,2,2,2,2,2,2,3,3,3,3,3,4,4,4,4,4,4,5,5,6,6,7,7,7,7,7,8,8,8,8,8 },
+          :int{1}, :int{1}, "", "?", ""
+        }
+        // Code coverage for UTF-8->SBCS conversion (ucnv_convertEx()).
+        // Test code path for non-roundtripping ASCII characters
+        // (try EBCDIC SBCS, and IBM PC SBCS with control code rotation).
+        {
+          "ibm-37",
+          "a\x85c",
+          :bin{       811583 },
+          :intvector{ 0,1,2 },
+          :int{1}, :int{0}, "", "?", ""
+        }
+        {
+          "ibm-850",
+          "a\x1ac",
+          :bin{       617f63 },
+          :intvector{ 0,1,2 },
+          :int{1}, :int{0}, "", "?", ""
+        }
+        // Code coverage for UTF-8->DBCS conversion (ucnv_convertEx()).
+        // Test code path for non-roundtripping ASCII characters
+        // (try IBM PC DBCS with control code rotation).
+        {
+          "ibm-943",
+          "a\x1ac\u30a1\x7ff",
+          :bin{       617f6383401c66 },
+          :intvector{ 0,1,2,3,3,4,5 },
+          :int{1}, :int{0}, "", "?", ""
+        }
+        // SCSU regression test.

         {
           "SCSU",
           "1\U00010001\u0085\U000500022\ud8003\udc014\ue001",
         {
           "SCSU",
           "1\U00010001\u0085\U000500022\ud8003\udc014\ue001",


@@ -692,6 +877,34 @@ conversion:table(nofallback) {
           :intvector{ 0,1,1,1,1,1,2 },
           :int{1}, :int{0}, "", "?=@$!@$", ""
         }
           :intvector{ 0,1,1,1,1,1,2 },
           :int{1}, :int{0}, "", "?=@$!@$", ""
         }

+        {
+          "windows-1252",
+          "1\U00010001\u0085\U000500022\ud8003\udc014\ue001",
+          :bin{ 311a1a1a321a331a341a },
+          :intvector{ 0,1,3,4,6,7,8,9,10,11 },
+          :int{1}, :int{0}, "", "?", ""
+        }
+        {
+          "windows-1252",
+          "\uD87E", // lone surrogate can cause an offset overflow
+          :bin{ 1a },
+          :intvector{ 0 },
+          :int{1}, :int{0}, "", "?", ""
+        }
+        {
+          "windows-1252",
+          "\uD87E", // lone surrogate can cause an offset overflow
+          :bin{ 6875683f },
+          :intvector{ 0,0,0,0 },
+          :int{1}, :int{0}, "", "?=huh?", "" // Use a long substitution character
+        }
+        {
+          "*test4",
+          "\u30ab", // An incomplete multi-codepoint character
+          :bin{ ff },
+          :intvector{ 0 },
+          :int{1}, :int{0}, "", "?", ""
+        }

         {
           "ibm-930", // stateful MBCS
           "a\ufdd0\u4e00\ufdd0e",
         {
           "ibm-930", // stateful MBCS
           "a\ufdd0\u4e00\ufdd0e",


@@ -977,6 +1190,14 @@ conversion:table(nofallback) {
           :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5 },
           :int{1}, :int{1}, "", "?\x00\x2f\x7e", ""
         }
           :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5 },
           :int{1}, :int{1}, "", "?\x00\x2f\x7e", ""
         }

+        // ISO-2022-KR
+        {
+          "ibm-25546",
+          "AB\uc88b\U00050005\uacccC",
+          :bin{           1b24294341420e41410f5c3530303035200e306a0f43 },
+          :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,3,3,3,3,3,3,5,5,5,6,6 },
+          :int{1}, :int{1}, "", "&S", ""
+        }

 
         // ISO-2022-JP-2 with G2 designator & SS2 shift
         {
 
         // ISO-2022-JP-2 with G2 designator & SS2 shift
         {


@@ -1270,6 +1491,18 @@ conversion:table(nofallback) {
 
         // simple sample, no error handling
         { "UTF-8", "a\U0010FFFF", :bin{ 61F48FBFBF }, :intvector{ 0, 1, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" }
 
         // simple sample, no error handling
         { "UTF-8", "a\U0010FFFF", :bin{ 61F48FBFBF }, :intvector{ 0, 1, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" }

+
+        // Verify that incomplete surrogates are handled as an error
+        { "UTF-8", "a\udc00", :bin{ 61efbfbd }, :intvector{ 0, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" }
+        { "UTF-8", "a\ud800", :bin{ 61efbfbd }, :intvector{ 0, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" }
+        { "UTF-8", "a\udc00b", :bin{ 61efbfbd62 }, :intvector{ 0, 1, 1, 1, 2 }, :int{1}, :int{0}, "", "", "" }
+        { "UTF-8", "a\ud800b", :bin{ 61efbfbd62 }, :intvector{ 0, 1, 1, 1, 2 }, :int{1}, :int{0}, "", "", "" }
+
+        // Code coverage for the EUC variants.
+        { "EUC-JP", "\u0061\u4edd\u5bec\ud801\udc01\ud801\u0061\u00a2", :bin{ 61a1b88ff4ae618ee0 }, :intvector{ 0, 1, 1, 2, 2, 2, 6, 7, 7 }, :int{1}, :int{0}, "", "0", "" }
+        { "EUC-JP", "\u0061\u4edd\u5bec\ud801\udc01\ud801\u0061\u00a2", :bin{ 61a1b88ff4aef4fef4fe618ee0 }, :intvector{ 0, 1, 1, 2, 2, 2, 3, 3, 5, 5, 6, 7, 7 }, :int{1}, :int{0}, "", "", "" }
+        { "EUC-TW", "\u0061\u2295\u5BF2\ud801\udc01\ud801\u0061\u8706\u008a", :bin{ 61a2d38ea2dce561e6ca8a }, :intvector{ 0, 1, 1, 2, 2, 2, 2, 6, 7, 7, 8 }, :int{1}, :int{0}, "", "0", "" }
+        { "EUC-TW", "\u0061\u2295\u5BF2\ud801\udc01\ud801\u0061\u8706\u008a", :bin{ 61a2d38ea2dce5fdfefdfe61e6ca8a }, :intvector{ 0, 1, 1, 2, 2, 2, 2, 3, 3, 5, 5, 6, 7, 7, 8 }, :int{1}, :int{0}, "", "", "" }

       }
     }
 
       }
     }
 


@@ -1367,16 +1600,29 @@ conversion:table(nofallback) {
         // versions of ISO-2022-JP
         {
           "ISO-2022-JP",
         // versions of ISO-2022-JP
         {
           "ISO-2022-JP",

-          "[\x00-\x0d\x10-\x1a\x1c-\x7f\u0391-\u03a1\uff61-\uff9f\u4e00\u4e01\uffe5]",
-          "[\x0e\x0f\x1b\u0100-\u0113\u0385-\u038a\u4e02\u4e27-\u4e29\uffe6-\U0010ffff]",
+          "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa5\u0391-\u03a1\u2015\u203e\u4e00\u4e01\uffe5]",
+          "[\x0e\x0f\x1b\u0100-\u0113\u0385-\u038a\u2014\u301c\u4e02\u4e27-\u4e29\u4fe0\u663b\u9eb5\ufa0e-\ufa2d\uff61-\uff9f\uffe4\uffe6-\U0010ffff]",

           :int{0}
         }
         {
           "ISO-2022-JP-2",
           :int{0}
         }
         {
           "ISO-2022-JP-2",

-          "[\x00-\x0d\x10-\x1a\x1c-\u0113\u0385-\u038a\u0390-\u03a1\uff61-\uff9f\u4e00-\u4e05\uffe6]",
-          "[\x0e\x0f\x1b\uffe7-\U0010ffff]",
+          "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa0-\u0113\u0384-\u0386\u0388-\u038a\u0390-\u03a1\u203e\u4e00-\u4e05\u4fe0\u663b\uffe6]",
+          "[\x0e\x0f\x1b\uff61-\uff9f\uffe4\uffe7-\U0010ffff]",

           :int{0}
         }
           :int{0}
         }

+        {
+          "JIS7",
+          "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa0-\u0113\u0384-\u0386\u0388-\u038a\u0390-\u03a1\u203e\u4e00-\u4e05\u4fe0\u663b\uff61-\uff9f\uffe6]",
+          "[\x0e\x0f\x1b\uffe4\uffe7-\U0010ffff]",
+          :int{0}
+        }
+        // with fallbacks
+        {
+          "ISO-2022-JP",
+          "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa5\u0391-\u03a1\u2014\u2015\u203e\u301c\u4e00\u4e01\u4fe0\u9eb5\uff61-\uff9f\uffe5]",
+          "[\x0e\x0f\x1b\xa6\u0100-\u0113\u0385-\u038a\u4e02\u4e27-\u4e29\u663b\ufa0e-\ufa2d\uffe4\uffe6-\U0010ffff]",
+          :int{1}
+        }

 
         // versions of ISO-2022-CN
         {
 
         // versions of ISO-2022-CN
         {


@@ -1392,6 +1638,43 @@ conversion:table(nofallback) {
           :int{0}
         }
 
           :int{0}
         }
 

+        // HZ
+        {
+          "HZ",
+          "[\u0410-\u044f\u4e00\u4e01\u4e03]",
+          "[\u4e02\u4e04-\u4e06\uac00-\ud7ff]",
+          :int{0}
+        }
+        
+        // LMBCS
+        {
+          "LMBCS",
+          "[\x00-\U0010ffff]",
+          "[]",
+          :int{0}
+        }
+        
+        // ISCII,version=0 (note: all versions of ISCII generates the same Unicodeset)
+        {
+          "iscii-dev",
+          "[\x00-\xa0\u0909-\u0939\u0993-\u09a8\u0a13-\u0a28\u0a93-\u0aa8\u0ae6-\u0aef\u0b05-\u0b0c\u0b13-\u0b28\u0bae-\u0bb5\u0c12-\u0c28\u0c92-\u0ca8\u0d12-\u0d28]",
+          "[\u0971-\u0975\u09e4\u0a4e-\u0a58\u0a80\u0b72-\u0b7a\u0bfb-\u0bff\u0c70-\u0c7f\u0c80\u0d00]",
+          :int{0}
+        }
+        
+        {
+          "iso-8859-1",
+          "[\x00-\xff]",
+          "[\u0100-\u01ff]",
+          :int{0}
+        }
+        
+        {
+          "us-ascii",
+          "[\x00-\x7f]",
+          "[\u0100-\u01ff]",
+          :int{0}
+        }

         // DBCS-only
         {
           "ibm-971",
         // DBCS-only
         {
           "ibm-971",