X-Git-Url: https://git.saurik.com/wxWidgets.git/blobdiff_plain/8899b155a1e4fa5f4b90e1f3bebe28088ea46bc9..3c01c5951189e13b8b4a5d7b288b54d4a57a30f8:/tests/mbconv/mbconvtest.cpp diff --git a/tests/mbconv/mbconvtest.cpp b/tests/mbconv/mbconvtest.cpp index 8bb3fd878e..5260887d7c 100644 --- a/tests/mbconv/mbconvtest.cpp +++ b/tests/mbconv/mbconvtest.cpp @@ -1,10 +1,10 @@ /////////////////////////////////////////////////////////////////////////////// // Name: tests/mbconv/main.cpp // Purpose: wxMBConv unit test -// Author: Vadim Zeitlin +// Author: Vadim Zeitlin, Mike Wetherell, Vince Harron // Created: 14.02.04 // RCS-ID: $Id$ -// Copyright: (c) 2003 TT-Solutions +// Copyright: (c) 2003 TT-Solutions, (c) 2005 Mike Wetherell, Vince Harron /////////////////////////////////////////////////////////////////////////////// // ---------------------------------------------------------------------------- @@ -23,6 +23,38 @@ #include "wx/strconv.h" #include "wx/string.h" +#include "wx/txtstrm.h" +#include "wx/mstream.h" + +#if defined wxHAVE_TCHAR_SUPPORT && !defined HAVE_WCHAR_H + #define HAVE_WCHAR_H +#endif + +// ---------------------------------------------------------------------------- +// Some wide character constants. "\uXXXX" escapes aren't supported by old +// compilers such as VC++ 5 and g++ 2.95. +// ---------------------------------------------------------------------------- + +wchar_t u41[] = { 0x41, 0 }; +wchar_t u7f[] = { 0x7f, 0 }; + +wchar_t u80[] = { 0x80, 0 }; +wchar_t u391[] = { 0x391, 0 }; +wchar_t u7ff[] = { 0x7ff, 0 }; + +wchar_t u800[] = { 0x800, 0 }; +wchar_t u2620[] = { 0x2620, 0 }; +wchar_t ufffd[] = { 0xfffd, 0 }; + +#if SIZEOF_WCHAR_T == 4 +wchar_t u10000[] = { 0x10000, 0 }; +wchar_t u1000a5[] = { 0x1000a5, 0 }; +wchar_t u10fffd[] = { 0x10fffd, 0 }; +#else +wchar_t u10000[] = { 0xd800, 0xdc00, 0 }; +wchar_t u1000a5[] = { 0xdbc0, 0xdca5, 0 }; +wchar_t u10fffd[] = { 0xdbff, 0xdffd, 0 }; +#endif // ---------------------------------------------------------------------------- // test class @@ -35,10 +67,147 @@ public: private: CPPUNIT_TEST_SUITE( MBConvTestCase ); + CPPUNIT_TEST( UTF32LETests ); + CPPUNIT_TEST( UTF32BETests ); CPPUNIT_TEST( WC2CP1250 ); + CPPUNIT_TEST( UTF7Tests ); + CPPUNIT_TEST( UTF8Tests ); + CPPUNIT_TEST( UTF16LETests ); + CPPUNIT_TEST( UTF16BETests ); + CPPUNIT_TEST( CP932Tests ); + CPPUNIT_TEST( CP1252Tests ); // depends on UTF8 Decoder functioning correctly + CPPUNIT_TEST( LibcTests ); + CPPUNIT_TEST( IconvTests ); + CPPUNIT_TEST( Latin1Tests ); + CPPUNIT_TEST( FontmapTests ); + CPPUNIT_TEST( BufSize ); +#ifdef HAVE_WCHAR_H + CPPUNIT_TEST( UTF8_41 ); + CPPUNIT_TEST( UTF8_7f ); + CPPUNIT_TEST( UTF8_80 ); + CPPUNIT_TEST( UTF8_c2_7f ); + CPPUNIT_TEST( UTF8_c2_80 ); + CPPUNIT_TEST( UTF8_ce_91 ); + CPPUNIT_TEST( UTF8_df_bf ); + CPPUNIT_TEST( UTF8_df_c0 ); + CPPUNIT_TEST( UTF8_e0_a0_7f ); + CPPUNIT_TEST( UTF8_e0_a0_80 ); + CPPUNIT_TEST( UTF8_e2_98_a0 ); + CPPUNIT_TEST( UTF8_ef_bf_bd ); + CPPUNIT_TEST( UTF8_ef_bf_c0 ); + CPPUNIT_TEST( UTF8_f0_90_80_7f ); + CPPUNIT_TEST( UTF8_f0_90_80_80 ); + CPPUNIT_TEST( UTF8_f4_8f_bf_bd ); + CPPUNIT_TEST( UTF8PUA_f4_80_82_a5 ); + CPPUNIT_TEST( UTF8Octal_backslash245 ); +#endif // HAVE_WCHAR_H CPPUNIT_TEST_SUITE_END(); void WC2CP1250(); + void UTF7Tests(); + void UTF8Tests(); + void UTF16LETests(); + void UTF16BETests(); + void UTF32LETests(); + void UTF32BETests(); + void CP932Tests(); + void CP1252Tests(); + void LibcTests(); + void FontmapTests(); + void BufSize(); + void IconvTests(); + void Latin1Tests(); + + // verifies that the specified multibyte sequence decodes to the specified wchar_t sequence + void TestDecoder( + const wchar_t* wideBuffer, // the same character sequence as multiBuffer, encoded as wchar_t + size_t wideChars, // the number of wide characters at wideBuffer + const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" + size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" + wxMBConv& converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence + int sizeofNull // number of bytes occupied by terminating null in this encoding + ); + + // verifies that the specified wchar_t sequence encodes to the specified multibyte sequence + void TestEncoder( + const wchar_t* wideBuffer, // the same character sequence as multiBuffer, encoded as wchar_t + size_t wideChars, // the number of wide characters at wideBuffer + const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" + size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" + wxMBConv& converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence + int sizeofNull // number of bytes occupied by terminating null in this encoding + ); + +#if wxUSE_UNICODE && wxUSE_STREAMS + // use wxTextInputStream to exercise wxMBConv interface + // (this reveals some bugs in certain wxMBConv subclasses) + void TestStreamDecoder( + const wchar_t* wideBuffer, // the same character sequence as multiBuffer, encoded as wchar_t + size_t wideChars, // the number of wide characters at wideBuffer + const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" + size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" + wxMBConv& converter // the wxMBConv object that can decode multiBuffer into a wide character sequence + ); + + // use wxTextOutputStream to exercise wxMBConv interface + // (this reveals some bugs in certain wxMBConv subclasses) + void TestStreamEncoder( + const wchar_t* wideBuffer, // the same character sequence as multiBuffer, encoded as wchar_t + size_t wideChars, // the number of wide characters at wideBuffer + const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" + size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" + wxMBConv& converter // the wxMBConv object that can decode multiBuffer into a wide character sequence + ); +#endif + + // tests the encoding and decoding capability of an wxMBConv object + // + // decodes the utf-8 bytes into wide characters + // encodes the wide characters to compare against input multiBuffer + // decodes the multiBuffer to compare against wide characters + // decodes the multiBuffer into wide characters + void TestCoder( + const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" + size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" + const char* utf8Buffer, // the same character sequence as multiBuffer, encoded as UTF-8 + size_t utf8Bytes, // the byte length of the UTF-8 encoded character sequence + wxMBConv& converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence + int sizeofNull // the number of bytes occupied by a terminating null in the converter's encoding + ); + +#ifdef HAVE_WCHAR_H + // UTF-8 tests. Test the first, last and one in the middle for sequences + // of each length + void UTF8_41() { UTF8("\x41", u41); } + void UTF8_7f() { UTF8("\x7f", u7f); } + void UTF8_80() { UTF8("\x80", NULL); } + + void UTF8_c2_7f() { UTF8("\xc2\x7f", NULL); } + void UTF8_c2_80() { UTF8("\xc2\x80", u80); } + void UTF8_ce_91() { UTF8("\xce\x91", u391); } + void UTF8_df_bf() { UTF8("\xdf\xbf", u7ff); } + void UTF8_df_c0() { UTF8("\xdf\xc0", NULL); } + + void UTF8_e0_a0_7f() { UTF8("\xe0\xa0\x7f", NULL); } + void UTF8_e0_a0_80() { UTF8("\xe0\xa0\x80", u800); } + void UTF8_e2_98_a0() { UTF8("\xe2\x98\xa0", u2620); } + void UTF8_ef_bf_bd() { UTF8("\xef\xbf\xbd", ufffd); } + void UTF8_ef_bf_c0() { UTF8("\xef\xbf\xc0", NULL); } + + void UTF8_f0_90_80_7f() { UTF8("\xf0\x90\x80\x7f", NULL); } + void UTF8_f0_90_80_80() { UTF8("\xf0\x90\x80\x80", u10000); } + void UTF8_f4_8f_bf_bd() { UTF8("\xf4\x8f\xbf\xbd", u10fffd); } + + // test 'escaping the escape characters' for the two escaping schemes + void UTF8PUA_f4_80_82_a5() { UTF8PUA("\xf4\x80\x82\xa5", u1000a5); } + void UTF8Octal_backslash245() { UTF8Octal("\\245", L"\\245"); } + + // implementation for the utf-8 tests (see comments below) + void UTF8(const char *charSequence, const wchar_t *wideSequence); + void UTF8PUA(const char *charSequence, const wchar_t *wideSequence); + void UTF8Octal(const char *charSequence, const wchar_t *wideSequence); + void UTF8(const char *charSequence, const wchar_t *wideSequence, int option); +#endif // HAVE_WCHAR_H DECLARE_NO_COPY_CLASS(MBConvTestCase) }; @@ -48,6 +217,7 @@ CPPUNIT_TEST_SUITE_REGISTRATION( MBConvTestCase ); // also include in it's own registry so that these tests can be run alone CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( MBConvTestCase, "MBConvTestCase" ); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( MBConvTestCase, "MBConv" ); void MBConvTestCase::WC2CP1250() { @@ -75,3 +245,1055 @@ void MBConvTestCase::WC2CP1250() } } } + +// Print an unsigned character array as a C unsigned character array. +// NB: Please don't remove this function even though it's not used anywhere, +// it's very useful when debugging a failed test. +wxString CByteArrayFormat( const void* data, size_t len, const wxChar* name ) +{ + const unsigned char* bytes = (unsigned char*)data; + wxString result; + + result.Printf( _T("static const unsigned char %s[%i] = \n{"), name, (int)len ); + + for ( size_t i = 0; i < len; i++ ) + { + if ( i != 0 ) + { + result.append( _T(",") ); + } + if ((i%16)==0) + { + result.append( _T("\n ") ); + } + wxString byte = wxString::Format( _T("0x%02x"), bytes[i] ); + result.append(byte); + } + result.append( _T("\n};\n") ); + return result; +} + +// The following bytes represent the same string, containing Japanese and English +// characters, encoded in several different formats. + +// encoded by iconv +static const unsigned char welcome_utf7_iconv[84] = +{ + 0x57,0x65,0x6c,0x63,0x6f,0x6d,0x65,0x20,0x74,0x6f,0x20,0x6f,0x75,0x72,0x20,0x63, + 0x79,0x62,0x65,0x72,0x20,0x73,0x70,0x61,0x63,0x65,0x20,0x66,0x6f,0x72,0x63,0x65, + 0x2e,0x20,0x20,0x2b,0x4d,0x46,0x6b,0x77,0x55,0x49,0x74,0x6d,0x57,0x39,0x38,0x77, + 0x61,0x35,0x62,0x37,0x69,0x6e,0x45,0x77,0x6b,0x6a,0x42,0x5a,0x4d,0x49,0x73,0x77, + 0x65,0x7a,0x42,0x47,0x4d,0x45,0x77,0x77,0x52,0x44,0x42,0x45,0x4d,0x47,0x63,0x77, + 0x57,0x54,0x41,0x43 +}; +// encoded by wxWindows (iconv can decode this successfully) +static const unsigned char welcome_utf7_wx[109] = +{ + 0x57,0x65,0x6c,0x63,0x6f,0x6d,0x65,0x2b,0x41,0x43,0x41,0x2d,0x74,0x6f,0x2b,0x41, + 0x43,0x41,0x2d,0x6f,0x75,0x72,0x2b,0x41,0x43,0x41,0x2d,0x63,0x79,0x62,0x65,0x72, + 0x2b,0x41,0x43,0x41,0x2d,0x73,0x70,0x61,0x63,0x65,0x2b,0x41,0x43,0x41,0x2d,0x66, + 0x6f,0x72,0x63,0x65,0x2e,0x2b,0x41,0x43,0x41,0x41,0x49,0x44,0x42,0x5a,0x4d,0x46, + 0x43,0x4c,0x5a,0x6c,0x76,0x66,0x4d,0x47,0x75,0x57,0x2b,0x34,0x70,0x78,0x4d,0x4a, + 0x49,0x77,0x57,0x54,0x43,0x4c,0x4d,0x48,0x73,0x77,0x52,0x6a,0x42,0x4d,0x4d,0x45, + 0x51,0x77,0x52,0x44,0x42,0x6e,0x4d,0x46,0x6b,0x77,0x41,0x67,0x2d +}; +// encoded by iconv +static const unsigned char welcome_utf8[89] = +{ + 0x57,0x65,0x6c,0x63,0x6f,0x6d,0x65,0x20,0x74,0x6f,0x20,0x6f,0x75,0x72,0x20,0x63, + 0x79,0x62,0x65,0x72,0x20,0x73,0x70,0x61,0x63,0x65,0x20,0x66,0x6f,0x72,0x63,0x65, + 0x2e,0x20,0x20,0xe3,0x81,0x99,0xe3,0x81,0x90,0xe8,0xad,0xa6,0xe5,0xaf,0x9f,0xe3, + 0x81,0xab,0xe9,0x9b,0xbb,0xe8,0xa9,0xb1,0xe3,0x82,0x92,0xe3,0x81,0x99,0xe3,0x82, + 0x8b,0xe3,0x81,0xbb,0xe3,0x81,0x86,0xe3,0x81,0x8c,0xe3,0x81,0x84,0xe3,0x81,0x84, + 0xe3,0x81,0xa7,0xe3,0x81,0x99,0xe3,0x80,0x82 +}; +// encoded by iconv +static const unsigned char welcome_utf16le[106] = +{ + 0x57,0x00,0x65,0x00,0x6c,0x00,0x63,0x00,0x6f,0x00,0x6d,0x00,0x65,0x00,0x20,0x00, + 0x74,0x00,0x6f,0x00,0x20,0x00,0x6f,0x00,0x75,0x00,0x72,0x00,0x20,0x00,0x63,0x00, + 0x79,0x00,0x62,0x00,0x65,0x00,0x72,0x00,0x20,0x00,0x73,0x00,0x70,0x00,0x61,0x00, + 0x63,0x00,0x65,0x00,0x20,0x00,0x66,0x00,0x6f,0x00,0x72,0x00,0x63,0x00,0x65,0x00, + 0x2e,0x00,0x20,0x00,0x20,0x00,0x59,0x30,0x50,0x30,0x66,0x8b,0xdf,0x5b,0x6b,0x30, + 0xfb,0x96,0x71,0x8a,0x92,0x30,0x59,0x30,0x8b,0x30,0x7b,0x30,0x46,0x30,0x4c,0x30, + 0x44,0x30,0x44,0x30,0x67,0x30,0x59,0x30,0x02,0x30 +}; +// encoded by iconv +static const unsigned char welcome_utf16be[106] = +{ + 0x00,0x57,0x00,0x65,0x00,0x6c,0x00,0x63,0x00,0x6f,0x00,0x6d,0x00,0x65,0x00,0x20, + 0x00,0x74,0x00,0x6f,0x00,0x20,0x00,0x6f,0x00,0x75,0x00,0x72,0x00,0x20,0x00,0x63, + 0x00,0x79,0x00,0x62,0x00,0x65,0x00,0x72,0x00,0x20,0x00,0x73,0x00,0x70,0x00,0x61, + 0x00,0x63,0x00,0x65,0x00,0x20,0x00,0x66,0x00,0x6f,0x00,0x72,0x00,0x63,0x00,0x65, + 0x00,0x2e,0x00,0x20,0x00,0x20,0x30,0x59,0x30,0x50,0x8b,0x66,0x5b,0xdf,0x30,0x6b, + 0x96,0xfb,0x8a,0x71,0x30,0x92,0x30,0x59,0x30,0x8b,0x30,0x7b,0x30,0x46,0x30,0x4c, + 0x30,0x44,0x30,0x44,0x30,0x67,0x30,0x59,0x30,0x02 +}; +// encoded by iconv +static const unsigned char welcome_utf32le[212] = +{ + 0x57,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x63,0x00,0x00,0x00, + 0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x20,0x00,0x00,0x00, + 0x74,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, + 0x75,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x63,0x00,0x00,0x00, + 0x79,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x72,0x00,0x00,0x00, + 0x20,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x61,0x00,0x00,0x00, + 0x63,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x66,0x00,0x00,0x00, + 0x6f,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x65,0x00,0x00,0x00, + 0x2e,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x59,0x30,0x00,0x00, + 0x50,0x30,0x00,0x00,0x66,0x8b,0x00,0x00,0xdf,0x5b,0x00,0x00,0x6b,0x30,0x00,0x00, + 0xfb,0x96,0x00,0x00,0x71,0x8a,0x00,0x00,0x92,0x30,0x00,0x00,0x59,0x30,0x00,0x00, + 0x8b,0x30,0x00,0x00,0x7b,0x30,0x00,0x00,0x46,0x30,0x00,0x00,0x4c,0x30,0x00,0x00, + 0x44,0x30,0x00,0x00,0x44,0x30,0x00,0x00,0x67,0x30,0x00,0x00,0x59,0x30,0x00,0x00, + 0x02,0x30,0x00,0x00 +}; +// encoded by iconv +static const unsigned char welcome_utf32be[212] = +{ + 0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x63, + 0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x20, + 0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x6f, + 0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x63, + 0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x72, + 0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x61, + 0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x66, + 0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x65, + 0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x20,0x00,0x00,0x30,0x59, + 0x00,0x00,0x30,0x50,0x00,0x00,0x8b,0x66,0x00,0x00,0x5b,0xdf,0x00,0x00,0x30,0x6b, + 0x00,0x00,0x96,0xfb,0x00,0x00,0x8a,0x71,0x00,0x00,0x30,0x92,0x00,0x00,0x30,0x59, + 0x00,0x00,0x30,0x8b,0x00,0x00,0x30,0x7b,0x00,0x00,0x30,0x46,0x00,0x00,0x30,0x4c, + 0x00,0x00,0x30,0x44,0x00,0x00,0x30,0x44,0x00,0x00,0x30,0x67,0x00,0x00,0x30,0x59, + 0x00,0x00,0x30,0x02 +}; +// encoded by iconv +static const unsigned char welcome_cp932[71] = +{ + 0x57,0x65,0x6c,0x63,0x6f,0x6d,0x65,0x20,0x74,0x6f,0x20,0x6f,0x75,0x72,0x20,0x63, + 0x79,0x62,0x65,0x72,0x20,0x73,0x70,0x61,0x63,0x65,0x20,0x66,0x6f,0x72,0x63,0x65, + 0x2e,0x20,0x20,0x82,0xb7,0x82,0xae,0x8c,0x78,0x8e,0x40,0x82,0xc9,0x93,0x64,0x98, + 0x62,0x82,0xf0,0x82,0xb7,0x82,0xe9,0x82,0xd9,0x82,0xa4,0x82,0xaa,0x82,0xa2,0x82, + 0xa2,0x82,0xc5,0x82,0xb7,0x81,0x42 +}; + +#if wxBYTE_ORDER == wxBIG_ENDIAN + #if SIZEOF_WCHAR_T == 2 + #define welcome_wchar_t welcome_utf16be + #elif SIZEOF_WCHAR_T == 4 + #define welcome_wchar_t welcome_utf32be + #endif +#elif wxBYTE_ORDER == wxLITTLE_ENDIAN + #if SIZEOF_WCHAR_T == 2 + #define welcome_wchar_t welcome_utf16le + #elif SIZEOF_WCHAR_T == 4 + #define welcome_wchar_t welcome_utf32le + #endif +#endif + +void MBConvTestCase::UTF7Tests() +{ +#if 0 + wxCSConv convUTF7(wxFONTENCODING_UTF7); +#else + wxMBConvUTF7 convUTF7; +#endif + + TestDecoder + ( + (const wchar_t*)welcome_wchar_t, + sizeof(welcome_wchar_t)/sizeof(wchar_t), + (const char*)welcome_utf7_iconv, + sizeof(welcome_utf7_iconv), + convUTF7, + 1 + ); + TestDecoder + ( + (const wchar_t*)welcome_wchar_t, + sizeof(welcome_wchar_t)/sizeof(wchar_t), + (const char*)welcome_utf7_wx, + sizeof(welcome_utf7_wx), + convUTF7, + 1 + ); +#if 0 + // wxWidget's UTF-7 encoder generates different byte sequences than iconv's. + // but both seem to be equally legal. + // This test won't work and that's okay. + TestEncoder + ( + (const wchar_t*)welcome_wchar_t, + sizeof(welcome_wchar_t)/sizeof(wchar_t), + (const char*)welcome_utf7_iconv, + sizeof(welcome_utf7_iconv), + convUTF7, + 1 + ); +#endif + TestEncoder + ( + (const wchar_t*)welcome_wchar_t, + sizeof(welcome_wchar_t)/sizeof(wchar_t), + (const char*)welcome_utf7_wx, + sizeof(welcome_utf7_wx), + convUTF7, + 1 + ); +} + +void MBConvTestCase::UTF8Tests() +{ + TestDecoder + ( + (const wchar_t*)welcome_wchar_t, + sizeof(welcome_wchar_t)/sizeof(wchar_t), + (const char*)welcome_utf8, + sizeof(welcome_utf8), + wxConvUTF8, + 1 + ); + TestEncoder + ( + (const wchar_t*)welcome_wchar_t, + sizeof(welcome_wchar_t)/sizeof(wchar_t), + (const char*)welcome_utf8, + sizeof(welcome_utf8), + wxConvUTF8, + 1 + ); +} + +void MBConvTestCase::UTF16LETests() +{ + wxMBConvUTF16LE convUTF16LE; + TestDecoder + ( + (const wchar_t*)welcome_wchar_t, + sizeof(welcome_wchar_t)/sizeof(wchar_t), + (const char*)welcome_utf16le, + sizeof(welcome_utf16le), + convUTF16LE, + 2 + ); + TestEncoder + ( + (const wchar_t*)welcome_wchar_t, + sizeof(welcome_wchar_t)/sizeof(wchar_t), + (const char*)welcome_utf16le, + sizeof(welcome_utf16le), + convUTF16LE, + 2 + ); +} + +void MBConvTestCase::UTF16BETests() +{ + wxMBConvUTF16BE convUTF16BE; + TestDecoder + ( + (const wchar_t*)welcome_wchar_t, + sizeof(welcome_wchar_t)/sizeof(wchar_t), + (const char*)welcome_utf16be, + sizeof(welcome_utf16be), + convUTF16BE, + 2 + ); + TestEncoder + ( + (const wchar_t*)welcome_wchar_t, + sizeof(welcome_wchar_t)/sizeof(wchar_t), + (const char*)welcome_utf16be, + sizeof(welcome_utf16be), + convUTF16BE, + 2 + ); +} + +void MBConvTestCase::UTF32LETests() +{ + wxMBConvUTF32LE convUTF32LE; + TestDecoder + ( + (const wchar_t*)welcome_wchar_t, + sizeof(welcome_wchar_t)/sizeof(wchar_t), + (const char*)welcome_utf32le, + sizeof(welcome_utf32le), + convUTF32LE, + 4 + ); + TestEncoder + ( + (const wchar_t*)welcome_wchar_t, + sizeof(welcome_wchar_t)/sizeof(wchar_t), + (const char*)welcome_utf32le, + sizeof(welcome_utf32le), + convUTF32LE, + 4 + ); +} + +void MBConvTestCase::UTF32BETests() +{ + wxMBConvUTF32BE convUTF32BE; + TestDecoder + ( + (const wchar_t*)welcome_wchar_t, + sizeof(welcome_wchar_t)/sizeof(wchar_t), + (const char*)welcome_utf32be, + sizeof(welcome_utf32be), + convUTF32BE, + 4 + ); + TestEncoder + ( + (const wchar_t*)welcome_wchar_t, + sizeof(welcome_wchar_t)/sizeof(wchar_t), + (const char*)welcome_utf32be, + sizeof(welcome_utf32be), + convUTF32BE, + 4 + ); +} + +void MBConvTestCase::CP932Tests() +{ + wxCSConv convCP932( wxFONTENCODING_CP932 ); + TestDecoder + ( + (const wchar_t*)welcome_wchar_t, + sizeof(welcome_wchar_t)/sizeof(wchar_t), + (const char*)welcome_cp932, + sizeof(welcome_cp932), + convCP932, + 1 + ); + TestEncoder + ( + (const wchar_t*)welcome_wchar_t, + sizeof(welcome_wchar_t)/sizeof(wchar_t), + (const char*)welcome_cp932, + sizeof(welcome_cp932), + convCP932, + 1 + ); +} + +// a character sequence encoded as iso8859-1 (iconv) +static const unsigned char iso8859_1[251] = +{ + 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14, + 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24, + 0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34, + 0x35,0x36,0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,0x40,0x41,0x42,0x43,0x44, + 0x45,0x46,0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54, + 0x55,0x56,0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,0x60,0x61,0x62,0x63,0x64, + 0x65,0x66,0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74, + 0x75,0x76,0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,0x80,0x81,0x82,0x83,0x84, + 0x85,0x86,0x87,0x88,0x89,0x8a,0x8b,0x8c,0x8d,0x8e,0x8f,0x90,0x91,0x92,0x93,0x94, + 0x95,0x96,0x97,0x98,0x99,0x9a,0x9b,0x9c,0x9d,0x9e,0x9f,0xa0,0xa1,0xa2,0xa3,0xa4, + 0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xab,0xac,0xad,0xae,0xaf,0xb0,0xb1,0xb2,0xb3,0xb4, + 0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xbb,0xbc,0xbd,0xbe,0xbf,0xc0,0xc1,0xc2,0xc3,0xc4, + 0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xcb,0xcc,0xcd,0xce,0xcf,0xd0,0xd1,0xd2,0xd3,0xd4, + 0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xdb,0xdc,0xdd,0xde,0xdf,0xe0,0xe1,0xe2,0xe3,0xe4, + 0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xeb,0xec,0xed,0xee,0xef,0xf0,0xf1,0xf2,0xf3,0xf4, + 0xf5,0xf6,0xf7,0xf8,0xf9,0xfa,0xfb,0xfc,0xfd,0xfe,0xff +}; +// the above character sequence encoded as UTF-8 (iconv) +static const unsigned char iso8859_1_utf8[379] = +{ + 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14, + 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24, + 0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34, + 0x35,0x36,0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,0x40,0x41,0x42,0x43,0x44, + 0x45,0x46,0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54, + 0x55,0x56,0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,0x60,0x61,0x62,0x63,0x64, + 0x65,0x66,0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74, + 0x75,0x76,0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,0xc2,0x80,0xc2,0x81,0xc2, + 0x82,0xc2,0x83,0xc2,0x84,0xc2,0x85,0xc2,0x86,0xc2,0x87,0xc2,0x88,0xc2,0x89,0xc2, + 0x8a,0xc2,0x8b,0xc2,0x8c,0xc2,0x8d,0xc2,0x8e,0xc2,0x8f,0xc2,0x90,0xc2,0x91,0xc2, + 0x92,0xc2,0x93,0xc2,0x94,0xc2,0x95,0xc2,0x96,0xc2,0x97,0xc2,0x98,0xc2,0x99,0xc2, + 0x9a,0xc2,0x9b,0xc2,0x9c,0xc2,0x9d,0xc2,0x9e,0xc2,0x9f,0xc2,0xa0,0xc2,0xa1,0xc2, + 0xa2,0xc2,0xa3,0xc2,0xa4,0xc2,0xa5,0xc2,0xa6,0xc2,0xa7,0xc2,0xa8,0xc2,0xa9,0xc2, + 0xaa,0xc2,0xab,0xc2,0xac,0xc2,0xad,0xc2,0xae,0xc2,0xaf,0xc2,0xb0,0xc2,0xb1,0xc2, + 0xb2,0xc2,0xb3,0xc2,0xb4,0xc2,0xb5,0xc2,0xb6,0xc2,0xb7,0xc2,0xb8,0xc2,0xb9,0xc2, + 0xba,0xc2,0xbb,0xc2,0xbc,0xc2,0xbd,0xc2,0xbe,0xc2,0xbf,0xc3,0x80,0xc3,0x81,0xc3, + 0x82,0xc3,0x83,0xc3,0x84,0xc3,0x85,0xc3,0x86,0xc3,0x87,0xc3,0x88,0xc3,0x89,0xc3, + 0x8a,0xc3,0x8b,0xc3,0x8c,0xc3,0x8d,0xc3,0x8e,0xc3,0x8f,0xc3,0x90,0xc3,0x91,0xc3, + 0x92,0xc3,0x93,0xc3,0x94,0xc3,0x95,0xc3,0x96,0xc3,0x97,0xc3,0x98,0xc3,0x99,0xc3, + 0x9a,0xc3,0x9b,0xc3,0x9c,0xc3,0x9d,0xc3,0x9e,0xc3,0x9f,0xc3,0xa0,0xc3,0xa1,0xc3, + 0xa2,0xc3,0xa3,0xc3,0xa4,0xc3,0xa5,0xc3,0xa6,0xc3,0xa7,0xc3,0xa8,0xc3,0xa9,0xc3, + 0xaa,0xc3,0xab,0xc3,0xac,0xc3,0xad,0xc3,0xae,0xc3,0xaf,0xc3,0xb0,0xc3,0xb1,0xc3, + 0xb2,0xc3,0xb3,0xc3,0xb4,0xc3,0xb5,0xc3,0xb6,0xc3,0xb7,0xc3,0xb8,0xc3,0xb9,0xc3, + 0xba,0xc3,0xbb,0xc3,0xbc,0xc3,0xbd,0xc3,0xbe,0xc3,0xbf +}; + +// a character sequence encoded as CP1252 (iconv) +static const unsigned char CP1252[246] = +{ + 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14, + 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24, + 0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34, + 0x35,0x36,0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,0x40,0x41,0x42,0x43,0x44, + 0x45,0x46,0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54, + 0x55,0x56,0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,0x60,0x61,0x62,0x63,0x64, + 0x65,0x66,0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74, + 0x75,0x76,0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,0xa0,0xa1,0xa2,0xa3,0xa4, + 0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xab,0xac,0xad,0xae,0xaf,0xb0,0xb1,0xb2,0xb3,0xb4, + 0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xbb,0xbc,0xbd,0xbe,0xbf,0xc0,0xc1,0xc2,0xc3,0xc4, + 0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xcb,0xcc,0xcd,0xce,0xcf,0xd0,0xd1,0xd2,0xd3,0xd4, + 0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xdb,0xdc,0xdd,0xde,0xdf,0xe0,0xe1,0xe2,0xe3,0xe4, + 0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xeb,0xec,0xed,0xee,0xef,0xf0,0xf1,0xf2,0xf3,0xf4, + 0xf5,0xf6,0xf7,0xf8,0xf9,0xfa,0xfb,0xfc,0xfd,0xfe,0xff,0x8c,0x9c,0x8a,0x9a,0x9f, + 0x8e,0x9e,0x83,0x88,0x98,0x96,0x97,0x91,0x92,0x82,0x93,0x94,0x84,0x86,0x87,0x95, + 0x85,0x89,0x8b,0x9b,0x80,0x99 +}; +// the above character sequence encoded as UTF-8 (iconv) +static const unsigned char CP1252_utf8[386] = +{ + 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14, + 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24, + 0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34, + 0x35,0x36,0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,0x40,0x41,0x42,0x43,0x44, + 0x45,0x46,0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54, + 0x55,0x56,0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,0x60,0x61,0x62,0x63,0x64, + 0x65,0x66,0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74, + 0x75,0x76,0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,0xc2,0xa0,0xc2,0xa1,0xc2, + 0xa2,0xc2,0xa3,0xc2,0xa4,0xc2,0xa5,0xc2,0xa6,0xc2,0xa7,0xc2,0xa8,0xc2,0xa9,0xc2, + 0xaa,0xc2,0xab,0xc2,0xac,0xc2,0xad,0xc2,0xae,0xc2,0xaf,0xc2,0xb0,0xc2,0xb1,0xc2, + 0xb2,0xc2,0xb3,0xc2,0xb4,0xc2,0xb5,0xc2,0xb6,0xc2,0xb7,0xc2,0xb8,0xc2,0xb9,0xc2, + 0xba,0xc2,0xbb,0xc2,0xbc,0xc2,0xbd,0xc2,0xbe,0xc2,0xbf,0xc3,0x80,0xc3,0x81,0xc3, + 0x82,0xc3,0x83,0xc3,0x84,0xc3,0x85,0xc3,0x86,0xc3,0x87,0xc3,0x88,0xc3,0x89,0xc3, + 0x8a,0xc3,0x8b,0xc3,0x8c,0xc3,0x8d,0xc3,0x8e,0xc3,0x8f,0xc3,0x90,0xc3,0x91,0xc3, + 0x92,0xc3,0x93,0xc3,0x94,0xc3,0x95,0xc3,0x96,0xc3,0x97,0xc3,0x98,0xc3,0x99,0xc3, + 0x9a,0xc3,0x9b,0xc3,0x9c,0xc3,0x9d,0xc3,0x9e,0xc3,0x9f,0xc3,0xa0,0xc3,0xa1,0xc3, + 0xa2,0xc3,0xa3,0xc3,0xa4,0xc3,0xa5,0xc3,0xa6,0xc3,0xa7,0xc3,0xa8,0xc3,0xa9,0xc3, + 0xaa,0xc3,0xab,0xc3,0xac,0xc3,0xad,0xc3,0xae,0xc3,0xaf,0xc3,0xb0,0xc3,0xb1,0xc3, + 0xb2,0xc3,0xb3,0xc3,0xb4,0xc3,0xb5,0xc3,0xb6,0xc3,0xb7,0xc3,0xb8,0xc3,0xb9,0xc3, + 0xba,0xc3,0xbb,0xc3,0xbc,0xc3,0xbd,0xc3,0xbe,0xc3,0xbf,0xc5,0x92,0xc5,0x93,0xc5, + 0xa0,0xc5,0xa1,0xc5,0xb8,0xc5,0xbd,0xc5,0xbe,0xc6,0x92,0xcb,0x86,0xcb,0x9c,0xe2, + 0x80,0x93,0xe2,0x80,0x94,0xe2,0x80,0x98,0xe2,0x80,0x99,0xe2,0x80,0x9a,0xe2,0x80, + 0x9c,0xe2,0x80,0x9d,0xe2,0x80,0x9e,0xe2,0x80,0xa0,0xe2,0x80,0xa1,0xe2,0x80,0xa2, + 0xe2,0x80,0xa6,0xe2,0x80,0xb0,0xe2,0x80,0xb9,0xe2,0x80,0xba,0xe2,0x82,0xac,0xe2, + 0x84,0xa2 +}; + +// this is unused currently so avoid warnings about this +#if 0 + +// a character sequence encoded as iso8859-5 (iconv) +static const unsigned char iso8859_5[251] = +{ + 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14, + 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24, + 0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34, + 0x35,0x36,0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,0x40,0x41,0x42,0x43,0x44, + 0x45,0x46,0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54, + 0x55,0x56,0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,0x60,0x61,0x62,0x63,0x64, + 0x65,0x66,0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74, + 0x75,0x76,0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,0x80,0x81,0x82,0x83,0x84, + 0x85,0x86,0x87,0x88,0x89,0x8a,0x8b,0x8c,0x8d,0x8e,0x8f,0x90,0x91,0x92,0x93,0x94, + 0x95,0x96,0x97,0x98,0x99,0x9a,0x9b,0x9c,0x9d,0x9e,0x9f,0xa0,0xfd,0xad,0xa1,0xa2, + 0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xab,0xac,0xae,0xaf,0xb0,0xb1,0xb2,0xb3, + 0xb4,0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xbb,0xbc,0xbd,0xbe,0xbf,0xc0,0xc1,0xc2,0xc3, + 0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xcb,0xcc,0xcd,0xce,0xcf,0xd0,0xd1,0xd2,0xd3, + 0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xdb,0xdc,0xdd,0xde,0xdf,0xe0,0xe1,0xe2,0xe3, + 0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xeb,0xec,0xed,0xee,0xef,0xf1,0xf2,0xf3,0xf4, + 0xf5,0xf6,0xf7,0xf8,0xf9,0xfa,0xfb,0xfc,0xfe,0xff,0xf0 +}; +// the above character sequence encoded as UTF-8 (iconv) +static const unsigned char iso8859_5_utf8[380] = +{ + 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14, + 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24, + 0x25,0x26,0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34, + 0x35,0x36,0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,0x40,0x41,0x42,0x43,0x44, + 0x45,0x46,0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54, + 0x55,0x56,0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,0x60,0x61,0x62,0x63,0x64, + 0x65,0x66,0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74, + 0x75,0x76,0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,0xc2,0x80,0xc2,0x81,0xc2, + 0x82,0xc2,0x83,0xc2,0x84,0xc2,0x85,0xc2,0x86,0xc2,0x87,0xc2,0x88,0xc2,0x89,0xc2, + 0x8a,0xc2,0x8b,0xc2,0x8c,0xc2,0x8d,0xc2,0x8e,0xc2,0x8f,0xc2,0x90,0xc2,0x91,0xc2, + 0x92,0xc2,0x93,0xc2,0x94,0xc2,0x95,0xc2,0x96,0xc2,0x97,0xc2,0x98,0xc2,0x99,0xc2, + 0x9a,0xc2,0x9b,0xc2,0x9c,0xc2,0x9d,0xc2,0x9e,0xc2,0x9f,0xc2,0xa0,0xc2,0xa7,0xc2, + 0xad,0xd0,0x81,0xd0,0x82,0xd0,0x83,0xd0,0x84,0xd0,0x85,0xd0,0x86,0xd0,0x87,0xd0, + 0x88,0xd0,0x89,0xd0,0x8a,0xd0,0x8b,0xd0,0x8c,0xd0,0x8e,0xd0,0x8f,0xd0,0x90,0xd0, + 0x91,0xd0,0x92,0xd0,0x93,0xd0,0x94,0xd0,0x95,0xd0,0x96,0xd0,0x97,0xd0,0x98,0xd0, + 0x99,0xd0,0x9a,0xd0,0x9b,0xd0,0x9c,0xd0,0x9d,0xd0,0x9e,0xd0,0x9f,0xd0,0xa0,0xd0, + 0xa1,0xd0,0xa2,0xd0,0xa3,0xd0,0xa4,0xd0,0xa5,0xd0,0xa6,0xd0,0xa7,0xd0,0xa8,0xd0, + 0xa9,0xd0,0xaa,0xd0,0xab,0xd0,0xac,0xd0,0xad,0xd0,0xae,0xd0,0xaf,0xd0,0xb0,0xd0, + 0xb1,0xd0,0xb2,0xd0,0xb3,0xd0,0xb4,0xd0,0xb5,0xd0,0xb6,0xd0,0xb7,0xd0,0xb8,0xd0, + 0xb9,0xd0,0xba,0xd0,0xbb,0xd0,0xbc,0xd0,0xbd,0xd0,0xbe,0xd0,0xbf,0xd1,0x80,0xd1, + 0x81,0xd1,0x82,0xd1,0x83,0xd1,0x84,0xd1,0x85,0xd1,0x86,0xd1,0x87,0xd1,0x88,0xd1, + 0x89,0xd1,0x8a,0xd1,0x8b,0xd1,0x8c,0xd1,0x8d,0xd1,0x8e,0xd1,0x8f,0xd1,0x91,0xd1, + 0x92,0xd1,0x93,0xd1,0x94,0xd1,0x95,0xd1,0x96,0xd1,0x97,0xd1,0x98,0xd1,0x99,0xd1, + 0x9a,0xd1,0x9b,0xd1,0x9c,0xd1,0x9e,0xd1,0x9f,0xe2,0x84,0x96 +}; +#endif // 0 + +// DecodeUTF8 +// decodes the specified *unterminated* UTF-8 byte array +wxWCharBuffer DecodeUTF8( + const void* data, // an unterminated UTF-8 encoded byte array + size_t size // the byte length of data + ) +{ + // the decoder requires a null terminated buffer. + // the input data is not null terminated. + // copy to null terminated buffer + + wxCharBuffer nullTerminated( size+1 ); + memcpy( nullTerminated.data(), data, size ); + nullTerminated.data()[size] = 0; + return wxConvUTF8.cMB2WC(nullTerminated.data()); +} + +// tests the encoding and decoding capability of an wxMBConv object +// +// decodes the utf-8 bytes into wide characters +// encodes the wide characters to compare against input multiBuffer +// decodes the multiBuffer to compare against wide characters +// decodes the multiBuffer into wide characters +void MBConvTestCase::TestCoder( + const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" + size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" + const char* utf8Buffer, // the same character sequence as multiBuffer, encoded as UTF-8 + size_t utf8Bytes, // the byte length of the UTF-8 encoded character sequence + wxMBConv& converter, // the wxMBConv object thta can decode multiBuffer into a wide character sequence + int sizeofNull // the number of bytes occupied by a terminating null in the converter's encoding + ) +{ + // wide character size and endian-ess varies from platform to platform + // compiler support for wide character literals varies from compiler to compiler + // so we should store the wide character version as UTF-8 and depend on + // the UTF-8 converter's ability to decode it to platform specific wide characters + // this test is invalid if the UTF-8 converter can't decode + wxWCharBuffer wideBuffer((size_t)0); + wideBuffer = DecodeUTF8( utf8Buffer, utf8Bytes ); + size_t wideChars = wxWcslen( wideBuffer.data() ); + + TestDecoder + ( + wideBuffer.data(), + wideChars, + (const char*)multiBuffer, + multiBytes, + converter, + sizeofNull + ); + TestEncoder + ( + wideBuffer.data(), + wideChars, + (const char*)multiBuffer, + multiBytes, + converter, + sizeofNull + ); +} + + +WXDLLIMPEXP_BASE wxMBConv* new_wxMBConv_wxwin( const char* name ); + +void MBConvTestCase::FontmapTests() +{ +#ifdef wxUSE_FONTMAP + wxMBConv* converter = new_wxMBConv_wxwin("CP1252"); + if ( !converter ) + { + return; + } + TestCoder( + (const char*)CP1252, + sizeof(CP1252), + (const char*)CP1252_utf8, + sizeof(CP1252_utf8), + *converter, + 1 + ); + delete converter; +#endif +} + +void MBConvTestCase::BufSize() +{ + wxCSConv conv1251(_T("CP1251")); + CPPUNIT_ASSERT( conv1251.IsOk() ); + const char *cp1251text = + "\313\301\326\305\324\323\321 \325\304\301\336\316\331\315"; + + const size_t lenW = conv1251.MB2WC(NULL, cp1251text, 0); + CPPUNIT_ASSERT_EQUAL( strlen(cp1251text), lenW ); + wxWCharBuffer wbuf(lenW + 1); // allocates lenW + 2 characters + wbuf.data()[lenW + 1] = L'!'; + + // lenW is not enough because it's the length and we need the size + CPPUNIT_ASSERT_EQUAL( + wxCONV_FAILED, conv1251.MB2WC(wbuf.data(), cp1251text, lenW) ); + + // lenW+1 is just fine + CPPUNIT_ASSERT( + conv1251.MB2WC(wbuf.data(), cp1251text, lenW + 1) != wxCONV_FAILED ); + + // of course, greater values work too + CPPUNIT_ASSERT( + conv1251.MB2WC(wbuf.data(), cp1251text, lenW + 2) != wxCONV_FAILED ); + + // but they shouldn't write more stuff to the buffer + CPPUNIT_ASSERT_EQUAL( L'!', wbuf[lenW + 1] ); + + + // test in the other direction too, using an encoding with multibyte NUL + wxCSConv convUTF16(_T("UTF-16LE")); + CPPUNIT_ASSERT( convUTF16.IsOk() ); + const wchar_t *utf16text = L"Hello"; + + const size_t lenMB = convUTF16.WC2MB(NULL, utf16text, 0); + CPPUNIT_ASSERT_EQUAL( wcslen(utf16text)*2, lenMB ); + wxCharBuffer buf(lenMB + 2); // it only adds 1 for NUL on its own, we need 2 + // for NUL and an extra one for the guard byte + buf.data()[lenMB + 2] = '?'; + + CPPUNIT_ASSERT_EQUAL( + wxCONV_FAILED, convUTF16.WC2MB(buf.data(), utf16text, lenMB) ); + CPPUNIT_ASSERT_EQUAL( + wxCONV_FAILED, convUTF16.WC2MB(buf.data(), utf16text, lenMB + 1) ); + CPPUNIT_ASSERT( + convUTF16.WC2MB(buf.data(), utf16text, lenMB + 2) != wxCONV_FAILED ); + CPPUNIT_ASSERT( + convUTF16.WC2MB(buf.data(), utf16text, lenMB + 3) != wxCONV_FAILED ); + CPPUNIT_ASSERT_EQUAL( '?', buf[lenMB + 2] ); +} + + +WXDLLIMPEXP_BASE wxMBConv* new_wxMBConv_iconv( const char* name ); + +void MBConvTestCase::IconvTests() +{ +#ifdef HAVE_ICONV + wxMBConv* converter = new_wxMBConv_iconv("CP932"); + if ( !converter ) + { + return; + } + TestCoder( + (const char*)welcome_cp932, + sizeof(welcome_cp932), + (const char*)welcome_utf8, + sizeof(welcome_utf8), + *converter, + 1 + ); + delete converter; +#endif +} + +void MBConvTestCase::Latin1Tests() +{ + TestCoder( + (const char*)iso8859_1, + sizeof(iso8859_1), + (const char*)iso8859_1_utf8, + sizeof(iso8859_1_utf8), + wxConvISO8859_1, + 1 + ); + + static const char nulstr[] = "foo\0bar\0"; + static const size_t mbLen = WXSIZEOF(nulstr) - 1; + size_t wcLen; + wxConvISO8859_1.cMB2WC(nulstr, mbLen, &wcLen); + CPPUNIT_ASSERT_EQUAL( mbLen, wcLen ); +} + +void MBConvTestCase::CP1252Tests() +{ + wxCSConv convCP1252( wxFONTENCODING_CP1252 ); + TestCoder( + (const char*)CP1252, + sizeof(CP1252), + (const char*)CP1252_utf8, + sizeof(CP1252_utf8), + convCP1252, + 1 + ); +} + +void MBConvTestCase::LibcTests() +{ + // There isn't a locale that all systems support (except "C"), so leave + // this one disabled for non-Windows systems for the moment, until + // a solution can be found. +#ifdef __WXMSW__ + +#ifdef __WXMSW__ + setlocale( LC_ALL, "English_United States.1252" ); + const unsigned char* systemMB = CP1252; + size_t systemMB_size = sizeof(CP1252); + const unsigned char* systemMB_utf8 = CP1252_utf8; + size_t systemMB_utf8_size = sizeof(CP1252_utf8); +#else + setlocale( LC_ALL, "en_US.iso8859-1" ); + const unsigned char* systemMB = iso8859_1; + size_t systemMB_size = sizeof(iso8859_1); + const unsigned char* systemMB_utf8 = iso8859_1_utf8; + size_t systemMB_utf8_size = sizeof(iso8859_1_utf8); +#endif + wxMBConvLibc convLibc; + TestCoder( + (const char*)systemMB, + systemMB_size, + (const char*)systemMB_utf8, + systemMB_utf8_size, + convLibc, + 1 + ); + +#endif // __WXMSW__ +} + +// verifies that the specified mb sequences decode to the specified wc sequence +void MBConvTestCase::TestDecoder( + const wchar_t* wideBuffer, // the same character sequence as multiBuffer, encoded as wchar_t + size_t wideChars, // the number of wide characters at wideBuffer + const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" + size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" + wxMBConv& converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence + int sizeofNull // number of bytes occupied by terminating null in this encoding + ) +{ + const unsigned UNINITIALIZED = 0xcd; + + // copy the input bytes into a null terminated buffer + wxCharBuffer inputCopy( multiBytes+sizeofNull ); + memcpy( inputCopy.data(), multiBuffer, multiBytes ); + memset( &inputCopy.data()[multiBytes], 0, sizeofNull ); + + // calculate the output size + size_t outputWritten = converter.MB2WC + ( + 0, + (const char*)inputCopy.data(), + 0 + ); + // make sure the correct output length was calculated + CPPUNIT_ASSERT_EQUAL( wideChars, outputWritten ); + + // convert the string + size_t guardChars = 8; // to make sure we're not overrunning the output buffer + size_t nullCharacters = 1; + size_t outputBufferChars = outputWritten + nullCharacters + guardChars; + wxWCharBuffer outputBuffer(outputBufferChars); + memset( outputBuffer.data(), UNINITIALIZED, outputBufferChars*sizeof(wchar_t) ); + + outputWritten = converter.MB2WC + ( + outputBuffer.data(), + (const char*)inputCopy.data(), + outputBufferChars + ); + // make sure the correct number of characters were outputs + CPPUNIT_ASSERT_EQUAL( wideChars, outputWritten ); + + // make sure the characters generated are correct + CPPUNIT_ASSERT( 0 == memcmp( outputBuffer, wideBuffer, wideChars*sizeof(wchar_t) ) ); + + // the output buffer should be null terminated + CPPUNIT_ASSERT( outputBuffer[outputWritten] == 0 ); + + // make sure the rest of the output buffer is untouched + for ( size_t i = (wideChars+1)*sizeof(wchar_t); i < (outputBufferChars*sizeof(wchar_t)); i++ ) + { + CPPUNIT_ASSERT( ((unsigned char*)outputBuffer.data())[i] == UNINITIALIZED ); + } + +#if wxUSE_UNICODE && wxUSE_STREAMS + TestStreamDecoder( wideBuffer, wideChars, multiBuffer, multiBytes, converter ); +#endif +} + +// verifies that the specified wc sequences encodes to the specified mb sequence +void MBConvTestCase::TestEncoder( + const wchar_t* wideBuffer, // the same character sequence as multiBuffer, encoded as wchar_t + size_t wideChars, // the number of wide characters at wideBuffer + const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" + size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" + wxMBConv& converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence + int sizeofNull // number of bytes occupied by terminating null in this encoding + ) +{ + const unsigned UNINITIALIZED = 0xcd; + + // copy the input bytes into a null terminated buffer + wxWCharBuffer inputCopy( wideChars + 1 ); + memcpy( inputCopy.data(), wideBuffer, (wideChars*sizeof(wchar_t)) ); + inputCopy.data()[wideChars] = 0; + + // calculate the output size + size_t outputWritten = converter.WC2MB + ( + 0, + (const wchar_t*)inputCopy.data(), + 0 + ); + // make sure the correct output length was calculated + CPPUNIT_ASSERT_EQUAL( multiBytes, outputWritten ); + + // convert the string + size_t guardBytes = 8; // to make sure we're not overrunning the output buffer + size_t outputBufferSize = outputWritten + sizeofNull + guardBytes; + wxCharBuffer outputBuffer(outputBufferSize); + memset( outputBuffer.data(), UNINITIALIZED, outputBufferSize ); + + outputWritten = converter.WC2MB + ( + outputBuffer.data(), + (const wchar_t*)inputCopy.data(), + outputBufferSize + ); + + // make sure the correct number of characters were output + CPPUNIT_ASSERT_EQUAL( multiBytes, outputWritten ); + + // make sure the characters generated are correct + CPPUNIT_ASSERT( 0 == memcmp( outputBuffer, multiBuffer, multiBytes ) ); + + size_t i; + + // the output buffer should be null terminated + for ( i = multiBytes; i < multiBytes + sizeofNull; i++ ) + { + CPPUNIT_ASSERT( ((unsigned char*)outputBuffer.data())[i] == 0 ); + } + + // make sure the rest of the output buffer is untouched + for ( i = multiBytes + sizeofNull; i < outputBufferSize; i++ ) + { + CPPUNIT_ASSERT( ((unsigned char*)outputBuffer.data())[i] == UNINITIALIZED ); + } + +#if wxUSE_UNICODE && wxUSE_STREAMS + TestStreamEncoder( wideBuffer, wideChars, multiBuffer, multiBytes, converter ); +#endif +} + +#if wxUSE_UNICODE && wxUSE_STREAMS +// use wxTextInputStream to exercise wxMBConv interface +// (this reveals some bugs in certain wxMBConv subclasses) +void MBConvTestCase::TestStreamDecoder( + const wchar_t* wideBuffer, // the same character sequence as multiBuffer, encoded as wchar_t + size_t wideChars, // the number of wide characters at wideBuffer + const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" + size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" + wxMBConv& converter // the wxMBConv object that can decode multiBuffer into a wide character sequence + ) +{ + // this isn't meant to test wxMemoryInputStream or wxTextInputStream + // it's meant to test the way wxTextInputStream uses wxMBConv + // (which has exposed some problems with wxMBConv) + wxMemoryInputStream memoryInputStream( multiBuffer, multiBytes ); + wxTextInputStream textInputStream( memoryInputStream, wxT(""), converter ); + for ( size_t i = 0; i < wideChars; i++ ) + { + wxChar wc = textInputStream.GetChar(); + CPPUNIT_ASSERT_EQUAL_MESSAGE( + std::string(wxString::Format("At position %lu", (unsigned long)i)), + wideBuffer[i], + wc + ); + } + CPPUNIT_ASSERT( 0 == textInputStream.GetChar() ); + CPPUNIT_ASSERT( memoryInputStream.Eof() ); +} +#endif + +#if wxUSE_UNICODE && wxUSE_STREAMS +// use wxTextInputStream to exercise wxMBConv interface +// (this reveals some bugs in certain wxMBConv subclasses) +void MBConvTestCase::TestStreamEncoder( + const wchar_t* wideBuffer, // the same character sequence as multiBuffer, encoded as wchar_t + size_t wideChars, // the number of wide characters at wideBuffer + const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" + size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" + wxMBConv& converter // the wxMBConv object that can decode multiBuffer into a wide character sequence + ) +{ + // this isn't meant to test wxMemoryOutputStream or wxTextOutputStream + // it's meant to test the way wxTextOutputStream uses wxMBConv + // (which has exposed some problems with wxMBConv) + wxMemoryOutputStream memoryOutputStream; + // wxEOL_UNIX will pass \n \r unchanged + wxTextOutputStream textOutputStream( memoryOutputStream, wxEOL_UNIX, converter ); + for ( size_t i = 0; i < wideChars; i++ ) + { + textOutputStream.PutChar( wideBuffer[i] ); + } + + textOutputStream.Flush(); + + CPPUNIT_ASSERT_EQUAL( multiBytes, size_t(memoryOutputStream.TellO()) ); + wxCharBuffer copy( memoryOutputStream.TellO() ); + memoryOutputStream.CopyTo( copy.data(), memoryOutputStream.TellO()); + CPPUNIT_ASSERT_EQUAL( 0, memcmp( copy.data(), multiBuffer, multiBytes ) ); +} +#endif + + +// ---------------------------------------------------------------------------- +// UTF-8 tests +// ---------------------------------------------------------------------------- + +#ifdef HAVE_WCHAR_H + +// Check that 'charSequence' translates to 'wideSequence' and back. +// Invalid sequences can be tested by giving NULL for 'wideSequence'. Even +// invalid sequences should roundtrip when an option is given and this is +// checked. +// +void MBConvTestCase::UTF8(const char *charSequence, + const wchar_t *wideSequence) +{ + UTF8(charSequence, wideSequence, wxMBConvUTF8::MAP_INVALID_UTF8_NOT); + UTF8(charSequence, wideSequence, wxMBConvUTF8::MAP_INVALID_UTF8_TO_PUA); + UTF8(charSequence, wideSequence, wxMBConvUTF8::MAP_INVALID_UTF8_TO_OCTAL); +} + +// Use this alternative when 'charSequence' contains a PUA character. Such +// sequences should still roundtrip ok, and this is checked. +// +void MBConvTestCase::UTF8PUA(const char *charSequence, + const wchar_t *wideSequence) +{ + UTF8(charSequence, wideSequence, wxMBConvUTF8::MAP_INVALID_UTF8_NOT); + UTF8(charSequence, NULL, wxMBConvUTF8::MAP_INVALID_UTF8_TO_PUA); + UTF8(charSequence, wideSequence, wxMBConvUTF8::MAP_INVALID_UTF8_TO_OCTAL); +} + +// Use this alternative when 'charSequence' contains an octal escape sequence. +// Such sequences should still roundtrip ok, and this is checked. +// +void MBConvTestCase::UTF8Octal(const char *charSequence, + const wchar_t *wideSequence) +{ + UTF8(charSequence, wideSequence, wxMBConvUTF8::MAP_INVALID_UTF8_NOT); + UTF8(charSequence, wideSequence, wxMBConvUTF8::MAP_INVALID_UTF8_TO_PUA); + UTF8(charSequence, NULL, wxMBConvUTF8::MAP_INVALID_UTF8_TO_OCTAL); +} + +// in case wcscpy is missing +// +static wchar_t *wx_wcscpy(wchar_t *dest, const wchar_t *src) +{ + wchar_t *d = dest; + while ((*d++ = *src++) != 0) + ; + return dest; +} + +// in case wcscat is missing +// +static wchar_t *wx_wcscat(wchar_t *dest, const wchar_t *src) +{ + wchar_t *d = dest; + while (*d) + d++; + while ((*d++ = *src++) != 0) + ; + return dest; +} + +// in case wcscmp is missing +// +static int wx_wcscmp(const wchar_t *s1, const wchar_t *s2) +{ + while (*s1 == *s2 && *s1 != 0) + { + s1++; + s2++; + } + return *s1 - *s2; +} + +// in case wcslen is missing +// +static size_t wx_wcslen(const wchar_t *s) +{ + const wchar_t *t = s; + while (*t != 0) + t++; + return t - s; +} + +// include the option in the error messages so it's possible to see which +// test failed +#define UTF8ASSERT(expr) CPPUNIT_ASSERT_MESSAGE(#expr + errmsg, expr) + +// The test implementation +// +void MBConvTestCase::UTF8(const char *charSequence, + const wchar_t *wideSequence, + int option) +{ + const size_t BUFSIZE = 128; + wxASSERT(strlen(charSequence) * 3 + 10 < BUFSIZE); + char bytes[BUFSIZE]; + + // include the option in the error messages so it's possible to see + // which test failed + sprintf(bytes, " (with option == %d)", option); + std::string errmsg(bytes); + + // put the charSequence at the start, middle and end of a string + strcpy(bytes, charSequence); + strcat(bytes, "ABC"); + strcat(bytes, charSequence); + strcat(bytes, "XYZ"); + strcat(bytes, charSequence); + + // translate it into wide characters + wxMBConvUTF8 utf8(option); + wchar_t widechars[BUFSIZE]; + size_t lenResult = utf8.MB2WC(NULL, bytes, 0); + size_t result = utf8.MB2WC(widechars, bytes, BUFSIZE); + UTF8ASSERT(result == lenResult); + + // check we got the expected result + if (wideSequence) { + UTF8ASSERT(result != (size_t)-1); + wxASSERT(result < BUFSIZE); + + wchar_t expected[BUFSIZE]; + wx_wcscpy(expected, wideSequence); + wx_wcscat(expected, L"ABC"); + wx_wcscat(expected, wideSequence); + wx_wcscat(expected, L"XYZ"); + wx_wcscat(expected, wideSequence); + + UTF8ASSERT(wx_wcscmp(widechars, expected) == 0); + UTF8ASSERT(wx_wcslen(widechars) == result); + } + else { + // If 'wideSequence' is NULL, then the result is expected to be + // invalid. Normally that is as far as we can go, but if there is an + // option then the conversion should succeed anyway, and it should be + // possible to translate back to the original + if (!option) { + UTF8ASSERT(result == (size_t)-1); + return; + } + else { + UTF8ASSERT(result != (size_t)-1); + } + } + + // translate it back and check we get the original + char bytesAgain[BUFSIZE]; + size_t lenResultAgain = utf8.WC2MB(NULL, widechars, 0); + size_t resultAgain = utf8.WC2MB(bytesAgain, widechars, BUFSIZE); + UTF8ASSERT(resultAgain == lenResultAgain); + UTF8ASSERT(resultAgain != (size_t)-1); + wxASSERT(resultAgain < BUFSIZE); + + UTF8ASSERT(strcmp(bytes, bytesAgain) == 0); + UTF8ASSERT(strlen(bytesAgain) == resultAgain); +} + +#endif // HAVE_WCHAR_H