X-Git-Url: https://git.saurik.com/wxWidgets.git/blobdiff_plain/8f1158912fc28ed542d50e7205a3f961b8c764b7..603cfe42185349e119ba4c69bd56a79998babcc4:/tests/mbconv/mbconvtest.cpp diff --git a/tests/mbconv/mbconvtest.cpp b/tests/mbconv/mbconvtest.cpp index 632ee3c8ba..93f3ea020f 100644 --- a/tests/mbconv/mbconvtest.cpp +++ b/tests/mbconv/mbconvtest.cpp @@ -78,7 +78,10 @@ private: CPPUNIT_TEST( CP1252Tests ); // depends on UTF8 Decoder functioning correctly CPPUNIT_TEST( LibcTests ); CPPUNIT_TEST( IconvTests ); + CPPUNIT_TEST( Latin1Tests ); CPPUNIT_TEST( FontmapTests ); + CPPUNIT_TEST( BufSize ); + CPPUNIT_TEST( FromWCharTests ); #ifdef HAVE_WCHAR_H CPPUNIT_TEST( UTF8_41 ); CPPUNIT_TEST( UTF8_7f ); @@ -112,7 +115,10 @@ private: void CP1252Tests(); void LibcTests(); void FontmapTests(); + void BufSize(); + void FromWCharTests(); void IconvTests(); + void Latin1Tests(); // verifies that the specified multibyte sequence decodes to the specified wchar_t sequence void TestDecoder( @@ -120,7 +126,7 @@ private: size_t wideChars, // the number of wide characters at wideBuffer const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" - wxMBConv* converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence + wxMBConv& converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence int sizeofNull // number of bytes occupied by terminating null in this encoding ); @@ -130,7 +136,7 @@ private: size_t wideChars, // the number of wide characters at wideBuffer const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" - wxMBConv* converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence + wxMBConv& converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence int sizeofNull // number of bytes occupied by terminating null in this encoding ); @@ -142,7 +148,7 @@ private: size_t wideChars, // the number of wide characters at wideBuffer const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" - wxMBConv* converter // the wxMBConv object that can decode multiBuffer into a wide character sequence + wxMBConv& converter // the wxMBConv object that can decode multiBuffer into a wide character sequence ); // use wxTextOutputStream to exercise wxMBConv interface @@ -152,7 +158,7 @@ private: size_t wideChars, // the number of wide characters at wideBuffer const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" - wxMBConv* converter // the wxMBConv object that can decode multiBuffer into a wide character sequence + wxMBConv& converter // the wxMBConv object that can decode multiBuffer into a wide character sequence ); #endif @@ -167,7 +173,7 @@ private: size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" const char* utf8Buffer, // the same character sequence as multiBuffer, encoded as UTF-8 size_t utf8Bytes, // the byte length of the UTF-8 encoded character sequence - wxMBConv* converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence + wxMBConv& converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence int sizeofNull // the number of bytes occupied by a terminating null in the converter's encoding ); @@ -211,8 +217,9 @@ private: // register in the unnamed registry so that these tests are run by default CPPUNIT_TEST_SUITE_REGISTRATION( MBConvTestCase ); -// also include in it's own registry so that these tests can be run alone +// also include in its own registry so that these tests can be run alone CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( MBConvTestCase, "MBConvTestCase" ); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( MBConvTestCase, "MBConv" ); void MBConvTestCase::WC2CP1250() { @@ -241,36 +248,38 @@ void MBConvTestCase::WC2CP1250() } } -// print an unsigned character array as a C unsigned character array +// Print an unsigned character array as a C unsigned character array. +// NB: Please don't remove this function even though it's not used anywhere, +// it's very useful when debugging a failed test. wxString CByteArrayFormat( const void* data, size_t len, const wxChar* name ) { const unsigned char* bytes = (unsigned char*)data; wxString result; - result.Printf( _T("const static unsigned char %s[%i] = \n{"), name, (int)len ); + result.Printf( wxT("static const unsigned char %s[%i] = \n{"), name, (int)len ); for ( size_t i = 0; i < len; i++ ) { if ( i != 0 ) { - result.append( _T(",") ); + result.append( wxT(",") ); } if ((i%16)==0) { - result.append( _T("\n ") ); + result.append( wxT("\n ") ); } - wxString byte = wxString::Format( _T("0x%02x"), bytes[i] ); + wxString byte = wxString::Format( wxT("0x%02x"), bytes[i] ); result.append(byte); } - result.append( _T("\n};\n") ); + result.append( wxT("\n};\n") ); return result; } -// The following bytes represent the same string, containing Japanese and English +// The following bytes represent the same string, containing Japanese and English // characters, encoded in several different formats. // encoded by iconv -const static unsigned char welcome_utf7_iconv[84] = +static const unsigned char welcome_utf7_iconv[84] = { 0x57,0x65,0x6c,0x63,0x6f,0x6d,0x65,0x20,0x74,0x6f,0x20,0x6f,0x75,0x72,0x20,0x63, 0x79,0x62,0x65,0x72,0x20,0x73,0x70,0x61,0x63,0x65,0x20,0x66,0x6f,0x72,0x63,0x65, @@ -280,7 +289,7 @@ const static unsigned char welcome_utf7_iconv[84] = 0x57,0x54,0x41,0x43 }; // encoded by wxWindows (iconv can decode this successfully) -const static unsigned char welcome_utf7_wx[109] = +static const unsigned char welcome_utf7_wx[109] = { 0x57,0x65,0x6c,0x63,0x6f,0x6d,0x65,0x2b,0x41,0x43,0x41,0x2d,0x74,0x6f,0x2b,0x41, 0x43,0x41,0x2d,0x6f,0x75,0x72,0x2b,0x41,0x43,0x41,0x2d,0x63,0x79,0x62,0x65,0x72, @@ -291,7 +300,7 @@ const static unsigned char welcome_utf7_wx[109] = 0x51,0x77,0x52,0x44,0x42,0x6e,0x4d,0x46,0x6b,0x77,0x41,0x67,0x2d }; // encoded by iconv -const static unsigned char welcome_utf8[89] = +static const unsigned char welcome_utf8[89] = { 0x57,0x65,0x6c,0x63,0x6f,0x6d,0x65,0x20,0x74,0x6f,0x20,0x6f,0x75,0x72,0x20,0x63, 0x79,0x62,0x65,0x72,0x20,0x73,0x70,0x61,0x63,0x65,0x20,0x66,0x6f,0x72,0x63,0x65, @@ -301,7 +310,7 @@ const static unsigned char welcome_utf8[89] = 0xe3,0x81,0xa7,0xe3,0x81,0x99,0xe3,0x80,0x82 }; // encoded by iconv -const static unsigned char welcome_utf16le[106] = +static const unsigned char welcome_utf16le[106] = { 0x57,0x00,0x65,0x00,0x6c,0x00,0x63,0x00,0x6f,0x00,0x6d,0x00,0x65,0x00,0x20,0x00, 0x74,0x00,0x6f,0x00,0x20,0x00,0x6f,0x00,0x75,0x00,0x72,0x00,0x20,0x00,0x63,0x00, @@ -312,7 +321,7 @@ const static unsigned char welcome_utf16le[106] = 0x44,0x30,0x44,0x30,0x67,0x30,0x59,0x30,0x02,0x30 }; // encoded by iconv -const static unsigned char welcome_utf16be[106] = +static const unsigned char welcome_utf16be[106] = { 0x00,0x57,0x00,0x65,0x00,0x6c,0x00,0x63,0x00,0x6f,0x00,0x6d,0x00,0x65,0x00,0x20, 0x00,0x74,0x00,0x6f,0x00,0x20,0x00,0x6f,0x00,0x75,0x00,0x72,0x00,0x20,0x00,0x63, @@ -323,7 +332,7 @@ const static unsigned char welcome_utf16be[106] = 0x30,0x44,0x30,0x44,0x30,0x67,0x30,0x59,0x30,0x02 }; // encoded by iconv -const static unsigned char welcome_utf32le[212] = +static const unsigned char welcome_utf32le[212] = { 0x57,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x63,0x00,0x00,0x00, 0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x20,0x00,0x00,0x00, @@ -341,7 +350,7 @@ const static unsigned char welcome_utf32le[212] = 0x02,0x30,0x00,0x00 }; // encoded by iconv -const static unsigned char welcome_utf32be[212] = +static const unsigned char welcome_utf32be[212] = { 0x00,0x00,0x00,0x57,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x63, 0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x20, @@ -359,7 +368,7 @@ const static unsigned char welcome_utf32be[212] = 0x00,0x00,0x30,0x02 }; // encoded by iconv -const static unsigned char welcome_cp932[71] = +static const unsigned char welcome_cp932[71] = { 0x57,0x65,0x6c,0x63,0x6f,0x6d,0x65,0x20,0x74,0x6f,0x20,0x6f,0x75,0x72,0x20,0x63, 0x79,0x62,0x65,0x72,0x20,0x73,0x70,0x61,0x63,0x65,0x20,0x66,0x6f,0x72,0x63,0x65, @@ -384,45 +393,51 @@ const static unsigned char welcome_cp932[71] = void MBConvTestCase::UTF7Tests() { +#if 0 + wxCSConv convUTF7(wxFONTENCODING_UTF7); +#else + wxMBConvUTF7 convUTF7; +#endif + TestDecoder - ( - (const wchar_t*)welcome_wchar_t, + ( + (const wchar_t*)welcome_wchar_t, sizeof(welcome_wchar_t)/sizeof(wchar_t), - (const char*)welcome_utf7_iconv, - sizeof(welcome_utf7_iconv), - &wxConvUTF7, + (const char*)welcome_utf7_iconv, + sizeof(welcome_utf7_iconv), + convUTF7, 1 ); TestDecoder - ( - (const wchar_t*)welcome_wchar_t, + ( + (const wchar_t*)welcome_wchar_t, sizeof(welcome_wchar_t)/sizeof(wchar_t), - (const char*)welcome_utf7_wx, - sizeof(welcome_utf7_wx), - &wxConvUTF7, + (const char*)welcome_utf7_wx, + sizeof(welcome_utf7_wx), + convUTF7, 1 ); -#if 0 - // wxWidget's UTF-7 encoder generates different byte sequences than iconv's. +#if 0 + // wxWidget's UTF-7 encoder generates different byte sequences than iconv's. // but both seem to be equally legal. // This test won't work and that's okay. TestEncoder - ( - (const wchar_t*)welcome_wchar_t, + ( + (const wchar_t*)welcome_wchar_t, sizeof(welcome_wchar_t)/sizeof(wchar_t), - (const char*)welcome_utf7_iconv, - sizeof(welcome_utf7_iconv), - &wxConvUTF7, + (const char*)welcome_utf7_iconv, + sizeof(welcome_utf7_iconv), + convUTF7, 1 ); #endif TestEncoder - ( - (const wchar_t*)welcome_wchar_t, + ( + (const wchar_t*)welcome_wchar_t, sizeof(welcome_wchar_t)/sizeof(wchar_t), - (const char*)welcome_utf7_wx, - sizeof(welcome_utf7_wx), - &wxConvUTF7, + (const char*)welcome_utf7_wx, + sizeof(welcome_utf7_wx), + convUTF7, 1 ); } @@ -430,21 +445,21 @@ void MBConvTestCase::UTF7Tests() void MBConvTestCase::UTF8Tests() { TestDecoder - ( - (const wchar_t*)welcome_wchar_t, + ( + (const wchar_t*)welcome_wchar_t, sizeof(welcome_wchar_t)/sizeof(wchar_t), - (const char*)welcome_utf8, - sizeof(welcome_utf8), - &wxConvUTF8, + (const char*)welcome_utf8, + sizeof(welcome_utf8), + wxConvUTF8, 1 ); TestEncoder - ( - (const wchar_t*)welcome_wchar_t, + ( + (const wchar_t*)welcome_wchar_t, sizeof(welcome_wchar_t)/sizeof(wchar_t), - (const char*)welcome_utf8, - sizeof(welcome_utf8), - &wxConvUTF8, + (const char*)welcome_utf8, + sizeof(welcome_utf8), + wxConvUTF8, 1 ); } @@ -453,21 +468,21 @@ void MBConvTestCase::UTF16LETests() { wxMBConvUTF16LE convUTF16LE; TestDecoder - ( - (const wchar_t*)welcome_wchar_t, + ( + (const wchar_t*)welcome_wchar_t, sizeof(welcome_wchar_t)/sizeof(wchar_t), - (const char*)welcome_utf16le, - sizeof(welcome_utf16le), - &convUTF16LE, + (const char*)welcome_utf16le, + sizeof(welcome_utf16le), + convUTF16LE, 2 ); TestEncoder - ( - (const wchar_t*)welcome_wchar_t, + ( + (const wchar_t*)welcome_wchar_t, sizeof(welcome_wchar_t)/sizeof(wchar_t), - (const char*)welcome_utf16le, - sizeof(welcome_utf16le), - &convUTF16LE, + (const char*)welcome_utf16le, + sizeof(welcome_utf16le), + convUTF16LE, 2 ); } @@ -476,21 +491,21 @@ void MBConvTestCase::UTF16BETests() { wxMBConvUTF16BE convUTF16BE; TestDecoder - ( - (const wchar_t*)welcome_wchar_t, + ( + (const wchar_t*)welcome_wchar_t, sizeof(welcome_wchar_t)/sizeof(wchar_t), - (const char*)welcome_utf16be, - sizeof(welcome_utf16be), - &convUTF16BE, + (const char*)welcome_utf16be, + sizeof(welcome_utf16be), + convUTF16BE, 2 ); TestEncoder - ( - (const wchar_t*)welcome_wchar_t, + ( + (const wchar_t*)welcome_wchar_t, sizeof(welcome_wchar_t)/sizeof(wchar_t), - (const char*)welcome_utf16be, - sizeof(welcome_utf16be), - &convUTF16BE, + (const char*)welcome_utf16be, + sizeof(welcome_utf16be), + convUTF16BE, 2 ); } @@ -499,21 +514,21 @@ void MBConvTestCase::UTF32LETests() { wxMBConvUTF32LE convUTF32LE; TestDecoder - ( - (const wchar_t*)welcome_wchar_t, + ( + (const wchar_t*)welcome_wchar_t, sizeof(welcome_wchar_t)/sizeof(wchar_t), - (const char*)welcome_utf32le, - sizeof(welcome_utf32le), - &convUTF32LE, + (const char*)welcome_utf32le, + sizeof(welcome_utf32le), + convUTF32LE, 4 ); TestEncoder - ( - (const wchar_t*)welcome_wchar_t, + ( + (const wchar_t*)welcome_wchar_t, sizeof(welcome_wchar_t)/sizeof(wchar_t), - (const char*)welcome_utf32le, - sizeof(welcome_utf32le), - &convUTF32LE, + (const char*)welcome_utf32le, + sizeof(welcome_utf32le), + convUTF32LE, 4 ); } @@ -522,21 +537,21 @@ void MBConvTestCase::UTF32BETests() { wxMBConvUTF32BE convUTF32BE; TestDecoder - ( - (const wchar_t*)welcome_wchar_t, + ( + (const wchar_t*)welcome_wchar_t, sizeof(welcome_wchar_t)/sizeof(wchar_t), - (const char*)welcome_utf32be, - sizeof(welcome_utf32be), - &convUTF32BE, + (const char*)welcome_utf32be, + sizeof(welcome_utf32be), + convUTF32BE, 4 ); TestEncoder - ( - (const wchar_t*)welcome_wchar_t, + ( + (const wchar_t*)welcome_wchar_t, sizeof(welcome_wchar_t)/sizeof(wchar_t), - (const char*)welcome_utf32be, - sizeof(welcome_utf32be), - &convUTF32BE, + (const char*)welcome_utf32be, + sizeof(welcome_utf32be), + convUTF32BE, 4 ); } @@ -545,27 +560,27 @@ void MBConvTestCase::CP932Tests() { wxCSConv convCP932( wxFONTENCODING_CP932 ); TestDecoder - ( - (const wchar_t*)welcome_wchar_t, + ( + (const wchar_t*)welcome_wchar_t, sizeof(welcome_wchar_t)/sizeof(wchar_t), - (const char*)welcome_cp932, - sizeof(welcome_cp932), - &convCP932, + (const char*)welcome_cp932, + sizeof(welcome_cp932), + convCP932, 1 ); TestEncoder - ( - (const wchar_t*)welcome_wchar_t, + ( + (const wchar_t*)welcome_wchar_t, sizeof(welcome_wchar_t)/sizeof(wchar_t), - (const char*)welcome_cp932, - sizeof(welcome_cp932), - &convCP932, + (const char*)welcome_cp932, + sizeof(welcome_cp932), + convCP932, 1 ); } // a character sequence encoded as iso8859-1 (iconv) -static const unsigned char iso8859_1[251] = +static const unsigned char iso8859_1[251] = { 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14, 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24, @@ -585,7 +600,7 @@ static const unsigned char iso8859_1[251] = 0xf5,0xf6,0xf7,0xf8,0xf9,0xfa,0xfb,0xfc,0xfd,0xfe,0xff }; // the above character sequence encoded as UTF-8 (iconv) -static const unsigned char iso8859_1_utf8[379] = +static const unsigned char iso8859_1_utf8[379] = { 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14, 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24, @@ -614,7 +629,7 @@ static const unsigned char iso8859_1_utf8[379] = }; // a character sequence encoded as CP1252 (iconv) -static const unsigned char CP1252[246] = +static const unsigned char CP1252[246] = { 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14, 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24, @@ -634,7 +649,7 @@ static const unsigned char CP1252[246] = 0x85,0x89,0x8b,0x9b,0x80,0x99 }; // the above character sequence encoded as UTF-8 (iconv) -static const unsigned char CP1252_utf8[386] = +static const unsigned char CP1252_utf8[386] = { 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14, 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24, @@ -663,8 +678,11 @@ static const unsigned char CP1252_utf8[386] = 0x84,0xa2 }; +// this is unused currently so avoid warnings about this +#if 0 + // a character sequence encoded as iso8859-5 (iconv) -static const unsigned char iso8859_5[251] = +static const unsigned char iso8859_5[251] = { 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14, 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24, @@ -684,7 +702,7 @@ static const unsigned char iso8859_5[251] = 0xf5,0xf6,0xf7,0xf8,0xf9,0xfa,0xfb,0xfc,0xfe,0xff,0xf0 }; // the above character sequence encoded as UTF-8 (iconv) -static const unsigned char iso8859_5_utf8[380] = +static const unsigned char iso8859_5_utf8[380] = { 0x05,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14, 0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x22,0x23,0x24, @@ -711,6 +729,7 @@ static const unsigned char iso8859_5_utf8[380] = 0x92,0xd1,0x93,0xd1,0x94,0xd1,0x95,0xd1,0x96,0xd1,0x97,0xd1,0x98,0xd1,0x99,0xd1, 0x9a,0xd1,0x9b,0xd1,0x9c,0xd1,0x9e,0xd1,0x9f,0xe2,0x84,0x96 }; +#endif // 0 // DecodeUTF8 // decodes the specified *unterminated* UTF-8 byte array @@ -740,7 +759,7 @@ void MBConvTestCase::TestCoder( size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" const char* utf8Buffer, // the same character sequence as multiBuffer, encoded as UTF-8 size_t utf8Bytes, // the byte length of the UTF-8 encoded character sequence - wxMBConv* converter, // the wxMBConv object thta can decode multiBuffer into a wide character sequence + wxMBConv& converter, // the wxMBConv object thta can decode multiBuffer into a wide character sequence int sizeofNull // the number of bytes occupied by a terminating null in the converter's encoding ) { @@ -749,24 +768,23 @@ void MBConvTestCase::TestCoder( // so we should store the wide character version as UTF-8 and depend on // the UTF-8 converter's ability to decode it to platform specific wide characters // this test is invalid if the UTF-8 converter can't decode - wxWCharBuffer wideBuffer((size_t)0); - wideBuffer = DecodeUTF8( utf8Buffer, utf8Bytes ); - size_t wideChars = wxWcslen( wideBuffer.data() ); + const wxWCharBuffer wideBuffer(DecodeUTF8(utf8Buffer, utf8Bytes)); + const size_t wideChars = wxWcslen(wideBuffer); TestDecoder - ( - wideBuffer.data(), + ( + wideBuffer.data(), wideChars, - (const char*)multiBuffer, + multiBuffer, multiBytes, converter, sizeofNull ); TestEncoder - ( - wideBuffer.data(), + ( + wideBuffer.data(), wideChars, - (const char*)multiBuffer, + multiBuffer, multiBytes, converter, sizeofNull @@ -774,12 +792,12 @@ void MBConvTestCase::TestCoder( } -WXDLLIMPEXP_BASE wxMBConv* new_wxMBConv_wxwin( const wxChar* name ); +WXDLLIMPEXP_BASE wxMBConv* new_wxMBConv_wxwin( const char* name ); void MBConvTestCase::FontmapTests() { #ifdef wxUSE_FONTMAP - wxMBConv* converter = new_wxMBConv_wxwin( _T("CP1252") ); + wxMBConv* converter = new_wxMBConv_wxwin("CP1252"); if ( !converter ) { return; @@ -789,20 +807,134 @@ void MBConvTestCase::FontmapTests() sizeof(CP1252), (const char*)CP1252_utf8, sizeof(CP1252_utf8), - converter, + *converter, 1 ); delete converter; #endif } +void MBConvTestCase::BufSize() +{ + wxCSConv conv1251(wxT("CP1251")); + CPPUNIT_ASSERT( conv1251.IsOk() ); + const char *cp1251text = + "\313\301\326\305\324\323\321 \325\304\301\336\316\331\315"; + + const size_t lenW = conv1251.MB2WC(NULL, cp1251text, 0); + CPPUNIT_ASSERT_EQUAL( strlen(cp1251text), lenW ); + wxWCharBuffer wbuf(lenW + 1); // allocates lenW + 2 characters + wbuf.data()[lenW + 1] = L'!'; + + // lenW is not enough because it's the length and we need the size + CPPUNIT_ASSERT_EQUAL( + wxCONV_FAILED, conv1251.MB2WC(wbuf.data(), cp1251text, lenW) ); + + // lenW+1 is just fine + CPPUNIT_ASSERT( + conv1251.MB2WC(wbuf.data(), cp1251text, lenW + 1) != wxCONV_FAILED ); + + // of course, greater values work too + CPPUNIT_ASSERT( + conv1251.MB2WC(wbuf.data(), cp1251text, lenW + 2) != wxCONV_FAILED ); + + // but they shouldn't write more stuff to the buffer + CPPUNIT_ASSERT_EQUAL( L'!', wbuf[lenW + 1] ); + + + // test in the other direction too, using an encoding with multibyte NUL + wxCSConv convUTF16(wxT("UTF-16LE")); + CPPUNIT_ASSERT( convUTF16.IsOk() ); + const wchar_t *utf16text = L"Hello"; + + const size_t lenMB = convUTF16.WC2MB(NULL, utf16text, 0); + CPPUNIT_ASSERT_EQUAL( wcslen(utf16text)*2, lenMB ); + wxCharBuffer buf(lenMB + 2); // it only adds 1 for NUL on its own, we need 2 + // for NUL and an extra one for the guard byte + buf.data()[lenMB + 2] = '?'; + + CPPUNIT_ASSERT_EQUAL( + wxCONV_FAILED, convUTF16.WC2MB(buf.data(), utf16text, lenMB) ); + CPPUNIT_ASSERT_EQUAL( + wxCONV_FAILED, convUTF16.WC2MB(buf.data(), utf16text, lenMB + 1) ); + CPPUNIT_ASSERT( + convUTF16.WC2MB(buf.data(), utf16text, lenMB + 2) != wxCONV_FAILED ); + CPPUNIT_ASSERT( + convUTF16.WC2MB(buf.data(), utf16text, lenMB + 3) != wxCONV_FAILED ); + CPPUNIT_ASSERT_EQUAL( '?', buf[lenMB + 2] ); +} -WXDLLIMPEXP_BASE wxMBConv* new_wxMBConv_iconv( const wxChar* name ); +void MBConvTestCase::FromWCharTests() +{ + wxCSConv conv950("CP950"); + char mbuf[10]; + // U+4e00 is 2 bytes (0xa4 0x40) in cp950 + wchar_t wbuf[] = { 0x4e00, 0, 0x4e00, 0 }; + + // test simple ASCII text + memset(mbuf, '!', sizeof(mbuf)); + CPPUNIT_ASSERT_EQUAL( wxCONV_FAILED, conv950.FromWChar(mbuf, 0, L"a", 1)); + CPPUNIT_ASSERT_EQUAL( '!', mbuf[0]); + + memset(mbuf, '!', sizeof(mbuf)); + CPPUNIT_ASSERT_EQUAL( 1, conv950.FromWChar(mbuf, 1, L"a", 1)); + CPPUNIT_ASSERT_EQUAL( 'a', mbuf[0]); + CPPUNIT_ASSERT_EQUAL( '!', mbuf[1]); + + memset(mbuf, '!', sizeof(mbuf)); + CPPUNIT_ASSERT_EQUAL( wxCONV_FAILED, conv950.FromWChar(mbuf, 1, L"a", 2)); + + memset(mbuf, '!', sizeof(mbuf)); + CPPUNIT_ASSERT_EQUAL( 2, conv950.FromWChar(mbuf, 2, L"a", 2)); + CPPUNIT_ASSERT_EQUAL( 'a', mbuf[0]); + CPPUNIT_ASSERT_EQUAL( '\0', mbuf[1]); + CPPUNIT_ASSERT_EQUAL( '!', mbuf[2]); + + // test non-ASCII text, 1 wchar -> 2 char + memset(mbuf, '!', sizeof(mbuf)); + CPPUNIT_ASSERT_EQUAL( wxCONV_FAILED, conv950.FromWChar(mbuf, 0, wbuf, 1)); + + memset(mbuf, '!', sizeof(mbuf)); + CPPUNIT_ASSERT_EQUAL( wxCONV_FAILED, conv950.FromWChar(mbuf, 1, wbuf, 1)); + + memset(mbuf, '!', sizeof(mbuf)); + CPPUNIT_ASSERT_EQUAL( 2, conv950.FromWChar(mbuf, 2, wbuf, 1)); + CPPUNIT_ASSERT_EQUAL( '!', mbuf[2]); + + memset(mbuf, '!', sizeof(mbuf)); + CPPUNIT_ASSERT_EQUAL( wxCONV_FAILED, conv950.FromWChar(mbuf, 2, wbuf, 2)); + + memset(mbuf, '!', sizeof(mbuf)); + CPPUNIT_ASSERT_EQUAL( 3, conv950.FromWChar(mbuf, 3, wbuf, 2)); + CPPUNIT_ASSERT_EQUAL( '\0', mbuf[2]); + CPPUNIT_ASSERT_EQUAL( '!', mbuf[3]); + + // test text with embedded NUL-character and srcLen specified + memset(mbuf, '!', sizeof(mbuf)); + CPPUNIT_ASSERT_EQUAL( wxCONV_FAILED, conv950.FromWChar(mbuf, 3, wbuf, 3)); + + memset(mbuf, '!', sizeof(mbuf)); + CPPUNIT_ASSERT_EQUAL( wxCONV_FAILED, conv950.FromWChar(mbuf, 4, wbuf, 3)); + CPPUNIT_ASSERT_EQUAL( 5, conv950.FromWChar(mbuf, 5, wbuf, 3)); + CPPUNIT_ASSERT_EQUAL( '\0', mbuf[2]); + CPPUNIT_ASSERT_EQUAL( '!', mbuf[5]); + + memset(mbuf, '!', sizeof(mbuf)); + CPPUNIT_ASSERT_EQUAL( wxCONV_FAILED, conv950.FromWChar(mbuf, 5, wbuf, 4)); + + memset(mbuf, '!', sizeof(mbuf)); + CPPUNIT_ASSERT_EQUAL( 6, conv950.FromWChar(mbuf, 6, wbuf, 4)); + CPPUNIT_ASSERT_EQUAL( '\0', mbuf[2]); + CPPUNIT_ASSERT_EQUAL( '\0', mbuf[5]); + CPPUNIT_ASSERT_EQUAL( '!', mbuf[6]); +} + +WXDLLIMPEXP_BASE wxMBConv* new_wxMBConv_iconv( const char* name ); void MBConvTestCase::IconvTests() { #ifdef HAVE_ICONV - wxMBConv* converter = new_wxMBConv_iconv( _T("CP932") ); + wxMBConv* converter = new_wxMBConv_iconv("CP932"); if ( !converter ) { return; @@ -812,13 +944,31 @@ void MBConvTestCase::IconvTests() sizeof(welcome_cp932), (const char*)welcome_utf8, sizeof(welcome_utf8), - converter, + *converter, 1 ); delete converter; #endif } +void MBConvTestCase::Latin1Tests() +{ + TestCoder( + (const char*)iso8859_1, + sizeof(iso8859_1), + (const char*)iso8859_1_utf8, + sizeof(iso8859_1_utf8), + wxConvISO8859_1, + 1 + ); + + static const char nulstr[] = "foo\0bar\0"; + static const size_t mbLen = WXSIZEOF(nulstr) - 1; + size_t wcLen; + wxConvISO8859_1.cMB2WC(nulstr, mbLen, &wcLen); + CPPUNIT_ASSERT_EQUAL( mbLen, wcLen ); +} + void MBConvTestCase::CP1252Tests() { wxCSConv convCP1252( wxFONTENCODING_CP1252 ); @@ -827,42 +977,30 @@ void MBConvTestCase::CP1252Tests() sizeof(CP1252), (const char*)CP1252_utf8, sizeof(CP1252_utf8), - &convCP1252, + convCP1252, 1 ); } void MBConvTestCase::LibcTests() { - // There isn't a locale that all systems support (except "C"), so leave - // this one disabled for non-Windows systems for the moment, until - // a solution can be found. -#ifdef __WXMSW__ - -#ifdef __WXMSW__ - setlocale( LC_ALL, "English_United States.1252" ); - const unsigned char* systemMB = CP1252; - size_t systemMB_size = sizeof(CP1252); - const unsigned char* systemMB_utf8 = CP1252_utf8; - size_t systemMB_utf8_size = sizeof(CP1252_utf8); -#else - setlocale( LC_ALL, "en_US.iso8859-1" ); - const unsigned char* systemMB = iso8859_1; - size_t systemMB_size = sizeof(iso8859_1); - const unsigned char* systemMB_utf8 = iso8859_1_utf8; - size_t systemMB_utf8_size = sizeof(iso8859_1_utf8); -#endif + // The locale name are OS-dependent so this test is done only under Windows + // when using MSVC (surprisingly it fails with MinGW, even though it's + // supposed to use the same CRT -- no idea why and unfortunately gdb is too + // flaky to debug it) +#ifdef __VISUALC__ + LocaleSetter loc("English_United States.1252"); + wxMBConvLibc convLibc; TestCoder( - (const char*)systemMB, - systemMB_size, - (const char*)systemMB_utf8, - systemMB_utf8_size, - &convLibc, + (const char*)CP1252, + sizeof(CP1252), + (const char*)CP1252_utf8, + sizeof(CP1252_utf8), + convLibc, 1 ); - -#endif // __WXMSW__ +#endif // __VISUALC__ } // verifies that the specified mb sequences decode to the specified wc sequence @@ -871,7 +1009,7 @@ void MBConvTestCase::TestDecoder( size_t wideChars, // the number of wide characters at wideBuffer const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" - wxMBConv* converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence + wxMBConv& converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence int sizeofNull // number of bytes occupied by terminating null in this encoding ) { @@ -883,14 +1021,19 @@ void MBConvTestCase::TestDecoder( memset( &inputCopy.data()[multiBytes], 0, sizeofNull ); // calculate the output size - size_t outputWritten = converter->MB2WC - ( - 0, - (const char*)inputCopy.data(), + size_t outputWritten = converter.MB2WC + ( + 0, + (const char*)inputCopy.data(), 0 ); // make sure the correct output length was calculated - CPPUNIT_ASSERT( outputWritten == wideChars ); + WX_ASSERT_EQUAL_MESSAGE + ( + ("while converting \"%s\"", multiBuffer), + wideChars, + outputWritten + ); // convert the string size_t guardChars = 8; // to make sure we're not overrunning the output buffer @@ -899,14 +1042,14 @@ void MBConvTestCase::TestDecoder( wxWCharBuffer outputBuffer(outputBufferChars); memset( outputBuffer.data(), UNINITIALIZED, outputBufferChars*sizeof(wchar_t) ); - outputWritten = converter->MB2WC - ( - outputBuffer.data(), - (const char*)inputCopy.data(), + outputWritten = converter.MB2WC + ( + outputBuffer.data(), + (const char*)inputCopy.data(), outputBufferChars ); // make sure the correct number of characters were outputs - CPPUNIT_ASSERT( outputWritten == wideChars ); + CPPUNIT_ASSERT_EQUAL( wideChars, outputWritten ); // make sure the characters generated are correct CPPUNIT_ASSERT( 0 == memcmp( outputBuffer, wideBuffer, wideChars*sizeof(wchar_t) ) ); @@ -931,7 +1074,7 @@ void MBConvTestCase::TestEncoder( size_t wideChars, // the number of wide characters at wideBuffer const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" - wxMBConv* converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence + wxMBConv& converter, // the wxMBConv object that can decode multiBuffer into a wide character sequence int sizeofNull // number of bytes occupied by terminating null in this encoding ) { @@ -943,14 +1086,14 @@ void MBConvTestCase::TestEncoder( inputCopy.data()[wideChars] = 0; // calculate the output size - size_t outputWritten = converter->WC2MB - ( - 0, - (const wchar_t*)inputCopy.data(), + size_t outputWritten = converter.WC2MB + ( + 0, + (const wchar_t*)inputCopy.data(), 0 ); // make sure the correct output length was calculated - CPPUNIT_ASSERT( outputWritten == multiBytes ); + CPPUNIT_ASSERT_EQUAL( multiBytes, outputWritten ); // convert the string size_t guardBytes = 8; // to make sure we're not overrunning the output buffer @@ -958,27 +1101,29 @@ void MBConvTestCase::TestEncoder( wxCharBuffer outputBuffer(outputBufferSize); memset( outputBuffer.data(), UNINITIALIZED, outputBufferSize ); - outputWritten = converter->WC2MB - ( - outputBuffer.data(), - (const wchar_t*)inputCopy.data(), - outputBufferSize + outputWritten = converter.WC2MB + ( + outputBuffer.data(), + (const wchar_t*)inputCopy.data(), + outputBufferSize ); // make sure the correct number of characters were output - CPPUNIT_ASSERT( outputWritten == multiBytes ); + CPPUNIT_ASSERT_EQUAL( multiBytes, outputWritten ); // make sure the characters generated are correct CPPUNIT_ASSERT( 0 == memcmp( outputBuffer, multiBuffer, multiBytes ) ); + size_t i; + // the output buffer should be null terminated - for ( size_t i = multiBytes; i < multiBytes + sizeofNull; i++ ) + for ( i = multiBytes; i < multiBytes + sizeofNull; i++ ) { CPPUNIT_ASSERT( ((unsigned char*)outputBuffer.data())[i] == 0 ); } // make sure the rest of the output buffer is untouched - for ( size_t i = multiBytes + sizeofNull; i < outputBufferSize; i++ ) + for ( i = multiBytes + sizeofNull; i < outputBufferSize; i++ ) { CPPUNIT_ASSERT( ((unsigned char*)outputBuffer.data())[i] == UNINITIALIZED ); } @@ -996,18 +1141,22 @@ void MBConvTestCase::TestStreamDecoder( size_t wideChars, // the number of wide characters at wideBuffer const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" - wxMBConv* converter // the wxMBConv object that can decode multiBuffer into a wide character sequence + wxMBConv& converter // the wxMBConv object that can decode multiBuffer into a wide character sequence ) { // this isn't meant to test wxMemoryInputStream or wxTextInputStream // it's meant to test the way wxTextInputStream uses wxMBConv // (which has exposed some problems with wxMBConv) wxMemoryInputStream memoryInputStream( multiBuffer, multiBytes ); - wxTextInputStream textInputStream( memoryInputStream, wxT(""), *converter ); + wxTextInputStream textInputStream( memoryInputStream, wxT(""), converter ); for ( size_t i = 0; i < wideChars; i++ ) { wxChar wc = textInputStream.GetChar(); - CPPUNIT_ASSERT( wc == wideBuffer[i] ); + CPPUNIT_ASSERT_EQUAL_MESSAGE( + std::string(wxString::Format("At position %lu", (unsigned long)i)), + wideBuffer[i], + wc + ); } CPPUNIT_ASSERT( 0 == textInputStream.GetChar() ); CPPUNIT_ASSERT( memoryInputStream.Eof() ); @@ -1022,7 +1171,7 @@ void MBConvTestCase::TestStreamEncoder( size_t wideChars, // the number of wide characters at wideBuffer const char* multiBuffer, // a multibyte encoded character sequence that can be decoded by "converter" size_t multiBytes, // the byte length of the multibyte character sequence that can be decoded by "converter" - wxMBConv* converter // the wxMBConv object that can decode multiBuffer into a wide character sequence + wxMBConv& converter // the wxMBConv object that can decode multiBuffer into a wide character sequence ) { // this isn't meant to test wxMemoryOutputStream or wxTextOutputStream @@ -1030,15 +1179,18 @@ void MBConvTestCase::TestStreamEncoder( // (which has exposed some problems with wxMBConv) wxMemoryOutputStream memoryOutputStream; // wxEOL_UNIX will pass \n \r unchanged - wxTextOutputStream textOutputStream( memoryOutputStream, wxEOL_UNIX, *converter ); + wxTextOutputStream textOutputStream( memoryOutputStream, wxEOL_UNIX, converter ); for ( size_t i = 0; i < wideChars; i++ ) { textOutputStream.PutChar( wideBuffer[i] ); } - CPPUNIT_ASSERT( memoryOutputStream.TellO() == multiBytes ); + + textOutputStream.Flush(); + + CPPUNIT_ASSERT_EQUAL( multiBytes, size_t(memoryOutputStream.TellO()) ); wxCharBuffer copy( memoryOutputStream.TellO() ); memoryOutputStream.CopyTo( copy.data(), memoryOutputStream.TellO()); - CPPUNIT_ASSERT( 0 == memcmp( copy.data(), multiBuffer, multiBytes ) ); + CPPUNIT_ASSERT_EQUAL( 0, memcmp( copy.data(), multiBuffer, multiBytes ) ); } #endif @@ -1088,7 +1240,7 @@ void MBConvTestCase::UTF8Octal(const char *charSequence, // static wchar_t *wx_wcscpy(wchar_t *dest, const wchar_t *src) { - wchar_t *d = dest; + wchar_t *d = dest; while ((*d++ = *src++) != 0) ; return dest; @@ -1098,7 +1250,7 @@ static wchar_t *wx_wcscpy(wchar_t *dest, const wchar_t *src) // static wchar_t *wx_wcscat(wchar_t *dest, const wchar_t *src) { - wchar_t *d = dest; + wchar_t *d = dest; while (*d) d++; while ((*d++ = *src++) != 0) @@ -1106,6 +1258,28 @@ static wchar_t *wx_wcscat(wchar_t *dest, const wchar_t *src) return dest; } +// in case wcscmp is missing +// +static int wx_wcscmp(const wchar_t *s1, const wchar_t *s2) +{ + while (*s1 == *s2 && *s1 != 0) + { + s1++; + s2++; + } + return *s1 - *s2; +} + +// in case wcslen is missing +// +static size_t wx_wcslen(const wchar_t *s) +{ + const wchar_t *t = s; + while (*t != 0) + t++; + return t - s; +} + // include the option in the error messages so it's possible to see which // test failed #define UTF8ASSERT(expr) CPPUNIT_ASSERT_MESSAGE(#expr + errmsg, expr) @@ -1119,12 +1293,12 @@ void MBConvTestCase::UTF8(const char *charSequence, const size_t BUFSIZE = 128; wxASSERT(strlen(charSequence) * 3 + 10 < BUFSIZE); char bytes[BUFSIZE]; - + // include the option in the error messages so it's possible to see // which test failed sprintf(bytes, " (with option == %d)", option); std::string errmsg(bytes); - + // put the charSequence at the start, middle and end of a string strcpy(bytes, charSequence); strcat(bytes, "ABC"); @@ -1151,8 +1325,8 @@ void MBConvTestCase::UTF8(const char *charSequence, wx_wcscat(expected, L"XYZ"); wx_wcscat(expected, wideSequence); - UTF8ASSERT(wcscmp(widechars, expected) == 0); - UTF8ASSERT(wcslen(widechars) == result); + UTF8ASSERT(wx_wcscmp(widechars, expected) == 0); + UTF8ASSERT(wx_wcslen(widechars) == result); } else { // If 'wideSequence' is NULL, then the result is expected to be