tests/strings/unicode.cpp

   1 ///////////////////////////////////////////////////////////////////////////////
   2 // Name:        tests/strings/unicode.cpp
   3 // Purpose:     Unicode unit test
   4 // Author:      Vadim Zeitlin, Wlodzimierz ABX Skiba
   5 // Created:     2004-04-28
   6 // RCS-ID:      $Id$
   7 // Copyright:   (c) 2004 Vadim Zeitlin, Wlodzimierz Skiba
   8 ///////////////////////////////////////////////////////////////////////////////
   9
  10 // ----------------------------------------------------------------------------
  11 // headers
  12 // ----------------------------------------------------------------------------
  13
  14 #include "testprec.h"
  15
  16 #ifdef __BORLANDC__
  17     #pragma hdrstop
  18 #endif
  19
  20 #ifndef WX_PRECOMP
  21 #endif // WX_PRECOMP
  22
  23 // ----------------------------------------------------------------------------
  24 // local functions
  25 // ----------------------------------------------------------------------------
  26
  27 #if wxUSE_WCHAR_T && !wxUSE_UNICODE
  28
  29 // in case wcscmp is missing
  30 static int wx_wcscmp(const wchar_t *s1, const wchar_t *s2)
  31 {
  32     while (*s1 == *s2 && *s1 != 0)
  33     {
  34         s1++;
  35         s2++;
  36     }
  37     return *s1 - *s2;
  38 }
  39
  40 #endif // wxUSE_WCHAR_T && !wxUSE_UNICODE
  41
  42 // ----------------------------------------------------------------------------
  43 // test class
  44 // ----------------------------------------------------------------------------
  45
  46 class UnicodeTestCase : public CppUnit::TestCase
  47 {
  48 public:
  49     UnicodeTestCase();
  50
  51 private:
  52     CPPUNIT_TEST_SUITE( UnicodeTestCase );
  53         CPPUNIT_TEST( ToFromAscii );
  54 #if wxUSE_WCHAR_T
  55         CPPUNIT_TEST( ConstructorsWithConversion );
  56         CPPUNIT_TEST( ConversionEmpty );
  57         CPPUNIT_TEST( ConversionWithNULs );
  58         CPPUNIT_TEST( ConversionUTF7 );
  59         CPPUNIT_TEST( ConversionUTF8 );
  60         CPPUNIT_TEST( ConversionUTF16 );
  61         CPPUNIT_TEST( ConversionUTF32 );
  62         CPPUNIT_TEST( IsConvOk );
  63 #endif // wxUSE_WCHAR_T
  64     CPPUNIT_TEST_SUITE_END();
  65
  66     void ToFromAscii();
  67 #if wxUSE_WCHAR_T
  68     void ConstructorsWithConversion();
  69     void ConversionEmpty();
  70     void ConversionWithNULs();
  71     void ConversionUTF7();
  72     void ConversionUTF8();
  73     void ConversionUTF16();
  74     void ConversionUTF32();
  75     void IsConvOk();
  76
  77     // test if converting s using the given encoding gives ws and vice versa
  78     //
  79     // if either of the first 2 arguments is NULL, the conversion is supposed
  80     // to fail
  81     void DoTestConversion(const char *s, const wchar_t *w, wxMBConv& conv);
  82 #endif // wxUSE_WCHAR_T
  83
  84
  85     DECLARE_NO_COPY_CLASS(UnicodeTestCase)
  86 };
  87
  88 // register in the unnamed registry so that these tests are run by default
  89 CPPUNIT_TEST_SUITE_REGISTRATION( UnicodeTestCase );
  90
  91 // also include in it's own registry so that these tests can be run alone
  92 CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( UnicodeTestCase, "Unicode" );
  93
  94 UnicodeTestCase::UnicodeTestCase()
  95 {
  96 }
  97
  98 void UnicodeTestCase::ToFromAscii()
  99 {
 100
 101 #define TEST_TO_FROM_ASCII(txt)                              \
 102     {                                                        \
 103         static const char *msg = txt;                        \
 104         wxString s = wxString::FromAscii(msg);               \
 105         CPPUNIT_ASSERT( strcmp( s.ToAscii() , msg ) == 0 );  \
 106     }
 107
 108     TEST_TO_FROM_ASCII( "Hello, world!" );
 109     TEST_TO_FROM_ASCII( "additional \" special \t test \\ component \n :-)" );
 110 }
 111
 112 #if wxUSE_WCHAR_T
 113 void UnicodeTestCase::ConstructorsWithConversion()
 114 {
 115     // the string "Déjà" in UTF-8 and wchar_t:
 116     const unsigned char utf8Buf[] = {0x44,0xC3,0xA9,0x6A,0xC3,0xA0,0};
 117     const wchar_t wchar[] = {0x44,0xE9,0x6A,0xE0,0};
 118     const unsigned char utf8subBuf[] = {0x44,0xC3,0xA9,0x6A,0}; // just "Déj"
 119     const char *utf8 = (char *)utf8Buf;
 120     const char *utf8sub = (char *)utf8subBuf;
 121
 122     wxString s1(utf8, wxConvUTF8);
 123     wxString s2(wchar, wxConvUTF8);
 124
 125 #if wxUSE_UNICODE
 126     CPPUNIT_ASSERT( s1 == wchar );
 127     CPPUNIT_ASSERT( s2 == wchar );
 128 #else
 129     CPPUNIT_ASSERT( s1 == utf8 );
 130     CPPUNIT_ASSERT( s2 == utf8 );
 131 #endif
 132
 133     wxString sub(utf8sub, wxConvUTF8); // "Dej" substring
 134     wxString s3(utf8, wxConvUTF8, 4);
 135     wxString s4(wchar, wxConvUTF8, 3);
 136
 137     CPPUNIT_ASSERT( s3 == sub );
 138     CPPUNIT_ASSERT( s4 == sub );
 139
 140 #if wxUSE_UNICODE
 141     CPPUNIT_ASSERT ( wxString("\t[pl]open.format.Sformatuj dyskietkê=gfloppy %f",
 142                                wxConvUTF8) == wxT("") ); //should stop at pos 35
 143 #endif
 144
 145
 146     // test using Unicode strings together with char* strings (this must work
 147     // in ANSI mode as well, of course):
 148     wxString s5("ascii");
 149     CPPUNIT_ASSERT( s5 == "ascii" );
 150
 151     s5 += " value";
 152
 153     CPPUNIT_ASSERT( strcmp(s5.mb_str(), "ascii value") == 0 );
 154     CPPUNIT_ASSERT( s5 == "ascii value" );
 155     CPPUNIT_ASSERT( s5 != "SomethingElse" );
 156 }
 157
 158 void UnicodeTestCase::ConversionEmpty()
 159 {
 160     size_t len;
 161
 162 #if wxUSE_UNICODE
 163     wxCharBuffer buf = wxConvLibc.cWC2MB(L"", 0, &len);
 164 #else // !wxUSE_UNICODE
 165     wxWCharBuffer wbuf = wxConvLibc.cMB2WC("", 0, &len);
 166 #endif // wxUSE_UNICODE/!wxUSE_UNICODE
 167
 168     CPPUNIT_ASSERT(len == 0);
 169 }
 170
 171 void UnicodeTestCase::ConversionWithNULs()
 172 {
 173 #if wxUSE_UNICODE
 174     static const size_t lenNulString = 10;
 175
 176     wxString szTheString(L"The\0String", wxConvLibc, lenNulString);
 177     wxCharBuffer theBuffer = szTheString.mb_str();
 178
 179     CPPUNIT_ASSERT( memcmp(theBuffer.data(), "The\0String",
 180                     lenNulString + 1) == 0 );
 181
 182     wxString szTheString2("The\0String", wxConvLocal, lenNulString);
 183     CPPUNIT_ASSERT_EQUAL( lenNulString, szTheString2.length() );
 184     CPPUNIT_ASSERT( wxTmemcmp(szTheString2.c_str(), L"The\0String",
 185                     lenNulString + 1) == 0 );
 186 #else // !wxUSE_UNICODE
 187     wxString szTheString(wxT("TheString"));
 188     szTheString.insert(3, 1, '\0');
 189     wxWCharBuffer theBuffer = szTheString.wc_str(wxConvLibc);
 190
 191     CPPUNIT_ASSERT( memcmp(theBuffer.data(), L"The\0String", 11 * sizeof(wchar_t)) == 0 );
 192
 193     wxString szLocalTheString(wxT("TheString"));
 194     szLocalTheString.insert(3, 1, '\0');
 195     wxWCharBuffer theLocalBuffer = szLocalTheString.wc_str(wxConvLocal);
 196
 197     CPPUNIT_ASSERT( memcmp(theLocalBuffer.data(), L"The\0String", 11 * sizeof(wchar_t)) == 0 );
 198 #endif // wxUSE_UNICODE/!wxUSE_UNICODE
 199 }
 200
 201 void
 202 UnicodeTestCase::DoTestConversion(const char *s,
 203                                   const wchar_t *ws,
 204                                   wxMBConv& conv)
 205 {
 206 #if wxUSE_UNICODE
 207     if ( ws )
 208     {
 209         wxCharBuffer buf = conv.cWC2MB(ws, (size_t)-1, NULL);
 210
 211         CPPUNIT_ASSERT( strcmp(buf, s) == 0 );
 212     }
 213 #else // wxUSE_UNICODE
 214     if ( s )
 215     {
 216         wxWCharBuffer wbuf = conv.cMB2WC(s, (size_t)-1, NULL);
 217
 218         if ( ws )
 219         {
 220             CPPUNIT_ASSERT( wbuf.data() );
 221             CPPUNIT_ASSERT( wx_wcscmp(wbuf, ws) == 0 );
 222         }
 223         else // conversion is supposed to fail
 224         {
 225             CPPUNIT_ASSERT_EQUAL( (wchar_t *)NULL, wbuf.data() );
 226         }
 227     }
 228 #endif // wxUSE_UNICODE/!wxUSE_UNICODE
 229 }
 230
 231 struct StringConversionData
 232 {
 233     const char *str;
 234     const wchar_t *wcs;
 235 };
 236
 237 void UnicodeTestCase::ConversionUTF7()
 238 {
 239     static const StringConversionData utf7data[] =
 240     {
 241         { "+-", L"+" },
 242         { "+--", L"+-" },
 243
 244 #ifdef wxHAVE_U_ESCAPE
 245         { "+AKM-", L"\u00a3" },
 246 #endif // wxHAVE_U_ESCAPE
 247
 248         // the following are invalid UTF-7 sequences
 249         { "+", NULL },
 250         { "+~", NULL },
 251         { "a+", NULL },
 252     };
 253
 254     wxCSConv conv(_T("utf-7"));
 255     for ( size_t n = 0; n < WXSIZEOF(utf7data); n++ )
 256     {
 257         const StringConversionData& d = utf7data[n];
 258
 259         // converting to/from UTF-7 using iconv() currently doesn't work
 260         // because of several problems:
 261         //  - GetMBNulLen() doesn't return correct result (iconv converts L'\0'
 262         //    to an incomplete and anyhow nonsensical "+AA" string)
 263         //  - iconv refuses to convert "+-" (although it converts "+-\n" just
 264         //    fine, go figure)
 265         //
 266         // I have no idea how to fix this so just disable the test for now
 267 #if 0
 268         DoTestConversion(d.str, d.wcs, conv);
 269 #endif
 270         DoTestConversion(d.str, d.wcs, wxConvUTF7);
 271     }
 272 }
 273
 274 void UnicodeTestCase::ConversionUTF8()
 275 {
 276     static const StringConversionData utf8data[] =
 277     {
 278 #ifdef wxHAVE_U_ESCAPE
 279         { "\xc2\xa3", L"\u00a3" },
 280 #endif
 281         { "\xc2", NULL },
 282     };
 283
 284     wxCSConv conv(_T("utf-8"));
 285     for ( size_t n = 0; n < WXSIZEOF(utf8data); n++ )
 286     {
 287         const StringConversionData& d = utf8data[n];
 288         DoTestConversion(d.str, d.wcs, conv);
 289         DoTestConversion(d.str, d.wcs, wxConvUTF8);
 290     }
 291 }
 292
 293 void UnicodeTestCase::ConversionUTF16()
 294 {
 295     static const StringConversionData utf16data[] =
 296     {
 297 #ifdef wxHAVE_U_ESCAPE
 298         { "\x04\x1f\x04\x40\x04\x38\x04\x32\x04\x35\x04\x42\0\0",
 299           L"\u041f\u0440\u0438\u0432\u0435\u0442" },
 300         { "\x01\0\0b\x01\0\0a\x01\0\0r\0\0", L"\u0100b\u0100a\u0100r" },
 301 #endif
 302         { "\0f\0o\0o\0\0", L"foo" },
 303     };
 304
 305     wxCSConv conv(wxFONTENCODING_UTF16BE);
 306     for ( size_t n = 0; n < WXSIZEOF(utf16data); n++ )
 307     {
 308         const StringConversionData& d = utf16data[n];
 309         DoTestConversion(d.str, d.wcs, conv);
 310     }
 311
 312     // special case: this string has consecutive NULs inside it which don't
 313     // terminate the string, this exposed a bug in our conversion code which
 314     // got confused in this case
 315     size_t len;
 316     wxWCharBuffer wbuf(conv.cMB2WC("\x01\0\0B\0C" /* A macron BC */, 6, &len));
 317     CPPUNIT_ASSERT_EQUAL( (size_t)3, len );
 318 }
 319
 320 void UnicodeTestCase::ConversionUTF32()
 321 {
 322     static const StringConversionData utf32data[] =
 323     {
 324 #ifdef wxHAVE_U_ESCAPE
 325         {
 326             "\0\0\x04\x1f\0\0\x04\x40\0\0\x04\x38\0\0\x04\x32\0\0\x04\x35\0\0\x04\x42\0\0\0\0",
 327           L"\u041f\u0440\u0438\u0432\u0435\u0442" },
 328 #endif
 329         { "\0\0\0f\0\0\0o\0\0\0o\0\0\0\0", L"foo" },
 330     };
 331
 332     wxCSConv conv(wxFONTENCODING_UTF32BE);
 333     for ( size_t n = 0; n < WXSIZEOF(utf32data); n++ )
 334     {
 335         const StringConversionData& d = utf32data[n];
 336         DoTestConversion(d.str, d.wcs, conv);
 337     }
 338
 339     size_t len;
 340     wxWCharBuffer wbuf(conv.cMB2WC("\0\0\x01\0\0\0\0B\0\0\0C" /* A macron BC */,
 341                                    12, &len));
 342     CPPUNIT_ASSERT_EQUAL( (size_t)3, len );
 343 }
 344
 345 void UnicodeTestCase::IsConvOk()
 346 {
 347     CPPUNIT_ASSERT( wxCSConv(wxFONTENCODING_SYSTEM).IsOk() );
 348     CPPUNIT_ASSERT( wxCSConv(_T("UTF-8")).IsOk() );
 349     CPPUNIT_ASSERT( !wxCSConv(_T("NoSuchConversion")).IsOk() );
 350
 351 #ifdef __WINDOWS__
 352     CPPUNIT_ASSERT( wxCSConv(_T("WINDOWS-437")).IsOk() );
 353 #endif
 354 }
 355
 356 #endif // wxUSE_WCHAR_T
 357