tests/strings/unicode.cpp

   1 ///////////////////////////////////////////////////////////////////////////////
   2 // Name:        tests/strings/unicode.cpp
   3 // Purpose:     Unicode unit test
   4 // Author:      Vadim Zeitlin, Wlodzimierz ABX Skiba
   5 // Created:     2004-04-28
   6 // RCS-ID:      $Id$
   7 // Copyright:   (c) 2004 Vadim Zeitlin, Wlodzimierz Skiba
   8 ///////////////////////////////////////////////////////////////////////////////
   9
  10 // ----------------------------------------------------------------------------
  11 // headers
  12 // ----------------------------------------------------------------------------
  13
  14 #include "testprec.h"
  15
  16 #ifdef __BORLANDC__
  17     #pragma hdrstop
  18 #endif
  19
  20 #ifndef WX_PRECOMP
  21     #include "wx/wx.h"
  22 #endif // WX_PRECOMP
  23
  24 // ----------------------------------------------------------------------------
  25 // local functions
  26 // ----------------------------------------------------------------------------
  27
  28 #if wxUSE_WCHAR_T && !wxUSE_UNICODE
  29
  30 // in case wcscmp is missing
  31 static int wx_wcscmp(const wchar_t *s1, const wchar_t *s2)
  32 {
  33     while (*s1 == *s2 && *s1 != 0)
  34     {
  35         s1++;
  36         s2++;
  37     }
  38     return *s1 - *s2;
  39 }
  40
  41 #endif // wxUSE_WCHAR_T && !wxUSE_UNICODE
  42
  43 // ----------------------------------------------------------------------------
  44 // test class
  45 // ----------------------------------------------------------------------------
  46
  47 class UnicodeTestCase : public CppUnit::TestCase
  48 {
  49 public:
  50     UnicodeTestCase();
  51
  52 private:
  53     CPPUNIT_TEST_SUITE( UnicodeTestCase );
  54         CPPUNIT_TEST( ToFromAscii );
  55 #if wxUSE_WCHAR_T
  56         CPPUNIT_TEST( ConstructorsWithConversion );
  57         CPPUNIT_TEST( ConversionEmpty );
  58         CPPUNIT_TEST( ConversionWithNULs );
  59         CPPUNIT_TEST( ConversionUTF7 );
  60         CPPUNIT_TEST( ConversionUTF8 );
  61         CPPUNIT_TEST( ConversionUTF16 );
  62         CPPUNIT_TEST( ConversionUTF32 );
  63         CPPUNIT_TEST( IsConvOk );
  64 #endif // wxUSE_WCHAR_T
  65 #if wxUSE_UNICODE
  66         CPPUNIT_TEST( Iteration );
  67 #endif
  68     CPPUNIT_TEST_SUITE_END();
  69
  70     void ToFromAscii();
  71 #if wxUSE_WCHAR_T
  72     void ConstructorsWithConversion();
  73     void ConversionEmpty();
  74     void ConversionWithNULs();
  75     void ConversionUTF7();
  76     void ConversionUTF8();
  77     void ConversionUTF16();
  78     void ConversionUTF32();
  79     void IsConvOk();
  80 #if wxUSE_UNICODE
  81     void Iteration();
  82 #endif
  83
  84     // test if converting s using the given encoding gives ws and vice versa
  85     //
  86     // if either of the first 2 arguments is NULL, the conversion is supposed
  87     // to fail
  88     void DoTestConversion(const char *s, const wchar_t *w, wxMBConv& conv);
  89 #endif // wxUSE_WCHAR_T
  90
  91
  92     DECLARE_NO_COPY_CLASS(UnicodeTestCase)
  93 };
  94
  95 // register in the unnamed registry so that these tests are run by default
  96 CPPUNIT_TEST_SUITE_REGISTRATION( UnicodeTestCase );
  97
  98 // also include in it's own registry so that these tests can be run alone
  99 CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( UnicodeTestCase, "UnicodeTestCase" );
 100
 101 UnicodeTestCase::UnicodeTestCase()
 102 {
 103 }
 104
 105 void UnicodeTestCase::ToFromAscii()
 106 {
 107
 108 #define TEST_TO_FROM_ASCII(txt)                              \
 109     {                                                        \
 110         static const char *msg = txt;                        \
 111         wxString s = wxString::FromAscii(msg);               \
 112         CPPUNIT_ASSERT( strcmp( s.ToAscii() , msg ) == 0 );  \
 113     }
 114
 115     TEST_TO_FROM_ASCII( "Hello, world!" );
 116     TEST_TO_FROM_ASCII( "additional \" special \t test \\ component \n :-)" );
 117 }
 118
 119 #if wxUSE_WCHAR_T
 120 void UnicodeTestCase::ConstructorsWithConversion()
 121 {
 122     // the string "Déjà" in UTF-8 and wchar_t:
 123     const unsigned char utf8Buf[] = {0x44,0xC3,0xA9,0x6A,0xC3,0xA0,0};
 124     const wchar_t wchar[] = {0x44,0xE9,0x6A,0xE0,0};
 125     const unsigned char utf8subBuf[] = {0x44,0xC3,0xA9,0x6A,0}; // just "Déj"
 126     const char *utf8 = (char *)utf8Buf;
 127     const char *utf8sub = (char *)utf8subBuf;
 128
 129     wxString s1(utf8, wxConvUTF8);
 130     wxString s2(wchar, wxConvUTF8);
 131
 132 #if wxUSE_UNICODE
 133     WX_ASSERT_STR_EQUAL( wchar, s1 );
 134     WX_ASSERT_STR_EQUAL( wchar, s2 );
 135 #else
 136     WX_ASSERT_STR_EQUAL( utf8, s1 );
 137     WX_ASSERT_STR_EQUAL( utf8, s2 );
 138 #endif
 139
 140     wxString sub(utf8sub, wxConvUTF8); // "Dej" substring
 141     wxString s3(utf8, wxConvUTF8, 4);
 142     wxString s4(wchar, wxConvUTF8, 3);
 143
 144     CPPUNIT_ASSERT_EQUAL( sub, s3 );
 145     CPPUNIT_ASSERT_EQUAL( sub, s4 );
 146
 147 #if wxUSE_UNICODE
 148     // conversion should stop with failure at pos 35
 149     wxString s("\t[pl]open.format.Sformatuj dyskietkê=gfloppy %f", wxConvUTF8);
 150     CPPUNIT_ASSERT( s.empty() );
 151 #endif
 152
 153
 154     // test using Unicode strings together with char* strings (this must work
 155     // in ANSI mode as well, of course):
 156     wxString s5("ascii");
 157     WX_ASSERT_STR_EQUAL( "ascii", s5 );
 158
 159     s5 += " value";
 160
 161     CPPUNIT_ASSERT( strcmp(s5.mb_str(), "ascii value") == 0 );
 162     WX_ASSERT_STR_EQUAL( "ascii value", s5 );
 163     CPPUNIT_ASSERT( s5 != "SomethingElse" );
 164 }
 165
 166 void UnicodeTestCase::ConversionEmpty()
 167 {
 168     size_t len;
 169
 170 #if wxUSE_UNICODE
 171     wxCharBuffer buf = wxConvLibc.cWC2MB(L"", 0, &len);
 172 #else // !wxUSE_UNICODE
 173     wxWCharBuffer wbuf = wxConvLibc.cMB2WC("", 0, &len);
 174 #endif // wxUSE_UNICODE/!wxUSE_UNICODE
 175
 176     CPPUNIT_ASSERT(len == 0);
 177 }
 178
 179 void UnicodeTestCase::ConversionWithNULs()
 180 {
 181 #if wxUSE_UNICODE
 182     static const size_t lenNulString = 10;
 183
 184     wxString szTheString(L"The\0String", wxConvLibc, lenNulString);
 185     wxCharBuffer theBuffer = szTheString.mb_str();
 186
 187     CPPUNIT_ASSERT( memcmp(theBuffer.data(), "The\0String",
 188                     lenNulString + 1) == 0 );
 189
 190     wxString szTheString2("The\0String", wxConvLocal, lenNulString);
 191     CPPUNIT_ASSERT_EQUAL( lenNulString, szTheString2.length() );
 192     CPPUNIT_ASSERT( wxTmemcmp(szTheString2.c_str(), L"The\0String",
 193                     lenNulString + 1) == 0 );
 194 #else // !wxUSE_UNICODE
 195     wxString szTheString(wxT("TheString"));
 196     szTheString.insert(3, 1, '\0');
 197     wxWCharBuffer theBuffer = szTheString.wc_str(wxConvLibc);
 198
 199     CPPUNIT_ASSERT( memcmp(theBuffer.data(), L"The\0String", 11 * sizeof(wchar_t)) == 0 );
 200
 201     wxString szLocalTheString(wxT("TheString"));
 202     szLocalTheString.insert(3, 1, '\0');
 203     wxWCharBuffer theLocalBuffer = szLocalTheString.wc_str(wxConvLocal);
 204
 205     CPPUNIT_ASSERT( memcmp(theLocalBuffer.data(), L"The\0String", 11 * sizeof(wchar_t)) == 0 );
 206 #endif // wxUSE_UNICODE/!wxUSE_UNICODE
 207 }
 208
 209 void
 210 UnicodeTestCase::DoTestConversion(const char *s,
 211                                   const wchar_t *ws,
 212                                   wxMBConv& conv)
 213 {
 214 #if wxUSE_UNICODE
 215     if ( ws )
 216     {
 217         wxCharBuffer buf = conv.cWC2MB(ws, (size_t)-1, NULL);
 218
 219         CPPUNIT_ASSERT( strcmp(buf, s) == 0 );
 220     }
 221 #else // wxUSE_UNICODE
 222     if ( s )
 223     {
 224         wxWCharBuffer wbuf = conv.cMB2WC(s, (size_t)-1, NULL);
 225
 226         if ( ws )
 227         {
 228             CPPUNIT_ASSERT( wbuf.data() );
 229             CPPUNIT_ASSERT( wx_wcscmp(wbuf, ws) == 0 );
 230         }
 231         else // conversion is supposed to fail
 232         {
 233             CPPUNIT_ASSERT_EQUAL( (wchar_t *)NULL, wbuf.data() );
 234         }
 235     }
 236 #endif // wxUSE_UNICODE/!wxUSE_UNICODE
 237 }
 238
 239 struct StringConversionData
 240 {
 241     const char *str;
 242     const wchar_t *wcs;
 243 };
 244
 245 void UnicodeTestCase::ConversionUTF7()
 246 {
 247     static const StringConversionData utf7data[] =
 248     {
 249         { "+-", L"+" },
 250         { "+--", L"+-" },
 251
 252 #ifdef wxHAVE_U_ESCAPE
 253         { "+AKM-", L"\u00a3" },
 254 #endif // wxHAVE_U_ESCAPE
 255
 256         // the following are invalid UTF-7 sequences
 257         { "+", NULL },
 258         { "+~", NULL },
 259         { "a+", NULL },
 260     };
 261
 262     wxCSConv conv(_T("utf-7"));
 263     for ( size_t n = 0; n < WXSIZEOF(utf7data); n++ )
 264     {
 265         const StringConversionData& d = utf7data[n];
 266
 267         // converting to/from UTF-7 using iconv() currently doesn't work
 268         // because of several problems:
 269         //  - GetMBNulLen() doesn't return correct result (iconv converts L'\0'
 270         //    to an incomplete and anyhow nonsensical "+AA" string)
 271         //  - iconv refuses to convert "+-" (although it converts "+-\n" just
 272         //    fine, go figure)
 273         //
 274         // I have no idea how to fix this so just disable the test for now
 275 #if 0
 276         DoTestConversion(d.str, d.wcs, conv);
 277 #endif
 278         DoTestConversion(d.str, d.wcs, wxConvUTF7);
 279     }
 280 }
 281
 282 void UnicodeTestCase::ConversionUTF8()
 283 {
 284     static const StringConversionData utf8data[] =
 285     {
 286 #ifdef wxHAVE_U_ESCAPE
 287         { "\xc2\xa3", L"\u00a3" },
 288 #endif
 289         { "\xc2", NULL },
 290     };
 291
 292     wxCSConv conv(_T("utf-8"));
 293     for ( size_t n = 0; n < WXSIZEOF(utf8data); n++ )
 294     {
 295         const StringConversionData& d = utf8data[n];
 296         DoTestConversion(d.str, d.wcs, conv);
 297         DoTestConversion(d.str, d.wcs, wxConvUTF8);
 298     }
 299 }
 300
 301 void UnicodeTestCase::ConversionUTF16()
 302 {
 303     static const StringConversionData utf16data[] =
 304     {
 305 #ifdef wxHAVE_U_ESCAPE
 306         { "\x04\x1f\x04\x40\x04\x38\x04\x32\x04\x35\x04\x42\0\0",
 307           L"\u041f\u0440\u0438\u0432\u0435\u0442" },
 308         { "\x01\0\0b\x01\0\0a\x01\0\0r\0\0", L"\u0100b\u0100a\u0100r" },
 309 #endif
 310         { "\0f\0o\0o\0\0", L"foo" },
 311     };
 312
 313     wxCSConv conv(wxFONTENCODING_UTF16BE);
 314     for ( size_t n = 0; n < WXSIZEOF(utf16data); n++ )
 315     {
 316         const StringConversionData& d = utf16data[n];
 317         DoTestConversion(d.str, d.wcs, conv);
 318     }
 319
 320     // special case: this string has consecutive NULs inside it which don't
 321     // terminate the string, this exposed a bug in our conversion code which
 322     // got confused in this case
 323     size_t len;
 324     wxWCharBuffer wbuf(conv.cMB2WC("\x01\0\0B\0C" /* A macron BC */, 6, &len));
 325     CPPUNIT_ASSERT_EQUAL( (size_t)3, len );
 326 }
 327
 328 void UnicodeTestCase::ConversionUTF32()
 329 {
 330     static const StringConversionData utf32data[] =
 331     {
 332 #ifdef wxHAVE_U_ESCAPE
 333         {
 334             "\0\0\x04\x1f\0\0\x04\x40\0\0\x04\x38\0\0\x04\x32\0\0\x04\x35\0\0\x04\x42\0\0\0\0",
 335           L"\u041f\u0440\u0438\u0432\u0435\u0442" },
 336 #endif
 337         { "\0\0\0f\0\0\0o\0\0\0o\0\0\0\0", L"foo" },
 338     };
 339
 340     wxCSConv conv(wxFONTENCODING_UTF32BE);
 341     for ( size_t n = 0; n < WXSIZEOF(utf32data); n++ )
 342     {
 343         const StringConversionData& d = utf32data[n];
 344         DoTestConversion(d.str, d.wcs, conv);
 345     }
 346
 347     size_t len;
 348     wxWCharBuffer wbuf(conv.cMB2WC("\0\0\x01\0\0\0\0B\0\0\0C" /* A macron BC */,
 349                                    12, &len));
 350     CPPUNIT_ASSERT_EQUAL( (size_t)3, len );
 351 }
 352
 353 void UnicodeTestCase::IsConvOk()
 354 {
 355     CPPUNIT_ASSERT( wxCSConv(wxFONTENCODING_SYSTEM).IsOk() );
 356     CPPUNIT_ASSERT( wxCSConv(_T("UTF-8")).IsOk() );
 357     CPPUNIT_ASSERT( !wxCSConv(_T("NoSuchConversion")).IsOk() );
 358
 359 #ifdef __WINDOWS__
 360     CPPUNIT_ASSERT( wxCSConv(_T("WINDOWS-437")).IsOk() );
 361 #endif
 362 }
 363
 364 #endif // wxUSE_WCHAR_T
 365
 366 #if wxUSE_UNICODE
 367 void UnicodeTestCase::Iteration()
 368 {
 369     // "czech" in Czech ("cestina"):
 370     static const char *textUTF8 = "\304\215e\305\241tina";
 371     static const wchar_t textUTF16[] = {0x10D, 0x65, 0x161, 0x74, 0x69, 0x6E, 0x61, 0};
 372
 373     wxString text(wxString::FromUTF8(textUTF8));
 374     CPPUNIT_ASSERT( wxStrcmp(text.wc_str(), textUTF16) == 0 );
 375
 376     // verify the string was decoded correctly:
 377     {
 378         size_t idx = 0;
 379         for ( wxString::const_iterator i = text.begin(); i != text.end(); ++i, ++idx )
 380         {
 381             CPPUNIT_ASSERT( *i == textUTF16[idx] );
 382         }
 383     }
 384
 385     // overwrite the string with something that is shorter in UTF-8:
 386     {
 387         for ( wxString::iterator i = text.begin(); i != text.end(); ++i )
 388             *i = 'x';
 389     }
 390
 391     // restore the original text now:
 392     {
 393         wxString::iterator end1 = text.end();
 394         wxString::const_iterator end2 = text.end();
 395
 396         size_t idx = 0;
 397         for ( wxString::iterator i = text.begin(); i != text.end(); ++i, ++idx )
 398         {
 399             *i = textUTF16[idx];
 400
 401             CPPUNIT_ASSERT( end1 == text.end() );
 402             CPPUNIT_ASSERT( end2 == text.end() );
 403         }
 404
 405         CPPUNIT_ASSERT( end1 == text.end() );
 406         CPPUNIT_ASSERT( end2 == text.end() );
 407     }
 408
 409     // and verify it again:
 410     {
 411         size_t idx = 0;
 412         for ( wxString::const_iterator i = text.begin(); i != text.end(); ++i, ++idx )
 413         {
 414             CPPUNIT_ASSERT( *i == textUTF16[idx] );
 415         }
 416     }
 417 }
 418 #endif // wxUSE_UNICODE