X-Git-Url: https://git.saurik.com/wxWidgets.git/blobdiff_plain/e1bfe89eee413a45848f7db54121aeb35ebc51d5..d4d8988c72e26ea5ef68b87a53691da7c3751a95:/src/common/strconv.cpp diff --git a/src/common/strconv.cpp b/src/common/strconv.cpp index 2e3c0c8eaa..7b0e78a69d 100644 --- a/src/common/strconv.cpp +++ b/src/common/strconv.cpp @@ -1,5 +1,5 @@ ///////////////////////////////////////////////////////////////////////////// -// Name: strconv.cpp +// Name: src/common/strconv.cpp // Purpose: Unicode conversion classes // Author: Ove Kaaven, Robert Roebling, Vadim Zeitlin, Vaclav Slavik, // Ryan Norton, Fredrik Roubert (UTF7) @@ -20,10 +20,6 @@ // headers // ---------------------------------------------------------------------------- -#if defined(__GNUG__) && !defined(NO_GCC_PRAGMA) - #pragma implementation "strconv.h" -#endif - // For compilers that support precompilation, includes "wx.h". #include "wx/wxprec.h" @@ -40,11 +36,8 @@ #if wxUSE_WCHAR_T -#ifdef __WXMSW__ - #include "wx/msw/private.h" -#endif - #ifdef __WINDOWS__ + #include "wx/msw/private.h" #include "wx/msw/missing.h" #endif @@ -60,10 +53,6 @@ #define wxHAVE_WIN32_MB2WC #endif // __WIN32__ but !__WXMICROWIN__ -// ---------------------------------------------------------------------------- -// headers -// ---------------------------------------------------------------------------- - #ifdef __SALFORDC__ #include #endif @@ -78,39 +67,19 @@ #include "wx/utils.h" #ifdef __WXMAC__ +#ifndef __DARWIN__ #include #include #include +#endif #include "wx/mac/private.h" // includes mac headers #endif -// ---------------------------------------------------------------------------- -// macros -// ---------------------------------------------------------------------------- -#define BSWAP_UCS4(str, len) { unsigned _c; for (_c=0; _c=0xdfff)) + else if ((input[1]<0xdc00) || (input[1]>0xdfff)) { output = *input; return (size_t)-1; @@ -270,15 +239,15 @@ const wxWCharBuffer wxMBConv::cMB2WC(const char *szString, size_t nStringLen, si } //Increment to next (sub)string - //Note that we have to use strlen here instead of nLen - //here because XX2XX gives us the size of the output buffer, - //not neccessarly the length of the string + //Note that we have to use strlen instead of nLen here + //because XX2XX gives us the size of the output buffer, + //which is not necessarily the length of the string szPos += strlen(szPos) + 1; } //success - return actual length and the buffer *pOutSize = nActualLength; - return theBuffer; + return theBuffer; } const wxCharBuffer wxMBConv::cWC2MB(const wchar_t *szString, size_t nStringLen, size_t* pOutSize) const @@ -313,7 +282,7 @@ const wxCharBuffer wxMBConv::cWC2MB(const wchar_t *szString, size_t nStringLen, //Increase the actual length (+1 for current null character) nActualLength += nLen + 1; - + //if buffer too big, realloc the buffer if (nActualLength > (nCurrentSize+1)) { @@ -332,15 +301,15 @@ const wxCharBuffer wxMBConv::cWC2MB(const wchar_t *szString, size_t nStringLen, } //Increment to next (sub)string - //Note that we have to use wxWcslen here instead of nLen - //here because XX2XX gives us the size of the output buffer, - //not neccessarly the length of the string + //Note that we have to use wxWcslen instead of nLen here + //because XX2XX gives us the size of the output buffer, + //which is not necessarily the length of the string szPos += wxWcslen(szPos) + 1; } //success - return actual length and the buffer *pOutSize = nActualLength; - return theBuffer; + return theBuffer; } // ---------------------------------------------------------------------------- @@ -357,42 +326,41 @@ size_t wxMBConvLibc::WC2MB(char *buf, const wchar_t *psz, size_t n) const return wxWC2MB(buf, psz, n); } +#ifdef __UNIX__ + // ---------------------------------------------------------------------------- -// wxConvBrokenFileNames is made for GTK2 in Unicode mode when -// files are accidentally written in an encoding which is not -// the system encoding. Typically, the system encoding will be -// UTF8 but there might be files stored in ISO8859-1 in disk. +// wxConvBrokenFileNames // ---------------------------------------------------------------------------- -class wxConvBrokenFileNames: public wxMBConvLibc +wxConvBrokenFileNames::wxConvBrokenFileNames(const wxChar *charset) { -public: - virtual size_t MB2WC(wchar_t *outputBuf, const char *psz, size_t outputSize) const; - virtual size_t WC2MB(char *outputBuf, const wchar_t *psz, size_t outputSize) const; -}; + if ( !charset || wxStricmp(charset, _T("UTF-8")) == 0 + || wxStricmp(charset, _T("UTF8")) == 0 ) + m_conv = new wxMBConvUTF8(wxMBConvUTF8::MAP_INVALID_UTF8_TO_OCTAL); + else + m_conv = new wxCSConv(charset); +} -size_t wxConvBrokenFileNames::MB2WC(wchar_t *outputBuf, const char *psz, size_t outputSize) const +size_t +wxConvBrokenFileNames::MB2WC(wchar_t *outputBuf, + const char *psz, + size_t outputSize) const { -#if 0 - if (we find some invalid characters) - { - Convert to Unicode range. - } - else -#endif - return wxMBConvLibc::MB2WC( outputBuf, psz, outputSize ); + return m_conv->MB2WC( outputBuf, psz, outputSize ); } -size_t wxConvBrokenFileNames::WC2MB(char *outputBuf, const wchar_t *psz, size_t outputSize) const +size_t +wxConvBrokenFileNames::WC2MB(char *outputBuf, + const wchar_t *psz, + size_t outputSize) const { -#if 0 - Convert back from Unicode range. -#endif - return wxMBConvLibc::WC2MB( outputBuf, psz, outputSize ); + return m_conv->WC2MB( outputBuf, psz, outputSize ); } +#endif + // ---------------------------------------------------------------------------- -// UTF-7 +// UTF-7 // ---------------------------------------------------------------------------- // Implementation (C) 2004 Fredrik Roubert @@ -545,7 +513,7 @@ size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t *psz, size_t n) const } #ifndef WC_UTF16 else if (((wxUint32)cc) > 0xffff) - { + { // no surrogate pair generation (yet?) return (size_t)-1; } @@ -559,7 +527,7 @@ size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t *psz, size_t n) const { // BASE64 encode string unsigned int lsb, d, l; - for (d = 0, l = 0;; psz++) + for (d = 0, l = 0; /*nothing*/; psz++) { for (lsb = 0; lsb < 2; lsb ++) { @@ -602,12 +570,19 @@ size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t *psz, size_t n) const static wxUint32 utf8_max[]= { 0x7f, 0x7ff, 0xffff, 0x1fffff, 0x3ffffff, 0x7fffffff, 0xffffffff }; +// boundaries of the private use area we use to (temporarily) remap invalid +// characters invalid in a UTF-8 encoded string +const wxUint32 wxUnicodePUA = 0x100000; +const wxUint32 wxUnicodePUAEnd = wxUnicodePUA + 256; + size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const { size_t len = 0; while (*psz && ((!buf) || (len < n))) { + const char *opsz = psz; + bool invalid = false; unsigned char cc = *psz++, fc = cc; unsigned cnt; for (cnt = 0; fc & 0x80; cnt++) @@ -618,6 +593,15 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const if (buf) *buf++ = cc; len++; + + // escape the escape character for octal escapes + if ((m_options & MAP_INVALID_UTF8_TO_OCTAL) + && cc == '\\' && (!buf || len < n)) + { + if (buf) + *buf++ = cc; + len++; + } } else { @@ -625,7 +609,7 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const if (!cnt) { // invalid UTF-8 sequence - return (size_t)-1; + invalid = true; } else { @@ -633,32 +617,93 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const wxUint32 res = cc & (0x3f >> cnt); while (cnt--) { - cc = *psz++; + cc = *psz; if ((cc & 0xC0) != 0x80) { // invalid UTF-8 sequence - return (size_t)-1; + invalid = true; + break; } + psz++; res = (res << 6) | (cc & 0x3f); } - if (res <= utf8_max[ocnt]) + if (invalid || res <= utf8_max[ocnt]) { // illegal UTF-8 encoding - return (size_t)-1; + invalid = true; + } + else if ((m_options & MAP_INVALID_UTF8_TO_PUA) && + res >= wxUnicodePUA && res < wxUnicodePUAEnd) + { + // if one of our PUA characters turns up externally + // it must also be treated as an illegal sequence + // (a bit like you have to escape an escape character) + invalid = true; } + else + { #ifdef WC_UTF16 - // cast is ok because wchar_t == wxUuint16 if WC_UTF16 - size_t pa = encode_utf16(res, (wxUint16 *)buf); - if (pa == (size_t)-1) - return (size_t)-1; - if (buf) - buf += pa; - len += pa; + // cast is ok because wchar_t == wxUuint16 if WC_UTF16 + size_t pa = encode_utf16(res, (wxUint16 *)buf); + if (pa == (size_t)-1) + { + invalid = true; + } + else + { + if (buf) + buf += pa; + len += pa; + } #else // !WC_UTF16 - if (buf) - *buf++ = res; - len++; + if (buf) + *buf++ = (wchar_t)res; + len++; #endif // WC_UTF16/!WC_UTF16 + } + } + if (invalid) + { + if (m_options & MAP_INVALID_UTF8_TO_PUA) + { + while (opsz < psz && (!buf || len < n)) + { +#ifdef WC_UTF16 + // cast is ok because wchar_t == wxUuint16 if WC_UTF16 + size_t pa = encode_utf16((unsigned char)*opsz + wxUnicodePUA, (wxUint16 *)buf); + wxASSERT(pa != (size_t)-1); + if (buf) + buf += pa; + opsz++; + len += pa; +#else + if (buf) + *buf++ = (wchar_t)(wxUnicodePUA + (unsigned char)*opsz); + opsz++; + len++; +#endif + } + } + else if (m_options & MAP_INVALID_UTF8_TO_OCTAL) + { + while (opsz < psz && (!buf || len < n)) + { + if ( buf && len + 3 < n ) + { + unsigned char on = *opsz; + *buf++ = L'\\'; + *buf++ = (wchar_t)( L'0' + on / 0100 ); + *buf++ = (wchar_t)( L'0' + (on % 0100) / 010 ); + *buf++ = (wchar_t)( L'0' + on % 010 ); + } + opsz++; + len += 4; + } + } + else // MAP_INVALID_UTF8_NOT + { + return (size_t)-1; + } } } } @@ -667,6 +712,11 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const return len; } +static inline bool isoctal(wchar_t wch) +{ + return L'0' <= wch && wch <= L'7'; +} + size_t wxMBConvUTF8::WC2MB(char *buf, const wchar_t *psz, size_t n) const { size_t len = 0; @@ -681,36 +731,67 @@ size_t wxMBConvUTF8::WC2MB(char *buf, const wchar_t *psz, size_t n) const #else cc=(*psz++) & 0x7fffffff; #endif - unsigned cnt; - for (cnt = 0; cc > utf8_max[cnt]; cnt++) {} - if (!cnt) + + if ( (m_options & MAP_INVALID_UTF8_TO_PUA) + && cc >= wxUnicodePUA && cc < wxUnicodePUAEnd ) { - // plain ASCII char if (buf) - *buf++ = (char) cc; + *buf++ = (char)(cc - wxUnicodePUA); len++; } + else if ( (m_options & MAP_INVALID_UTF8_TO_OCTAL) + && cc == L'\\' && psz[0] == L'\\' ) + { + if (buf) + *buf++ = (char)cc; + psz++; + len++; + } + else if ( (m_options & MAP_INVALID_UTF8_TO_OCTAL) && + cc == L'\\' && + isoctal(psz[0]) && isoctal(psz[1]) && isoctal(psz[2]) ) + { + if (buf) + { + *buf++ = (char) ((psz[0] - L'0')*0100 + + (psz[1] - L'0')*010 + + (psz[2] - L'0')); + } + psz += 3; + len++; + } else { - len += cnt + 1; - if (buf) + unsigned cnt; + for (cnt = 0; cc > utf8_max[cnt]; cnt++) {} + if (!cnt) { - *buf++ = (char) ((-128 >> cnt) | ((cc >> (cnt * 6)) & (0x3f >> cnt))); - while (cnt--) - *buf++ = (char) (0x80 | ((cc >> (cnt * 6)) & 0x3f)); + // plain ASCII char + if (buf) + *buf++ = (char) cc; + len++; + } + + else + { + len += cnt + 1; + if (buf) + { + *buf++ = (char) ((-128 >> cnt) | ((cc >> (cnt * 6)) & (0x3f >> cnt))); + while (cnt--) + *buf++ = (char) (0x80 | ((cc >> (cnt * 6)) & 0x3f)); + } } } } - if (buf && (lenIsOk() ) + { + delete result; + return 0; + } + return result; +} + +wxString wxMBConv_iconv::ms_wcCharsetName; bool wxMBConv_iconv::ms_wcNeedsSwap = false; wxMBConv_iconv::wxMBConv_iconv(const wxChar *name) { - // Do it the hard way - char cname[100]; - for (size_t i = 0; i < wxStrlen(name)+1; i++) - cname[i] = (char) name[i]; + // iconv operates with chars, not wxChars, but luckily it uses only ASCII + // names for the charsets + const wxCharBuffer cname(wxString(name).ToAscii()); // check for charset that represents wchar_t: - if (ms_wcCharsetName == NULL) + if ( ms_wcCharsetName.empty() ) { - ms_wcNeedsSwap = false; + wxLogTrace(TRACE_STRCONV, _T("Looking for wide char codeset:")); - // try charset with explicit bytesex info (e.g. "UCS-4LE"): - ms_wcCharsetName = WC_NAME_BEST; - m2w = iconv_open(ms_wcCharsetName, cname); +#if wxUSE_FONTMAP + const wxChar **names = wxFontMapperBase::GetAllEncodingNames(WC_ENC); +#else // !wxUSE_FONTMAP + static const wxChar *names[] = + { +#if SIZEOF_WCHAR_T == 4 + _T("UCS-4"), +#elif SIZEOF_WCHAR_T = 2 + _T("UCS-2"), +#endif + NULL + }; +#endif // wxUSE_FONTMAP/!wxUSE_FONTMAP - if (m2w == (iconv_t)-1) + for ( ; *names && ms_wcCharsetName.empty(); ++names ) { - // try charset w/o bytesex info (e.g. "UCS4") - // and check for bytesex ourselves: - ms_wcCharsetName = WC_NAME; - m2w = iconv_open(ms_wcCharsetName, cname); + const wxString nameCS(*names); - // last bet, try if it knows WCHAR_T pseudo-charset - if (m2w == (iconv_t)-1) - { - ms_wcCharsetName = "WCHAR_T"; - m2w = iconv_open(ms_wcCharsetName, cname); - } + // first try charset with explicit bytesex info (e.g. "UCS-4LE"): + wxString nameXE(nameCS); + #ifdef WORDS_BIGENDIAN + nameXE += _T("BE"); + #else // little endian + nameXE += _T("LE"); + #endif - if (m2w != (iconv_t)-1) + wxLogTrace(TRACE_STRCONV, _T(" trying charset \"%s\""), + nameXE.c_str()); + + m2w = iconv_open(nameXE.ToAscii(), cname); + if ( m2w == ICONV_T_INVALID ) { - char buf[2], *bufPtr; - wchar_t wbuf[2], *wbufPtr; - size_t insz, outsz; - size_t res; - - buf[0] = 'A'; - buf[1] = 0; - wbuf[0] = 0; - insz = 2; - outsz = SIZEOF_WCHAR_T * 2; - wbufPtr = wbuf; - bufPtr = buf; - - res = iconv(m2w, ICONV_CHAR_CAST(&bufPtr), &insz, - (char**)&wbufPtr, &outsz); - - if (ICONV_FAILED(res, insz)) - { - ms_wcCharsetName = NULL; - wxLogLastError(wxT("iconv")); - wxLogError(_("Conversion to charset '%s' doesn't work."), name); - } - else + // try charset w/o bytesex info (e.g. "UCS4") + wxLogTrace(TRACE_STRCONV, _T(" trying charset \"%s\""), + nameCS.c_str()); + m2w = iconv_open(nameCS.ToAscii(), cname); + + // and check for bytesex ourselves: + if ( m2w != ICONV_T_INVALID ) { - ms_wcNeedsSwap = wbuf[0] != (wchar_t)buf[0]; + char buf[2], *bufPtr; + wchar_t wbuf[2], *wbufPtr; + size_t insz, outsz; + size_t res; + + buf[0] = 'A'; + buf[1] = 0; + wbuf[0] = 0; + insz = 2; + outsz = SIZEOF_WCHAR_T * 2; + wbufPtr = wbuf; + bufPtr = buf; + + res = iconv(m2w, ICONV_CHAR_CAST(&bufPtr), &insz, + (char**)&wbufPtr, &outsz); + + if (ICONV_FAILED(res, insz)) + { + wxLogLastError(wxT("iconv")); + wxLogError(_("Conversion to charset '%s' doesn't work."), + nameCS.c_str()); + } + else // ok, can convert to this encoding, remember it + { + ms_wcCharsetName = nameCS; + ms_wcNeedsSwap = wbuf[0] != (wchar_t)buf[0]; + } } } - else + else // use charset not requiring byte swapping { - ms_wcCharsetName = NULL; - - // VS: we must not output an error here, since wxWidgets will safely - // fall back to using wxEncodingConverter. - wxLogTrace(wxT("strconv"), wxT("Impossible to convert to/from charset '%s' with iconv, falling back to wxEncodingConverter."), name); - //wxLogError( + ms_wcCharsetName = nameXE; } } - wxLogTrace(wxT("strconv"), wxT("wchar_t charset is '%s', needs swap: %i"), ms_wcCharsetName, ms_wcNeedsSwap); + + wxLogTrace(TRACE_STRCONV, + wxT("iconv wchar_t charset is \"%s\"%s"), + ms_wcCharsetName.empty() ? _T("") + : ms_wcCharsetName.c_str(), + ms_wcNeedsSwap ? _T(" (needs swap)") + : _T("")); } else // we already have ms_wcCharsetName { - m2w = iconv_open(ms_wcCharsetName, cname); + m2w = iconv_open(ms_wcCharsetName.ToAscii(), cname); } - // NB: don't ever pass NULL to iconv_open(), it may crash! - if ( ms_wcCharsetName ) + if ( ms_wcCharsetName.empty() ) { - w2m = iconv_open( cname, ms_wcCharsetName); + w2m = ICONV_T_INVALID; } else { - w2m = (iconv_t)-1; + w2m = iconv_open(cname, ms_wcCharsetName.ToAscii()); + if ( w2m == ICONV_T_INVALID ) + { + wxLogTrace(TRACE_STRCONV, + wxT("\"%s\" -> \"%s\" works but not the converse!?"), + ms_wcCharsetName.c_str(), cname.data()); + } } } wxMBConv_iconv::~wxMBConv_iconv() { - if ( m2w != (iconv_t)-1 ) + if ( m2w != ICONV_T_INVALID ) iconv_close(m2w); - if ( w2m != (iconv_t)-1 ) + if ( w2m != ICONV_T_INVALID ) iconv_close(w2m); } @@ -1344,7 +1477,7 @@ size_t wxMBConv_iconv::MB2WC(wchar_t *buf, const char *psz, size_t n) const // as MB<->WC conversion would fail "randomly". wxMutexLocker lock(wxConstCast(this, wxMBConv_iconv)->m_iconvMutex); #endif - + size_t inbuf = strlen(psz); size_t outbuf = n * SIZEOF_WCHAR_T; size_t res, cres; @@ -1363,7 +1496,8 @@ size_t wxMBConv_iconv::MB2WC(wchar_t *buf, const char *psz, size_t n) const if (ms_wcNeedsSwap) { // convert to native endianness - WC_BSWAP(buf /* _not_ bufPtr */, res) + for ( unsigned i = 0; i < res; i++ ) + buf[n] = WC_BSWAP(buf[i]); } // NB: iconv was given only strlen(psz) characters on input, and so @@ -1393,7 +1527,7 @@ size_t wxMBConv_iconv::MB2WC(wchar_t *buf, const char *psz, size_t n) const if (ICONV_FAILED(cres, inbuf)) { //VS: it is ok if iconv fails, hence trace only - wxLogTrace(wxT("strconv"), wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode())); + wxLogTrace(TRACE_STRCONV, wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode())); return (size_t)-1; } @@ -1406,8 +1540,9 @@ size_t wxMBConv_iconv::WC2MB(char *buf, const wchar_t *psz, size_t n) const // NB: explained in MB2WC wxMutexLocker lock(wxConstCast(this, wxMBConv_iconv)->m_iconvMutex); #endif - - size_t inbuf = wxWcslen(psz) * SIZEOF_WCHAR_T; + + size_t inlen = wxWcslen(psz); + size_t inbuf = inlen * SIZEOF_WCHAR_T; size_t outbuf = n; size_t res, cres; @@ -1416,13 +1551,13 @@ size_t wxMBConv_iconv::WC2MB(char *buf, const wchar_t *psz, size_t n) const if (ms_wcNeedsSwap) { // need to copy to temp buffer to switch endianness - // this absolutely doesn't rock! - // (no, doing WC_BSWAP twice on the original buffer won't help, as it + // (doing WC_BSWAP twice on the original buffer won't help, as it // could be in read-only memory, or be accessed in some other thread) - tmpbuf=(wchar_t*)malloc((inbuf+1)*SIZEOF_WCHAR_T); - memcpy(tmpbuf,psz,(inbuf+1)*SIZEOF_WCHAR_T); - WC_BSWAP(tmpbuf, inbuf) - psz=tmpbuf; + tmpbuf = (wchar_t *)malloc(inbuf + SIZEOF_WCHAR_T); + for ( size_t i = 0; i < inlen; i++ ) + tmpbuf[n] = WC_BSWAP(psz[i]); + tmpbuf[inlen] = L'\0'; + psz = tmpbuf; } if (buf) @@ -1460,8 +1595,7 @@ size_t wxMBConv_iconv::WC2MB(char *buf, const wchar_t *psz, size_t n) const if (ICONV_FAILED(cres, inbuf)) { - //VS: it is ok if iconv fails, hence trace only - wxLogTrace(wxT("strconv"), wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode())); + wxLogTrace(TRACE_STRCONV, wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode())); return (size_t)-1; } @@ -1518,7 +1652,28 @@ public: // own wxMBConvUTF7 doesn't detect errors (e.g. lone "+" which is // explicitly ill-formed according to RFC 2152) neither so we don't // even have any fallback here... - int flags = m_CodePage == CP_UTF7 ? 0 : MB_ERR_INVALID_CHARS; + // + // Moreover, MB_ERR_INVALID_CHARS is only supported on Win 2K SP4 or + // Win XP or newer and if it is specified on older versions, conversion + // from CP_UTF8 (which can have flags only 0 or MB_ERR_INVALID_CHARS) + // fails. So we can only use the flag on newer Windows versions. + // Additionally, the flag is not supported by UTF7, symbol and CJK + // encodings. See here: + // http://blogs.msdn.com/michkap/archive/2005/04/19/409566.aspx + // http://msdn.microsoft.com/library/en-us/intl/unicode_17si.asp + int flags = 0; + if ( m_CodePage != CP_UTF7 && m_CodePage != CP_SYMBOL && + m_CodePage < 50000 && + IsAtLeastWin2kSP4() ) + { + flags = MB_ERR_INVALID_CHARS; + } + else if ( m_CodePage == CP_UTF8 ) + { + // Avoid round-trip in the special case of UTF-8 by using our + // own UTF-8 conversion code: + return wxMBConvUTF8().MB2WC(buf, psz, n); + } const size_t len = ::MultiByteToWideChar ( @@ -1529,11 +1684,40 @@ public: buf, // output string buf ? n : 0 // size of output buffer ); + if ( !len ) + { + // function totally failed + return (size_t)-1; + } + + // if we were really converting and didn't use MB_ERR_INVALID_CHARS, + // check if we succeeded, by doing a double trip: + if ( !flags && buf ) + { + wxCharBuffer mbBuf(n); + if ( ::WideCharToMultiByte + ( + m_CodePage, + 0, + buf, + -1, + mbBuf.data(), + n, + NULL, + NULL + ) == 0 || + strcmp(mbBuf, psz) != 0 ) + { + // we didn't obtain the same thing we started from, hence + // the conversion was lossy and we consider that it failed + return (size_t)-1; + } + } // note that it returns count of written chars for buf != NULL and size // of the needed buffer for buf == NULL so in either case the length of // the string (which never includes the terminating NUL) is one less - return len ? len - 1 : (size_t)-1; + return len - 1; } size_t WC2MB(char *buf, const wchar_t *pwz, size_t n) const @@ -1648,6 +1832,33 @@ private: return s_isWin98Or2k == 1; } + static bool IsAtLeastWin2kSP4() + { +#ifdef __WXWINCE__ + return false; +#else + static int s_isAtLeastWin2kSP4 = -1; + + if ( s_isAtLeastWin2kSP4 == -1 ) + { + OSVERSIONINFOEX ver; + + memset(&ver, 0, sizeof(ver)); + ver.dwOSVersionInfoSize = sizeof(ver); + GetVersionEx((OSVERSIONINFO*)&ver); + + s_isAtLeastWin2kSP4 = + ((ver.dwMajorVersion > 5) || // Vista+ + (ver.dwMajorVersion == 5 && ver.dwMinorVersion > 0) || // XP/2003 + (ver.dwMajorVersion == 5 && ver.dwMinorVersion == 0 && + ver.wServicePackMajor >= 4)) // 2000 SP4+ + ? 1 : 0; + } + + return s_isAtLeastWin2kSP4 == 1; +#endif + } + long m_CodePage; }; @@ -1979,9 +2190,9 @@ public: #if SIZEOF_WCHAR_T == 4 UniChar* szUniCharBuffer = new UniChar[nOutSize]; #endif - + CFStringGetCharacters(theString, theRange, szUniCharBuffer); - + CFRelease(theString); szUniCharBuffer[nOutLength] = '\0' ; @@ -1991,20 +2202,20 @@ public: converter.MB2WC(szOut, (const char*)szUniCharBuffer , nOutSize ) ; delete[] szUniCharBuffer; #endif - + return nOutLength; } size_t WC2MB(char *szOut, const wchar_t *szUnConv, size_t nOutSize) const { wxASSERT(szUnConv); - + size_t nRealOutSize; size_t nBufSize = wxWcslen(szUnConv); UniChar* szUniBuffer = (UniChar*) szUnConv; #if SIZEOF_WCHAR_T == 4 - wxMBConvUTF16BE converter ; + wxMBConvUTF16 converter ; nBufSize = converter.WC2MB( NULL , szUnConv , 0 ); szUniBuffer = new UniChar[ (nBufSize / sizeof(UniChar)) + 1] ; converter.WC2MB( (char*) szUniBuffer , szUnConv, nBufSize + sizeof(UniChar)) ; @@ -2026,7 +2237,7 @@ public: { if (szOut != NULL) CFStringGetCharacters(theString, CFRangeMake(0, nOutSize - 1), (UniChar*) szOut); - + nRealOutSize = CFStringGetLength(theString) + 1; } else @@ -2039,7 +2250,7 @@ public: //0 tells CFString to return NULL if it meets such a character false, //not an external representation (UInt8*) szOut, - nOutSize, + nOutSize, (CFIndex*) &nRealOutSize ); } @@ -2055,7 +2266,7 @@ public: bool IsOk() const { - return m_encoding != kCFStringEncodingInvalidId && + return m_encoding != kCFStringEncodingInvalidId && CFStringIsEncodingAvailable(m_encoding); } @@ -2140,7 +2351,7 @@ public: // we have to terminate here, because n might be larger for the trailing zero, and if UniChar // is not properly terminated we get random characters at the end ubuf[byteOutLen / sizeof( UniChar ) ] = 0 ; - wxMBConvUTF16BE converter ; + wxMBConvUTF16 converter ; res = converter.MB2WC( (buf ? buf : tbuf) , (const char*)ubuf , n ) ; free( ubuf ) ; #else @@ -2173,7 +2384,7 @@ public: ByteCount byteBufferLen = n ; UniChar* ubuf = NULL ; #if SIZEOF_WCHAR_T == 4 - wxMBConvUTF16BE converter ; + wxMBConvUTF16 converter ; size_t unicharlen = converter.WC2MB( NULL , psz , 0 ) ; byteInLen = unicharlen ; ubuf = (UniChar*) malloc( byteInLen + 2 ) ; @@ -2193,7 +2404,7 @@ public: if ( buf && res < n) { buf[res] = 0; - + //we need to double-trip to verify it didn't insert any ? in place //of bogus characters wxWCharBuffer wcBuf(n); @@ -2294,6 +2505,18 @@ public: DECLARE_NO_COPY_CLASS(wxMBConv_wxwin) }; +// make the constructors available for unit testing +WXDLLIMPEXP_BASE wxMBConv* new_wxMBConv_wxwin( const wxChar* name ) +{ + wxMBConv_wxwin* result = new wxMBConv_wxwin( name ); + if ( !result->IsOk() ) + { + delete result; + return 0; + } + return result; +} + #endif // wxUSE_FONTMAP // ============================================================================ @@ -2316,7 +2539,11 @@ wxCSConv::wxCSConv(const wxChar *charset) SetName(charset); } +#if wxUSE_FONTMAP + m_encoding = wxFontMapperBase::GetEncodingFromName(charset); +#else m_encoding = wxFONTENCODING_SYSTEM; +#endif } wxCSConv::wxCSConv(wxFontEncoding encoding) @@ -2375,12 +2602,29 @@ void wxCSConv::SetName(const wxChar *charset) } } +#if wxUSE_FONTMAP +#include "wx/hashmap.h" + +WX_DECLARE_HASH_MAP( wxFontEncoding, wxString, wxIntegerHash, wxIntegerEqual, + wxEncodingNameCache ); + +static wxEncodingNameCache gs_nameCache; +#endif + wxMBConv *wxCSConv::DoCreate() const { +#if wxUSE_FONTMAP + wxLogTrace(TRACE_STRCONV, + wxT("creating conversion for %s"), + (m_name ? m_name + : wxFontMapperBase::GetEncodingName(m_encoding).c_str())); +#endif // wxUSE_FONTMAP + // check for the special case of ASCII or ISO8859-1 charset: as we have // special knowledge of it anyhow, we don't need to create a special // conversion object - if ( m_encoding == wxFONTENCODING_ISO8859_1 ) + if ( m_encoding == wxFONTENCODING_ISO8859_1 || + m_encoding == wxFONTENCODING_DEFAULT ) { // don't convert at all return NULL; @@ -2401,17 +2645,53 @@ wxMBConv *wxCSConv::DoCreate() const #endif // !wxUSE_FONTMAP { wxString name(m_name); + wxFontEncoding encoding(m_encoding); + + if ( !name.empty() ) + { + wxMBConv_iconv *conv = new wxMBConv_iconv(name); + if ( conv->IsOk() ) + return conv; + + delete conv; #if wxUSE_FONTMAP - if ( name.empty() ) - name = wxFontMapperBase::Get()->GetEncodingName(m_encoding); + encoding = + wxFontMapperBase::Get()->CharsetToEncoding(name, false); #endif // wxUSE_FONTMAP + } +#if wxUSE_FONTMAP + { + const wxEncodingNameCache::iterator it = gs_nameCache.find(encoding); + if ( it != gs_nameCache.end() ) + { + if ( it->second.empty() ) + return NULL; - wxMBConv_iconv *conv = new wxMBConv_iconv(name); - if ( conv->IsOk() ) - return conv; + wxMBConv_iconv *conv = new wxMBConv_iconv(it->second); + if ( conv->IsOk() ) + return conv; - delete conv; + delete conv; + } + + const wxChar** names = wxFontMapperBase::GetAllEncodingNames(encoding); + + for ( ; *names; ++names ) + { + wxMBConv_iconv *conv = new wxMBConv_iconv(*names); + if ( conv->IsOk() ) + { + gs_nameCache[encoding] = *names; + return conv; + } + + delete conv; + } + + gs_nameCache[encoding] = _T(""); // cache the failure + } +#endif // wxUSE_FONTMAP } #endif // HAVE_ICONV @@ -2432,7 +2712,7 @@ wxMBConv *wxCSConv::DoCreate() const #if defined(__WXMAC__) { // leave UTF16 and UTF32 to the built-ins of wx - if ( m_name || ( m_encoding < wxFONTENCODING_UTF16BE || + if ( m_name || ( m_encoding < wxFONTENCODING_UTF16BE || ( m_encoding >= wxFONTENCODING_MACMIN && m_encoding <= wxFONTENCODING_MACMAX ) ) ) { @@ -2636,9 +2916,9 @@ WXDLLIMPEXP_DATA_BASE(wxMBConvUTF8&) wxConvUTF8 = wxConvUTF8Obj; WXDLLIMPEXP_DATA_BASE(wxMBConv *) wxConvCurrent = &wxConvLibcObj; WXDLLIMPEXP_DATA_BASE(wxMBConv *) wxConvFileName = & #ifdef __WXOSX__ - wxConvUTF8Obj; + wxConvUTF8Obj; #else - wxConvLibcObj; + wxConvLibcObj; #endif @@ -2651,5 +2931,3 @@ WXDLLIMPEXP_DATA_BASE(wxMBConv) wxConvLibc, wxConvUTF8; #endif // wxUSE_WCHAR_T/!wxUSE_WCHAR_T - -