X-Git-Url: https://git.saurik.com/wxWidgets.git/blobdiff_plain/c543817b3280c560d53307c0fa98e7a99aa00bf9..86948c99a6f0fd177b09b3db0ef702739ec62a27:/src/common/strconv.cpp diff --git a/src/common/strconv.cpp b/src/common/strconv.cpp index c75888b960..7b0e78a69d 100644 --- a/src/common/strconv.cpp +++ b/src/common/strconv.cpp @@ -1,5 +1,5 @@ ///////////////////////////////////////////////////////////////////////////// -// Name: strconv.cpp +// Name: src/common/strconv.cpp // Purpose: Unicode conversion classes // Author: Ove Kaaven, Robert Roebling, Vadim Zeitlin, Vaclav Slavik, // Ryan Norton, Fredrik Roubert (UTF7) @@ -20,10 +20,6 @@ // headers // ---------------------------------------------------------------------------- -#if defined(__GNUG__) && !defined(NO_GCC_PRAGMA) - #pragma implementation "strconv.h" -#endif - // For compilers that support precompilation, includes "wx.h". #include "wx/wxprec.h" @@ -40,11 +36,8 @@ #if wxUSE_WCHAR_T -#ifdef __WXMSW__ - #include "wx/msw/private.h" -#endif - #ifdef __WINDOWS__ + #include "wx/msw/private.h" #include "wx/msw/missing.h" #endif @@ -60,16 +53,13 @@ #define wxHAVE_WIN32_MB2WC #endif // __WIN32__ but !__WXMICROWIN__ -// ---------------------------------------------------------------------------- -// headers -// ---------------------------------------------------------------------------- - #ifdef __SALFORDC__ #include #endif #ifdef HAVE_ICONV #include + #include "wx/thread.h" #endif #include "wx/encconv.h" @@ -77,39 +67,19 @@ #include "wx/utils.h" #ifdef __WXMAC__ +#ifndef __DARWIN__ #include #include #include +#endif #include "wx/mac/private.h" // includes mac headers #endif -// ---------------------------------------------------------------------------- -// macros -// ---------------------------------------------------------------------------- -#define BSWAP_UCS4(str, len) { unsigned _c; for (_c=0; _c=0xdfff)) + else if ((input[1]<0xdc00) || (input[1]>0xdfff)) { output = *input; return (size_t)-1; @@ -217,14 +187,18 @@ const wxCharBuffer wxMBConv::cWC2MB(const wchar_t *pwz) const return buf; } -size_t wxMBConv::MB2WC(wchar_t* szBuffer, const char* szString, - size_t outsize, size_t nStringLen) const +const wxWCharBuffer wxMBConv::cMB2WC(const char *szString, size_t nStringLen, size_t* pOutSize) const { + wxASSERT(pOutSize != NULL); + const char* szEnd = szString + nStringLen + 1; const char* szPos = szString; const char* szStart = szPos; size_t nActualLength = 0; + size_t nCurrentSize = nStringLen; //try normal size first (should never resize?) + + wxWCharBuffer theBuffer(nCurrentSize); //Convert the string until the length() is reached, continuing the //loop every time a null character is reached @@ -237,38 +211,57 @@ size_t wxMBConv::MB2WC(wchar_t* szBuffer, const char* szString, //Invalid conversion? if( nLen == (size_t)-1 ) - return nLen; + { + *pOutSize = 0; + theBuffer.data()[0u] = wxT('\0'); + return theBuffer; + } + //Increase the actual length (+1 for current null character) nActualLength += nLen + 1; - //Only copy data in if buffer size is big enough - if (szBuffer != NULL && - nActualLength <= outsize) + //if buffer too big, realloc the buffer + if (nActualLength > (nCurrentSize+1)) { - //Convert the current (sub)string - if ( MB2WC(&szBuffer[szPos - szStart], szPos, nLen + 1) == (size_t)-1 ) - return (size_t)-1; + wxWCharBuffer theNewBuffer(nCurrentSize << 1); + memcpy(theNewBuffer.data(), theBuffer.data(), nCurrentSize * sizeof(wchar_t)); + theBuffer = theNewBuffer; + nCurrentSize <<= 1; + } + + //Convert the current (sub)string + if ( MB2WC(&theBuffer.data()[szPos - szStart], szPos, nLen + 1) == (size_t)-1 ) + { + *pOutSize = 0; + theBuffer.data()[0u] = wxT('\0'); + return theBuffer; } //Increment to next (sub)string - //Note that we have to use strlen here instead of nLen - //here because XX2XX gives us the size of the output buffer, - //not neccessarly the length of the string + //Note that we have to use strlen instead of nLen here + //because XX2XX gives us the size of the output buffer, + //which is not necessarily the length of the string szPos += strlen(szPos) + 1; } - return nActualLength - 1; //success - return actual length + //success - return actual length and the buffer + *pOutSize = nActualLength; + return theBuffer; } -size_t wxMBConv::WC2MB(char* szBuffer, const wchar_t* szString, - size_t outsize, size_t nStringLen) const +const wxCharBuffer wxMBConv::cWC2MB(const wchar_t *szString, size_t nStringLen, size_t* pOutSize) const { + wxASSERT(pOutSize != NULL); + const wchar_t* szEnd = szString + nStringLen + 1; const wchar_t* szPos = szString; const wchar_t* szStart = szPos; size_t nActualLength = 0; + size_t nCurrentSize = nStringLen << 2; //try * 4 first + + wxCharBuffer theBuffer(nCurrentSize); //Convert the string until the length() is reached, continuing the //loop every time a null character is reached @@ -281,28 +274,42 @@ size_t wxMBConv::WC2MB(char* szBuffer, const wchar_t* szString, //Invalid conversion? if( nLen == (size_t)-1 ) - return nLen; + { + *pOutSize = 0; + theBuffer.data()[0u] = wxT('\0'); + return theBuffer; + } //Increase the actual length (+1 for current null character) nActualLength += nLen + 1; - - //Only copy data in if buffer size is big enough - if (szBuffer != NULL && - nActualLength <= outsize) + + //if buffer too big, realloc the buffer + if (nActualLength > (nCurrentSize+1)) { - //Convert the current (sub)string - if(WC2MB(&szBuffer[szPos - szStart], szPos, nLen + 1) == (size_t)-1 ) - return (size_t)-1; + wxCharBuffer theNewBuffer(nCurrentSize << 1); + memcpy(theNewBuffer.data(), theBuffer.data(), nCurrentSize); + theBuffer = theNewBuffer; + nCurrentSize <<= 1; + } + + //Convert the current (sub)string + if(WC2MB(&theBuffer.data()[szPos - szStart], szPos, nLen + 1) == (size_t)-1 ) + { + *pOutSize = 0; + theBuffer.data()[0u] = wxT('\0'); + return theBuffer; } //Increment to next (sub)string - //Note that we have to use wxWcslen here instead of nLen - //here because XX2XX gives us the size of the output buffer, - //not neccessarly the length of the string + //Note that we have to use wxWcslen instead of nLen here + //because XX2XX gives us the size of the output buffer, + //which is not necessarily the length of the string szPos += wxWcslen(szPos) + 1; } - return nActualLength - 1; //success - return actual length + //success - return actual length and the buffer + *pOutSize = nActualLength; + return theBuffer; } // ---------------------------------------------------------------------------- @@ -318,8 +325,42 @@ size_t wxMBConvLibc::WC2MB(char *buf, const wchar_t *psz, size_t n) const { return wxWC2MB(buf, psz, n); } + +#ifdef __UNIX__ + +// ---------------------------------------------------------------------------- +// wxConvBrokenFileNames +// ---------------------------------------------------------------------------- + +wxConvBrokenFileNames::wxConvBrokenFileNames(const wxChar *charset) +{ + if ( !charset || wxStricmp(charset, _T("UTF-8")) == 0 + || wxStricmp(charset, _T("UTF8")) == 0 ) + m_conv = new wxMBConvUTF8(wxMBConvUTF8::MAP_INVALID_UTF8_TO_OCTAL); + else + m_conv = new wxCSConv(charset); +} + +size_t +wxConvBrokenFileNames::MB2WC(wchar_t *outputBuf, + const char *psz, + size_t outputSize) const +{ + return m_conv->MB2WC( outputBuf, psz, outputSize ); +} + +size_t +wxConvBrokenFileNames::WC2MB(char *outputBuf, + const wchar_t *psz, + size_t outputSize) const +{ + return m_conv->WC2MB( outputBuf, psz, outputSize ); +} + +#endif + // ---------------------------------------------------------------------------- -// UTF-7 +// UTF-7 // ---------------------------------------------------------------------------- // Implementation (C) 2004 Fredrik Roubert @@ -365,7 +406,6 @@ static const unsigned char utf7unb64[] = size_t wxMBConvUTF7::MB2WC(wchar_t *buf, const char *psz, size_t n) const { - size_t len = 0; while (*psz && ((!buf) || (len < n))) @@ -399,7 +439,7 @@ size_t wxMBConvUTF7::MB2WC(wchar_t *buf, const char *psz, size_t n) const d += cc; for (l += 6; l >= 8; lsb = !lsb) { - c = (d >> (l -= 8)) % 256; + c = (unsigned char)((d >> (l -= 8)) % 256); if (lsb) { if (buf) @@ -408,7 +448,7 @@ size_t wxMBConvUTF7::MB2WC(wchar_t *buf, const char *psz, size_t n) const } else if (buf) - *buf = c << 8; + *buf = (wchar_t)(c << 8); } } if (*psz == '-') @@ -455,8 +495,7 @@ static const unsigned char utf7encode[128] = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 3, 3 }; -size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t -*psz, size_t n) const +size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t *psz, size_t n) const { @@ -473,12 +512,8 @@ size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t len++; } #ifndef WC_UTF16 -#ifdef __VMS - else if (cc > 0xffff) -#else - else if (cc > ((const wchar_t)0xffff)) -#endif - { + else if (((wxUint32)cc) > 0xffff) + { // no surrogate pair generation (yet?) return (size_t)-1; } @@ -492,7 +527,7 @@ size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t { // BASE64 encode string unsigned int lsb, d, l; - for (d = 0, l = 0;; psz++) + for (d = 0, l = 0; /*nothing*/; psz++) { for (lsb = 0; lsb < 2; lsb ++) { @@ -535,12 +570,19 @@ size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t static wxUint32 utf8_max[]= { 0x7f, 0x7ff, 0xffff, 0x1fffff, 0x3ffffff, 0x7fffffff, 0xffffffff }; +// boundaries of the private use area we use to (temporarily) remap invalid +// characters invalid in a UTF-8 encoded string +const wxUint32 wxUnicodePUA = 0x100000; +const wxUint32 wxUnicodePUAEnd = wxUnicodePUA + 256; + size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const { size_t len = 0; while (*psz && ((!buf) || (len < n))) { + const char *opsz = psz; + bool invalid = false; unsigned char cc = *psz++, fc = cc; unsigned cnt; for (cnt = 0; fc & 0x80; cnt++) @@ -551,6 +593,15 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const if (buf) *buf++ = cc; len++; + + // escape the escape character for octal escapes + if ((m_options & MAP_INVALID_UTF8_TO_OCTAL) + && cc == '\\' && (!buf || len < n)) + { + if (buf) + *buf++ = cc; + len++; + } } else { @@ -558,7 +609,7 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const if (!cnt) { // invalid UTF-8 sequence - return (size_t)-1; + invalid = true; } else { @@ -566,32 +617,93 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const wxUint32 res = cc & (0x3f >> cnt); while (cnt--) { - cc = *psz++; + cc = *psz; if ((cc & 0xC0) != 0x80) { // invalid UTF-8 sequence - return (size_t)-1; + invalid = true; + break; } + psz++; res = (res << 6) | (cc & 0x3f); } - if (res <= utf8_max[ocnt]) + if (invalid || res <= utf8_max[ocnt]) { // illegal UTF-8 encoding - return (size_t)-1; + invalid = true; + } + else if ((m_options & MAP_INVALID_UTF8_TO_PUA) && + res >= wxUnicodePUA && res < wxUnicodePUAEnd) + { + // if one of our PUA characters turns up externally + // it must also be treated as an illegal sequence + // (a bit like you have to escape an escape character) + invalid = true; } + else + { #ifdef WC_UTF16 - // cast is ok because wchar_t == wxUuint16 if WC_UTF16 - size_t pa = encode_utf16(res, (wxUint16 *)buf); - if (pa == (size_t)-1) - return (size_t)-1; - if (buf) - buf += pa; - len += pa; + // cast is ok because wchar_t == wxUuint16 if WC_UTF16 + size_t pa = encode_utf16(res, (wxUint16 *)buf); + if (pa == (size_t)-1) + { + invalid = true; + } + else + { + if (buf) + buf += pa; + len += pa; + } #else // !WC_UTF16 - if (buf) - *buf++ = res; - len++; + if (buf) + *buf++ = (wchar_t)res; + len++; #endif // WC_UTF16/!WC_UTF16 + } + } + if (invalid) + { + if (m_options & MAP_INVALID_UTF8_TO_PUA) + { + while (opsz < psz && (!buf || len < n)) + { +#ifdef WC_UTF16 + // cast is ok because wchar_t == wxUuint16 if WC_UTF16 + size_t pa = encode_utf16((unsigned char)*opsz + wxUnicodePUA, (wxUint16 *)buf); + wxASSERT(pa != (size_t)-1); + if (buf) + buf += pa; + opsz++; + len += pa; +#else + if (buf) + *buf++ = (wchar_t)(wxUnicodePUA + (unsigned char)*opsz); + opsz++; + len++; +#endif + } + } + else if (m_options & MAP_INVALID_UTF8_TO_OCTAL) + { + while (opsz < psz && (!buf || len < n)) + { + if ( buf && len + 3 < n ) + { + unsigned char on = *opsz; + *buf++ = L'\\'; + *buf++ = (wchar_t)( L'0' + on / 0100 ); + *buf++ = (wchar_t)( L'0' + (on % 0100) / 010 ); + *buf++ = (wchar_t)( L'0' + on % 010 ); + } + opsz++; + len += 4; + } + } + else // MAP_INVALID_UTF8_NOT + { + return (size_t)-1; + } } } } @@ -600,6 +712,11 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const return len; } +static inline bool isoctal(wchar_t wch) +{ + return L'0' <= wch && wch <= L'7'; +} + size_t wxMBConvUTF8::WC2MB(char *buf, const wchar_t *psz, size_t n) const { size_t len = 0; @@ -614,36 +731,67 @@ size_t wxMBConvUTF8::WC2MB(char *buf, const wchar_t *psz, size_t n) const #else cc=(*psz++) & 0x7fffffff; #endif - unsigned cnt; - for (cnt = 0; cc > utf8_max[cnt]; cnt++) {} - if (!cnt) + + if ( (m_options & MAP_INVALID_UTF8_TO_PUA) + && cc >= wxUnicodePUA && cc < wxUnicodePUAEnd ) { - // plain ASCII char if (buf) - *buf++ = (char) cc; + *buf++ = (char)(cc - wxUnicodePUA); len++; } + else if ( (m_options & MAP_INVALID_UTF8_TO_OCTAL) + && cc == L'\\' && psz[0] == L'\\' ) + { + if (buf) + *buf++ = (char)cc; + psz++; + len++; + } + else if ( (m_options & MAP_INVALID_UTF8_TO_OCTAL) && + cc == L'\\' && + isoctal(psz[0]) && isoctal(psz[1]) && isoctal(psz[2]) ) + { + if (buf) + { + *buf++ = (char) ((psz[0] - L'0')*0100 + + (psz[1] - L'0')*010 + + (psz[2] - L'0')); + } + psz += 3; + len++; + } else { - len += cnt + 1; - if (buf) + unsigned cnt; + for (cnt = 0; cc > utf8_max[cnt]; cnt++) {} + if (!cnt) { - *buf++ = (char) ((-128 >> cnt) | ((cc >> (cnt * 6)) & (0x3f >> cnt))); - while (cnt--) - *buf++ = (char) (0x80 | ((cc >> (cnt * 6)) & 0x3f)); + // plain ASCII char + if (buf) + *buf++ = (char) cc; + len++; + } + + else + { + len += cnt + 1; + if (buf) + { + *buf++ = (char) ((-128 >> cnt) | ((cc >> (cnt * 6)) & (0x3f >> cnt))); + while (cnt--) + *buf++ = (char) (0x80 | ((cc >> (cnt * 6)) & 0x3f)); + } } } } - if (buf && (lenIsOk() ) + { + delete result; + return 0; + } + return result; +} + +wxString wxMBConv_iconv::ms_wcCharsetName; bool wxMBConv_iconv::ms_wcNeedsSwap = false; wxMBConv_iconv::wxMBConv_iconv(const wxChar *name) { - // Do it the hard way - char cname[100]; - for (size_t i = 0; i < wxStrlen(name)+1; i++) - cname[i] = (char) name[i]; + // iconv operates with chars, not wxChars, but luckily it uses only ASCII + // names for the charsets + const wxCharBuffer cname(wxString(name).ToAscii()); // check for charset that represents wchar_t: - if (ms_wcCharsetName == NULL) + if ( ms_wcCharsetName.empty() ) { - ms_wcNeedsSwap = false; + wxLogTrace(TRACE_STRCONV, _T("Looking for wide char codeset:")); - // try charset with explicit bytesex info (e.g. "UCS-4LE"): - ms_wcCharsetName = WC_NAME_BEST; - m2w = iconv_open(ms_wcCharsetName, cname); +#if wxUSE_FONTMAP + const wxChar **names = wxFontMapperBase::GetAllEncodingNames(WC_ENC); +#else // !wxUSE_FONTMAP + static const wxChar *names[] = + { +#if SIZEOF_WCHAR_T == 4 + _T("UCS-4"), +#elif SIZEOF_WCHAR_T = 2 + _T("UCS-2"), +#endif + NULL + }; +#endif // wxUSE_FONTMAP/!wxUSE_FONTMAP - if (m2w == (iconv_t)-1) + for ( ; *names && ms_wcCharsetName.empty(); ++names ) { - // try charset w/o bytesex info (e.g. "UCS4") - // and check for bytesex ourselves: - ms_wcCharsetName = WC_NAME; - m2w = iconv_open(ms_wcCharsetName, cname); + const wxString nameCS(*names); - // last bet, try if it knows WCHAR_T pseudo-charset - if (m2w == (iconv_t)-1) - { - ms_wcCharsetName = "WCHAR_T"; - m2w = iconv_open(ms_wcCharsetName, cname); - } + // first try charset with explicit bytesex info (e.g. "UCS-4LE"): + wxString nameXE(nameCS); + #ifdef WORDS_BIGENDIAN + nameXE += _T("BE"); + #else // little endian + nameXE += _T("LE"); + #endif - if (m2w != (iconv_t)-1) + wxLogTrace(TRACE_STRCONV, _T(" trying charset \"%s\""), + nameXE.c_str()); + + m2w = iconv_open(nameXE.ToAscii(), cname); + if ( m2w == ICONV_T_INVALID ) { - char buf[2], *bufPtr; - wchar_t wbuf[2], *wbufPtr; - size_t insz, outsz; - size_t res; - - buf[0] = 'A'; - buf[1] = 0; - wbuf[0] = 0; - insz = 2; - outsz = SIZEOF_WCHAR_T * 2; - wbufPtr = wbuf; - bufPtr = buf; - - res = iconv(m2w, ICONV_CHAR_CAST(&bufPtr), &insz, - (char**)&wbufPtr, &outsz); - - if (ICONV_FAILED(res, insz)) - { - ms_wcCharsetName = NULL; - wxLogLastError(wxT("iconv")); - wxLogError(_("Conversion to charset '%s' doesn't work."), name); - } - else + // try charset w/o bytesex info (e.g. "UCS4") + wxLogTrace(TRACE_STRCONV, _T(" trying charset \"%s\""), + nameCS.c_str()); + m2w = iconv_open(nameCS.ToAscii(), cname); + + // and check for bytesex ourselves: + if ( m2w != ICONV_T_INVALID ) { - ms_wcNeedsSwap = wbuf[0] != (wchar_t)buf[0]; + char buf[2], *bufPtr; + wchar_t wbuf[2], *wbufPtr; + size_t insz, outsz; + size_t res; + + buf[0] = 'A'; + buf[1] = 0; + wbuf[0] = 0; + insz = 2; + outsz = SIZEOF_WCHAR_T * 2; + wbufPtr = wbuf; + bufPtr = buf; + + res = iconv(m2w, ICONV_CHAR_CAST(&bufPtr), &insz, + (char**)&wbufPtr, &outsz); + + if (ICONV_FAILED(res, insz)) + { + wxLogLastError(wxT("iconv")); + wxLogError(_("Conversion to charset '%s' doesn't work."), + nameCS.c_str()); + } + else // ok, can convert to this encoding, remember it + { + ms_wcCharsetName = nameCS; + ms_wcNeedsSwap = wbuf[0] != (wchar_t)buf[0]; + } } } - else + else // use charset not requiring byte swapping { - ms_wcCharsetName = NULL; - - // VS: we must not output an error here, since wxWidgets will safely - // fall back to using wxEncodingConverter. - wxLogTrace(wxT("strconv"), wxT("Impossible to convert to/from charset '%s' with iconv, falling back to wxEncodingConverter."), name); - //wxLogError( + ms_wcCharsetName = nameXE; } } - wxLogTrace(wxT("strconv"), wxT("wchar_t charset is '%s', needs swap: %i"), ms_wcCharsetName, ms_wcNeedsSwap); + + wxLogTrace(TRACE_STRCONV, + wxT("iconv wchar_t charset is \"%s\"%s"), + ms_wcCharsetName.empty() ? _T("") + : ms_wcCharsetName.c_str(), + ms_wcNeedsSwap ? _T(" (needs swap)") + : _T("")); } else // we already have ms_wcCharsetName { - m2w = iconv_open(ms_wcCharsetName, cname); + m2w = iconv_open(ms_wcCharsetName.ToAscii(), cname); } - // NB: don't ever pass NULL to iconv_open(), it may crash! - if ( ms_wcCharsetName ) + if ( ms_wcCharsetName.empty() ) { - w2m = iconv_open( cname, ms_wcCharsetName); + w2m = ICONV_T_INVALID; } else { - w2m = (iconv_t)-1; + w2m = iconv_open(cname, ms_wcCharsetName.ToAscii()); + if ( w2m == ICONV_T_INVALID ) + { + wxLogTrace(TRACE_STRCONV, + wxT("\"%s\" -> \"%s\" works but not the converse!?"), + ms_wcCharsetName.c_str(), cname.data()); + } } } wxMBConv_iconv::~wxMBConv_iconv() { - if ( m2w != (iconv_t)-1 ) + if ( m2w != ICONV_T_INVALID ) iconv_close(m2w); - if ( w2m != (iconv_t)-1 ) + if ( w2m != ICONV_T_INVALID ) iconv_close(w2m); } size_t wxMBConv_iconv::MB2WC(wchar_t *buf, const char *psz, size_t n) const { +#if wxUSE_THREADS + // NB: iconv() is MT-safe, but each thread must use it's own iconv_t handle. + // Unfortunately there is a couple of global wxCSConv objects such as + // wxConvLocal that are used all over wx code, so we have to make sure + // the handle is used by at most one thread at the time. Otherwise + // only a few wx classes would be safe to use from non-main threads + // as MB<->WC conversion would fail "randomly". + wxMutexLocker lock(wxConstCast(this, wxMBConv_iconv)->m_iconvMutex); +#endif + size_t inbuf = strlen(psz); size_t outbuf = n * SIZEOF_WCHAR_T; size_t res, cres; @@ -1281,7 +1496,8 @@ size_t wxMBConv_iconv::MB2WC(wchar_t *buf, const char *psz, size_t n) const if (ms_wcNeedsSwap) { // convert to native endianness - WC_BSWAP(buf /* _not_ bufPtr */, res) + for ( unsigned i = 0; i < res; i++ ) + buf[n] = WC_BSWAP(buf[i]); } // NB: iconv was given only strlen(psz) characters on input, and so @@ -1311,7 +1527,7 @@ size_t wxMBConv_iconv::MB2WC(wchar_t *buf, const char *psz, size_t n) const if (ICONV_FAILED(cres, inbuf)) { //VS: it is ok if iconv fails, hence trace only - wxLogTrace(wxT("strconv"), wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode())); + wxLogTrace(TRACE_STRCONV, wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode())); return (size_t)-1; } @@ -1320,7 +1536,13 @@ size_t wxMBConv_iconv::MB2WC(wchar_t *buf, const char *psz, size_t n) const size_t wxMBConv_iconv::WC2MB(char *buf, const wchar_t *psz, size_t n) const { - size_t inbuf = wxWcslen(psz) * SIZEOF_WCHAR_T; +#if wxUSE_THREADS + // NB: explained in MB2WC + wxMutexLocker lock(wxConstCast(this, wxMBConv_iconv)->m_iconvMutex); +#endif + + size_t inlen = wxWcslen(psz); + size_t inbuf = inlen * SIZEOF_WCHAR_T; size_t outbuf = n; size_t res, cres; @@ -1329,13 +1551,13 @@ size_t wxMBConv_iconv::WC2MB(char *buf, const wchar_t *psz, size_t n) const if (ms_wcNeedsSwap) { // need to copy to temp buffer to switch endianness - // this absolutely doesn't rock! - // (no, doing WC_BSWAP twice on the original buffer won't help, as it + // (doing WC_BSWAP twice on the original buffer won't help, as it // could be in read-only memory, or be accessed in some other thread) - tmpbuf=(wchar_t*)malloc((inbuf+1)*SIZEOF_WCHAR_T); - memcpy(tmpbuf,psz,(inbuf+1)*SIZEOF_WCHAR_T); - WC_BSWAP(tmpbuf, inbuf) - psz=tmpbuf; + tmpbuf = (wchar_t *)malloc(inbuf + SIZEOF_WCHAR_T); + for ( size_t i = 0; i < inlen; i++ ) + tmpbuf[n] = WC_BSWAP(psz[i]); + tmpbuf[inlen] = L'\0'; + psz = tmpbuf; } if (buf) @@ -1373,8 +1595,7 @@ size_t wxMBConv_iconv::WC2MB(char *buf, const wchar_t *psz, size_t n) const if (ICONV_FAILED(cres, inbuf)) { - //VS: it is ok if iconv fails, hence trace only - wxLogTrace(wxT("strconv"), wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode())); + wxLogTrace(TRACE_STRCONV, wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode())); return (size_t)-1; } @@ -1423,20 +1644,80 @@ public: // and break the library itself, e.g. wxTextInputStream::NextChar() // wouldn't work if reading an incomplete MB char didn't result in an // error + // + // note however that using MB_ERR_INVALID_CHARS with CP_UTF7 results in + // an error (tested under Windows Server 2003) and apparently it is + // done on purpose, i.e. the function accepts any input in this case + // and although I'd prefer to return error on ill-formed output, our + // own wxMBConvUTF7 doesn't detect errors (e.g. lone "+" which is + // explicitly ill-formed according to RFC 2152) neither so we don't + // even have any fallback here... + // + // Moreover, MB_ERR_INVALID_CHARS is only supported on Win 2K SP4 or + // Win XP or newer and if it is specified on older versions, conversion + // from CP_UTF8 (which can have flags only 0 or MB_ERR_INVALID_CHARS) + // fails. So we can only use the flag on newer Windows versions. + // Additionally, the flag is not supported by UTF7, symbol and CJK + // encodings. See here: + // http://blogs.msdn.com/michkap/archive/2005/04/19/409566.aspx + // http://msdn.microsoft.com/library/en-us/intl/unicode_17si.asp + int flags = 0; + if ( m_CodePage != CP_UTF7 && m_CodePage != CP_SYMBOL && + m_CodePage < 50000 && + IsAtLeastWin2kSP4() ) + { + flags = MB_ERR_INVALID_CHARS; + } + else if ( m_CodePage == CP_UTF8 ) + { + // Avoid round-trip in the special case of UTF-8 by using our + // own UTF-8 conversion code: + return wxMBConvUTF8().MB2WC(buf, psz, n); + } + const size_t len = ::MultiByteToWideChar ( m_CodePage, // code page - MB_ERR_INVALID_CHARS, // flags: fall on error + flags, // flags: fall on error psz, // input string -1, // its length (NUL-terminated) buf, // output string buf ? n : 0 // size of output buffer ); + if ( !len ) + { + // function totally failed + return (size_t)-1; + } + + // if we were really converting and didn't use MB_ERR_INVALID_CHARS, + // check if we succeeded, by doing a double trip: + if ( !flags && buf ) + { + wxCharBuffer mbBuf(n); + if ( ::WideCharToMultiByte + ( + m_CodePage, + 0, + buf, + -1, + mbBuf.data(), + n, + NULL, + NULL + ) == 0 || + strcmp(mbBuf, psz) != 0 ) + { + // we didn't obtain the same thing we started from, hence + // the conversion was lossy and we consider that it failed + return (size_t)-1; + } + } // note that it returns count of written chars for buf != NULL and size // of the needed buffer for buf == NULL so in either case the length of // the string (which never includes the terminating NUL) is one less - return len ? len - 1 : (size_t)-1; + return len - 1; } size_t WC2MB(char *buf, const wchar_t *pwz, size_t n) const @@ -1551,6 +1832,33 @@ private: return s_isWin98Or2k == 1; } + static bool IsAtLeastWin2kSP4() + { +#ifdef __WXWINCE__ + return false; +#else + static int s_isAtLeastWin2kSP4 = -1; + + if ( s_isAtLeastWin2kSP4 == -1 ) + { + OSVERSIONINFOEX ver; + + memset(&ver, 0, sizeof(ver)); + ver.dwOSVersionInfoSize = sizeof(ver); + GetVersionEx((OSVERSIONINFO*)&ver); + + s_isAtLeastWin2kSP4 = + ((ver.dwMajorVersion > 5) || // Vista+ + (ver.dwMajorVersion == 5 && ver.dwMinorVersion > 0) || // XP/2003 + (ver.dwMajorVersion == 5 && ver.dwMinorVersion == 0 && + ver.wServicePackMajor >= 4)) // 2000 SP4+ + ? 1 : 0; + } + + return s_isAtLeastWin2kSP4 == 1; +#endif + } + long m_CodePage; }; @@ -1834,10 +2142,12 @@ public: Init(CFStringGetSystemEncoding()) ; } +#if wxUSE_FONTMAP wxMBConv_cocoa(const wxChar* name) { - Init( wxCFStringEncFromFontEnc(wxFontMapper::Get()->CharsetToEncoding(name, false) ) ) ; + Init( wxCFStringEncFromFontEnc(wxFontMapperBase::Get()->CharsetToEncoding(name, false) ) ) ; } +#endif wxMBConv_cocoa(wxFontEncoding encoding) { @@ -1880,9 +2190,9 @@ public: #if SIZEOF_WCHAR_T == 4 UniChar* szUniCharBuffer = new UniChar[nOutSize]; #endif - + CFStringGetCharacters(theString, theRange, szUniCharBuffer); - + CFRelease(theString); szUniCharBuffer[nOutLength] = '\0' ; @@ -1892,20 +2202,20 @@ public: converter.MB2WC(szOut, (const char*)szUniCharBuffer , nOutSize ) ; delete[] szUniCharBuffer; #endif - + return nOutLength; } size_t WC2MB(char *szOut, const wchar_t *szUnConv, size_t nOutSize) const { wxASSERT(szUnConv); - + size_t nRealOutSize; size_t nBufSize = wxWcslen(szUnConv); UniChar* szUniBuffer = (UniChar*) szUnConv; #if SIZEOF_WCHAR_T == 4 - wxMBConvUTF16BE converter ; + wxMBConvUTF16 converter ; nBufSize = converter.WC2MB( NULL , szUnConv , 0 ); szUniBuffer = new UniChar[ (nBufSize / sizeof(UniChar)) + 1] ; converter.WC2MB( (char*) szUniBuffer , szUnConv, nBufSize + sizeof(UniChar)) ; @@ -1927,7 +2237,7 @@ public: { if (szOut != NULL) CFStringGetCharacters(theString, CFRangeMake(0, nOutSize - 1), (UniChar*) szOut); - + nRealOutSize = CFStringGetLength(theString) + 1; } else @@ -1940,7 +2250,7 @@ public: //0 tells CFString to return NULL if it meets such a character false, //not an external representation (UInt8*) szOut, - nOutSize, + nOutSize, (CFIndex*) &nRealOutSize ); } @@ -1956,7 +2266,7 @@ public: bool IsOk() const { - return m_encoding != kCFStringEncodingInvalidId && + return m_encoding != kCFStringEncodingInvalidId && CFStringIsEncodingAvailable(m_encoding); } @@ -1980,10 +2290,12 @@ public: Init(CFStringGetSystemEncoding()) ; } +#if wxUSE_FONTMAP wxMBConv_mac(const wxChar* name) { - Init( wxMacGetSystemEncFromFontEnc(wxFontMapper::Get()->CharsetToEncoding(name, false) ) ) ; + Init( wxMacGetSystemEncFromFontEnc(wxFontMapperBase::Get()->CharsetToEncoding(name, false) ) ) ; } +#endif wxMBConv_mac(wxFontEncoding encoding) { @@ -2039,7 +2351,7 @@ public: // we have to terminate here, because n might be larger for the trailing zero, and if UniChar // is not properly terminated we get random characters at the end ubuf[byteOutLen / sizeof( UniChar ) ] = 0 ; - wxMBConvUTF16BE converter ; + wxMBConvUTF16 converter ; res = converter.MB2WC( (buf ? buf : tbuf) , (const char*)ubuf , n ) ; free( ubuf ) ; #else @@ -2072,7 +2384,7 @@ public: ByteCount byteBufferLen = n ; UniChar* ubuf = NULL ; #if SIZEOF_WCHAR_T == 4 - wxMBConvUTF16BE converter ; + wxMBConvUTF16 converter ; size_t unicharlen = converter.WC2MB( NULL , psz , 0 ) ; byteInLen = unicharlen ; ubuf = (UniChar*) malloc( byteInLen + 2 ) ; @@ -2092,7 +2404,7 @@ public: if ( buf && res < n) { buf[res] = 0; - + //we need to double-trip to verify it didn't insert any ? in place //of bogus characters wxWCharBuffer wcBuf(n); @@ -2144,7 +2456,7 @@ public: wxMBConv_wxwin(const wxChar* name) { if (name) - m_enc = wxFontMapper::Get()->CharsetToEncoding(name, false); + m_enc = wxFontMapperBase::Get()->CharsetToEncoding(name, false); else m_enc = wxFONTENCODING_SYSTEM; @@ -2162,7 +2474,10 @@ public: { size_t inbuf = strlen(psz); if (buf) - m2w.Convert(psz,buf); + { + if (!m2w.Convert(psz,buf)) + return (size_t)-1; + } return inbuf; } @@ -2170,7 +2485,10 @@ public: { const size_t inbuf = wxWcslen(psz); if (buf) - w2m.Convert(psz,buf); + { + if (!w2m.Convert(psz,buf)) + return (size_t)-1; + } return inbuf; } @@ -2187,6 +2505,18 @@ public: DECLARE_NO_COPY_CLASS(wxMBConv_wxwin) }; +// make the constructors available for unit testing +WXDLLIMPEXP_BASE wxMBConv* new_wxMBConv_wxwin( const wxChar* name ) +{ + wxMBConv_wxwin* result = new wxMBConv_wxwin( name ); + if ( !result->IsOk() ) + { + delete result; + return 0; + } + return result; +} + #endif // wxUSE_FONTMAP // ============================================================================ @@ -2209,7 +2539,11 @@ wxCSConv::wxCSConv(const wxChar *charset) SetName(charset); } +#if wxUSE_FONTMAP + m_encoding = wxFontMapperBase::GetEncodingFromName(charset); +#else m_encoding = wxFONTENCODING_SYSTEM; +#endif } wxCSConv::wxCSConv(wxFontEncoding encoding) @@ -2268,12 +2602,29 @@ void wxCSConv::SetName(const wxChar *charset) } } +#if wxUSE_FONTMAP +#include "wx/hashmap.h" + +WX_DECLARE_HASH_MAP( wxFontEncoding, wxString, wxIntegerHash, wxIntegerEqual, + wxEncodingNameCache ); + +static wxEncodingNameCache gs_nameCache; +#endif + wxMBConv *wxCSConv::DoCreate() const { +#if wxUSE_FONTMAP + wxLogTrace(TRACE_STRCONV, + wxT("creating conversion for %s"), + (m_name ? m_name + : wxFontMapperBase::GetEncodingName(m_encoding).c_str())); +#endif // wxUSE_FONTMAP + // check for the special case of ASCII or ISO8859-1 charset: as we have // special knowledge of it anyhow, we don't need to create a special // conversion object - if ( m_encoding == wxFONTENCODING_ISO8859_1 ) + if ( m_encoding == wxFONTENCODING_ISO8859_1 || + m_encoding == wxFONTENCODING_DEFAULT ) { // don't convert at all return NULL; @@ -2294,17 +2645,53 @@ wxMBConv *wxCSConv::DoCreate() const #endif // !wxUSE_FONTMAP { wxString name(m_name); + wxFontEncoding encoding(m_encoding); + + if ( !name.empty() ) + { + wxMBConv_iconv *conv = new wxMBConv_iconv(name); + if ( conv->IsOk() ) + return conv; + + delete conv; #if wxUSE_FONTMAP - if ( name.empty() ) - name = wxFontMapper::Get()->GetEncodingName(m_encoding); + encoding = + wxFontMapperBase::Get()->CharsetToEncoding(name, false); #endif // wxUSE_FONTMAP + } +#if wxUSE_FONTMAP + { + const wxEncodingNameCache::iterator it = gs_nameCache.find(encoding); + if ( it != gs_nameCache.end() ) + { + if ( it->second.empty() ) + return NULL; - wxMBConv_iconv *conv = new wxMBConv_iconv(name); - if ( conv->IsOk() ) - return conv; + wxMBConv_iconv *conv = new wxMBConv_iconv(it->second); + if ( conv->IsOk() ) + return conv; - delete conv; + delete conv; + } + + const wxChar** names = wxFontMapperBase::GetAllEncodingNames(encoding); + + for ( ; *names; ++names ) + { + wxMBConv_iconv *conv = new wxMBConv_iconv(*names); + if ( conv->IsOk() ) + { + gs_nameCache[encoding] = *names; + return conv; + } + + delete conv; + } + + gs_nameCache[encoding] = _T(""); // cache the failure + } +#endif // wxUSE_FONTMAP } #endif // HAVE_ICONV @@ -2324,11 +2711,17 @@ wxMBConv *wxCSConv::DoCreate() const #endif // wxHAVE_WIN32_MB2WC #if defined(__WXMAC__) { - if ( m_name || ( m_encoding < wxFONTENCODING_UTF16BE ) ) + // leave UTF16 and UTF32 to the built-ins of wx + if ( m_name || ( m_encoding < wxFONTENCODING_UTF16BE || + ( m_encoding >= wxFONTENCODING_MACMIN && m_encoding <= wxFONTENCODING_MACMAX ) ) ) { +#if wxUSE_FONTMAP wxMBConv_mac *conv = m_name ? new wxMBConv_mac(m_name) : new wxMBConv_mac(m_encoding); +#else + wxMBConv_mac *conv = new wxMBConv_mac(m_encoding); +#endif if ( conv->IsOk() ) return conv; @@ -2341,8 +2734,12 @@ wxMBConv *wxCSConv::DoCreate() const if ( m_name || ( m_encoding <= wxFONTENCODING_UTF16 ) ) { +#if wxUSE_FONTMAP wxMBConv_cocoa *conv = m_name ? new wxMBConv_cocoa(m_name) : new wxMBConv_cocoa(m_encoding); +#else + wxMBConv_cocoa *conv = new wxMBConv_cocoa(m_encoding); +#endif if ( conv->IsOk() ) return conv; @@ -2358,7 +2755,7 @@ wxMBConv *wxCSConv::DoCreate() const // use "false" to suppress interactive dialogs -- we can be called from // anywhere and popping up a dialog from here is the last thing we want to // do - enc = wxFontMapper::Get()->CharsetToEncoding(m_name, false); + enc = wxFontMapperBase::Get()->CharsetToEncoding(m_name, false); } #endif // wxUSE_FONTMAP @@ -2414,7 +2811,7 @@ wxMBConv *wxCSConv::DoCreate() const m_name ? m_name : #if wxUSE_FONTMAP - wxFontMapper::GetEncodingDescription(m_encoding).c_str() + wxFontMapperBase::GetEncodingDescription(m_encoding).c_str() #else // !wxUSE_FONTMAP wxString::Format(_("encoding %s"), m_encoding).c_str() #endif // wxUSE_FONTMAP/!wxUSE_FONTMAP @@ -2511,13 +2908,19 @@ static wxCSConv wxConvISO8859_1Obj(wxFONTENCODING_ISO8859_1); static wxMBConvUTF7 wxConvUTF7Obj; static wxMBConvUTF8 wxConvUTF8Obj; - WXDLLIMPEXP_DATA_BASE(wxMBConv&) wxConvLibc = wxConvLibcObj; WXDLLIMPEXP_DATA_BASE(wxCSConv&) wxConvLocal = wxConvLocalObj; WXDLLIMPEXP_DATA_BASE(wxCSConv&) wxConvISO8859_1 = wxConvISO8859_1Obj; WXDLLIMPEXP_DATA_BASE(wxMBConvUTF7&) wxConvUTF7 = wxConvUTF7Obj; WXDLLIMPEXP_DATA_BASE(wxMBConvUTF8&) wxConvUTF8 = wxConvUTF8Obj; WXDLLIMPEXP_DATA_BASE(wxMBConv *) wxConvCurrent = &wxConvLibcObj; +WXDLLIMPEXP_DATA_BASE(wxMBConv *) wxConvFileName = & +#ifdef __WXOSX__ + wxConvUTF8Obj; +#else + wxConvLibcObj; +#endif + #else // !wxUSE_WCHAR_T @@ -2528,5 +2931,3 @@ WXDLLIMPEXP_DATA_BASE(wxMBConv) wxConvLibc, wxConvUTF8; #endif // wxUSE_WCHAR_T/!wxUSE_WCHAR_T - -