X-Git-Url: https://git.saurik.com/wxWidgets.git/blobdiff_plain/3698ae71952311f7f0dc4594e0f666e07cc0a103..ef25e5a4c8102c9d981e3f720c27462c72102b54:/src/common/strconv.cpp diff --git a/src/common/strconv.cpp b/src/common/strconv.cpp index db0147d6bc..e50eddf3f9 100644 --- a/src/common/strconv.cpp +++ b/src/common/strconv.cpp @@ -1,5 +1,5 @@ ///////////////////////////////////////////////////////////////////////////// -// Name: strconv.cpp +// Name: src/common/strconv.cpp // Purpose: Unicode conversion classes // Author: Ove Kaaven, Robert Roebling, Vadim Zeitlin, Vaclav Slavik, // Ryan Norton, Fredrik Roubert (UTF7) @@ -20,10 +20,6 @@ // headers // ---------------------------------------------------------------------------- -#if defined(__GNUG__) && !defined(NO_GCC_PRAGMA) - #pragma implementation "strconv.h" -#endif - // For compilers that support precompilation, includes "wx.h". #include "wx/wxprec.h" @@ -40,11 +36,8 @@ #if wxUSE_WCHAR_T -#ifdef __WXMSW__ - #include "wx/msw/private.h" -#endif - #ifdef __WINDOWS__ + #include "wx/msw/private.h" #include "wx/msw/missing.h" #endif @@ -55,18 +48,11 @@ #include #include #include -#ifdef HAVE_LANGINFO_H - #include -#endif #if defined(__WIN32__) && !defined(__WXMICROWIN__) #define wxHAVE_WIN32_MB2WC #endif // __WIN32__ but !__WXMICROWIN__ -// ---------------------------------------------------------------------------- -// headers -// ---------------------------------------------------------------------------- - #ifdef __SALFORDC__ #include #endif @@ -81,39 +67,19 @@ #include "wx/utils.h" #ifdef __WXMAC__ +#ifndef __DARWIN__ #include #include #include +#endif #include "wx/mac/private.h" // includes mac headers #endif -// ---------------------------------------------------------------------------- -// macros -// ---------------------------------------------------------------------------- -#define BSWAP_UCS4(str, len) { unsigned _c; for (_c=0; _c=0xdfff)) + else if ((input[1]<0xdc00) || (input[1]>0xdfff)) { output = *input; return (size_t)-1; @@ -221,129 +187,161 @@ const wxCharBuffer wxMBConv::cWC2MB(const wchar_t *pwz) const return buf; } -const wxWCharBuffer wxMBConv::cMB2WC(const char *szString, size_t nStringLen, size_t* pOutSize) const +const wxWCharBuffer +wxMBConv::cMB2WC(const char *in, size_t inLen, size_t *outLen) const { - wxASSERT(pOutSize != NULL); + // the currently accumulated wide characters + wxWCharBuffer wbuf; - const char* szEnd = szString + nStringLen + 1; - const char* szPos = szString; - const char* szStart = szPos; + // the current length of wbuf + size_t lenBuf = 0; - size_t nActualLength = 0; - size_t nCurrentSize = nStringLen; //try normal size first (should never resize?) + // we need to know the representation of L'\0' for this conversion + size_t nulLen; + const char * const nul = GetMBNul(&nulLen); + if ( nulLen == (size_t)-1 || nulLen == 0 ) + return wxWCharBuffer(); - wxWCharBuffer theBuffer(nCurrentSize); + // make a copy of the input string unless it is already properly + // NUL-terminated + wxCharBuffer bufTmp; - //Convert the string until the length() is reached, continuing the - //loop every time a null character is reached - while(szPos != szEnd) + // now we can compute the input size if we were not given it: notice that + // in this case the string must be properly NUL-terminated, of course, as + // otherwise we have no way of knowing how long it is + if ( inLen == (size_t)-1 ) { - wxASSERT(szPos < szEnd); //something is _really_ screwed up if this rings true - - //Get the length of the current (sub)string - size_t nLen = MB2WC(NULL, szPos, 0); + // not the most efficient algorithm but it shouldn't matter as normally + // there are not many NULs in the string and so normally memcmp() + // should stop on the first character + const char *p = in; + while ( memcmp(p, nul, nulLen) != 0 ) + p++; - //Invalid conversion? - if( nLen == (size_t)-1 ) + inLen = p - in + nulLen; + } + else // we already have the size + { + // check if it's not already NUL-terminated too to avoid the copy + if ( inLen < nulLen || memcmp(in + inLen - nulLen, nul, nulLen) != 0 ) { - *pOutSize = 0; - theBuffer.data()[0u] = wxT('\0'); - return theBuffer; + // make a copy in order to properly NUL-terminate the string + bufTmp = wxCharBuffer(inLen + nulLen - 1 /* 1 will be added */); + memcpy(bufTmp.data(), in, inLen); + memcpy(bufTmp.data() + inLen, nul, nulLen); } + } + if ( bufTmp ) + in = bufTmp; - //Increase the actual length (+1 for current null character) - nActualLength += nLen + 1; - - //if buffer too big, realloc the buffer - if (nActualLength > (nCurrentSize+1)) + for ( const char * const inEnd = in + inLen;; ) + { + // try to convert the current chunk if anything left + size_t lenChunk = in < inEnd ? MB2WC(NULL, in, 0) : 0; + if ( lenChunk == 0 ) { - wxWCharBuffer theNewBuffer(nCurrentSize << 1); - memcpy(theNewBuffer.data(), theBuffer.data(), nCurrentSize * sizeof(wchar_t)); - theBuffer = theNewBuffer; - nCurrentSize <<= 1; - } + // nothing left in the input string, conversion succeeded + if ( outLen ) + { + // we shouldn't include the last NUL in the result length + *outLen = lenBuf ? lenBuf - 1 : 0; + } - //Convert the current (sub)string - if ( MB2WC(&theBuffer.data()[szPos - szStart], szPos, nLen + 1) == (size_t)-1 ) - { - *pOutSize = 0; - theBuffer.data()[0u] = wxT('\0'); - return theBuffer; + return wbuf; } - //Increment to next (sub)string - //Note that we have to use strlen here instead of nLen - //here because XX2XX gives us the size of the output buffer, - //not neccessarly the length of the string - szPos += strlen(szPos) + 1; + if ( lenChunk == (size_t)-1 ) + break; + + const size_t lenBufNew = lenBuf + lenChunk; + if ( !wbuf.extend(lenBufNew) ) + break; + + lenChunk = MB2WC(wbuf.data() + lenBuf, in, lenChunk + 1 /* for NUL */); + if ( lenChunk == (size_t)-1 ) + break; + + // +! for the embedded NUL (if something follows) + lenBuf = lenBufNew + 1; + + // advance the input pointer past the end of this chunk + while ( memcmp(in, nul, nulLen) != 0 ) + in++; + + in += nulLen; // skipping over its terminator as well } - //success - return actual length and the buffer - *pOutSize = nActualLength; - return theBuffer; + // conversion failed + if ( outLen ) + *outLen = 0; + + return wxWCharBuffer(); } -const wxCharBuffer wxMBConv::cWC2MB(const wchar_t *szString, size_t nStringLen, size_t* pOutSize) const +const wxCharBuffer +wxMBConv::cWC2MB(const wchar_t *in, size_t inLen, size_t *outLen) const { - wxASSERT(pOutSize != NULL); - - const wchar_t* szEnd = szString + nStringLen + 1; - const wchar_t* szPos = szString; - const wchar_t* szStart = szPos; + // the currently accumulated multibyte characters + wxCharBuffer buf; - size_t nActualLength = 0; - size_t nCurrentSize = nStringLen << 2; //try * 4 first + // the current length of buf + size_t lenBuf = 0; - wxCharBuffer theBuffer(nCurrentSize); - - //Convert the string until the length() is reached, continuing the - //loop every time a null character is reached - while(szPos != szEnd) + // make a copy of the input string unless it is already properly + // NUL-terminated + // + // if we don't know its length we have no choice but to assume that it is, + // indeed, properly terminated + wxWCharBuffer bufTmp; + if ( inLen == (size_t)-1 ) + { + inLen = wxWcslen(in) + 1; + } + else if ( inLen != 0 && in[inLen - 1] != L'\0' ) { - wxASSERT(szPos < szEnd); //something is _really_ screwed up if this rings true + // make a copy in order to properly NUL-terminate the string + bufTmp = wxWCharBuffer(inLen); + memcpy(bufTmp.data(), in, inLen*sizeof(wchar_t)); + } - //Get the length of the current (sub)string - size_t nLen = WC2MB(NULL, szPos, 0); + if ( bufTmp ) + in = bufTmp; - //Invalid conversion? - if( nLen == (size_t)-1 ) + for ( const wchar_t * const inEnd = in + inLen;; ) + { + // try to convert the current chunk, if anything left + size_t lenChunk = in < inEnd ? WC2MB(NULL, in, 0) : 0; + if ( lenChunk == 0 ) { - *pOutSize = 0; - theBuffer.data()[0u] = wxT('\0'); - return theBuffer; + // nothing left in the input string, conversion succeeded + if ( outLen ) + *outLen = lenBuf ? lenBuf - 1 : lenBuf; + + return buf; } - //Increase the actual length (+1 for current null character) - nActualLength += nLen + 1; + if ( lenChunk == (size_t)-1 ) + break; - //if buffer too big, realloc the buffer - if (nActualLength > (nCurrentSize+1)) - { - wxCharBuffer theNewBuffer(nCurrentSize << 1); - memcpy(theNewBuffer.data(), theBuffer.data(), nCurrentSize); - theBuffer = theNewBuffer; - nCurrentSize <<= 1; - } + const size_t lenBufNew = lenBuf + lenChunk; + if ( !buf.extend(lenBufNew) ) + break; - //Convert the current (sub)string - if(WC2MB(&theBuffer.data()[szPos - szStart], szPos, nLen + 1) == (size_t)-1 ) - { - *pOutSize = 0; - theBuffer.data()[0u] = wxT('\0'); - return theBuffer; - } + lenChunk = WC2MB(buf.data() + lenBuf, in, lenChunk + 1 /* for NUL */); + if ( lenChunk == (size_t)-1 ) + break; - //Increment to next (sub)string - //Note that we have to use wxWcslen here instead of nLen - //here because XX2XX gives us the size of the output buffer, - //not neccessarly the length of the string - szPos += wxWcslen(szPos) + 1; + // chunk successfully converted, go to the next one + in += wxWcslen(in) + 1 /* skip NUL too */; + lenBuf = lenBufNew + 1; } - //success - return actual length and the buffer - *pOutSize = nActualLength; - return theBuffer; + // conversion failed + if ( outLen ) + *outLen = 0; + + return wxCharBuffer(); } // ---------------------------------------------------------------------------- @@ -361,48 +359,21 @@ size_t wxMBConvLibc::WC2MB(char *buf, const wchar_t *psz, size_t n) const } // ---------------------------------------------------------------------------- -// wxConvBrokenFileNames is made for GTK2 in Unicode mode when -// files are accidentally written in an encoding which is not -// the system encoding. Typically, the system encoding will be -// UTF8 but there might be files stored in ISO8859-1 on disk. +// wxConvBrokenFileNames // ---------------------------------------------------------------------------- -class wxConvBrokenFileNames: public wxMBConvLibc -{ -public: - wxConvBrokenFileNames() : m_utf8conv(wxMBConvUTF8::MAP_INVALID_UTF8_TO_OCTAL) { } - virtual size_t MB2WC(wchar_t *outputBuf, const char *psz, size_t outputSize) const; - virtual size_t WC2MB(char *outputBuf, const wchar_t *psz, size_t outputSize) const; - inline bool UseUTF8() const; -private: - wxMBConvUTF8 m_utf8conv; -}; +#ifdef __UNIX__ -bool wxConvBrokenFileNames::UseUTF8() const +wxConvBrokenFileNames::wxConvBrokenFileNames(const wxChar *charset) { -#if defined HAVE_LANGINFO_H && defined CODESET - char *codeset = nl_langinfo(CODESET); - return strcmp(codeset, "UTF-8") == 0; -#else - return false; -#endif -} - -size_t wxConvBrokenFileNames::MB2WC(wchar_t *outputBuf, const char *psz, size_t outputSize) const -{ - if (UseUTF8()) - return m_utf8conv.MB2WC( outputBuf, psz, outputSize ); + if ( !charset || wxStricmp(charset, _T("UTF-8")) == 0 + || wxStricmp(charset, _T("UTF8")) == 0 ) + m_conv = new wxMBConvUTF8(wxMBConvUTF8::MAP_INVALID_UTF8_TO_OCTAL); else - return wxMBConvLibc::MB2WC( outputBuf, psz, outputSize ); + m_conv = new wxCSConv(charset); } -size_t wxConvBrokenFileNames::WC2MB(char *outputBuf, const wchar_t *psz, size_t outputSize) const -{ - if (UseUTF8()) - return m_utf8conv.WC2MB( outputBuf, psz, outputSize ); - else - return wxMBConvLibc::WC2MB( outputBuf, psz, outputSize ); -} +#endif // __UNIX__ // ---------------------------------------------------------------------------- // UTF-7 @@ -453,7 +424,7 @@ size_t wxMBConvUTF7::MB2WC(wchar_t *buf, const char *psz, size_t n) const { size_t len = 0; - while (*psz && ((!buf) || (len < n))) + while ( *psz && (!buf || (len < n)) ) { unsigned char cc = *psz++; if (cc != '+') @@ -471,20 +442,19 @@ size_t wxMBConvUTF7::MB2WC(wchar_t *buf, const char *psz, size_t n) const len++; psz++; } - else + else // start of BASE64 encoded string { - // BASE64 encoded string - bool lsb; - unsigned char c; + bool lsb, ok; unsigned int d, l; - for (lsb = false, d = 0, l = 0; - (cc = utf7unb64[(unsigned char)*psz]) != 0xff; psz++) + for ( ok = lsb = false, d = 0, l = 0; + (cc = utf7unb64[(unsigned char)*psz]) != 0xff; + psz++ ) { d <<= 6; d += cc; for (l += 6; l >= 8; lsb = !lsb) { - c = (unsigned char)((d >> (l -= 8)) % 256); + unsigned char c = (unsigned char)((d >> (l -= 8)) % 256); if (lsb) { if (buf) @@ -492,16 +462,29 @@ size_t wxMBConvUTF7::MB2WC(wchar_t *buf, const char *psz, size_t n) const len ++; } else + { if (buf) *buf = (wchar_t)(c << 8); + } + + ok = true; } } + + if ( !ok ) + { + // in valid UTF7 we should have valid characters after '+' + return (size_t)-1; + } + if (*psz == '-') psz++; } } - if (buf && (len < n)) - *buf = 0; + + if ( buf && (len < n) ) + *buf = '\0'; + return len; } @@ -542,8 +525,6 @@ static const unsigned char utf7encode[128] = size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t *psz, size_t n) const { - - size_t len = 0; while (*psz && ((!buf) || (len < n))) @@ -558,7 +539,7 @@ size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t *psz, size_t n) const } #ifndef WC_UTF16 else if (((wxUint32)cc) > 0xffff) - { + { // no surrogate pair generation (yet?) return (size_t)-1; } @@ -572,7 +553,7 @@ size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t *psz, size_t n) const { // BASE64 encode string unsigned int lsb, d, l; - for (d = 0, l = 0;; psz++) + for (d = 0, l = 0; /*nothing*/; psz++) { for (lsb = 0; lsb < 2; lsb ++) { @@ -638,6 +619,15 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const if (buf) *buf++ = cc; len++; + + // escape the escape character for octal escapes + if ((m_options & MAP_INVALID_UTF8_TO_OCTAL) + && cc == '\\' && (!buf || len < n)) + { + if (buf) + *buf++ = cc; + len++; + } } else { @@ -693,7 +683,7 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const } #else // !WC_UTF16 if (buf) - *buf++ = res; + *buf++ = (wchar_t)res; len++; #endif // WC_UTF16/!WC_UTF16 } @@ -714,7 +704,7 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const len += pa; #else if (buf) - *buf++ = wxUnicodePUA + (unsigned char)*opsz; + *buf++ = (wchar_t)(wxUnicodePUA + (unsigned char)*opsz); opsz++; len++; #endif @@ -726,11 +716,11 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const { if ( buf && len + 3 < n ) { - unsigned char n = *opsz; + unsigned char on = *opsz; *buf++ = L'\\'; - *buf++ = L'0' + n / 0100; - *buf++ = L'0' + (n % 0100) / 010; - *buf++ = L'0' + n % 010; + *buf++ = (wchar_t)( L'0' + on / 0100 ); + *buf++ = (wchar_t)( L'0' + (on % 0100) / 010 ); + *buf++ = (wchar_t)( L'0' + on % 010 ); } opsz++; len += 4; @@ -775,15 +765,23 @@ size_t wxMBConvUTF8::WC2MB(char *buf, const wchar_t *psz, size_t n) const *buf++ = (char)(cc - wxUnicodePUA); len++; } + else if ( (m_options & MAP_INVALID_UTF8_TO_OCTAL) + && cc == L'\\' && psz[0] == L'\\' ) + { + if (buf) + *buf++ = (char)cc; + psz++; + len++; + } else if ( (m_options & MAP_INVALID_UTF8_TO_OCTAL) && cc == L'\\' && isoctal(psz[0]) && isoctal(psz[1]) && isoctal(psz[2]) ) { if (buf) { - *buf++ = (char) (psz[0] - L'0')*0100 + - (psz[1] - L'0')*010 + - (psz[2] - L'0'); + *buf++ = (char) ((psz[0] - L'0')*0100 + + (psz[1] - L'0')*010 + + (psz[2] - L'0')); } psz += 3; @@ -878,20 +876,24 @@ size_t wxMBConvUTF16straight::WC2MB(char *buf, const wchar_t *psz, size_t n) con // swap 16bit MB to 16bit String size_t wxMBConvUTF16swap::MB2WC(wchar_t *buf, const char *psz, size_t n) const { - size_t len=0; + size_t len = 0; - while (*(wxUint16*)psz && (!buf || len < n)) + // UTF16 string must be terminated by 2 NULs as single NULs may occur + // inside the string + while ( (psz[0] || psz[1]) && (!buf || len < n) ) { - if (buf) + if ( buf ) { ((char *)buf)[0] = psz[1]; ((char *)buf)[1] = psz[0]; buf++; } len++; - psz += sizeof(wxUint16); + psz += 2; } - if (buf && lenIsOk() ) + { + delete result; + return 0; + } + return result; +} + +wxString wxMBConv_iconv::ms_wcCharsetName; bool wxMBConv_iconv::ms_wcNeedsSwap = false; wxMBConv_iconv::wxMBConv_iconv(const wxChar *name) { - // Do it the hard way - char cname[100]; - for (size_t i = 0; i < wxStrlen(name)+1; i++) - cname[i] = (char) name[i]; + m_nulLen = (size_t)-2; + + // iconv operates with chars, not wxChars, but luckily it uses only ASCII + // names for the charsets + const wxCharBuffer cname(wxString(name).ToAscii()); // check for charset that represents wchar_t: - if (ms_wcCharsetName == NULL) + if ( ms_wcCharsetName.empty() ) { - ms_wcNeedsSwap = false; + wxLogTrace(TRACE_STRCONV, _T("Looking for wide char codeset:")); - // try charset with explicit bytesex info (e.g. "UCS-4LE"): - ms_wcCharsetName = WC_NAME_BEST; - m2w = iconv_open(ms_wcCharsetName, cname); +#if wxUSE_FONTMAP + const wxChar **names = wxFontMapperBase::GetAllEncodingNames(WC_ENC); +#else // !wxUSE_FONTMAP + static const wxChar *names[] = + { +#if SIZEOF_WCHAR_T == 4 + _T("UCS-4"), +#elif SIZEOF_WCHAR_T = 2 + _T("UCS-2"), +#endif + NULL + }; +#endif // wxUSE_FONTMAP/!wxUSE_FONTMAP - if (m2w == (iconv_t)-1) + for ( ; *names && ms_wcCharsetName.empty(); ++names ) { - // try charset w/o bytesex info (e.g. "UCS4") - // and check for bytesex ourselves: - ms_wcCharsetName = WC_NAME; - m2w = iconv_open(ms_wcCharsetName, cname); + const wxString nameCS(*names); - // last bet, try if it knows WCHAR_T pseudo-charset - if (m2w == (iconv_t)-1) - { - ms_wcCharsetName = "WCHAR_T"; - m2w = iconv_open(ms_wcCharsetName, cname); - } + // first try charset with explicit bytesex info (e.g. "UCS-4LE"): + wxString nameXE(nameCS); + #ifdef WORDS_BIGENDIAN + nameXE += _T("BE"); + #else // little endian + nameXE += _T("LE"); + #endif - if (m2w != (iconv_t)-1) + wxLogTrace(TRACE_STRCONV, _T(" trying charset \"%s\""), + nameXE.c_str()); + + m2w = iconv_open(nameXE.ToAscii(), cname); + if ( m2w == ICONV_T_INVALID ) { - char buf[2], *bufPtr; - wchar_t wbuf[2], *wbufPtr; - size_t insz, outsz; - size_t res; - - buf[0] = 'A'; - buf[1] = 0; - wbuf[0] = 0; - insz = 2; - outsz = SIZEOF_WCHAR_T * 2; - wbufPtr = wbuf; - bufPtr = buf; - - res = iconv(m2w, ICONV_CHAR_CAST(&bufPtr), &insz, - (char**)&wbufPtr, &outsz); - - if (ICONV_FAILED(res, insz)) - { - ms_wcCharsetName = NULL; - wxLogLastError(wxT("iconv")); - wxLogError(_("Conversion to charset '%s' doesn't work."), name); - } - else + // try charset w/o bytesex info (e.g. "UCS4") + wxLogTrace(TRACE_STRCONV, _T(" trying charset \"%s\""), + nameCS.c_str()); + m2w = iconv_open(nameCS.ToAscii(), cname); + + // and check for bytesex ourselves: + if ( m2w != ICONV_T_INVALID ) { - ms_wcNeedsSwap = wbuf[0] != (wchar_t)buf[0]; + char buf[2], *bufPtr; + wchar_t wbuf[2], *wbufPtr; + size_t insz, outsz; + size_t res; + + buf[0] = 'A'; + buf[1] = 0; + wbuf[0] = 0; + insz = 2; + outsz = SIZEOF_WCHAR_T * 2; + wbufPtr = wbuf; + bufPtr = buf; + + res = iconv(m2w, ICONV_CHAR_CAST(&bufPtr), &insz, + (char**)&wbufPtr, &outsz); + + if (ICONV_FAILED(res, insz)) + { + wxLogLastError(wxT("iconv")); + wxLogError(_("Conversion to charset '%s' doesn't work."), + nameCS.c_str()); + } + else // ok, can convert to this encoding, remember it + { + ms_wcCharsetName = nameCS; + ms_wcNeedsSwap = wbuf[0] != (wchar_t)buf[0]; + } } } - else + else // use charset not requiring byte swapping { - ms_wcCharsetName = NULL; - - // VS: we must not output an error here, since wxWidgets will safely - // fall back to using wxEncodingConverter. - wxLogTrace(wxT("strconv"), wxT("Impossible to convert to/from charset '%s' with iconv, falling back to wxEncodingConverter."), name); - //wxLogError( + ms_wcCharsetName = nameXE; } } - wxLogTrace(wxT("strconv"), wxT("wchar_t charset is '%s', needs swap: %i"), ms_wcCharsetName, ms_wcNeedsSwap); + + wxLogTrace(TRACE_STRCONV, + wxT("iconv wchar_t charset is \"%s\"%s"), + ms_wcCharsetName.empty() ? _T("") + : ms_wcCharsetName.c_str(), + ms_wcNeedsSwap ? _T(" (needs swap)") + : _T("")); } else // we already have ms_wcCharsetName { - m2w = iconv_open(ms_wcCharsetName, cname); + m2w = iconv_open(ms_wcCharsetName.ToAscii(), cname); } - // NB: don't ever pass NULL to iconv_open(), it may crash! - if ( ms_wcCharsetName ) + if ( ms_wcCharsetName.empty() ) { - w2m = iconv_open( cname, ms_wcCharsetName); + w2m = ICONV_T_INVALID; } else { - w2m = (iconv_t)-1; + w2m = iconv_open(cname, ms_wcCharsetName.ToAscii()); + if ( w2m == ICONV_T_INVALID ) + { + wxLogTrace(TRACE_STRCONV, + wxT("\"%s\" -> \"%s\" works but not the converse!?"), + ms_wcCharsetName.c_str(), cname.data()); + } } } wxMBConv_iconv::~wxMBConv_iconv() { - if ( m2w != (iconv_t)-1 ) + if ( m2w != ICONV_T_INVALID ) iconv_close(m2w); - if ( w2m != (iconv_t)-1 ) + if ( w2m != ICONV_T_INVALID ) iconv_close(w2m); } @@ -1472,7 +1536,8 @@ size_t wxMBConv_iconv::MB2WC(wchar_t *buf, const char *psz, size_t n) const if (ms_wcNeedsSwap) { // convert to native endianness - WC_BSWAP(buf /* _not_ bufPtr */, res) + for ( unsigned i = 0; i < res; i++ ) + buf[n] = WC_BSWAP(buf[i]); } // NB: iconv was given only strlen(psz) characters on input, and so @@ -1502,7 +1567,7 @@ size_t wxMBConv_iconv::MB2WC(wchar_t *buf, const char *psz, size_t n) const if (ICONV_FAILED(cres, inbuf)) { //VS: it is ok if iconv fails, hence trace only - wxLogTrace(wxT("strconv"), wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode())); + wxLogTrace(TRACE_STRCONV, wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode())); return (size_t)-1; } @@ -1516,7 +1581,8 @@ size_t wxMBConv_iconv::WC2MB(char *buf, const wchar_t *psz, size_t n) const wxMutexLocker lock(wxConstCast(this, wxMBConv_iconv)->m_iconvMutex); #endif - size_t inbuf = wxWcslen(psz) * SIZEOF_WCHAR_T; + size_t inlen = wxWcslen(psz); + size_t inbuf = inlen * SIZEOF_WCHAR_T; size_t outbuf = n; size_t res, cres; @@ -1525,13 +1591,13 @@ size_t wxMBConv_iconv::WC2MB(char *buf, const wchar_t *psz, size_t n) const if (ms_wcNeedsSwap) { // need to copy to temp buffer to switch endianness - // this absolutely doesn't rock! - // (no, doing WC_BSWAP twice on the original buffer won't help, as it + // (doing WC_BSWAP twice on the original buffer won't help, as it // could be in read-only memory, or be accessed in some other thread) - tmpbuf=(wchar_t*)malloc((inbuf+1)*SIZEOF_WCHAR_T); - memcpy(tmpbuf,psz,(inbuf+1)*SIZEOF_WCHAR_T); - WC_BSWAP(tmpbuf, inbuf) - psz=tmpbuf; + tmpbuf = (wchar_t *)malloc(inbuf + SIZEOF_WCHAR_T); + for ( size_t i = 0; i < inlen; i++ ) + tmpbuf[n] = WC_BSWAP(psz[i]); + tmpbuf[inlen] = L'\0'; + psz = tmpbuf; } if (buf) @@ -1569,14 +1635,43 @@ size_t wxMBConv_iconv::WC2MB(char *buf, const wchar_t *psz, size_t n) const if (ICONV_FAILED(cres, inbuf)) { - //VS: it is ok if iconv fails, hence trace only - wxLogTrace(wxT("strconv"), wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode())); + wxLogTrace(TRACE_STRCONV, wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode())); return (size_t)-1; } return res; } +const char *wxMBConv_iconv::GetMBNul(size_t *nulLen) const +{ + if ( m_nulLen == (size_t)-2 ) + { + wxMBConv_iconv * const self = wxConstCast(this, wxMBConv_iconv); + +#if wxUSE_THREADS + // NB: explained in MB2WC + wxMutexLocker lock(self->m_iconvMutex); +#endif + + wchar_t *wnul = L""; + size_t inLen = sizeof(wchar_t), + outLen = WXSIZEOF(m_nulBuf); + const char *in = (char *)wnul; + char *out = self->m_nulBuf; + if ( iconv(w2m, &in, &inLen, &out, &outLen) == (size_t)-1 ) + { + self->m_nulLen = (size_t)-1; + } + else // ok + { + self->m_nulLen = out - m_nulBuf; + } + } + + *nulLen = m_nulLen; + return m_nulBuf; +} + #endif // HAVE_ICONV @@ -1598,19 +1693,22 @@ public: wxMBConv_win32() { m_CodePage = CP_ACP; + m_nulLen = (size_t)-2; } #if wxUSE_FONTMAP wxMBConv_win32(const wxChar* name) { m_CodePage = wxCharsetToCodepage(name); + m_nulLen = (size_t)-2; } wxMBConv_win32(wxFontEncoding encoding) { m_CodePage = wxEncodingToCodepage(encoding); + m_nulLen = (size_t)-2; } -#endif +#endif // wxUSE_FONTMAP size_t MB2WC(wchar_t *buf, const char *psz, size_t n) const { @@ -1627,7 +1725,28 @@ public: // own wxMBConvUTF7 doesn't detect errors (e.g. lone "+" which is // explicitly ill-formed according to RFC 2152) neither so we don't // even have any fallback here... - int flags = m_CodePage == CP_UTF7 ? 0 : MB_ERR_INVALID_CHARS; + // + // Moreover, MB_ERR_INVALID_CHARS is only supported on Win 2K SP4 or + // Win XP or newer and if it is specified on older versions, conversion + // from CP_UTF8 (which can have flags only 0 or MB_ERR_INVALID_CHARS) + // fails. So we can only use the flag on newer Windows versions. + // Additionally, the flag is not supported by UTF7, symbol and CJK + // encodings. See here: + // http://blogs.msdn.com/michkap/archive/2005/04/19/409566.aspx + // http://msdn.microsoft.com/library/en-us/intl/unicode_17si.asp + int flags = 0; + if ( m_CodePage != CP_UTF7 && m_CodePage != CP_SYMBOL && + m_CodePage < 50000 && + IsAtLeastWin2kSP4() ) + { + flags = MB_ERR_INVALID_CHARS; + } + else if ( m_CodePage == CP_UTF8 ) + { + // Avoid round-trip in the special case of UTF-8 by using our + // own UTF-8 conversion code: + return wxMBConvUTF8().MB2WC(buf, psz, n); + } const size_t len = ::MultiByteToWideChar ( @@ -1638,11 +1757,41 @@ public: buf, // output string buf ? n : 0 // size of output buffer ); + if ( !len ) + { + // function totally failed + return (size_t)-1; + } + + // if we were really converting and didn't use MB_ERR_INVALID_CHARS, + // check if we succeeded, by doing a double trip: + if ( !flags && buf ) + { + const size_t mbLen = strlen(psz); + wxCharBuffer mbBuf(mbLen); + if ( ::WideCharToMultiByte + ( + m_CodePage, + 0, + buf, + -1, + mbBuf.data(), + mbLen + 1, // size in bytes, not length + NULL, + NULL + ) == 0 || + strcmp(mbBuf, psz) != 0 ) + { + // we didn't obtain the same thing we started from, hence + // the conversion was lossy and we consider that it failed + return (size_t)-1; + } + } // note that it returns count of written chars for buf != NULL and size // of the needed buffer for buf == NULL so in either case the length of // the string (which never includes the terminating NUL) is one less - return len ? len - 1 : (size_t)-1; + return len - 1; } size_t WC2MB(char *buf, const wchar_t *pwz, size_t n) const @@ -1757,7 +1906,62 @@ private: return s_isWin98Or2k == 1; } + static bool IsAtLeastWin2kSP4() + { +#ifdef __WXWINCE__ + return false; +#else + static int s_isAtLeastWin2kSP4 = -1; + + if ( s_isAtLeastWin2kSP4 == -1 ) + { + OSVERSIONINFOEX ver; + + memset(&ver, 0, sizeof(ver)); + ver.dwOSVersionInfoSize = sizeof(ver); + GetVersionEx((OSVERSIONINFO*)&ver); + + s_isAtLeastWin2kSP4 = + ((ver.dwMajorVersion > 5) || // Vista+ + (ver.dwMajorVersion == 5 && ver.dwMinorVersion > 0) || // XP/2003 + (ver.dwMajorVersion == 5 && ver.dwMinorVersion == 0 && + ver.wServicePackMajor >= 4)) // 2000 SP4+ + ? 1 : 0; + } + + return s_isAtLeastWin2kSP4 == 1; +#endif + } + + virtual const char *GetMBNul(size_t *nulLen) const + { + if ( m_nulLen == (size_t)-2 ) + { + wxMBConv_win32 * const self = wxConstCast(this, wxMBConv_win32); + + self->m_nulLen = ::WideCharToMultiByte + ( + m_CodePage, // code page + 0, // no flags + L"", // input string + 1, // translate just NUL + self->m_nulBuf, // output buffer + WXSIZEOF(m_nulBuf), // and its size + NULL, // "replacement" char + NULL // [out] was it used? + ); + + if ( m_nulLen == 0 ) + self->m_nulLen = (size_t)-1; + } + + *nulLen = m_nulLen; + return m_nulBuf; + } + long m_CodePage; + size_t m_nulLen; + char m_nulBuf[8]; }; #endif // wxHAVE_WIN32_MB2WC @@ -2113,7 +2317,7 @@ public: UniChar* szUniBuffer = (UniChar*) szUnConv; #if SIZEOF_WCHAR_T == 4 - wxMBConvUTF16BE converter ; + wxMBConvUTF16 converter ; nBufSize = converter.WC2MB( NULL , szUnConv , 0 ); szUniBuffer = new UniChar[ (nBufSize / sizeof(UniChar)) + 1] ; converter.WC2MB( (char*) szUniBuffer , szUnConv, nBufSize + sizeof(UniChar)) ; @@ -2249,7 +2453,7 @@ public: // we have to terminate here, because n might be larger for the trailing zero, and if UniChar // is not properly terminated we get random characters at the end ubuf[byteOutLen / sizeof( UniChar ) ] = 0 ; - wxMBConvUTF16BE converter ; + wxMBConvUTF16 converter ; res = converter.MB2WC( (buf ? buf : tbuf) , (const char*)ubuf , n ) ; free( ubuf ) ; #else @@ -2282,7 +2486,7 @@ public: ByteCount byteBufferLen = n ; UniChar* ubuf = NULL ; #if SIZEOF_WCHAR_T == 4 - wxMBConvUTF16BE converter ; + wxMBConvUTF16 converter ; size_t unicharlen = converter.WC2MB( NULL , psz , 0 ) ; byteInLen = unicharlen ; ubuf = (UniChar*) malloc( byteInLen + 2 ) ; @@ -2397,12 +2601,45 @@ public: wxFontEncoding m_enc; wxEncodingConverter m2w, w2m; +private: + virtual const char *GetMBNul(size_t *nulLen) const + { + switch ( m_enc ) + { + case wxFONTENCODING_UTF16BE: + case wxFONTENCODING_UTF16LE: + *nulLen = 2; + return "\0"; + + case wxFONTENCODING_UTF32BE: + case wxFONTENCODING_UTF32LE: + *nulLen = 4; + return "\0\0\0"; + + default: + *nulLen = 1; + return ""; + } + } + // were we initialized successfully? bool m_ok; DECLARE_NO_COPY_CLASS(wxMBConv_wxwin) }; +// make the constructors available for unit testing +WXDLLIMPEXP_BASE wxMBConv* new_wxMBConv_wxwin( const wxChar* name ) +{ + wxMBConv_wxwin* result = new wxMBConv_wxwin( name ); + if ( !result->IsOk() ) + { + delete result; + return 0; + } + return result; +} + #endif // wxUSE_FONTMAP // ============================================================================ @@ -2425,7 +2662,11 @@ wxCSConv::wxCSConv(const wxChar *charset) SetName(charset); } +#if wxUSE_FONTMAP + m_encoding = wxFontMapperBase::GetEncodingFromName(charset); +#else m_encoding = wxFONTENCODING_SYSTEM; +#endif } wxCSConv::wxCSConv(wxFontEncoding encoding) @@ -2484,12 +2725,29 @@ void wxCSConv::SetName(const wxChar *charset) } } +#if wxUSE_FONTMAP +#include "wx/hashmap.h" + +WX_DECLARE_HASH_MAP( wxFontEncoding, wxString, wxIntegerHash, wxIntegerEqual, + wxEncodingNameCache ); + +static wxEncodingNameCache gs_nameCache; +#endif + wxMBConv *wxCSConv::DoCreate() const { +#if wxUSE_FONTMAP + wxLogTrace(TRACE_STRCONV, + wxT("creating conversion for %s"), + (m_name ? m_name + : wxFontMapperBase::GetEncodingName(m_encoding).c_str())); +#endif // wxUSE_FONTMAP + // check for the special case of ASCII or ISO8859-1 charset: as we have // special knowledge of it anyhow, we don't need to create a special // conversion object - if ( m_encoding == wxFONTENCODING_ISO8859_1 ) + if ( m_encoding == wxFONTENCODING_ISO8859_1 || + m_encoding == wxFONTENCODING_DEFAULT ) { // don't convert at all return NULL; @@ -2510,17 +2768,53 @@ wxMBConv *wxCSConv::DoCreate() const #endif // !wxUSE_FONTMAP { wxString name(m_name); + wxFontEncoding encoding(m_encoding); + + if ( !name.empty() ) + { + wxMBConv_iconv *conv = new wxMBConv_iconv(name); + if ( conv->IsOk() ) + return conv; + + delete conv; #if wxUSE_FONTMAP - if ( name.empty() ) - name = wxFontMapperBase::Get()->GetEncodingName(m_encoding); + encoding = + wxFontMapperBase::Get()->CharsetToEncoding(name, false); #endif // wxUSE_FONTMAP + } +#if wxUSE_FONTMAP + { + const wxEncodingNameCache::iterator it = gs_nameCache.find(encoding); + if ( it != gs_nameCache.end() ) + { + if ( it->second.empty() ) + return NULL; - wxMBConv_iconv *conv = new wxMBConv_iconv(name); - if ( conv->IsOk() ) - return conv; + wxMBConv_iconv *conv = new wxMBConv_iconv(it->second); + if ( conv->IsOk() ) + return conv; - delete conv; + delete conv; + } + + const wxChar** names = wxFontMapperBase::GetAllEncodingNames(encoding); + + for ( ; *names; ++names ) + { + wxMBConv_iconv *conv = new wxMBConv_iconv(*names); + if ( conv->IsOk() ) + { + gs_nameCache[encoding] = *names; + return conv; + } + + delete conv; + } + + gs_nameCache[encoding] = _T(""); // cache the failure + } +#endif // wxUSE_FONTMAP } #endif // HAVE_ICONV @@ -2720,6 +3014,20 @@ size_t wxCSConv::WC2MB(char *buf, const wchar_t *psz, size_t n) const return len; } +const char *wxCSConv::GetMBNul(size_t *nulLen) const +{ + CreateConvIfNeeded(); + + if ( m_convReal ) + { + // cast needed just to call private function of m_convReal + return ((wxCSConv *)m_convReal)->GetMBNul(nulLen); + } + + *nulLen = 1; + return ""; +} + // ---------------------------------------------------------------------------- // globals // ---------------------------------------------------------------------------- @@ -2736,7 +3044,6 @@ static wxCSConv wxConvLocalObj(wxFONTENCODING_SYSTEM); static wxCSConv wxConvISO8859_1Obj(wxFONTENCODING_ISO8859_1); static wxMBConvUTF7 wxConvUTF7Obj; static wxMBConvUTF8 wxConvUTF8Obj; -static wxConvBrokenFileNames wxConvBrokenFileNamesObj; WXDLLIMPEXP_DATA_BASE(wxMBConv&) wxConvLibc = wxConvLibcObj; WXDLLIMPEXP_DATA_BASE(wxCSConv&) wxConvLocal = wxConvLocalObj; @@ -2747,8 +3054,6 @@ WXDLLIMPEXP_DATA_BASE(wxMBConv *) wxConvCurrent = &wxConvLibcObj; WXDLLIMPEXP_DATA_BASE(wxMBConv *) wxConvFileName = & #ifdef __WXOSX__ wxConvUTF8Obj; -#elif __WXGTK20__ - wxConvBrokenFileNamesObj; #else wxConvLibcObj; #endif @@ -2763,5 +3068,3 @@ WXDLLIMPEXP_DATA_BASE(wxMBConv) wxConvLibc, wxConvUTF8; #endif // wxUSE_WCHAR_T/!wxUSE_WCHAR_T - -