X-Git-Url: https://git.saurik.com/wxWidgets.git/blobdiff_plain/17a1ebd101f0653e69736416a2a28d0ada423141..356410fc88856bffdb7687d7d37e214a02af63e7:/src/common/strconv.cpp?ds=sidebyside diff --git a/src/common/strconv.cpp b/src/common/strconv.cpp index 63de22405c..960aa42566 100644 --- a/src/common/strconv.cpp +++ b/src/common/strconv.cpp @@ -1,5 +1,5 @@ ///////////////////////////////////////////////////////////////////////////// -// Name: strconv.cpp +// Name: src/common/strconv.cpp // Purpose: Unicode conversion classes // Author: Ove Kaaven, Robert Roebling, Vadim Zeitlin, Vaclav Slavik, // Ryan Norton, Fredrik Roubert (UTF7) @@ -78,6 +78,10 @@ #define TRACE_STRCONV _T("strconv") +#if SIZEOF_WCHAR_T == 2 + #define WC_UTF16 +#endif + // ============================================================================ // implementation // ============================================================================ @@ -183,129 +187,161 @@ const wxCharBuffer wxMBConv::cWC2MB(const wchar_t *pwz) const return buf; } -const wxWCharBuffer wxMBConv::cMB2WC(const char *szString, size_t nStringLen, size_t* pOutSize) const +const wxWCharBuffer +wxMBConv::cMB2WC(const char *in, size_t inLen, size_t *outLen) const { - wxASSERT(pOutSize != NULL); + // the currently accumulated wide characters + wxWCharBuffer wbuf; - const char* szEnd = szString + nStringLen + 1; - const char* szPos = szString; - const char* szStart = szPos; + // the current length of wbuf + size_t lenBuf = 0; - size_t nActualLength = 0; - size_t nCurrentSize = nStringLen; //try normal size first (should never resize?) + // we need to know the representation of L'\0' for this conversion + size_t nulLen; + const char * const nul = GetMBNul(&nulLen); + if ( nulLen == (size_t)-1 || nulLen == 0 ) + return wxWCharBuffer(); - wxWCharBuffer theBuffer(nCurrentSize); + // make a copy of the input string unless it is already properly + // NUL-terminated + wxCharBuffer bufTmp; - //Convert the string until the length() is reached, continuing the - //loop every time a null character is reached - while(szPos != szEnd) + // now we can compute the input size if we were not given it: notice that + // in this case the string must be properly NUL-terminated, of course, as + // otherwise we have no way of knowing how long it is + if ( inLen == (size_t)-1 ) { - wxASSERT(szPos < szEnd); //something is _really_ screwed up if this rings true - - //Get the length of the current (sub)string - size_t nLen = MB2WC(NULL, szPos, 0); + // not the most efficient algorithm but it shouldn't matter as normally + // there are not many NULs in the string and so normally memcmp() + // should stop on the first character + const char *p = in; + while ( memcmp(p, nul, nulLen) != 0 ) + p++; - //Invalid conversion? - if( nLen == (size_t)-1 ) + inLen = p - in + nulLen; + } + else // we already have the size + { + // check if it's not already NUL-terminated too to avoid the copy + if ( inLen < nulLen || memcmp(in + inLen - nulLen, nul, nulLen) != 0 ) { - *pOutSize = 0; - theBuffer.data()[0u] = wxT('\0'); - return theBuffer; + // make a copy in order to properly NUL-terminate the string + bufTmp = wxCharBuffer(inLen + nulLen - 1 /* 1 will be added */); + memcpy(bufTmp.data(), in, inLen); + memcpy(bufTmp.data() + inLen, nul, nulLen); } + } + if ( bufTmp ) + in = bufTmp; - //Increase the actual length (+1 for current null character) - nActualLength += nLen + 1; - - //if buffer too big, realloc the buffer - if (nActualLength > (nCurrentSize+1)) + for ( const char * const inEnd = in + inLen;; ) + { + // try to convert the current chunk if anything left + size_t lenChunk = in < inEnd ? MB2WC(NULL, in, 0) : 0; + if ( lenChunk == 0 ) { - wxWCharBuffer theNewBuffer(nCurrentSize << 1); - memcpy(theNewBuffer.data(), theBuffer.data(), nCurrentSize * sizeof(wchar_t)); - theBuffer = theNewBuffer; - nCurrentSize <<= 1; - } + // nothing left in the input string, conversion succeeded + if ( outLen ) + { + // we shouldn't include the last NUL in the result length + *outLen = lenBuf ? lenBuf - 1 : 0; + } - //Convert the current (sub)string - if ( MB2WC(&theBuffer.data()[szPos - szStart], szPos, nLen + 1) == (size_t)-1 ) - { - *pOutSize = 0; - theBuffer.data()[0u] = wxT('\0'); - return theBuffer; + return wbuf; } - //Increment to next (sub)string - //Note that we have to use strlen instead of nLen here - //because XX2XX gives us the size of the output buffer, - //which is not necessarily the length of the string - szPos += strlen(szPos) + 1; + if ( lenChunk == (size_t)-1 ) + break; + + const size_t lenBufNew = lenBuf + lenChunk; + if ( !wbuf.extend(lenBufNew) ) + break; + + lenChunk = MB2WC(wbuf.data() + lenBuf, in, lenChunk + 1 /* for NUL */); + if ( lenChunk == (size_t)-1 ) + break; + + // +! for the embedded NUL (if something follows) + lenBuf = lenBufNew + 1; + + // advance the input pointer past the end of this chunk + while ( memcmp(in, nul, nulLen) != 0 ) + in++; + + in += nulLen; // skipping over its terminator as well } - //success - return actual length and the buffer - *pOutSize = nActualLength; - return theBuffer; + // conversion failed + if ( outLen ) + *outLen = 0; + + return wxWCharBuffer(); } -const wxCharBuffer wxMBConv::cWC2MB(const wchar_t *szString, size_t nStringLen, size_t* pOutSize) const +const wxCharBuffer +wxMBConv::cWC2MB(const wchar_t *in, size_t inLen, size_t *outLen) const { - wxASSERT(pOutSize != NULL); - - const wchar_t* szEnd = szString + nStringLen + 1; - const wchar_t* szPos = szString; - const wchar_t* szStart = szPos; + // the currently accumulated multibyte characters + wxCharBuffer buf; - size_t nActualLength = 0; - size_t nCurrentSize = nStringLen << 2; //try * 4 first + // the current length of buf + size_t lenBuf = 0; - wxCharBuffer theBuffer(nCurrentSize); - - //Convert the string until the length() is reached, continuing the - //loop every time a null character is reached - while(szPos != szEnd) + // make a copy of the input string unless it is already properly + // NUL-terminated + // + // if we don't know its length we have no choice but to assume that it is, + // indeed, properly terminated + wxWCharBuffer bufTmp; + if ( inLen == (size_t)-1 ) { - wxASSERT(szPos < szEnd); //something is _really_ screwed up if this rings true + inLen = wxWcslen(in) + 1; + } + else if ( inLen != 0 && in[inLen - 1] != L'\0' ) + { + // make a copy in order to properly NUL-terminate the string + bufTmp = wxWCharBuffer(inLen); + memcpy(bufTmp.data(), in, inLen*sizeof(wchar_t)); + } - //Get the length of the current (sub)string - size_t nLen = WC2MB(NULL, szPos, 0); + if ( bufTmp ) + in = bufTmp; - //Invalid conversion? - if( nLen == (size_t)-1 ) + for ( const wchar_t * const inEnd = in + inLen;; ) + { + // try to convert the current chunk, if anything left + size_t lenChunk = in < inEnd ? WC2MB(NULL, in, 0) : 0; + if ( lenChunk == 0 ) { - *pOutSize = 0; - theBuffer.data()[0u] = wxT('\0'); - return theBuffer; + // nothing left in the input string, conversion succeeded + if ( outLen ) + *outLen = lenBuf ? lenBuf - 1 : lenBuf; + + return buf; } - //Increase the actual length (+1 for current null character) - nActualLength += nLen + 1; + if ( lenChunk == (size_t)-1 ) + break; - //if buffer too big, realloc the buffer - if (nActualLength > (nCurrentSize+1)) - { - wxCharBuffer theNewBuffer(nCurrentSize << 1); - memcpy(theNewBuffer.data(), theBuffer.data(), nCurrentSize); - theBuffer = theNewBuffer; - nCurrentSize <<= 1; - } + const size_t lenBufNew = lenBuf + lenChunk; + if ( !buf.extend(lenBufNew) ) + break; - //Convert the current (sub)string - if(WC2MB(&theBuffer.data()[szPos - szStart], szPos, nLen + 1) == (size_t)-1 ) - { - *pOutSize = 0; - theBuffer.data()[0u] = wxT('\0'); - return theBuffer; - } + lenChunk = WC2MB(buf.data() + lenBuf, in, lenChunk + 1 /* for NUL */); + if ( lenChunk == (size_t)-1 ) + break; - //Increment to next (sub)string - //Note that we have to use wxWcslen instead of nLen here - //because XX2XX gives us the size of the output buffer, - //which is not necessarily the length of the string - szPos += wxWcslen(szPos) + 1; + // chunk successfully converted, go to the next one + in += wxWcslen(in) + 1 /* skip NUL too */; + lenBuf = lenBufNew + 1; } - //success - return actual length and the buffer - *pOutSize = nActualLength; - return theBuffer; + // conversion failed + if ( outLen ) + *outLen = 0; + + return wxCharBuffer(); } // ---------------------------------------------------------------------------- @@ -322,12 +358,12 @@ size_t wxMBConvLibc::WC2MB(char *buf, const wchar_t *psz, size_t n) const return wxWC2MB(buf, psz, n); } -#ifdef __UNIX__ - // ---------------------------------------------------------------------------- // wxConvBrokenFileNames // ---------------------------------------------------------------------------- +#ifdef __UNIX__ + wxConvBrokenFileNames::wxConvBrokenFileNames(const wxChar *charset) { if ( !charset || wxStricmp(charset, _T("UTF-8")) == 0 @@ -337,23 +373,7 @@ wxConvBrokenFileNames::wxConvBrokenFileNames(const wxChar *charset) m_conv = new wxCSConv(charset); } -size_t -wxConvBrokenFileNames::MB2WC(wchar_t *outputBuf, - const char *psz, - size_t outputSize) const -{ - return m_conv->MB2WC( outputBuf, psz, outputSize ); -} - -size_t -wxConvBrokenFileNames::WC2MB(char *outputBuf, - const wchar_t *psz, - size_t outputSize) const -{ - return m_conv->WC2MB( outputBuf, psz, outputSize ); -} - -#endif +#endif // __UNIX__ // ---------------------------------------------------------------------------- // UTF-7 @@ -404,7 +424,7 @@ size_t wxMBConvUTF7::MB2WC(wchar_t *buf, const char *psz, size_t n) const { size_t len = 0; - while (*psz && ((!buf) || (len < n))) + while ( *psz && (!buf || (len < n)) ) { unsigned char cc = *psz++; if (cc != '+') @@ -422,20 +442,19 @@ size_t wxMBConvUTF7::MB2WC(wchar_t *buf, const char *psz, size_t n) const len++; psz++; } - else + else // start of BASE64 encoded string { - // BASE64 encoded string - bool lsb; - unsigned char c; + bool lsb, ok; unsigned int d, l; - for (lsb = false, d = 0, l = 0; - (cc = utf7unb64[(unsigned char)*psz]) != 0xff; psz++) + for ( ok = lsb = false, d = 0, l = 0; + (cc = utf7unb64[(unsigned char)*psz]) != 0xff; + psz++ ) { d <<= 6; d += cc; for (l += 6; l >= 8; lsb = !lsb) { - c = (unsigned char)((d >> (l -= 8)) % 256); + unsigned char c = (unsigned char)((d >> (l -= 8)) % 256); if (lsb) { if (buf) @@ -443,16 +462,29 @@ size_t wxMBConvUTF7::MB2WC(wchar_t *buf, const char *psz, size_t n) const len ++; } else + { if (buf) *buf = (wchar_t)(c << 8); + } + + ok = true; } } + + if ( !ok ) + { + // in valid UTF7 we should have valid characters after '+' + return (size_t)-1; + } + if (*psz == '-') psz++; } } - if (buf && (len < n)) - *buf = 0; + + if ( buf && (len < n) ) + *buf = '\0'; + return len; } @@ -493,8 +525,6 @@ static const unsigned char utf7encode[128] = size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t *psz, size_t n) const { - - size_t len = 0; while (*psz && ((!buf) || (len < n))) @@ -523,7 +553,7 @@ size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t *psz, size_t n) const { // BASE64 encode string unsigned int lsb, d, l; - for (d = 0, l = 0;; psz++) + for (d = 0, l = 0; /*nothing*/; psz++) { for (lsb = 0; lsb < 2; lsb ++) { @@ -653,7 +683,7 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const } #else // !WC_UTF16 if (buf) - *buf++ = res; + *buf++ = (wchar_t)res; len++; #endif // WC_UTF16/!WC_UTF16 } @@ -674,7 +704,7 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const len += pa; #else if (buf) - *buf++ = wxUnicodePUA + (unsigned char)*opsz; + *buf++ = (wchar_t)(wxUnicodePUA + (unsigned char)*opsz); opsz++; len++; #endif @@ -846,20 +876,24 @@ size_t wxMBConvUTF16straight::WC2MB(char *buf, const wchar_t *psz, size_t n) con // swap 16bit MB to 16bit String size_t wxMBConvUTF16swap::MB2WC(wchar_t *buf, const char *psz, size_t n) const { - size_t len=0; + size_t len = 0; - while (*(wxUint16*)psz && (!buf || len < n)) + // UTF16 string must be terminated by 2 NULs as single NULs may occur + // inside the string + while ( (psz[0] || psz[1]) && (!buf || len < n) ) { - if (buf) + if ( buf ) { ((char *)buf)[0] = psz[1]; ((char *)buf)[1] = psz[0]; buf++; } len++; - psz += sizeof(wxUint16); + psz += 2; } - if (buf && lenm_iconvMutex); +#endif + + wchar_t *wnul = L""; + size_t inLen = sizeof(wchar_t), + outLen = WXSIZEOF(m_nulBuf); + char *in = (char *)wnul, + *out = self->m_nulBuf; + if ( iconv(w2m, &in, &inLen, &out, &outLen) == (size_t)-1 ) + { + self->m_nulLen = (size_t)-1; + } + else // ok + { + self->m_nulLen = out - m_nulBuf; + } + } + + *nulLen = m_nulLen; + return m_nulBuf; +} + #endif // HAVE_ICONV @@ -1613,19 +1693,22 @@ public: wxMBConv_win32() { m_CodePage = CP_ACP; + m_nulLen = (size_t)-2; } #if wxUSE_FONTMAP wxMBConv_win32(const wxChar* name) { m_CodePage = wxCharsetToCodepage(name); + m_nulLen = (size_t)-2; } wxMBConv_win32(wxFontEncoding encoding) { m_CodePage = wxEncodingToCodepage(encoding); + m_nulLen = (size_t)-2; } -#endif +#endif // wxUSE_FONTMAP size_t MB2WC(wchar_t *buf, const char *psz, size_t n) const { @@ -1642,7 +1725,28 @@ public: // own wxMBConvUTF7 doesn't detect errors (e.g. lone "+" which is // explicitly ill-formed according to RFC 2152) neither so we don't // even have any fallback here... - int flags = m_CodePage == CP_UTF7 ? 0 : MB_ERR_INVALID_CHARS; + // + // Moreover, MB_ERR_INVALID_CHARS is only supported on Win 2K SP4 or + // Win XP or newer and if it is specified on older versions, conversion + // from CP_UTF8 (which can have flags only 0 or MB_ERR_INVALID_CHARS) + // fails. So we can only use the flag on newer Windows versions. + // Additionally, the flag is not supported by UTF7, symbol and CJK + // encodings. See here: + // http://blogs.msdn.com/michkap/archive/2005/04/19/409566.aspx + // http://msdn.microsoft.com/library/en-us/intl/unicode_17si.asp + int flags = 0; + if ( m_CodePage != CP_UTF7 && m_CodePage != CP_SYMBOL && + m_CodePage < 50000 && + IsAtLeastWin2kSP4() ) + { + flags = MB_ERR_INVALID_CHARS; + } + else if ( m_CodePage == CP_UTF8 ) + { + // Avoid round-trip in the special case of UTF-8 by using our + // own UTF-8 conversion code: + return wxMBConvUTF8().MB2WC(buf, psz, n); + } const size_t len = ::MultiByteToWideChar ( @@ -1653,11 +1757,41 @@ public: buf, // output string buf ? n : 0 // size of output buffer ); + if ( !len ) + { + // function totally failed + return (size_t)-1; + } + + // if we were really converting and didn't use MB_ERR_INVALID_CHARS, + // check if we succeeded, by doing a double trip: + if ( !flags && buf ) + { + const size_t mbLen = strlen(psz); + wxCharBuffer mbBuf(mbLen); + if ( ::WideCharToMultiByte + ( + m_CodePage, + 0, + buf, + -1, + mbBuf.data(), + mbLen + 1, // size in bytes, not length + NULL, + NULL + ) == 0 || + strcmp(mbBuf, psz) != 0 ) + { + // we didn't obtain the same thing we started from, hence + // the conversion was lossy and we consider that it failed + return (size_t)-1; + } + } // note that it returns count of written chars for buf != NULL and size // of the needed buffer for buf == NULL so in either case the length of // the string (which never includes the terminating NUL) is one less - return len ? len - 1 : (size_t)-1; + return len - 1; } size_t WC2MB(char *buf, const wchar_t *pwz, size_t n) const @@ -1772,7 +1906,62 @@ private: return s_isWin98Or2k == 1; } + static bool IsAtLeastWin2kSP4() + { +#ifdef __WXWINCE__ + return false; +#else + static int s_isAtLeastWin2kSP4 = -1; + + if ( s_isAtLeastWin2kSP4 == -1 ) + { + OSVERSIONINFOEX ver; + + memset(&ver, 0, sizeof(ver)); + ver.dwOSVersionInfoSize = sizeof(ver); + GetVersionEx((OSVERSIONINFO*)&ver); + + s_isAtLeastWin2kSP4 = + ((ver.dwMajorVersion > 5) || // Vista+ + (ver.dwMajorVersion == 5 && ver.dwMinorVersion > 0) || // XP/2003 + (ver.dwMajorVersion == 5 && ver.dwMinorVersion == 0 && + ver.wServicePackMajor >= 4)) // 2000 SP4+ + ? 1 : 0; + } + + return s_isAtLeastWin2kSP4 == 1; +#endif + } + + virtual const char *GetMBNul(size_t *nulLen) const + { + if ( m_nulLen == (size_t)-2 ) + { + wxMBConv_win32 * const self = wxConstCast(this, wxMBConv_win32); + + self->m_nulLen = ::WideCharToMultiByte + ( + m_CodePage, // code page + 0, // no flags + L"", // input string + 1, // translate just NUL + self->m_nulBuf, // output buffer + WXSIZEOF(m_nulBuf), // and its size + NULL, // "replacement" char + NULL // [out] was it used? + ); + + if ( m_nulLen == 0 ) + self->m_nulLen = (size_t)-1; + } + + *nulLen = m_nulLen; + return m_nulBuf; + } + long m_CodePage; + size_t m_nulLen; + char m_nulBuf[8]; }; #endif // wxHAVE_WIN32_MB2WC @@ -2412,6 +2601,27 @@ public: wxFontEncoding m_enc; wxEncodingConverter m2w, w2m; +private: + virtual const char *GetMBNul(size_t *nulLen) const + { + switch ( m_enc ) + { + case wxFONTENCODING_UTF16BE: + case wxFONTENCODING_UTF16LE: + *nulLen = 2; + return "\0"; + + case wxFONTENCODING_UTF32BE: + case wxFONTENCODING_UTF32LE: + *nulLen = 4; + return "\0\0\0"; + + default: + *nulLen = 1; + return ""; + } + } + // were we initialized successfully? bool m_ok; @@ -2452,7 +2662,11 @@ wxCSConv::wxCSConv(const wxChar *charset) SetName(charset); } +#if wxUSE_FONTMAP + m_encoding = wxFontMapperBase::GetEncodingFromName(charset); +#else m_encoding = wxFONTENCODING_SYSTEM; +#endif } wxCSConv::wxCSConv(wxFontEncoding encoding) @@ -2532,7 +2746,8 @@ wxMBConv *wxCSConv::DoCreate() const // check for the special case of ASCII or ISO8859-1 charset: as we have // special knowledge of it anyhow, we don't need to create a special // conversion object - if ( m_encoding == wxFONTENCODING_ISO8859_1 ) + if ( m_encoding == wxFONTENCODING_ISO8859_1 || + m_encoding == wxFONTENCODING_DEFAULT ) { // don't convert at all return NULL; @@ -2799,6 +3014,20 @@ size_t wxCSConv::WC2MB(char *buf, const wchar_t *psz, size_t n) const return len; } +const char *wxCSConv::GetMBNul(size_t *nulLen) const +{ + CreateConvIfNeeded(); + + if ( m_convReal ) + { + // cast needed just to call private function of m_convReal + return ((wxCSConv *)m_convReal)->GetMBNul(nulLen); + } + + *nulLen = 1; + return ""; +} + // ---------------------------------------------------------------------------- // globals // ---------------------------------------------------------------------------- @@ -2839,5 +3068,3 @@ WXDLLIMPEXP_DATA_BASE(wxMBConv) wxConvLibc, wxConvUTF8; #endif // wxUSE_WCHAR_T/!wxUSE_WCHAR_T - -