X-Git-Url: https://git.saurik.com/wxWidgets.git/blobdiff_plain/d6f2a8911e509fd9e61f881cc881a97f5aa05ae8..726cc8697cd44a41b43257f05ca8cdd42b71a711:/src/common/strconv.cpp diff --git a/src/common/strconv.cpp b/src/common/strconv.cpp index f2364b7fc3..8fc90a869e 100644 --- a/src/common/strconv.cpp +++ b/src/common/strconv.cpp @@ -28,8 +28,6 @@ #include "wx/strconv.h" -#if wxUSE_WCHAR_T - #ifndef __WXWINCE__ #include #endif @@ -44,10 +42,6 @@ #define wxHAVE_WIN32_MB2WC #endif -#ifdef __SALFORDC__ - #include -#endif - #ifdef HAVE_ICONV #include #include "wx/thread.h" @@ -57,11 +51,11 @@ #include "wx/fontmap.h" #ifdef __DARWIN__ -#include "wx/mac/corefoundation/private/strconv_cf.h" +#include "wx/osx/core/private/strconv_cf.h" #endif //def __DARWIN__ -#define TRACE_STRCONV _T("strconv") +#define TRACE_STRCONV wxT("strconv") // WC_UTF16 is defined only if sizeof(wchar_t) == 2, otherwise it's supposed to // be 4 bytes @@ -146,7 +140,7 @@ static wxUint32 wxDecodeSurrogate(const wxDecodeSurrogate_t **pSrc) { wxUint32 out; const size_t - n = decode_utf16(wx_reinterpret_cast(const wxUint16 *, *pSrc), out); + n = decode_utf16(reinterpret_cast(*pSrc), out); if ( n == wxCONV_FAILED ) *pSrc = NULL; else @@ -164,11 +158,15 @@ wxMBConv::ToWChar(wchar_t *dst, size_t dstLen, const char *src, size_t srcLen) const { // although new conversion classes are supposed to implement this function - // directly, the existins ones only implement the old MB2WC() and so, to + // directly, the existing ones only implement the old MB2WC() and so, to // avoid to have to rewrite all conversion classes at once, we provide a // default (but not efficient) implementation of this one in terms of the // old function by copying the input to ensure that it's NUL-terminated and // then using MB2WC() to convert it + // + // moreover, some conversion classes simply can't implement ToWChar() + // directly, the primary example is wxConvLibc: mbstowcs() only handles + // NUL-terminated strings // the number of chars [which would be] written to dst [if it were not NULL] size_t dstWritten = 0; @@ -209,6 +207,21 @@ wxMBConv::ToWChar(wchar_t *dst, size_t dstLen, srcEnd = NULL; } + // the idea of this code is straightforward: it converts a NUL-terminated + // chunk of the string during each iteration and updates the output buffer + // with the result + // + // all the complication come from the fact that this function, for + // historical reasons, must behave in 2 subtly different ways when it's + // called with a fixed number of characters and when it's called for the + // entire NUL-terminated string: in the former case (srcEnd != NULL) we + // must count all characters we convert, NUL or not; but in the latter we + // do not count the trailing NUL -- but still count all the NULs inside the + // string + // + // so for the (simple) former case we just always count the trailing NUL, + // but for the latter we need to wait until we see if there is going to be + // another loop iteration and only count it then for ( ;; ) { // try to convert the current chunk @@ -216,11 +229,11 @@ wxMBConv::ToWChar(wchar_t *dst, size_t dstLen, if ( lenChunk == wxCONV_FAILED ) return wxCONV_FAILED; - lenChunk++; // for the L'\0' at the end of this chunk - dstWritten += lenChunk; + if ( !srcEnd ) + dstWritten++; - if ( lenChunk == 1 ) + if ( !lenChunk ) { // nothing left in the input string, conversion succeeded break; @@ -231,20 +244,25 @@ wxMBConv::ToWChar(wchar_t *dst, size_t dstLen, if ( dstWritten > dstLen ) return wxCONV_FAILED; - if ( MB2WC(dst, src, lenChunk) == wxCONV_FAILED ) + // +1 is for trailing NUL + if ( MB2WC(dst, src, lenChunk + 1) == wxCONV_FAILED ) return wxCONV_FAILED; dst += lenChunk; + if ( !srcEnd ) + dst++; } if ( !srcEnd ) { // we convert just one chunk in this case as this is the entire - // string anyhow + // string anyhow (and we don't count the trailing NUL in this case) break; } - // advance the input pointer past the end of this chunk + // advance the input pointer past the end of this chunk: notice that we + // will always stop before srcEnd because we know that the chunk is + // always properly NUL-terminated while ( NotAllNULs(src, nulLen) ) { // notice that we must skip over multiple bytes here as we suppose @@ -254,11 +272,18 @@ wxMBConv::ToWChar(wchar_t *dst, size_t dstLen, src += nulLen; } - src += nulLen; // skipping over its terminator as well + // if the buffer ends before this NUL, we shouldn't count it in our + // output so skip the code below + if ( src == srcEnd ) + break; + + // do count this terminator as it's inside the buffer we convert + dstWritten++; + if ( dst ) + dst++; + + src += nulLen; // skip the terminator itself - // note that ">=" (and not just "==") is needed here as the terminator - // we skipped just above could be inside or just after the buffer - // delimited by inEnd if ( src >= srcEnd ) break; } @@ -273,13 +298,15 @@ wxMBConv::FromWChar(char *dst, size_t dstLen, // the number of chars [which would be] written to dst [if it were not NULL] size_t dstWritten = 0; + // if we don't know its length we have no choice but to assume that it is + // NUL-terminated (notice that it can still be NUL-terminated even if + // explicit length is given but it doesn't change our return value) + const bool isNulTerminated = srcLen == wxNO_LEN; + // make a copy of the input string unless it is already properly // NUL-terminated - // - // if we don't know its length we have no choice but to assume that it is, - // indeed, properly terminated wxWCharBuffer bufTmp; - if ( srcLen == wxNO_LEN ) + if ( isNulTerminated ) { srcLen = wxWcslen(src) + 1; } @@ -294,27 +321,65 @@ wxMBConv::FromWChar(char *dst, size_t dstLen, const size_t lenNul = GetMBNulLen(); for ( const wchar_t * const srcEnd = src + srcLen; src < srcEnd; - src += wxWcslen(src) + 1 /* skip L'\0' too */ ) + src++ /* skip L'\0' too */ ) { // try to convert the current chunk size_t lenChunk = WC2MB(NULL, src, 0); - if ( lenChunk == wxCONV_FAILED ) return wxCONV_FAILED; - lenChunk += lenNul; dstWritten += lenChunk; + const wchar_t * const + chunkEnd = isNulTerminated ? srcEnd - 1 : src + wxWcslen(src); + + // our return value accounts for the trailing NUL(s), unlike that of + // WC2MB(), however don't do it for the last NUL we artificially added + // ourselves above + if ( chunkEnd < srcEnd ) + dstWritten += lenNul; + if ( dst ) { if ( dstWritten > dstLen ) return wxCONV_FAILED; - if ( WC2MB(dst, src, lenChunk) == wxCONV_FAILED ) + // if we know that there is enough space in the destination buffer + // (because we accounted for lenNul in dstWritten above), we can + // convert directly in place -- but otherwise we need another + // temporary buffer to ensure that we don't overwrite the output + wxCharBuffer dstBuf; + char *dstTmp; + if ( chunkEnd == srcEnd ) + { + dstBuf = wxCharBuffer(lenChunk + lenNul - 1); + dstTmp = dstBuf.data(); + } + else + { + dstTmp = dst; + } + + if ( WC2MB(dstTmp, src, lenChunk + lenNul) == wxCONV_FAILED ) return wxCONV_FAILED; + if ( dstTmp != dst ) + { + // copy everything up to but excluding the terminating NUL(s) + // into the real output buffer + memcpy(dst, dstTmp, lenChunk); + + // micro-optimization: if dstTmp != dst it means that chunkEnd + // == srcEnd and so we're done, no need to update anything below + break; + } + dst += lenChunk; + if ( chunkEnd < srcEnd ) + dst += lenNul; } + + src = chunkEnd; } return dstWritten; @@ -391,13 +456,23 @@ wxMBConv::cMB2WC(const char *inBuff, size_t inLen, size_t *outLen) const const size_t dstLen = ToWChar(NULL, 0, inBuff, inLen); if ( dstLen != wxCONV_FAILED ) { - wxWCharBuffer wbuf(dstLen - 1); + // notice that we allocate space for dstLen+1 wide characters here + // because we want the buffer to always be NUL-terminated, even if the + // input isn't (as otherwise the caller has no way to know its length) + wxWCharBuffer wbuf(dstLen); + wbuf.data()[dstLen] = L'\0'; if ( ToWChar(wbuf.data(), dstLen, inBuff, inLen) != wxCONV_FAILED ) { if ( outLen ) { *outLen = dstLen; - if ( wbuf[dstLen - 1] == L'\0' ) + + // we also need to handle NUL-terminated input strings + // specially: for them the output is the length of the string + // excluding the trailing NUL, however if we're asked to + // convert a specific number of characters we return the length + // of the resulting output even if it's NUL-terminated + if ( inLen == wxNO_LEN ) (*outLen)--; } @@ -417,21 +492,22 @@ wxMBConv::cWC2MB(const wchar_t *inBuff, size_t inLen, size_t *outLen) const size_t dstLen = FromWChar(NULL, 0, inBuff, inLen); if ( dstLen != wxCONV_FAILED ) { - // special case of empty input: can't allocate 0 size buffer below as - // wxCharBuffer insists on NUL-terminating it - wxCharBuffer buf(dstLen ? dstLen - 1 : 1); + const size_t nulLen = GetMBNulLen(); + + // as above, ensure that the buffer is always NUL-terminated, even if + // the input is not + wxCharBuffer buf(dstLen + nulLen - 1); + memset(buf.data() + dstLen, 0, nulLen); if ( FromWChar(buf.data(), dstLen, inBuff, inLen) != wxCONV_FAILED ) { if ( outLen ) { *outLen = dstLen; - const size_t nulLen = GetMBNulLen(); - if ( dstLen >= nulLen && - !NotAllNULs(buf.data() + dstLen - nulLen, nulLen) ) + if ( inLen == wxNO_LEN ) { - // in this case the output is NUL-terminated and we're not - // supposed to count NUL + // in this case both input and output are NUL-terminated + // and we're not supposed to count NUL *outLen -= nulLen; } } @@ -446,6 +522,42 @@ wxMBConv::cWC2MB(const wchar_t *inBuff, size_t inLen, size_t *outLen) const return wxCharBuffer(); } +const wxWCharBuffer wxMBConv::cMB2WC(const wxScopedCharBuffer& buf) const +{ + const size_t srcLen = buf.length(); + if ( srcLen ) + { + const size_t dstLen = ToWChar(NULL, 0, buf, srcLen); + if ( dstLen != wxCONV_FAILED ) + { + wxWCharBuffer wbuf(dstLen); + wbuf.data()[dstLen] = L'\0'; + if ( ToWChar(wbuf.data(), dstLen, buf, srcLen) != wxCONV_FAILED ) + return wbuf; + } + } + + return wxScopedWCharBuffer::CreateNonOwned(L"", 0); +} + +const wxCharBuffer wxMBConv::cWC2MB(const wxScopedWCharBuffer& wbuf) const +{ + const size_t srcLen = wbuf.length(); + if ( srcLen ) + { + const size_t dstLen = FromWChar(NULL, 0, wbuf, srcLen); + if ( dstLen != wxCONV_FAILED ) + { + wxCharBuffer buf(dstLen); + buf.data()[dstLen] = '\0'; + if ( FromWChar(buf.data(), dstLen, wbuf, srcLen) != wxCONV_FAILED ) + return buf; + } + } + + return wxScopedCharBuffer::CreateNonOwned("", 0); +} + // ---------------------------------------------------------------------------- // wxMBConvLibc // ---------------------------------------------------------------------------- @@ -468,8 +580,8 @@ size_t wxMBConvLibc::WC2MB(char *buf, const wchar_t *psz, size_t n) const wxConvBrokenFileNames::wxConvBrokenFileNames(const wxString& charset) { - if ( wxStricmp(charset, _T("UTF-8")) == 0 || - wxStricmp(charset, _T("UTF8")) == 0 ) + if ( wxStricmp(charset, wxT("UTF-8")) == 0 || + wxStricmp(charset, wxT("UTF8")) == 0 ) m_conv = new wxMBConvUTF8(wxMBConvUTF8::MAP_INVALID_UTF8_TO_PUA); else m_conv = new wxCSConv(charset); @@ -482,6 +594,8 @@ wxConvBrokenFileNames::wxConvBrokenFileNames(const wxString& charset) // ---------------------------------------------------------------------------- // Implementation (C) 2004 Fredrik Roubert +// +// Changes to work in streaming mode (C) 2008 Vadim Zeitlin // // BASE64 decoding table @@ -522,70 +636,149 @@ static const unsigned char utf7unb64[] = 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -size_t wxMBConvUTF7::MB2WC(wchar_t *buf, const char *psz, size_t n) const +size_t wxMBConvUTF7::ToWChar(wchar_t *dst, size_t dstLen, + const char *src, size_t srcLen) const { + DecoderState stateOrig, + *statePtr; + if ( srcLen == wxNO_LEN ) + { + // convert the entire string, up to and including the trailing NUL + srcLen = strlen(src) + 1; + + // when working on the entire strings we don't update nor use the shift + // state from the previous call + statePtr = &stateOrig; + } + else // when working with partial strings we do use the shift state + { + statePtr = const_cast(&m_stateDecoder); + + // also save the old state to be able to rollback to it on error + stateOrig = m_stateDecoder; + } + + // but to simplify the code below we use this variable in both cases + DecoderState& state = *statePtr; + + + // number of characters [which would have been] written to dst [if it were + // not NULL] size_t len = 0; - while ( *psz && (!buf || (len < n)) ) + const char * const srcEnd = src + srcLen; + + while ( (src < srcEnd) && (!dst || (len < dstLen)) ) { - unsigned char cc = *psz++; - if (cc != '+') - { - // plain ASCII char - if (buf) - *buf++ = cc; - len++; - } - else if (*psz == '-') - { - // encoded plus sign - if (buf) - *buf++ = cc; - len++; - psz++; - } - else // start of BASE64 encoded string + const unsigned char cc = *src++; + + if ( state.IsShifted() ) { - bool lsb, ok; - unsigned int d, l; - for ( ok = lsb = false, d = 0, l = 0; - (cc = utf7unb64[(unsigned char)*psz]) != 0xff; - psz++ ) + const unsigned char dc = utf7unb64[cc]; + if ( dc == 0xff ) { - d <<= 6; - d += cc; - for (l += 6; l >= 8; lsb = !lsb) + // end of encoded part, check that nothing was left: there can + // be up to 4 bits of 0 padding but nothing else (we also need + // to check isLSB as we count bits modulo 8 while a valid UTF-7 + // encoded sequence must contain an integral number of UTF-16 + // characters) + if ( state.isLSB || state.bit > 4 || + (state.accum & ((1 << state.bit) - 1)) ) { - unsigned char c = (unsigned char)((d >> (l -= 8)) % 256); - if (lsb) + if ( !len ) + state = stateOrig; + + return wxCONV_FAILED; + } + + state.ToDirect(); + + // re-parse this character normally below unless it's '-' which + // is consumed by the decoder + if ( cc == '-' ) + continue; + } + else // valid encoded character + { + // mini base64 decoder: each character is 6 bits + state.bit += 6; + state.accum <<= 6; + state.accum += dc; + + if ( state.bit >= 8 ) + { + // got the full byte, consume it + state.bit -= 8; + unsigned char b = (state.accum >> state.bit) & 0x00ff; + + if ( state.isLSB ) { - if (buf) - *buf++ |= c; - len ++; + // we've got the full word, output it + if ( dst ) + *dst++ = (state.msb << 8) | b; + len++; + state.isLSB = false; } - else + else // MSB { - if (buf) - *buf = (wchar_t)(c << 8); + // just store it while we wait for LSB + state.msb = b; + state.isLSB = true; } - - ok = true; } } + } - if ( !ok ) + if ( state.IsDirect() ) + { + // start of an encoded segment? + if ( cc == '+' ) { - // in valid UTF7 we should have valid characters after '+' - return wxCONV_FAILED; + if ( *src == '-' ) + { + // just the encoded plus sign, don't switch to shifted mode + if ( dst ) + *dst++ = '+'; + len++; + src++; + } + else if ( utf7unb64[(unsigned)*src] == 0xff ) + { + // empty encoded chunks are not allowed + if ( !len ) + state = stateOrig; + + return wxCONV_FAILED; + } + else // base-64 encoded chunk follows + { + state.ToShifted(); + } } + else // not '+' + { + // only printable 7 bit ASCII characters (with the exception of + // NUL, TAB, CR and LF) can be used directly + if ( cc >= 0x7f || (cc < ' ' && + !(cc == '\0' || cc == '\t' || cc == '\r' || cc == '\n')) ) + return wxCONV_FAILED; - if (*psz == '-') - psz++; + if ( dst ) + *dst++ = cc; + len++; + } } } - if ( buf && (len < n) ) - *buf = '\0'; + if ( !len ) + { + // as we didn't read any characters we should be called with the same + // data (followed by some more new data) again later so don't save our + // state + state = stateOrig; + + return wxCONV_FAILED; + } return len; } @@ -615,7 +808,7 @@ static const unsigned char utf7enb64[] = // static const unsigned char utf7encode[128] = { - 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 3, 3, 2, 3, 3, + 0, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, @@ -625,21 +818,72 @@ static const unsigned char utf7encode[128] = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 3, 3 }; -size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t *psz, size_t n) const +static inline bool wxIsUTF7Direct(wchar_t wc) +{ + return wc < 0x80 && utf7encode[wc] < 1; +} + +size_t wxMBConvUTF7::FromWChar(char *dst, size_t dstLen, + const wchar_t *src, size_t srcLen) const { + EncoderState stateOrig, + *statePtr; + if ( srcLen == wxNO_LEN ) + { + // we don't apply the stored state when operating on entire strings at + // once + statePtr = &stateOrig; + + srcLen = wxWcslen(src) + 1; + } + else // do use the mode we left the output in previously + { + stateOrig = m_stateEncoder; + statePtr = const_cast(&m_stateEncoder); + } + + EncoderState& state = *statePtr; + + size_t len = 0; - while (*psz && ((!buf) || (len < n))) + const wchar_t * const srcEnd = src + srcLen; + while ( src < srcEnd && (!dst || len < dstLen) ) { - wchar_t cc = *psz++; - if (cc < 0x80 && utf7encode[cc] < 1) + wchar_t cc = *src++; + if ( wxIsUTF7Direct(cc) ) { - // plain ASCII char - if (buf) - *buf++ = (char)cc; + if ( state.IsShifted() ) + { + // pad with zeros the last encoded block if necessary + if ( state.bit ) + { + if ( dst ) + *dst++ = utf7enb64[((state.accum % 16) << (6 - state.bit)) % 64]; + len++; + } + + state.ToDirect(); + if ( dst ) + *dst++ = '-'; + len++; + } + + if ( dst ) + *dst++ = (char)cc; len++; } + else if ( cc == '+' && state.IsDirect() ) + { + if ( dst ) + { + *dst++ = '+'; + *dst++ = '-'; + } + + len += 2; + } #ifndef WC_UTF16 else if (((wxUint32)cc) > 0xffff) { @@ -649,52 +893,45 @@ size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t *psz, size_t n) const #endif else { - if (buf) - *buf++ = '+'; + if ( state.IsDirect() ) + { + state.ToShifted(); - len++; - if (cc != '+') + if ( dst ) + *dst++ = '+'; + len++; + } + + // BASE64 encode string + for ( ;; ) { - // BASE64 encode string - unsigned int lsb, d, l; - for (d = 0, l = 0; /*nothing*/; psz++) + for ( unsigned lsb = 0; lsb < 2; lsb++ ) { - for (lsb = 0; lsb < 2; lsb ++) - { - d <<= 8; - d += lsb ? cc & 0xff : (cc & 0xff00) >> 8; + state.accum <<= 8; + state.accum += lsb ? cc & 0xff : (cc & 0xff00) >> 8; - for (l += 8; l >= 6; ) - { - l -= 6; - if (buf) - *buf++ = utf7enb64[(d >> l) % 64]; - len++; - } + for (state.bit += 8; state.bit >= 6; ) + { + state.bit -= 6; + if ( dst ) + *dst++ = utf7enb64[(state.accum >> state.bit) % 64]; + len++; } - - cc = *psz; - if (!(cc) || (cc < 0x80 && utf7encode[cc] < 1)) - break; } - if (l != 0) - { - if (buf) - *buf++ = utf7enb64[((d % 16) << (6 - l)) % 64]; + if ( src == srcEnd || wxIsUTF7Direct(cc = *src) ) + break; - len++; - } + src++; } - - if (buf) - *buf++ = '-'; - len++; } } - if (buf && (len < n)) - *buf = 0; + // we need to restore the original encoder state if we were called just to + // calculate the amount of space needed as we will presumably be called + // again to really convert the data now + if ( !dst ) + state = stateOrig; return len; } @@ -703,7 +940,7 @@ size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t *psz, size_t n) const // UTF-8 // ---------------------------------------------------------------------------- -static wxUint32 utf8_max[]= +static const wxUint32 utf8_max[]= { 0x7f, 0x7ff, 0xffff, 0x1fffff, 0x3ffffff, 0x7fffffff, 0xffffffff }; // boundaries of the private use area we use to (temporarily) remap invalid @@ -712,7 +949,7 @@ const wxUint32 wxUnicodePUA = 0x100000; const wxUint32 wxUnicodePUAEnd = wxUnicodePUA + 256; // this table gives the length of the UTF-8 encoding from its first character: -unsigned char tableUtf8Lengths[256] = { +const unsigned char tableUtf8Lengths[256] = { // single-byte sequences (ASCII): 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 00..0F 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 10..1F @@ -758,7 +995,7 @@ wxMBConvStrictUTF8::ToWChar(wchar_t *dst, size_t dstLen, for ( const char *p = src; ; p++ ) { - if ( !(srcLen == wxNO_LEN ? *p : srcLen) ) + if ( (srcLen == wxNO_LEN ? !*p : !srcLen) ) { // all done successfully, just add the trailing NULL if we are not // using explicit length @@ -778,58 +1015,73 @@ wxMBConvStrictUTF8::ToWChar(wchar_t *dst, size_t dstLen, return written; } - unsigned char c = *p; - unsigned len = tableUtf8Lengths[c]; - if ( !len ) + if ( out && !dstLen-- ) break; - if ( srcLen < len ) // the test works for wxNO_LEN too - break; + wxUint32 code; + unsigned char c = *p; - if ( srcLen != wxNO_LEN ) - srcLen -= len; + if ( c < 0x80 ) + { + if ( srcLen == 0 ) // the test works for wxNO_LEN too + break; - if ( out && !dstLen-- ) - break; + if ( srcLen != wxNO_LEN ) + srcLen--; + code = c; + } + else + { + unsigned len = tableUtf8Lengths[c]; + if ( !len ) + break; - // Char. number range | UTF-8 octet sequence - // (hexadecimal) | (binary) - // ----------------------+--------------------------------------------- - // 0000 0000 - 0000 007F | 0xxxxxxx - // 0000 0080 - 0000 07FF | 110xxxxx 10xxxxxx - // 0000 0800 - 0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx - // 0001 0000 - 0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - // - // Code point value is stored in bits marked with 'x', lowest-order bit - // of the value on the right side in the diagram above. - // (from RFC 3629) + if ( srcLen < len ) // the test works for wxNO_LEN too + break; - // mask to extract lead byte's value ('x' bits above), by sequence length: - static const unsigned char leadValueMask[] = { 0x7F, 0x1F, 0x0F, 0x07 }; + if ( srcLen != wxNO_LEN ) + srcLen -= len; - // mask and value of lead byte's most significant bits, by length: - static const unsigned char leadMarkerMask[] = { 0x80, 0xE0, 0xF0, 0xF8 }; - static const unsigned char leadMarkerVal[] = { 0x00, 0xC0, 0xE0, 0xF0 }; + // Char. number range | UTF-8 octet sequence + // (hexadecimal) | (binary) + // ----------------------+---------------------------------------- + // 0000 0000 - 0000 007F | 0xxxxxxx + // 0000 0080 - 0000 07FF | 110xxxxx 10xxxxxx + // 0000 0800 - 0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx + // 0001 0000 - 0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + // + // Code point value is stored in bits marked with 'x', + // lowest-order bit of the value on the right side in the diagram + // above. (from RFC 3629) - len--; // it's more convenient to work with 0-based length here + // mask to extract lead byte's value ('x' bits above), by sequence + // length: + static const unsigned char leadValueMask[] = { 0x7F, 0x1F, 0x0F, 0x07 }; - // extract the lead byte's value bits: - if ( (c & leadMarkerMask[len]) != leadMarkerVal[len] ) - break; + // mask and value of lead byte's most significant bits, by length: + static const unsigned char leadMarkerMask[] = { 0x80, 0xE0, 0xF0, 0xF8 }; + static const unsigned char leadMarkerVal[] = { 0x00, 0xC0, 0xE0, 0xF0 }; - wxUint32 code = c & leadValueMask[len]; + len--; // it's more convenient to work with 0-based length here - // all remaining bytes, if any, are handled in the same way regardless of - // sequence's length: - for ( ; len; --len ) - { - c = *++p; - if ( (c & 0xC0) != 0x80 ) - return wxCONV_FAILED; + // extract the lead byte's value bits: + if ( (c & leadMarkerMask[len]) != leadMarkerVal[len] ) + break; - code <<= 6; - code |= c & 0x3F; + code = c & leadValueMask[len]; + + // all remaining bytes, if any, are handled in the same way + // regardless of sequence's length: + for ( ; len; --len ) + { + c = *++p; + if ( (c & 0xC0) != 0x80 ) + return wxCONV_FAILED; + + code <<= 6; + code |= c & 0x3F; + } } #ifdef WC_UTF16 @@ -863,7 +1115,7 @@ wxMBConvStrictUTF8::FromWChar(char *dst, size_t dstLen, for ( const wchar_t *wp = src; ; wp++ ) { - if ( !(srcLen == wxNO_LEN ? *wp : srcLen--) ) + if ( (srcLen == wxNO_LEN ? !*wp : !srcLen) ) { // all done successfully, just add the trailing NULL if we are not // using explicit length @@ -883,6 +1135,8 @@ wxMBConvStrictUTF8::FromWChar(char *dst, size_t dstLen, return written; } + if ( srcLen != wxNO_LEN ) + srcLen--; wxUint32 code; #ifdef WC_UTF16 @@ -951,7 +1205,7 @@ wxMBConvStrictUTF8::FromWChar(char *dst, size_t dstLen, } else { - wxFAIL_MSG( _T("trying to encode undefined Unicode character") ); + wxFAIL_MSG( wxT("trying to encode undefined Unicode character") ); break; } @@ -968,14 +1222,15 @@ wxMBConvStrictUTF8::FromWChar(char *dst, size_t dstLen, return wxCONV_FAILED; } -size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const +size_t wxMBConvUTF8::ToWChar(wchar_t *buf, size_t n, + const char *psz, size_t srcLen) const { if ( m_options == MAP_INVALID_UTF8_NOT ) - return wxMBConvStrictUTF8::MB2WC(buf, psz, n); + return wxMBConvStrictUTF8::ToWChar(buf, n, psz, srcLen); size_t len = 0; - while (*psz && ((!buf) || (len < n))) + while ((srcLen == wxNO_LEN ? *psz : srcLen--) && ((!buf) || (len < n))) { const char *opsz = psz; bool invalid = false; @@ -1109,10 +1364,10 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const } } - if (buf && (len < n)) + if (srcLen == wxNO_LEN && buf && (len < n)) *buf = 0; - return len; + return len + 1; } static inline bool isoctal(wchar_t wch) @@ -1120,14 +1375,15 @@ static inline bool isoctal(wchar_t wch) return L'0' <= wch && wch <= L'7'; } -size_t wxMBConvUTF8::WC2MB(char *buf, const wchar_t *psz, size_t n) const +size_t wxMBConvUTF8::FromWChar(char *buf, size_t n, + const wchar_t *psz, size_t srcLen) const { if ( m_options == MAP_INVALID_UTF8_NOT ) - return wxMBConvStrictUTF8::WC2MB(buf, psz, n); + return wxMBConvStrictUTF8::FromWChar(buf, n, psz, srcLen); size_t len = 0; - while (*psz && ((!buf) || (len < n))) + while ((srcLen == wxNO_LEN ? *psz : srcLen--) && ((!buf) || (len < n))) { wxUint32 cc; @@ -1195,10 +1451,10 @@ size_t wxMBConvUTF8::WC2MB(char *buf, const wchar_t *psz, size_t n) const } } - if (buf && (len < n)) + if (srcLen == wxNO_LEN && buf && (len < n)) *buf = 0; - return len; + return len + 1; } // ============================================================================ @@ -1219,7 +1475,7 @@ size_t wxMBConvUTF16Base::GetLength(const char *src, size_t srcLen) if ( srcLen == wxNO_LEN ) { // count the number of bytes in input, including the trailing NULs - const wxUint16 *inBuff = wx_reinterpret_cast(const wxUint16 *, src); + const wxUint16 *inBuff = reinterpret_cast(src); for ( srcLen = 1; *inBuff++; srcLen++ ) ; @@ -1303,7 +1559,7 @@ wxMBConvUTF16swap::ToWChar(wchar_t *dst, size_t dstLen, if ( dstLen < srcLen ) return wxCONV_FAILED; - const wxUint16 *inBuff = wx_reinterpret_cast(const wxUint16 *, src); + const wxUint16 *inBuff = reinterpret_cast(src); for ( size_t n = 0; n < srcLen; n++, inBuff++ ) { *dst++ = wxUINT16_SWAP_ALWAYS(*inBuff); @@ -1327,7 +1583,7 @@ wxMBConvUTF16swap::FromWChar(char *dst, size_t dstLen, if ( dstLen < srcLen ) return wxCONV_FAILED; - wxUint16 *outBuff = wx_reinterpret_cast(wxUint16 *, dst); + wxUint16 *outBuff = reinterpret_cast(dst); for ( size_t n = 0; n < srcLen; n += BYTES_PER_CHAR, src++ ) { *outBuff++ = wxUINT16_SWAP_ALWAYS(*src); @@ -1361,7 +1617,7 @@ wxMBConvUTF16straight::ToWChar(wchar_t *dst, size_t dstLen, } size_t outLen = 0; - const wxUint16 *inBuff = wx_reinterpret_cast(const wxUint16 *, src); + const wxUint16 *inBuff = reinterpret_cast(src); for ( const wxUint16 * const inEnd = inBuff + inLen; inBuff < inEnd; ) { const wxUint32 ch = wxDecodeSurrogate(&inBuff); @@ -1386,7 +1642,7 @@ wxMBConvUTF16straight::FromWChar(char *dst, size_t dstLen, srcLen = wxWcslen(src) + 1; size_t outLen = 0; - wxUint16 *outBuff = wx_reinterpret_cast(wxUint16 *, dst); + wxUint16 *outBuff = reinterpret_cast(dst); for ( size_t n = 0; n < srcLen; n++ ) { wxUint16 cc[2]; @@ -1434,7 +1690,7 @@ wxMBConvUTF16swap::ToWChar(wchar_t *dst, size_t dstLen, } size_t outLen = 0; - const wxUint16 *inBuff = wx_reinterpret_cast(const wxUint16 *, src); + const wxUint16 *inBuff = reinterpret_cast(src); for ( const wxUint16 * const inEnd = inBuff + inLen; inBuff < inEnd; ) { wxUint32 ch; @@ -1469,7 +1725,7 @@ wxMBConvUTF16swap::FromWChar(char *dst, size_t dstLen, srcLen = wxWcslen(src) + 1; size_t outLen = 0; - wxUint16 *outBuff = wx_reinterpret_cast(wxUint16 *, dst); + wxUint16 *outBuff = reinterpret_cast(dst); for ( const wchar_t *srcEnd = src + srcLen; src < srcEnd; src++ ) { wxUint16 cc[2]; @@ -1520,7 +1776,7 @@ size_t wxMBConvUTF32Base::GetLength(const char *src, size_t srcLen) if ( srcLen == wxNO_LEN ) { // count the number of bytes in input, including the trailing NULs - const wxUint32 *inBuff = wx_reinterpret_cast(const wxUint32 *, src); + const wxUint32 *inBuff = reinterpret_cast(src); for ( srcLen = 1; *inBuff++; srcLen++ ) ; @@ -1551,7 +1807,7 @@ wxMBConvUTF32straight::ToWChar(wchar_t *dst, size_t dstLen, if ( srcLen == wxNO_LEN ) return wxCONV_FAILED; - const wxUint32 *inBuff = wx_reinterpret_cast(const wxUint32 *, src); + const wxUint32 *inBuff = reinterpret_cast(src); const size_t inLen = srcLen / BYTES_PER_CHAR; size_t outLen = 0; for ( size_t n = 0; n < inLen; n++ ) @@ -1598,7 +1854,7 @@ wxMBConvUTF32straight::FromWChar(char *dst, size_t dstLen, return srcLen * BYTES_PER_CHAR; } - wxUint32 *outBuff = wx_reinterpret_cast(wxUint32 *, dst); + wxUint32 *outBuff = reinterpret_cast(dst); size_t outLen = 0; for ( const wchar_t * const srcEnd = src + srcLen; src < srcEnd; ) { @@ -1629,7 +1885,7 @@ wxMBConvUTF32swap::ToWChar(wchar_t *dst, size_t dstLen, if ( srcLen == wxNO_LEN ) return wxCONV_FAILED; - const wxUint32 *inBuff = wx_reinterpret_cast(const wxUint32 *, src); + const wxUint32 *inBuff = reinterpret_cast(src); const size_t inLen = srcLen / BYTES_PER_CHAR; size_t outLen = 0; for ( size_t n = 0; n < inLen; n++, inBuff++ ) @@ -1676,7 +1932,7 @@ wxMBConvUTF32swap::FromWChar(char *dst, size_t dstLen, return srcLen*BYTES_PER_CHAR; } - wxUint32 *outBuff = wx_reinterpret_cast(wxUint32 *, dst); + wxUint32 *outBuff = reinterpret_cast(dst); size_t outLen = 0; for ( const wchar_t * const srcEnd = src + srcLen; src < srcEnd; ) { @@ -1761,7 +2017,7 @@ wxMBConvUTF32swap::ToWChar(wchar_t *dst, size_t dstLen, if ( dstLen < srcLen ) return wxCONV_FAILED; - const wxUint32 *inBuff = wx_reinterpret_cast(const wxUint32 *, src); + const wxUint32 *inBuff = reinterpret_cast(src); for ( size_t n = 0; n < srcLen; n++, inBuff++ ) { *dst++ = wxUINT32_SWAP_ALWAYS(*inBuff); @@ -1785,7 +2041,7 @@ wxMBConvUTF32swap::FromWChar(char *dst, size_t dstLen, if ( dstLen < srcLen ) return wxCONV_FAILED; - wxUint32 *outBuff = wx_reinterpret_cast(wxUint32 *, dst); + wxUint32 *outBuff = reinterpret_cast(dst); for ( size_t n = 0; n < srcLen; n += BYTES_PER_CHAR, src++ ) { *outBuff++ = wxUINT32_SWAP_ALWAYS(*src); @@ -1844,10 +2100,11 @@ public: wxMBConv_iconv(const char *name); virtual ~wxMBConv_iconv(); - virtual size_t MB2WC(wchar_t *buf, const char *psz, size_t n) const; - virtual size_t WC2MB(char *buf, const wchar_t *psz, size_t n) const; - - // classify this encoding as explained in wxMBConv::GetMBNulLen() comment + // implement base class virtual methods + virtual size_t ToWChar(wchar_t *dst, size_t dstLen, + const char *src, size_t srcLen = wxNO_LEN) const; + virtual size_t FromWChar(char *dst, size_t dstLen, + const wchar_t *src, size_t srcLen = wxNO_LEN) const; virtual size_t GetMBNulLen() const; #if wxUSE_UNICODE_UTF8 @@ -1856,7 +2113,7 @@ public: virtual wxMBConv *Clone() const { - wxMBConv_iconv *p = new wxMBConv_iconv(m_name.ToAscii()); + wxMBConv_iconv *p = new wxMBConv_iconv(m_name); p->m_minMBCharWidth = m_minMBCharWidth; return p; } @@ -1886,7 +2143,7 @@ private: // name of the encoding handled by this conversion - wxString m_name; + const char *m_name; // cached result of GetMBNulLen(); set to 0 meaning "unknown" // initially @@ -1910,28 +2167,28 @@ wxString wxMBConv_iconv::ms_wcCharsetName; bool wxMBConv_iconv::ms_wcNeedsSwap = false; wxMBConv_iconv::wxMBConv_iconv(const char *name) - : m_name(name) + : m_name(wxStrdup(name)) { m_minMBCharWidth = 0; // check for charset that represents wchar_t: if ( ms_wcCharsetName.empty() ) { - wxLogTrace(TRACE_STRCONV, _T("Looking for wide char codeset:")); + wxLogTrace(TRACE_STRCONV, wxT("Looking for wide char codeset:")); #if wxUSE_FONTMAP - const wxChar **names = wxFontMapperBase::GetAllEncodingNames(WC_ENC); + const wxChar *const *names = wxFontMapperBase::GetAllEncodingNames(WC_ENC); #else // !wxUSE_FONTMAP - static const wxChar *names_static[] = + static const wxChar *const names_static[] = { #if SIZEOF_WCHAR_T == 4 - _T("UCS-4"), -#elif SIZEOF_WCHAR_T = 2 - _T("UCS-2"), + wxT("UCS-4"), +#elif SIZEOF_WCHAR_T == 2 + wxT("UCS-2"), #endif NULL }; - const wxChar **names = names_static; + const wxChar *const *names = names_static; #endif // wxUSE_FONTMAP/!wxUSE_FONTMAP for ( ; *names && ms_wcCharsetName.empty(); ++names ) @@ -1942,19 +2199,19 @@ wxMBConv_iconv::wxMBConv_iconv(const char *name) wxString nameXE(nameCS); #ifdef WORDS_BIGENDIAN - nameXE += _T("BE"); + nameXE += wxT("BE"); #else // little endian - nameXE += _T("LE"); + nameXE += wxT("LE"); #endif - wxLogTrace(TRACE_STRCONV, _T(" trying charset \"%s\""), + wxLogTrace(TRACE_STRCONV, wxT(" trying charset \"%s\""), nameXE.c_str()); m2w = iconv_open(nameXE.ToAscii(), name); if ( m2w == ICONV_T_INVALID ) { // try charset w/o bytesex info (e.g. "UCS4") - wxLogTrace(TRACE_STRCONV, _T(" trying charset \"%s\""), + wxLogTrace(TRACE_STRCONV, wxT(" trying charset \"%s\""), nameCS.c_str()); m2w = iconv_open(nameCS.ToAscii(), name); @@ -1962,7 +2219,7 @@ wxMBConv_iconv::wxMBConv_iconv(const char *name) if ( m2w != ICONV_T_INVALID ) { char buf[2], *bufPtr; - wchar_t wbuf[2], *wbufPtr; + wchar_t wbuf[2]; size_t insz, outsz; size_t res; @@ -1971,12 +2228,12 @@ wxMBConv_iconv::wxMBConv_iconv(const char *name) wbuf[0] = 0; insz = 2; outsz = SIZEOF_WCHAR_T * 2; - wbufPtr = wbuf; + char* wbufPtr = (char*)wbuf; bufPtr = buf; res = iconv( m2w, ICONV_CHAR_CAST(&bufPtr), &insz, - (char**)&wbufPtr, &outsz); + &wbufPtr, &outsz); if (ICONV_FAILED(res, insz)) { @@ -2001,8 +2258,8 @@ wxMBConv_iconv::wxMBConv_iconv(const char *name) wxT("iconv wchar_t charset is \"%s\"%s"), ms_wcCharsetName.empty() ? wxString("") : ms_wcCharsetName, - ms_wcNeedsSwap ? _T(" (needs swap)") - : _T("")); + ms_wcNeedsSwap ? wxT(" (needs swap)") + : wxT("")); } else // we already have ms_wcCharsetName { @@ -2027,39 +2284,55 @@ wxMBConv_iconv::wxMBConv_iconv(const char *name) wxMBConv_iconv::~wxMBConv_iconv() { + free(const_cast(m_name)); + if ( m2w != ICONV_T_INVALID ) iconv_close(m2w); if ( w2m != ICONV_T_INVALID ) iconv_close(w2m); } -size_t wxMBConv_iconv::MB2WC(wchar_t *buf, const char *psz, size_t n) const +size_t +wxMBConv_iconv::ToWChar(wchar_t *dst, size_t dstLen, + const char *src, size_t srcLen) const { - // find the string length: notice that must be done differently for - // NUL-terminated strings and UTF-16/32 which are terminated with 2/4 NULs - size_t inbuf; - const size_t nulLen = GetMBNulLen(); - switch ( nulLen ) + if ( srcLen == wxNO_LEN ) { - default: - return wxCONV_FAILED; + // find the string length: notice that must be done differently for + // NUL-terminated strings and UTF-16/32 which are terminated with 2/4 + // consecutive NULs + const size_t nulLen = GetMBNulLen(); + switch ( nulLen ) + { + default: + return wxCONV_FAILED; - case 1: - inbuf = strlen(psz); // arguably more optimized than our version - break; + case 1: + srcLen = strlen(src); // arguably more optimized than our version + break; - case 2: - case 4: - // for UTF-16/32 not only we need to have 2/4 consecutive NULs but - // they also have to start at character boundary and not span two - // adjacent characters - const char *p; - for ( p = psz; NotAllNULs(p, nulLen); p += nulLen ) - ; - inbuf = p - psz; - break; + case 2: + case 4: + // for UTF-16/32 not only we need to have 2/4 consecutive NULs + // but they also have to start at character boundary and not + // span two adjacent characters + const char *p; + for ( p = src; NotAllNULs(p, nulLen); p += nulLen ) + ; + srcLen = p - src; + break; + } + + // when we're determining the length of the string ourselves we count + // the terminating NUL(s) as part of it and always NUL-terminate the + // output + srcLen += nulLen; } + // we express length in the number of (wide) characters but iconv always + // counts buffer sizes it in bytes + dstLen *= SIZEOF_WCHAR_T; + #if wxUSE_THREADS // NB: iconv() is MT-safe, but each thread must use its own iconv_t handle. // Unfortunately there are a couple of global wxCSConv objects such as @@ -2070,53 +2343,51 @@ size_t wxMBConv_iconv::MB2WC(wchar_t *buf, const char *psz, size_t n) const wxMutexLocker lock(wxConstCast(this, wxMBConv_iconv)->m_iconvMutex); #endif // wxUSE_THREADS - size_t outbuf = n * SIZEOF_WCHAR_T; size_t res, cres; - // VS: Use these instead of psz, buf because iconv() modifies its arguments: - wchar_t *bufPtr = buf; - const char *pszPtr = psz; + const char *pszPtr = src; - if (buf) + if ( dst ) { + char* bufPtr = (char*)dst; + // have destination buffer, convert there + size_t dstLenOrig = dstLen; cres = iconv(m2w, - ICONV_CHAR_CAST(&pszPtr), &inbuf, - (char**)&bufPtr, &outbuf); - res = n - (outbuf / SIZEOF_WCHAR_T); + ICONV_CHAR_CAST(&pszPtr), &srcLen, + &bufPtr, &dstLen); + + // convert the number of bytes converted as returned by iconv to the + // number of (wide) characters converted that we need + res = (dstLenOrig - dstLen) / SIZEOF_WCHAR_T; if (ms_wcNeedsSwap) { // convert to native endianness for ( unsigned i = 0; i < res; i++ ) - buf[n] = WC_BSWAP(buf[i]); + dst[i] = WC_BSWAP(dst[i]); } - - // NUL-terminate the string if there is any space left - if (res < n) - buf[res] = 0; } - else + else // no destination buffer { - // no destination buffer... convert using temp buffer - // to calculate destination buffer requirement - wchar_t tbuf[8]; + // convert using temp buffer to calculate the size of the buffer needed + wchar_t tbuf[256]; res = 0; do { - bufPtr = tbuf; - outbuf = 8 * SIZEOF_WCHAR_T; + char* bufPtr = (char*)tbuf; + dstLen = 8 * SIZEOF_WCHAR_T; cres = iconv(m2w, - ICONV_CHAR_CAST(&pszPtr), &inbuf, - (char**)&bufPtr, &outbuf ); + ICONV_CHAR_CAST(&pszPtr), &srcLen, + &bufPtr, &dstLen ); - res += 8 - (outbuf / SIZEOF_WCHAR_T); + res += 8 - (dstLen / SIZEOF_WCHAR_T); } while ((cres == (size_t)-1) && (errno == E2BIG)); } - if (ICONV_FAILED(cres, inbuf)) + if (ICONV_FAILED(cres, srcLen)) { //VS: it is ok if iconv fails, hence trace only wxLogTrace(TRACE_STRCONV, wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode())); @@ -2126,16 +2397,19 @@ size_t wxMBConv_iconv::MB2WC(wchar_t *buf, const char *psz, size_t n) const return res; } -size_t wxMBConv_iconv::WC2MB(char *buf, const wchar_t *psz, size_t n) const +size_t wxMBConv_iconv::FromWChar(char *dst, size_t dstLen, + const wchar_t *src, size_t srcLen) const { #if wxUSE_THREADS // NB: explained in MB2WC wxMutexLocker lock(wxConstCast(this, wxMBConv_iconv)->m_iconvMutex); #endif - size_t inlen = wxWcslen(psz); - size_t inbuf = inlen * SIZEOF_WCHAR_T; - size_t outbuf = n; + if ( srcLen == wxNO_LEN ) + srcLen = wxWcslen(src) + 1; + + size_t inbuflen = srcLen * SIZEOF_WCHAR_T; + size_t outbuflen = dstLen; size_t res, cres; wchar_t *tmpbuf = 0; @@ -2143,43 +2417,36 @@ size_t wxMBConv_iconv::WC2MB(char *buf, const wchar_t *psz, size_t n) const if (ms_wcNeedsSwap) { // need to copy to temp buffer to switch endianness - // (doing WC_BSWAP twice on the original buffer won't help, as it + // (doing WC_BSWAP twice on the original buffer won't work, as it // could be in read-only memory, or be accessed in some other thread) - tmpbuf = (wchar_t *)malloc(inbuf + SIZEOF_WCHAR_T); - for ( size_t i = 0; i < inlen; i++ ) - tmpbuf[n] = WC_BSWAP(psz[i]); + tmpbuf = (wchar_t *)malloc(inbuflen); + for ( size_t i = 0; i < srcLen; i++ ) + tmpbuf[i] = WC_BSWAP(src[i]); - tmpbuf[inlen] = L'\0'; - psz = tmpbuf; + src = tmpbuf; } - if (buf) + char* inbuf = (char*)src; + if ( dst ) { // have destination buffer, convert there - cres = iconv( w2m, ICONV_CHAR_CAST(&psz), &inbuf, &buf, &outbuf ); + cres = iconv(w2m, ICONV_CHAR_CAST(&inbuf), &inbuflen, &dst, &outbuflen); - res = n - outbuf; - - // NB: iconv was given only wcslen(psz) characters on input, and so - // it couldn't convert the trailing zero. Let's do it ourselves - // if there's some room left for it in the output buffer. - if (res < n) - buf[0] = 0; + res = dstLen - outbuflen; } - else + else // no destination buffer { - // no destination buffer: convert using temp buffer - // to calculate destination buffer requirement - char tbuf[16]; + // convert using temp buffer to calculate the size of the buffer needed + char tbuf[256]; res = 0; do { - buf = tbuf; - outbuf = 16; + dst = tbuf; + outbuflen = WXSIZEOF(tbuf); - cres = iconv( w2m, ICONV_CHAR_CAST(&psz), &inbuf, &buf, &outbuf ); + cres = iconv(w2m, ICONV_CHAR_CAST(&inbuf), &inbuflen, &dst, &outbuflen); - res += 16 - outbuf; + res += WXSIZEOF(tbuf) - outbuflen; } while ((cres == (size_t)-1) && (errno == E2BIG)); } @@ -2189,7 +2456,7 @@ size_t wxMBConv_iconv::WC2MB(char *buf, const wchar_t *psz, size_t n) const free(tmpbuf); } - if (ICONV_FAILED(cres, inbuf)) + if (ICONV_FAILED(cres, inbuflen)) { wxLogTrace(TRACE_STRCONV, wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode())); return wxCONV_FAILED; @@ -2411,26 +2678,38 @@ public: return wxCONV_FAILED; } - // if we were really converting, check if we succeeded - if ( buf ) + // we did something, check if we really succeeded + if ( flags ) { - if ( flags ) + // check if the conversion failed, i.e. if any replacements + // were done + if ( usedDef ) + return wxCONV_FAILED; + } + else // we must resort to double tripping... + { + // first we need to ensure that we really have the MB data: this is + // not the case if we're called with NULL buffer, in which case we + // need to do the conversion yet again + wxCharBuffer bufDef; + if ( !buf ) { - // check if the conversion failed, i.e. if any replacements - // were done - if ( usedDef ) + bufDef = wxCharBuffer(len); + buf = bufDef.data(); + if ( !::WideCharToMultiByte(m_CodePage, flags, pwz, -1, + buf, len, NULL, NULL) ) return wxCONV_FAILED; } - else // we must resort to double tripping... + + if ( !n ) + n = wcslen(pwz); + wxWCharBuffer wcBuf(n); + if ( MB2WC(wcBuf.data(), buf, n + 1) == wxCONV_FAILED || + wcscmp(wcBuf, pwz) != 0 ) { - wxWCharBuffer wcBuf(n); - if ( MB2WC(wcBuf.data(), buf, n) == wxCONV_FAILED || - wcscmp(wcBuf, pwz) != 0 ) - { - // we didn't obtain the same thing we started from, hence - // the conversion was lossy and we consider that it failed - return wxCONV_FAILED; - } + // we didn't obtain the same thing we started from, hence + // the conversion was lossy and we consider that it failed + return wxCONV_FAILED; } } @@ -2458,7 +2737,7 @@ public: switch ( len ) { default: - wxLogDebug(_T("Unexpected NUL length %d"), len); + wxLogDebug(wxT("Unexpected NUL length %d"), len); self->m_minMBCharWidth = (size_t)-1; break; @@ -2505,7 +2784,7 @@ private: break; } - wxASSERT_MSG( s_isWin98Or2k != -1, _T("should be set above") ); + wxASSERT_MSG( s_isWin98Or2k != -1, wxT("should be set above") ); } return s_isWin98Or2k == 1; @@ -2640,7 +2919,7 @@ private: // were we initialized successfully? bool m_ok; - DECLARE_NO_COPY_CLASS(wxMBConv_wxwin) + wxDECLARE_NO_COPY_CLASS(wxMBConv_wxwin); }; // make the constructors available for unit testing @@ -2666,7 +2945,41 @@ void wxCSConv::Init() { m_name = NULL; m_convReal = NULL; - m_deferred = true; +} + +void wxCSConv::SetEncoding(wxFontEncoding encoding) +{ + switch ( encoding ) + { + case wxFONTENCODING_MAX: + case wxFONTENCODING_SYSTEM: + if ( m_name ) + { + // It's ok to not have encoding value if we have a name for it. + m_encoding = wxFONTENCODING_SYSTEM; + } + else // No name neither. + { + // Fall back to the system default encoding in this case (not + // sure how much sense does this make but this is how the old + // code used to behave). +#if wxUSE_INTL + m_encoding = wxLocale::GetSystemEncoding(); + if ( m_encoding == wxFONTENCODING_SYSTEM ) +#endif // wxUSE_INTL + m_encoding = wxFONTENCODING_ISO8859_1; + } + break; + + case wxFONTENCODING_DEFAULT: + // wxFONTENCODING_DEFAULT is same as US-ASCII in this context + m_encoding = wxFONTENCODING_ISO8859_1; + break; + + default: + // Just use the provided encoding. + m_encoding = encoding; + } } wxCSConv::wxCSConv(const wxString& charset) @@ -2679,24 +2992,28 @@ wxCSConv::wxCSConv(const wxString& charset) } #if wxUSE_FONTMAP - m_encoding = wxFontMapperBase::GetEncodingFromName(charset); + SetEncoding(wxFontMapperBase::GetEncodingFromName(charset)); #else - m_encoding = wxFONTENCODING_SYSTEM; + SetEncoding(wxFONTENCODING_SYSTEM); #endif + + m_convReal = DoCreate(); } wxCSConv::wxCSConv(wxFontEncoding encoding) { if ( encoding == wxFONTENCODING_MAX || encoding == wxFONTENCODING_DEFAULT ) { - wxFAIL_MSG( _T("invalid encoding value in wxCSConv ctor") ); + wxFAIL_MSG( wxT("invalid encoding value in wxCSConv ctor") ); encoding = wxFONTENCODING_SYSTEM; } Init(); - m_encoding = encoding; + SetEncoding(encoding); + + m_convReal = DoCreate(); } wxCSConv::~wxCSConv() @@ -2710,7 +3027,9 @@ wxCSConv::wxCSConv(const wxCSConv& conv) Init(); SetName(conv.m_name); - m_encoding = conv.m_encoding; + SetEncoding(conv.m_encoding); + + m_convReal = DoCreate(); } wxCSConv& wxCSConv::operator=(const wxCSConv& conv) @@ -2718,7 +3037,9 @@ wxCSConv& wxCSConv::operator=(const wxCSConv& conv) Clear(); SetName(conv.m_name); - m_encoding = conv.m_encoding; + SetEncoding(conv.m_encoding); + + m_convReal = DoCreate(); return *this; } @@ -2726,19 +3047,15 @@ wxCSConv& wxCSConv::operator=(const wxCSConv& conv) void wxCSConv::Clear() { free(m_name); - delete m_convReal; - m_name = NULL; - m_convReal = NULL; + + wxDELETE(m_convReal); } void wxCSConv::SetName(const char *charset) { - if (charset) - { + if ( charset ) m_name = wxStrdup(charset); - m_deferred = true; - } } #if wxUSE_FONTMAP @@ -2761,8 +3078,7 @@ wxMBConv *wxCSConv::DoCreate() const // check for the special case of ASCII or ISO8859-1 charset: as we have // special knowledge of it anyhow, we don't need to create a special // conversion object - if ( m_encoding == wxFONTENCODING_ISO8859_1 || - m_encoding == wxFONTENCODING_DEFAULT ) + if ( m_encoding == wxFONTENCODING_ISO8859_1 ) { // don't convert at all return NULL; @@ -2814,7 +3130,7 @@ wxMBConv *wxCSConv::DoCreate() const delete conv; } - const wxChar** names = wxFontMapperBase::GetAllEncodingNames(encoding); + const wxChar* const* names = wxFontMapperBase::GetAllEncodingNames(encoding); // CS : in case this does not return valid names (eg for MacRoman) // encoding got a 'failure' entry in the cache all the same, // although it just has to be created using a different method, so @@ -2837,7 +3153,7 @@ wxMBConv *wxCSConv::DoCreate() const delete conv; } - gs_nameCache[encoding] = _T(""); // cache the failure + gs_nameCache[encoding] = wxT(""); // cache the failure } } #endif // wxUSE_FONTMAP @@ -2927,62 +3243,18 @@ wxMBConv *wxCSConv::DoCreate() const delete conv; } -#endif // wxUSE_FONTMAP - - // NB: This is a hack to prevent deadlock. What could otherwise happen - // in Unicode build: wxConvLocal creation ends up being here - // because of some failure and logs the error. But wxLog will try to - // attach a timestamp, for which it will need wxConvLocal (to convert - // time to char* and then wchar_t*), but that fails, tries to log the - // error, but wxLog has an (already locked) critical section that - // guards the static buffer. - static bool alreadyLoggingError = false; - if (!alreadyLoggingError) - { - alreadyLoggingError = true; - wxLogError(_("Cannot convert from the charset '%s'!"), - m_name ? m_name - : -#if wxUSE_FONTMAP - (const char*)wxFontMapperBase::GetEncodingDescription(m_encoding).ToAscii() -#else // !wxUSE_FONTMAP - (const char*)wxString::Format(_("encoding %i"), m_encoding).ToAscii() -#endif // wxUSE_FONTMAP/!wxUSE_FONTMAP - ); - alreadyLoggingError = false; - } + wxLogTrace(TRACE_STRCONV, + wxT("encoding \"%s\" is not supported by this system"), + (m_name ? wxString(m_name) + : wxFontMapperBase::GetEncodingName(m_encoding))); +#endif // wxUSE_FONTMAP return NULL; } -void wxCSConv::CreateConvIfNeeded() const -{ - if ( m_deferred ) - { - wxCSConv *self = (wxCSConv *)this; // const_cast - - // if we don't have neither the name nor the encoding, use the default - // encoding for this system - if ( !m_name && m_encoding == wxFONTENCODING_SYSTEM ) - { -#if wxUSE_INTL - self->m_encoding = wxLocale::GetSystemEncoding(); -#else - // fallback to some reasonable default: - self->m_encoding = wxFONTENCODING_ISO8859_1; -#endif // wxUSE_INTL - } - - self->m_convReal = DoCreate(); - self->m_deferred = false; - } -} - bool wxCSConv::IsOk() const { - CreateConvIfNeeded(); - // special case: no convReal created for wxFONTENCODING_ISO8859_1 if ( m_encoding == wxFONTENCODING_ISO8859_1 ) return true; // always ok as we do it ourselves @@ -2995,85 +3267,65 @@ bool wxCSConv::IsOk() const size_t wxCSConv::ToWChar(wchar_t *dst, size_t dstLen, const char *src, size_t srcLen) const { - CreateConvIfNeeded(); - if (m_convReal) return m_convReal->ToWChar(dst, dstLen, src, srcLen); // latin-1 (direct) - return wxMBConv::ToWChar(dst, dstLen, src, srcLen); -} + if ( srcLen == wxNO_LEN ) + srcLen = strlen(src) + 1; // take trailing NUL too -size_t wxCSConv::FromWChar(char *dst, size_t dstLen, - const wchar_t *src, size_t srcLen) const -{ - CreateConvIfNeeded(); + if ( dst ) + { + if ( dstLen < srcLen ) + return wxCONV_FAILED; - if (m_convReal) - return m_convReal->FromWChar(dst, dstLen, src, srcLen); + for ( size_t n = 0; n < srcLen; n++ ) + dst[n] = (unsigned char)(src[n]); + } - // latin-1 (direct) - return wxMBConv::FromWChar(dst, dstLen, src, srcLen); + return srcLen; } -size_t wxCSConv::MB2WC(wchar_t *buf, const char *psz, size_t n) const +size_t wxCSConv::FromWChar(char *dst, size_t dstLen, + const wchar_t *src, size_t srcLen) const { - CreateConvIfNeeded(); - if (m_convReal) - return m_convReal->MB2WC(buf, psz, n); + return m_convReal->FromWChar(dst, dstLen, src, srcLen); // latin-1 (direct) - size_t len = strlen(psz); + if ( srcLen == wxNO_LEN ) + srcLen = wxWcslen(src) + 1; - if (buf) + if ( dst ) { - for (size_t c = 0; c <= len; c++) - buf[c] = (unsigned char)(psz[c]); - } - - return len; -} - -size_t wxCSConv::WC2MB(char *buf, const wchar_t *psz, size_t n) const -{ - CreateConvIfNeeded(); - - if (m_convReal) - return m_convReal->WC2MB(buf, psz, n); + if ( dstLen < srcLen ) + return wxCONV_FAILED; - // latin-1 (direct) - const size_t len = wxWcslen(psz); - if (buf) - { - for (size_t c = 0; c <= len; c++) + for ( size_t n = 0; n < srcLen; n++ ) { - if (psz[c] > 0xFF) + if ( src[n] > 0xFF ) return wxCONV_FAILED; - buf[c] = (char)psz[c]; + dst[n] = (char)src[n]; } + } - else + else // still need to check the input validity { - for (size_t c = 0; c <= len; c++) + for ( size_t n = 0; n < srcLen; n++ ) { - if (psz[c] > 0xFF) + if ( src[n] > 0xFF ) return wxCONV_FAILED; } } - return len; + return srcLen; } size_t wxCSConv::GetMBNulLen() const { - CreateConvIfNeeded(); - if ( m_convReal ) - { return m_convReal->GetMBNulLen(); - } // otherwise, we are ISO-8859-1 return 1; @@ -3082,12 +3334,8 @@ size_t wxCSConv::GetMBNulLen() const #if wxUSE_UNICODE_UTF8 bool wxCSConv::IsUTF8() const { - CreateConvIfNeeded(); - if ( m_convReal ) - { return m_convReal->IsUTF8(); - } // otherwise, we are ISO-8859-1 return false; @@ -3157,14 +3405,27 @@ wxCharBuffer wxSafeConvertWX2MB(const wchar_t *ws) #define WX_DEFINE_GLOBAL_CONV(klass, name, ctor_args) \ WX_DEFINE_GLOBAL_CONV2(klass, klass, name, ctor_args) +#ifdef __INTELC__ + // disable warning "variable 'xxx' was declared but never referenced" + #pragma warning(disable: 177) +#endif // Intel C++ + #ifdef __WINDOWS__ WX_DEFINE_GLOBAL_CONV2(wxMBConv, wxMBConv_win32, wxConvLibc, wxEMPTY_PARAMETER_VALUE); +#elif 0 // defined(__WXOSX__) + WX_DEFINE_GLOBAL_CONV2(wxMBConv, wxMBConv_cf, wxConvLibc, (wxFONTENCODING_UTF8)); #else WX_DEFINE_GLOBAL_CONV2(wxMBConv, wxMBConvLibc, wxConvLibc, wxEMPTY_PARAMETER_VALUE); #endif -WX_DEFINE_GLOBAL_CONV(wxMBConvStrictUTF8, wxConvUTF8, wxEMPTY_PARAMETER_VALUE); -WX_DEFINE_GLOBAL_CONV(wxMBConvUTF7, wxConvUTF7, wxEMPTY_PARAMETER_VALUE); +// NB: we can't use wxEMPTY_PARAMETER_VALUE as final argument here because it's +// passed to WX_DEFINE_GLOBAL_CONV2 after a macro expansion and so still +// provokes an error message about "not enough macro parameters"; and we +// can't use "()" here as the name##Obj declaration would be parsed as a +// function declaration then, so use a semicolon and live with an extra +// empty statement (and hope that no compilers warns about this) +WX_DEFINE_GLOBAL_CONV(wxMBConvStrictUTF8, wxConvUTF8, ;); +WX_DEFINE_GLOBAL_CONV(wxMBConvUTF7, wxConvUTF7, ;); WX_DEFINE_GLOBAL_CONV(wxCSConv, wxConvLocal, (wxFONTENCODING_SYSTEM)); WX_DEFINE_GLOBAL_CONV(wxCSConv, wxConvISO8859_1, (wxFONTENCODING_ISO8859_1)); @@ -3173,8 +3434,9 @@ WXDLLIMPEXP_DATA_BASE(wxMBConv *) wxConvCurrent = wxGet_wxConvLibcPtr(); WXDLLIMPEXP_DATA_BASE(wxMBConv *) wxConvUI = wxGet_wxConvLocalPtr(); #ifdef __DARWIN__ -// The xnu kernel always communicates file paths in decomposed UTF-8. -// WARNING: Are we sure that CFString's conversion will cause decomposition? +// It is important to use this conversion object under Darwin as it ensures +// that Unicode strings are (re)composed correctly even though xnu kernel uses +// decomposed form internally (at least for the file names). static wxMBConv_cf wxConvMacUTF8DObj(wxFONTENCODING_UTF8); #endif @@ -3184,14 +3446,3 @@ WXDLLIMPEXP_DATA_BASE(wxMBConv *) wxConvFileName = #else // !__DARWIN__ wxGet_wxConvLibcPtr(); #endif // __DARWIN__/!__DARWIN__ - -#else // !wxUSE_WCHAR_T - -// FIXME-UTF8: remove this, wxUSE_WCHAR_T is required now -// stand-ins in absence of wchar_t -WXDLLIMPEXP_DATA_BASE(wxMBConv) wxConvLibc, - wxConvISO8859_1, - wxConvLocal, - wxConvUTF8; - -#endif // wxUSE_WCHAR_T/!wxUSE_WCHAR_T