fixed wxMBConv_iconv::GetMBNul()

[wxWidgets.git] / src / common / strconv.cpp

diff --git a/src/common/strconv.cpp b/src/common/strconv.cpp
index 63de22405cdd3213bf2ba09bba09adbffe278b64..960aa425663e830e92795366266632988b8f3dcc 100644 (file)
--- a/src/common/strconv.cpp
+++ b/src/common/strconv.cpp
@@ -1,5 +1,5 @@
 /////////////////////////////////////////////////////////////////////////////
 /////////////////////////////////////////////////////////////////////////////

-// Name:        strconv.cpp
+// Name:        src/common/strconv.cpp

 // Purpose:     Unicode conversion classes
 // Author:      Ove Kaaven, Robert Roebling, Vadim Zeitlin, Vaclav Slavik,
 //              Ryan Norton, Fredrik Roubert (UTF7)
 // Purpose:     Unicode conversion classes
 // Author:      Ove Kaaven, Robert Roebling, Vadim Zeitlin, Vaclav Slavik,
 //              Ryan Norton, Fredrik Roubert (UTF7)


@@ -78,6 +78,10 @@
 
 #define TRACE_STRCONV _T("strconv")
 
 
 #define TRACE_STRCONV _T("strconv")
 

+#if SIZEOF_WCHAR_T == 2
+    #define WC_UTF16
+#endif
+

 // ============================================================================
 // implementation
 // ============================================================================
 // ============================================================================
 // implementation
 // ============================================================================


@@ -183,129 +187,161 @@ const wxCharBuffer wxMBConv::cWC2MB(const wchar_t *pwz) const
     return buf;
 }
 
     return buf;
 }
 

-const wxWCharBuffer wxMBConv::cMB2WC(const char *szString, size_t nStringLen, size_t* pOutSize) const
+const wxWCharBuffer
+wxMBConv::cMB2WC(const char *in, size_t inLen, size_t *outLen) const

 {
 {

-    wxASSERT(pOutSize != NULL);
+    // the currently accumulated wide characters
+    wxWCharBuffer wbuf;

 
 

-    const char* szEnd = szString + nStringLen + 1;
-    const char* szPos = szString;
-    const char* szStart = szPos;
+    // the current length of wbuf
+    size_t lenBuf = 0;

 
 

-    size_t nActualLength = 0;
-    size_t nCurrentSize = nStringLen; //try normal size first (should never resize?)
+    // we need to know the representation of L'\0' for this conversion
+    size_t nulLen;
+    const char * const nul = GetMBNul(&nulLen);
+    if ( nulLen == (size_t)-1 || nulLen == 0 )
+        return wxWCharBuffer();

 
 

-    wxWCharBuffer theBuffer(nCurrentSize);
+    // make a copy of the input string unless it is already properly
+    // NUL-terminated
+    wxCharBuffer bufTmp;

 
 

-    //Convert the string until the length() is reached, continuing the
-    //loop every time a null character is reached
-    while(szPos != szEnd)
+    // now we can compute the input size if we were not given it: notice that
+    // in this case the string must be properly NUL-terminated, of course, as
+    // otherwise we have no way of knowing how long it is
+    if ( inLen == (size_t)-1 )

     {
     {

-        wxASSERT(szPos < szEnd); //something is _really_ screwed up if this rings true
-
-        //Get the length of the current (sub)string
-        size_t nLen = MB2WC(NULL, szPos, 0);
+        // not the most efficient algorithm but it shouldn't matter as normally
+        // there are not many NULs in the string and so normally memcmp()
+        // should stop on the first character
+        const char *p = in;
+        while ( memcmp(p, nul, nulLen) != 0 )
+            p++;

 
 

-        //Invalid conversion?
-        if( nLen == (size_t)-1 )
+        inLen = p - in + nulLen;
+    }
+    else // we already have the size
+    {
+        // check if it's not already NUL-terminated too to avoid the copy
+        if ( inLen < nulLen || memcmp(in + inLen - nulLen, nul, nulLen) != 0 )

         {
         {

-            *pOutSize = 0;
-            theBuffer.data()[0u] = wxT('\0');
-            return theBuffer;
+            // make a copy in order to properly NUL-terminate the string
+            bufTmp = wxCharBuffer(inLen + nulLen - 1 /* 1 will be added */);
+            memcpy(bufTmp.data(), in, inLen);
+            memcpy(bufTmp.data() + inLen, nul, nulLen);

         }
         }

+    }

 
 

+    if ( bufTmp )
+        in = bufTmp;

 
 

-        //Increase the actual length (+1 for current null character)
-        nActualLength += nLen + 1;
-
-        //if buffer too big, realloc the buffer
-        if (nActualLength > (nCurrentSize+1))
+    for ( const char * const inEnd = in + inLen;; )
+    {
+        // try to convert the current chunk if anything left
+        size_t lenChunk = in < inEnd ? MB2WC(NULL, in, 0) : 0;
+        if ( lenChunk == 0 )

         {
         {

-            wxWCharBuffer theNewBuffer(nCurrentSize << 1);
-            memcpy(theNewBuffer.data(), theBuffer.data(), nCurrentSize * sizeof(wchar_t));
-            theBuffer = theNewBuffer;
-            nCurrentSize <<= 1;
-        }
+            // nothing left in the input string, conversion succeeded
+            if ( outLen )
+            {
+                // we shouldn't include the last NUL in the result length
+                *outLen = lenBuf ? lenBuf - 1 : 0;
+            }

 
 

-        //Convert the current (sub)string
-        if ( MB2WC(&theBuffer.data()[szPos - szStart], szPos, nLen + 1) == (size_t)-1 )
-        {
-            *pOutSize = 0;
-            theBuffer.data()[0u] = wxT('\0');
-            return theBuffer;
+            return wbuf;

         }
 
         }
 

-        //Increment to next (sub)string
-        //Note that we have to use strlen instead of nLen here
-        //because XX2XX gives us the size of the output buffer,
-        //which is not necessarily the length of the string
-        szPos += strlen(szPos) + 1;
+        if ( lenChunk == (size_t)-1 )
+            break;
+
+        const size_t lenBufNew = lenBuf + lenChunk;
+        if ( !wbuf.extend(lenBufNew) )
+            break;
+
+        lenChunk = MB2WC(wbuf.data() + lenBuf, in, lenChunk + 1 /* for NUL */);
+        if ( lenChunk == (size_t)-1 )
+            break;
+
+        // +! for the embedded NUL (if something follows)
+        lenBuf = lenBufNew + 1;
+
+        // advance the input pointer past the end of this chunk
+        while ( memcmp(in, nul, nulLen) != 0 )
+            in++;
+
+        in += nulLen; // skipping over its terminator as well

     }
 
     }
 

-    //success - return actual length and the buffer
-    *pOutSize = nActualLength;
-    return theBuffer;
+    // conversion failed
+    if ( outLen )
+        *outLen = 0;
+
+    return wxWCharBuffer();

 }
 
 }
 

-const wxCharBuffer wxMBConv::cWC2MB(const wchar_t *szString, size_t nStringLen, size_t* pOutSize) const
+const wxCharBuffer
+wxMBConv::cWC2MB(const wchar_t *in, size_t inLen, size_t *outLen) const

 {
 {

-    wxASSERT(pOutSize != NULL);
-
-    const wchar_t* szEnd = szString + nStringLen + 1;
-    const wchar_t* szPos = szString;
-    const wchar_t* szStart = szPos;
+    // the currently accumulated multibyte characters
+    wxCharBuffer buf;

 
 

-    size_t nActualLength = 0;
-    size_t nCurrentSize = nStringLen << 2; //try * 4 first
+    // the current length of buf
+    size_t lenBuf = 0;

 
 

-    wxCharBuffer theBuffer(nCurrentSize);
-
-    //Convert the string until the length() is reached, continuing the
-    //loop every time a null character is reached
-    while(szPos != szEnd)
+    // make a copy of the input string unless it is already properly
+    // NUL-terminated
+    //
+    // if we don't know its length we have no choice but to assume that it is,
+    // indeed, properly terminated
+    wxWCharBuffer bufTmp;
+    if ( inLen == (size_t)-1 )

     {
     {

-        wxASSERT(szPos < szEnd); //something is _really_ screwed up if this rings true
+        inLen = wxWcslen(in) + 1;
+    }
+    else if ( inLen != 0 && in[inLen - 1] != L'\0' )
+    {
+        // make a copy in order to properly NUL-terminate the string
+        bufTmp = wxWCharBuffer(inLen);
+        memcpy(bufTmp.data(), in, inLen*sizeof(wchar_t));
+    }

 
 

-        //Get the length of the current (sub)string
-        size_t nLen = WC2MB(NULL, szPos, 0);
+    if ( bufTmp )
+        in = bufTmp;

 
 

-        //Invalid conversion?
-        if( nLen == (size_t)-1 )
+    for ( const wchar_t * const inEnd = in + inLen;; )
+    {
+        // try to convert the current chunk, if anything left
+        size_t lenChunk = in < inEnd ? WC2MB(NULL, in, 0) : 0;
+        if ( lenChunk == 0 )

         {
         {

-            *pOutSize = 0;
-            theBuffer.data()[0u] = wxT('\0');
-            return theBuffer;
+            // nothing left in the input string, conversion succeeded
+            if ( outLen )
+                *outLen = lenBuf ? lenBuf - 1 : lenBuf;
+
+            return buf;

         }
 
         }
 

-        //Increase the actual length (+1 for current null character)
-        nActualLength += nLen + 1;
+        if ( lenChunk == (size_t)-1 )
+            break;

 
 

-        //if buffer too big, realloc the buffer
-        if (nActualLength > (nCurrentSize+1))
-        {
-            wxCharBuffer theNewBuffer(nCurrentSize << 1);
-            memcpy(theNewBuffer.data(), theBuffer.data(), nCurrentSize);
-            theBuffer = theNewBuffer;
-            nCurrentSize <<= 1;
-        }
+        const size_t lenBufNew = lenBuf + lenChunk;
+        if ( !buf.extend(lenBufNew) )
+            break;

 
 

-        //Convert the current (sub)string
-        if(WC2MB(&theBuffer.data()[szPos - szStart], szPos, nLen + 1) == (size_t)-1 )
-        {
-            *pOutSize = 0;
-            theBuffer.data()[0u] = wxT('\0');
-            return theBuffer;
-        }
+        lenChunk = WC2MB(buf.data() + lenBuf, in, lenChunk + 1 /* for NUL */);
+        if ( lenChunk == (size_t)-1 )
+            break;

 
 

-        //Increment to next (sub)string
-        //Note that we have to use wxWcslen instead of nLen here
-        //because XX2XX gives us the size of the output buffer,
-        //which is not necessarily the length of the string
-        szPos += wxWcslen(szPos) + 1;
+        // chunk successfully converted, go to the next one
+        in += wxWcslen(in) + 1 /* skip NUL too */;
+        lenBuf = lenBufNew + 1;

     }
 
     }
 

-    //success - return actual length and the buffer
-    *pOutSize = nActualLength;
-    return theBuffer;
+    // conversion failed
+    if ( outLen )
+        *outLen = 0;
+
+    return wxCharBuffer();

 }
 
 // ----------------------------------------------------------------------------
 }
 
 // ----------------------------------------------------------------------------


@@ -322,12 +358,12 @@ size_t wxMBConvLibc::WC2MB(char *buf, const wchar_t *psz, size_t n) const
     return wxWC2MB(buf, psz, n);
 }
 
     return wxWC2MB(buf, psz, n);
 }
 

-#ifdef __UNIX__
-

 // ----------------------------------------------------------------------------
 // wxConvBrokenFileNames
 // ----------------------------------------------------------------------------
 
 // ----------------------------------------------------------------------------
 // wxConvBrokenFileNames
 // ----------------------------------------------------------------------------
 

+#ifdef __UNIX__
+

 wxConvBrokenFileNames::wxConvBrokenFileNames(const wxChar *charset)
 {
     if ( !charset || wxStricmp(charset, _T("UTF-8")) == 0
 wxConvBrokenFileNames::wxConvBrokenFileNames(const wxChar *charset)
 {
     if ( !charset || wxStricmp(charset, _T("UTF-8")) == 0


@@ -337,23 +373,7 @@ wxConvBrokenFileNames::wxConvBrokenFileNames(const wxChar *charset)
         m_conv = new wxCSConv(charset);
 }
 
         m_conv = new wxCSConv(charset);
 }
 

-size_t
-wxConvBrokenFileNames::MB2WC(wchar_t *outputBuf,
-                             const char *psz,
-                             size_t outputSize) const
-{
-    return m_conv->MB2WC( outputBuf, psz, outputSize );
-}
-
-size_t
-wxConvBrokenFileNames::WC2MB(char *outputBuf,
-                             const wchar_t *psz,
-                             size_t outputSize) const
-{
-    return m_conv->WC2MB( outputBuf, psz, outputSize );
-}
-
-#endif
+#endif // __UNIX__

 
 // ----------------------------------------------------------------------------
 // UTF-7
 
 // ----------------------------------------------------------------------------
 // UTF-7


@@ -404,7 +424,7 @@ size_t wxMBConvUTF7::MB2WC(wchar_t *buf, const char *psz, size_t n) const
 {
     size_t len = 0;
 
 {
     size_t len = 0;
 

-    while (*psz && ((!buf) || (len < n)))
+    while ( *psz && (!buf || (len < n)) )

     {
         unsigned char cc = *psz++;
         if (cc != '+')
     {
         unsigned char cc = *psz++;
         if (cc != '+')


@@ -422,20 +442,19 @@ size_t wxMBConvUTF7::MB2WC(wchar_t *buf, const char *psz, size_t n) const
             len++;
             psz++;
         }
             len++;
             psz++;
         }

-        else
+        else // start of BASE64 encoded string

         {
         {

-            // BASE64 encoded string
-            bool lsb;
-            unsigned char c;
+            bool lsb, ok;

             unsigned int d, l;
             unsigned int d, l;

-            for (lsb = false, d = 0, l = 0;
-                (cc = utf7unb64[(unsigned char)*psz]) != 0xff; psz++)
+            for ( ok = lsb = false, d = 0, l = 0;
+                  (cc = utf7unb64[(unsigned char)*psz]) != 0xff;
+                  psz++ )

             {
                 d <<= 6;
                 d += cc;
                 for (l += 6; l >= 8; lsb = !lsb)
                 {
             {
                 d <<= 6;
                 d += cc;
                 for (l += 6; l >= 8; lsb = !lsb)
                 {

-                    c = (unsigned char)((d >> (l -= 8)) % 256);
+                    unsigned char c = (unsigned char)((d >> (l -= 8)) % 256);

                     if (lsb)
                     {
                         if (buf)
                     if (lsb)
                     {
                         if (buf)


@@ -443,16 +462,29 @@ size_t wxMBConvUTF7::MB2WC(wchar_t *buf, const char *psz, size_t n) const
                         len ++;
                     }
                     else
                         len ++;
                     }
                     else

+                    {

                         if (buf)
                             *buf = (wchar_t)(c << 8);
                         if (buf)
                             *buf = (wchar_t)(c << 8);

+                    }
+
+                    ok = true;

                 }
             }
                 }
             }

+
+            if ( !ok )
+            {
+                // in valid UTF7 we should have valid characters after '+'
+                return (size_t)-1;
+            }
+

             if (*psz == '-')
                 psz++;
         }
     }
             if (*psz == '-')
                 psz++;
         }
     }

-    if (buf && (len < n))
-        *buf = 0;
+
+    if ( buf && (len < n) )
+        *buf = '\0';
+

     return len;
 }
 
     return len;
 }
 


@@ -493,8 +525,6 @@ static const unsigned char utf7encode[128] =
 
 size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t *psz, size_t n) const
 {
 
 size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t *psz, size_t n) const
 {

-
-

     size_t len = 0;
 
     while (*psz && ((!buf) || (len < n)))
     size_t len = 0;
 
     while (*psz && ((!buf) || (len < n)))


@@ -523,7 +553,7 @@ size_t wxMBConvUTF7::WC2MB(char *buf, const wchar_t *psz, size_t n) const
             {
                 // BASE64 encode string
                 unsigned int lsb, d, l;
             {
                 // BASE64 encode string
                 unsigned int lsb, d, l;

-                for (d = 0, l = 0;; psz++)
+                for (d = 0, l = 0; /*nothing*/; psz++)

                 {
                     for (lsb = 0; lsb < 2; lsb ++)
                     {
                 {
                     for (lsb = 0; lsb < 2; lsb ++)
                     {


@@ -653,7 +683,7 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const
                     }
 #else // !WC_UTF16
                     if (buf)
                     }
 #else // !WC_UTF16
                     if (buf)

-                        *buf++ = res;
+                        *buf++ = (wchar_t)res;

                     len++;
 #endif // WC_UTF16/!WC_UTF16
                 }
                     len++;
 #endif // WC_UTF16/!WC_UTF16
                 }


@@ -674,7 +704,7 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const
                         len += pa;
 #else
                         if (buf)
                         len += pa;
 #else
                         if (buf)

-                            *buf++ = wxUnicodePUA + (unsigned char)*opsz;
+                            *buf++ = (wchar_t)(wxUnicodePUA + (unsigned char)*opsz);

                         opsz++;
                         len++;
 #endif
                         opsz++;
                         len++;
 #endif


@@ -846,20 +876,24 @@ size_t wxMBConvUTF16straight::WC2MB(char *buf, const wchar_t *psz, size_t n) con
 // swap 16bit MB to 16bit String
 size_t wxMBConvUTF16swap::MB2WC(wchar_t *buf, const char *psz, size_t n) const
 {
 // swap 16bit MB to 16bit String
 size_t wxMBConvUTF16swap::MB2WC(wchar_t *buf, const char *psz, size_t n) const
 {

-    size_t len=0;
+    size_t len = 0;

 
 

-    while (*(wxUint16*)psz && (!buf || len < n))
+    // UTF16 string must be terminated by 2 NULs as single NULs may occur
+    // inside the string
+    while ( (psz[0] || psz[1]) && (!buf || len < n) )

     {
     {

-        if (buf)
+        if ( buf )

         {
             ((char *)buf)[0] = psz[1];
             ((char *)buf)[1] = psz[0];
             buf++;
         }
         len++;
         {
             ((char *)buf)[0] = psz[1];
             ((char *)buf)[1] = psz[0];
             buf++;
         }
         len++;

-        psz += sizeof(wxUint16);
+        psz += 2;

     }
     }

-    if (buf && len<n)   *buf=0;
+
+    if ( buf && len < n )
+        *buf = L'\0';

 
     return len;
 }
 
     return len;
 }


@@ -868,19 +902,21 @@ size_t wxMBConvUTF16swap::MB2WC(wchar_t *buf, const char *psz, size_t n) const
 // swap 16bit MB to 16bit String
 size_t wxMBConvUTF16swap::WC2MB(char *buf, const wchar_t *psz, size_t n) const
 {
 // swap 16bit MB to 16bit String
 size_t wxMBConvUTF16swap::WC2MB(char *buf, const wchar_t *psz, size_t n) const
 {

-    size_t len=0;
+    size_t len = 0;

 
 

-    while (*psz && (!buf || len < n))
+    while ( *psz && (!buf || len < n) )

     {
     {

-        if (buf)
+        if ( buf )

         {
             *buf++ = ((char*)psz)[1];
             *buf++ = ((char*)psz)[0];
         }
         {
             *buf++ = ((char*)psz)[1];
             *buf++ = ((char*)psz)[0];
         }

-        len += sizeof(wxUint16);
+        len += 2;

         psz++;
     }
         psz++;
     }

-    if (buf && len<=n-sizeof(wxUint16))   *(wxUint16*)buf=0;
+
+    if ( buf && len < n )
+        *buf = '\0';

 
     return len;
 }
 
     return len;
 }


@@ -902,7 +938,7 @@ size_t wxMBConvUTF16straight::MB2WC(wchar_t *buf, const char *psz, size_t n) con
             return pa;
 
         if (buf)
             return pa;
 
         if (buf)

-            *buf++ = cc;
+            *buf++ = (wchar_t)cc;

         len++;
         psz += pa * sizeof(wxUint16);
     }
         len++;
         psz += pa * sizeof(wxUint16);
     }


@@ -962,7 +998,7 @@ size_t wxMBConvUTF16swap::MB2WC(wchar_t *buf, const char *psz, size_t n) const
             return pa;
 
         if (buf)
             return pa;
 
         if (buf)

-            *buf++ = cc;
+            *buf++ = (wchar_t)cc;

 
         len++;
         psz += pa * sizeof(wxUint16);
 
         len++;
         psz += pa * sizeof(wxUint16);


@@ -1163,7 +1199,7 @@ size_t wxMBConvUTF32straight::MB2WC(wchar_t *buf, const char *psz, size_t n) con
     while (*(wxUint32*)psz && (!buf || len < n))
     {
         if (buf)
     while (*(wxUint32*)psz && (!buf || len < n))
     {
         if (buf)

-            *buf++ = *(wxUint32*)psz;
+            *buf++ = (wchar_t)(*(wxUint32*)psz);

         len++;
         psz += sizeof(wxUint32);
     }
         len++;
         psz += sizeof(wxUint32);
     }


@@ -1316,6 +1352,8 @@ protected:
 #endif
 
 private:
 #endif
 
 private:

+    virtual const char *GetMBNul(size_t *nulLen) const;
+

     // the name (for iconv_open()) of a wide char charset -- if none is
     // available on this machine, it will remain NULL
     static wxString ms_wcCharsetName;
     // the name (for iconv_open()) of a wide char charset -- if none is
     // available on this machine, it will remain NULL
     static wxString ms_wcCharsetName;


@@ -1323,6 +1361,10 @@ private:
     // true if the wide char encoding we use (i.e. ms_wcCharsetName) has
     // different endian-ness than the native one
     static bool ms_wcNeedsSwap;
     // true if the wide char encoding we use (i.e. ms_wcCharsetName) has
     // different endian-ness than the native one
     static bool ms_wcNeedsSwap;

+
+    // NUL representation
+    size_t m_nulLen;
+    char m_nulBuf[8];

 };
 
 // make the constructor available for unit testing
 };
 
 // make the constructor available for unit testing


@@ -1342,6 +1384,8 @@ bool wxMBConv_iconv::ms_wcNeedsSwap = false;
 
 wxMBConv_iconv::wxMBConv_iconv(const wxChar *name)
 {
 
 wxMBConv_iconv::wxMBConv_iconv(const wxChar *name)
 {

+    m_nulLen = (size_t)-2;
+

     // iconv operates with chars, not wxChars, but luckily it uses only ASCII
     // names for the charsets
     const wxCharBuffer cname(wxString(name).ToAscii());
     // iconv operates with chars, not wxChars, but luckily it uses only ASCII
     // names for the charsets
     const wxCharBuffer cname(wxString(name).ToAscii());


@@ -1349,6 +1393,8 @@ wxMBConv_iconv::wxMBConv_iconv(const wxChar *name)
     // check for charset that represents wchar_t:
     if ( ms_wcCharsetName.empty() )
     {
     // check for charset that represents wchar_t:
     if ( ms_wcCharsetName.empty() )
     {

+        wxLogTrace(TRACE_STRCONV, _T("Looking for wide char codeset:"));
+

 #if wxUSE_FONTMAP
         const wxChar **names = wxFontMapperBase::GetAllEncodingNames(WC_ENC);
 #else // !wxUSE_FONTMAP
 #if wxUSE_FONTMAP
         const wxChar **names = wxFontMapperBase::GetAllEncodingNames(WC_ENC);
 #else // !wxUSE_FONTMAP


@@ -1363,7 +1409,7 @@ wxMBConv_iconv::wxMBConv_iconv(const wxChar *name)
         };
 #endif // wxUSE_FONTMAP/!wxUSE_FONTMAP
 
         };
 #endif // wxUSE_FONTMAP/!wxUSE_FONTMAP
 

-        for ( ; *names; ++names )
+        for ( ; *names && ms_wcCharsetName.empty(); ++names )

         {
             const wxString nameCS(*names);
 
         {
             const wxString nameCS(*names);
 


@@ -1375,10 +1421,15 @@ wxMBConv_iconv::wxMBConv_iconv(const wxChar *name)
                 nameXE += _T("LE");
             #endif
 
                 nameXE += _T("LE");
             #endif
 

+            wxLogTrace(TRACE_STRCONV, _T("  trying charset \"%s\""),
+                       nameXE.c_str());
+

             m2w = iconv_open(nameXE.ToAscii(), cname);
             if ( m2w == ICONV_T_INVALID )
             {
                 // try charset w/o bytesex info (e.g. "UCS4")
             m2w = iconv_open(nameXE.ToAscii(), cname);
             if ( m2w == ICONV_T_INVALID )
             {
                 // try charset w/o bytesex info (e.g. "UCS4")

+                wxLogTrace(TRACE_STRCONV, _T("  trying charset \"%s\""),
+                           nameCS.c_str());

                 m2w = iconv_open(nameCS.ToAscii(), cname);
 
                 // and check for bytesex ourselves:
                 m2w = iconv_open(nameCS.ToAscii(), cname);
 
                 // and check for bytesex ourselves:


@@ -1584,7 +1635,6 @@ size_t wxMBConv_iconv::WC2MB(char *buf, const wchar_t *psz, size_t n) const
 
     if (ICONV_FAILED(cres, inbuf))
     {
 
     if (ICONV_FAILED(cres, inbuf))
     {

-        //VS: it is ok if iconv fails, hence trace only

         wxLogTrace(TRACE_STRCONV, wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode()));
         return (size_t)-1;
     }
         wxLogTrace(TRACE_STRCONV, wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode()));
         return (size_t)-1;
     }


@@ -1592,6 +1642,36 @@ size_t wxMBConv_iconv::WC2MB(char *buf, const wchar_t *psz, size_t n) const
     return res;
 }
 
     return res;
 }
 

+const char *wxMBConv_iconv::GetMBNul(size_t *nulLen) const
+{
+    if ( m_nulLen == (size_t)-2 )
+    {
+        wxMBConv_iconv * const self = wxConstCast(this, wxMBConv_iconv);
+
+#if wxUSE_THREADS
+        // NB: explained in MB2WC
+        wxMutexLocker lock(self->m_iconvMutex);
+#endif
+
+        wchar_t *wnul = L"";
+        size_t inLen = sizeof(wchar_t),
+               outLen = WXSIZEOF(m_nulBuf);
+        char *in = (char *)wnul,
+             *out = self->m_nulBuf;
+        if ( iconv(w2m, &in, &inLen, &out, &outLen) == (size_t)-1 )
+        {
+            self->m_nulLen = (size_t)-1;
+        }
+        else // ok
+        {
+            self->m_nulLen = out - m_nulBuf;
+        }
+    }
+
+    *nulLen = m_nulLen;
+    return m_nulBuf;
+}
+

 #endif // HAVE_ICONV
 
 
 #endif // HAVE_ICONV
 
 


@@ -1613,19 +1693,22 @@ public:
     wxMBConv_win32()
     {
         m_CodePage = CP_ACP;
     wxMBConv_win32()
     {
         m_CodePage = CP_ACP;

+        m_nulLen = (size_t)-2;

     }
 
 #if wxUSE_FONTMAP
     wxMBConv_win32(const wxChar* name)
     {
         m_CodePage = wxCharsetToCodepage(name);
     }
 
 #if wxUSE_FONTMAP
     wxMBConv_win32(const wxChar* name)
     {
         m_CodePage = wxCharsetToCodepage(name);

+        m_nulLen = (size_t)-2;

     }
 
     wxMBConv_win32(wxFontEncoding encoding)
     {
         m_CodePage = wxEncodingToCodepage(encoding);
     }
 
     wxMBConv_win32(wxFontEncoding encoding)
     {
         m_CodePage = wxEncodingToCodepage(encoding);

+        m_nulLen = (size_t)-2;

     }
     }

-#endif
+#endif // wxUSE_FONTMAP

 
     size_t MB2WC(wchar_t *buf, const char *psz, size_t n) const
     {
 
     size_t MB2WC(wchar_t *buf, const char *psz, size_t n) const
     {


@@ -1642,7 +1725,28 @@ public:
         // own wxMBConvUTF7 doesn't detect errors (e.g. lone "+" which is
         // explicitly ill-formed according to RFC 2152) neither so we don't
         // even have any fallback here...
         // own wxMBConvUTF7 doesn't detect errors (e.g. lone "+" which is
         // explicitly ill-formed according to RFC 2152) neither so we don't
         // even have any fallback here...

-        int flags = m_CodePage == CP_UTF7 ? 0 : MB_ERR_INVALID_CHARS;
+        //
+        // Moreover, MB_ERR_INVALID_CHARS is only supported on Win 2K SP4 or
+        // Win XP or newer and if it is specified on older versions, conversion
+        // from CP_UTF8 (which can have flags only 0 or MB_ERR_INVALID_CHARS)
+        // fails. So we can only use the flag on newer Windows versions.
+        // Additionally, the flag is not supported by UTF7, symbol and CJK
+        // encodings. See here:
+        //     http://blogs.msdn.com/michkap/archive/2005/04/19/409566.aspx
+        //     http://msdn.microsoft.com/library/en-us/intl/unicode_17si.asp
+        int flags = 0;
+        if ( m_CodePage != CP_UTF7 && m_CodePage != CP_SYMBOL &&
+             m_CodePage < 50000 &&
+             IsAtLeastWin2kSP4() )
+        {
+            flags = MB_ERR_INVALID_CHARS;
+        }
+        else if ( m_CodePage == CP_UTF8 )
+        {
+            // Avoid round-trip in the special case of UTF-8 by using our
+            // own UTF-8 conversion code:
+            return wxMBConvUTF8().MB2WC(buf, psz, n);
+        }

 
         const size_t len = ::MultiByteToWideChar
                              (
 
         const size_t len = ::MultiByteToWideChar
                              (


@@ -1653,11 +1757,41 @@ public:
                                 buf,            // output string
                                 buf ? n : 0     // size of output buffer
                              );
                                 buf,            // output string
                                 buf ? n : 0     // size of output buffer
                              );

+        if ( !len )
+        {
+            // function totally failed
+            return (size_t)-1;
+        }
+
+        // if we were really converting and didn't use MB_ERR_INVALID_CHARS,
+        // check if we succeeded, by doing a double trip:
+        if ( !flags && buf )
+        {
+            const size_t mbLen = strlen(psz);
+            wxCharBuffer mbBuf(mbLen);
+            if ( ::WideCharToMultiByte
+                   (
+                      m_CodePage,
+                      0,
+                      buf,
+                      -1,
+                      mbBuf.data(),
+                      mbLen + 1,        // size in bytes, not length
+                      NULL,
+                      NULL
+                   ) == 0 ||
+                  strcmp(mbBuf, psz) != 0 )
+            {
+                // we didn't obtain the same thing we started from, hence
+                // the conversion was lossy and we consider that it failed
+                return (size_t)-1;
+            }
+        }

 
         // note that it returns count of written chars for buf != NULL and size
         // of the needed buffer for buf == NULL so in either case the length of
         // the string (which never includes the terminating NUL) is one less
 
         // note that it returns count of written chars for buf != NULL and size
         // of the needed buffer for buf == NULL so in either case the length of
         // the string (which never includes the terminating NUL) is one less

-        return len ? len - 1 : (size_t)-1;
+        return len - 1;

     }
 
     size_t WC2MB(char *buf, const wchar_t *pwz, size_t n) const
     }
 
     size_t WC2MB(char *buf, const wchar_t *pwz, size_t n) const


@@ -1772,7 +1906,62 @@ private:
         return s_isWin98Or2k == 1;
     }
 
         return s_isWin98Or2k == 1;
     }
 

+    static bool IsAtLeastWin2kSP4()
+    {
+#ifdef __WXWINCE__
+        return false;
+#else
+        static int s_isAtLeastWin2kSP4 = -1;
+
+        if ( s_isAtLeastWin2kSP4 == -1 )
+        {
+            OSVERSIONINFOEX ver;
+
+            memset(&ver, 0, sizeof(ver));
+            ver.dwOSVersionInfoSize = sizeof(ver);
+            GetVersionEx((OSVERSIONINFO*)&ver);
+
+            s_isAtLeastWin2kSP4 =
+              ((ver.dwMajorVersion > 5) || // Vista+
+               (ver.dwMajorVersion == 5 && ver.dwMinorVersion > 0) || // XP/2003
+               (ver.dwMajorVersion == 5 && ver.dwMinorVersion == 0 &&
+               ver.wServicePackMajor >= 4)) // 2000 SP4+
+              ? 1 : 0;
+        }
+
+        return s_isAtLeastWin2kSP4 == 1;
+#endif
+    }
+
+    virtual const char *GetMBNul(size_t *nulLen) const
+    {
+        if ( m_nulLen == (size_t)-2 )
+        {
+            wxMBConv_win32 * const self = wxConstCast(this, wxMBConv_win32);
+
+            self->m_nulLen = ::WideCharToMultiByte
+                               (
+                                    m_CodePage,         // code page
+                                    0,                  // no flags
+                                    L"",                // input string
+                                    1,                  // translate just NUL
+                                    self->m_nulBuf,     // output buffer
+                                    WXSIZEOF(m_nulBuf), // and its size
+                                    NULL,               // "replacement" char
+                                    NULL                // [out] was it used?
+                               );
+
+            if ( m_nulLen == 0 )
+                self->m_nulLen = (size_t)-1;
+        }
+
+        *nulLen = m_nulLen;
+        return m_nulBuf;
+    }
+

     long m_CodePage;
     long m_CodePage;

+    size_t m_nulLen;
+    char m_nulBuf[8];

 };
 
 #endif // wxHAVE_WIN32_MB2WC
 };
 
 #endif // wxHAVE_WIN32_MB2WC


@@ -2412,6 +2601,27 @@ public:
     wxFontEncoding m_enc;
     wxEncodingConverter m2w, w2m;
 
     wxFontEncoding m_enc;
     wxEncodingConverter m2w, w2m;
 

+private:
+    virtual const char *GetMBNul(size_t *nulLen) const
+    {
+        switch ( m_enc )
+        {
+            case wxFONTENCODING_UTF16BE:
+            case wxFONTENCODING_UTF16LE:
+                *nulLen = 2;
+                return "\0";
+
+            case wxFONTENCODING_UTF32BE:
+            case wxFONTENCODING_UTF32LE:
+                *nulLen = 4;
+                return "\0\0\0";
+
+            default:
+                *nulLen = 1;
+                return "";
+        }
+    }
+

     // were we initialized successfully?
     bool m_ok;
 
     // were we initialized successfully?
     bool m_ok;
 


@@ -2452,7 +2662,11 @@ wxCSConv::wxCSConv(const wxChar *charset)
         SetName(charset);
     }
 
         SetName(charset);
     }
 

+#if wxUSE_FONTMAP
+    m_encoding = wxFontMapperBase::GetEncodingFromName(charset);
+#else

     m_encoding = wxFONTENCODING_SYSTEM;
     m_encoding = wxFONTENCODING_SYSTEM;

+#endif

 }
 
 wxCSConv::wxCSConv(wxFontEncoding encoding)
 }
 
 wxCSConv::wxCSConv(wxFontEncoding encoding)


@@ -2532,7 +2746,8 @@ wxMBConv *wxCSConv::DoCreate() const
     // check for the special case of ASCII or ISO8859-1 charset: as we have
     // special knowledge of it anyhow, we don't need to create a special
     // conversion object
     // check for the special case of ASCII or ISO8859-1 charset: as we have
     // special knowledge of it anyhow, we don't need to create a special
     // conversion object

-    if ( m_encoding == wxFONTENCODING_ISO8859_1 )
+    if ( m_encoding == wxFONTENCODING_ISO8859_1 ||
+            m_encoding == wxFONTENCODING_DEFAULT )

     {
         // don't convert at all
         return NULL;
     {
         // don't convert at all
         return NULL;


@@ -2799,6 +3014,20 @@ size_t wxCSConv::WC2MB(char *buf, const wchar_t *psz, size_t n) const
     return len;
 }
 
     return len;
 }
 

+const char *wxCSConv::GetMBNul(size_t *nulLen) const
+{
+    CreateConvIfNeeded();
+
+    if ( m_convReal )
+    {
+        // cast needed just to call private function of m_convReal
+        return ((wxCSConv *)m_convReal)->GetMBNul(nulLen);
+    }
+
+    *nulLen = 1;
+    return "";
+}
+

 // ----------------------------------------------------------------------------
 // globals
 // ----------------------------------------------------------------------------
 // ----------------------------------------------------------------------------
 // globals
 // ----------------------------------------------------------------------------


@@ -2839,5 +3068,3 @@ WXDLLIMPEXP_DATA_BASE(wxMBConv) wxConvLibc,
                                 wxConvUTF8;
 
 #endif // wxUSE_WCHAR_T/!wxUSE_WCHAR_T
                                 wxConvUTF8;
 
 #endif // wxUSE_WCHAR_T/!wxUSE_WCHAR_T

-
-