1 /////////////////////////////////////////////////////////////////////////////
2 // Name: src/common/strconv.cpp
3 // Purpose: Unicode conversion classes
4 // Author: Ove Kaaven, Robert Roebling, Vadim Zeitlin, Vaclav Slavik,
5 // Ryan Norton, Fredrik Roubert (UTF7)
9 // Copyright: (c) 1999 Ove Kaaven, Robert Roebling, Vaclav Slavik
10 // (c) 2000-2003 Vadim Zeitlin
11 // (c) 2004 Ryan Norton, Fredrik Roubert
12 // Licence: wxWindows licence
13 /////////////////////////////////////////////////////////////////////////////
15 // For compilers that support precompilation, includes "wx.h".
16 #include "wx/wxprec.h"
24 #include "wx/strconv.h"
29 #include "wx/msw/private.h"
30 #include "wx/msw/missing.h"
41 #if defined(__WIN32__) && !defined(__WXMICROWIN__)
42 #define wxHAVE_WIN32_MB2WC
51 #include "wx/thread.h"
54 #include "wx/encconv.h"
55 #include "wx/fontmap.h"
59 #include <ATSUnicode.h>
60 #include <TextCommon.h>
61 #include <TextEncodingConverter.h>
64 // includes Mac headers
65 #include "wx/mac/private.h"
69 #define TRACE_STRCONV _T("strconv")
71 // WC_UTF16 is defined only if sizeof(wchar_t) == 2, otherwise it's supposed to
73 #if SIZEOF_WCHAR_T == 2
78 // ============================================================================
80 // ============================================================================
82 // helper function of cMB2WC(): check if n bytes at this location are all NUL
83 static bool NotAllNULs(const char *p
, size_t n
)
85 while ( n
&& *p
++ == '\0' )
91 // ----------------------------------------------------------------------------
92 // UTF-16 en/decoding to/from UCS-4 with surrogates handling
93 // ----------------------------------------------------------------------------
95 static size_t encode_utf16(wxUint32 input
, wxUint16
*output
)
100 *output
= (wxUint16
) input
;
104 else if (input
>= 0x110000)
106 return wxCONV_FAILED
;
112 *output
++ = (wxUint16
) ((input
>> 10) + 0xd7c0);
113 *output
= (wxUint16
) ((input
& 0x3ff) + 0xdc00);
120 static size_t decode_utf16(const wxUint16
* input
, wxUint32
& output
)
122 if ((*input
< 0xd800) || (*input
> 0xdfff))
127 else if ((input
[1] < 0xdc00) || (input
[1] > 0xdfff))
130 return wxCONV_FAILED
;
134 output
= ((input
[0] - 0xd7c0) << 10) + (input
[1] - 0xdc00);
140 typedef wchar_t wxDecodeSurrogate_t
;
142 typedef wxUint16 wxDecodeSurrogate_t
;
143 #endif // WC_UTF16/!WC_UTF16
145 // returns the next UTF-32 character from the wchar_t buffer and advances the
146 // pointer to the character after this one
148 // if an invalid character is found, *pSrc is set to NULL, the caller must
150 static wxUint32
wxDecodeSurrogate(const wxDecodeSurrogate_t
**pSrc
)
154 n
= decode_utf16(wx_reinterpret_cast(const wxUint16
*, *pSrc
), out
);
155 if ( n
== wxCONV_FAILED
)
163 // ----------------------------------------------------------------------------
165 // ----------------------------------------------------------------------------
168 wxMBConv::ToWChar(wchar_t *dst
, size_t dstLen
,
169 const char *src
, size_t srcLen
) const
171 // although new conversion classes are supposed to implement this function
172 // directly, the existins ones only implement the old MB2WC() and so, to
173 // avoid to have to rewrite all conversion classes at once, we provide a
174 // default (but not efficient) implementation of this one in terms of the
175 // old function by copying the input to ensure that it's NUL-terminated and
176 // then using MB2WC() to convert it
178 // the number of chars [which would be] written to dst [if it were not NULL]
179 size_t dstWritten
= 0;
181 // the number of NULs terminating this string
182 size_t nulLen
= 0; // not really needed, but just to avoid warnings
184 // if we were not given the input size we just have to assume that the
185 // string is properly terminated as we have no way of knowing how long it
186 // is anyhow, but if we do have the size check whether there are enough
190 if ( srcLen
!= wxNO_LEN
)
192 // we need to know how to find the end of this string
193 nulLen
= GetMBNulLen();
194 if ( nulLen
== wxCONV_FAILED
)
195 return wxCONV_FAILED
;
197 // if there are enough NULs we can avoid the copy
198 if ( srcLen
< nulLen
|| NotAllNULs(src
+ srcLen
- nulLen
, nulLen
) )
200 // make a copy in order to properly NUL-terminate the string
201 bufTmp
= wxCharBuffer(srcLen
+ nulLen
- 1 /* 1 will be added */);
202 char * const p
= bufTmp
.data();
203 memcpy(p
, src
, srcLen
);
204 for ( char *s
= p
+ srcLen
; s
< p
+ srcLen
+ nulLen
; s
++ )
210 srcEnd
= src
+ srcLen
;
212 else // quit after the first loop iteration
219 // try to convert the current chunk
220 size_t lenChunk
= MB2WC(NULL
, src
, 0);
221 if ( lenChunk
== wxCONV_FAILED
)
222 return wxCONV_FAILED
;
224 lenChunk
++; // for the L'\0' at the end of this chunk
226 dstWritten
+= lenChunk
;
230 // nothing left in the input string, conversion succeeded
236 if ( dstWritten
> dstLen
)
237 return wxCONV_FAILED
;
239 if ( MB2WC(dst
, src
, lenChunk
) == wxCONV_FAILED
)
240 return wxCONV_FAILED
;
247 // we convert just one chunk in this case as this is the entire
252 // advance the input pointer past the end of this chunk
253 while ( NotAllNULs(src
, nulLen
) )
255 // notice that we must skip over multiple bytes here as we suppose
256 // that if NUL takes 2 or 4 bytes, then all the other characters do
257 // too and so if advanced by a single byte we might erroneously
258 // detect sequences of NUL bytes in the middle of the input
262 src
+= nulLen
; // skipping over its terminator as well
264 // note that ">=" (and not just "==") is needed here as the terminator
265 // we skipped just above could be inside or just after the buffer
266 // delimited by inEnd
275 wxMBConv::FromWChar(char *dst
, size_t dstLen
,
276 const wchar_t *src
, size_t srcLen
) const
278 // the number of chars [which would be] written to dst [if it were not NULL]
279 size_t dstWritten
= 0;
281 // make a copy of the input string unless it is already properly
284 // if we don't know its length we have no choice but to assume that it is,
285 // indeed, properly terminated
286 wxWCharBuffer bufTmp
;
287 if ( srcLen
== wxNO_LEN
)
289 srcLen
= wxWcslen(src
) + 1;
291 else if ( srcLen
!= 0 && src
[srcLen
- 1] != L
'\0' )
293 // make a copy in order to properly NUL-terminate the string
294 bufTmp
= wxWCharBuffer(srcLen
);
295 memcpy(bufTmp
.data(), src
, srcLen
* sizeof(wchar_t));
299 const size_t lenNul
= GetMBNulLen();
300 for ( const wchar_t * const srcEnd
= src
+ srcLen
;
302 src
+= wxWcslen(src
) + 1 /* skip L'\0' too */ )
304 // try to convert the current chunk
305 size_t lenChunk
= WC2MB(NULL
, src
, 0);
307 if ( lenChunk
== wxCONV_FAILED
)
308 return wxCONV_FAILED
;
311 dstWritten
+= lenChunk
;
315 if ( dstWritten
> dstLen
)
316 return wxCONV_FAILED
;
318 if ( WC2MB(dst
, src
, lenChunk
) == wxCONV_FAILED
)
319 return wxCONV_FAILED
;
328 size_t wxMBConv::MB2WC(wchar_t *outBuff
, const char *inBuff
, size_t outLen
) const
330 size_t rc
= ToWChar(outBuff
, outLen
, inBuff
);
331 if ( rc
!= wxCONV_FAILED
)
333 // ToWChar() returns the buffer length, i.e. including the trailing
334 // NUL, while this method doesn't take it into account
341 size_t wxMBConv::WC2MB(char *outBuff
, const wchar_t *inBuff
, size_t outLen
) const
343 size_t rc
= FromWChar(outBuff
, outLen
, inBuff
);
344 if ( rc
!= wxCONV_FAILED
)
352 wxMBConv::~wxMBConv()
354 // nothing to do here (necessary for Darwin linking probably)
357 const wxWCharBuffer
wxMBConv::cMB2WC(const char *psz
) const
361 // calculate the length of the buffer needed first
362 const size_t nLen
= MB2WC(NULL
, psz
, 0);
363 if ( nLen
!= wxCONV_FAILED
)
365 // now do the actual conversion
366 wxWCharBuffer
buf(nLen
/* +1 added implicitly */);
368 // +1 for the trailing NULL
369 if ( MB2WC(buf
.data(), psz
, nLen
+ 1) != wxCONV_FAILED
)
374 return wxWCharBuffer();
377 const wxCharBuffer
wxMBConv::cWC2MB(const wchar_t *pwz
) const
381 const size_t nLen
= WC2MB(NULL
, pwz
, 0);
382 if ( nLen
!= wxCONV_FAILED
)
384 // extra space for trailing NUL(s)
385 static const size_t extraLen
= GetMaxMBNulLen();
387 wxCharBuffer
buf(nLen
+ extraLen
- 1);
388 if ( WC2MB(buf
.data(), pwz
, nLen
+ extraLen
) != wxCONV_FAILED
)
393 return wxCharBuffer();
397 wxMBConv::cMB2WC(const char *inBuff
, size_t inLen
, size_t *outLen
) const
399 const size_t dstLen
= ToWChar(NULL
, 0, inBuff
, inLen
);
400 if ( dstLen
!= wxCONV_FAILED
)
402 wxWCharBuffer
wbuf(dstLen
- 1);
403 if ( ToWChar(wbuf
.data(), dstLen
, inBuff
, inLen
) != wxCONV_FAILED
)
408 if ( wbuf
[dstLen
- 1] == L
'\0' )
419 return wxWCharBuffer();
423 wxMBConv::cWC2MB(const wchar_t *inBuff
, size_t inLen
, size_t *outLen
) const
425 size_t dstLen
= FromWChar(NULL
, 0, inBuff
, inLen
);
426 if ( dstLen
!= wxCONV_FAILED
)
428 // special case of empty input: can't allocate 0 size buffer below as
429 // wxCharBuffer insists on NUL-terminating it
430 wxCharBuffer
buf(dstLen
? dstLen
- 1 : 1);
431 if ( FromWChar(buf
.data(), dstLen
, inBuff
, inLen
) != wxCONV_FAILED
)
437 const size_t nulLen
= GetMBNulLen();
438 if ( dstLen
>= nulLen
&&
439 !NotAllNULs(buf
.data() + dstLen
- nulLen
, nulLen
) )
441 // in this case the output is NUL-terminated and we're not
442 // supposed to count NUL
454 return wxCharBuffer();
457 // ----------------------------------------------------------------------------
459 // ----------------------------------------------------------------------------
461 size_t wxMBConvLibc::MB2WC(wchar_t *buf
, const char *psz
, size_t n
) const
463 return wxMB2WC(buf
, psz
, n
);
466 size_t wxMBConvLibc::WC2MB(char *buf
, const wchar_t *psz
, size_t n
) const
468 return wxWC2MB(buf
, psz
, n
);
471 // ----------------------------------------------------------------------------
472 // wxConvBrokenFileNames
473 // ----------------------------------------------------------------------------
477 wxConvBrokenFileNames::wxConvBrokenFileNames(const wxChar
*charset
)
479 if ( !charset
|| wxStricmp(charset
, _T("UTF-8")) == 0
480 || wxStricmp(charset
, _T("UTF8")) == 0 )
481 m_conv
= new wxMBConvUTF8(wxMBConvUTF8::MAP_INVALID_UTF8_TO_OCTAL
);
483 m_conv
= new wxCSConv(charset
);
488 // ----------------------------------------------------------------------------
490 // ----------------------------------------------------------------------------
492 // Implementation (C) 2004 Fredrik Roubert
495 // BASE64 decoding table
497 static const unsigned char utf7unb64
[] =
499 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
500 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
501 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
502 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
503 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
504 0xff, 0xff, 0xff, 0x3e, 0xff, 0xff, 0xff, 0x3f,
505 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b,
506 0x3c, 0x3d, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
507 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
508 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
509 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16,
510 0x17, 0x18, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff,
511 0xff, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20,
512 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28,
513 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30,
514 0x31, 0x32, 0x33, 0xff, 0xff, 0xff, 0xff, 0xff,
515 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
516 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
517 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
518 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
519 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
520 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
521 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
522 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
523 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
524 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
525 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
526 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
527 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
528 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
529 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
530 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
533 size_t wxMBConvUTF7::MB2WC(wchar_t *buf
, const char *psz
, size_t n
) const
537 while ( *psz
&& (!buf
|| (len
< n
)) )
539 unsigned char cc
= *psz
++;
547 else if (*psz
== '-')
555 else // start of BASE64 encoded string
559 for ( ok
= lsb
= false, d
= 0, l
= 0;
560 (cc
= utf7unb64
[(unsigned char)*psz
]) != 0xff;
565 for (l
+= 6; l
>= 8; lsb
= !lsb
)
567 unsigned char c
= (unsigned char)((d
>> (l
-= 8)) % 256);
577 *buf
= (wchar_t)(c
<< 8);
586 // in valid UTF7 we should have valid characters after '+'
587 return wxCONV_FAILED
;
595 if ( buf
&& (len
< n
) )
602 // BASE64 encoding table
604 static const unsigned char utf7enb64
[] =
606 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
607 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
608 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
609 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
610 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
611 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
612 'w', 'x', 'y', 'z', '0', '1', '2', '3',
613 '4', '5', '6', '7', '8', '9', '+', '/'
617 // UTF-7 encoding table
619 // 0 - Set D (directly encoded characters)
620 // 1 - Set O (optional direct characters)
621 // 2 - whitespace characters (optional)
622 // 3 - special characters
624 static const unsigned char utf7encode
[128] =
626 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 3, 3, 2, 3, 3,
627 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
628 2, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 3, 0, 0, 0, 3,
629 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0,
630 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
631 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 1, 1, 1,
632 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
633 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 3, 3
636 size_t wxMBConvUTF7::WC2MB(char *buf
, const wchar_t *psz
, size_t n
) const
640 while (*psz
&& ((!buf
) || (len
< n
)))
643 if (cc
< 0x80 && utf7encode
[cc
] < 1)
652 else if (((wxUint32
)cc
) > 0xffff)
654 // no surrogate pair generation (yet?)
655 return wxCONV_FAILED
;
666 // BASE64 encode string
667 unsigned int lsb
, d
, l
;
668 for (d
= 0, l
= 0; /*nothing*/; psz
++)
670 for (lsb
= 0; lsb
< 2; lsb
++)
673 d
+= lsb
? cc
& 0xff : (cc
& 0xff00) >> 8;
675 for (l
+= 8; l
>= 6; )
679 *buf
++ = utf7enb64
[(d
>> l
) % 64];
685 if (!(cc
) || (cc
< 0x80 && utf7encode
[cc
] < 1))
692 *buf
++ = utf7enb64
[((d
% 16) << (6 - l
)) % 64];
704 if (buf
&& (len
< n
))
710 // ----------------------------------------------------------------------------
712 // ----------------------------------------------------------------------------
714 static wxUint32 utf8_max
[]=
715 { 0x7f, 0x7ff, 0xffff, 0x1fffff, 0x3ffffff, 0x7fffffff, 0xffffffff };
717 // boundaries of the private use area we use to (temporarily) remap invalid
718 // characters invalid in a UTF-8 encoded string
719 const wxUint32 wxUnicodePUA
= 0x100000;
720 const wxUint32 wxUnicodePUAEnd
= wxUnicodePUA
+ 256;
722 size_t wxMBConvUTF8::MB2WC(wchar_t *buf
, const char *psz
, size_t n
) const
726 while (*psz
&& ((!buf
) || (len
< n
)))
728 const char *opsz
= psz
;
729 bool invalid
= false;
730 unsigned char cc
= *psz
++, fc
= cc
;
732 for (cnt
= 0; fc
& 0x80; cnt
++)
742 // escape the escape character for octal escapes
743 if ((m_options
& MAP_INVALID_UTF8_TO_OCTAL
)
744 && cc
== '\\' && (!buf
|| len
< n
))
756 // invalid UTF-8 sequence
761 unsigned ocnt
= cnt
- 1;
762 wxUint32 res
= cc
& (0x3f >> cnt
);
766 if ((cc
& 0xC0) != 0x80)
768 // invalid UTF-8 sequence
774 res
= (res
<< 6) | (cc
& 0x3f);
777 if (invalid
|| res
<= utf8_max
[ocnt
])
779 // illegal UTF-8 encoding
782 else if ((m_options
& MAP_INVALID_UTF8_TO_PUA
) &&
783 res
>= wxUnicodePUA
&& res
< wxUnicodePUAEnd
)
785 // if one of our PUA characters turns up externally
786 // it must also be treated as an illegal sequence
787 // (a bit like you have to escape an escape character)
793 // cast is ok because wchar_t == wxUuint16 if WC_UTF16
794 size_t pa
= encode_utf16(res
, (wxUint16
*)buf
);
795 if (pa
== wxCONV_FAILED
)
807 *buf
++ = (wchar_t)res
;
809 #endif // WC_UTF16/!WC_UTF16
815 if (m_options
& MAP_INVALID_UTF8_TO_PUA
)
817 while (opsz
< psz
&& (!buf
|| len
< n
))
820 // cast is ok because wchar_t == wxUuint16 if WC_UTF16
821 size_t pa
= encode_utf16((unsigned char)*opsz
+ wxUnicodePUA
, (wxUint16
*)buf
);
822 wxASSERT(pa
!= wxCONV_FAILED
);
829 *buf
++ = (wchar_t)(wxUnicodePUA
+ (unsigned char)*opsz
);
835 else if (m_options
& MAP_INVALID_UTF8_TO_OCTAL
)
837 while (opsz
< psz
&& (!buf
|| len
< n
))
839 if ( buf
&& len
+ 3 < n
)
841 unsigned char on
= *opsz
;
843 *buf
++ = (wchar_t)( L
'0' + on
/ 0100 );
844 *buf
++ = (wchar_t)( L
'0' + (on
% 0100) / 010 );
845 *buf
++ = (wchar_t)( L
'0' + on
% 010 );
852 else // MAP_INVALID_UTF8_NOT
854 return wxCONV_FAILED
;
860 if (buf
&& (len
< n
))
866 static inline bool isoctal(wchar_t wch
)
868 return L
'0' <= wch
&& wch
<= L
'7';
871 size_t wxMBConvUTF8::WC2MB(char *buf
, const wchar_t *psz
, size_t n
) const
875 while (*psz
&& ((!buf
) || (len
< n
)))
880 // cast is ok for WC_UTF16
881 size_t pa
= decode_utf16((const wxUint16
*)psz
, cc
);
882 psz
+= (pa
== wxCONV_FAILED
) ? 1 : pa
;
884 cc
= (*psz
++) & 0x7fffffff;
887 if ( (m_options
& MAP_INVALID_UTF8_TO_PUA
)
888 && cc
>= wxUnicodePUA
&& cc
< wxUnicodePUAEnd
)
891 *buf
++ = (char)(cc
- wxUnicodePUA
);
894 else if ( (m_options
& MAP_INVALID_UTF8_TO_OCTAL
)
895 && cc
== L
'\\' && psz
[0] == L
'\\' )
902 else if ( (m_options
& MAP_INVALID_UTF8_TO_OCTAL
) &&
904 isoctal(psz
[0]) && isoctal(psz
[1]) && isoctal(psz
[2]) )
908 *buf
++ = (char) ((psz
[0] - L
'0') * 0100 +
909 (psz
[1] - L
'0') * 010 +
919 for (cnt
= 0; cc
> utf8_max
[cnt
]; cnt
++)
935 *buf
++ = (char) ((-128 >> cnt
) | ((cc
>> (cnt
* 6)) & (0x3f >> cnt
)));
937 *buf
++ = (char) (0x80 | ((cc
>> (cnt
* 6)) & 0x3f));
943 if (buf
&& (len
< n
))
949 // ============================================================================
951 // ============================================================================
953 #ifdef WORDS_BIGENDIAN
954 #define wxMBConvUTF16straight wxMBConvUTF16BE
955 #define wxMBConvUTF16swap wxMBConvUTF16LE
957 #define wxMBConvUTF16swap wxMBConvUTF16BE
958 #define wxMBConvUTF16straight wxMBConvUTF16LE
962 size_t wxMBConvUTF16Base::GetLength(const char *src
, size_t srcLen
)
964 if ( srcLen
== wxNO_LEN
)
966 // count the number of bytes in input, including the trailing NULs
967 const wxUint16
*inBuff
= wx_reinterpret_cast(const wxUint16
*, src
);
968 for ( srcLen
= 1; *inBuff
++; srcLen
++ )
971 srcLen
*= BYTES_PER_CHAR
;
973 else // we already have the length
975 // we can only convert an entire number of UTF-16 characters
976 if ( srcLen
% BYTES_PER_CHAR
)
977 return wxCONV_FAILED
;
983 // case when in-memory representation is UTF-16 too
986 // ----------------------------------------------------------------------------
987 // conversions without endianness change
988 // ----------------------------------------------------------------------------
991 wxMBConvUTF16straight::ToWChar(wchar_t *dst
, size_t dstLen
,
992 const char *src
, size_t srcLen
) const
994 // set up the scene for using memcpy() (which is presumably more efficient
995 // than copying the bytes one by one)
996 srcLen
= GetLength(src
, srcLen
);
997 if ( srcLen
== wxNO_LEN
)
998 return wxCONV_FAILED
;
1000 const size_t inLen
= srcLen
/ BYTES_PER_CHAR
;
1003 if ( dstLen
< inLen
)
1004 return wxCONV_FAILED
;
1006 memcpy(dst
, src
, srcLen
);
1013 wxMBConvUTF16straight::FromWChar(char *dst
, size_t dstLen
,
1014 const wchar_t *src
, size_t srcLen
) const
1016 if ( srcLen
== wxNO_LEN
)
1017 srcLen
= wxWcslen(src
) + 1;
1019 srcLen
*= BYTES_PER_CHAR
;
1023 if ( dstLen
< srcLen
)
1024 return wxCONV_FAILED
;
1026 memcpy(dst
, src
, srcLen
);
1032 // ----------------------------------------------------------------------------
1033 // endian-reversing conversions
1034 // ----------------------------------------------------------------------------
1037 wxMBConvUTF16swap::ToWChar(wchar_t *dst
, size_t dstLen
,
1038 const char *src
, size_t srcLen
) const
1040 srcLen
= GetLength(src
, srcLen
);
1041 if ( srcLen
== wxNO_LEN
)
1042 return wxCONV_FAILED
;
1044 srcLen
/= BYTES_PER_CHAR
;
1048 if ( dstLen
< srcLen
)
1049 return wxCONV_FAILED
;
1051 const wxUint16
*inBuff
= wx_reinterpret_cast(const wxUint16
*, src
);
1052 for ( size_t n
= 0; n
< srcLen
; n
++, inBuff
++ )
1054 *dst
++ = wxUINT16_SWAP_ALWAYS(*inBuff
);
1062 wxMBConvUTF16swap::FromWChar(char *dst
, size_t dstLen
,
1063 const wchar_t *src
, size_t srcLen
) const
1065 if ( srcLen
== wxNO_LEN
)
1066 srcLen
= wxWcslen(src
) + 1;
1068 srcLen
*= BYTES_PER_CHAR
;
1072 if ( dstLen
< srcLen
)
1073 return wxCONV_FAILED
;
1075 wxUint16
*outBuff
= wx_reinterpret_cast(wxUint16
*, dst
);
1076 for ( size_t n
= 0; n
< srcLen
; n
+= BYTES_PER_CHAR
, src
++ )
1078 *outBuff
++ = wxUINT16_SWAP_ALWAYS(*src
);
1085 #else // !WC_UTF16: wchar_t is UTF-32
1087 // ----------------------------------------------------------------------------
1088 // conversions without endianness change
1089 // ----------------------------------------------------------------------------
1092 wxMBConvUTF16straight::ToWChar(wchar_t *dst
, size_t dstLen
,
1093 const char *src
, size_t srcLen
) const
1095 srcLen
= GetLength(src
, srcLen
);
1096 if ( srcLen
== wxNO_LEN
)
1097 return wxCONV_FAILED
;
1099 const size_t inLen
= srcLen
/ BYTES_PER_CHAR
;
1102 // optimization: return maximal space which could be needed for this
1103 // string even if the real size could be smaller if the buffer contains
1109 const wxUint16
*inBuff
= wx_reinterpret_cast(const wxUint16
*, src
);
1110 for ( const wxUint16
* const inEnd
= inBuff
+ inLen
; inBuff
< inEnd
; )
1112 const wxUint32 ch
= wxDecodeSurrogate(&inBuff
);
1114 return wxCONV_FAILED
;
1116 if ( ++outLen
> dstLen
)
1117 return wxCONV_FAILED
;
1127 wxMBConvUTF16straight::FromWChar(char *dst
, size_t dstLen
,
1128 const wchar_t *src
, size_t srcLen
) const
1130 if ( srcLen
== wxNO_LEN
)
1131 srcLen
= wxWcslen(src
) + 1;
1134 wxUint16
*outBuff
= wx_reinterpret_cast(wxUint16
*, dst
);
1135 for ( size_t n
= 0; n
< srcLen
; n
++ )
1138 const size_t numChars
= encode_utf16(*src
++, cc
);
1139 if ( numChars
== wxCONV_FAILED
)
1140 return wxCONV_FAILED
;
1142 outLen
+= numChars
* BYTES_PER_CHAR
;
1145 if ( outLen
> dstLen
)
1146 return wxCONV_FAILED
;
1149 if ( numChars
== 2 )
1151 // second character of a surrogate
1160 // ----------------------------------------------------------------------------
1161 // endian-reversing conversions
1162 // ----------------------------------------------------------------------------
1165 wxMBConvUTF16swap::ToWChar(wchar_t *dst
, size_t dstLen
,
1166 const char *src
, size_t srcLen
) const
1168 srcLen
= GetLength(src
, srcLen
);
1169 if ( srcLen
== wxNO_LEN
)
1170 return wxCONV_FAILED
;
1172 const size_t inLen
= srcLen
/ BYTES_PER_CHAR
;
1175 // optimization: return maximal space which could be needed for this
1176 // string even if the real size could be smaller if the buffer contains
1182 const wxUint16
*inBuff
= wx_reinterpret_cast(const wxUint16
*, src
);
1183 for ( const wxUint16
* const inEnd
= inBuff
+ inLen
; inBuff
< inEnd
; )
1188 tmp
[0] = wxUINT16_SWAP_ALWAYS(*inBuff
);
1190 tmp
[1] = wxUINT16_SWAP_ALWAYS(*inBuff
);
1192 const size_t numChars
= decode_utf16(tmp
, ch
);
1193 if ( numChars
== wxCONV_FAILED
)
1194 return wxCONV_FAILED
;
1196 if ( numChars
== 2 )
1199 if ( ++outLen
> dstLen
)
1200 return wxCONV_FAILED
;
1210 wxMBConvUTF16swap::FromWChar(char *dst
, size_t dstLen
,
1211 const wchar_t *src
, size_t srcLen
) const
1213 if ( srcLen
== wxNO_LEN
)
1214 srcLen
= wxWcslen(src
) + 1;
1217 wxUint16
*outBuff
= wx_reinterpret_cast(wxUint16
*, dst
);
1218 for ( const wchar_t *srcEnd
= src
+ srcLen
; src
< srcEnd
; src
++ )
1221 const size_t numChars
= encode_utf16(*src
, cc
);
1222 if ( numChars
== wxCONV_FAILED
)
1223 return wxCONV_FAILED
;
1225 outLen
+= numChars
* BYTES_PER_CHAR
;
1228 if ( outLen
> dstLen
)
1229 return wxCONV_FAILED
;
1231 *outBuff
++ = wxUINT16_SWAP_ALWAYS(cc
[0]);
1232 if ( numChars
== 2 )
1234 // second character of a surrogate
1235 *outBuff
++ = wxUINT16_SWAP_ALWAYS(cc
[1]);
1243 #endif // WC_UTF16/!WC_UTF16
1246 // ============================================================================
1248 // ============================================================================
1250 #ifdef WORDS_BIGENDIAN
1251 #define wxMBConvUTF32straight wxMBConvUTF32BE
1252 #define wxMBConvUTF32swap wxMBConvUTF32LE
1254 #define wxMBConvUTF32swap wxMBConvUTF32BE
1255 #define wxMBConvUTF32straight wxMBConvUTF32LE
1259 WXDLLIMPEXP_DATA_BASE(wxMBConvUTF32LE
) wxConvUTF32LE
;
1260 WXDLLIMPEXP_DATA_BASE(wxMBConvUTF32BE
) wxConvUTF32BE
;
1263 size_t wxMBConvUTF32Base::GetLength(const char *src
, size_t srcLen
)
1265 if ( srcLen
== wxNO_LEN
)
1267 // count the number of bytes in input, including the trailing NULs
1268 const wxUint32
*inBuff
= wx_reinterpret_cast(const wxUint32
*, src
);
1269 for ( srcLen
= 1; *inBuff
++; srcLen
++ )
1272 srcLen
*= BYTES_PER_CHAR
;
1274 else // we already have the length
1276 // we can only convert an entire number of UTF-32 characters
1277 if ( srcLen
% BYTES_PER_CHAR
)
1278 return wxCONV_FAILED
;
1284 // case when in-memory representation is UTF-16
1287 // ----------------------------------------------------------------------------
1288 // conversions without endianness change
1289 // ----------------------------------------------------------------------------
1292 wxMBConvUTF32straight::ToWChar(wchar_t *dst
, size_t dstLen
,
1293 const char *src
, size_t srcLen
) const
1295 srcLen
= GetLength(src
, srcLen
);
1296 if ( srcLen
== wxNO_LEN
)
1297 return wxCONV_FAILED
;
1299 const wxUint32
*inBuff
= wx_reinterpret_cast(const wxUint32
*, src
);
1300 const size_t inLen
= srcLen
/ BYTES_PER_CHAR
;
1302 for ( size_t n
= 0; n
< inLen
; n
++ )
1305 const size_t numChars
= encode_utf16(*inBuff
++, cc
);
1306 if ( numChars
== wxCONV_FAILED
)
1307 return wxCONV_FAILED
;
1312 if ( outLen
> dstLen
)
1313 return wxCONV_FAILED
;
1316 if ( numChars
== 2 )
1318 // second character of a surrogate
1328 wxMBConvUTF32straight::FromWChar(char *dst
, size_t dstLen
,
1329 const wchar_t *src
, size_t srcLen
) const
1331 if ( srcLen
== wxNO_LEN
)
1332 srcLen
= wxWcslen(src
) + 1;
1336 // optimization: return maximal space which could be needed for this
1337 // string instead of the exact amount which could be less if there are
1338 // any surrogates in the input
1340 // we consider that surrogates are rare enough to make it worthwhile to
1341 // avoid running the loop below at the cost of slightly extra memory
1343 return srcLen
* BYTES_PER_CHAR
;
1346 wxUint32
*outBuff
= wx_reinterpret_cast(wxUint32
*, dst
);
1348 for ( const wchar_t * const srcEnd
= src
+ srcLen
; src
< srcEnd
; )
1350 const wxUint32 ch
= wxDecodeSurrogate(&src
);
1352 return wxCONV_FAILED
;
1354 outLen
+= BYTES_PER_CHAR
;
1356 if ( outLen
> dstLen
)
1357 return wxCONV_FAILED
;
1365 // ----------------------------------------------------------------------------
1366 // endian-reversing conversions
1367 // ----------------------------------------------------------------------------
1370 wxMBConvUTF32swap::ToWChar(wchar_t *dst
, size_t dstLen
,
1371 const char *src
, size_t srcLen
) const
1373 srcLen
= GetLength(src
, srcLen
);
1374 if ( srcLen
== wxNO_LEN
)
1375 return wxCONV_FAILED
;
1377 const wxUint32
*inBuff
= wx_reinterpret_cast(const wxUint32
*, src
);
1378 const size_t inLen
= srcLen
/ BYTES_PER_CHAR
;
1380 for ( size_t n
= 0; n
< inLen
; n
++, inBuff
++ )
1383 const size_t numChars
= encode_utf16(wxUINT32_SWAP_ALWAYS(*inBuff
), cc
);
1384 if ( numChars
== wxCONV_FAILED
)
1385 return wxCONV_FAILED
;
1390 if ( outLen
> dstLen
)
1391 return wxCONV_FAILED
;
1394 if ( numChars
== 2 )
1396 // second character of a surrogate
1406 wxMBConvUTF32swap::FromWChar(char *dst
, size_t dstLen
,
1407 const wchar_t *src
, size_t srcLen
) const
1409 if ( srcLen
== wxNO_LEN
)
1410 srcLen
= wxWcslen(src
) + 1;
1414 // optimization: return maximal space which could be needed for this
1415 // string instead of the exact amount which could be less if there are
1416 // any surrogates in the input
1418 // we consider that surrogates are rare enough to make it worthwhile to
1419 // avoid running the loop below at the cost of slightly extra memory
1421 return srcLen
*BYTES_PER_CHAR
;
1424 wxUint32
*outBuff
= wx_reinterpret_cast(wxUint32
*, dst
);
1426 for ( const wchar_t * const srcEnd
= src
+ srcLen
; src
< srcEnd
; )
1428 const wxUint32 ch
= wxDecodeSurrogate(&src
);
1430 return wxCONV_FAILED
;
1432 outLen
+= BYTES_PER_CHAR
;
1434 if ( outLen
> dstLen
)
1435 return wxCONV_FAILED
;
1437 *outBuff
++ = wxUINT32_SWAP_ALWAYS(ch
);
1443 #else // !WC_UTF16: wchar_t is UTF-32
1445 // ----------------------------------------------------------------------------
1446 // conversions without endianness change
1447 // ----------------------------------------------------------------------------
1450 wxMBConvUTF32straight::ToWChar(wchar_t *dst
, size_t dstLen
,
1451 const char *src
, size_t srcLen
) const
1453 // use memcpy() as it should be much faster than hand-written loop
1454 srcLen
= GetLength(src
, srcLen
);
1455 if ( srcLen
== wxNO_LEN
)
1456 return wxCONV_FAILED
;
1458 const size_t inLen
= srcLen
/BYTES_PER_CHAR
;
1461 if ( dstLen
< inLen
)
1462 return wxCONV_FAILED
;
1464 memcpy(dst
, src
, srcLen
);
1471 wxMBConvUTF32straight::FromWChar(char *dst
, size_t dstLen
,
1472 const wchar_t *src
, size_t srcLen
) const
1474 if ( srcLen
== wxNO_LEN
)
1475 srcLen
= wxWcslen(src
) + 1;
1477 srcLen
*= BYTES_PER_CHAR
;
1481 if ( dstLen
< srcLen
)
1482 return wxCONV_FAILED
;
1484 memcpy(dst
, src
, srcLen
);
1490 // ----------------------------------------------------------------------------
1491 // endian-reversing conversions
1492 // ----------------------------------------------------------------------------
1495 wxMBConvUTF32swap::ToWChar(wchar_t *dst
, size_t dstLen
,
1496 const char *src
, size_t srcLen
) const
1498 srcLen
= GetLength(src
, srcLen
);
1499 if ( srcLen
== wxNO_LEN
)
1500 return wxCONV_FAILED
;
1502 srcLen
/= BYTES_PER_CHAR
;
1506 if ( dstLen
< srcLen
)
1507 return wxCONV_FAILED
;
1509 const wxUint32
*inBuff
= wx_reinterpret_cast(const wxUint32
*, src
);
1510 for ( size_t n
= 0; n
< srcLen
; n
++, inBuff
++ )
1512 *dst
++ = wxUINT32_SWAP_ALWAYS(*inBuff
);
1520 wxMBConvUTF32swap::FromWChar(char *dst
, size_t dstLen
,
1521 const wchar_t *src
, size_t srcLen
) const
1523 if ( srcLen
== wxNO_LEN
)
1524 srcLen
= wxWcslen(src
) + 1;
1526 srcLen
*= BYTES_PER_CHAR
;
1530 if ( dstLen
< srcLen
)
1531 return wxCONV_FAILED
;
1533 wxUint32
*outBuff
= wx_reinterpret_cast(wxUint32
*, dst
);
1534 for ( size_t n
= 0; n
< srcLen
; n
+= BYTES_PER_CHAR
, src
++ )
1536 *outBuff
++ = wxUINT32_SWAP_ALWAYS(*src
);
1543 #endif // WC_UTF16/!WC_UTF16
1546 // ============================================================================
1547 // The classes doing conversion using the iconv_xxx() functions
1548 // ============================================================================
1552 // VS: glibc 2.1.3 is broken in that iconv() conversion to/from UCS4 fails with
1553 // E2BIG if output buffer is _exactly_ as big as needed. Such case is
1554 // (unless there's yet another bug in glibc) the only case when iconv()
1555 // returns with (size_t)-1 (which means error) and says there are 0 bytes
1556 // left in the input buffer -- when _real_ error occurs,
1557 // bytes-left-in-input buffer is non-zero. Hence, this alternative test for
1559 // [This bug does not appear in glibc 2.2.]
1560 #if defined(__GLIBC__) && __GLIBC__ == 2 && __GLIBC_MINOR__ <= 1
1561 #define ICONV_FAILED(cres, bufLeft) ((cres == (size_t)-1) && \
1562 (errno != E2BIG || bufLeft != 0))
1564 #define ICONV_FAILED(cres, bufLeft) (cres == (size_t)-1)
1567 #define ICONV_CHAR_CAST(x) ((ICONV_CONST char **)(x))
1569 #define ICONV_T_INVALID ((iconv_t)-1)
1571 #if SIZEOF_WCHAR_T == 4
1572 #define WC_BSWAP wxUINT32_SWAP_ALWAYS
1573 #define WC_ENC wxFONTENCODING_UTF32
1574 #elif SIZEOF_WCHAR_T == 2
1575 #define WC_BSWAP wxUINT16_SWAP_ALWAYS
1576 #define WC_ENC wxFONTENCODING_UTF16
1577 #else // sizeof(wchar_t) != 2 nor 4
1578 // does this ever happen?
1579 #error "Unknown sizeof(wchar_t): please report this to wx-dev@lists.wxwindows.org"
1582 // ----------------------------------------------------------------------------
1583 // wxMBConv_iconv: encapsulates an iconv character set
1584 // ----------------------------------------------------------------------------
1586 class wxMBConv_iconv
: public wxMBConv
1589 wxMBConv_iconv(const wxChar
*name
);
1590 virtual ~wxMBConv_iconv();
1592 virtual size_t MB2WC(wchar_t *buf
, const char *psz
, size_t n
) const;
1593 virtual size_t WC2MB(char *buf
, const wchar_t *psz
, size_t n
) const;
1595 // classify this encoding as explained in wxMBConv::GetMBNulLen() comment
1596 virtual size_t GetMBNulLen() const;
1598 virtual wxMBConv
*Clone() const
1600 wxMBConv_iconv
*p
= new wxMBConv_iconv(m_name
);
1601 p
->m_minMBCharWidth
= m_minMBCharWidth
;
1606 { return (m2w
!= ICONV_T_INVALID
) && (w2m
!= ICONV_T_INVALID
); }
1609 // the iconv handlers used to translate from multibyte
1610 // to wide char and in the other direction
1615 // guards access to m2w and w2m objects
1616 wxMutex m_iconvMutex
;
1620 // the name (for iconv_open()) of a wide char charset -- if none is
1621 // available on this machine, it will remain NULL
1622 static wxString ms_wcCharsetName
;
1624 // true if the wide char encoding we use (i.e. ms_wcCharsetName) has
1625 // different endian-ness than the native one
1626 static bool ms_wcNeedsSwap
;
1629 // name of the encoding handled by this conversion
1632 // cached result of GetMBNulLen(); set to 0 meaning "unknown"
1634 size_t m_minMBCharWidth
;
1637 // make the constructor available for unit testing
1638 WXDLLIMPEXP_BASE wxMBConv
* new_wxMBConv_iconv( const wxChar
* name
)
1640 wxMBConv_iconv
* result
= new wxMBConv_iconv( name
);
1641 if ( !result
->IsOk() )
1650 wxString
wxMBConv_iconv::ms_wcCharsetName
;
1651 bool wxMBConv_iconv::ms_wcNeedsSwap
= false;
1653 wxMBConv_iconv::wxMBConv_iconv(const wxChar
*name
)
1656 m_minMBCharWidth
= 0;
1658 // iconv operates with chars, not wxChars, but luckily it uses only ASCII
1659 // names for the charsets
1660 const wxCharBuffer
cname(wxString(name
).ToAscii());
1662 // check for charset that represents wchar_t:
1663 if ( ms_wcCharsetName
.empty() )
1665 wxLogTrace(TRACE_STRCONV
, _T("Looking for wide char codeset:"));
1668 const wxChar
**names
= wxFontMapperBase::GetAllEncodingNames(WC_ENC
);
1669 #else // !wxUSE_FONTMAP
1670 static const wxChar
*names
[] =
1672 #if SIZEOF_WCHAR_T == 4
1674 #elif SIZEOF_WCHAR_T = 2
1679 #endif // wxUSE_FONTMAP/!wxUSE_FONTMAP
1681 for ( ; *names
&& ms_wcCharsetName
.empty(); ++names
)
1683 const wxString
nameCS(*names
);
1685 // first try charset with explicit bytesex info (e.g. "UCS-4LE"):
1686 wxString
nameXE(nameCS
);
1688 #ifdef WORDS_BIGENDIAN
1690 #else // little endian
1694 wxLogTrace(TRACE_STRCONV
, _T(" trying charset \"%s\""),
1697 m2w
= iconv_open(nameXE
.ToAscii(), cname
);
1698 if ( m2w
== ICONV_T_INVALID
)
1700 // try charset w/o bytesex info (e.g. "UCS4")
1701 wxLogTrace(TRACE_STRCONV
, _T(" trying charset \"%s\""),
1703 m2w
= iconv_open(nameCS
.ToAscii(), cname
);
1705 // and check for bytesex ourselves:
1706 if ( m2w
!= ICONV_T_INVALID
)
1708 char buf
[2], *bufPtr
;
1709 wchar_t wbuf
[2], *wbufPtr
;
1717 outsz
= SIZEOF_WCHAR_T
* 2;
1722 m2w
, ICONV_CHAR_CAST(&bufPtr
), &insz
,
1723 (char**)&wbufPtr
, &outsz
);
1725 if (ICONV_FAILED(res
, insz
))
1727 wxLogLastError(wxT("iconv"));
1728 wxLogError(_("Conversion to charset '%s' doesn't work."),
1731 else // ok, can convert to this encoding, remember it
1733 ms_wcCharsetName
= nameCS
;
1734 ms_wcNeedsSwap
= wbuf
[0] != (wchar_t)buf
[0];
1738 else // use charset not requiring byte swapping
1740 ms_wcCharsetName
= nameXE
;
1744 wxLogTrace(TRACE_STRCONV
,
1745 wxT("iconv wchar_t charset is \"%s\"%s"),
1746 ms_wcCharsetName
.empty() ? _T("<none>")
1747 : ms_wcCharsetName
.c_str(),
1748 ms_wcNeedsSwap
? _T(" (needs swap)")
1751 else // we already have ms_wcCharsetName
1753 m2w
= iconv_open(ms_wcCharsetName
.ToAscii(), cname
);
1756 if ( ms_wcCharsetName
.empty() )
1758 w2m
= ICONV_T_INVALID
;
1762 w2m
= iconv_open(cname
, ms_wcCharsetName
.ToAscii());
1763 if ( w2m
== ICONV_T_INVALID
)
1765 wxLogTrace(TRACE_STRCONV
,
1766 wxT("\"%s\" -> \"%s\" works but not the converse!?"),
1767 ms_wcCharsetName
.c_str(), cname
.data());
1772 wxMBConv_iconv::~wxMBConv_iconv()
1774 if ( m2w
!= ICONV_T_INVALID
)
1776 if ( w2m
!= ICONV_T_INVALID
)
1780 size_t wxMBConv_iconv::MB2WC(wchar_t *buf
, const char *psz
, size_t n
) const
1782 // find the string length: notice that must be done differently for
1783 // NUL-terminated strings and UTF-16/32 which are terminated with 2/4 NULs
1785 const size_t nulLen
= GetMBNulLen();
1789 return wxCONV_FAILED
;
1792 inbuf
= strlen(psz
); // arguably more optimized than our version
1797 // for UTF-16/32 not only we need to have 2/4 consecutive NULs but
1798 // they also have to start at character boundary and not span two
1799 // adjacent characters
1801 for ( p
= psz
; NotAllNULs(p
, nulLen
); p
+= nulLen
)
1808 // NB: iconv() is MT-safe, but each thread must use it's own iconv_t handle.
1809 // Unfortunately there is a couple of global wxCSConv objects such as
1810 // wxConvLocal that are used all over wx code, so we have to make sure
1811 // the handle is used by at most one thread at the time. Otherwise
1812 // only a few wx classes would be safe to use from non-main threads
1813 // as MB<->WC conversion would fail "randomly".
1814 wxMutexLocker
lock(wxConstCast(this, wxMBConv_iconv
)->m_iconvMutex
);
1815 #endif // wxUSE_THREADS
1817 size_t outbuf
= n
* SIZEOF_WCHAR_T
;
1819 // VS: Use these instead of psz, buf because iconv() modifies its arguments:
1820 wchar_t *bufPtr
= buf
;
1821 const char *pszPtr
= psz
;
1825 // have destination buffer, convert there
1827 ICONV_CHAR_CAST(&pszPtr
), &inbuf
,
1828 (char**)&bufPtr
, &outbuf
);
1829 res
= n
- (outbuf
/ SIZEOF_WCHAR_T
);
1833 // convert to native endianness
1834 for ( unsigned i
= 0; i
< res
; i
++ )
1835 buf
[n
] = WC_BSWAP(buf
[i
]);
1838 // NUL-terminate the string if there is any space left
1844 // no destination buffer... convert using temp buffer
1845 // to calculate destination buffer requirement
1852 outbuf
= 8 * SIZEOF_WCHAR_T
;
1855 ICONV_CHAR_CAST(&pszPtr
), &inbuf
,
1856 (char**)&bufPtr
, &outbuf
);
1858 res
+= 8 - (outbuf
/ SIZEOF_WCHAR_T
);
1860 while ((cres
== (size_t)-1) && (errno
== E2BIG
));
1863 if (ICONV_FAILED(cres
, inbuf
))
1865 //VS: it is ok if iconv fails, hence trace only
1866 wxLogTrace(TRACE_STRCONV
, wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode()));
1867 return wxCONV_FAILED
;
1873 size_t wxMBConv_iconv::WC2MB(char *buf
, const wchar_t *psz
, size_t n
) const
1876 // NB: explained in MB2WC
1877 wxMutexLocker
lock(wxConstCast(this, wxMBConv_iconv
)->m_iconvMutex
);
1880 size_t inlen
= wxWcslen(psz
);
1881 size_t inbuf
= inlen
* SIZEOF_WCHAR_T
;
1885 wchar_t *tmpbuf
= 0;
1889 // need to copy to temp buffer to switch endianness
1890 // (doing WC_BSWAP twice on the original buffer won't help, as it
1891 // could be in read-only memory, or be accessed in some other thread)
1892 tmpbuf
= (wchar_t *)malloc(inbuf
+ SIZEOF_WCHAR_T
);
1893 for ( size_t i
= 0; i
< inlen
; i
++ )
1894 tmpbuf
[n
] = WC_BSWAP(psz
[i
]);
1896 tmpbuf
[inlen
] = L
'\0';
1902 // have destination buffer, convert there
1903 cres
= iconv( w2m
, ICONV_CHAR_CAST(&psz
), &inbuf
, &buf
, &outbuf
);
1907 // NB: iconv was given only wcslen(psz) characters on input, and so
1908 // it couldn't convert the trailing zero. Let's do it ourselves
1909 // if there's some room left for it in the output buffer.
1915 // no destination buffer: convert using temp buffer
1916 // to calculate destination buffer requirement
1924 cres
= iconv( w2m
, ICONV_CHAR_CAST(&psz
), &inbuf
, &buf
, &outbuf
);
1928 while ((cres
== (size_t)-1) && (errno
== E2BIG
));
1936 if (ICONV_FAILED(cres
, inbuf
))
1938 wxLogTrace(TRACE_STRCONV
, wxT("iconv failed: %s"), wxSysErrorMsg(wxSysErrorCode()));
1939 return wxCONV_FAILED
;
1945 size_t wxMBConv_iconv::GetMBNulLen() const
1947 if ( m_minMBCharWidth
== 0 )
1949 wxMBConv_iconv
* const self
= wxConstCast(this, wxMBConv_iconv
);
1952 // NB: explained in MB2WC
1953 wxMutexLocker
lock(self
->m_iconvMutex
);
1956 wchar_t *wnul
= L
"";
1957 char buf
[8]; // should be enough for NUL in any encoding
1958 size_t inLen
= sizeof(wchar_t),
1959 outLen
= WXSIZEOF(buf
);
1960 char *inBuff
= (char *)wnul
;
1961 char *outBuff
= buf
;
1962 if ( iconv(w2m
, ICONV_CHAR_CAST(&inBuff
), &inLen
, &outBuff
, &outLen
) == (size_t)-1 )
1964 self
->m_minMBCharWidth
= (size_t)-1;
1968 self
->m_minMBCharWidth
= outBuff
- buf
;
1972 return m_minMBCharWidth
;
1975 #endif // HAVE_ICONV
1978 // ============================================================================
1979 // Win32 conversion classes
1980 // ============================================================================
1982 #ifdef wxHAVE_WIN32_MB2WC
1986 extern WXDLLIMPEXP_BASE
long wxCharsetToCodepage(const wxChar
*charset
);
1987 extern WXDLLIMPEXP_BASE
long wxEncodingToCodepage(wxFontEncoding encoding
);
1990 class wxMBConv_win32
: public wxMBConv
1995 m_CodePage
= CP_ACP
;
1996 m_minMBCharWidth
= 0;
1999 wxMBConv_win32(const wxMBConv_win32
& conv
)
2001 m_CodePage
= conv
.m_CodePage
;
2002 m_minMBCharWidth
= conv
.m_minMBCharWidth
;
2006 wxMBConv_win32(const wxChar
* name
)
2008 m_CodePage
= wxCharsetToCodepage(name
);
2009 m_minMBCharWidth
= 0;
2012 wxMBConv_win32(wxFontEncoding encoding
)
2014 m_CodePage
= wxEncodingToCodepage(encoding
);
2015 m_minMBCharWidth
= 0;
2017 #endif // wxUSE_FONTMAP
2019 virtual size_t MB2WC(wchar_t *buf
, const char *psz
, size_t n
) const
2021 // note that we have to use MB_ERR_INVALID_CHARS flag as it without it
2022 // the behaviour is not compatible with the Unix version (using iconv)
2023 // and break the library itself, e.g. wxTextInputStream::NextChar()
2024 // wouldn't work if reading an incomplete MB char didn't result in an
2027 // Moreover, MB_ERR_INVALID_CHARS is only supported on Win 2K SP4 or
2028 // Win XP or newer and it is not supported for UTF-[78] so we always
2029 // use our own conversions in this case. See
2030 // http://blogs.msdn.com/michkap/archive/2005/04/19/409566.aspx
2031 // http://msdn.microsoft.com/library/en-us/intl/unicode_17si.asp
2032 if ( m_CodePage
== CP_UTF8
)
2034 return wxConvUTF8
.MB2WC(buf
, psz
, n
);
2037 if ( m_CodePage
== CP_UTF7
)
2039 return wxConvUTF7
.MB2WC(buf
, psz
, n
);
2043 if ( (m_CodePage
< 50000 && m_CodePage
!= CP_SYMBOL
) &&
2044 IsAtLeastWin2kSP4() )
2046 flags
= MB_ERR_INVALID_CHARS
;
2049 const size_t len
= ::MultiByteToWideChar
2051 m_CodePage
, // code page
2052 flags
, // flags: fall on error
2053 psz
, // input string
2054 -1, // its length (NUL-terminated)
2055 buf
, // output string
2056 buf
? n
: 0 // size of output buffer
2060 // function totally failed
2061 return wxCONV_FAILED
;
2064 // if we were really converting and didn't use MB_ERR_INVALID_CHARS,
2065 // check if we succeeded, by doing a double trip:
2066 if ( !flags
&& buf
)
2068 const size_t mbLen
= strlen(psz
);
2069 wxCharBuffer
mbBuf(mbLen
);
2070 if ( ::WideCharToMultiByte
2077 mbLen
+ 1, // size in bytes, not length
2081 strcmp(mbBuf
, psz
) != 0 )
2083 // we didn't obtain the same thing we started from, hence
2084 // the conversion was lossy and we consider that it failed
2085 return wxCONV_FAILED
;
2089 // note that it returns count of written chars for buf != NULL and size
2090 // of the needed buffer for buf == NULL so in either case the length of
2091 // the string (which never includes the terminating NUL) is one less
2095 virtual size_t WC2MB(char *buf
, const wchar_t *pwz
, size_t n
) const
2098 we have a problem here: by default, WideCharToMultiByte() may
2099 replace characters unrepresentable in the target code page with bad
2100 quality approximations such as turning "1/2" symbol (U+00BD) into
2101 "1" for the code pages which don't have it and we, obviously, want
2102 to avoid this at any price
2104 the trouble is that this function does it _silently_, i.e. it won't
2105 even tell us whether it did or not... Win98/2000 and higher provide
2106 WC_NO_BEST_FIT_CHARS but it doesn't work for the older systems and
2107 we have to resort to a round trip, i.e. check that converting back
2108 results in the same string -- this is, of course, expensive but
2109 otherwise we simply can't be sure to not garble the data.
2112 // determine if we can rely on WC_NO_BEST_FIT_CHARS: according to MSDN
2113 // it doesn't work with CJK encodings (which we test for rather roughly
2114 // here...) nor with UTF-7/8 nor, of course, with Windows versions not
2116 BOOL usedDef
wxDUMMY_INITIALIZE(false);
2119 if ( CanUseNoBestFit() && m_CodePage
< 50000 )
2121 // it's our lucky day
2122 flags
= WC_NO_BEST_FIT_CHARS
;
2123 pUsedDef
= &usedDef
;
2125 else // old system or unsupported encoding
2131 const size_t len
= ::WideCharToMultiByte
2133 m_CodePage
, // code page
2134 flags
, // either none or no best fit
2135 pwz
, // input string
2136 -1, // it is (wide) NUL-terminated
2137 buf
, // output buffer
2138 buf
? n
: 0, // and its size
2139 NULL
, // default "replacement" char
2140 pUsedDef
// [out] was it used?
2145 // function totally failed
2146 return wxCONV_FAILED
;
2149 // if we were really converting, check if we succeeded
2154 // check if the conversion failed, i.e. if any replacements
2157 return wxCONV_FAILED
;
2159 else // we must resort to double tripping...
2161 wxWCharBuffer
wcBuf(n
);
2162 if ( MB2WC(wcBuf
.data(), buf
, n
) == wxCONV_FAILED
||
2163 wcscmp(wcBuf
, pwz
) != 0 )
2165 // we didn't obtain the same thing we started from, hence
2166 // the conversion was lossy and we consider that it failed
2167 return wxCONV_FAILED
;
2172 // see the comment above for the reason of "len - 1"
2176 virtual size_t GetMBNulLen() const
2178 if ( m_minMBCharWidth
== 0 )
2180 int len
= ::WideCharToMultiByte
2182 m_CodePage
, // code page
2184 L
"", // input string
2185 1, // translate just the NUL
2186 NULL
, // output buffer
2188 NULL
, // no replacement char
2189 NULL
// [out] don't care if it was used
2192 wxMBConv_win32
* const self
= wxConstCast(this, wxMBConv_win32
);
2196 wxLogDebug(_T("Unexpected NUL length %d"), len
);
2197 self
->m_minMBCharWidth
= (size_t)-1;
2201 self
->m_minMBCharWidth
= (size_t)-1;
2207 self
->m_minMBCharWidth
= len
;
2212 return m_minMBCharWidth
;
2215 virtual wxMBConv
*Clone() const { return new wxMBConv_win32(*this); }
2217 bool IsOk() const { return m_CodePage
!= -1; }
2220 static bool CanUseNoBestFit()
2222 static int s_isWin98Or2k
= -1;
2224 if ( s_isWin98Or2k
== -1 )
2227 switch ( wxGetOsVersion(&verMaj
, &verMin
) )
2230 s_isWin98Or2k
= verMaj
>= 4 && verMin
>= 10;
2234 s_isWin98Or2k
= verMaj
>= 5;
2238 // unknown: be conservative by default
2243 wxASSERT_MSG( s_isWin98Or2k
!= -1, _T("should be set above") );
2246 return s_isWin98Or2k
== 1;
2249 static bool IsAtLeastWin2kSP4()
2254 static int s_isAtLeastWin2kSP4
= -1;
2256 if ( s_isAtLeastWin2kSP4
== -1 )
2258 OSVERSIONINFOEX ver
;
2260 memset(&ver
, 0, sizeof(ver
));
2261 ver
.dwOSVersionInfoSize
= sizeof(ver
);
2262 GetVersionEx((OSVERSIONINFO
*)&ver
);
2264 s_isAtLeastWin2kSP4
=
2265 ((ver
.dwMajorVersion
> 5) || // Vista+
2266 (ver
.dwMajorVersion
== 5 && ver
.dwMinorVersion
> 0) || // XP/2003
2267 (ver
.dwMajorVersion
== 5 && ver
.dwMinorVersion
== 0 &&
2268 ver
.wServicePackMajor
>= 4)) // 2000 SP4+
2272 return s_isAtLeastWin2kSP4
== 1;
2277 // the code page we're working with
2280 // cached result of GetMBNulLen(), set to 0 initially meaning
2282 size_t m_minMBCharWidth
;
2285 #endif // wxHAVE_WIN32_MB2WC
2287 // ============================================================================
2288 // Cocoa conversion classes
2289 // ============================================================================
2291 #if defined(__WXCOCOA__)
2293 // RN: There is no UTF-32 support in either Core Foundation or Cocoa.
2294 // Strangely enough, internally Core Foundation uses
2295 // UTF-32 internally quite a bit - its just not public (yet).
2297 #include <CoreFoundation/CFString.h>
2298 #include <CoreFoundation/CFStringEncodingExt.h>
2300 CFStringEncoding
wxCFStringEncFromFontEnc(wxFontEncoding encoding
)
2302 CFStringEncoding enc
= kCFStringEncodingInvalidId
;
2306 case wxFONTENCODING_DEFAULT
:
2307 enc
= CFStringGetSystemEncoding();
2310 case wxFONTENCODING_ISO8859_1
:
2311 enc
= kCFStringEncodingISOLatin1
;
2313 case wxFONTENCODING_ISO8859_2
:
2314 enc
= kCFStringEncodingISOLatin2
;
2316 case wxFONTENCODING_ISO8859_3
:
2317 enc
= kCFStringEncodingISOLatin3
;
2319 case wxFONTENCODING_ISO8859_4
:
2320 enc
= kCFStringEncodingISOLatin4
;
2322 case wxFONTENCODING_ISO8859_5
:
2323 enc
= kCFStringEncodingISOLatinCyrillic
;
2325 case wxFONTENCODING_ISO8859_6
:
2326 enc
= kCFStringEncodingISOLatinArabic
;
2328 case wxFONTENCODING_ISO8859_7
:
2329 enc
= kCFStringEncodingISOLatinGreek
;
2331 case wxFONTENCODING_ISO8859_8
:
2332 enc
= kCFStringEncodingISOLatinHebrew
;
2334 case wxFONTENCODING_ISO8859_9
:
2335 enc
= kCFStringEncodingISOLatin5
;
2337 case wxFONTENCODING_ISO8859_10
:
2338 enc
= kCFStringEncodingISOLatin6
;
2340 case wxFONTENCODING_ISO8859_11
:
2341 enc
= kCFStringEncodingISOLatinThai
;
2343 case wxFONTENCODING_ISO8859_13
:
2344 enc
= kCFStringEncodingISOLatin7
;
2346 case wxFONTENCODING_ISO8859_14
:
2347 enc
= kCFStringEncodingISOLatin8
;
2349 case wxFONTENCODING_ISO8859_15
:
2350 enc
= kCFStringEncodingISOLatin9
;
2353 case wxFONTENCODING_KOI8
:
2354 enc
= kCFStringEncodingKOI8_R
;
2356 case wxFONTENCODING_ALTERNATIVE
: // MS-DOS CP866
2357 enc
= kCFStringEncodingDOSRussian
;
2360 // case wxFONTENCODING_BULGARIAN :
2364 case wxFONTENCODING_CP437
:
2365 enc
= kCFStringEncodingDOSLatinUS
;
2367 case wxFONTENCODING_CP850
:
2368 enc
= kCFStringEncodingDOSLatin1
;
2370 case wxFONTENCODING_CP852
:
2371 enc
= kCFStringEncodingDOSLatin2
;
2373 case wxFONTENCODING_CP855
:
2374 enc
= kCFStringEncodingDOSCyrillic
;
2376 case wxFONTENCODING_CP866
:
2377 enc
= kCFStringEncodingDOSRussian
;
2379 case wxFONTENCODING_CP874
:
2380 enc
= kCFStringEncodingDOSThai
;
2382 case wxFONTENCODING_CP932
:
2383 enc
= kCFStringEncodingDOSJapanese
;
2385 case wxFONTENCODING_CP936
:
2386 enc
= kCFStringEncodingDOSChineseSimplif
;
2388 case wxFONTENCODING_CP949
:
2389 enc
= kCFStringEncodingDOSKorean
;
2391 case wxFONTENCODING_CP950
:
2392 enc
= kCFStringEncodingDOSChineseTrad
;
2394 case wxFONTENCODING_CP1250
:
2395 enc
= kCFStringEncodingWindowsLatin2
;
2397 case wxFONTENCODING_CP1251
:
2398 enc
= kCFStringEncodingWindowsCyrillic
;
2400 case wxFONTENCODING_CP1252
:
2401 enc
= kCFStringEncodingWindowsLatin1
;
2403 case wxFONTENCODING_CP1253
:
2404 enc
= kCFStringEncodingWindowsGreek
;
2406 case wxFONTENCODING_CP1254
:
2407 enc
= kCFStringEncodingWindowsLatin5
;
2409 case wxFONTENCODING_CP1255
:
2410 enc
= kCFStringEncodingWindowsHebrew
;
2412 case wxFONTENCODING_CP1256
:
2413 enc
= kCFStringEncodingWindowsArabic
;
2415 case wxFONTENCODING_CP1257
:
2416 enc
= kCFStringEncodingWindowsBalticRim
;
2418 // This only really encodes to UTF7 (if that) evidently
2419 // case wxFONTENCODING_UTF7 :
2420 // enc = kCFStringEncodingNonLossyASCII ;
2422 case wxFONTENCODING_UTF8
:
2423 enc
= kCFStringEncodingUTF8
;
2425 case wxFONTENCODING_EUC_JP
:
2426 enc
= kCFStringEncodingEUC_JP
;
2428 case wxFONTENCODING_UTF16
:
2429 enc
= kCFStringEncodingUnicode
;
2431 case wxFONTENCODING_MACROMAN
:
2432 enc
= kCFStringEncodingMacRoman
;
2434 case wxFONTENCODING_MACJAPANESE
:
2435 enc
= kCFStringEncodingMacJapanese
;
2437 case wxFONTENCODING_MACCHINESETRAD
:
2438 enc
= kCFStringEncodingMacChineseTrad
;
2440 case wxFONTENCODING_MACKOREAN
:
2441 enc
= kCFStringEncodingMacKorean
;
2443 case wxFONTENCODING_MACARABIC
:
2444 enc
= kCFStringEncodingMacArabic
;
2446 case wxFONTENCODING_MACHEBREW
:
2447 enc
= kCFStringEncodingMacHebrew
;
2449 case wxFONTENCODING_MACGREEK
:
2450 enc
= kCFStringEncodingMacGreek
;
2452 case wxFONTENCODING_MACCYRILLIC
:
2453 enc
= kCFStringEncodingMacCyrillic
;
2455 case wxFONTENCODING_MACDEVANAGARI
:
2456 enc
= kCFStringEncodingMacDevanagari
;
2458 case wxFONTENCODING_MACGURMUKHI
:
2459 enc
= kCFStringEncodingMacGurmukhi
;
2461 case wxFONTENCODING_MACGUJARATI
:
2462 enc
= kCFStringEncodingMacGujarati
;
2464 case wxFONTENCODING_MACORIYA
:
2465 enc
= kCFStringEncodingMacOriya
;
2467 case wxFONTENCODING_MACBENGALI
:
2468 enc
= kCFStringEncodingMacBengali
;
2470 case wxFONTENCODING_MACTAMIL
:
2471 enc
= kCFStringEncodingMacTamil
;
2473 case wxFONTENCODING_MACTELUGU
:
2474 enc
= kCFStringEncodingMacTelugu
;
2476 case wxFONTENCODING_MACKANNADA
:
2477 enc
= kCFStringEncodingMacKannada
;
2479 case wxFONTENCODING_MACMALAJALAM
:
2480 enc
= kCFStringEncodingMacMalayalam
;
2482 case wxFONTENCODING_MACSINHALESE
:
2483 enc
= kCFStringEncodingMacSinhalese
;
2485 case wxFONTENCODING_MACBURMESE
:
2486 enc
= kCFStringEncodingMacBurmese
;
2488 case wxFONTENCODING_MACKHMER
:
2489 enc
= kCFStringEncodingMacKhmer
;
2491 case wxFONTENCODING_MACTHAI
:
2492 enc
= kCFStringEncodingMacThai
;
2494 case wxFONTENCODING_MACLAOTIAN
:
2495 enc
= kCFStringEncodingMacLaotian
;
2497 case wxFONTENCODING_MACGEORGIAN
:
2498 enc
= kCFStringEncodingMacGeorgian
;
2500 case wxFONTENCODING_MACARMENIAN
:
2501 enc
= kCFStringEncodingMacArmenian
;
2503 case wxFONTENCODING_MACCHINESESIMP
:
2504 enc
= kCFStringEncodingMacChineseSimp
;
2506 case wxFONTENCODING_MACTIBETAN
:
2507 enc
= kCFStringEncodingMacTibetan
;
2509 case wxFONTENCODING_MACMONGOLIAN
:
2510 enc
= kCFStringEncodingMacMongolian
;
2512 case wxFONTENCODING_MACETHIOPIC
:
2513 enc
= kCFStringEncodingMacEthiopic
;
2515 case wxFONTENCODING_MACCENTRALEUR
:
2516 enc
= kCFStringEncodingMacCentralEurRoman
;
2518 case wxFONTENCODING_MACVIATNAMESE
:
2519 enc
= kCFStringEncodingMacVietnamese
;
2521 case wxFONTENCODING_MACARABICEXT
:
2522 enc
= kCFStringEncodingMacExtArabic
;
2524 case wxFONTENCODING_MACSYMBOL
:
2525 enc
= kCFStringEncodingMacSymbol
;
2527 case wxFONTENCODING_MACDINGBATS
:
2528 enc
= kCFStringEncodingMacDingbats
;
2530 case wxFONTENCODING_MACTURKISH
:
2531 enc
= kCFStringEncodingMacTurkish
;
2533 case wxFONTENCODING_MACCROATIAN
:
2534 enc
= kCFStringEncodingMacCroatian
;
2536 case wxFONTENCODING_MACICELANDIC
:
2537 enc
= kCFStringEncodingMacIcelandic
;
2539 case wxFONTENCODING_MACROMANIAN
:
2540 enc
= kCFStringEncodingMacRomanian
;
2542 case wxFONTENCODING_MACCELTIC
:
2543 enc
= kCFStringEncodingMacCeltic
;
2545 case wxFONTENCODING_MACGAELIC
:
2546 enc
= kCFStringEncodingMacGaelic
;
2548 // case wxFONTENCODING_MACKEYBOARD :
2549 // enc = kCFStringEncodingMacKeyboardGlyphs ;
2553 // because gcc is picky
2560 class wxMBConv_cocoa
: public wxMBConv
2565 Init(CFStringGetSystemEncoding()) ;
2568 wxMBConv_cocoa(const wxMBConv_cocoa
& conv
)
2570 m_encoding
= conv
.m_encoding
;
2574 wxMBConv_cocoa(const wxChar
* name
)
2576 Init( wxCFStringEncFromFontEnc(wxFontMapperBase::Get()->CharsetToEncoding(name
, false) ) ) ;
2580 wxMBConv_cocoa(wxFontEncoding encoding
)
2582 Init( wxCFStringEncFromFontEnc(encoding
) );
2589 void Init( CFStringEncoding encoding
)
2591 m_encoding
= encoding
;
2594 size_t MB2WC(wchar_t * szOut
, const char * szUnConv
, size_t nOutSize
) const
2598 CFStringRef theString
= CFStringCreateWithBytes (
2599 NULL
, //the allocator
2600 (const UInt8
*)szUnConv
,
2603 false //no BOM/external representation
2606 wxASSERT(theString
);
2608 size_t nOutLength
= CFStringGetLength(theString
);
2612 CFRelease(theString
);
2616 CFRange theRange
= { 0, nOutSize
};
2618 #if SIZEOF_WCHAR_T == 4
2619 UniChar
* szUniCharBuffer
= new UniChar
[nOutSize
];
2622 CFStringGetCharacters(theString
, theRange
, szUniCharBuffer
);
2624 CFRelease(theString
);
2626 szUniCharBuffer
[nOutLength
] = '\0';
2628 #if SIZEOF_WCHAR_T == 4
2629 wxMBConvUTF16 converter
;
2630 converter
.MB2WC( szOut
, (const char*)szUniCharBuffer
, nOutSize
);
2631 delete [] szUniCharBuffer
;
2637 size_t WC2MB(char *szOut
, const wchar_t *szUnConv
, size_t nOutSize
) const
2641 size_t nRealOutSize
;
2642 size_t nBufSize
= wxWcslen(szUnConv
);
2643 UniChar
* szUniBuffer
= (UniChar
*) szUnConv
;
2645 #if SIZEOF_WCHAR_T == 4
2646 wxMBConvUTF16 converter
;
2647 nBufSize
= converter
.WC2MB( NULL
, szUnConv
, 0 );
2648 szUniBuffer
= new UniChar
[ (nBufSize
/ sizeof(UniChar
)) + 1];
2649 converter
.WC2MB( (char*) szUniBuffer
, szUnConv
, nBufSize
+ sizeof(UniChar
));
2650 nBufSize
/= sizeof(UniChar
);
2653 CFStringRef theString
= CFStringCreateWithCharactersNoCopy(
2657 kCFAllocatorNull
//deallocator - we want to deallocate it ourselves
2660 wxASSERT(theString
);
2662 //Note that CER puts a BOM when converting to unicode
2663 //so we check and use getchars instead in that case
2664 if (m_encoding
== kCFStringEncodingUnicode
)
2667 CFStringGetCharacters(theString
, CFRangeMake(0, nOutSize
- 1), (UniChar
*) szOut
);
2669 nRealOutSize
= CFStringGetLength(theString
) + 1;
2675 CFRangeMake(0, CFStringGetLength(theString
)),
2677 0, //what to put in characters that can't be converted -
2678 //0 tells CFString to return NULL if it meets such a character
2679 false, //not an external representation
2682 (CFIndex
*) &nRealOutSize
2686 CFRelease(theString
);
2688 #if SIZEOF_WCHAR_T == 4
2689 delete[] szUniBuffer
;
2692 return nRealOutSize
- 1;
2695 virtual wxMBConv
*Clone() const { return new wxMBConv_cocoa(*this); }
2699 return m_encoding
!= kCFStringEncodingInvalidId
&&
2700 CFStringIsEncodingAvailable(m_encoding
);
2704 CFStringEncoding m_encoding
;
2707 #endif // defined(__WXCOCOA__)
2709 // ============================================================================
2710 // Mac conversion classes
2711 // ============================================================================
2713 #if defined(__WXMAC__) && defined(TARGET_CARBON)
2715 class wxMBConv_mac
: public wxMBConv
2720 Init(CFStringGetSystemEncoding()) ;
2723 wxMBConv_mac(const wxMBConv_mac
& conv
)
2725 Init(conv
.m_char_encoding
);
2729 wxMBConv_mac(const wxChar
* name
)
2731 Init( wxMacGetSystemEncFromFontEnc( wxFontMapperBase::Get()->CharsetToEncoding(name
, false) ) );
2735 wxMBConv_mac(wxFontEncoding encoding
)
2737 Init( wxMacGetSystemEncFromFontEnc(encoding
) );
2742 OSStatus status
= noErr
;
2743 status
= TECDisposeConverter(m_MB2WC_converter
);
2744 status
= TECDisposeConverter(m_WC2MB_converter
);
2748 void Init( TextEncodingBase encoding
)
2750 OSStatus status
= noErr
;
2751 m_char_encoding
= encoding
;
2752 m_unicode_encoding
= CreateTextEncoding(kTextEncodingUnicodeDefault
, 0, kUnicode16BitFormat
) ;
2754 status
= TECCreateConverter(&m_MB2WC_converter
,
2756 m_unicode_encoding
);
2757 status
= TECCreateConverter(&m_WC2MB_converter
,
2762 size_t MB2WC(wchar_t *buf
, const char *psz
, size_t n
) const
2764 OSStatus status
= noErr
;
2765 ByteCount byteOutLen
;
2766 ByteCount byteInLen
= strlen(psz
) + 1;
2767 wchar_t *tbuf
= NULL
;
2768 UniChar
* ubuf
= NULL
;
2773 // Apple specs say at least 32
2774 n
= wxMax( 32, byteInLen
) ;
2775 tbuf
= (wchar_t*) malloc( n
* SIZEOF_WCHAR_T
) ;
2778 ByteCount byteBufferLen
= n
* sizeof( UniChar
) ;
2780 #if SIZEOF_WCHAR_T == 4
2781 ubuf
= (UniChar
*) malloc( byteBufferLen
+ 2 ) ;
2783 ubuf
= (UniChar
*) (buf
? buf
: tbuf
) ;
2786 status
= TECConvertText(
2787 m_MB2WC_converter
, (ConstTextPtr
) psz
, byteInLen
, &byteInLen
,
2788 (TextPtr
) ubuf
, byteBufferLen
, &byteOutLen
);
2790 #if SIZEOF_WCHAR_T == 4
2791 // we have to terminate here, because n might be larger for the trailing zero, and if UniChar
2792 // is not properly terminated we get random characters at the end
2793 ubuf
[byteOutLen
/ sizeof( UniChar
) ] = 0 ;
2794 wxMBConvUTF16 converter
;
2795 res
= converter
.MB2WC( (buf
? buf
: tbuf
), (const char*)ubuf
, n
) ;
2798 res
= byteOutLen
/ sizeof( UniChar
) ;
2804 if ( buf
&& res
< n
)
2810 size_t WC2MB(char *buf
, const wchar_t *psz
, size_t n
) const
2812 OSStatus status
= noErr
;
2813 ByteCount byteOutLen
;
2814 ByteCount byteInLen
= wxWcslen(psz
) * SIZEOF_WCHAR_T
;
2820 // Apple specs say at least 32
2821 n
= wxMax( 32, ((byteInLen
/ SIZEOF_WCHAR_T
) * 8) + SIZEOF_WCHAR_T
);
2822 tbuf
= (char*) malloc( n
) ;
2825 ByteCount byteBufferLen
= n
;
2826 UniChar
* ubuf
= NULL
;
2828 #if SIZEOF_WCHAR_T == 4
2829 wxMBConvUTF16 converter
;
2830 size_t unicharlen
= converter
.WC2MB( NULL
, psz
, 0 ) ;
2831 byteInLen
= unicharlen
;
2832 ubuf
= (UniChar
*) malloc( byteInLen
+ 2 ) ;
2833 converter
.WC2MB( (char*) ubuf
, psz
, unicharlen
+ 2 ) ;
2835 ubuf
= (UniChar
*) psz
;
2838 status
= TECConvertText(
2839 m_WC2MB_converter
, (ConstTextPtr
) ubuf
, byteInLen
, &byteInLen
,
2840 (TextPtr
) (buf
? buf
: tbuf
), byteBufferLen
, &byteOutLen
);
2842 #if SIZEOF_WCHAR_T == 4
2849 size_t res
= byteOutLen
;
2850 if ( buf
&& res
< n
)
2854 //we need to double-trip to verify it didn't insert any ? in place
2855 //of bogus characters
2856 wxWCharBuffer
wcBuf(n
);
2857 size_t pszlen
= wxWcslen(psz
);
2858 if ( MB2WC(wcBuf
.data(), buf
, n
) == wxCONV_FAILED
||
2859 wxWcslen(wcBuf
) != pszlen
||
2860 memcmp(wcBuf
, psz
, pszlen
* sizeof(wchar_t)) != 0 )
2862 // we didn't obtain the same thing we started from, hence
2863 // the conversion was lossy and we consider that it failed
2864 return wxCONV_FAILED
;
2871 virtual wxMBConv
*Clone() const { return new wxMBConv_mac(*this); }
2874 { return m_MB2WC_converter
!= NULL
&& m_WC2MB_converter
!= NULL
; }
2877 TECObjectRef m_MB2WC_converter
;
2878 TECObjectRef m_WC2MB_converter
;
2880 TextEncodingBase m_char_encoding
;
2881 TextEncodingBase m_unicode_encoding
;
2884 #endif // defined(__WXMAC__) && defined(TARGET_CARBON)
2886 // ============================================================================
2887 // wxEncodingConverter based conversion classes
2888 // ============================================================================
2892 class wxMBConv_wxwin
: public wxMBConv
2897 m_ok
= m2w
.Init(m_enc
, wxFONTENCODING_UNICODE
) &&
2898 w2m
.Init(wxFONTENCODING_UNICODE
, m_enc
);
2902 // temporarily just use wxEncodingConverter stuff,
2903 // so that it works while a better implementation is built
2904 wxMBConv_wxwin(const wxChar
* name
)
2907 m_enc
= wxFontMapperBase::Get()->CharsetToEncoding(name
, false);
2909 m_enc
= wxFONTENCODING_SYSTEM
;
2914 wxMBConv_wxwin(wxFontEncoding enc
)
2921 size_t MB2WC(wchar_t *buf
, const char *psz
, size_t WXUNUSED(n
)) const
2923 size_t inbuf
= strlen(psz
);
2926 if (!m2w
.Convert(psz
, buf
))
2927 return wxCONV_FAILED
;
2932 size_t WC2MB(char *buf
, const wchar_t *psz
, size_t WXUNUSED(n
)) const
2934 const size_t inbuf
= wxWcslen(psz
);
2937 if (!w2m
.Convert(psz
, buf
))
2938 return wxCONV_FAILED
;
2944 virtual size_t GetMBNulLen() const
2948 case wxFONTENCODING_UTF16BE
:
2949 case wxFONTENCODING_UTF16LE
:
2952 case wxFONTENCODING_UTF32BE
:
2953 case wxFONTENCODING_UTF32LE
:
2961 virtual wxMBConv
*Clone() const { return new wxMBConv_wxwin(m_enc
); }
2963 bool IsOk() const { return m_ok
; }
2966 wxFontEncoding m_enc
;
2967 wxEncodingConverter m2w
, w2m
;
2970 // were we initialized successfully?
2973 DECLARE_NO_COPY_CLASS(wxMBConv_wxwin
)
2976 // make the constructors available for unit testing
2977 WXDLLIMPEXP_BASE wxMBConv
* new_wxMBConv_wxwin( const wxChar
* name
)
2979 wxMBConv_wxwin
* result
= new wxMBConv_wxwin( name
);
2980 if ( !result
->IsOk() )
2989 #endif // wxUSE_FONTMAP
2991 // ============================================================================
2992 // wxCSConv implementation
2993 // ============================================================================
2995 void wxCSConv::Init()
3002 wxCSConv::wxCSConv(const wxChar
*charset
)
3012 m_encoding
= wxFontMapperBase::GetEncodingFromName(charset
);
3014 m_encoding
= wxFONTENCODING_SYSTEM
;
3018 wxCSConv::wxCSConv(wxFontEncoding encoding
)
3020 if ( encoding
== wxFONTENCODING_MAX
|| encoding
== wxFONTENCODING_DEFAULT
)
3022 wxFAIL_MSG( _T("invalid encoding value in wxCSConv ctor") );
3024 encoding
= wxFONTENCODING_SYSTEM
;
3029 m_encoding
= encoding
;
3032 wxCSConv::~wxCSConv()
3037 wxCSConv::wxCSConv(const wxCSConv
& conv
)
3042 SetName(conv
.m_name
);
3043 m_encoding
= conv
.m_encoding
;
3046 wxCSConv
& wxCSConv::operator=(const wxCSConv
& conv
)
3050 SetName(conv
.m_name
);
3051 m_encoding
= conv
.m_encoding
;
3056 void wxCSConv::Clear()
3065 void wxCSConv::SetName(const wxChar
*charset
)
3069 m_name
= wxStrdup(charset
);
3075 #include "wx/hashmap.h"
3077 WX_DECLARE_HASH_MAP( wxFontEncoding
, wxString
, wxIntegerHash
, wxIntegerEqual
,
3078 wxEncodingNameCache
);
3080 static wxEncodingNameCache gs_nameCache
;
3083 wxMBConv
*wxCSConv::DoCreate() const
3086 wxLogTrace(TRACE_STRCONV
,
3087 wxT("creating conversion for %s"),
3089 : wxFontMapperBase::GetEncodingName(m_encoding
).c_str()));
3090 #endif // wxUSE_FONTMAP
3092 // check for the special case of ASCII or ISO8859-1 charset: as we have
3093 // special knowledge of it anyhow, we don't need to create a special
3094 // conversion object
3095 if ( m_encoding
== wxFONTENCODING_ISO8859_1
||
3096 m_encoding
== wxFONTENCODING_DEFAULT
)
3098 // don't convert at all
3102 // we trust OS to do conversion better than we can so try external
3103 // conversion methods first
3105 // the full order is:
3106 // 1. OS conversion (iconv() under Unix or Win32 API)
3107 // 2. hard coded conversions for UTF
3108 // 3. wxEncodingConverter as fall back
3114 #endif // !wxUSE_FONTMAP
3116 wxString
name(m_name
);
3117 wxFontEncoding
encoding(m_encoding
);
3119 if ( !name
.empty() )
3121 wxMBConv_iconv
*conv
= new wxMBConv_iconv(name
);
3129 wxFontMapperBase::Get()->CharsetToEncoding(name
, false);
3130 #endif // wxUSE_FONTMAP
3134 const wxEncodingNameCache::iterator it
= gs_nameCache
.find(encoding
);
3135 if ( it
!= gs_nameCache
.end() )
3137 if ( it
->second
.empty() )
3140 wxMBConv_iconv
*conv
= new wxMBConv_iconv(it
->second
);
3147 const wxChar
** names
= wxFontMapperBase::GetAllEncodingNames(encoding
);
3149 for ( ; *names
; ++names
)
3151 wxMBConv_iconv
*conv
= new wxMBConv_iconv(*names
);
3154 gs_nameCache
[encoding
] = *names
;
3161 gs_nameCache
[encoding
] = _T(""); // cache the failure
3163 #endif // wxUSE_FONTMAP
3165 #endif // HAVE_ICONV
3167 #ifdef wxHAVE_WIN32_MB2WC
3170 wxMBConv_win32
*conv
= m_name
? new wxMBConv_win32(m_name
)
3171 : new wxMBConv_win32(m_encoding
);
3180 #endif // wxHAVE_WIN32_MB2WC
3182 #if defined(__WXMAC__)
3184 // leave UTF16 and UTF32 to the built-ins of wx
3185 if ( m_name
|| ( m_encoding
< wxFONTENCODING_UTF16BE
||
3186 ( m_encoding
>= wxFONTENCODING_MACMIN
&& m_encoding
<= wxFONTENCODING_MACMAX
) ) )
3189 wxMBConv_mac
*conv
= m_name
? new wxMBConv_mac(m_name
)
3190 : new wxMBConv_mac(m_encoding
);
3192 wxMBConv_mac
*conv
= new wxMBConv_mac(m_encoding
);
3202 #if defined(__WXCOCOA__)
3204 if ( m_name
|| ( m_encoding
<= wxFONTENCODING_UTF16
) )
3207 wxMBConv_cocoa
*conv
= m_name
? new wxMBConv_cocoa(m_name
)
3208 : new wxMBConv_cocoa(m_encoding
);
3210 wxMBConv_cocoa
*conv
= new wxMBConv_cocoa(m_encoding
);
3221 wxFontEncoding enc
= m_encoding
;
3223 if ( enc
== wxFONTENCODING_SYSTEM
&& m_name
)
3225 // use "false" to suppress interactive dialogs -- we can be called from
3226 // anywhere and popping up a dialog from here is the last thing we want to
3228 enc
= wxFontMapperBase::Get()->CharsetToEncoding(m_name
, false);
3230 #endif // wxUSE_FONTMAP
3234 case wxFONTENCODING_UTF7
:
3235 return new wxMBConvUTF7
;
3237 case wxFONTENCODING_UTF8
:
3238 return new wxMBConvUTF8
;
3240 case wxFONTENCODING_UTF16BE
:
3241 return new wxMBConvUTF16BE
;
3243 case wxFONTENCODING_UTF16LE
:
3244 return new wxMBConvUTF16LE
;
3246 case wxFONTENCODING_UTF32BE
:
3247 return new wxMBConvUTF32BE
;
3249 case wxFONTENCODING_UTF32LE
:
3250 return new wxMBConvUTF32LE
;
3253 // nothing to do but put here to suppress gcc warnings
3260 wxMBConv_wxwin
*conv
= m_name
? new wxMBConv_wxwin(m_name
)
3261 : new wxMBConv_wxwin(m_encoding
);
3267 #endif // wxUSE_FONTMAP
3269 // NB: This is a hack to prevent deadlock. What could otherwise happen
3270 // in Unicode build: wxConvLocal creation ends up being here
3271 // because of some failure and logs the error. But wxLog will try to
3272 // attach timestamp, for which it will need wxConvLocal (to convert
3273 // time to char* and then wchar_t*), but that fails, tries to log
3274 // error, but wxLog has a (already locked) critical section that
3275 // guards static buffer.
3276 static bool alreadyLoggingError
= false;
3277 if (!alreadyLoggingError
)
3279 alreadyLoggingError
= true;
3280 wxLogError(_("Cannot convert from the charset '%s'!"),
3284 wxFontMapperBase::GetEncodingDescription(m_encoding
).c_str()
3285 #else // !wxUSE_FONTMAP
3286 wxString::Format(_("encoding %s"), m_encoding
).c_str()
3287 #endif // wxUSE_FONTMAP/!wxUSE_FONTMAP
3290 alreadyLoggingError
= false;
3296 void wxCSConv::CreateConvIfNeeded() const
3300 wxCSConv
*self
= (wxCSConv
*)this; // const_cast
3303 // if we don't have neither the name nor the encoding, use the default
3304 // encoding for this system
3305 if ( !m_name
&& m_encoding
== wxFONTENCODING_SYSTEM
)
3307 self
->m_name
= wxStrdup(wxLocale::GetSystemEncodingName());
3309 #endif // wxUSE_INTL
3311 self
->m_convReal
= DoCreate();
3312 self
->m_deferred
= false;
3316 size_t wxCSConv::MB2WC(wchar_t *buf
, const char *psz
, size_t n
) const
3318 CreateConvIfNeeded();
3321 return m_convReal
->MB2WC(buf
, psz
, n
);
3324 size_t len
= strlen(psz
);
3328 for (size_t c
= 0; c
<= len
; c
++)
3329 buf
[c
] = (unsigned char)(psz
[c
]);
3335 size_t wxCSConv::WC2MB(char *buf
, const wchar_t *psz
, size_t n
) const
3337 CreateConvIfNeeded();
3340 return m_convReal
->WC2MB(buf
, psz
, n
);
3343 const size_t len
= wxWcslen(psz
);
3346 for (size_t c
= 0; c
<= len
; c
++)
3349 return wxCONV_FAILED
;
3351 buf
[c
] = (char)psz
[c
];
3356 for (size_t c
= 0; c
<= len
; c
++)
3359 return wxCONV_FAILED
;
3366 size_t wxCSConv::GetMBNulLen() const
3368 CreateConvIfNeeded();
3372 return m_convReal
->GetMBNulLen();
3378 // ----------------------------------------------------------------------------
3380 // ----------------------------------------------------------------------------
3383 static wxMBConv_win32 wxConvLibcObj
;
3384 #elif defined(__WXMAC__) && !defined(__MACH__)
3385 static wxMBConv_mac wxConvLibcObj
;
3387 static wxMBConvLibc wxConvLibcObj
;
3390 static wxCSConv
wxConvLocalObj(wxFONTENCODING_SYSTEM
);
3391 static wxCSConv
wxConvISO8859_1Obj(wxFONTENCODING_ISO8859_1
);
3392 static wxMBConvUTF7 wxConvUTF7Obj
;
3393 static wxMBConvUTF8 wxConvUTF8Obj
;
3395 WXDLLIMPEXP_DATA_BASE(wxMBConv
&) wxConvLibc
= wxConvLibcObj
;
3396 WXDLLIMPEXP_DATA_BASE(wxCSConv
&) wxConvLocal
= wxConvLocalObj
;
3397 WXDLLIMPEXP_DATA_BASE(wxCSConv
&) wxConvISO8859_1
= wxConvISO8859_1Obj
;
3398 WXDLLIMPEXP_DATA_BASE(wxMBConvUTF7
&) wxConvUTF7
= wxConvUTF7Obj
;
3399 WXDLLIMPEXP_DATA_BASE(wxMBConvUTF8
&) wxConvUTF8
= wxConvUTF8Obj
;
3400 WXDLLIMPEXP_DATA_BASE(wxMBConv
*) wxConvCurrent
= &wxConvLibcObj
;
3401 WXDLLIMPEXP_DATA_BASE(wxMBConv
*) wxConvUI
= &wxConvLocal
;
3402 WXDLLIMPEXP_DATA_BASE(wxMBConv
*) wxConvFileName
= &
3409 #else // !wxUSE_WCHAR_T
3411 // stand-ins in absence of wchar_t
3412 WXDLLIMPEXP_DATA_BASE(wxMBConv
) wxConvLibc
,
3417 #endif // wxUSE_WCHAR_T/!wxUSE_WCHAR_T