X-Git-Url: https://git.saurik.com/wxWidgets.git/blobdiff_plain/6001e347baf63a91be525f3b9beeefde4bc16f19..7018409531500bd1850c1ab3f0107007802bbc5a:/src/common/strconv.cpp diff --git a/src/common/strconv.cpp b/src/common/strconv.cpp index e61b9fd56c..69f9a7a85a 100644 --- a/src/common/strconv.cpp +++ b/src/common/strconv.cpp @@ -9,6 +9,14 @@ // Licence: wxWindows license ///////////////////////////////////////////////////////////////////////////// +// ============================================================================ +// declarations +// ============================================================================ + +// ---------------------------------------------------------------------------- +// headers +// ---------------------------------------------------------------------------- + #ifdef __GNUG__ #pragma implementation "strconv.h" #endif @@ -20,6 +28,11 @@ #pragma hdrstop #endif +#ifdef __WXMSW__ + #include "wx/msw/private.h" +#endif + +#include #include #include #include @@ -28,27 +41,91 @@ #include #endif +#ifdef HAVE_ICONV_H + #include +#endif +#ifdef HAVE_LANGINFO_H + #include +#endif + +#ifdef __WXMSW__ + #include +#endif + #include "wx/debug.h" #include "wx/strconv.h" -//---------------------------------------------------------------------------- -// wxConvCurrent -//---------------------------------------------------------------------------- +#if defined(WORDS_BIGENDIAN) || defined(__STDC_ISO_10646__) +#define BSWAP_UCS4(str, len) +#define BSWAP_UCS2(str, len) +#else +#define BSWAP_UCS4(str, len) { unsigned _c; for (_c=0; _c=0x110000) { + return (size_t)-1; + } else { + if (output) { + *output++ = (input >> 10)+0xd7c0; + *output++ = (input&0x3ff)+0xdc00; + } + return 2; + } +} + +static size_t decode_utf16(wxUint16*input,wxUint32&output) +{ + if ((*input<0xd800) || (*input>0xdfff)) { + output = *input; + return 1; + } else + if ((input[1]<0xdc00) || (input[1]>=0xdfff)) { + output = *input; + return (size_t)-1; + } else { + output = ((input[0] - 0xd7c0) << 10) + (input[1] - 0xdc00); + return 2; + } +} + +#endif // WC_UTF16 -//---------------------------------------------------------------------------- +// ---------------------------------------------------------------------------- // wxMBConv -//---------------------------------------------------------------------------- +// ---------------------------------------------------------------------------- WXDLLEXPORT_DATA(wxMBConv) wxConvLibc; @@ -64,33 +141,37 @@ size_t wxMBConv::WC2MB(char *buf, const wchar_t *psz, size_t n) const const wxWCharBuffer wxMBConv::cMB2WC(const char *psz) const { - if (psz) + if (psz) { size_t nLen = MB2WC((wchar_t *) NULL, psz, 0); + if (nLen == (size_t)-1) + return wxWCharBuffer((wchar_t *) NULL); wxWCharBuffer buf(nLen); MB2WC((wchar_t *)(const wchar_t *) buf, psz, nLen); return buf; - } - else + } + else return wxWCharBuffer((wchar_t *) NULL); } const wxCharBuffer wxMBConv::cWC2MB(const wchar_t *psz) const { - if (psz) + if (psz) { size_t nLen = WC2MB((char *) NULL, psz, 0); + if (nLen == (size_t)-1) + return wxCharBuffer((char *) NULL); wxCharBuffer buf(nLen); WC2MB((char *)(const char *) buf, psz, nLen); return buf; - } - else + } + else return wxCharBuffer((char *) NULL); } -//---------------------------------------------------------------------------- +// ---------------------------------------------------------------------------- // standard file conversion -//---------------------------------------------------------------------------- +// ---------------------------------------------------------------------------- WXDLLEXPORT_DATA(wxMBConvFile) wxConvFile; @@ -105,11 +186,11 @@ size_t wxMBConvFile::WC2MB(char *buf, const wchar_t *psz, size_t n) const return wxWC2MB(buf, psz, n); } -#ifdef __WXGTK12__ - -//---------------------------------------------------------------------------- +// ---------------------------------------------------------------------------- // standard gdk conversion -//---------------------------------------------------------------------------- +// ---------------------------------------------------------------------------- + +#ifdef __WXGTK12__ WXDLLEXPORT_DATA(wxMBConvGdk) wxConvGdk; @@ -138,6 +219,7 @@ size_t wxMBConvGdk::WC2MB(char *buf, const wchar_t *psz, size_t n) const } return len; } + #endif // GTK > 1.0 // ---------------------------------------------------------------------------- @@ -171,13 +253,13 @@ size_t wxMBConvUTF7::WC2MB(char * WXUNUSED(buf), return 0; } -//---------------------------------------------------------------------------- +// ---------------------------------------------------------------------------- // UTF-8 -//---------------------------------------------------------------------------- +// ---------------------------------------------------------------------------- WXDLLEXPORT_DATA(wxMBConvUTF8) wxConvUTF8; -static unsigned long utf8_max[]={0x7f,0x7ff,0xffff,0x1fffff,0x3ffffff,0x7fffffff,0xffffffff}; +static wxUint32 utf8_max[]={0x7f,0x7ff,0xffff,0x1fffff,0x3ffffff,0x7fffffff,0xffffffff}; size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const { @@ -198,7 +280,7 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const return (size_t)-1; } else { unsigned ocnt=cnt-1; - unsigned long res=cc&(0x3f>>cnt); + wxUint32 res=cc&(0x3f>>cnt); while (cnt--) { cc = *psz++; if ((cc&0xC0)!=0x80) { @@ -211,8 +293,16 @@ size_t wxMBConvUTF8::MB2WC(wchar_t *buf, const char *psz, size_t n) const // illegal UTF-8 encoding return (size_t)-1; } +#ifdef WC_UTF16 + size_t pa = encode_utf16(res, buf); + if (pa == (size_t)-1) + return (size_t)-1; + if (buf) buf+=pa; + len+=pa; +#else if (buf) *buf++=res; len++; +#endif } } } @@ -225,7 +315,13 @@ size_t wxMBConvUTF8::WC2MB(char *buf, const wchar_t *psz, size_t n) const size_t len = 0; while (*psz && ((!buf) || (lenutf8_max[cnt]; cnt++); if (!cnt) { @@ -249,121 +345,223 @@ size_t wxMBConvUTF8::WC2MB(char *buf, const wchar_t *psz, size_t n) const // specified character set // ---------------------------------------------------------------------------- -#ifndef WX_PRECOMP - #include "wx/dynarray.h" - #include "wx/filefn.h" - #include "wx/textfile.h" - #include "wx/tokenzr.h" - #include "wx/utils.h" +WXDLLEXPORT_DATA(wxCSConv) wxConvLocal((const wxChar *)NULL); + +#include "wx/encconv.h" +#include "wx/fontmap.h" + +// TODO: add some tables here +// - perhaps common encodings to common codepages (for Win32) +// - perhaps common encodings to objects ("UTF8" -> wxConvUTF8) +// - move wxEncodingConverter meat in here + +#ifdef __WIN32__ +#include "wx/msw/registry.h" +// this should work if M$ Internet Exploiter is installed +static long CharsetToCodepage(const wxChar *name) +{ + if (!name) return GetACP(); + long CP=-1; + wxString cn(name); + do { + wxString path(wxT("MIME\\Database\\Charset\\")); + path += cn; + wxRegKey key(wxRegKey::HKCR,path); + + /* two cases: either there's an AliasForCharset string, + * or there are Codepage and InternetEncoding dwords. + * The InternetEncoding gives us the actual encoding, + * the Codepage just says which Windows character set to + * use when displaying the data. + */ + if (key.QueryValue(wxT("InternetEncoding"),&CP)) break; + // no encoding, see if it's an alias + if (!key.QueryValue(wxT("AliasForCharset"),cn)) break; + } while (1); + return CP; +} #endif class wxCharacterSet { public: - wxArrayString names; - wchar_t *data; + const wxChar*cname; + wxCharacterSet(const wxChar*name) : cname(name) {} + virtual ~wxCharacterSet() {} + virtual size_t MB2WC(wchar_t*buf, const char*psz, size_t n) { return (size_t)-1; } + virtual size_t WC2MB(char*buf, const wchar_t*psz, size_t n) { return (size_t)-1; } + virtual bool usable() { return FALSE; } }; -WX_DECLARE_OBJARRAY(wxCharacterSet, wxCSArray); -#include "wx/arrimpl.cpp" -WX_DEFINE_OBJARRAY(wxCSArray); - -static wxCSArray wxCharsets; +class ID_CharSet : public wxCharacterSet +{ +public: + wxMBConv*work; + ID_CharSet(const wxChar*name,wxMBConv*cnv) : wxCharacterSet(name), work(cnv) {} + size_t MB2WC(wchar_t*buf, const char*psz, size_t n) + { return work ? work->MB2WC(buf,psz,n) : (size_t)-1; } + size_t WC2MB(char*buf, const wchar_t*psz, size_t n) + { return work ? work->WC2MB(buf,psz,n) : (size_t)-1; } + bool usable() { return work!=NULL; } +}; -static void wxLoadCharacterSets(void) +#ifdef HAVE_ICONV_H +class IC_CharSet : public wxCharacterSet { - static bool already_loaded = FALSE; - - if (already_loaded) return; - - already_loaded = TRUE; -#if defined(__UNIX__) && wxUSE_TEXTFILE - // search through files in /usr/share/i18n/charmaps - wxString fname; - for (fname = ::wxFindFirstFile(_T("/usr/share/i18n/charmaps/*")); - !fname.IsEmpty(); - fname = ::wxFindNextFile()) { - wxTextFile cmap(fname); - if (cmap.Open()) { - wxCharacterSet *cset = new wxCharacterSet; - wxString comchar,escchar; - bool in_charset = FALSE; - - // wxFprintf(stderr,_T("Loaded: %s\n"),fname.c_str()); - - wxString line; - for (line = cmap.GetFirstLine(); - !cmap.Eof(); - line = cmap.GetNextLine()) { - // wxFprintf(stderr,_T("line contents: %s\n"),line.c_str()); - wxStringTokenizer token(line); - wxString cmd = token.GetNextToken(); - if (cmd == comchar) { - if (token.GetNextToken() == _T("alias")) - cset->names.Add(token.GetNextToken()); - } - else if (cmd == _T("")) - cset->names.Add(token.GetNextToken()); - else if (cmd == _T("")) - comchar = token.GetNextToken(); - else if (cmd == _T("")) - escchar = token.GetNextToken(); - else if (cmd == _T("")) { - delete cset; - cset = (wxCharacterSet *) NULL; - break; // we don't support multibyte charsets ourselves (yet) - } - else if (cmd == _T("CHARMAP")) { - cset->data = (wchar_t *)calloc(256, sizeof(wchar_t)); - in_charset = TRUE; - } - else if (cmd == _T("END")) { - if (token.GetNextToken() == _T("CHARMAP")) - in_charset = FALSE; - } - else if (in_charset) { - // format: /x00 NULL (NUL) - // /x41 LATIN CAPITAL LETTER A - wxString hex = token.GetNextToken(); - // skip whitespace (why doesn't wxStringTokenizer do this?) - while (wxIsEmpty(hex) && token.HasMoreTokens()) hex = token.GetNextToken(); - wxString uni = token.GetNextToken(); - // skip whitespace again - while (wxIsEmpty(uni) && token.HasMoreTokens()) uni = token.GetNextToken(); - if ((hex.Len() > 2) && (wxString(hex.GetChar(0)) == escchar) && (hex.GetChar(1) == _T('x')) && - (uni.Left(2) == _T("=0) { - unsigned long uni1 = ::wxHexToDec(uni.Mid(2,2)); - unsigned long uni2 = ::wxHexToDec(uni.Mid(4,2)); - cset->data[pos] = (uni1 << 16) | uni2; - // wxFprintf(stderr,_T("char %02x mapped to %04x (%c)\n"),pos,cset->data[pos],cset->data[pos]); - } - } - } - } - if (cset) { - cset->names.Shrink(); - wxCharsets.Add(cset); - } +public: + iconv_t m2w, w2m; + IC_CharSet(const wxChar*name) : wxCharacterSet(name), m2w((iconv_t)-1), w2m((iconv_t)-1) {} + ~IC_CharSet() { + if (m2w!=(iconv_t)-1) iconv_close(m2w); + if (w2m!=(iconv_t)-1) iconv_close(w2m); + } + void LoadM2W() { if (m2w==(iconv_t)-1) m2w=iconv_open(WC_NAME,wxConvLibc.cWX2MB(cname)); } + void LoadW2M() { if (w2m==(iconv_t)-1) w2m=iconv_open(wxConvLibc.cWX2MB(cname),WC_NAME); } + size_t MB2WC(wchar_t*buf, const char*psz, size_t n) { + LoadM2W(); + size_t inbuf = strlen(psz); + size_t outbuf = n*SIZEOF_WCHAR_T; + size_t res, cres; + fprintf(stderr,"IC Convert to WC using %s\n",(const char*)wxConvLibc.cWX2MB(cname)); + if (buf) { + // have destination buffer, convert there + cres = iconv(m2w,&psz,&inbuf,(char**)&buf,&outbuf); + res = n-(outbuf/SIZEOF_WCHAR_T); + // convert to native endianness + WC_BSWAP(buf, res) + } else { + // no destination buffer... convert using temp buffer + // to calculate destination buffer requirement + wchar_t tbuf[8]; + res = 0; + do { + buf = tbuf; outbuf = 8*SIZEOF_WCHAR_T; + cres = iconv(m2w,&psz,&inbuf,(char**)&buf,&outbuf); + res += 8-(outbuf/SIZEOF_WCHAR_T); + } while ((cres==(size_t)-1) && (errno==E2BIG)); } + if (cres==(size_t)-1) return (size_t)-1; + return res; } + size_t WC2MB(char*buf, const wchar_t*psz, size_t n) { + LoadW2M(); +#if defined(__BORLANDC__) && (__BORLANDC__ > 0x530) + size_t inbuf = std::wcslen(psz); +#else + size_t inbuf = ::wcslen(psz); +#endif + size_t outbuf = n; + size_t res, cres; + fprintf(stderr,"IC Convert from WC using %s\n",(const char*)wxConvLibc.cWX2MB(cname)); +#ifdef WC_NEED_BSWAP + // need to copy to temp buffer to switch endianness + // this absolutely doesn't rock! + // (no, doing WC_BSWAP twice on the original buffer won't help, as it + // could be in read-only memory, or be accessed in some other thread) + wchar_t*tmpbuf=(wchar_t*)malloc((inbuf+1)*SIZEOF_WCHAR_T); + memcpy(tmpbuf,psz,(inbuf+1)*SIZEOF_WCHAR_T); + WC_BSWAP(tmpbuf, inbuf) + psz=tmpbuf; +#endif + if (buf) { + // have destination buffer, convert there + cres = iconv(w2m,(const char**)&psz,&inbuf,&buf,&outbuf); + res = n-outbuf; + } else { + // no destination buffer... convert using temp buffer + // to calculate destination buffer requirement + char tbuf[16]; + res = 0; + do { + buf = tbuf; outbuf = 16; + cres = iconv(w2m,(const char**)&psz,&inbuf,&buf,&outbuf); + res += 16 - outbuf; + } while ((cres==(size_t)-1) && (errno==E2BIG)); + } +#ifdef WC_NEED_BSWAP + free(tmpbuf); +#endif + if (cres==(size_t)-1) return (size_t)-1; + return res; + } + bool usable() { return TRUE; } +}; #endif - wxCharsets.Shrink(); -} -static wxCharacterSet *wxFindCharacterSet(const wxChar *charset) +#ifdef __WIN32__ +class CP_CharSet : public wxCharacterSet { - if (!charset) return (wxCharacterSet *)NULL; - wxLoadCharacterSets(); - for (size_t n=0; nCharsetToEncoding(name, FALSE); + m2w.Init(enc, wxFONTENCODING_UNICODE); + w2m.Init(wxFONTENCODING_UNICODE, enc); + } + size_t MB2WC(wchar_t*buf, const char*psz, size_t n) { + size_t inbuf = strlen(psz); + fprintf(stderr,"EC Convert to WC using %d\n",enc); + if (buf) m2w.Convert(psz,buf); + return inbuf; + } + size_t WC2MB(char*buf, const wchar_t*psz, size_t n) { +#if defined(__BORLANDC__) && (__BORLANDC__ > 0x530) + size_t inbuf = std::wcslen(psz); +#else + size_t inbuf = ::wcslen(psz); +#endif + fprintf(stderr,"EC Convert from WC using %d\n",enc); + if (buf) w2m.Convert(psz,buf); + return inbuf; + } + bool usable() { return (enc!=wxFONTENCODING_SYSTEM) && (enc!=wxFONTENCODING_DEFAULT); } +}; + +static wxCharacterSet *wxGetCharacterSet(const wxChar *name) +{ + wxCharacterSet *cset = NULL; + if (name) { + if (!wxStricmp(name, wxT("UTF8")) || !wxStricmp(name, wxT("UTF-8"))) { + cset = new ID_CharSet(name, &wxConvUTF8); + } else { +#ifdef HAVE_ICONV_H + cset = new IC_CharSet(name); // may not take NULL +#endif + } + } + if (cset && cset->usable()) return cset; + if (cset) delete cset; +#ifdef __WIN32__ + cset = new CP_CharSet(name); // may take NULL + if (cset->usable()) return cset; +#endif + if (cset) delete cset; + cset = new EC_CharSet(name); + if (cset->usable()) return cset; + delete cset; + return NULL; +} wxCSConv::wxCSConv(const wxChar *charset) { @@ -376,46 +574,42 @@ wxCSConv::wxCSConv(const wxChar *charset) wxCSConv::~wxCSConv() { if (m_name) free(m_name); + if (m_cset) delete m_cset; } void wxCSConv::SetName(const wxChar *charset) { if (charset) { -#ifdef __UNIX__ - // first, convert the character set name to standard form - wxString codeset; - if (wxString(charset,3).CmpNoCase(_T("ISO")) == 0) { - // make sure it's represented in the standard form: ISO_8859-1 - codeset = _T("ISO_"); - charset += 3; - if ((*charset == _T('-')) || (*charset == _T('_'))) charset++; - if (wxStrlen(charset)>4) { - if (wxString(charset,4) == _T("8859")) { - codeset << _T("8859-"); - if (*charset == _T('-')) charset++; - } - } - } - codeset << charset; - codeset.MakeUpper(); - m_name = wxStrdup(codeset.c_str()); + m_name = wxStrdup(charset); m_deferred = TRUE; -#endif } } void wxCSConv::LoadNow() { -// wxPrintf(_T("Conversion request\n")); +// wxPrintf(wxT("Conversion request\n")); if (m_deferred) { if (!m_name) { #ifdef __UNIX__ - wxChar *lang = wxGetenv(_T("LANG")); - wxChar *dot = lang ? wxStrchr(lang, _T('.')) : (wxChar *)NULL; - if (dot) SetName(dot+1); +#if defined(HAVE_LANGINFO_H) && defined(CODESET) + // GNU libc provides current character set this way + char*alang = nl_langinfo(CODESET); + if (alang) SetName(wxConvLibc.cMB2WX(alang)); + else +#endif + // if we can't get at the character set directly, + // try to see if it's in the environment variables + // (in most cases this won't work, but I was out of ideas) + { + wxChar *lang = wxGetenv(wxT("LC_ALL")); + if (!lang) lang = wxGetenv(wxT("LC_CTYPE")); + if (!lang) lang = wxGetenv(wxT("LANG")); + wxChar *dot = lang ? wxStrchr(lang, wxT('.')) : (wxChar *)NULL; + if (dot) SetName(dot+1); + } #endif } - m_cset = wxFindCharacterSet(m_name); + m_cset = wxGetCharacterSet(m_name); m_deferred = FALSE; } } @@ -423,41 +617,80 @@ void wxCSConv::LoadNow() size_t wxCSConv::MB2WC(wchar_t *buf, const char *psz, size_t n) const { ((wxCSConv *)this)->LoadNow(); // discard constness + if (m_cset) + return m_cset->MB2WC(buf, psz, n); + + // latin-1 (direct) + size_t len=strlen(psz); if (buf) { - if (m_cset) { - for (size_t c=0; cdata[(unsigned char)(psz[c])]; - } else { - // latin-1 (direct) - for (size_t c=0; cLoadNow(); // discard constness + if (m_cset) + return m_cset->WC2MB(buf, psz, n); + + // latin-1 (direct) +#if defined(__BORLANDC__) && (__BORLANDC__ > 0x530) + size_t len=std::wcslen(psz); +#else + size_t len=::wcslen(psz); +#endif if (buf) { - if (m_cset) { - for (size_t c=0; cdata[n] != psz[c]); n++); - buf[c] = (n>0xff) ? '?' : n; - } - } else { - // latin-1 (direct) - for (size_t c=0; c0xff) ? '?' : psz[c]; - } - return n; + for (size_t c=0; c<=len; c++) + buf[c] = (psz[c]>0xff) ? '?' : psz[c]; } - return wcslen(psz); + return len; } +#ifdef HAVE_ICONV_H +class IC_CharSetConverter +{ +public: + iconv_t cnv; + IC_CharSetConverter(IC_CharSet*from,IC_CharSet*to) { + cnv=iconv_open(wxConvLibc.cWX2MB(to->cname),wxConvLibc.cWX2MB(from->cname)); + } + ~IC_CharSetConverter() { + if (cnv!=(iconv_t)-1) iconv_close(cnv); + } + size_t Convert(char*buf, const char*psz, size_t n) { + size_t inbuf = strlen(psz); + size_t outbuf = n; + size_t res = iconv(cnv,&psz,&inbuf,&buf,&outbuf); + if (res==(size_t)-1) return (size_t)-1; + return n-outbuf; + } +}; #endif - //wxUSE_WCHAR_T + +class EC_CharSetConverter +{ +public: + wxEncodingConverter cnv; + EC_CharSetConverter(EC_CharSet*from,EC_CharSet*to) { + cnv.Init(from->enc,to->enc); + } + size_t Convert(char*buf, const char*psz, size_t n) { + size_t inbuf = strlen(psz); + if (buf) cnv.Convert(psz,buf); + return inbuf; + } +}; + +#else // !wxUSE_WCHAR_T + +// ---------------------------------------------------------------------------- +// stand-ins in absence of wchar_t +// ---------------------------------------------------------------------------- + +WXDLLEXPORT_DATA(wxMBConv) wxConvLibc, wxConvFile; + +#endif // wxUSE_WCHAR_T