src/common/convauto.cpp

   1 ///////////////////////////////////////////////////////////////////////////////
   2 // Name:        src/common/convauto.cpp
   3 // Purpose:     implementation of wxConvAuto
   4 // Author:      Vadim Zeitlin
   5 // Created:     2006-04-04
   6 // RCS-ID:      $Id$
   7 // Copyright:   (c) 2006 Vadim Zeitlin <vadim@wxwindows.org>
   8 // Licence:     wxWindows licence
   9 ///////////////////////////////////////////////////////////////////////////////
  10
  11 // ============================================================================
  12 // declarations
  13 // ============================================================================
  14
  15 // ----------------------------------------------------------------------------
  16 // headers
  17 // ----------------------------------------------------------------------------
  18
  19 // for compilers that support precompilation, includes "wx.h".
  20 #include "wx/wxprec.h"
  21
  22 #ifdef __BORLANDC__
  23     #pragma hdrstop
  24 #endif
  25
  26 #include "wx/convauto.h"
  27
  28 // we use latin1 by default as it seems the least bad choice: the files we need
  29 // to detect input of don't always come from the user system (they are often
  30 // received from other machines) and so using wxFONTENCODING_SYSTEM doesn't
  31 // seem to be a good idea and there is no other reasonable alternative
  32 wxFontEncoding wxConvAuto::ms_defaultMBEncoding = wxFONTENCODING_ISO8859_1;
  33
  34 namespace
  35 {
  36
  37 const char BOM_UTF32BE[] = { '\x00', '\x00', '\xFE', '\xFF' };
  38 const char BOM_UTF32LE[] = { '\xFF', '\xFE', '\x00', '\x00' };
  39 const char BOM_UTF16BE[] = { '\xFE', '\xFF'                 };
  40 const char BOM_UTF16LE[] = { '\xFF', '\xFE'                 };
  41 const char BOM_UTF8[]    = { '\xEF', '\xBB', '\xBF'         };
  42
  43 } // anonymous namespace
  44
  45 // ============================================================================
  46 // implementation
  47 // ============================================================================
  48
  49 /* static */
  50 void wxConvAuto::SetFallbackEncoding(wxFontEncoding enc)
  51 {
  52     wxASSERT_MSG( enc != wxFONTENCODING_DEFAULT,
  53                   wxT("wxFONTENCODING_DEFAULT doesn't make sense here") );
  54
  55     ms_defaultMBEncoding = enc;
  56 }
  57
  58 /* static */
  59 const char* wxConvAuto::GetBOMChars(wxBOM bom, size_t* count)
  60 {
  61     wxCHECK_MSG( count , NULL, wxS("count pointer must be provided") );
  62
  63     switch ( bom )
  64     {
  65         case wxBOM_UTF32BE: *count = WXSIZEOF(BOM_UTF32BE); return BOM_UTF32BE;
  66         case wxBOM_UTF32LE: *count = WXSIZEOF(BOM_UTF32LE); return BOM_UTF32LE;
  67         case wxBOM_UTF16BE: *count = WXSIZEOF(BOM_UTF16BE); return BOM_UTF16BE;
  68         case wxBOM_UTF16LE: *count = WXSIZEOF(BOM_UTF16LE); return BOM_UTF16LE;
  69         case wxBOM_UTF8   : *count = WXSIZEOF(BOM_UTF8   ); return BOM_UTF8;
  70         case wxBOM_Unknown:
  71         case wxBOM_None:
  72             wxFAIL_MSG( wxS("Invalid BOM type") );
  73             return NULL;
  74     }
  75
  76     wxFAIL_MSG( wxS("Unknown BOM type") );
  77     return NULL;
  78 }
  79
  80 /* static */
  81 wxBOM wxConvAuto::DetectBOM(const char *src, size_t srcLen)
  82 {
  83     // examine the buffer for BOM presence
  84     //
  85     // quoting from http://www.unicode.org/faq/utf_bom.html#BOM:
  86     //
  87     //  Bytes           Encoding Form
  88     //
  89     //  00 00 FE FF     UTF-32, big-endian
  90     //  FF FE 00 00     UTF-32, little-endian
  91     //  FE FF           UTF-16, big-endian
  92     //  FF FE           UTF-16, little-endian
  93     //  EF BB BF        UTF-8
  94     //
  95     // as some BOMs are prefixes of other ones we may need to read more bytes
  96     // to disambiguate them
  97
  98     switch ( srcLen )
  99     {
 100         case 0:
 101             return wxBOM_Unknown;
 102
 103         case 1:
 104             if ( src[0] == '\x00' || src[0] == '\xFF' ||
 105                  src[0] == '\xFE' || src[0] == '\xEF')
 106             {
 107                 // this could be a BOM but we don't know yet
 108                 return wxBOM_Unknown;
 109             }
 110             break;
 111
 112         case 2:
 113         case 3:
 114             if ( src[0] == '\xEF' && src[1] == '\xBB' )
 115             {
 116                 if ( srcLen == 3 )
 117                     return src[2] == '\xBF' ? wxBOM_UTF8 : wxBOM_None;
 118
 119                 return wxBOM_Unknown;
 120             }
 121
 122             if ( src[0] == '\xFE' && src[1] == '\xFF' )
 123                 return wxBOM_UTF16BE;
 124
 125             if ( src[0] == '\xFF' && src[1] == '\xFE' )
 126             {
 127                 // if the next byte is 0, it could be an UTF-32LE BOM but if it
 128                 // isn't we can be sure it's UTF-16LE
 129                 if ( srcLen == 3 && src[2] != '\x00' )
 130                     return wxBOM_UTF16LE;
 131
 132                 return wxBOM_Unknown;
 133             }
 134
 135             if ( src[0] == '\x00' && src[1] == '\x00' )
 136             {
 137                 // this could only be UTF-32BE, check that the data we have so
 138                 // far allows for it
 139                 if ( srcLen == 3 && src[2] != '\xFE' )
 140                     return wxBOM_None;
 141
 142                 return wxBOM_Unknown;
 143             }
 144             break;
 145
 146         default:
 147             // we have at least 4 characters so we may finally decide whether
 148             // we have a BOM or not
 149             if ( src[0] == '\xEF' && src[1] == '\xBB' && src[2] == '\xBF' )
 150                 return wxBOM_UTF8;
 151
 152             if ( src[0] == '\x00' && src[1] == '\x00' &&
 153                  src[2] == '\xFE' && src[3] == '\xFF' )
 154                 return wxBOM_UTF32BE;
 155
 156             if ( src[0] == '\xFF' && src[1] == '\xFE' &&
 157                  src[2] == '\x00' && src[3] == '\x00' )
 158                 return wxBOM_UTF32LE;
 159
 160             if ( src[0] == '\xFE' && src[1] == '\xFF' )
 161                 return wxBOM_UTF16BE;
 162
 163             if ( src[0] == '\xFF' && src[1] == '\xFE' )
 164                 return wxBOM_UTF16LE;
 165     }
 166
 167     return wxBOM_None;
 168 }
 169
 170 void wxConvAuto::InitFromBOM(wxBOM bomType)
 171 {
 172     m_consumedBOM = false;
 173
 174     switch ( bomType )
 175     {
 176         case wxBOM_Unknown:
 177             wxFAIL_MSG( "shouldn't be called for this BOM type" );
 178             break;
 179
 180         case wxBOM_None:
 181             // use the default
 182             break;
 183
 184         case wxBOM_UTF32BE:
 185             m_conv = new wxMBConvUTF32BE;
 186             m_ownsConv = true;
 187             break;
 188
 189         case wxBOM_UTF32LE:
 190             m_conv = new wxMBConvUTF32LE;
 191             m_ownsConv = true;
 192             break;
 193
 194         case wxBOM_UTF16BE:
 195             m_conv = new wxMBConvUTF16BE;
 196             m_ownsConv = true;
 197             break;
 198
 199         case wxBOM_UTF16LE:
 200             m_conv = new wxMBConvUTF16LE;
 201             m_ownsConv = true;
 202             break;
 203
 204         case wxBOM_UTF8:
 205             InitWithUTF8();
 206             break;
 207
 208         default:
 209             wxFAIL_MSG( "unknown BOM type" );
 210     }
 211
 212     if ( !m_conv )
 213     {
 214         // we end up here if there is no BOM or we didn't recognize it somehow
 215         // (this shouldn't happen but still don't crash if it does), so use the
 216         // default encoding
 217         InitWithUTF8();
 218         m_consumedBOM = true; // as there is nothing to consume
 219     }
 220 }
 221
 222 void wxConvAuto::SkipBOM(const char **src, size_t *len) const
 223 {
 224     int ofs;
 225     switch ( m_bomType )
 226     {
 227         case wxBOM_Unknown:
 228             wxFAIL_MSG( "shouldn't be called for this BOM type" );
 229             return;
 230
 231         case wxBOM_None:
 232             ofs = 0;
 233             break;
 234
 235         case wxBOM_UTF32BE:
 236         case wxBOM_UTF32LE:
 237             ofs = 4;
 238             break;
 239
 240         case wxBOM_UTF16BE:
 241         case wxBOM_UTF16LE:
 242             ofs = 2;
 243             break;
 244
 245         case wxBOM_UTF8:
 246             ofs = 3;
 247             break;
 248
 249         default:
 250             wxFAIL_MSG( "unknown BOM type" );
 251             return;
 252     }
 253
 254     *src += ofs;
 255     if ( *len != (size_t)-1 )
 256         *len -= ofs;
 257 }
 258
 259 bool wxConvAuto::InitFromInput(const char *src, size_t len)
 260 {
 261     m_bomType = DetectBOM(src, len == wxNO_LEN ? strlen(src) : len);
 262     if ( m_bomType == wxBOM_Unknown )
 263         return false;
 264
 265     InitFromBOM(m_bomType);
 266
 267     return true;
 268 }
 269
 270 size_t
 271 wxConvAuto::ToWChar(wchar_t *dst, size_t dstLen,
 272                     const char *src, size_t srcLen) const
 273 {
 274     // we check BOM and create the appropriate conversion the first time we're
 275     // called but we also need to ensure that the BOM is skipped not only
 276     // during this initial call but also during the first call with non-NULL
 277     // dst as typically we're first called with NULL dst to calculate the
 278     // needed buffer size
 279     wxConvAuto *self = const_cast<wxConvAuto *>(this);
 280
 281
 282     if ( !m_conv )
 283     {
 284         if ( !self->InitFromInput(src, srcLen) )
 285         {
 286             // there is not enough data to determine whether we have a BOM or
 287             // not, so fail for now -- the caller is supposed to call us again
 288             // with more data
 289             return wxCONV_FAILED;
 290         }
 291     }
 292
 293     if ( !m_consumedBOM )
 294     {
 295         SkipBOM(&src, &srcLen);
 296         if ( srcLen == 0 )
 297         {
 298             // there is nothing left except the BOM so we'd return 0 below but
 299             // this is unexpected: decoding a non-empty string must either fail
 300             // or return something non-empty, in particular this would break
 301             // the code in wxTextInputStream::NextChar()
 302             //
 303             // so still return an error as we need some more data to be able to
 304             // decode it
 305             return wxCONV_FAILED;
 306         }
 307     }
 308
 309     // try to convert using the auto-detected encoding
 310     size_t rc = m_conv->ToWChar(dst, dstLen, src, srcLen);
 311     if ( rc == wxCONV_FAILED && m_bomType == wxBOM_None )
 312     {
 313         // if the conversion failed but we didn't really detect anything and
 314         // simply tried UTF-8 by default, retry it using the fall-back
 315         if ( m_encDefault != wxFONTENCODING_MAX )
 316         {
 317             if ( m_ownsConv )
 318                 delete m_conv;
 319
 320             self->m_conv = new wxCSConv(m_encDefault == wxFONTENCODING_DEFAULT
 321                                             ? GetFallbackEncoding()
 322                                             : m_encDefault);
 323             self->m_ownsConv = true;
 324
 325             rc = m_conv->ToWChar(dst, dstLen, src, srcLen);
 326         }
 327     }
 328
 329     // don't skip the BOM again the next time if we really consumed it
 330     if ( rc != wxCONV_FAILED && dst && !m_consumedBOM )
 331         self->m_consumedBOM = true;
 332
 333     return rc;
 334 }
 335
 336 size_t
 337 wxConvAuto::FromWChar(char *dst, size_t dstLen,
 338                       const wchar_t *src, size_t srcLen) const
 339 {
 340     if ( !m_conv )
 341     {
 342         // default to UTF-8 for the multibyte output
 343         const_cast<wxConvAuto *>(this)->InitWithUTF8();
 344     }
 345
 346     return m_conv->FromWChar(dst, dstLen, src, srcLen);
 347 }