X-Git-Url: https://git.saurik.com/wxWidgets.git/blobdiff_plain/a4a44612a654667fbd40e01d39c7482820fb851b..29fbbf8967e4c4b3174a4bc45a9007b92ecb0de7:/src/common/string.cpp diff --git a/src/common/string.cpp b/src/common/string.cpp index ab47f8c300..6b65824622 100644 --- a/src/common/string.cpp +++ b/src/common/string.cpp @@ -10,13 +10,6 @@ // Licence: wxWindows licence ///////////////////////////////////////////////////////////////////////////// -/* - * About ref counting: - * 1) all empty strings use g_strEmpty, nRefs = -1 (set in Init()) - * 2) AllocBuffer() sets nRefs to 1, Lock() increments it by one - * 3) Unlock() decrements nRefs and frees memory if it goes to 0 - */ - // =========================================================================== // headers, declarations, constants // =========================================================================== @@ -30,6 +23,7 @@ #ifndef WX_PRECOMP #include "wx/string.h" + #include "wx/wxcrtvararg.h" #endif #include @@ -41,10 +35,6 @@ #include #include -#ifdef __SALFORDC__ - #include -#endif - #include "wx/hashmap.h" // string handling functions used by wxString: @@ -78,11 +68,10 @@ const size_t wxString::npos = (size_t) -1; wxSTD ostream& operator<<(wxSTD ostream& os, const wxCStrData& str) { -// FIXME-UTF8: always, not only if wxUSE_UNICODE -#if wxUSE_UNICODE && !defined(__BORLANDC__) - return os << str.AsWChar(); +#if wxUSE_UNICODE && !wxUSE_UNICODE_UTF8 + return os << (const char *)str.AsCharBuf(); #else - return os << str.AsChar(); + return os << str.AsInternal(); #endif } @@ -103,401 +92,32 @@ wxSTD ostream& operator<<(wxSTD ostream& os, const wxWCharBuffer& str) } #endif -#endif // wxUSE_STD_IOSTREAM - -// =========================================================================== -// wxString class core -// =========================================================================== - -#if wxUSE_UNICODE_UTF8 - -// --------------------------------------------------------------------------- -// UTF-8 operations -// --------------------------------------------------------------------------- - -// -// Table 3.1B from Unicode spec: Legal UTF-8 Byte Sequences -// -// Code Points | 1st Byte | 2nd Byte | 3rd Byte | 4th Byte | -// -------------------+----------+----------+----------+----------+ -// U+0000..U+007F | 00..7F | | | | -// U+0080..U+07FF | C2..DF | 80..BF | | | -// U+0800..U+0FFF | E0 | A0..BF | 80..BF | | -// U+1000..U+FFFF | E1..EF | 80..BF | 80..BF | | -// U+10000..U+3FFFF | F0 | 90..BF | 80..BF | 80..BF | -// U+40000..U+FFFFF | F1..F3 | 80..BF | 80..BF | 80..BF | -// U+100000..U+10FFFF | F4 | 80..8F | 80..BF | 80..BF | -// -------------------+----------+----------+----------+----------+ - -bool wxString::IsValidUtf8String(const char *str) -{ - if ( !str ) - return true; // empty string is UTF8 string - - const unsigned char *c = (const unsigned char*)str; - - for ( ; *c; ++c ) - { - unsigned char b = *c; - - if ( b <= 0x7F ) // 00..7F - continue; - - else if ( b < 0xC2 ) // invalid lead bytes: 80..C1 - return false; - - // two-byte sequences: - else if ( b <= 0xDF ) // C2..DF - { - b = *(++c); - if ( !(b >= 0x80 && b <= 0xBF ) ) - return false; - } - - // three-byte sequences: - else if ( b == 0xE0 ) - { - b = *(++c); - if ( !(b >= 0xA0 && b <= 0xBF ) ) - return false; - b = *(++c); - if ( !(b >= 0x80 && b <= 0xBF ) ) - return false; - } - else if ( b <= 0xEF ) // E1..EF - { - for ( int i = 0; i < 2; ++i ) - { - b = *(++c); - if ( !(b >= 0x80 && b <= 0xBF ) ) - return false; - } - } - - // four-byte sequences: - else if ( b == 0xF0 ) - { - b = *(++c); - if ( !(b >= 0x90 && b <= 0xBF ) ) - return false; - for ( int i = 0; i < 2; ++i ) - { - b = *(++c); - if ( !(b >= 0x80 && b <= 0xBF ) ) - return false; - } - } - else if ( b <= 0xF3 ) // F1..F3 - { - for ( int i = 0; i < 3; ++i ) - { - b = *(++c); - if ( !(b >= 0x80 && b <= 0xBF ) ) - return false; - } - } - else if ( b == 0xF4 ) - { - b = *(++c); - if ( !(b >= 0x80 && b <= 0x8F ) ) - return false; - for ( int i = 0; i < 2; ++i ) - { - b = *(++c); - if ( !(b >= 0x80 && b <= 0xBF ) ) - return false; - } - } - else // otherwise, it's invalid lead byte - return false; - } - - return true; -} - -#ifdef __WXDEBUG__ -/* static */ -bool wxString::IsValidUtf8LeadByte(unsigned char c) -{ - return (c <= 0x7F) || (c >= 0xC2 && c <= 0xF4); -} -#endif - -unsigned char wxString::ms_utf8IterTable[256] = { - // single-byte sequences (ASCII): - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 00..0F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 10..1F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 20..2F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 30..3F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 40..4F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 50..5F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 60..6F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 70..7F - - // these are invalid, we use step 1 to skip - // over them (should never happen): - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 80..8F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 90..9F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A0..AF - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B0..BF - 1, 1, // C0,C1 - - // two-byte sequences: - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C2..CF - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // D0..DF - - // three-byte sequences: - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // E0..EF - - // four-byte sequences: - 4, 4, 4, 4, 4, // F0..F4 - - // these are invalid again (5- or 6-byte - // sequences and sequences for code points - // above U+10FFFF, as restricted by RFC 3629): - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // F5..FF -}; +#if wxUSE_UNICODE && defined(HAVE_WOSTREAM) -/* static */ -void wxString::DecIter(wxStringImpl::const_iterator& i) +wxSTD wostream& operator<<(wxSTD wostream& wos, const wxString& str) { - wxASSERT( IsValidUtf8LeadByte(*i) ); - - // Non-lead bytes are all in the 0x80..0xBF range (i.e. 10xxxxxx in - // binary), so we just have to go back until we hit a byte that is either - // < 0x80 (i.e. 0xxxxxxx in binary) or 0xC0..0xFF (11xxxxxx in binary; this - // includes some invalid values, but we can ignore it here, because we - // assume valid UTF-8 input for the purpose of efficient implementation). - --i; - while ( ((*i) & 0xC0) == 0x80 /* 2 highest bits are '10' */ ) - --i; + return wos << str.wc_str(); } -/* static */ -void wxString::DecIter(wxStringImpl::iterator& i) +wxSTD wostream& operator<<(wxSTD wostream& wos, const wxCStrData& str) { - // FIXME-UTF8: use template instead - wxASSERT( IsValidUtf8LeadByte(*i) ); - --i; - while ( ((*i) & 0xC0) == 0x80 /* 2 highest bits are '10' */ ) - --i; + return wos << str.AsWChar(); } -/* static */ -wxStringImpl::const_iterator -wxString::AddToIter(wxStringImpl::const_iterator i, int n) +wxSTD wostream& operator<<(wxSTD wostream& wos, const wxWCharBuffer& str) { - wxStringImpl::const_iterator out(i); - - if ( n > 0 ) - { - for ( int j = 0; j < n; ++j ) - IncIter(out); - } - else if ( n < 0 ) - { - for ( int j = 0; j > n; --j ) - DecIter(out); - } - - return out; + return wos << str.data(); } -wxStringImpl::iterator -wxString::AddToIter(wxStringImpl::iterator i, int n) -{ - // FIXME-UTF8: use template instead - wxStringImpl::iterator out(i); - - if ( n > 0 ) - { - for ( int j = 0; j < n; ++j ) - IncIter(out); - } - else if ( n < 0 ) - { - for ( int j = 0; j > n; --j ) - DecIter(out); - } - - return out; -} - - -/* static */ -int wxString::DiffIters(wxStringImpl::const_iterator i1, - wxStringImpl::const_iterator i2) -{ - int dist = 0; - - if ( i1 < i2 ) - { - while ( i1 != i2 ) - { - IncIter(i1); - dist--; - } - } - else if ( i2 < i1 ) - { - while ( i2 != i1 ) - { - IncIter(i2); - dist++; - } - } - - return dist; -} - -int wxString::DiffIters(wxStringImpl::iterator i1, wxStringImpl::iterator i2) -{ - // FIXME-UTF8: use template instead - int dist = 0; - - if ( i1 < i2 ) - { - while ( i1 != i2 ) - { - IncIter(i1); - dist--; - } - } - else if ( i2 < i1 ) - { - while ( i2 != i1 ) - { - IncIter(i2); - dist++; - } - } - - return dist; -} - -/* static */ -wxString::Utf8CharBuffer wxString::EncodeChar(wxUniChar ch) -{ - Utf8CharBuffer buf; - char *out = buf.data; - - wxUniChar::value_type code = ch.GetValue(); - - // Char. number range | UTF-8 octet sequence - // (hexadecimal) | (binary) - // ----------------------+--------------------------------------------- - // 0000 0000 - 0000 007F | 0xxxxxxx - // 0000 0080 - 0000 07FF | 110xxxxx 10xxxxxx - // 0000 0800 - 0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx - // 0001 0000 - 0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - // - // Code point value is stored in bits marked with 'x', lowest-order bit - // of the value on the right side in the diagram above. - // (from RFC 3629) - - if ( code <= 0x7F ) - { - out[1] = 0; - out[0] = (char)code; - } - else if ( code <= 0x07FF ) - { - out[2] = 0; - // NB: this line takes 6 least significant bits, encodes them as - // 10xxxxxx and discards them so that the next byte can be encoded: - out[1] = 0x80 | (code & 0x3F); code >>= 6; - out[0] = 0xC0 | code; - } - else if ( code < 0xFFFF ) - { - out[3] = 0; - out[2] = 0x80 | (code & 0x3F); code >>= 6; - out[1] = 0x80 | (code & 0x3F); code >>= 6; - out[0] = 0xE0 | code; - } - else if ( code <= 0x10FFFF ) - { - out[4] = 0; - out[3] = 0x80 | (code & 0x3F); code >>= 6; - out[2] = 0x80 | (code & 0x3F); code >>= 6; - out[1] = 0x80 | (code & 0x3F); code >>= 6; - out[0] = 0xF0 | code; - } - else - { - wxFAIL_MSG( _T("trying to encode undefined Unicode character") ); - out[0] = 0; - } - - return buf; -} - -/* static */ -wxUniChar wxUniCharRef::DecodeChar(wxStringImpl::const_iterator i) -{ - wxASSERT( wxString::IsValidUtf8LeadByte(*i) ); // FIXME-UTF8: no "wxString::" - - wxUniChar::value_type code = 0; - size_t len = wxString::GetUtf8CharLength(*i); - wxASSERT_MSG( len <= 4, _T("invalid UTF-8 sequence length") ); - - // Char. number range | UTF-8 octet sequence - // (hexadecimal) | (binary) - // ----------------------+--------------------------------------------- - // 0000 0000 - 0000 007F | 0xxxxxxx - // 0000 0080 - 0000 07FF | 110xxxxx 10xxxxxx - // 0000 0800 - 0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx - // 0001 0000 - 0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - // - // Code point value is stored in bits marked with 'x', lowest-order bit - // of the value on the right side in the diagram above. - // (from RFC 3629) - - // mask to extract lead byte's value ('x' bits above), by sequence's length: - static const unsigned char s_leadValueMask[4] = { 0x7F, 0x1F, 0x0F, 0x07 }; -#ifdef __WXDEBUG__ - // mask and value of lead byte's most significant bits, by length: - static const unsigned char s_leadMarkerMask[4] = { 0x80, 0xE0, 0xF0, 0xF8 }; - static const unsigned char s_leadMarkerVal[4] = { 0x00, 0xC0, 0xE0, 0xF0 }; -#endif - - // extract the lead byte's value bits: - wxASSERT_MSG( ((unsigned char)*i & s_leadMarkerMask[len-1]) == - s_leadMarkerVal[len-1], - _T("invalid UTF-8 lead byte") ); - code = (unsigned char)*i & s_leadValueMask[len-1]; - - // all remaining bytes, if any, are handled in the same way regardless of - // sequence's length: - for ( ++i ; len > 1; --len, ++i ) - { - wxASSERT_MSG( ((unsigned char)*i & 0xC0) == 0x80, - _T("invalid UTF-8 byte") ); - - code <<= 6; - code |= (unsigned char)*i & 0x3F; - } - - return wxUniChar(code); -} - -/* static */ -wxCharBuffer wxString::EncodeNChars(size_t n, wxUniChar ch) -{ - Utf8CharBuffer once(EncodeChar(ch)); - // the IncIter() table can be used to determine the length of ch's encoding: - size_t len = ms_utf8IterTable[(unsigned char)once.data[0]]; +#endif // wxUSE_UNICODE && defined(HAVE_WOSTREAM) - wxCharBuffer buf(n * len); - char *ptr = buf.data(); - for ( size_t i = 0; i < n; i++, ptr += len ) - { - memcpy(ptr, once.data, len); - } +#endif // wxUSE_STD_IOSTREAM - return buf; -} +// =========================================================================== +// wxString class core +// =========================================================================== +#if wxUSE_UNICODE_UTF8 void wxString::PosLenToImpl(size_t pos, size_t len, size_t *implPos, size_t *implLen) const @@ -507,7 +127,7 @@ void wxString::PosLenToImpl(size_t pos, size_t len, else { const_iterator i = begin() + pos; - *implPos = wxStringImpl::const_iterator(i) - m_impl.begin(); + *implPos = wxStringImpl::const_iterator(i.impl()) - m_impl.begin(); if ( len == npos ) *implLen = npos; else @@ -518,8 +138,7 @@ void wxString::PosLenToImpl(size_t pos, size_t len, if ( pos + len > length() ) len = length() - pos; - *implLen = wxStringImpl::const_iterator(i + len) - - wxStringImpl::const_iterator(i); + *implLen = (i + len).impl() - i.impl(); } } } @@ -615,15 +234,48 @@ wxString::~wxString() } #endif -#if wxUSE_UNICODE +#if wxUSE_UNICODE && !wxUSE_UTF8_LOCALE_ONLY const char* wxCStrData::AsChar() const { +#if wxUSE_UNICODE_UTF8 + if ( wxLocaleIsUtf8 ) + return AsInternal(); +#endif + // under non-UTF8 locales, we have to convert the internal UTF-8 + // representation using wxConvLibc and cache the result + wxString *str = wxConstCast(m_str, wxString); // convert the string: + // + // FIXME-UTF8: we'd like to do the conversion in the existing buffer (if we + // have it) but it's unfortunately not obvious to implement + // because we don't know how big buffer do we need for the + // given string length (in case of multibyte encodings, e.g. + // ISO-2022-JP or UTF-8 when internal representation is wchar_t) + // + // One idea would be to store more than just m_convertedToChar + // in wxString: then we could record the length of the string + // which was converted the last time and try to reuse the same + // buffer if the current length is not greater than it (this + // could still fail because string could have been modified in + // place but it would work most of the time, so we'd do it and + // only allocate the new buffer if in-place conversion returned + // an error). We could also store a bit saying if the string + // was modified since the last conversion (and update it in all + // operation modifying the string, of course) to avoid unneeded + // consequential conversions. But both of these ideas require + // adding more fields to wxString and require profiling results + // to be sure that we really gain enough from them to justify + // doing it. wxCharBuffer buf(str->mb_str()); - // FIXME-UTF8: do the conversion in-place in the existing buffer + // if it failed, return empty string and not NULL to avoid crashes in code + // written with either wxWidgets 2 wxString or std::string behaviour in + // mind: neither of them ever returns NULL and so we shouldn't neither + if ( !buf ) + return ""; + if ( str->m_convertedToChar && strlen(buf) == strlen(str->m_convertedToChar) ) { @@ -639,7 +291,7 @@ const char* wxCStrData::AsChar() const // and keep it: return str->m_convertedToChar + m_offset; } -#endif // wxUSE_UNICODE +#endif // wxUSE_UNICODE && !wxUSE_UTF8_LOCALE_ONLY #if !wxUSE_UNICODE_WCHAR const wchar_t* wxCStrData::AsWChar() const @@ -649,6 +301,10 @@ const wchar_t* wxCStrData::AsWChar() const // convert the string: wxWCharBuffer buf(str->wc_str()); + // notice that here, unlike above in AsChar(), conversion can't fail as our + // internal UTF-8 is always well-formed -- or the string was corrupted and + // all bets are off anyhow + // FIXME-UTF8: do the conversion in-place in the existing buffer if ( str->m_convertedToWChar && wxWcslen(buf) == wxWcslen(str->m_convertedToWChar) ) @@ -701,14 +357,23 @@ wxString::SubstrBufFromMB wxString::ConvertStr(const char *psz, size_t nLength, wxString::SubstrBufFromMB wxString::ConvertStr(const char *psz, size_t nLength, const wxMBConv& conv) { - // FIXME-UTF8: return as-is without copying under UTF8 locale, return - // converted string under other locales - needs wxCharBuffer - // changes - // anything to do? if ( !psz || nLength == 0 ) return SubstrBufFromMB("", 0); + // if psz is already in UTF-8, we don't have to do the roundtrip to + // wchar_t* and back: + if ( conv.IsUTF8() ) + { + // we need to validate the input because UTF8 iterators assume valid + // UTF-8 sequence and psz may be invalid: + if ( wxStringOperations::IsValidUtf8String(psz, nLength) ) + { + return SubstrBufFromMB(wxCharBuffer::CreateNonOwned(psz), nLength); + } + // else: do the roundtrip through wchar_t* + } + if ( nLength == npos ) nLength = wxNO_LEN; @@ -719,7 +384,7 @@ wxString::SubstrBufFromMB wxString::ConvertStr(const char *psz, size_t nLength, return SubstrBufFromMB("", 0); // and then to UTF-8: - SubstrBufFromMB buf(ConvertStr(wcBuf, wcLen, wxConvUTF8)); + SubstrBufFromMB buf(ConvertStr(wcBuf, wcLen, wxMBConvStrictUTF8())); // widechar -> UTF-8 conversion isn't supposed to ever fail: wxASSERT_MSG( buf.data, _T("conversion to UTF-8 failed") ); @@ -761,26 +426,32 @@ const wxCharBuffer wxString::mb_str(const wxMBConv& conv) const const wxWCharBuffer wxString::wc_str() const { - return wxConvUTF8.cMB2WC(m_impl.c_str(), - m_impl.length() + 1 /* size, not length */, - NULL); + return wxMBConvStrictUTF8().cMB2WC + ( + m_impl.c_str(), + m_impl.length() + 1, // size, not length + NULL + ); } const wxCharBuffer wxString::mb_str(const wxMBConv& conv) const { - // FIXME-UTF8: optimize the case when conv==wxConvUTF8 or wxConvLibc - // under UTF8 locale + if ( conv.IsUTF8() ) + return wxCharBuffer::CreateNonOwned(m_impl.c_str()); + // FIXME-UTF8: use wc_str() here once we have buffers with length size_t wcLen; - wxWCharBuffer wcBuf( - wxConvUTF8.cMB2WC(m_impl.c_str(), - m_impl.length() + 1 /* size, not length */, - &wcLen)); + wxWCharBuffer wcBuf(wxMBConvStrictUTF8().cMB2WC + ( + m_impl.c_str(), + m_impl.length() + 1, // size + &wcLen + )); if ( !wcLen ) return wxCharBuffer(""); - return conv.cWC2MB(wcBuf, wcLen, NULL); + return conv.cWC2MB(wcBuf, wcLen+1, NULL); } #else // ANSI @@ -804,7 +475,7 @@ bool wxString::Shrink() // deprecated compatibility code: #if WXWIN_COMPATIBILITY_2_8 && !wxUSE_STL_BASED_WXSTRING && !wxUSE_UNICODE_UTF8 -wxChar *wxString::GetWriteBuf(size_t nLen) +wxStringCharType *wxString::GetWriteBuf(size_t nLen) { return DoGetWriteBuf(nLen); } @@ -943,6 +614,12 @@ wxString operator+(const wchar_t *pwz, const wxString& str) // string comparison // --------------------------------------------------------------------------- +bool wxString::IsSameAs(wxUniChar c, bool compareWithCase) const +{ + return (length() == 1) && (compareWithCase ? GetChar(0u) == c + : wxToupper(GetChar(0u)) == wxToupper(c)); +} + #ifdef HAVE_STD_STRING_COMPARE // NB: Comparison code (both if HAVE_STD_STRING_COMPARE and if not) works with @@ -1312,13 +989,12 @@ int wxString::CmpNoCase(const wxString& s) const { // FIXME-UTF8: use wxUniChar::ToLower/ToUpper once added - size_t idx = 0; const_iterator i1 = begin(); const_iterator end1 = end(); const_iterator i2 = s.begin(); const_iterator end2 = s.end(); - for ( ; i1 != end1 && i2 != end2; ++idx, ++i1, ++i2 ) + for ( ; i1 != end1 && i2 != end2; ++i1, ++i2 ) { wxUniChar lower1 = (wxChar)wxTolower(*i1); wxUniChar lower2 = (wxChar)wxTolower(*i2); @@ -1345,63 +1021,69 @@ int wxString::CmpNoCase(const wxString& s) const #endif #endif -wxString wxString::FromAscii(const char *ascii) +wxString wxString::FromAscii(const char *ascii, size_t len) { - if (!ascii) + if (!ascii || len == 0) return wxEmptyString; - size_t len = strlen( ascii ); wxString res; - if ( len ) { - wxStringBuffer buf(res, len); - - wchar_t *dest = buf; + wxStringInternalBuffer buf(res, len); + wxStringCharType *dest = buf; - for ( ;; ) + for ( ; len > 0; --len ) { - if ( (*dest++ = (wchar_t)(unsigned char)*ascii++) == L'\0' ) - break; + unsigned char c = (unsigned char)*ascii++; + wxASSERT_MSG( c < 0x80, + _T("Non-ASCII value passed to FromAscii().") ); + + *dest++ = (wchar_t)c; } } return res; } -wxString wxString::FromAscii(const char ascii) +wxString wxString::FromAscii(const char *ascii) +{ + return FromAscii(ascii, wxStrlen(ascii)); +} + +wxString wxString::FromAscii(char ascii) { // What do we do with '\0' ? - wxString res; - res += (wchar_t)(unsigned char) ascii; + unsigned char c = (unsigned char)ascii; - return res; + wxASSERT_MSG( c < 0x80, _T("Non-ASCII value passed to FromAscii().") ); + + // NB: the cast to wchar_t causes interpretation of 'ascii' as Latin1 value + return wxString(wxUniChar((wchar_t)c)); } const wxCharBuffer wxString::ToAscii() const { // this will allocate enough space for the terminating NUL too wxCharBuffer buffer(length()); - - char *dest = buffer.data(); - const wchar_t *pwc = c_str(); - for ( ;; ) + for ( const_iterator i = begin(); i != end(); ++i ) { - *dest++ = (char)(*pwc > SCHAR_MAX ? wxT('_') : *pwc); + wxUniChar c(*i); + // FIXME-UTF8: unify substituted char ('_') with wxUniChar ('?') + *dest++ = c.IsAscii() ? (char)c : '_'; // the output string can't have embedded NULs anyhow, so we can safely // stop at first of them even if we do have any - if ( !*pwc++ ) + if ( !c ) break; } return buffer; } -#endif // Unicode +#endif // wxUSE_UNICODE // extract string of length nCount starting at nFirst wxString wxString::Mid(size_t nFirst, size_t nCount) const @@ -1437,28 +1119,15 @@ wxString wxString::Mid(size_t nFirst, size_t nCount) const // check that the string starts with prefix and return the rest of the string // in the provided pointer if it is not NULL, otherwise return false -bool wxString::StartsWith(const wxChar *prefix, wxString *rest) const +bool wxString::StartsWith(const wxString& prefix, wxString *rest) const { - wxASSERT_MSG( prefix, _T("invalid parameter in wxString::StartsWith") ); - - // first check if the beginning of the string matches the prefix: note - // that we don't have to check that we don't run out of this string as - // when we reach the terminating NUL, either prefix string ends too (and - // then it's ok) or we break out of the loop because there is no match - const wxChar *p = c_str(); - while ( *prefix ) - { - if ( *prefix++ != *p++ ) - { - // no match - return false; - } - } + if ( compare(0, prefix.length(), prefix) != 0 ) + return false; if ( rest ) { // put the rest of the string into provided pointer - *rest = p; + rest->assign(*this, prefix.length(), npos); } return true; @@ -1467,11 +1136,9 @@ bool wxString::StartsWith(const wxChar *prefix, wxString *rest) const // check that the string ends with suffix and return the rest of it in the // provided pointer if it is not NULL, otherwise return false -bool wxString::EndsWith(const wxChar *suffix, wxString *rest) const +bool wxString::EndsWith(const wxString& suffix, wxString *rest) const { - wxASSERT_MSG( suffix, _T("invalid parameter in wxString::EndssWith") ); - - int start = length() - wxStrlen(suffix); + int start = length() - suffix.length(); if ( start < 0 || compare(start, npos, suffix) != 0 ) return false; @@ -1676,9 +1343,9 @@ wxString& wxString::MakeLower() // --------------------------------------------------------------------------- // some compilers (VC++ 6.0 not to name them) return true for a call to -// isspace('ê') in the C locale which seems to be broken to me, but we have to -// live with this by checking that the character is a 7 bit one - even if this -// may fail to detect some spaces (I don't know if Unicode doesn't have +// isspace('\xEA') in the C locale which seems to be broken to me, but we have +// to live with this by checking that the character is a 7 bit one - even if +// this may fail to detect some spaces (I don't know if Unicode doesn't have // space-like symbols somewhere except in the first 128 chars), it is arguably // still better than trimming away accented letters inline int wxSafeIsspace(wxChar ch) { return (ch < 127) && wxIsspace(ch); } @@ -1699,7 +1366,7 @@ wxString& wxString::Trim(bool bFromRight) // find last non-space character reverse_iterator psz = rbegin(); while ( (psz != rend()) && wxSafeIsspace(*psz) ) - psz++; + ++psz; // truncate at trailing space start erase(psz.base(), end()); @@ -1709,7 +1376,7 @@ wxString& wxString::Trim(bool bFromRight) // find first non-space character iterator psz = begin(); while ( (psz != end()) && wxSafeIsspace(*psz) ) - psz++; + ++psz; // fix up data and length erase(begin(), psz); @@ -1763,84 +1430,84 @@ int wxString::Find(wxUniChar ch, bool bFromEnd) const // conversion to numbers // ---------------------------------------------------------------------------- -// the implementation of all the functions below is exactly the same so factor -// it out - -template -bool wxStringToIntType(const wxChar *start, - T *val, - int base, - F func) -{ - wxCHECK_MSG( val, false, _T("NULL output pointer") ); - wxASSERT_MSG( !base || (base > 1 && base <= 36), _T("invalid base") ); +// The implementation of all the functions below is exactly the same so factor +// it out. Note that number extraction works correctly on UTF-8 strings, so +// we can use wxStringCharType and wx_str() for maximum efficiency. #ifndef __WXWINCE__ - errno = 0; + #define DO_IF_NOT_WINCE(x) x +#else + #define DO_IF_NOT_WINCE(x) #endif - wxChar *end; - *val = (*func)(start, &end, base); +#define WX_STRING_TO_INT_TYPE(out, base, func, T) \ + wxCHECK_MSG( out, false, _T("NULL output pointer") ); \ + wxASSERT_MSG( !base || (base > 1 && base <= 36), _T("invalid base") ); \ + \ + DO_IF_NOT_WINCE( errno = 0; ) \ + \ + const wxStringCharType *start = wx_str(); \ + wxStringCharType *end; \ + T val = func(start, &end, base); \ + \ + /* return true only if scan was stopped by the terminating NUL and */ \ + /* if the string was not empty to start with and no under/overflow */ \ + /* occurred: */ \ + if ( *end || end == start DO_IF_NOT_WINCE(|| errno == ERANGE) ) \ + return false; \ + *out = val; \ + return true - // return true only if scan was stopped by the terminating NUL and if the - // string was not empty to start with and no under/overflow occurred - return !*end && (end != start) -#ifndef __WXWINCE__ - && (errno != ERANGE) -#endif - ; -} - -bool wxString::ToLong(long *val, int base) const +bool wxString::ToLong(long *pVal, int base) const { - return wxStringToIntType((const wxChar*)c_str(), val, base, wxStrtol); + WX_STRING_TO_INT_TYPE(pVal, base, wxStrtol, long); } -bool wxString::ToULong(unsigned long *val, int base) const +bool wxString::ToULong(unsigned long *pVal, int base) const { - return wxStringToIntType((const wxChar*)c_str(), val, base, wxStrtoul); + WX_STRING_TO_INT_TYPE(pVal, base, wxStrtoul, unsigned long); } -bool wxString::ToLongLong(wxLongLong_t *val, int base) const +bool wxString::ToLongLong(wxLongLong_t *pVal, int base) const { - return wxStringToIntType((const wxChar*)c_str(), val, base, wxStrtoll); + WX_STRING_TO_INT_TYPE(pVal, base, wxStrtoll, wxLongLong_t); } -bool wxString::ToULongLong(wxULongLong_t *val, int base) const +bool wxString::ToULongLong(wxULongLong_t *pVal, int base) const { - return wxStringToIntType((const wxChar*)c_str(), val, base, wxStrtoull); + WX_STRING_TO_INT_TYPE(pVal, base, wxStrtoull, wxULongLong_t); } -bool wxString::ToDouble(double *val) const +bool wxString::ToDouble(double *pVal) const { - wxCHECK_MSG( val, false, _T("NULL pointer in wxString::ToDouble") ); + wxCHECK_MSG( pVal, false, _T("NULL output pointer") ); -#ifndef __WXWINCE__ - errno = 0; -#endif + DO_IF_NOT_WINCE( errno = 0; ) const wxChar *start = c_str(); wxChar *end; - *val = wxStrtod(start, &end); + double val = wxStrtod(start, &end); // return true only if scan was stopped by the terminating NUL and if the // string was not empty to start with and no under/overflow occurred - return !*end && (end != start) -#ifndef __WXWINCE__ - && (errno != ERANGE) -#endif - ; + if ( *end || end == start DO_IF_NOT_WINCE(|| errno == ERANGE) ) + return false; + + *pVal = val; + + return true; } // --------------------------------------------------------------------------- // formatted output // --------------------------------------------------------------------------- +#if !wxUSE_UTF8_LOCALE_ONLY /* static */ #ifdef wxNEEDS_WXSTRING_PRINTF_MIXIN -wxString wxStringPrintfMixinBase::DoFormat(const wxChar *format, ...) +wxString wxStringPrintfMixinBase::DoFormatWchar(const wxChar *format, ...) #else -wxString wxString::DoFormat(const wxChar *format, ...) +wxString wxString::DoFormatWchar(const wxChar *format, ...) #endif { va_list argptr; @@ -1853,6 +1520,23 @@ wxString wxString::DoFormat(const wxChar *format, ...) return s; } +#endif // !wxUSE_UTF8_LOCALE_ONLY + +#if wxUSE_UNICODE_UTF8 +/* static */ +wxString wxString::DoFormatUtf8(const char *format, ...) +{ + va_list argptr; + va_start(argptr, format); + + wxString s; + s.PrintfV(format, argptr); + + va_end(argptr); + + return s; +} +#endif // wxUSE_UNICODE_UTF8 /* static */ wxString wxString::FormatV(const wxString& format, va_list argptr) @@ -1862,10 +1546,11 @@ wxString wxString::FormatV(const wxString& format, va_list argptr) return s; } +#if !wxUSE_UTF8_LOCALE_ONLY #ifdef wxNEEDS_WXSTRING_PRINTF_MIXIN -int wxStringPrintfMixinBase::DoPrintf(const wxChar *format, ...) +int wxStringPrintfMixinBase::DoPrintfWchar(const wxChar *format, ...) #else -int wxString::DoPrintf(const wxChar *format, ...) +int wxString::DoPrintfWchar(const wxChar *format, ...) #endif { va_list argptr; @@ -1886,19 +1571,106 @@ int wxString::DoPrintf(const wxChar *format, ...) return iLen; } +#endif // !wxUSE_UTF8_LOCALE_ONLY -int wxString::PrintfV(const wxString& format, va_list argptr) +#if wxUSE_UNICODE_UTF8 +int wxString::DoPrintfUtf8(const char *format, ...) +{ + va_list argptr; + va_start(argptr, format); + + int iLen = PrintfV(format, argptr); + + va_end(argptr); + + return iLen; +} +#endif // wxUSE_UNICODE_UTF8 + +/* + Uses wxVsnprintf and places the result into the this string. + + In ANSI build, wxVsnprintf is effectively vsnprintf but in Unicode build + it is vswprintf. Due to a discrepancy between vsnprintf and vswprintf in + the ISO C99 (and thus SUSv3) standard the return value for the case of + an undersized buffer is inconsistent. For conforming vsnprintf + implementations the function must return the number of characters that + would have been printed had the buffer been large enough. For conforming + vswprintf implementations the function must return a negative number + and set errno. + + What vswprintf sets errno to is undefined but Darwin seems to set it to + EOVERFLOW. The only expected errno are EILSEQ and EINVAL. Both of + those are defined in the standard and backed up by several conformance + statements. Note that ENOMEM mentioned in the manual page does not + apply to swprintf, only wprintf and fwprintf. + + Official manual page: + http://www.opengroup.org/onlinepubs/009695399/functions/swprintf.html + + Some conformance statements (AIX, Solaris): + http://www.opengroup.org/csq/view.mhtml?RID=ibm%2FSD1%2F3 + http://www.theopengroup.org/csq/view.mhtml?norationale=1&noreferences=1&RID=Fujitsu%2FSE2%2F10 + + Since EILSEQ and EINVAL are rather common but EOVERFLOW is not and since + EILSEQ and EINVAL are specifically defined to mean the error is other than + an undersized buffer and no other errno are defined we treat those two + as meaning hard errors and everything else gets the old behavior which + is to keep looping and increasing buffer size until the function succeeds. + + In practice it's impossible to determine before compilation which behavior + may be used. The vswprintf function may have vsnprintf-like behavior or + vice-versa. Behavior detected on one release can theoretically change + with an updated release. Not to mention that configure testing for it + would require the test to be run on the host system, not the build system + which makes cross compilation difficult. Therefore, we make no assumptions + about behavior and try our best to handle every known case, including the + case where wxVsnprintf returns a negative number and fails to set errno. + + There is yet one more non-standard implementation and that is our own. + Fortunately, that can be detected at compile-time. + + On top of all that, ISO C99 explicitly defines snprintf to write a null + character to the last position of the specified buffer. That would be at + at the given buffer size minus 1. It is supposed to do this even if it + turns out that the buffer is sized too small. + + Darwin (tested on 10.5) follows the C99 behavior exactly. + + Glibc 2.6 almost follows the C99 behavior except vswprintf never sets + errno even when it fails. However, it only seems to ever fail due + to an undersized buffer. +*/ +#if wxUSE_UNICODE_UTF8 +template +#else +// we only need one version in non-UTF8 builds and at least two Windows +// compilers have problems with this function template, so use just one +// normal function here +#endif +static int DoStringPrintfV(wxString& str, + const wxString& format, va_list argptr) { int size = 1024; for ( ;; ) { - wxStringBuffer tmp(*this, size + 1); +#if wxUSE_UNICODE_UTF8 + BufferType tmp(str, size + 1); + typename BufferType::CharType *buf = tmp; +#else + wxStringBuffer tmp(str, size + 1); wxChar *buf = tmp; +#endif if ( !buf ) { // out of memory + + // in UTF-8 build, leaving uninitialized junk in the buffer + // could result in invalid non-empty UTF-8 string, so just + // reset the string to empty on failure: + buf[0] = '\0'; return -1; } @@ -1906,12 +1678,19 @@ int wxString::PrintfV(const wxString& format, va_list argptr) // only a copy va_list argptrcopy; wxVaCopy(argptrcopy, argptr); - int len = wxVsnprintf(buf, size, (const wxChar*)/*FIXME-UTF8*/format, argptrcopy); + +#ifndef __WXWINCE__ + // Set errno to 0 to make it determinate if wxVsnprintf fails to set it. + errno = 0; +#endif + int len = wxVsnprintf(buf, size, format, argptrcopy); va_end(argptrcopy); // some implementations of vsnprintf() don't NUL terminate // the string if there is not enough space for it so // always do it manually + // FIXME: This really seems to be the wrong and would be an off-by-one + // bug except the code above allocates an extra character. buf[size] = _T('\0'); // vsnprintf() may return either -1 (traditional Unix behaviour) or the @@ -1919,27 +1698,47 @@ int wxString::PrintfV(const wxString& format, va_list argptr) // buffer were large enough (newer standards such as Unix98) if ( len < 0 ) { + // NB: wxVsnprintf() may call either wxCRT_VsnprintfW or + // wxCRT_VsnprintfA in UTF-8 build; wxUSE_WXVSNPRINTF + // is true if *both* of them use our own implementation, + // otherwise we can't be sure #if wxUSE_WXVSNPRINTF // we know that our own implementation of wxVsnprintf() returns -1 // only for a format error - thus there's something wrong with // the user's format string + buf[0] = '\0'; return -1; -#else // assume that system version only returns error if not enough space +#else // possibly using system version + // assume it only returns error if there is not enough space, but + // as we don't know how much we need, double the current size of + // the buffer +#ifndef __WXWINCE__ + if( (errno == EILSEQ) || (errno == EINVAL) ) + // If errno was set to one of the two well-known hard errors + // then fail immediately to avoid an infinite loop. + return -1; + else +#endif // __WXWINCE__ // still not enough, as we don't know how much we need, double the // current size of the buffer - size *= 2; + size *= 2; #endif // wxUSE_WXVSNPRINTF/!wxUSE_WXVSNPRINTF } else if ( len >= size ) { #if wxUSE_WXVSNPRINTF - // we know that our own implementation of wxVsnprintf() returns + // we know that our own implementation of wxVsnprintf() returns // size+1 when there's not enough space but that's not the size // of the required buffer! size *= 2; // so we just double the current size of the buffer #else // some vsnprintf() implementations NUL-terminate the buffer and // some don't in len == size case, to be safe always add 1 + // FIXME: I don't quite understand this comment. The vsnprintf + // function is specifically defined to return the number of + // characters printed not including the null terminator. + // So OF COURSE you need to add 1 to get the right buffer size. + // The following line is definitely correct, no question. size = len + 1; #endif } @@ -1950,9 +1749,34 @@ int wxString::PrintfV(const wxString& format, va_list argptr) } // we could have overshot - Shrink(); + str.Shrink(); - return length(); + return str.length(); +} + +int wxString::PrintfV(const wxString& format, va_list argptr) +{ +#if wxUSE_UNICODE_UTF8 + #if wxUSE_STL_BASED_WXSTRING + typedef wxStringTypeBuffer Utf8Buffer; + #else + typedef wxStringInternalBuffer Utf8Buffer; + #endif +#endif + +#if wxUSE_UTF8_LOCALE_ONLY + return DoStringPrintfV(*this, format, argptr); +#else + #if wxUSE_UNICODE_UTF8 + if ( wxLocaleIsUtf8 ) + return DoStringPrintfV(*this, format, argptr); + else + // wxChar* version + return DoStringPrintfV(*this, format, argptr); + #else + return DoStringPrintfV(*this, format, argptr); + #endif // UTF8/WCHAR +#endif } // ---------------------------------------------------------------------------- @@ -2124,3 +1948,32 @@ wxString wxString::Upper() const // convert to lower case, return the copy of the string wxString wxString::Lower() const { wxString s(*this); return s.MakeLower(); } + +// ---------------------------------------------------------------------------- +// wxUTF8StringBuffer +// ---------------------------------------------------------------------------- + +#if wxUSE_UNICODE_WCHAR +wxUTF8StringBuffer::~wxUTF8StringBuffer() +{ + wxMBConvStrictUTF8 conv; + size_t wlen = conv.ToWChar(NULL, 0, m_buf); + wxCHECK_RET( wlen != wxCONV_FAILED, "invalid UTF-8 data in string buffer?" ); + + wxStringInternalBuffer wbuf(m_str, wlen); + conv.ToWChar(wbuf, wlen, m_buf); +} + +wxUTF8StringBufferLength::~wxUTF8StringBufferLength() +{ + wxCHECK_RET(m_lenSet, "length not set"); + + wxMBConvStrictUTF8 conv; + size_t wlen = conv.ToWChar(NULL, 0, m_buf, m_len); + wxCHECK_RET( wlen != wxCONV_FAILED, "invalid UTF-8 data in string buffer?" ); + + wxStringInternalBufferLength wbuf(m_str, wlen); + conv.ToWChar(wbuf, wlen, m_buf, m_len); + wbuf.SetLength(wlen); +} +#endif // wxUSE_UNICODE_WCHAR