]>
git.saurik.com Git - wxWidgets.git/blob - src/common/stringops.cpp
1 /////////////////////////////////////////////////////////////////////////////
2 // Name: src/common/stringops.cpp
3 // Purpose: implementation of wxString primitive operations
4 // Author: Vaclav Slavik
8 // Copyright: (c) 2007 REA Elektronik GmbH
9 // Licence: wxWindows licence
10 /////////////////////////////////////////////////////////////////////////////
12 // ===========================================================================
14 // ===========================================================================
16 // For compilers that support precompilation, includes "wx.h".
17 #include "wx/wxprec.h"
24 #include "wx/stringops.h"
27 // ===========================================================================
29 // ===========================================================================
31 #if wxUSE_UNICODE_UTF8
33 // ---------------------------------------------------------------------------
34 // UTF-8 sequences lengths
35 // ---------------------------------------------------------------------------
37 unsigned char wxStringOperationsUtf8::ms_utf8IterTable
[256] = {
38 // single-byte sequences (ASCII):
39 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 00..0F
40 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 10..1F
41 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 20..2F
42 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 30..3F
43 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 40..4F
44 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 50..5F
45 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 60..6F
46 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 70..7F
48 // these are invalid, we use step 1 to skip
49 // over them (should never happen):
50 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 80..8F
51 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 90..9F
52 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A0..AF
53 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B0..BF
56 // two-byte sequences:
57 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C2..CF
58 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // D0..DF
60 // three-byte sequences:
61 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // E0..EF
63 // four-byte sequences:
64 4, 4, 4, 4, 4, // F0..F4
66 // these are invalid again (5- or 6-byte
67 // sequences and sequences for code points
68 // above U+10FFFF, as restricted by RFC 3629):
69 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // F5..FF
72 // ---------------------------------------------------------------------------
74 // ---------------------------------------------------------------------------
77 // Table 3.1B from Unicode spec: Legal UTF-8 Byte Sequences
79 // Code Points | 1st Byte | 2nd Byte | 3rd Byte | 4th Byte |
80 // -------------------+----------+----------+----------+----------+
81 // U+0000..U+007F | 00..7F | | | |
82 // U+0080..U+07FF | C2..DF | 80..BF | | |
83 // U+0800..U+0FFF | E0 | A0..BF | 80..BF | |
84 // U+1000..U+FFFF | E1..EF | 80..BF | 80..BF | |
85 // U+10000..U+3FFFF | F0 | 90..BF | 80..BF | 80..BF |
86 // U+40000..U+FFFFF | F1..F3 | 80..BF | 80..BF | 80..BF |
87 // U+100000..U+10FFFF | F4 | 80..8F | 80..BF | 80..BF |
88 // -------------------+----------+----------+----------+----------+
90 bool wxStringOperationsUtf8::IsValidUtf8String(const char *str
)
93 return true; // empty string is UTF8 string
95 const unsigned char *c
= (const unsigned char*)str
;
101 if ( b
<= 0x7F ) // 00..7F
104 else if ( b
< 0xC2 ) // invalid lead bytes: 80..C1
107 // two-byte sequences:
108 else if ( b
<= 0xDF ) // C2..DF
111 if ( !(b
>= 0x80 && b
<= 0xBF ) )
115 // three-byte sequences:
116 else if ( b
== 0xE0 )
119 if ( !(b
>= 0xA0 && b
<= 0xBF ) )
122 if ( !(b
>= 0x80 && b
<= 0xBF ) )
125 else if ( b
<= 0xEF ) // E1..EF
127 for ( int i
= 0; i
< 2; ++i
)
130 if ( !(b
>= 0x80 && b
<= 0xBF ) )
135 // four-byte sequences:
136 else if ( b
== 0xF0 )
139 if ( !(b
>= 0x90 && b
<= 0xBF ) )
141 for ( int i
= 0; i
< 2; ++i
)
144 if ( !(b
>= 0x80 && b
<= 0xBF ) )
148 else if ( b
<= 0xF3 ) // F1..F3
150 for ( int i
= 0; i
< 3; ++i
)
153 if ( !(b
>= 0x80 && b
<= 0xBF ) )
157 else if ( b
== 0xF4 )
160 if ( !(b
>= 0x80 && b
<= 0x8F ) )
162 for ( int i
= 0; i
< 2; ++i
)
165 if ( !(b
>= 0x80 && b
<= 0xBF ) )
169 else // otherwise, it's invalid lead byte
177 bool wxStringOperationsUtf8::IsValidUtf8LeadByte(unsigned char c
)
179 return (c
<= 0x7F) || (c
>= 0xC2 && c
<= 0xF4);
184 wxStringOperationsUtf8::Utf8CharBuffer
185 wxStringOperationsUtf8::EncodeChar(const wxUniChar
& ch
)
188 char *out
= buf
.data
;
190 wxUniChar::value_type code
= ch
.GetValue();
192 // Char. number range | UTF-8 octet sequence
193 // (hexadecimal) | (binary)
194 // ----------------------+---------------------------------------------
195 // 0000 0000 - 0000 007F | 0xxxxxxx
196 // 0000 0080 - 0000 07FF | 110xxxxx 10xxxxxx
197 // 0000 0800 - 0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
198 // 0001 0000 - 0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
200 // Code point value is stored in bits marked with 'x', lowest-order bit
201 // of the value on the right side in the diagram above.
209 else if ( code
<= 0x07FF )
212 // NB: this line takes 6 least significant bits, encodes them as
213 // 10xxxxxx and discards them so that the next byte can be encoded:
214 out
[1] = 0x80 | (code
& 0x3F); code
>>= 6;
215 out
[0] = 0xC0 | code
;
217 else if ( code
< 0xFFFF )
220 out
[2] = 0x80 | (code
& 0x3F); code
>>= 6;
221 out
[1] = 0x80 | (code
& 0x3F); code
>>= 6;
222 out
[0] = 0xE0 | code
;
224 else if ( code
<= 0x10FFFF )
227 out
[3] = 0x80 | (code
& 0x3F); code
>>= 6;
228 out
[2] = 0x80 | (code
& 0x3F); code
>>= 6;
229 out
[1] = 0x80 | (code
& 0x3F); code
>>= 6;
230 out
[0] = 0xF0 | code
;
234 wxFAIL_MSG( _T("trying to encode undefined Unicode character") );
242 wxStringOperationsUtf8::DecodeChar(wxStringImpl::const_iterator i
)
244 wxASSERT( IsValidUtf8LeadByte(*i
) );
246 wxUniChar::value_type code
= 0;
247 size_t len
= GetUtf8CharLength(*i
);
248 wxASSERT_MSG( len
<= 4, _T("invalid UTF-8 sequence length") );
250 // Char. number range | UTF-8 octet sequence
251 // (hexadecimal) | (binary)
252 // ----------------------+---------------------------------------------
253 // 0000 0000 - 0000 007F | 0xxxxxxx
254 // 0000 0080 - 0000 07FF | 110xxxxx 10xxxxxx
255 // 0000 0800 - 0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
256 // 0001 0000 - 0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
258 // Code point value is stored in bits marked with 'x', lowest-order bit
259 // of the value on the right side in the diagram above.
262 // mask to extract lead byte's value ('x' bits above), by sequence's length:
263 static const unsigned char s_leadValueMask
[4] = { 0x7F, 0x1F, 0x0F, 0x07 };
265 // mask and value of lead byte's most significant bits, by length:
266 static const unsigned char s_leadMarkerMask
[4] = { 0x80, 0xE0, 0xF0, 0xF8 };
267 static const unsigned char s_leadMarkerVal
[4] = { 0x00, 0xC0, 0xE0, 0xF0 };
270 // extract the lead byte's value bits:
271 wxASSERT_MSG( ((unsigned char)*i
& s_leadMarkerMask
[len
-1]) ==
272 s_leadMarkerVal
[len
-1],
273 _T("invalid UTF-8 lead byte") );
274 code
= (unsigned char)*i
& s_leadValueMask
[len
-1];
276 // all remaining bytes, if any, are handled in the same way regardless of
277 // sequence's length:
278 for ( ++i
; len
> 1; --len
, ++i
)
280 wxASSERT_MSG( ((unsigned char)*i
& 0xC0) == 0x80,
281 _T("invalid UTF-8 byte") );
284 code
|= (unsigned char)*i
& 0x3F;
287 return wxUniChar(code
);
290 wxCharBuffer
wxStringOperationsUtf8::EncodeNChars(size_t n
, const wxUniChar
& ch
)
292 Utf8CharBuffer
once(EncodeChar(ch
));
293 // the IncIter() table can be used to determine the length of ch's encoding:
294 size_t len
= ms_utf8IterTable
[(unsigned char)once
.data
[0]];
296 wxCharBuffer
buf(n
* len
);
297 char *ptr
= buf
.data();
298 for ( size_t i
= 0; i
< n
; i
++, ptr
+= len
)
300 memcpy(ptr
, once
.data
, len
);
306 #endif // wxUSE_UNICODE_UTF8