]>
git.saurik.com Git - wxWidgets.git/blob - src/common/stringops.cpp
1 /////////////////////////////////////////////////////////////////////////////
2 // Name: src/common/stringops.cpp
3 // Purpose: implementation of wxString primitive operations
4 // Author: Vaclav Slavik
8 // Copyright: (c) 2007 REA Elektronik GmbH
9 // Licence: wxWindows licence
10 /////////////////////////////////////////////////////////////////////////////
12 // ===========================================================================
14 // ===========================================================================
16 // For compilers that support precompilation, includes "wx.h".
17 #include "wx/wxprec.h"
24 #include "wx/stringops.h"
27 // ===========================================================================
29 // ===========================================================================
31 #if wxUSE_UNICODE_UTF8
33 // ---------------------------------------------------------------------------
34 // UTF-8 sequences lengths
35 // ---------------------------------------------------------------------------
37 unsigned char wxStringOperationsUtf8::ms_utf8IterTable
[256] = {
38 // single-byte sequences (ASCII):
39 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 00..0F
40 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 10..1F
41 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 20..2F
42 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 30..3F
43 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 40..4F
44 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 50..5F
45 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 60..6F
46 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 70..7F
48 // these are invalid, we use step 1 to skip
49 // over them (should never happen):
50 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 80..8F
51 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 90..9F
52 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // A0..AF
53 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // B0..BF
56 // two-byte sequences:
57 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C2..CF
58 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // D0..DF
60 // three-byte sequences:
61 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // E0..EF
63 // four-byte sequences:
64 4, 4, 4, 4, 4, // F0..F4
66 // these are invalid again (5- or 6-byte
67 // sequences and sequences for code points
68 // above U+10FFFF, as restricted by RFC 3629):
69 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // F5..FF
72 // ---------------------------------------------------------------------------
74 // ---------------------------------------------------------------------------
77 // Table 3.1B from Unicode spec: Legal UTF-8 Byte Sequences
79 // Code Points | 1st Byte | 2nd Byte | 3rd Byte | 4th Byte |
80 // -------------------+----------+----------+----------+----------+
81 // U+0000..U+007F | 00..7F | | | |
82 // U+0080..U+07FF | C2..DF | 80..BF | | |
83 // U+0800..U+0FFF | E0 | A0..BF | 80..BF | |
84 // U+1000..U+FFFF | E1..EF | 80..BF | 80..BF | |
85 // U+10000..U+3FFFF | F0 | 90..BF | 80..BF | 80..BF |
86 // U+40000..U+FFFFF | F1..F3 | 80..BF | 80..BF | 80..BF |
87 // U+100000..U+10FFFF | F4 | 80..8F | 80..BF | 80..BF |
88 // -------------------+----------+----------+----------+----------+
90 bool wxStringOperationsUtf8::IsValidUtf8String(const char *str
, size_t len
)
93 return true; // empty string is UTF8 string
95 const unsigned char *c
= (const unsigned char*)str
;
96 const unsigned char * const end
= (len
== wxStringImpl::npos
) ? NULL
: c
+ len
;
98 for ( ; c
!= end
&& *c
; ++c
)
100 unsigned char b
= *c
;
104 // if the string is not NULL-terminated, verify we have enough
105 // bytes in it left for current character's encoding:
106 if ( c
+ ms_utf8IterTable
[*c
] > end
)
110 if ( b
<= 0x7F ) // 00..7F
113 else if ( b
< 0xC2 ) // invalid lead bytes: 80..C1
116 // two-byte sequences:
117 else if ( b
<= 0xDF ) // C2..DF
120 if ( !(b
>= 0x80 && b
<= 0xBF ) )
124 // three-byte sequences:
125 else if ( b
== 0xE0 )
128 if ( !(b
>= 0xA0 && b
<= 0xBF ) )
131 if ( !(b
>= 0x80 && b
<= 0xBF ) )
134 else if ( b
<= 0xEF ) // E1..EF
136 for ( int i
= 0; i
< 2; ++i
)
139 if ( !(b
>= 0x80 && b
<= 0xBF ) )
144 // four-byte sequences:
145 else if ( b
== 0xF0 )
148 if ( !(b
>= 0x90 && b
<= 0xBF ) )
150 for ( int i
= 0; i
< 2; ++i
)
153 if ( !(b
>= 0x80 && b
<= 0xBF ) )
157 else if ( b
<= 0xF3 ) // F1..F3
159 for ( int i
= 0; i
< 3; ++i
)
162 if ( !(b
>= 0x80 && b
<= 0xBF ) )
166 else if ( b
== 0xF4 )
169 if ( !(b
>= 0x80 && b
<= 0x8F ) )
171 for ( int i
= 0; i
< 2; ++i
)
174 if ( !(b
>= 0x80 && b
<= 0xBF ) )
178 else // otherwise, it's invalid lead byte
186 bool wxStringOperationsUtf8::IsValidUtf8LeadByte(unsigned char c
)
188 return (c
<= 0x7F) || (c
>= 0xC2 && c
<= 0xF4);
193 wxStringOperationsUtf8::Utf8CharBuffer
194 wxStringOperationsUtf8::EncodeChar(const wxUniChar
& ch
)
197 char *out
= buf
.data
;
199 wxUniChar::value_type code
= ch
.GetValue();
201 // Char. number range | UTF-8 octet sequence
202 // (hexadecimal) | (binary)
203 // ----------------------+---------------------------------------------
204 // 0000 0000 - 0000 007F | 0xxxxxxx
205 // 0000 0080 - 0000 07FF | 110xxxxx 10xxxxxx
206 // 0000 0800 - 0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
207 // 0001 0000 - 0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
209 // Code point value is stored in bits marked with 'x', lowest-order bit
210 // of the value on the right side in the diagram above.
218 else if ( code
<= 0x07FF )
221 // NB: this line takes 6 least significant bits, encodes them as
222 // 10xxxxxx and discards them so that the next byte can be encoded:
223 out
[1] = 0x80 | (code
& 0x3F); code
>>= 6;
224 out
[0] = 0xC0 | code
;
226 else if ( code
< 0xFFFF )
229 out
[2] = 0x80 | (code
& 0x3F); code
>>= 6;
230 out
[1] = 0x80 | (code
& 0x3F); code
>>= 6;
231 out
[0] = 0xE0 | code
;
233 else if ( code
<= 0x10FFFF )
236 out
[3] = 0x80 | (code
& 0x3F); code
>>= 6;
237 out
[2] = 0x80 | (code
& 0x3F); code
>>= 6;
238 out
[1] = 0x80 | (code
& 0x3F); code
>>= 6;
239 out
[0] = 0xF0 | code
;
243 wxFAIL_MSG( _T("trying to encode undefined Unicode character") );
251 wxStringOperationsUtf8::DecodeChar(wxStringImpl::const_iterator i
)
253 wxASSERT( IsValidUtf8LeadByte(*i
) );
255 wxUniChar::value_type code
= 0;
256 size_t len
= GetUtf8CharLength(*i
);
257 wxASSERT_MSG( len
<= 4, _T("invalid UTF-8 sequence length") );
259 // Char. number range | UTF-8 octet sequence
260 // (hexadecimal) | (binary)
261 // ----------------------+---------------------------------------------
262 // 0000 0000 - 0000 007F | 0xxxxxxx
263 // 0000 0080 - 0000 07FF | 110xxxxx 10xxxxxx
264 // 0000 0800 - 0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
265 // 0001 0000 - 0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
267 // Code point value is stored in bits marked with 'x', lowest-order bit
268 // of the value on the right side in the diagram above.
271 // mask to extract lead byte's value ('x' bits above), by sequence's length:
272 static const unsigned char s_leadValueMask
[4] = { 0x7F, 0x1F, 0x0F, 0x07 };
274 // mask and value of lead byte's most significant bits, by length:
275 static const unsigned char s_leadMarkerMask
[4] = { 0x80, 0xE0, 0xF0, 0xF8 };
276 static const unsigned char s_leadMarkerVal
[4] = { 0x00, 0xC0, 0xE0, 0xF0 };
279 // extract the lead byte's value bits:
280 wxASSERT_MSG( ((unsigned char)*i
& s_leadMarkerMask
[len
-1]) ==
281 s_leadMarkerVal
[len
-1],
282 _T("invalid UTF-8 lead byte") );
283 code
= (unsigned char)*i
& s_leadValueMask
[len
-1];
285 // all remaining bytes, if any, are handled in the same way regardless of
286 // sequence's length:
287 for ( ++i
; len
> 1; --len
, ++i
)
289 wxASSERT_MSG( ((unsigned char)*i
& 0xC0) == 0x80,
290 _T("invalid UTF-8 byte") );
293 code
|= (unsigned char)*i
& 0x3F;
296 return wxUniChar(code
);
299 wxCharBuffer
wxStringOperationsUtf8::EncodeNChars(size_t n
, const wxUniChar
& ch
)
301 Utf8CharBuffer
once(EncodeChar(ch
));
302 // the IncIter() table can be used to determine the length of ch's encoding:
303 size_t len
= ms_utf8IterTable
[(unsigned char)once
.data
[0]];
305 wxCharBuffer
buf(n
* len
);
306 char *ptr
= buf
.data();
307 for ( size_t i
= 0; i
< n
; i
++, ptr
+= len
)
309 memcpy(ptr
, once
.data
, len
);
315 #endif // wxUSE_UNICODE_UTF8