1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
4 ******************************************************************************
6 * Copyright (C) 1999-2012, International Business Machines
7 * Corporation and others. All Rights Reserved.
9 ******************************************************************************
10 * file name: utf_impl.cpp
12 * tab size: 8 (not used)
15 * created on: 1999sep13
16 * created by: Markus W. Scherer
18 * This file provides implementation functions for macros in the utfXX.h
19 * that would otherwise be too long as macros.
22 /* set import/export definitions */
27 #include "unicode/utypes.h"
28 #include "unicode/utf.h"
29 #include "unicode/utf8.h"
33 * Table of the number of utf8 trail bytes, indexed by the lead byte.
34 * Used by the deprecated macro UTF8_COUNT_TRAIL_BYTES, defined in utf_old.h
36 * The current macro, U8_COUNT_TRAIL_BYTES, does _not_ use this table.
38 * Note that this table cannot be removed, even if UTF8_COUNT_TRAIL_BYTES were
39 * changed to no longer use it. References to the table from expansions of UTF8_COUNT_TRAIL_BYTES
40 * may exist in old client code that must continue to run with newer icu library versions.
42 * This table could be replaced on many machines by
43 * a few lines of assembler code using an
44 * "index of first 0-bit from msb" instruction and
45 * one or two more integer instructions.
47 * For example, on an i386, do something like
49 * - NOT AL (8-bit, leave b15..b8==0..0, reverse only b7..b0)
51 * - BSR BX, AX (16-bit)
52 * - MOV AX, 6 (result)
53 * - JZ finish (ZF==1 if leadByte==0xff)
54 * - SUB AX, BX (result)
56 * (BSR: Bit Scan Reverse, scans for a 1-bit, starting from the MSB)
58 extern "C" U_EXPORT
const uint8_t
59 utf8_countTrailBytes
[256]={
60 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
61 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
62 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
63 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
65 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
66 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
67 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
68 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
70 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
71 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
72 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
73 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
76 // 2-byte lead bytes C2..DF
77 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
78 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
80 // 3-byte lead bytes E0..EF
81 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
82 // 4-byte lead bytes F0..F4
84 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
89 // Same values as UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_2, UTF_ERROR_VALUE,
90 // but without relying on the obsolete unicode/utf_old.h.
96 errorValue(int32_t count
, int8_t strict
) {
98 return utf8_errorValue
[count
];
99 } else if(strict
==-3) {
107 * Handle the non-inline part of the U8_NEXT() and U8_NEXT_FFFD() macros
108 * and their obsolete sibling UTF8_NEXT_CHAR_SAFE().
110 * U8_NEXT() supports NUL-terminated strings indicated via length<0.
112 * The "strict" parameter controls the error behavior:
113 * <0 "Safe" behavior of U8_NEXT():
114 * -1: All illegal byte sequences yield U_SENTINEL=-1.
115 * -2: Same as -1, except for lenient treatment of surrogate code points as legal.
116 * Some implementations use this for roundtripping of
117 * Unicode 16-bit strings that are not well-formed UTF-16, that is, they
118 * contain unpaired surrogates.
119 * -3: All illegal byte sequences yield U+FFFD.
120 * 0 Obsolete "safe" behavior of UTF8_NEXT_CHAR_SAFE(..., FALSE):
121 * All illegal byte sequences yield a positive code point such that this
122 * result code point would be encoded with the same number of bytes as
123 * the illegal sequence.
124 * >0 Obsolete "strict" behavior of UTF8_NEXT_CHAR_SAFE(..., TRUE):
125 * Same as the obsolete "safe" behavior, but non-characters are also treated
126 * like illegal sequences.
128 * Note that a UBool is the same as an int8_t.
130 U_CAPI UChar32 U_EXPORT2
131 utf8_nextCharSafeBody(const uint8_t *s
, int32_t *pi
, int32_t length
, UChar32 c
, UBool strict
) {
132 // *pi is one after byte c.
134 // length can be negative for NUL-terminated strings: Read and validate one byte at a time.
135 if(i
==length
|| c
>0xf4) {
136 // end of string, or not a lead byte
138 // Test for 4-byte sequences first because
139 // U8_NEXT() handles shorter valid sequences inline.
140 uint8_t t1
=s
[i
], t2
, t3
;
142 if(U8_IS_VALID_LEAD4_AND_T1(c
, t1
) &&
143 ++i
!=length
&& (t2
=s
[i
]-0x80)<=0x3f &&
144 ++i
!=length
&& (t3
=s
[i
]-0x80)<=0x3f) {
146 c
=(c
<<18)|((t1
&0x3f)<<12)|(t2
<<6)|t3
;
147 // strict: forbid non-characters like U+fffe
148 if(strict
<=0 || !U_IS_UNICODE_NONCHAR(c
)) {
157 if(U8_IS_VALID_LEAD3_AND_T1(c
, t1
) &&
158 ++i
!=length
&& (t2
=s
[i
]-0x80)<=0x3f) {
160 c
=(c
<<12)|((t1
&0x3f)<<6)|t2
;
161 // strict: forbid non-characters like U+fffe
162 if(strict
<=0 || !U_IS_UNICODE_NONCHAR(c
)) {
168 // strict=-2 -> lenient: allow surrogates
169 uint8_t t1
=s
[i
]-0x80, t2
;
170 if(t1
<=0x3f && (c
>0 || t1
>=0x20) &&
171 ++i
!=length
&& (t2
=s
[i
]-0x80)<=0x3f) {
173 return (c
<<12)|(t1
<<6)|t2
;
177 uint8_t t1
=s
[i
]-0x80;
180 return ((c
-0xc0)<<6)|t1
;
182 } // else 0x80<=c<0xc2 is not a lead byte
185 c
=errorValue(i
-*pi
, strict
);
190 U_CAPI
int32_t U_EXPORT2
191 utf8_appendCharSafeBody(uint8_t *s
, int32_t i
, int32_t length
, UChar32 c
, UBool
*pIsError
) {
192 if((uint32_t)(c
)<=0x7ff) {
194 (s
)[(i
)++]=(uint8_t)(((c
)>>6)|0xc0);
195 (s
)[(i
)++]=(uint8_t)(((c
)&0x3f)|0x80);
198 } else if((uint32_t)(c
)<=0xffff) {
199 /* Starting with Unicode 3.2, surrogate code points must not be encoded in UTF-8. */
200 if((i
)+2<(length
) && !U_IS_SURROGATE(c
)) {
201 (s
)[(i
)++]=(uint8_t)(((c
)>>12)|0xe0);
202 (s
)[(i
)++]=(uint8_t)((((c
)>>6)&0x3f)|0x80);
203 (s
)[(i
)++]=(uint8_t)(((c
)&0x3f)|0x80);
206 } else if((uint32_t)(c
)<=0x10ffff) {
208 (s
)[(i
)++]=(uint8_t)(((c
)>>18)|0xf0);
209 (s
)[(i
)++]=(uint8_t)((((c
)>>12)&0x3f)|0x80);
210 (s
)[(i
)++]=(uint8_t)((((c
)>>6)&0x3f)|0x80);
211 (s
)[(i
)++]=(uint8_t)(((c
)&0x3f)|0x80);
215 /* c>0x10ffff or not enough space, write an error value */
227 c
=utf8_errorValue
[length
-1];
228 U8_APPEND_UNSAFE(s
, offset
, c
);
235 U_CAPI UChar32 U_EXPORT2
236 utf8_prevCharSafeBody(const uint8_t *s
, int32_t start
, int32_t *pi
, UChar32 c
, UBool strict
) {
237 // *pi is the index of byte c.
239 if(U8_IS_TRAIL(c
) && i
>start
) {
244 return ((b1
-0xc0)<<6)|(c
&0x3f);
245 } else if(b1
<0xf0 ? U8_IS_VALID_LEAD3_AND_T1(b1
, c
) : U8_IS_VALID_LEAD4_AND_T1(b1
, c
)) {
246 // Truncated 3- or 4-byte sequence.
248 return errorValue(1, strict
);
250 } else if(U8_IS_TRAIL(b1
) && i
>start
) {
251 // Extract the value bits from the last trail byte.
254 if(0xe0<=b2
&& b2
<=0xf4) {
258 if(U8_IS_VALID_LEAD3_AND_T1(b2
, b1
)) {
260 c
=(b2
<<12)|((b1
&0x3f)<<6)|c
;
261 if(strict
<=0 || !U_IS_UNICODE_NONCHAR(c
)) {
264 // strict: forbid non-characters like U+fffe
265 return errorValue(2, strict
);
269 // strict=-2 -> lenient: allow surrogates
271 if((b2
>0 || b1
>=0x20)) {
273 return (b2
<<12)|(b1
<<6)|c
;
276 } else if(U8_IS_VALID_LEAD4_AND_T1(b2
, b1
)) {
277 // Truncated 4-byte sequence.
279 return errorValue(2, strict
);
281 } else if(U8_IS_TRAIL(b2
) && i
>start
) {
283 if(0xf0<=b3
&& b3
<=0xf4) {
285 if(U8_IS_VALID_LEAD4_AND_T1(b3
, b2
)) {
287 c
=(b3
<<18)|((b2
&0x3f)<<12)|((b1
&0x3f)<<6)|c
;
288 if(strict
<=0 || !U_IS_UNICODE_NONCHAR(c
)) {
291 // strict: forbid non-characters like U+fffe
292 return errorValue(3, strict
);
299 return errorValue(0, strict
);
302 U_CAPI
int32_t U_EXPORT2
303 utf8_back1SafeBody(const uint8_t *s
, int32_t start
, int32_t i
) {
304 // Same as utf8_prevCharSafeBody(..., strict=-1) minus assembling code points.
307 if(U8_IS_TRAIL(c
) && i
>start
) {
311 (b1
<0xf0 ? U8_IS_VALID_LEAD3_AND_T1(b1
, c
) : U8_IS_VALID_LEAD4_AND_T1(b1
, c
))) {
314 } else if(U8_IS_TRAIL(b1
) && i
>start
) {
316 if(0xe0<=b2
&& b2
<=0xf4) {
317 if(b2
<0xf0 ? U8_IS_VALID_LEAD3_AND_T1(b2
, b1
) : U8_IS_VALID_LEAD4_AND_T1(b2
, b1
)) {
320 } else if(U8_IS_TRAIL(b2
) && i
>start
) {
322 if(0xf0<=b3
&& b3
<=0xf4 && U8_IS_VALID_LEAD4_AND_T1(b3
, b2
)) {