1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
4 *******************************************************************************
6 * Copyright (C) 2004-2014, International Business Machines
7 * Corporation and others. All Rights Reserved.
9 *******************************************************************************
10 * file name: ucase.cpp
12 * tab size: 8 (not used)
15 * created on: 2004aug30
16 * created by: Markus W. Scherer
18 * Low-level Unicode character/string case mapping code.
19 * Much code moved here (and modified) from uchar.c.
22 #include "unicode/utypes.h"
23 #include "unicode/unistr.h"
24 #include "unicode/uset.h"
25 #include "unicode/udata.h" /* UDataInfo */
26 #include "unicode/utf16.h"
27 #include "ucmndata.h" /* DataHeader */
37 const int32_t *indexes
;
38 const uint16_t *exceptions
;
39 const uint16_t *unfold
;
42 uint8_t formatVersion
[4];
45 /* ucase_props_data.h is machine-generated by gencase --csource */
46 #define INCLUDED_FROM_UCASE_CPP
47 #include "ucase_props_data.h"
49 /* set of property starts for UnicodeSet ------------------------------------ */
51 static UBool U_CALLCONV
52 _enumPropertyStartsRange(const void *context
, UChar32 start
, UChar32
/*end*/, uint32_t /*value*/) {
53 /* add the start code point to the USet */
54 const USetAdder
*sa
=(const USetAdder
*)context
;
55 sa
->add(sa
->set
, start
);
59 U_CFUNC
void U_EXPORT2
60 ucase_addPropertyStarts(const USetAdder
*sa
, UErrorCode
*pErrorCode
) {
61 if(U_FAILURE(*pErrorCode
)) {
65 /* add the start code point of each same-value range of the trie */
66 utrie2_enum(&ucase_props_singleton
.trie
, NULL
, _enumPropertyStartsRange
, sa
);
68 /* add code points with hardcoded properties, plus the ones following them */
70 /* (none right now, see comment below) */
73 * Omit code points with hardcoded specialcasing properties
74 * because we do not build property UnicodeSets for them right now.
78 /* data access primitives --------------------------------------------------- */
80 #define GET_EXCEPTIONS(csp, props) ((csp)->exceptions+((props)>>UCASE_EXC_SHIFT))
82 #define PROPS_HAS_EXCEPTION(props) ((props)&UCASE_EXCEPTION)
84 /* number of bits in an 8-bit integer value */
85 static const uint8_t flagsOffset
[256]={
86 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
87 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
88 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
89 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
90 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
91 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
92 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
93 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
94 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
95 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
96 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
97 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
98 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
99 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
100 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
101 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
104 #define HAS_SLOT(flags, idx) ((flags)&(1<<(idx)))
105 #define SLOT_OFFSET(flags, idx) flagsOffset[(flags)&((1<<(idx))-1)]
108 * Get the value of an optional-value slot where HAS_SLOT(excWord, idx).
110 * @param excWord (in) initial exceptions word
111 * @param idx (in) desired slot index
112 * @param pExc16 (in/out) const uint16_t * after excWord=*pExc16++;
113 * moved to the last uint16_t of the value, use +1 for beginning of next slot
114 * @param value (out) int32_t or uint32_t output if hasSlot, otherwise not modified
116 #define GET_SLOT_VALUE(excWord, idx, pExc16, value) \
117 if(((excWord)&UCASE_EXC_DOUBLE_SLOTS)==0) { \
118 (pExc16)+=SLOT_OFFSET(excWord, idx); \
121 (pExc16)+=2*SLOT_OFFSET(excWord, idx); \
123 (value)=((value)<<16)|*pExc16; \
126 /* simple case mappings ----------------------------------------------------- */
128 U_CAPI UChar32 U_EXPORT2
129 ucase_tolower(UChar32 c
) {
130 uint16_t props
=UTRIE2_GET16(&ucase_props_singleton
.trie
, c
);
131 if(!PROPS_HAS_EXCEPTION(props
)) {
132 if(UCASE_GET_TYPE(props
)>=UCASE_UPPER
) {
133 c
+=UCASE_GET_DELTA(props
);
136 const uint16_t *pe
=GET_EXCEPTIONS(&ucase_props_singleton
, props
);
137 uint16_t excWord
=*pe
++;
138 if(HAS_SLOT(excWord
, UCASE_EXC_LOWER
)) {
139 GET_SLOT_VALUE(excWord
, UCASE_EXC_LOWER
, pe
, c
);
145 U_CAPI UChar32 U_EXPORT2
146 ucase_toupper(UChar32 c
) {
147 uint16_t props
=UTRIE2_GET16(&ucase_props_singleton
.trie
, c
);
148 if(!PROPS_HAS_EXCEPTION(props
)) {
149 if(UCASE_GET_TYPE(props
)==UCASE_LOWER
) {
150 c
+=UCASE_GET_DELTA(props
);
153 const uint16_t *pe
=GET_EXCEPTIONS(&ucase_props_singleton
, props
);
154 uint16_t excWord
=*pe
++;
155 if(HAS_SLOT(excWord
, UCASE_EXC_UPPER
)) {
156 GET_SLOT_VALUE(excWord
, UCASE_EXC_UPPER
, pe
, c
);
162 U_CAPI UChar32 U_EXPORT2
163 ucase_totitle(UChar32 c
) {
164 uint16_t props
=UTRIE2_GET16(&ucase_props_singleton
.trie
, c
);
165 if(!PROPS_HAS_EXCEPTION(props
)) {
166 if(UCASE_GET_TYPE(props
)==UCASE_LOWER
) {
167 c
+=UCASE_GET_DELTA(props
);
170 const uint16_t *pe
=GET_EXCEPTIONS(&ucase_props_singleton
, props
);
171 uint16_t excWord
=*pe
++;
173 if(HAS_SLOT(excWord
, UCASE_EXC_TITLE
)) {
175 } else if(HAS_SLOT(excWord
, UCASE_EXC_UPPER
)) {
180 GET_SLOT_VALUE(excWord
, idx
, pe
, c
);
185 static const UChar iDot
[2] = { 0x69, 0x307 };
186 static const UChar jDot
[2] = { 0x6a, 0x307 };
187 static const UChar iOgonekDot
[3] = { 0x12f, 0x307 };
188 static const UChar iDotGrave
[3] = { 0x69, 0x307, 0x300 };
189 static const UChar iDotAcute
[3] = { 0x69, 0x307, 0x301 };
190 static const UChar iDotTilde
[3] = { 0x69, 0x307, 0x303 };
193 U_CFUNC
void U_EXPORT2
194 ucase_addCaseClosure(UChar32 c
, const USetAdder
*sa
) {
198 * Hardcode the case closure of i and its relatives and ignore the
199 * data file data for these characters.
200 * The Turkic dotless i and dotted I with their case mapping conditions
201 * and case folding option make the related characters behave specially.
202 * This code matches their closure behavior to their case folding behavior.
207 /* regular i and I are in one equivalence class */
208 sa
->add(sa
->set
, 0x69);
211 sa
->add(sa
->set
, 0x49);
214 /* dotted I is in a class with <0069 0307> (for canonical equivalence with <0049 0307>) */
215 sa
->addString(sa
->set
, iDot
, 2);
218 /* dotless i is in a class by itself */
221 /* otherwise use the data file data */
225 props
=UTRIE2_GET16(&ucase_props_singleton
.trie
, c
);
226 if(!PROPS_HAS_EXCEPTION(props
)) {
227 if(UCASE_GET_TYPE(props
)!=UCASE_NONE
) {
228 /* add the one simple case mapping, no matter what type it is */
229 int32_t delta
=UCASE_GET_DELTA(props
);
231 sa
->add(sa
->set
, c
+delta
);
236 * c has exceptions, so there may be multiple simple and/or
237 * full case mappings. Add them all.
239 const uint16_t *pe0
, *pe
=GET_EXCEPTIONS(&ucase_props_singleton
, props
);
240 const UChar
*closure
;
241 uint16_t excWord
=*pe
++;
242 int32_t idx
, closureLength
, fullLength
, length
;
246 /* add all simple case mappings */
247 for(idx
=UCASE_EXC_LOWER
; idx
<=UCASE_EXC_TITLE
; ++idx
) {
248 if(HAS_SLOT(excWord
, idx
)) {
250 GET_SLOT_VALUE(excWord
, idx
, pe
, c
);
255 /* get the closure string pointer & length */
256 if(HAS_SLOT(excWord
, UCASE_EXC_CLOSURE
)) {
258 GET_SLOT_VALUE(excWord
, UCASE_EXC_CLOSURE
, pe
, closureLength
);
259 closureLength
&=UCASE_CLOSURE_MAX_LENGTH
; /* higher bits are reserved */
260 closure
=(const UChar
*)pe
+1; /* behind this slot, unless there are full case mappings */
266 /* add the full case folding */
267 if(HAS_SLOT(excWord
, UCASE_EXC_FULL_MAPPINGS
)) {
269 GET_SLOT_VALUE(excWord
, UCASE_EXC_FULL_MAPPINGS
, pe
, fullLength
);
271 /* start of full case mapping strings */
274 fullLength
&=0xffff; /* bits 16 and higher are reserved */
276 /* skip the lowercase result string */
277 pe
+=fullLength
&UCASE_FULL_LOWER
;
280 /* add the full case folding string */
281 length
=fullLength
&0xf;
283 sa
->addString(sa
->set
, (const UChar
*)pe
, length
);
287 /* skip the uppercase and titlecase strings */
293 closure
=(const UChar
*)pe
; /* behind full case mappings */
296 /* add each code point in the closure string */
297 for(idx
=0; idx
<closureLength
;) {
298 U16_NEXT_UNSAFE(closure
, idx
, c
);
305 * compare s, which has a length, with t, which has a maximum length or is NUL-terminated
306 * must be length>0 and max>0 and length<=max
308 static inline int32_t
309 strcmpMax(const UChar
*s
, int32_t length
, const UChar
*t
, int32_t max
) {
312 max
-=length
; /* we require length<=max, so no need to decrement max in the loop */
317 return 1; /* reached the end of t but not of s */
321 return c1
; /* return difference result */
324 /* ends with length==0 */
326 if(max
==0 || *t
==0) {
327 return 0; /* equal to length of both strings */
329 return -max
; /* return lengh difference */
333 U_CFUNC UBool U_EXPORT2
334 ucase_addStringCaseClosure(const UChar
*s
, int32_t length
, const USetAdder
*sa
) {
335 int32_t i
, start
, limit
, result
, unfoldRows
, unfoldRowWidth
, unfoldStringWidth
;
337 if(ucase_props_singleton
.unfold
==NULL
|| s
==NULL
) {
338 return FALSE
; /* no reverse case folding data, or no string */
341 /* the string is too short to find any match */
343 * more precise would be:
344 * if(!u_strHasMoreChar32Than(s, length, 1))
345 * but this does not make much practical difference because
346 * a single supplementary code point would just not be found
351 const uint16_t *unfold
=ucase_props_singleton
.unfold
;
352 unfoldRows
=unfold
[UCASE_UNFOLD_ROWS
];
353 unfoldRowWidth
=unfold
[UCASE_UNFOLD_ROW_WIDTH
];
354 unfoldStringWidth
=unfold
[UCASE_UNFOLD_STRING_WIDTH
];
355 unfold
+=unfoldRowWidth
;
357 if(length
>unfoldStringWidth
) {
358 /* the string is too long to find any match */
362 /* do a binary search for the string */
367 const UChar
*p
=reinterpret_cast<const UChar
*>(unfold
+(i
*unfoldRowWidth
));
368 result
=strcmpMax(s
, length
, p
, unfoldStringWidth
);
371 /* found the string: add each code point, and its case closure */
374 for(i
=unfoldStringWidth
; i
<unfoldRowWidth
&& p
[i
]!=0;) {
375 U16_NEXT_UNSAFE(p
, i
, c
);
377 ucase_addCaseClosure(c
, sa
);
380 } else if(result
<0) {
382 } else /* result>0 */ {
387 return FALSE
; /* string not found */
392 FullCaseFoldingIterator::FullCaseFoldingIterator()
393 : unfold(reinterpret_cast<const UChar
*>(ucase_props_singleton
.unfold
)),
394 unfoldRows(unfold
[UCASE_UNFOLD_ROWS
]),
395 unfoldRowWidth(unfold
[UCASE_UNFOLD_ROW_WIDTH
]),
396 unfoldStringWidth(unfold
[UCASE_UNFOLD_STRING_WIDTH
]),
398 rowCpIndex(unfoldStringWidth
) {
399 unfold
+=unfoldRowWidth
;
403 FullCaseFoldingIterator::next(UnicodeString
&full
) {
404 // Advance past the last-delivered code point.
405 const UChar
*p
=unfold
+(currentRow
*unfoldRowWidth
);
406 if(rowCpIndex
>=unfoldRowWidth
|| p
[rowCpIndex
]==0) {
409 rowCpIndex
=unfoldStringWidth
;
411 if(currentRow
>=unfoldRows
) { return U_SENTINEL
; }
412 // Set "full" to the NUL-terminated string in the first unfold column.
413 int32_t length
=unfoldStringWidth
;
414 while(length
>0 && p
[length
-1]==0) { --length
; }
415 full
.setTo(FALSE
, p
, length
);
416 // Return the code point.
418 U16_NEXT_UNSAFE(p
, rowCpIndex
, c
);
424 /** @return UCASE_NONE, UCASE_LOWER, UCASE_UPPER, UCASE_TITLE */
425 U_CAPI
int32_t U_EXPORT2
426 ucase_getType(UChar32 c
) {
427 uint16_t props
=UTRIE2_GET16(&ucase_props_singleton
.trie
, c
);
428 return UCASE_GET_TYPE(props
);
431 /** @return same as ucase_getType() and set bit 2 if c is case-ignorable */
432 U_CAPI
int32_t U_EXPORT2
433 ucase_getTypeOrIgnorable(UChar32 c
) {
434 uint16_t props
=UTRIE2_GET16(&ucase_props_singleton
.trie
, c
);
435 return UCASE_GET_TYPE_AND_IGNORABLE(props
);
438 /** @return UCASE_NO_DOT, UCASE_SOFT_DOTTED, UCASE_ABOVE, UCASE_OTHER_ACCENT */
439 static inline int32_t
440 getDotType(UChar32 c
) {
441 uint16_t props
=UTRIE2_GET16(&ucase_props_singleton
.trie
, c
);
442 if(!PROPS_HAS_EXCEPTION(props
)) {
443 return props
&UCASE_DOT_MASK
;
445 const uint16_t *pe
=GET_EXCEPTIONS(&ucase_props_singleton
, props
);
446 return (*pe
>>UCASE_EXC_DOT_SHIFT
)&UCASE_DOT_MASK
;
450 U_CAPI UBool U_EXPORT2
451 ucase_isSoftDotted(UChar32 c
) {
452 return (UBool
)(getDotType(c
)==UCASE_SOFT_DOTTED
);
455 U_CAPI UBool U_EXPORT2
456 ucase_isCaseSensitive(UChar32 c
) {
457 uint16_t props
=UTRIE2_GET16(&ucase_props_singleton
.trie
, c
);
458 return (UBool
)((props
&UCASE_SENSITIVE
)!=0);
461 /* string casing ------------------------------------------------------------ */
464 * These internal functions form the core of string case mappings.
465 * They map single code points to result code points or strings and take
466 * all necessary conditions (context, locale ID, options) into account.
468 * They do not iterate over the source or write to the destination
469 * so that the same functions are useful for non-standard string storage,
470 * such as in a Replaceable (for Transliterator) or UTF-8/32 strings etc.
471 * For the same reason, the "surrounding text" context is passed in as a
472 * UCaseContextIterator which does not make any assumptions about
473 * the underlying storage.
475 * This section contains helper functions that check for conditions
476 * in the input text surrounding the current code point
477 * according to SpecialCasing.txt.
479 * Each helper function gets the index
480 * - after the current code point if it looks at following text
481 * - before the current code point if it looks at preceding text
483 * Unicode 3.2 UAX 21 "Case Mappings" defines the conditions as follows:
486 * C is preceded by a sequence consisting of
487 * a cased letter and a case-ignorable sequence,
488 * and C is not followed by a sequence consisting of
489 * an ignorable sequence and then a cased letter.
492 * C is followed by one or more characters of combining class 230 (ABOVE)
493 * in the combining character sequence.
496 * The last preceding character with combining class of zero before C
498 * and there is no intervening combining character class 230 (ABOVE).
501 * C is followed by combining dot above (U+0307).
502 * Any sequence of characters with a combining class that is neither 0 nor 230
503 * may intervene between the current character and the combining dot above.
505 * The erratum from 2002-10-31 adds the condition
508 * The last preceding base character was an uppercase I, and there is no
509 * intervening combining character class 230 (ABOVE).
511 * (See Jitterbug 2344 and the comments on After_I below.)
513 * Helper definitions in Unicode 3.2 UAX 21:
515 * D1. A character C is defined to be cased
516 * if it meets any of the following criteria:
518 * - The general category of C is Titlecase Letter (Lt)
519 * - In [CoreProps], C has one of the properties Uppercase, or Lowercase
520 * - Given D = NFD(C), then it is not the case that:
521 * D = UCD_lower(D) = UCD_upper(D) = UCD_title(D)
522 * (This third criterium does not add any characters to the list
523 * for Unicode 3.2. Ignored.)
525 * D2. A character C is defined to be case-ignorable
526 * if it meets either of the following criteria:
528 * - The general category of C is
529 * Nonspacing Mark (Mn), or Enclosing Mark (Me), or Format Control (Cf), or
530 * Letter Modifier (Lm), or Symbol Modifier (Sk)
531 * - C is one of the following characters
533 * U+00AD SOFT HYPHEN (SHY)
534 * U+2019 RIGHT SINGLE QUOTATION MARK
535 * (the preferred character for apostrophe)
537 * D3. A case-ignorable sequence is a sequence of
538 * zero or more case-ignorable characters.
541 #define is_d(c) ((c)=='d' || (c)=='D')
542 #define is_e(c) ((c)=='e' || (c)=='E')
543 #define is_i(c) ((c)=='i' || (c)=='I')
544 #define is_l(c) ((c)=='l' || (c)=='L')
545 #define is_r(c) ((c)=='r' || (c)=='R')
546 #define is_t(c) ((c)=='t' || (c)=='T')
547 #define is_u(c) ((c)=='u' || (c)=='U')
548 #define is_z(c) ((c)=='z' || (c)=='Z')
551 #define is_sep(c) ((c)=='_' || (c)=='-' || (c)==0)
554 * Requires non-NULL locale ID but otherwise does the equivalent of
555 * checking for language codes as if uloc_getLanguage() were called:
556 * Accepts both 2- and 3-letter codes and accepts case variants.
559 ucase_getCaseLocale(const char *locale
) {
561 * This function used to use uloc_getLanguage(), but the current code
562 * removes the dependency of this low-level code on uloc implementation code
563 * and is faster because not the whole locale ID has to be
564 * examined and copied/transformed.
566 * Because this code does not want to depend on uloc, the caller must
567 * pass in a non-NULL locale, i.e., may need to call uloc_getDefault().
570 // Fastpath for English "en" which is often used for default (=root locale) case mappings,
571 // and for Chinese "zh": Very common but no special case mapping behavior.
572 // Then check lowercase vs. uppercase to reduce the number of comparisons
573 // for other locales without special behavior.
583 return UCASE_LOC_GREEK
;
586 // en, es, ... -> root
588 return UCASE_LOC_ROOT
;
589 #if U_CHARSET_FAMILY==U_ASCII_FAMILY
590 } else if(c
>='a') { // ASCII a-z = 0x61..0x7a, after A-Z
591 #elif U_CHARSET_FAMILY==U_EBCDIC_FAMILY
592 } else if(c
<='z') { // EBCDIC a-z = 0x81..0xa9 with two gaps, before A-Z
594 # error Unknown charset family!
606 return UCASE_LOC_TURKISH
;
618 return UCASE_LOC_TURKISH
;
630 return UCASE_LOC_LITHUANIAN
;
642 return UCASE_LOC_DUTCH
;
648 // Same code as for lowercase c but also check for 'E'.
658 return UCASE_LOC_TURKISH
;
670 return UCASE_LOC_TURKISH
;
682 return UCASE_LOC_LITHUANIAN
;
694 return UCASE_LOC_GREEK
;
706 return UCASE_LOC_DUTCH
;
711 return UCASE_LOC_ROOT
;
716 * {case-ignorable}* cased
718 * (dir determines looking forward/backward)
719 * If a character is case-ignorable, it is skipped regardless of whether
720 * it is also cased or not.
723 isFollowedByCasedLetter(UCaseContextIterator
*iter
, void *context
, int8_t dir
) {
730 for(/* dir!=0 sets direction */; (c
=iter(context
, dir
))>=0; dir
=0) {
731 int32_t type
=ucase_getTypeOrIgnorable(c
);
733 /* case-ignorable, continue with the loop */
734 } else if(type
!=UCASE_NONE
) {
735 return TRUE
; /* followed by cased letter */
737 return FALSE
; /* uncased and not case-ignorable */
741 return FALSE
; /* not followed by cased letter */
744 /* Is preceded by Soft_Dotted character with no intervening cc=230 ? */
746 isPrecededBySoftDotted(UCaseContextIterator
*iter
, void *context
) {
755 for(dir
=-1; (c
=iter(context
, dir
))>=0; dir
=0) {
756 dotType
=getDotType(c
);
757 if(dotType
==UCASE_SOFT_DOTTED
) {
758 return TRUE
; /* preceded by TYPE_i */
759 } else if(dotType
!=UCASE_OTHER_ACCENT
) {
760 return FALSE
; /* preceded by different base character (not TYPE_i), or intervening cc==230 */
764 return FALSE
; /* not preceded by TYPE_i */
768 * See Jitterbug 2344:
769 * The condition After_I for Turkic-lowercasing of U+0307 combining dot above
770 * is checked in ICU 2.0, 2.1, 2.6 but was not in 2.2 & 2.4 because
771 * we made those releases compatible with Unicode 3.2 which had not fixed
772 * a related bug in SpecialCasing.txt.
774 * From the Jitterbug 2344 text:
775 * ... this bug is listed as a Unicode erratum
776 * from 2002-10-31 at http://www.unicode.org/uni2errata/UnicodeErrata.html
778 * There are two errors in SpecialCasing.txt.
779 * 1. Missing semicolons on two lines. ... [irrelevant for ICU]
780 * 2. An incorrect context definition. Correct as follows:
781 * < 0307; ; 0307; 0307; tr After_Soft_Dotted; # COMBINING DOT ABOVE
782 * < 0307; ; 0307; 0307; az After_Soft_Dotted; # COMBINING DOT ABOVE
784 * > 0307; ; 0307; 0307; tr After_I; # COMBINING DOT ABOVE
785 * > 0307; ; 0307; 0307; az After_I; # COMBINING DOT ABOVE
786 * where the context After_I is defined as:
787 * The last preceding base character was an uppercase I, and there is no
788 * intervening combining character class 230 (ABOVE).
791 * Note that SpecialCasing.txt even in Unicode 3.2 described the condition as:
793 * # When lowercasing, remove dot_above in the sequence I + dot_above, which will turn into i.
794 * # This matches the behavior of the canonically equivalent I-dot_above
796 * See also the description in this place in older versions of uchar.c (revision 1.100).
798 * Markus W. Scherer 2003-feb-15
801 /* Is preceded by base character 'I' with no intervening cc=230 ? */
803 isPrecededBy_I(UCaseContextIterator
*iter
, void *context
) {
812 for(dir
=-1; (c
=iter(context
, dir
))>=0; dir
=0) {
814 return TRUE
; /* preceded by I */
816 dotType
=getDotType(c
);
817 if(dotType
!=UCASE_OTHER_ACCENT
) {
818 return FALSE
; /* preceded by different base character (not I), or intervening cc==230 */
822 return FALSE
; /* not preceded by I */
825 /* Is followed by one or more cc==230 ? */
827 isFollowedByMoreAbove(UCaseContextIterator
*iter
, void *context
) {
836 for(dir
=1; (c
=iter(context
, dir
))>=0; dir
=0) {
837 dotType
=getDotType(c
);
838 if(dotType
==UCASE_ABOVE
) {
839 return TRUE
; /* at least one cc==230 following */
840 } else if(dotType
!=UCASE_OTHER_ACCENT
) {
841 return FALSE
; /* next base character, no more cc==230 following */
845 return FALSE
; /* no more cc==230 following */
848 /* Is followed by a dot above (without cc==230 in between) ? */
850 isFollowedByDotAbove(UCaseContextIterator
*iter
, void *context
) {
859 for(dir
=1; (c
=iter(context
, dir
))>=0; dir
=0) {
863 dotType
=getDotType(c
);
864 if(dotType
!=UCASE_OTHER_ACCENT
) {
865 return FALSE
; /* next base character or cc==230 in between */
869 return FALSE
; /* no dot above following */
872 U_CAPI
int32_t U_EXPORT2
873 ucase_toFullLower(UChar32 c
,
874 UCaseContextIterator
*iter
, void *context
,
875 const UChar
**pString
,
877 // The sign of the result has meaning, input must be non-negative so that it can be returned as is.
880 uint16_t props
=UTRIE2_GET16(&ucase_props_singleton
.trie
, c
);
881 if(!PROPS_HAS_EXCEPTION(props
)) {
882 if(UCASE_GET_TYPE(props
)>=UCASE_UPPER
) {
883 result
=c
+UCASE_GET_DELTA(props
);
886 const uint16_t *pe
=GET_EXCEPTIONS(&ucase_props_singleton
, props
), *pe2
;
887 uint16_t excWord
=*pe
++;
892 if(excWord
&UCASE_EXC_CONDITIONAL_SPECIAL
) {
893 /* use hardcoded conditions and mappings */
896 * Test for conditional mappings first
897 * (otherwise the unconditional default mappings are always taken),
898 * then test for characters that have unconditional mappings in SpecialCasing.txt,
899 * then get the UnicodeData.txt mappings.
901 if( loc
==UCASE_LOC_LITHUANIAN
&&
902 /* base characters, find accents above */
903 (((c
==0x49 || c
==0x4a || c
==0x12e) &&
904 isFollowedByMoreAbove(iter
, context
)) ||
905 /* precomposed with accent above, no need to find one */
906 (c
==0xcc || c
==0xcd || c
==0x128))
911 # Lithuanian retains the dot in a lowercase i when followed by accents.
913 # Introduce an explicit dot above when lowercasing capital I's and J's
914 # whenever there are more accents above.
915 # (of the accents used in Lithuanian: grave, acute, tilde above, and ogonek)
917 0049; 0069 0307; 0049; 0049; lt More_Above; # LATIN CAPITAL LETTER I
918 004A; 006A 0307; 004A; 004A; lt More_Above; # LATIN CAPITAL LETTER J
919 012E; 012F 0307; 012E; 012E; lt More_Above; # LATIN CAPITAL LETTER I WITH OGONEK
920 00CC; 0069 0307 0300; 00CC; 00CC; lt; # LATIN CAPITAL LETTER I WITH GRAVE
921 00CD; 0069 0307 0301; 00CD; 00CD; lt; # LATIN CAPITAL LETTER I WITH ACUTE
922 0128; 0069 0307 0303; 0128; 0128; lt; # LATIN CAPITAL LETTER I WITH TILDE
925 case 0x49: /* LATIN CAPITAL LETTER I */
928 case 0x4a: /* LATIN CAPITAL LETTER J */
931 case 0x12e: /* LATIN CAPITAL LETTER I WITH OGONEK */
934 case 0xcc: /* LATIN CAPITAL LETTER I WITH GRAVE */
937 case 0xcd: /* LATIN CAPITAL LETTER I WITH ACUTE */
940 case 0x128: /* LATIN CAPITAL LETTER I WITH TILDE */
944 return 0; /* will not occur */
946 /* # Turkish and Azeri */
947 } else if(loc
==UCASE_LOC_TURKISH
&& c
==0x130) {
949 # I and i-dotless; I-dot and i are case pairs in Turkish and Azeri
950 # The following rules handle those cases.
952 0130; 0069; 0130; 0130; tr # LATIN CAPITAL LETTER I WITH DOT ABOVE
953 0130; 0069; 0130; 0130; az # LATIN CAPITAL LETTER I WITH DOT ABOVE
956 } else if(loc
==UCASE_LOC_TURKISH
&& c
==0x307 && isPrecededBy_I(iter
, context
)) {
958 # When lowercasing, remove dot_above in the sequence I + dot_above, which will turn into i.
959 # This matches the behavior of the canonically equivalent I-dot_above
961 0307; ; 0307; 0307; tr After_I; # COMBINING DOT ABOVE
962 0307; ; 0307; 0307; az After_I; # COMBINING DOT ABOVE
964 return 0; /* remove the dot (continue without output) */
965 } else if(loc
==UCASE_LOC_TURKISH
&& c
==0x49 && !isFollowedByDotAbove(iter
, context
)) {
967 # When lowercasing, unless an I is before a dot_above, it turns into a dotless i.
969 0049; 0131; 0049; 0049; tr Not_Before_Dot; # LATIN CAPITAL LETTER I
970 0049; 0131; 0049; 0049; az Not_Before_Dot; # LATIN CAPITAL LETTER I
973 } else if(c
==0x130) {
975 # Preserve canonical equivalence for I with dot. Turkic is handled below.
977 0130; 0069 0307; 0130; 0130; # LATIN CAPITAL LETTER I WITH DOT ABOVE
981 } else if( c
==0x3a3 &&
982 !isFollowedByCasedLetter(iter
, context
, 1) &&
983 isFollowedByCasedLetter(iter
, context
, -1) /* -1=preceded */
985 /* greek capital sigma maps depending on surrounding cased letters (see SpecialCasing.txt) */
987 # Special case for final form of sigma
989 03A3; 03C2; 03A3; 03A3; Final_Sigma; # GREEK CAPITAL LETTER SIGMA
991 return 0x3c2; /* greek small final sigma */
993 /* no known conditional special case mapping, use a normal mapping */
995 } else if(HAS_SLOT(excWord
, UCASE_EXC_FULL_MAPPINGS
)) {
996 GET_SLOT_VALUE(excWord
, UCASE_EXC_FULL_MAPPINGS
, pe
, full
);
997 full
&=UCASE_FULL_LOWER
;
999 /* set the output pointer to the lowercase mapping */
1000 *pString
=reinterpret_cast<const UChar
*>(pe
+1);
1002 /* return the string length */
1007 if(HAS_SLOT(excWord
, UCASE_EXC_LOWER
)) {
1008 GET_SLOT_VALUE(excWord
, UCASE_EXC_LOWER
, pe2
, result
);
1012 return (result
==c
) ? ~result
: result
;
1017 toUpperOrTitle(UChar32 c
,
1018 UCaseContextIterator
*iter
, void *context
,
1019 const UChar
**pString
,
1021 UBool upperNotTitle
) {
1022 // The sign of the result has meaning, input must be non-negative so that it can be returned as is.
1025 uint16_t props
=UTRIE2_GET16(&ucase_props_singleton
.trie
, c
);
1026 if(!PROPS_HAS_EXCEPTION(props
)) {
1027 if(UCASE_GET_TYPE(props
)==UCASE_LOWER
) {
1028 result
=c
+UCASE_GET_DELTA(props
);
1031 const uint16_t *pe
=GET_EXCEPTIONS(&ucase_props_singleton
, props
), *pe2
;
1032 uint16_t excWord
=*pe
++;
1037 if(excWord
&UCASE_EXC_CONDITIONAL_SPECIAL
) {
1038 /* use hardcoded conditions and mappings */
1039 if(loc
==UCASE_LOC_TURKISH
&& c
==0x69) {
1043 # I and i-dotless; I-dot and i are case pairs in Turkish and Azeri
1044 # The following rules handle those cases.
1046 # When uppercasing, i turns into a dotted capital I
1048 0069; 0069; 0130; 0130; tr; # LATIN SMALL LETTER I
1049 0069; 0069; 0130; 0130; az; # LATIN SMALL LETTER I
1052 } else if(loc
==UCASE_LOC_LITHUANIAN
&& c
==0x307 && isPrecededBySoftDotted(iter
, context
)) {
1056 # Lithuanian retains the dot in a lowercase i when followed by accents.
1058 # Remove DOT ABOVE after "i" with upper or titlecase
1060 0307; 0307; ; ; lt After_Soft_Dotted; # COMBINING DOT ABOVE
1062 return 0; /* remove the dot (continue without output) */
1064 /* no known conditional special case mapping, use a normal mapping */
1066 } else if(HAS_SLOT(excWord
, UCASE_EXC_FULL_MAPPINGS
)) {
1067 GET_SLOT_VALUE(excWord
, UCASE_EXC_FULL_MAPPINGS
, pe
, full
);
1069 /* start of full case mapping strings */
1072 /* skip the lowercase and case-folding result strings */
1073 pe
+=full
&UCASE_FULL_LOWER
;
1081 /* skip the uppercase result string */
1087 /* set the output pointer to the result string */
1088 *pString
=reinterpret_cast<const UChar
*>(pe
);
1090 /* return the string length */
1095 if(!upperNotTitle
&& HAS_SLOT(excWord
, UCASE_EXC_TITLE
)) {
1096 idx
=UCASE_EXC_TITLE
;
1097 } else if(HAS_SLOT(excWord
, UCASE_EXC_UPPER
)) {
1098 /* here, titlecase is same as uppercase */
1099 idx
=UCASE_EXC_UPPER
;
1103 GET_SLOT_VALUE(excWord
, idx
, pe2
, result
);
1106 return (result
==c
) ? ~result
: result
;
1109 U_CAPI
int32_t U_EXPORT2
1110 ucase_toFullUpper(UChar32 c
,
1111 UCaseContextIterator
*iter
, void *context
,
1112 const UChar
**pString
,
1113 int32_t caseLocale
) {
1114 return toUpperOrTitle(c
, iter
, context
, pString
, caseLocale
, TRUE
);
1117 U_CAPI
int32_t U_EXPORT2
1118 ucase_toFullTitle(UChar32 c
,
1119 UCaseContextIterator
*iter
, void *context
,
1120 const UChar
**pString
,
1121 int32_t caseLocale
) {
1122 return toUpperOrTitle(c
, iter
, context
, pString
, caseLocale
, FALSE
);
1125 /* case folding ------------------------------------------------------------- */
1128 * Case folding is similar to lowercasing.
1129 * The result may be a simple mapping, i.e., a single code point, or
1130 * a full mapping, i.e., a string.
1131 * If the case folding for a code point is the same as its simple (1:1) lowercase mapping,
1132 * then only the lowercase mapping is stored.
1134 * Some special cases are hardcoded because their conditions cannot be
1135 * parsed and processed from CaseFolding.txt.
1137 * Unicode 3.2 CaseFolding.txt specifies for its status field:
1139 # C: common case folding, common mappings shared by both simple and full mappings.
1140 # F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces.
1141 # S: simple case folding, mappings to single characters where different from F.
1142 # T: special case for uppercase I and dotted uppercase I
1143 # - For non-Turkic languages, this mapping is normally not used.
1144 # - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters.
1147 # A. To do a simple case folding, use the mappings with status C + S.
1148 # B. To do a full case folding, use the mappings with status C + F.
1150 # The mappings with status T can be used or omitted depending on the desired case-folding
1151 # behavior. (The default option is to exclude them.)
1153 * Unicode 3.2 has 'T' mappings as follows:
1155 0049; T; 0131; # LATIN CAPITAL LETTER I
1156 0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE
1158 * while the default mappings for these code points are:
1160 0049; C; 0069; # LATIN CAPITAL LETTER I
1161 0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE
1163 * U+0130 has no simple case folding (simple-case-folds to itself).
1166 /* return the simple case folding mapping for c */
1167 U_CAPI UChar32 U_EXPORT2
1168 ucase_fold(UChar32 c
, uint32_t options
) {
1169 uint16_t props
=UTRIE2_GET16(&ucase_props_singleton
.trie
, c
);
1170 if(!PROPS_HAS_EXCEPTION(props
)) {
1171 if(UCASE_GET_TYPE(props
)>=UCASE_UPPER
) {
1172 c
+=UCASE_GET_DELTA(props
);
1175 const uint16_t *pe
=GET_EXCEPTIONS(&ucase_props_singleton
, props
);
1176 uint16_t excWord
=*pe
++;
1178 if(excWord
&UCASE_EXC_CONDITIONAL_FOLD
) {
1179 /* special case folding mappings, hardcoded */
1180 if((options
&_FOLD_CASE_OPTIONS_MASK
)==U_FOLD_CASE_DEFAULT
) {
1181 /* default mappings */
1183 /* 0049; C; 0069; # LATIN CAPITAL LETTER I */
1185 } else if(c
==0x130) {
1186 /* no simple case folding for U+0130 */
1190 /* Turkic mappings */
1192 /* 0049; T; 0131; # LATIN CAPITAL LETTER I */
1194 } else if(c
==0x130) {
1195 /* 0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE */
1200 if(HAS_SLOT(excWord
, UCASE_EXC_FOLD
)) {
1202 } else if(HAS_SLOT(excWord
, UCASE_EXC_LOWER
)) {
1203 idx
=UCASE_EXC_LOWER
;
1207 GET_SLOT_VALUE(excWord
, idx
, pe
, c
);
1213 * Issue for canonical caseless match (UAX #21):
1214 * Turkic casefolding (using "T" mappings in CaseFolding.txt) does not preserve
1215 * canonical equivalence, unlike default-option casefolding.
1216 * For example, I-grave and I + grave fold to strings that are not canonically
1218 * For more details, see the comment in unorm_compare() in unorm.cpp
1219 * and the intermediate prototype changes for Jitterbug 2021.
1220 * (For example, revision 1.104 of uchar.c and 1.4 of CaseFolding.txt.)
1222 * This did not get fixed because it appears that it is not possible to fix
1223 * it for uppercase and lowercase characters (I-grave vs. i-grave)
1224 * together in a way that they still fold to common result strings.
1227 U_CAPI
int32_t U_EXPORT2
1228 ucase_toFullFolding(UChar32 c
,
1229 const UChar
**pString
,
1231 // The sign of the result has meaning, input must be non-negative so that it can be returned as is.
1234 uint16_t props
=UTRIE2_GET16(&ucase_props_singleton
.trie
, c
);
1235 if(!PROPS_HAS_EXCEPTION(props
)) {
1236 if(UCASE_GET_TYPE(props
)>=UCASE_UPPER
) {
1237 result
=c
+UCASE_GET_DELTA(props
);
1240 const uint16_t *pe
=GET_EXCEPTIONS(&ucase_props_singleton
, props
), *pe2
;
1241 uint16_t excWord
=*pe
++;
1246 if(excWord
&UCASE_EXC_CONDITIONAL_FOLD
) {
1247 /* use hardcoded conditions and mappings */
1248 if((options
&_FOLD_CASE_OPTIONS_MASK
)==U_FOLD_CASE_DEFAULT
) {
1249 /* default mappings */
1251 /* 0049; C; 0069; # LATIN CAPITAL LETTER I */
1253 } else if(c
==0x130) {
1254 /* 0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE */
1259 /* Turkic mappings */
1261 /* 0049; T; 0131; # LATIN CAPITAL LETTER I */
1263 } else if(c
==0x130) {
1264 /* 0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE */
1268 } else if(HAS_SLOT(excWord
, UCASE_EXC_FULL_MAPPINGS
)) {
1269 GET_SLOT_VALUE(excWord
, UCASE_EXC_FULL_MAPPINGS
, pe
, full
);
1271 /* start of full case mapping strings */
1274 /* skip the lowercase result string */
1275 pe
+=full
&UCASE_FULL_LOWER
;
1279 /* set the output pointer to the result string */
1280 *pString
=reinterpret_cast<const UChar
*>(pe
);
1282 /* return the string length */
1287 if(HAS_SLOT(excWord
, UCASE_EXC_FOLD
)) {
1289 } else if(HAS_SLOT(excWord
, UCASE_EXC_LOWER
)) {
1290 idx
=UCASE_EXC_LOWER
;
1294 GET_SLOT_VALUE(excWord
, idx
, pe2
, result
);
1297 return (result
==c
) ? ~result
: result
;
1300 /* case mapping properties API ---------------------------------------------- */
1302 /* public API (see uchar.h) */
1304 U_CAPI UBool U_EXPORT2
1305 u_isULowercase(UChar32 c
) {
1306 return (UBool
)(UCASE_LOWER
==ucase_getType(c
));
1309 U_CAPI UBool U_EXPORT2
1310 u_isUUppercase(UChar32 c
) {
1311 return (UBool
)(UCASE_UPPER
==ucase_getType(c
));
1314 /* Transforms the Unicode character to its lower case equivalent.*/
1315 U_CAPI UChar32 U_EXPORT2
1316 u_tolower(UChar32 c
) {
1317 return ucase_tolower(c
);
1320 /* Transforms the Unicode character to its upper case equivalent.*/
1321 U_CAPI UChar32 U_EXPORT2
1322 u_toupper(UChar32 c
) {
1323 return ucase_toupper(c
);
1326 /* Transforms the Unicode character to its title case equivalent.*/
1327 U_CAPI UChar32 U_EXPORT2
1328 u_totitle(UChar32 c
) {
1329 return ucase_totitle(c
);
1332 /* return the simple case folding mapping for c */
1333 U_CAPI UChar32 U_EXPORT2
1334 u_foldCase(UChar32 c
, uint32_t options
) {
1335 return ucase_fold(c
, options
);
1338 U_CFUNC
int32_t U_EXPORT2
1339 ucase_hasBinaryProperty(UChar32 c
, UProperty which
) {
1340 /* case mapping properties */
1341 const UChar
*resultString
;
1343 case UCHAR_LOWERCASE
:
1344 return (UBool
)(UCASE_LOWER
==ucase_getType(c
));
1345 case UCHAR_UPPERCASE
:
1346 return (UBool
)(UCASE_UPPER
==ucase_getType(c
));
1347 case UCHAR_SOFT_DOTTED
:
1348 return ucase_isSoftDotted(c
);
1349 case UCHAR_CASE_SENSITIVE
:
1350 return ucase_isCaseSensitive(c
);
1352 return (UBool
)(UCASE_NONE
!=ucase_getType(c
));
1353 case UCHAR_CASE_IGNORABLE
:
1354 return (UBool
)(ucase_getTypeOrIgnorable(c
)>>2);
1356 * Note: The following Changes_When_Xyz are defined as testing whether
1357 * the NFD form of the input changes when Xyz-case-mapped.
1358 * However, this simpler implementation of these properties,
1359 * ignoring NFD, passes the tests.
1360 * The implementation needs to be changed if the tests start failing.
1361 * When that happens, optimizations should be used to work with the
1362 * per-single-code point ucase_toFullXyz() functions unless
1363 * the NFD form has more than one code point,
1364 * and the property starts set needs to be the union of the
1365 * start sets for normalization and case mappings.
1367 case UCHAR_CHANGES_WHEN_LOWERCASED
:
1368 return (UBool
)(ucase_toFullLower(c
, NULL
, NULL
, &resultString
, UCASE_LOC_ROOT
)>=0);
1369 case UCHAR_CHANGES_WHEN_UPPERCASED
:
1370 return (UBool
)(ucase_toFullUpper(c
, NULL
, NULL
, &resultString
, UCASE_LOC_ROOT
)>=0);
1371 case UCHAR_CHANGES_WHEN_TITLECASED
:
1372 return (UBool
)(ucase_toFullTitle(c
, NULL
, NULL
, &resultString
, UCASE_LOC_ROOT
)>=0);
1373 /* case UCHAR_CHANGES_WHEN_CASEFOLDED: -- in uprops.c */
1374 case UCHAR_CHANGES_WHEN_CASEMAPPED
:
1376 ucase_toFullLower(c
, NULL
, NULL
, &resultString
, UCASE_LOC_ROOT
)>=0 ||
1377 ucase_toFullUpper(c
, NULL
, NULL
, &resultString
, UCASE_LOC_ROOT
)>=0 ||
1378 ucase_toFullTitle(c
, NULL
, NULL
, &resultString
, UCASE_LOC_ROOT
)>=0);