2 *******************************************************************************
4 * Copyright (C) 2001-2006, International Business Machines
5 * Corporation and others. All Rights Reserved.
7 *******************************************************************************
8 * file name: unormcmp.cpp
10 * tab size: 8 (not used)
13 * created on: 2004sep13
14 * created by: Markus W. Scherer
16 * unorm_compare() function moved here from unorm.cpp for better modularization.
17 * Depends on both normalization and case folding.
18 * Allows unorm.cpp to not depend on any character properties code.
21 #include "unicode/utypes.h"
23 #if !UCONFIG_NO_NORMALIZATION
25 #include "unicode/ustring.h"
26 #include "unicode/unorm.h"
27 #include "unicode/uniset.h"
34 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
36 /* compare canonically equivalent ------------------------------------------- */
39 * Compare two strings for canonical equivalence.
40 * Further options include case-insensitive comparison and
41 * code point order (as opposed to code unit order).
43 * In this function, canonical equivalence is optional as well.
44 * If canonical equivalence is tested, then both strings must fulfill
47 * Semantically, this is equivalent to
48 * strcmp[CodePointOrder](NFD(foldCase(s1)), NFD(foldCase(s2)))
49 * where code point order, NFD and foldCase are all optional.
51 * String comparisons almost always yield results before processing both strings
53 * They are generally more efficient working incrementally instead of
54 * performing the sub-processing (strlen, normalization, case-folding)
55 * on the entire strings first.
57 * It is also unnecessary to not normalize identical characters.
59 * This function works in principle as follows:
62 * get one code unit c1 from s1 (-1 if end of source)
63 * get one code unit c2 from s2 (-1 if end of source)
65 * if(either string finished) {
73 * try to decompose/case-fold c1/c2, and continue if one does;
75 * // still c1!=c2 and neither decomposes/case-folds, return result
79 * When a character decomposes, then the pointer for that source changes to
80 * the decomposition, pushing the previous pointer onto a stack.
81 * When the end of the decomposition is reached, then the code unit reader
82 * pops the previous source from the stack.
83 * (Same for case-folding.)
85 * This is complicated further by operating on variable-width UTF-16.
86 * The top part of the loop works on code units, while lookups for decomposition
87 * and case-folding need code points.
88 * Code points are assembled after the equality/end-of-source part.
89 * The source pointer is only advanced beyond all code units when the code point
90 * actually decomposes/case-folds.
92 * If we were on a trail surrogate unit when assembling a code point,
93 * and the code point decomposes/case-folds, then the decomposition/folding
94 * result must be compared with the part of the other string that corresponds to
95 * this string's lead surrogate.
96 * Since we only assemble a code point when hitting a trail unit when the
97 * preceding lead units were identical, we back up the other string by one unit
100 * The optional code point order comparison at the end works with
101 * the same fix-up as the other code point order comparison functions.
102 * See ustring.c and the comment near the end of this function.
104 * Assumption: A decomposition or case-folding result string never contains
105 * a single surrogate. This is a safe assumption in the Unicode Standard.
106 * Therefore, we do not need to check for surrogate pairs across
107 * decomposition/case-folding boundaries.
109 * Further assumptions (see verifications tstnorm.cpp):
110 * The API function checks for FCD first, while the core function
111 * first case-folds and then decomposes. This requires that case-folding does not
112 * un-FCD any strings.
114 * The API function may also NFD the input and turn off decomposition.
115 * This requires that case-folding does not un-NFD strings either.
117 * TODO If any of the above two assumptions is violated,
118 * then this entire code must be re-thought.
119 * If this happens, then a simple solution is to case-fold both strings up front
120 * and to turn off UNORM_INPUT_IS_FCD.
121 * We already do this when not both strings are in FCD because makeFCD
122 * would be a partial NFD before the case folding, which does not work.
123 * Note that all of this is only a problem when case-folding _and_
124 * canonical equivalence come together.
125 * (Comments in unorm_compare() are more up to date than this TODO.)
127 * This function could be moved to a different source file, at increased cost
128 * for calling the decomposition access function.
131 /* stack element for previous-level source/decomposition pointers */
132 struct CmpEquivLevel
{
133 const UChar
*start
, *s
, *limit
;
135 typedef struct CmpEquivLevel CmpEquivLevel
;
137 /* internal function */
139 unorm_cmpEquivFold(const UChar
*s1
, int32_t length1
,
140 const UChar
*s2
, int32_t length2
,
142 UErrorCode
*pErrorCode
) {
143 const UCaseProps
*csp
;
145 /* current-level start/limit - s1/s2 as current */
146 const UChar
*start1
, *start2
, *limit1
, *limit2
;
148 /* decomposition and case folding variables */
152 /* stacks of previous-level start/current/limit */
153 CmpEquivLevel stack1
[2], stack2
[2];
155 /* decomposition buffers for Hangul */
156 UChar decomp1
[4], decomp2
[4];
158 /* case folding buffers, only use current-level start/limit */
159 UChar fold1
[UCASE_MAX_STRING_LENGTH
+1], fold2
[UCASE_MAX_STRING_LENGTH
+1];
161 /* track which is the current level per string */
162 int32_t level1
, level2
;
164 /* current code units, and code points for lookups */
165 UChar32 c1
, c2
, cp1
, cp2
;
167 /* no argument error checking because this itself is not an API */
170 * assume that at least one of the options _COMPARE_EQUIV and U_COMPARE_IGNORE_CASE is set
171 * otherwise this function must behave exactly as uprv_strCompare()
172 * not checking for that here makes testing this function easier
175 /* normalization/properties data loaded? */
176 if( ((options
&_COMPARE_EQUIV
)!=0 && !unorm_haveData(pErrorCode
)) ||
177 U_FAILURE(*pErrorCode
)
181 if((options
&U_COMPARE_IGNORE_CASE
)!=0) {
182 csp
=ucase_getSingleton(pErrorCode
);
183 if(U_FAILURE(*pErrorCode
)) {
208 /* comparison loop */
211 * here a code unit value of -1 means "get another code unit"
212 * below it will mean "this source is finished"
216 /* get next code unit from string 1, post-increment */
218 if(s1
==limit1
|| ((c1
=*s1
)==0 && (limit1
==NULL
|| (options
&_STRNCMP_STYLE
)))) {
228 /* reached end of level buffer, pop one level */
231 start1
=stack1
[level1
].start
;
232 } while(start1
==NULL
);
234 limit1
=stack1
[level1
].limit
;
239 /* get next code unit from string 2, post-increment */
241 if(s2
==limit2
|| ((c2
=*s2
)==0 && (limit2
==NULL
|| (options
&_STRNCMP_STYLE
)))) {
251 /* reached end of level buffer, pop one level */
254 start2
=stack2
[level2
].start
;
255 } while(start2
==NULL
);
257 limit2
=stack2
[level2
].limit
;
263 * either variable c1, c2 is -1 only if the corresponding string is finished
267 return 0; /* c1==c2==-1 indicating end of strings */
269 c1
=c2
=-1; /* make us fetch new code units */
272 return -1; /* string 1 ends before string 2 */
274 return 1; /* string 2 ends before string 1 */
276 /* c1!=c2 && c1>=0 && c2>=0 */
278 /* get complete code points for c1, c2 for lookups if either is a surrogate */
280 if(U_IS_SURROGATE(c1
)) {
283 if(U_IS_SURROGATE_LEAD(c1
)) {
284 if(s1
!=limit1
&& U16_IS_TRAIL(c
=*s1
)) {
285 /* advance ++s1; only below if cp1 decomposes/case-folds */
286 cp1
=U16_GET_SUPPLEMENTARY(c1
, c
);
288 } else /* isTrail(c1) */ {
289 if(start1
<=(s1
-2) && U16_IS_LEAD(c
=*(s1
-2))) {
290 cp1
=U16_GET_SUPPLEMENTARY(c
, c1
);
296 if(U_IS_SURROGATE(c2
)) {
299 if(U_IS_SURROGATE_LEAD(c2
)) {
300 if(s2
!=limit2
&& U16_IS_TRAIL(c
=*s2
)) {
301 /* advance ++s2; only below if cp2 decomposes/case-folds */
302 cp2
=U16_GET_SUPPLEMENTARY(c2
, c
);
304 } else /* isTrail(c2) */ {
305 if(start2
<=(s2
-2) && U16_IS_LEAD(c
=*(s2
-2))) {
306 cp2
=U16_GET_SUPPLEMENTARY(c
, c2
);
312 * go down one level for each string
313 * continue with the main loop as soon as there is a real change
316 if( level1
==0 && (options
&U_COMPARE_IGNORE_CASE
) &&
317 (length
=ucase_toFullFolding(csp
, (UChar32
)cp1
, &p
, options
))>=0
319 /* cp1 case-folds to the code point "length" or to p[length] */
320 if(U_IS_SURROGATE(c1
)) {
321 if(U_IS_SURROGATE_LEAD(c1
)) {
322 /* advance beyond source surrogate pair if it case-folds */
324 } else /* isTrail(c1) */ {
326 * we got a supplementary code point when hitting its trail surrogate,
327 * therefore the lead surrogate must have been the same as in the other string;
328 * compare this decomposition with the lead surrogate in the other string
329 * remember that this simulates bulk text replacement:
330 * the decomposition would replace the entire code point
337 /* push current level pointers */
338 stack1
[0].start
=start1
;
340 stack1
[0].limit
=limit1
;
343 /* copy the folding result to fold1[] */
344 if(length
<=UCASE_MAX_STRING_LENGTH
) {
345 u_memcpy(fold1
, p
, length
);
348 U16_APPEND_UNSAFE(fold1
, i
, length
);
352 /* set next level pointers to case folding */
356 /* get ready to read from decomposition, continue with loop */
361 if( level2
==0 && (options
&U_COMPARE_IGNORE_CASE
) &&
362 (length
=ucase_toFullFolding(csp
, (UChar32
)cp2
, &p
, options
))>=0
364 /* cp2 case-folds to the code point "length" or to p[length] */
365 if(U_IS_SURROGATE(c2
)) {
366 if(U_IS_SURROGATE_LEAD(c2
)) {
367 /* advance beyond source surrogate pair if it case-folds */
369 } else /* isTrail(c2) */ {
371 * we got a supplementary code point when hitting its trail surrogate,
372 * therefore the lead surrogate must have been the same as in the other string;
373 * compare this decomposition with the lead surrogate in the other string
374 * remember that this simulates bulk text replacement:
375 * the decomposition would replace the entire code point
382 /* push current level pointers */
383 stack2
[0].start
=start2
;
385 stack2
[0].limit
=limit2
;
388 /* copy the folding result to fold2[] */
389 if(length
<=UCASE_MAX_STRING_LENGTH
) {
390 u_memcpy(fold2
, p
, length
);
393 U16_APPEND_UNSAFE(fold2
, i
, length
);
397 /* set next level pointers to case folding */
401 /* get ready to read from decomposition, continue with loop */
406 if( level1
<2 && (options
&_COMPARE_EQUIV
) &&
407 0!=(p
=unorm_getCanonicalDecomposition((UChar32
)cp1
, decomp1
, &length
))
409 /* cp1 decomposes into p[length] */
410 if(U_IS_SURROGATE(c1
)) {
411 if(U_IS_SURROGATE_LEAD(c1
)) {
412 /* advance beyond source surrogate pair if it decomposes */
414 } else /* isTrail(c1) */ {
416 * we got a supplementary code point when hitting its trail surrogate,
417 * therefore the lead surrogate must have been the same as in the other string;
418 * compare this decomposition with the lead surrogate in the other string
419 * remember that this simulates bulk text replacement:
420 * the decomposition would replace the entire code point
427 /* push current level pointers */
428 stack1
[level1
].start
=start1
;
430 stack1
[level1
].limit
=limit1
;
433 /* set empty intermediate level if skipped */
435 stack1
[level1
++].start
=NULL
;
438 /* set next level pointers to decomposition */
442 /* get ready to read from decomposition, continue with loop */
447 if( level2
<2 && (options
&_COMPARE_EQUIV
) &&
448 0!=(p
=unorm_getCanonicalDecomposition((UChar32
)cp2
, decomp2
, &length
))
450 /* cp2 decomposes into p[length] */
451 if(U_IS_SURROGATE(c2
)) {
452 if(U_IS_SURROGATE_LEAD(c2
)) {
453 /* advance beyond source surrogate pair if it decomposes */
455 } else /* isTrail(c2) */ {
457 * we got a supplementary code point when hitting its trail surrogate,
458 * therefore the lead surrogate must have been the same as in the other string;
459 * compare this decomposition with the lead surrogate in the other string
460 * remember that this simulates bulk text replacement:
461 * the decomposition would replace the entire code point
468 /* push current level pointers */
469 stack2
[level2
].start
=start2
;
471 stack2
[level2
].limit
=limit2
;
474 /* set empty intermediate level if skipped */
476 stack2
[level2
++].start
=NULL
;
479 /* set next level pointers to decomposition */
483 /* get ready to read from decomposition, continue with loop */
489 * no decomposition/case folding, max level for both sides:
490 * return difference result
492 * code point order comparison must not just return cp1-cp2
493 * because when single surrogates are present then the surrogate pairs
494 * that formed cp1 and cp2 may be from different string indexes
496 * example: { d800 d800 dc01 } vs. { d800 dc00 }, compare at second code units
497 * c1=d800 cp1=10001 c2=dc00 cp2=10000
498 * cp1-cp2>0 but c1-c2<0 and in fact in UTF-32 it is { d800 10001 } < { 10000 }
500 * therefore, use same fix-up as in ustring.c/uprv_strCompare()
501 * except: uprv_strCompare() fetches c=*s while this functions fetches c=*s++
502 * so we have slightly different pointer/start/limit comparisons here
505 if(c1
>=0xd800 && c2
>=0xd800 && (options
&U_COMPARE_CODE_POINT_ORDER
)) {
506 /* subtract 0x2800 from BMP code points to make them smaller than supplementary ones */
508 (c1
<=0xdbff && s1
!=limit1
&& U16_IS_TRAIL(*s1
)) ||
509 (U16_IS_TRAIL(c1
) && start1
!=(s1
-1) && U16_IS_LEAD(*(s1
-2)))
511 /* part of a surrogate pair, leave >=d800 */
513 /* BMP code point - may be surrogate code point - make <d800 */
518 (c2
<=0xdbff && s2
!=limit2
&& U16_IS_TRAIL(*s2
)) ||
519 (U16_IS_TRAIL(c2
) && start2
!=(s2
-1) && U16_IS_LEAD(*(s2
-2)))
521 /* part of a surrogate pair, leave >=d800 */
523 /* BMP code point - may be surrogate code point - make <d800 */
532 U_CAPI
int32_t U_EXPORT2
533 unorm_compare(const UChar
*s1
, int32_t length1
,
534 const UChar
*s2
, int32_t length2
,
536 UErrorCode
*pErrorCode
) {
537 UChar fcd1
[300], fcd2
[300];
539 const UnicodeSet
*nx
;
540 UNormalizationMode mode
;
544 /* argument checking */
545 if(pErrorCode
==0 || U_FAILURE(*pErrorCode
)) {
548 if(s1
==0 || length1
<-1 || s2
==0 || length2
<-1) {
549 *pErrorCode
=U_ILLEGAL_ARGUMENT_ERROR
;
553 if(!unorm_haveData(pErrorCode
)) {
556 if(!uprv_haveProperties(pErrorCode
)) {
560 normOptions
=(int32_t)(options
>>UNORM_COMPARE_NORM_OPTIONS_SHIFT
);
561 nx
=unorm_getNX(normOptions
, pErrorCode
);
562 if(U_FAILURE(*pErrorCode
)) {
567 options
|=_COMPARE_EQUIV
;
571 * UAX #21 Case Mappings, as fixed for Unicode version 4
572 * (see Jitterbug 2021), defines a canonical caseless match as
574 * A string X is a canonical caseless match
575 * for a string Y if and only if
576 * NFD(toCasefold(NFD(X))) = NFD(toCasefold(NFD(Y)))
578 * For better performance, we check for FCD (or let the caller tell us that
579 * both strings are in FCD) for the inner normalization.
580 * BasicNormalizerTest::FindFoldFCDExceptions() makes sure that
581 * case-folding preserves the FCD-ness of a string.
582 * The outer normalization is then only performed by unorm_cmpEquivFold()
583 * when there is a difference.
585 * Exception: When using the Turkic case-folding option, we do perform
586 * full NFD first. This is because in the Turkic case precomposed characters
587 * with 0049 capital I or 0069 small i fold differently whether they
588 * are first decomposed or not, so an FCD check - a check only for
589 * canonical order - is not sufficient.
591 if(options
&U_FOLD_CASE_EXCLUDE_SPECIAL_I
) {
593 options
&=~UNORM_INPUT_IS_FCD
;
598 if(!(options
&UNORM_INPUT_IS_FCD
)) {
599 int32_t _len1
, _len2
;
600 UBool isFCD1
, isFCD2
;
602 // check if s1 and/or s2 fulfill the FCD conditions
603 isFCD1
= UNORM_YES
==unorm_internalQuickCheck(s1
, length1
, mode
, TRUE
, nx
, pErrorCode
);
604 isFCD2
= UNORM_YES
==unorm_internalQuickCheck(s2
, length2
, mode
, TRUE
, nx
, pErrorCode
);
605 if(U_FAILURE(*pErrorCode
)) {
610 * ICU 2.4 had a further optimization:
611 * If both strings were not in FCD, then they were both NFD'ed,
612 * and the _COMPARE_EQUIV option was turned off.
613 * It is not entirely clear that this is valid with the current
614 * definition of the canonical caseless match.
615 * Therefore, ICU 2.6 removes that optimization.
619 _len1
=unorm_internalNormalizeWithNX(fcd1
, LENGTHOF(fcd1
),
621 mode
, normOptions
, nx
,
623 if(*pErrorCode
!=U_BUFFER_OVERFLOW_ERROR
) {
626 d1
=(UChar
*)uprv_malloc(_len1
*U_SIZEOF_UCHAR
);
628 *pErrorCode
=U_MEMORY_ALLOCATION_ERROR
;
632 *pErrorCode
=U_ZERO_ERROR
;
633 _len1
=unorm_internalNormalizeWithNX(d1
, _len1
,
635 mode
, normOptions
, nx
,
637 if(U_FAILURE(*pErrorCode
)) {
647 _len2
=unorm_internalNormalizeWithNX(fcd2
, LENGTHOF(fcd2
),
649 mode
, normOptions
, nx
,
651 if(*pErrorCode
!=U_BUFFER_OVERFLOW_ERROR
) {
654 d2
=(UChar
*)uprv_malloc(_len2
*U_SIZEOF_UCHAR
);
656 *pErrorCode
=U_MEMORY_ALLOCATION_ERROR
;
660 *pErrorCode
=U_ZERO_ERROR
;
661 _len2
=unorm_internalNormalizeWithNX(d2
, _len2
,
663 mode
, normOptions
, nx
,
665 if(U_FAILURE(*pErrorCode
)) {
675 if(U_SUCCESS(*pErrorCode
)) {
676 result
=unorm_cmpEquivFold(s1
, length1
, s2
, length2
, options
, pErrorCode
);
690 #endif /* #if !UCONFIG_NO_NORMALIZATION */