]> git.saurik.com Git - apple/icu.git/blob - icuSources/common/unormcmp.cpp
ICU-6.2.22.tar.gz
[apple/icu.git] / icuSources / common / unormcmp.cpp
1 /*
2 *******************************************************************************
3 *
4 * Copyright (C) 2001-2004, International Business Machines
5 * Corporation and others. All Rights Reserved.
6 *
7 *******************************************************************************
8 * file name: unormcmp.cpp
9 * encoding: US-ASCII
10 * tab size: 8 (not used)
11 * indentation:4
12 *
13 * created on: 2004sep13
14 * created by: Markus W. Scherer
15 *
16 * unorm_compare() function moved here from unorm.cpp for better modularization.
17 * Depends on both normalization and case folding.
18 * Allows unorm.cpp to not depend on any character properties code.
19 */
20
21 #include "unicode/utypes.h"
22
23 #if !UCONFIG_NO_NORMALIZATION
24
25 #include "unicode/ustring.h"
26 #include "unicode/unorm.h"
27 #include "unicode/uniset.h"
28 #include "unormimp.h"
29 #include "ucase.h"
30 #include "cmemory.h"
31
32 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
33
34 /* compare canonically equivalent ------------------------------------------- */
35
36 /*
37 * Compare two strings for canonical equivalence.
38 * Further options include case-insensitive comparison and
39 * code point order (as opposed to code unit order).
40 *
41 * In this function, canonical equivalence is optional as well.
42 * If canonical equivalence is tested, then both strings must fulfill
43 * the FCD check.
44 *
45 * Semantically, this is equivalent to
46 * strcmp[CodePointOrder](NFD(foldCase(s1)), NFD(foldCase(s2)))
47 * where code point order, NFD and foldCase are all optional.
48 *
49 * String comparisons almost always yield results before processing both strings
50 * completely.
51 * They are generally more efficient working incrementally instead of
52 * performing the sub-processing (strlen, normalization, case-folding)
53 * on the entire strings first.
54 *
55 * It is also unnecessary to not normalize identical characters.
56 *
57 * This function works in principle as follows:
58 *
59 * loop {
60 * get one code unit c1 from s1 (-1 if end of source)
61 * get one code unit c2 from s2 (-1 if end of source)
62 *
63 * if(either string finished) {
64 * return result;
65 * }
66 * if(c1==c2) {
67 * continue;
68 * }
69 *
70 * // c1!=c2
71 * try to decompose/case-fold c1/c2, and continue if one does;
72 *
73 * // still c1!=c2 and neither decomposes/case-folds, return result
74 * return c1-c2;
75 * }
76 *
77 * When a character decomposes, then the pointer for that source changes to
78 * the decomposition, pushing the previous pointer onto a stack.
79 * When the end of the decomposition is reached, then the code unit reader
80 * pops the previous source from the stack.
81 * (Same for case-folding.)
82 *
83 * This is complicated further by operating on variable-width UTF-16.
84 * The top part of the loop works on code units, while lookups for decomposition
85 * and case-folding need code points.
86 * Code points are assembled after the equality/end-of-source part.
87 * The source pointer is only advanced beyond all code units when the code point
88 * actually decomposes/case-folds.
89 *
90 * If we were on a trail surrogate unit when assembling a code point,
91 * and the code point decomposes/case-folds, then the decomposition/folding
92 * result must be compared with the part of the other string that corresponds to
93 * this string's lead surrogate.
94 * Since we only assemble a code point when hitting a trail unit when the
95 * preceding lead units were identical, we back up the other string by one unit
96 * in such a case.
97 *
98 * The optional code point order comparison at the end works with
99 * the same fix-up as the other code point order comparison functions.
100 * See ustring.c and the comment near the end of this function.
101 *
102 * Assumption: A decomposition or case-folding result string never contains
103 * a single surrogate. This is a safe assumption in the Unicode Standard.
104 * Therefore, we do not need to check for surrogate pairs across
105 * decomposition/case-folding boundaries.
106 *
107 * Further assumptions (see verifications tstnorm.cpp):
108 * The API function checks for FCD first, while the core function
109 * first case-folds and then decomposes. This requires that case-folding does not
110 * un-FCD any strings.
111 *
112 * The API function may also NFD the input and turn off decomposition.
113 * This requires that case-folding does not un-NFD strings either.
114 *
115 * TODO If any of the above two assumptions is violated,
116 * then this entire code must be re-thought.
117 * If this happens, then a simple solution is to case-fold both strings up front
118 * and to turn off UNORM_INPUT_IS_FCD.
119 * We already do this when not both strings are in FCD because makeFCD
120 * would be a partial NFD before the case folding, which does not work.
121 * Note that all of this is only a problem when case-folding _and_
122 * canonical equivalence come together.
123 * (Comments in unorm_compare() are more up to date than this TODO.)
124 *
125 * This function could be moved to a different source file, at increased cost
126 * for calling the decomposition access function.
127 */
128
129 /* stack element for previous-level source/decomposition pointers */
130 struct CmpEquivLevel {
131 const UChar *start, *s, *limit;
132 };
133 typedef struct CmpEquivLevel CmpEquivLevel;
134
135 /* internal function */
136 static int32_t
137 unorm_cmpEquivFold(const UChar *s1, int32_t length1,
138 const UChar *s2, int32_t length2,
139 uint32_t options,
140 UErrorCode *pErrorCode) {
141 UCaseProps *csp;
142
143 /* current-level start/limit - s1/s2 as current */
144 const UChar *start1, *start2, *limit1, *limit2;
145
146 /* decomposition and case folding variables */
147 const UChar *p;
148 int32_t length;
149
150 /* stacks of previous-level start/current/limit */
151 CmpEquivLevel stack1[2], stack2[2];
152
153 /* decomposition buffers for Hangul */
154 UChar decomp1[4], decomp2[4];
155
156 /* case folding buffers, only use current-level start/limit */
157 UChar fold1[UCASE_MAX_STRING_LENGTH+1], fold2[UCASE_MAX_STRING_LENGTH+1];
158
159 /* track which is the current level per string */
160 int32_t level1, level2;
161
162 /* current code units, and code points for lookups */
163 UChar32 c1, c2, cp1, cp2;
164
165 /* no argument error checking because this itself is not an API */
166
167 /*
168 * assume that at least one of the options _COMPARE_EQUIV and U_COMPARE_IGNORE_CASE is set
169 * otherwise this function must behave exactly as uprv_strCompare()
170 * not checking for that here makes testing this function easier
171 */
172
173 /* normalization/properties data loaded? */
174 if( ((options&_COMPARE_EQUIV)!=0 && !unorm_haveData(pErrorCode)) ||
175 U_FAILURE(*pErrorCode)
176 ) {
177 return 0;
178 }
179 if((options&U_COMPARE_IGNORE_CASE)!=0) {
180 csp=ucase_getSingleton(pErrorCode);
181 if(U_FAILURE(*pErrorCode)) {
182 return 0;
183 }
184 } else {
185 csp=NULL;
186 }
187
188 /* initialize */
189 start1=s1;
190 if(length1==-1) {
191 limit1=NULL;
192 } else {
193 limit1=s1+length1;
194 }
195
196 start2=s2;
197 if(length2==-1) {
198 limit2=NULL;
199 } else {
200 limit2=s2+length2;
201 }
202
203 level1=level2=0;
204 c1=c2=-1;
205
206 /* comparison loop */
207 for(;;) {
208 /*
209 * here a code unit value of -1 means "get another code unit"
210 * below it will mean "this source is finished"
211 */
212
213 if(c1<0) {
214 /* get next code unit from string 1, post-increment */
215 for(;;) {
216 if(s1==limit1 || ((c1=*s1)==0 && (limit1==NULL || (options&_STRNCMP_STYLE)))) {
217 if(level1==0) {
218 c1=-1;
219 break;
220 }
221 } else {
222 ++s1;
223 break;
224 }
225
226 /* reached end of level buffer, pop one level */
227 do {
228 --level1;
229 start1=stack1[level1].start;
230 } while(start1==NULL);
231 s1=stack1[level1].s;
232 limit1=stack1[level1].limit;
233 }
234 }
235
236 if(c2<0) {
237 /* get next code unit from string 2, post-increment */
238 for(;;) {
239 if(s2==limit2 || ((c2=*s2)==0 && (limit2==NULL || (options&_STRNCMP_STYLE)))) {
240 if(level2==0) {
241 c2=-1;
242 break;
243 }
244 } else {
245 ++s2;
246 break;
247 }
248
249 /* reached end of level buffer, pop one level */
250 do {
251 --level2;
252 start2=stack2[level2].start;
253 } while(start2==NULL);
254 s2=stack2[level2].s;
255 limit2=stack2[level2].limit;
256 }
257 }
258
259 /*
260 * compare c1 and c2
261 * either variable c1, c2 is -1 only if the corresponding string is finished
262 */
263 if(c1==c2) {
264 if(c1<0) {
265 return 0; /* c1==c2==-1 indicating end of strings */
266 }
267 c1=c2=-1; /* make us fetch new code units */
268 continue;
269 } else if(c1<0) {
270 return -1; /* string 1 ends before string 2 */
271 } else if(c2<0) {
272 return 1; /* string 2 ends before string 1 */
273 }
274 /* c1!=c2 && c1>=0 && c2>=0 */
275
276 /* get complete code points for c1, c2 for lookups if either is a surrogate */
277 cp1=c1;
278 if(U_IS_SURROGATE(c1)) {
279 UChar c;
280
281 if(U_IS_SURROGATE_LEAD(c1)) {
282 if(s1!=limit1 && U16_IS_TRAIL(c=*s1)) {
283 /* advance ++s1; only below if cp1 decomposes/case-folds */
284 cp1=U16_GET_SUPPLEMENTARY(c1, c);
285 }
286 } else /* isTrail(c1) */ {
287 if(start1<=(s1-2) && U16_IS_LEAD(c=*(s1-2))) {
288 cp1=U16_GET_SUPPLEMENTARY(c, c1);
289 }
290 }
291 }
292
293 cp2=c2;
294 if(U_IS_SURROGATE(c2)) {
295 UChar c;
296
297 if(U_IS_SURROGATE_LEAD(c2)) {
298 if(s2!=limit2 && U16_IS_TRAIL(c=*s2)) {
299 /* advance ++s2; only below if cp2 decomposes/case-folds */
300 cp2=U16_GET_SUPPLEMENTARY(c2, c);
301 }
302 } else /* isTrail(c2) */ {
303 if(start2<=(s2-2) && U16_IS_LEAD(c=*(s2-2))) {
304 cp2=U16_GET_SUPPLEMENTARY(c, c2);
305 }
306 }
307 }
308
309 /*
310 * go down one level for each string
311 * continue with the main loop as soon as there is a real change
312 */
313
314 if( level1==0 && (options&U_COMPARE_IGNORE_CASE) &&
315 (length=ucase_toFullFolding(csp, (UChar32)cp1, &p, options))>=0
316 ) {
317 /* cp1 case-folds to the code point "length" or to p[length] */
318 if(U_IS_SURROGATE(c1)) {
319 if(U_IS_SURROGATE_LEAD(c1)) {
320 /* advance beyond source surrogate pair if it case-folds */
321 ++s1;
322 } else /* isTrail(c1) */ {
323 /*
324 * we got a supplementary code point when hitting its trail surrogate,
325 * therefore the lead surrogate must have been the same as in the other string;
326 * compare this decomposition with the lead surrogate in the other string
327 * remember that this simulates bulk text replacement:
328 * the decomposition would replace the entire code point
329 */
330 --s2;
331 c2=*(s2-1);
332 }
333 }
334
335 /* push current level pointers */
336 stack1[0].start=start1;
337 stack1[0].s=s1;
338 stack1[0].limit=limit1;
339 ++level1;
340
341 /* copy the folding result to fold1[] */
342 if(length<=UCASE_MAX_STRING_LENGTH) {
343 u_memcpy(fold1, p, length);
344 } else {
345 int32_t i=0;
346 U16_APPEND_UNSAFE(fold1, i, length);
347 length=i;
348 }
349
350 /* set next level pointers to case folding */
351 start1=s1=fold1;
352 limit1=fold1+length;
353
354 /* get ready to read from decomposition, continue with loop */
355 c1=-1;
356 continue;
357 }
358
359 if( level2==0 && (options&U_COMPARE_IGNORE_CASE) &&
360 (length=ucase_toFullFolding(csp, (UChar32)cp2, &p, options))>=0
361 ) {
362 /* cp2 case-folds to the code point "length" or to p[length] */
363 if(U_IS_SURROGATE(c2)) {
364 if(U_IS_SURROGATE_LEAD(c2)) {
365 /* advance beyond source surrogate pair if it case-folds */
366 ++s2;
367 } else /* isTrail(c2) */ {
368 /*
369 * we got a supplementary code point when hitting its trail surrogate,
370 * therefore the lead surrogate must have been the same as in the other string;
371 * compare this decomposition with the lead surrogate in the other string
372 * remember that this simulates bulk text replacement:
373 * the decomposition would replace the entire code point
374 */
375 --s1;
376 c1=*(s1-1);
377 }
378 }
379
380 /* push current level pointers */
381 stack2[0].start=start2;
382 stack2[0].s=s2;
383 stack2[0].limit=limit2;
384 ++level2;
385
386 /* copy the folding result to fold2[] */
387 if(length<=UCASE_MAX_STRING_LENGTH) {
388 u_memcpy(fold2, p, length);
389 } else {
390 int32_t i=0;
391 U16_APPEND_UNSAFE(fold2, i, length);
392 length=i;
393 }
394
395 /* set next level pointers to case folding */
396 start2=s2=fold2;
397 limit2=fold2+length;
398
399 /* get ready to read from decomposition, continue with loop */
400 c2=-1;
401 continue;
402 }
403
404 if( level1<2 && (options&_COMPARE_EQUIV) &&
405 0!=(p=unorm_getCanonicalDecomposition((UChar32)cp1, decomp1, &length))
406 ) {
407 /* cp1 decomposes into p[length] */
408 if(U_IS_SURROGATE(c1)) {
409 if(U_IS_SURROGATE_LEAD(c1)) {
410 /* advance beyond source surrogate pair if it decomposes */
411 ++s1;
412 } else /* isTrail(c1) */ {
413 /*
414 * we got a supplementary code point when hitting its trail surrogate,
415 * therefore the lead surrogate must have been the same as in the other string;
416 * compare this decomposition with the lead surrogate in the other string
417 * remember that this simulates bulk text replacement:
418 * the decomposition would replace the entire code point
419 */
420 --s2;
421 c2=*(s2-1);
422 }
423 }
424
425 /* push current level pointers */
426 stack1[level1].start=start1;
427 stack1[level1].s=s1;
428 stack1[level1].limit=limit1;
429 ++level1;
430
431 /* set empty intermediate level if skipped */
432 if(level1<2) {
433 stack1[level1++].start=NULL;
434 }
435
436 /* set next level pointers to decomposition */
437 start1=s1=p;
438 limit1=p+length;
439
440 /* get ready to read from decomposition, continue with loop */
441 c1=-1;
442 continue;
443 }
444
445 if( level2<2 && (options&_COMPARE_EQUIV) &&
446 0!=(p=unorm_getCanonicalDecomposition((UChar32)cp2, decomp2, &length))
447 ) {
448 /* cp2 decomposes into p[length] */
449 if(U_IS_SURROGATE(c2)) {
450 if(U_IS_SURROGATE_LEAD(c2)) {
451 /* advance beyond source surrogate pair if it decomposes */
452 ++s2;
453 } else /* isTrail(c2) */ {
454 /*
455 * we got a supplementary code point when hitting its trail surrogate,
456 * therefore the lead surrogate must have been the same as in the other string;
457 * compare this decomposition with the lead surrogate in the other string
458 * remember that this simulates bulk text replacement:
459 * the decomposition would replace the entire code point
460 */
461 --s1;
462 c1=*(s1-1);
463 }
464 }
465
466 /* push current level pointers */
467 stack2[level2].start=start2;
468 stack2[level2].s=s2;
469 stack2[level2].limit=limit2;
470 ++level2;
471
472 /* set empty intermediate level if skipped */
473 if(level2<2) {
474 stack2[level2++].start=NULL;
475 }
476
477 /* set next level pointers to decomposition */
478 start2=s2=p;
479 limit2=p+length;
480
481 /* get ready to read from decomposition, continue with loop */
482 c2=-1;
483 continue;
484 }
485
486 /*
487 * no decomposition/case folding, max level for both sides:
488 * return difference result
489 *
490 * code point order comparison must not just return cp1-cp2
491 * because when single surrogates are present then the surrogate pairs
492 * that formed cp1 and cp2 may be from different string indexes
493 *
494 * example: { d800 d800 dc01 } vs. { d800 dc00 }, compare at second code units
495 * c1=d800 cp1=10001 c2=dc00 cp2=10000
496 * cp1-cp2>0 but c1-c2<0 and in fact in UTF-32 it is { d800 10001 } < { 10000 }
497 *
498 * therefore, use same fix-up as in ustring.c/uprv_strCompare()
499 * except: uprv_strCompare() fetches c=*s while this functions fetches c=*s++
500 * so we have slightly different pointer/start/limit comparisons here
501 */
502
503 if(c1>=0xd800 && c2>=0xd800 && (options&U_COMPARE_CODE_POINT_ORDER)) {
504 /* subtract 0x2800 from BMP code points to make them smaller than supplementary ones */
505 if(
506 (c1<=0xdbff && s1!=limit1 && U16_IS_TRAIL(*s1)) ||
507 (U16_IS_TRAIL(c1) && start1!=(s1-1) && U16_IS_LEAD(*(s1-2)))
508 ) {
509 /* part of a surrogate pair, leave >=d800 */
510 } else {
511 /* BMP code point - may be surrogate code point - make <d800 */
512 c1-=0x2800;
513 }
514
515 if(
516 (c2<=0xdbff && s2!=limit2 && U16_IS_TRAIL(*s2)) ||
517 (U16_IS_TRAIL(c2) && start2!=(s2-1) && U16_IS_LEAD(*(s2-2)))
518 ) {
519 /* part of a surrogate pair, leave >=d800 */
520 } else {
521 /* BMP code point - may be surrogate code point - make <d800 */
522 c2-=0x2800;
523 }
524 }
525
526 return c1-c2;
527 }
528 }
529
530 U_CAPI int32_t U_EXPORT2
531 unorm_compare(const UChar *s1, int32_t length1,
532 const UChar *s2, int32_t length2,
533 uint32_t options,
534 UErrorCode *pErrorCode) {
535 UChar fcd1[300], fcd2[300];
536 UChar *d1, *d2;
537 const UnicodeSet *nx;
538 UNormalizationMode mode;
539 int32_t normOptions;
540 int32_t result;
541
542 /* argument checking */
543 if(pErrorCode==0 || U_FAILURE(*pErrorCode)) {
544 return 0;
545 }
546 if(s1==0 || length1<-1 || s2==0 || length2<-1) {
547 *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR;
548 return 0;
549 }
550
551 if(!unorm_haveData(pErrorCode)) {
552 return 0;
553 }
554 if(!uprv_haveProperties(pErrorCode)) {
555 return 0;
556 }
557
558 normOptions=(int32_t)(options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT);
559 nx=unorm_getNX(normOptions, pErrorCode);
560 if(U_FAILURE(*pErrorCode)) {
561 return 0;
562 }
563
564 d1=d2=0;
565 options|=_COMPARE_EQUIV;
566 result=0;
567
568 /*
569 * UAX #21 Case Mappings, as fixed for Unicode version 4
570 * (see Jitterbug 2021), defines a canonical caseless match as
571 *
572 * A string X is a canonical caseless match
573 * for a string Y if and only if
574 * NFD(toCasefold(NFD(X))) = NFD(toCasefold(NFD(Y)))
575 *
576 * For better performance, we check for FCD (or let the caller tell us that
577 * both strings are in FCD) for the inner normalization.
578 * BasicNormalizerTest::FindFoldFCDExceptions() makes sure that
579 * case-folding preserves the FCD-ness of a string.
580 * The outer normalization is then only performed by unorm_cmpEquivFold()
581 * when there is a difference.
582 *
583 * Exception: When using the Turkic case-folding option, we do perform
584 * full NFD first. This is because in the Turkic case precomposed characters
585 * with 0049 capital I or 0069 small i fold differently whether they
586 * are first decomposed or not, so an FCD check - a check only for
587 * canonical order - is not sufficient.
588 */
589 if(options&U_FOLD_CASE_EXCLUDE_SPECIAL_I) {
590 mode=UNORM_NFD;
591 options&=~UNORM_INPUT_IS_FCD;
592 } else {
593 mode=UNORM_FCD;
594 }
595
596 if(!(options&UNORM_INPUT_IS_FCD)) {
597 int32_t _len1, _len2;
598 UBool isFCD1, isFCD2;
599
600 // check if s1 and/or s2 fulfill the FCD conditions
601 isFCD1= UNORM_YES==unorm_internalQuickCheck(s1, length1, mode, TRUE, nx, pErrorCode);
602 isFCD2= UNORM_YES==unorm_internalQuickCheck(s2, length2, mode, TRUE, nx, pErrorCode);
603 if(U_FAILURE(*pErrorCode)) {
604 return 0;
605 }
606
607 /*
608 * ICU 2.4 had a further optimization:
609 * If both strings were not in FCD, then they were both NFD'ed,
610 * and the _COMPARE_EQUIV option was turned off.
611 * It is not entirely clear that this is valid with the current
612 * definition of the canonical caseless match.
613 * Therefore, ICU 2.6 removes that optimization.
614 */
615
616 if(!isFCD1) {
617 _len1=unorm_internalNormalizeWithNX(fcd1, LENGTHOF(fcd1),
618 s1, length1,
619 mode, normOptions, nx,
620 pErrorCode);
621 if(*pErrorCode!=U_BUFFER_OVERFLOW_ERROR) {
622 s1=fcd1;
623 } else {
624 d1=(UChar *)uprv_malloc(_len1*U_SIZEOF_UCHAR);
625 if(d1==0) {
626 *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
627 goto cleanup;
628 }
629
630 *pErrorCode=U_ZERO_ERROR;
631 _len1=unorm_internalNormalizeWithNX(d1, _len1,
632 s1, length1,
633 mode, normOptions, nx,
634 pErrorCode);
635 if(U_FAILURE(*pErrorCode)) {
636 goto cleanup;
637 }
638
639 s1=d1;
640 }
641 length1=_len1;
642 }
643
644 if(!isFCD2) {
645 _len2=unorm_internalNormalizeWithNX(fcd2, LENGTHOF(fcd2),
646 s2, length2,
647 mode, normOptions, nx,
648 pErrorCode);
649 if(*pErrorCode!=U_BUFFER_OVERFLOW_ERROR) {
650 s2=fcd2;
651 } else {
652 d2=(UChar *)uprv_malloc(_len2*U_SIZEOF_UCHAR);
653 if(d2==0) {
654 *pErrorCode=U_MEMORY_ALLOCATION_ERROR;
655 goto cleanup;
656 }
657
658 *pErrorCode=U_ZERO_ERROR;
659 _len2=unorm_internalNormalizeWithNX(d2, _len2,
660 s2, length2,
661 mode, normOptions, nx,
662 pErrorCode);
663 if(U_FAILURE(*pErrorCode)) {
664 goto cleanup;
665 }
666
667 s2=d2;
668 }
669 length2=_len2;
670 }
671 }
672
673 if(U_SUCCESS(*pErrorCode)) {
674 result=unorm_cmpEquivFold(s1, length1, s2, length2, options, pErrorCode);
675 }
676
677 cleanup:
678 if(d1!=0) {
679 uprv_free(d1);
680 }
681 if(d2!=0) {
682 uprv_free(d2);
683 }
684
685 return result;
686 }
687
688 #endif /* #if !UCONFIG_NO_NORMALIZATION */