ICU-57132.0.1.tar.gz

[apple/icu.git] / icuSources / common / caniter.cpp

diff --git a/icuSources/common/caniter.cpp b/icuSources/common/caniter.cpp
index 04d48ba8fce52cda1ec6abc8074706ec183734a8..58b378181056fccc4f287417004d836dfdbe93f6 100644 (file)
--- a/icuSources/common/caniter.cpp
+++ b/icuSources/common/caniter.cpp
@@ -1,7 +1,7 @@
 /*
  *****************************************************************************
 /*
  *****************************************************************************

- * Copyright (C) 1996-2006, International Business Machines Corporation and  *
- * others. All Rights Reserved.                                              *
+ * Copyright (C) 1996-2015, International Business Machines Corporation and
+ * others. All Rights Reserved.

  *****************************************************************************
  */
 
  *****************************************************************************
  */
 


@@ -9,14 +9,16 @@
 
 #if !UCONFIG_NO_NORMALIZATION
 
 
 #if !UCONFIG_NO_NORMALIZATION
 

-#include "unicode/uset.h"
-#include "unicode/ustring.h"
-#include "hash.h"
-#include "unormimp.h"

 #include "unicode/caniter.h"
 #include "unicode/caniter.h"

-#include "unicode/normlzr.h"
+#include "unicode/normalizer2.h"

 #include "unicode/uchar.h"
 #include "unicode/uchar.h"

+#include "unicode/uniset.h"
+#include "unicode/usetiter.h"
+#include "unicode/ustring.h"
+#include "unicode/utf16.h"

 #include "cmemory.h"
 #include "cmemory.h"

+#include "hash.h"
+#include "normalizer2impl.h"

 
 /**
  * This class allows one to iterate through all the strings that are canonically equivalent to a given
 
 /**
  * This class allows one to iterate through all the strings that are canonically equivalent to a given


@@ -68,9 +70,11 @@ CanonicalIterator::CanonicalIterator(const UnicodeString &sourceStr, UErrorCode
     pieces_length(0),
     pieces_lengths(NULL),
     current(NULL),
     pieces_length(0),
     pieces_lengths(NULL),
     current(NULL),

-    current_length(0)
+    current_length(0),
+    nfd(*Normalizer2::getNFDInstance(status)),
+    nfcImpl(*Normalizer2Factory::getNFCImpl(status))

 {
 {

-    if(U_SUCCESS(status)) {
+    if(U_SUCCESS(status) && nfcImpl.ensureCanonIterData(status)) {

       setSource(sourceStr, status);
     }
 }
       setSource(sourceStr, status);
     }
 }


@@ -166,7 +170,7 @@ void CanonicalIterator::setSource(const UnicodeString &newSource, UErrorCode &st
     int32_t i = 0;
     UnicodeString *list = NULL;
 
     int32_t i = 0;
     UnicodeString *list = NULL;
 

-    Normalizer::normalize(newSource, UNORM_NFD, 0, source, status);
+    nfd.normalize(newSource, source, status);

     if(U_FAILURE(status)) {
       return;
     }
     if(U_FAILURE(status)) {
       return;
     }


@@ -204,16 +208,16 @@ void CanonicalIterator::setSource(const UnicodeString &newSource, UErrorCode &st
 
     // i should initialy be the number of code units at the 
     // start of the string
 
     // i should initialy be the number of code units at the 
     // start of the string

-    i = UTF16_CHAR_LENGTH(source.char32At(0));
+    i = U16_LENGTH(source.char32At(0));

     //int32_t i = 1;
     // find the segments
     // This code iterates through the source string and 
     // extracts segments that end up on a codepoint that
     // doesn't start any decompositions. (Analysis is done
     // on the NFD form - see above).
     //int32_t i = 1;
     // find the segments
     // This code iterates through the source string and 
     // extracts segments that end up on a codepoint that
     // doesn't start any decompositions. (Analysis is done
     // on the NFD form - see above).

-    for (; i < source.length(); i += UTF16_CHAR_LENGTH(cp)) {
+    for (; i < source.length(); i += U16_LENGTH(cp)) {

         cp = source.char32At(i);
         cp = source.char32At(i);

-        if (unorm_isCanonSafeStart(cp)) {
+        if (nfcImpl.isCanonSegmentStarter(cp)) {

             source.extract(start, i-start, list[list_length++]); // add up to i
             start = i;
         }
             source.extract(start, i-start, list[list_length++]); // add up to i
             start = i;
         }


@@ -285,12 +289,12 @@ void U_EXPORT2 CanonicalIterator::permute(UnicodeString &source, UBool skipZeros
     if(U_FAILURE(status)) {
         return;
     }
     if(U_FAILURE(status)) {
         return;
     }

-    subpermute.setValueDeleter(uhash_deleteUnicodeString);
+    subpermute.setValueDeleter(uprv_deleteUObject);

 
 

-    for (i = 0; i < source.length(); i += UTF16_CHAR_LENGTH(cp)) {
+    for (i = 0; i < source.length(); i += U16_LENGTH(cp)) {

         cp = source.char32At(i);
         const UHashElement *ne = NULL;
         cp = source.char32At(i);
         const UHashElement *ne = NULL;

-        int32_t el = -1;
+        int32_t el = UHASH_FIRST;

         UnicodeString subPermuteString = source;
 
         // optimization:
         UnicodeString subPermuteString = source;
 
         // optimization:


@@ -305,7 +309,7 @@ void U_EXPORT2 CanonicalIterator::permute(UnicodeString &source, UBool skipZeros
 
         // see what the permutations of the characters before and after this one are
         //Hashtable *subpermute = permute(source.substring(0,i) + source.substring(i + UTF16.getCharCount(cp)));
 
         // see what the permutations of the characters before and after this one are
         //Hashtable *subpermute = permute(source.substring(0,i) + source.substring(i + UTF16.getCharCount(cp)));

-        permute(subPermuteString.replace(i, UTF16_CHAR_LENGTH(cp), NULL, 0), skipZeros, &subpermute, status);
+        permute(subPermuteString.replace(i, U16_LENGTH(cp), NULL, 0), skipZeros, &subpermute, status);

         /* Test for buffer overflows */
         if(U_FAILURE(status)) {
             return;
         /* Test for buffer overflows */
         if(U_FAILURE(status)) {
             return;


@@ -342,9 +346,9 @@ UnicodeString* CanonicalIterator::getEquivalents(const UnicodeString &segment, i
     if (U_FAILURE(status)) {
         return 0;
     }
     if (U_FAILURE(status)) {
         return 0;
     }

-    result.setValueDeleter(uhash_deleteUnicodeString);
-    permutations.setValueDeleter(uhash_deleteUnicodeString);
-    basic.setValueDeleter(uhash_deleteUnicodeString);
+    result.setValueDeleter(uprv_deleteUObject);
+    permutations.setValueDeleter(uprv_deleteUObject);
+    basic.setValueDeleter(uprv_deleteUObject);

 
     UChar USeg[256];
     int32_t segLen = segment.extract(USeg, 256, status);
 
     UChar USeg[256];
     int32_t segLen = segment.extract(USeg, 256, status);


@@ -355,7 +359,7 @@ UnicodeString* CanonicalIterator::getEquivalents(const UnicodeString &segment, i
     // TODO: optimize by not permuting any class zero.
 
     const UHashElement *ne = NULL;
     // TODO: optimize by not permuting any class zero.
 
     const UHashElement *ne = NULL;

-    int32_t el = -1;
+    int32_t el = UHASH_FIRST;

     //Iterator it = basic.iterator();
     ne = basic.nextElement(el);
     //while (it.hasNext())
     //Iterator it = basic.iterator();
     ne = basic.nextElement(el);
     //while (it.hasNext())


@@ -366,7 +370,7 @@ UnicodeString* CanonicalIterator::getEquivalents(const UnicodeString &segment, i
         permutations.removeAll();
         permute(item, CANITER_SKIP_ZEROES, &permutations, status);
         const UHashElement *ne2 = NULL;
         permutations.removeAll();
         permute(item, CANITER_SKIP_ZEROES, &permutations, status);
         const UHashElement *ne2 = NULL;

-        int32_t el2 = -1;
+        int32_t el2 = UHASH_FIRST;

         //Iterator it2 = permutations.iterator();
         ne2 = permutations.nextElement(el2);
         //while (it2.hasNext())
         //Iterator it2 = permutations.iterator();
         ne2 = permutations.nextElement(el2);
         //while (it2.hasNext())


@@ -375,7 +379,7 @@ UnicodeString* CanonicalIterator::getEquivalents(const UnicodeString &segment, i
             //UnicodeString *possible = new UnicodeString(*((UnicodeString *)(ne2->value.pointer)));
             UnicodeString possible(*((UnicodeString *)(ne2->value.pointer)));
             UnicodeString attempt;
             //UnicodeString *possible = new UnicodeString(*((UnicodeString *)(ne2->value.pointer)));
             UnicodeString possible(*((UnicodeString *)(ne2->value.pointer)));
             UnicodeString attempt;

-            Normalizer::normalize(possible, UNORM_NFD, 0, attempt, status);
+            nfd.normalize(possible, attempt, status);

 
             // TODO: check if operator == is semanticaly the same as attempt.equals(segment)
             if (attempt==segment) {
 
             // TODO: check if operator == is semanticaly the same as attempt.equals(segment)
             if (attempt==segment) {


@@ -412,7 +416,7 @@ UnicodeString* CanonicalIterator::getEquivalents(const UnicodeString &segment, i
     }
     //result.toArray(finalResult);
     result_len = 0;
     }
     //result.toArray(finalResult);
     result_len = 0;

-    el = -1;
+    el = UHASH_FIRST;

     ne = result.nextElement(el);
     while(ne != NULL) {
         finalResult[result_len++] = *((UnicodeString *)(ne->value.pointer));
     ne = result.nextElement(el);
     while(ne != NULL) {
         finalResult[result_len++] = *((UnicodeString *)(ne->value.pointer));


@@ -435,30 +439,31 @@ Hashtable *CanonicalIterator::getEquivalents2(Hashtable *fillinResult, const UCh
 
     fillinResult->put(toPut, new UnicodeString(toPut), status);
 
 
     fillinResult->put(toPut, new UnicodeString(toPut), status);
 

-    USerializedSet starts;
+    UnicodeSet starts;

 
     // cycle through all the characters
 
     // cycle through all the characters

-    UChar32 cp, end = 0;
-    int32_t i = 0, j;
-    for (i = 0; i < segLen; i += UTF16_CHAR_LENGTH(cp)) {
+    UChar32 cp;
+    for (int32_t i = 0; i < segLen; i += U16_LENGTH(cp)) {

         // see if any character is at the start of some decomposition
         // see if any character is at the start of some decomposition

-        UTF_GET_CHAR(segment, 0, i, segLen, cp);
-        if (!unorm_getCanonStartSet(cp, &starts)) {
+        U16_GET(segment, 0, i, segLen, cp);
+        if (!nfcImpl.getCanonStartSet(cp, starts)) {

             continue;
         }
             continue;
         }

-        // if so, see which decompositions match 
-        for(j = 0, cp = end+1; cp <= end || uset_getSerializedRange(&starts, j++, &cp, &end); ++cp) {
+        // if so, see which decompositions match
+        UnicodeSetIterator iter(starts);
+        while (iter.next()) {
+            UChar32 cp2 = iter.getCodepoint();

             Hashtable remainder(status);
             Hashtable remainder(status);

-            remainder.setValueDeleter(uhash_deleteUnicodeString);
-            if (extract(&remainder, cp, segment, segLen, i, status) == NULL) {
+            remainder.setValueDeleter(uprv_deleteUObject);
+            if (extract(&remainder, cp2, segment, segLen, i, status) == NULL) {

                 continue;
             }
 
             // there were some matches, so add all the possibilities to the set.
             UnicodeString prefix(segment, i);
                 continue;
             }
 
             // there were some matches, so add all the possibilities to the set.
             UnicodeString prefix(segment, i);

-            prefix += cp;
+            prefix += cp2;

 
 

-            int32_t el = -1;
+            int32_t el = UHASH_FIRST;

             const UHashElement *ne = remainder.nextElement(el);
             while (ne != NULL) {
                 UnicodeString item = *((UnicodeString *)(ne->value.pointer));
             const UHashElement *ne = remainder.nextElement(el);
             while (ne != NULL) {
                 UnicodeString item = *((UnicodeString *)(ne->value.pointer));


@@ -499,73 +504,46 @@ Hashtable *CanonicalIterator::extract(Hashtable *fillinResult, UChar32 comp, con
         return NULL;
     }
 
         return NULL;
     }
 

-    const int32_t bufSize = 256;
-    int32_t bufLen = 0;
-    UChar temp[bufSize];
-
-    int32_t inputLen = 0, decompLen;
-    UChar stackBuffer[4];
-    const UChar *decomp;
-
-    U16_APPEND_UNSAFE(temp, inputLen, comp);
-    decomp = unorm_getCanonicalDecomposition(comp, stackBuffer, &decompLen);
-    if(decomp == NULL) {
-        /* copy temp */
-        stackBuffer[0] = temp[0];
-        if(inputLen > 1) {
-            stackBuffer[1] = temp[1];
-        }
-        decomp = stackBuffer;
-        decompLen = inputLen;
+    UnicodeString temp(comp);
+    int32_t inputLen=temp.length();
+    UnicodeString decompString;
+    nfd.normalize(temp, decompString, status);
+    if (U_FAILURE(status)) {
+        return NULL;

     }
     }

-
-    UChar *buff = temp+inputLen;
+    if (decompString.isBogus()) {
+        status = U_MEMORY_ALLOCATION_ERROR;
+        return NULL;
+    }
+    const UChar *decomp=decompString.getBuffer();
+    int32_t decompLen=decompString.length();

 
     // See if it matches the start of segment (at segmentPos)
     UBool ok = FALSE;
     UChar32 cp;
     int32_t decompPos = 0;
     UChar32 decompCp;
 
     // See if it matches the start of segment (at segmentPos)
     UBool ok = FALSE;
     UChar32 cp;
     int32_t decompPos = 0;
     UChar32 decompCp;

-    UTF_NEXT_CHAR(decomp, decompPos, decompLen, decompCp);
+    U16_NEXT(decomp, decompPos, decompLen, decompCp);

 
 

-    int32_t i;
-    UBool overflow = FALSE;
-
-    i = segmentPos;
+    int32_t i = segmentPos;

     while(i < segLen) {
     while(i < segLen) {

-        UTF_NEXT_CHAR(segment, i, segLen, cp);
+        U16_NEXT(segment, i, segLen, cp);

 
         if (cp == decompCp) { // if equal, eat another cp from decomp
 
             //if (PROGRESS) printf("  matches: %s\n", UToS(Tr(UnicodeString(cp))));
 
             if (decompPos == decompLen) { // done, have all decomp characters!
 
         if (cp == decompCp) { // if equal, eat another cp from decomp
 
             //if (PROGRESS) printf("  matches: %s\n", UToS(Tr(UnicodeString(cp))));
 
             if (decompPos == decompLen) { // done, have all decomp characters!

-                //u_strcat(buff+bufLen, segment+i);
-                uprv_memcpy(buff+bufLen, segment+i, (segLen-i)*sizeof(UChar));
-                bufLen+=segLen-i;
-
+                temp.append(segment+i, segLen-i);

                 ok = TRUE;
                 break;
             }
                 ok = TRUE;
                 break;
             }

-            UTF_NEXT_CHAR(decomp, decompPos, decompLen, decompCp);
+            U16_NEXT(decomp, decompPos, decompLen, decompCp);

         } else {
             //if (PROGRESS) printf("  buffer: %s\n", UToS(Tr(UnicodeString(cp))));
 
             // brute force approach
         } else {
             //if (PROGRESS) printf("  buffer: %s\n", UToS(Tr(UnicodeString(cp))));
 
             // brute force approach

-
-            U16_APPEND(buff, bufLen, bufSize, cp, overflow);
-
-            if(overflow) {
-                /*
-                 * ### TODO handle buffer overflow
-                 * The buffer is large, but an overflow may still happen with
-                 * unusual input (many combining marks?).
-                 * Reallocate buffer and continue.
-                 * markus 20020929
-                 */
-
-                overflow = FALSE;
-            }
+            temp.append(cp);

 
             /* TODO: optimize
             // since we know that the classes are monotonically increasing, after zero
 
             /* TODO: optimize
             // since we know that the classes are monotonically increasing, after zero


@@ -585,25 +563,20 @@ Hashtable *CanonicalIterator::extract(Hashtable *fillinResult, UChar32 comp, con
 
     //if (PROGRESS) printf("Matches\n");
 
 
     //if (PROGRESS) printf("Matches\n");
 

-    if (bufLen == 0) {
+    if (inputLen == temp.length()) {

         fillinResult->put(UnicodeString(), new UnicodeString(), status);
         return fillinResult; // succeed, but no remainder
     }
 
     // brute force approach
     // check to make sure result is canonically equivalent
         fillinResult->put(UnicodeString(), new UnicodeString(), status);
         return fillinResult; // succeed, but no remainder
     }
 
     // brute force approach
     // check to make sure result is canonically equivalent

-    int32_t tempLen = inputLen + bufLen;
-
-    UChar trial[bufSize];
-    unorm_decompose(trial, bufSize, temp, tempLen, FALSE, 0, &status);
-
-    if(U_FAILURE(status)
-        || uprv_memcmp(segment+segmentPos, trial, (segLen - segmentPos)*sizeof(UChar)) != 0)
-    {
+    UnicodeString trial;
+    nfd.normalize(temp, trial, status);
+    if(U_FAILURE(status) || trial.compare(segment+segmentPos, segLen - segmentPos) != 0) {

         return NULL;
     }
 
         return NULL;
     }
 

-    return getEquivalents2(fillinResult, buff, bufLen, status);
+    return getEquivalents2(fillinResult, temp.getBuffer()+inputLen, temp.length()-inputLen, status);

 }
 
 U_NAMESPACE_END
 }
 
 U_NAMESPACE_END