X-Git-Url: https://git.saurik.com/apple/icu.git/blobdiff_plain/73c04bcfe1096173b00431f0cdc742894b15eef0..ef6cf650f4a75c3f97de06b51fa104f2069b9ea2:/icuSources/common/caniter.cpp?ds=sidebyside diff --git a/icuSources/common/caniter.cpp b/icuSources/common/caniter.cpp index 04d48ba8..58b37818 100644 --- a/icuSources/common/caniter.cpp +++ b/icuSources/common/caniter.cpp @@ -1,7 +1,7 @@ /* ***************************************************************************** - * Copyright (C) 1996-2006, International Business Machines Corporation and * - * others. All Rights Reserved. * + * Copyright (C) 1996-2015, International Business Machines Corporation and + * others. All Rights Reserved. ***************************************************************************** */ @@ -9,14 +9,16 @@ #if !UCONFIG_NO_NORMALIZATION -#include "unicode/uset.h" -#include "unicode/ustring.h" -#include "hash.h" -#include "unormimp.h" #include "unicode/caniter.h" -#include "unicode/normlzr.h" +#include "unicode/normalizer2.h" #include "unicode/uchar.h" +#include "unicode/uniset.h" +#include "unicode/usetiter.h" +#include "unicode/ustring.h" +#include "unicode/utf16.h" #include "cmemory.h" +#include "hash.h" +#include "normalizer2impl.h" /** * This class allows one to iterate through all the strings that are canonically equivalent to a given @@ -68,9 +70,11 @@ CanonicalIterator::CanonicalIterator(const UnicodeString &sourceStr, UErrorCode pieces_length(0), pieces_lengths(NULL), current(NULL), - current_length(0) + current_length(0), + nfd(*Normalizer2::getNFDInstance(status)), + nfcImpl(*Normalizer2Factory::getNFCImpl(status)) { - if(U_SUCCESS(status)) { + if(U_SUCCESS(status) && nfcImpl.ensureCanonIterData(status)) { setSource(sourceStr, status); } } @@ -166,7 +170,7 @@ void CanonicalIterator::setSource(const UnicodeString &newSource, UErrorCode &st int32_t i = 0; UnicodeString *list = NULL; - Normalizer::normalize(newSource, UNORM_NFD, 0, source, status); + nfd.normalize(newSource, source, status); if(U_FAILURE(status)) { return; } @@ -204,16 +208,16 @@ void CanonicalIterator::setSource(const UnicodeString &newSource, UErrorCode &st // i should initialy be the number of code units at the // start of the string - i = UTF16_CHAR_LENGTH(source.char32At(0)); + i = U16_LENGTH(source.char32At(0)); //int32_t i = 1; // find the segments // This code iterates through the source string and // extracts segments that end up on a codepoint that // doesn't start any decompositions. (Analysis is done // on the NFD form - see above). - for (; i < source.length(); i += UTF16_CHAR_LENGTH(cp)) { + for (; i < source.length(); i += U16_LENGTH(cp)) { cp = source.char32At(i); - if (unorm_isCanonSafeStart(cp)) { + if (nfcImpl.isCanonSegmentStarter(cp)) { source.extract(start, i-start, list[list_length++]); // add up to i start = i; } @@ -285,12 +289,12 @@ void U_EXPORT2 CanonicalIterator::permute(UnicodeString &source, UBool skipZeros if(U_FAILURE(status)) { return; } - subpermute.setValueDeleter(uhash_deleteUnicodeString); + subpermute.setValueDeleter(uprv_deleteUObject); - for (i = 0; i < source.length(); i += UTF16_CHAR_LENGTH(cp)) { + for (i = 0; i < source.length(); i += U16_LENGTH(cp)) { cp = source.char32At(i); const UHashElement *ne = NULL; - int32_t el = -1; + int32_t el = UHASH_FIRST; UnicodeString subPermuteString = source; // optimization: @@ -305,7 +309,7 @@ void U_EXPORT2 CanonicalIterator::permute(UnicodeString &source, UBool skipZeros // see what the permutations of the characters before and after this one are //Hashtable *subpermute = permute(source.substring(0,i) + source.substring(i + UTF16.getCharCount(cp))); - permute(subPermuteString.replace(i, UTF16_CHAR_LENGTH(cp), NULL, 0), skipZeros, &subpermute, status); + permute(subPermuteString.replace(i, U16_LENGTH(cp), NULL, 0), skipZeros, &subpermute, status); /* Test for buffer overflows */ if(U_FAILURE(status)) { return; @@ -342,9 +346,9 @@ UnicodeString* CanonicalIterator::getEquivalents(const UnicodeString &segment, i if (U_FAILURE(status)) { return 0; } - result.setValueDeleter(uhash_deleteUnicodeString); - permutations.setValueDeleter(uhash_deleteUnicodeString); - basic.setValueDeleter(uhash_deleteUnicodeString); + result.setValueDeleter(uprv_deleteUObject); + permutations.setValueDeleter(uprv_deleteUObject); + basic.setValueDeleter(uprv_deleteUObject); UChar USeg[256]; int32_t segLen = segment.extract(USeg, 256, status); @@ -355,7 +359,7 @@ UnicodeString* CanonicalIterator::getEquivalents(const UnicodeString &segment, i // TODO: optimize by not permuting any class zero. const UHashElement *ne = NULL; - int32_t el = -1; + int32_t el = UHASH_FIRST; //Iterator it = basic.iterator(); ne = basic.nextElement(el); //while (it.hasNext()) @@ -366,7 +370,7 @@ UnicodeString* CanonicalIterator::getEquivalents(const UnicodeString &segment, i permutations.removeAll(); permute(item, CANITER_SKIP_ZEROES, &permutations, status); const UHashElement *ne2 = NULL; - int32_t el2 = -1; + int32_t el2 = UHASH_FIRST; //Iterator it2 = permutations.iterator(); ne2 = permutations.nextElement(el2); //while (it2.hasNext()) @@ -375,7 +379,7 @@ UnicodeString* CanonicalIterator::getEquivalents(const UnicodeString &segment, i //UnicodeString *possible = new UnicodeString(*((UnicodeString *)(ne2->value.pointer))); UnicodeString possible(*((UnicodeString *)(ne2->value.pointer))); UnicodeString attempt; - Normalizer::normalize(possible, UNORM_NFD, 0, attempt, status); + nfd.normalize(possible, attempt, status); // TODO: check if operator == is semanticaly the same as attempt.equals(segment) if (attempt==segment) { @@ -412,7 +416,7 @@ UnicodeString* CanonicalIterator::getEquivalents(const UnicodeString &segment, i } //result.toArray(finalResult); result_len = 0; - el = -1; + el = UHASH_FIRST; ne = result.nextElement(el); while(ne != NULL) { finalResult[result_len++] = *((UnicodeString *)(ne->value.pointer)); @@ -435,30 +439,31 @@ Hashtable *CanonicalIterator::getEquivalents2(Hashtable *fillinResult, const UCh fillinResult->put(toPut, new UnicodeString(toPut), status); - USerializedSet starts; + UnicodeSet starts; // cycle through all the characters - UChar32 cp, end = 0; - int32_t i = 0, j; - for (i = 0; i < segLen; i += UTF16_CHAR_LENGTH(cp)) { + UChar32 cp; + for (int32_t i = 0; i < segLen; i += U16_LENGTH(cp)) { // see if any character is at the start of some decomposition - UTF_GET_CHAR(segment, 0, i, segLen, cp); - if (!unorm_getCanonStartSet(cp, &starts)) { + U16_GET(segment, 0, i, segLen, cp); + if (!nfcImpl.getCanonStartSet(cp, starts)) { continue; } - // if so, see which decompositions match - for(j = 0, cp = end+1; cp <= end || uset_getSerializedRange(&starts, j++, &cp, &end); ++cp) { + // if so, see which decompositions match + UnicodeSetIterator iter(starts); + while (iter.next()) { + UChar32 cp2 = iter.getCodepoint(); Hashtable remainder(status); - remainder.setValueDeleter(uhash_deleteUnicodeString); - if (extract(&remainder, cp, segment, segLen, i, status) == NULL) { + remainder.setValueDeleter(uprv_deleteUObject); + if (extract(&remainder, cp2, segment, segLen, i, status) == NULL) { continue; } // there were some matches, so add all the possibilities to the set. UnicodeString prefix(segment, i); - prefix += cp; + prefix += cp2; - int32_t el = -1; + int32_t el = UHASH_FIRST; const UHashElement *ne = remainder.nextElement(el); while (ne != NULL) { UnicodeString item = *((UnicodeString *)(ne->value.pointer)); @@ -499,73 +504,46 @@ Hashtable *CanonicalIterator::extract(Hashtable *fillinResult, UChar32 comp, con return NULL; } - const int32_t bufSize = 256; - int32_t bufLen = 0; - UChar temp[bufSize]; - - int32_t inputLen = 0, decompLen; - UChar stackBuffer[4]; - const UChar *decomp; - - U16_APPEND_UNSAFE(temp, inputLen, comp); - decomp = unorm_getCanonicalDecomposition(comp, stackBuffer, &decompLen); - if(decomp == NULL) { - /* copy temp */ - stackBuffer[0] = temp[0]; - if(inputLen > 1) { - stackBuffer[1] = temp[1]; - } - decomp = stackBuffer; - decompLen = inputLen; + UnicodeString temp(comp); + int32_t inputLen=temp.length(); + UnicodeString decompString; + nfd.normalize(temp, decompString, status); + if (U_FAILURE(status)) { + return NULL; } - - UChar *buff = temp+inputLen; + if (decompString.isBogus()) { + status = U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + const UChar *decomp=decompString.getBuffer(); + int32_t decompLen=decompString.length(); // See if it matches the start of segment (at segmentPos) UBool ok = FALSE; UChar32 cp; int32_t decompPos = 0; UChar32 decompCp; - UTF_NEXT_CHAR(decomp, decompPos, decompLen, decompCp); + U16_NEXT(decomp, decompPos, decompLen, decompCp); - int32_t i; - UBool overflow = FALSE; - - i = segmentPos; + int32_t i = segmentPos; while(i < segLen) { - UTF_NEXT_CHAR(segment, i, segLen, cp); + U16_NEXT(segment, i, segLen, cp); if (cp == decompCp) { // if equal, eat another cp from decomp //if (PROGRESS) printf(" matches: %s\n", UToS(Tr(UnicodeString(cp)))); if (decompPos == decompLen) { // done, have all decomp characters! - //u_strcat(buff+bufLen, segment+i); - uprv_memcpy(buff+bufLen, segment+i, (segLen-i)*sizeof(UChar)); - bufLen+=segLen-i; - + temp.append(segment+i, segLen-i); ok = TRUE; break; } - UTF_NEXT_CHAR(decomp, decompPos, decompLen, decompCp); + U16_NEXT(decomp, decompPos, decompLen, decompCp); } else { //if (PROGRESS) printf(" buffer: %s\n", UToS(Tr(UnicodeString(cp)))); // brute force approach - - U16_APPEND(buff, bufLen, bufSize, cp, overflow); - - if(overflow) { - /* - * ### TODO handle buffer overflow - * The buffer is large, but an overflow may still happen with - * unusual input (many combining marks?). - * Reallocate buffer and continue. - * markus 20020929 - */ - - overflow = FALSE; - } + temp.append(cp); /* TODO: optimize // since we know that the classes are monotonically increasing, after zero @@ -585,25 +563,20 @@ Hashtable *CanonicalIterator::extract(Hashtable *fillinResult, UChar32 comp, con //if (PROGRESS) printf("Matches\n"); - if (bufLen == 0) { + if (inputLen == temp.length()) { fillinResult->put(UnicodeString(), new UnicodeString(), status); return fillinResult; // succeed, but no remainder } // brute force approach // check to make sure result is canonically equivalent - int32_t tempLen = inputLen + bufLen; - - UChar trial[bufSize]; - unorm_decompose(trial, bufSize, temp, tempLen, FALSE, 0, &status); - - if(U_FAILURE(status) - || uprv_memcmp(segment+segmentPos, trial, (segLen - segmentPos)*sizeof(UChar)) != 0) - { + UnicodeString trial; + nfd.normalize(temp, trial, status); + if(U_FAILURE(status) || trial.compare(segment+segmentPos, segLen - segmentPos) != 0) { return NULL; } - return getEquivalents2(fillinResult, buff, bufLen, status); + return getEquivalents2(fillinResult, temp.getBuffer()+inputLen, temp.length()-inputLen, status); } U_NAMESPACE_END