ICU-66108.tar.gz

[apple/icu.git] / icuSources / common / normalizer2.cpp

diff --git a/icuSources/common/normalizer2.cpp b/icuSources/common/normalizer2.cpp
index c345c06e21da923ca610b8a08f8dc7fa5236fdfe..6be7e0b21a2bf9ec31cf3e2767e6d2511fc04292 100644 (file)
--- a/icuSources/common/normalizer2.cpp
+++ b/icuSources/common/normalizer2.cpp
@@ -1,3 +1,5 @@
+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html

 /*
 *******************************************************************************
 *
 /*
 *******************************************************************************
 *


@@ -6,7 +8,7 @@
 *
 *******************************************************************************
 *   file name:  normalizer2.cpp
 *
 *******************************************************************************
 *   file name:  normalizer2.cpp

-*   encoding:   US-ASCII
+*   encoding:   UTF-8

 *   tab size:   8 (not used)
 *   indentation:4
 *
 *   tab size:   8 (not used)
 *   indentation:4
 *


@@ -18,7 +20,9 @@
 
 #if !UCONFIG_NO_NORMALIZATION
 
 
 #if !UCONFIG_NO_NORMALIZATION
 

+#include "unicode/edits.h"

 #include "unicode/normalizer2.h"
 #include "unicode/normalizer2.h"

+#include "unicode/stringoptions.h"

 #include "unicode/unistr.h"
 #include "unicode/unorm.h"
 #include "cstring.h"
 #include "unicode/unistr.h"
 #include "unicode/unorm.h"
 #include "cstring.h"


@@ -30,9 +34,11 @@
 
 using icu::Normalizer2Impl;
 
 
 using icu::Normalizer2Impl;
 

+#if NORM2_HARDCODE_NFC_DATA

 // NFC/NFD data machine-generated by gennorm2 --csource
 #define INCLUDED_FROM_NORMALIZER2_CPP
 #include "norm2_nfc_data.h"
 // NFC/NFD data machine-generated by gennorm2 --csource
 #define INCLUDED_FROM_NORMALIZER2_CPP
 #include "norm2_nfc_data.h"

+#endif

 
 U_NAMESPACE_BEGIN
 
 
 U_NAMESPACE_BEGIN
 


@@ -40,6 +46,20 @@ U_NAMESPACE_BEGIN
 
 Normalizer2::~Normalizer2() {}
 
 
 Normalizer2::~Normalizer2() {}
 

+void
+Normalizer2::normalizeUTF8(uint32_t /*options*/, StringPiece src, ByteSink &sink,
+                           Edits *edits, UErrorCode &errorCode) const {
+    if (U_FAILURE(errorCode)) {
+        return;
+    }
+    if (edits != nullptr) {
+        errorCode = U_UNSUPPORTED_ERROR;
+        return;
+    }
+    UnicodeString src16 = UnicodeString::fromUTF8(src);
+    normalize(src16, errorCode).toUTF8(sink);
+}
+

 UBool
 Normalizer2::getRawDecomposition(UChar32, UnicodeString &) const {
     return FALSE;
 UBool
 Normalizer2::getRawDecomposition(UChar32, UnicodeString &) const {
     return FALSE;


@@ -55,6 +75,11 @@ Normalizer2::getCombiningClass(UChar32 /*c*/) const {
     return 0;
 }
 
     return 0;
 }
 

+UBool
+Normalizer2::isNormalizedUTF8(StringPiece s, UErrorCode &errorCode) const {
+    return U_SUCCESS(errorCode) && isNormalized(UnicodeString::fromUTF8(s), errorCode);
+}
+

 // Normalizer2 implementation for the old UNORM_NONE.
 class NoopNormalizer2 : public Normalizer2 {
     virtual ~NoopNormalizer2();
 // Normalizer2 implementation for the old UNORM_NONE.
 class NoopNormalizer2 : public Normalizer2 {
     virtual ~NoopNormalizer2();


@@ -62,7 +87,7 @@ class NoopNormalizer2 : public Normalizer2 {
     virtual UnicodeString &
     normalize(const UnicodeString &src,
               UnicodeString &dest,
     virtual UnicodeString &
     normalize(const UnicodeString &src,
               UnicodeString &dest,

-              UErrorCode &errorCode) const {
+              UErrorCode &errorCode) const U_OVERRIDE {

         if(U_SUCCESS(errorCode)) {
             if(&dest!=&src) {
                 dest=src;
         if(U_SUCCESS(errorCode)) {
             if(&dest!=&src) {
                 dest=src;


@@ -72,10 +97,27 @@ class NoopNormalizer2 : public Normalizer2 {
         }
         return dest;
     }
         }
         return dest;
     }

+    virtual void
+    normalizeUTF8(uint32_t options, StringPiece src, ByteSink &sink,
+                  Edits *edits, UErrorCode &errorCode) const U_OVERRIDE {
+        if(U_SUCCESS(errorCode)) {
+            if (edits != nullptr) {
+                if ((options & U_EDITS_NO_RESET) == 0) {
+                    edits->reset();
+                }
+                edits->addUnchanged(src.length());
+            }
+            if ((options & U_OMIT_UNCHANGED_TEXT) == 0) {
+                sink.Append(src.data(), src.length());
+            }
+            sink.Flush();
+        }
+    }
+

     virtual UnicodeString &
     normalizeSecondAndAppend(UnicodeString &first,
                              const UnicodeString &second,
     virtual UnicodeString &
     normalizeSecondAndAppend(UnicodeString &first,
                              const UnicodeString &second,

-                             UErrorCode &errorCode) const {
+                             UErrorCode &errorCode) const U_OVERRIDE {

         if(U_SUCCESS(errorCode)) {
             if(&first!=&second) {
                 first.append(second);
         if(U_SUCCESS(errorCode)) {
             if(&first!=&second) {
                 first.append(second);


@@ -88,7 +130,7 @@ class NoopNormalizer2 : public Normalizer2 {
     virtual UnicodeString &
     append(UnicodeString &first,
            const UnicodeString &second,
     virtual UnicodeString &
     append(UnicodeString &first,
            const UnicodeString &second,

-           UErrorCode &errorCode) const {
+           UErrorCode &errorCode) const U_OVERRIDE {

         if(U_SUCCESS(errorCode)) {
             if(&first!=&second) {
                 first.append(second);
         if(U_SUCCESS(errorCode)) {
             if(&first!=&second) {
                 first.append(second);


@@ -99,25 +141,29 @@ class NoopNormalizer2 : public Normalizer2 {
         return first;
     }
     virtual UBool
         return first;
     }
     virtual UBool

-    getDecomposition(UChar32, UnicodeString &) const {
+    getDecomposition(UChar32, UnicodeString &) const U_OVERRIDE {

         return FALSE;
     }
         return FALSE;
     }

-    // No need to override the default getRawDecomposition().
+    // No need to U_OVERRIDE the default getRawDecomposition().

     virtual UBool
     virtual UBool

-    isNormalized(const UnicodeString &, UErrorCode &) const {
-        return TRUE;
+    isNormalized(const UnicodeString &, UErrorCode &errorCode) const U_OVERRIDE {
+        return U_SUCCESS(errorCode);
+    }
+    virtual UBool
+    isNormalizedUTF8(StringPiece, UErrorCode &errorCode) const U_OVERRIDE {
+        return U_SUCCESS(errorCode);

     }
     virtual UNormalizationCheckResult
     }
     virtual UNormalizationCheckResult

-    quickCheck(const UnicodeString &, UErrorCode &) const {
+    quickCheck(const UnicodeString &, UErrorCode &) const U_OVERRIDE {

         return UNORM_YES;
     }
     virtual int32_t
         return UNORM_YES;
     }
     virtual int32_t

-    spanQuickCheckYes(const UnicodeString &s, UErrorCode &) const {
+    spanQuickCheckYes(const UnicodeString &s, UErrorCode &) const U_OVERRIDE {

         return s.length();
     }
         return s.length();
     }

-    virtual UBool hasBoundaryBefore(UChar32) const { return TRUE; }
-    virtual UBool hasBoundaryAfter(UChar32) const { return TRUE; }
-    virtual UBool isInert(UChar32) const { return TRUE; }
+    virtual UBool hasBoundaryBefore(UChar32) const U_OVERRIDE { return TRUE; }
+    virtual UBool hasBoundaryAfter(UChar32) const U_OVERRIDE { return TRUE; }
+    virtual UBool isInert(UChar32) const U_OVERRIDE { return TRUE; }

 };
 
 NoopNormalizer2::~NoopNormalizer2() {}
 };
 
 NoopNormalizer2::~NoopNormalizer2() {}


@@ -132,6 +178,36 @@ FCDNormalizer2::~FCDNormalizer2() {}
 
 // instance cache ---------------------------------------------------------- ***
 
 
 // instance cache ---------------------------------------------------------- ***
 

+U_CDECL_BEGIN
+static UBool U_CALLCONV uprv_normalizer2_cleanup();
+U_CDECL_END
+
+static Normalizer2   *noopSingleton;
+static icu::UInitOnce noopInitOnce = U_INITONCE_INITIALIZER;
+
+static void U_CALLCONV initNoopSingleton(UErrorCode &errorCode) {
+    if(U_FAILURE(errorCode)) {
+        return;
+    }
+    noopSingleton=new NoopNormalizer2;
+    if(noopSingleton==NULL) {
+        errorCode=U_MEMORY_ALLOCATION_ERROR;
+        return;
+    }
+    ucln_common_registerCleanup(UCLN_COMMON_NORMALIZER2, uprv_normalizer2_cleanup);
+}
+
+const Normalizer2 *Normalizer2Factory::getNoopInstance(UErrorCode &errorCode) {
+    if(U_FAILURE(errorCode)) { return NULL; }
+    umtx_initOnce(noopInitOnce, &initNoopSingleton, errorCode);
+    return noopSingleton;
+}
+
+const Normalizer2Impl *
+Normalizer2Factory::getImpl(const Normalizer2 *norm2) {
+    return &((Normalizer2WithImpl *)norm2)->impl;
+}
+

 Norm2AllModes::~Norm2AllModes() {
     delete impl;
 }
 Norm2AllModes::~Norm2AllModes() {
     delete impl;
 }


@@ -151,6 +227,7 @@ Norm2AllModes::createInstance(Normalizer2Impl *impl, UErrorCode &errorCode) {
     return allModes;
 }
 
     return allModes;
 }
 

+#if NORM2_HARDCODE_NFC_DATA

 Norm2AllModes *
 Norm2AllModes::createNFCInstance(UErrorCode &errorCode) {
     if(U_FAILURE(errorCode)) {
 Norm2AllModes *
 Norm2AllModes::createNFCInstance(UErrorCode &errorCode) {
     if(U_FAILURE(errorCode)) {


@@ -166,48 +243,15 @@ Norm2AllModes::createNFCInstance(UErrorCode &errorCode) {
     return createInstance(impl, errorCode);
 }
 
     return createInstance(impl, errorCode);
 }
 

-U_CDECL_BEGIN
-static UBool U_CALLCONV uprv_normalizer2_cleanup();
-U_CDECL_END
-

 static Norm2AllModes *nfcSingleton;
 static Norm2AllModes *nfcSingleton;

-static Normalizer2   *noopSingleton;

 
 static icu::UInitOnce nfcInitOnce = U_INITONCE_INITIALIZER;
 
 static icu::UInitOnce nfcInitOnce = U_INITONCE_INITIALIZER;

-static icu::UInitOnce noopInitOnce = U_INITONCE_INITIALIZER;

 
 

-// UInitOnce singleton initialization functions

 static void U_CALLCONV initNFCSingleton(UErrorCode &errorCode) {
     nfcSingleton=Norm2AllModes::createNFCInstance(errorCode);
     ucln_common_registerCleanup(UCLN_COMMON_NORMALIZER2, uprv_normalizer2_cleanup);
 }
 
 static void U_CALLCONV initNFCSingleton(UErrorCode &errorCode) {
     nfcSingleton=Norm2AllModes::createNFCInstance(errorCode);
     ucln_common_registerCleanup(UCLN_COMMON_NORMALIZER2, uprv_normalizer2_cleanup);
 }
 

-static void U_CALLCONV initNoopSingleton(UErrorCode &errorCode) {
-    if(U_FAILURE(errorCode)) {
-        return;
-    }
-    noopSingleton=new NoopNormalizer2;
-    if(noopSingleton==NULL) {
-        errorCode=U_MEMORY_ALLOCATION_ERROR;
-        return;
-    }
-    ucln_common_registerCleanup(UCLN_COMMON_NORMALIZER2, uprv_normalizer2_cleanup);
-}
-
-U_CDECL_BEGIN
-
-static UBool U_CALLCONV uprv_normalizer2_cleanup() {
-    delete nfcSingleton;
-    nfcSingleton = NULL;
-    delete noopSingleton;
-    noopSingleton = NULL;
-    nfcInitOnce.reset(); 
-    noopInitOnce.reset(); 
-    return TRUE;
-}
-
-U_CDECL_END
-

 const Norm2AllModes *
 Norm2AllModes::getNFCInstance(UErrorCode &errorCode) {
     if(U_FAILURE(errorCode)) { return NULL; }
 const Norm2AllModes *
 Norm2AllModes::getNFCInstance(UErrorCode &errorCode) {
     if(U_FAILURE(errorCode)) { return NULL; }


@@ -237,23 +281,29 @@ const Normalizer2 *Normalizer2Factory::getFCCInstance(UErrorCode &errorCode) {
     return allModes!=NULL ? &allModes->fcc : NULL;
 }
 
     return allModes!=NULL ? &allModes->fcc : NULL;
 }
 

-const Normalizer2 *Normalizer2Factory::getNoopInstance(UErrorCode &errorCode) {
-    if(U_FAILURE(errorCode)) { return NULL; }
-    umtx_initOnce(noopInitOnce, &initNoopSingleton, errorCode);
-    return noopSingleton;
-}
-

 const Normalizer2Impl *
 Normalizer2Factory::getNFCImpl(UErrorCode &errorCode) {
     const Norm2AllModes *allModes=Norm2AllModes::getNFCInstance(errorCode);
     return allModes!=NULL ? allModes->impl : NULL;
 }
 const Normalizer2Impl *
 Normalizer2Factory::getNFCImpl(UErrorCode &errorCode) {
     const Norm2AllModes *allModes=Norm2AllModes::getNFCInstance(errorCode);
     return allModes!=NULL ? allModes->impl : NULL;
 }

+#endif  // NORM2_HARDCODE_NFC_DATA

 
 

-const Normalizer2Impl *
-Normalizer2Factory::getImpl(const Normalizer2 *norm2) {
-    return &((Normalizer2WithImpl *)norm2)->impl;
+U_CDECL_BEGIN
+
+static UBool U_CALLCONV uprv_normalizer2_cleanup() {
+    delete noopSingleton;
+    noopSingleton = NULL;
+    noopInitOnce.reset(); 
+#if NORM2_HARDCODE_NFC_DATA
+    delete nfcSingleton;
+    nfcSingleton = NULL;
+    nfcInitOnce.reset(); 
+#endif
+    return TRUE;

 }
 
 }
 

+U_CDECL_END
+

 U_NAMESPACE_END
 
 // C API ------------------------------------------------------------------- ***
 U_NAMESPACE_END
 
 // C API ------------------------------------------------------------------- ***