ICU-64260.0.1.tar.gz

[apple/icu.git] / icuSources / common / brkeng.cpp

diff --git a/icuSources/common/brkeng.cpp b/icuSources/common/brkeng.cpp
index 2419be5f6c5243ebbb951831850db7051aad48d2..80e11585f82a50c7cb39fd45bd240fe07a5de9f2 100644 (file)
--- a/icuSources/common/brkeng.cpp
+++ b/icuSources/common/brkeng.cpp
@@ -1,7 +1,9 @@
-/**
+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+/*

  ************************************************************************************
  ************************************************************************************

- * Copyright (C) 2006-2007, International Business Machines Corporation and others. *
- * All Rights Reserved.                                                             *
+ * Copyright (C) 2006-2016, International Business Machines Corporation
+ * and others. All Rights Reserved.

  ************************************************************************************
  */
 
  ************************************************************************************
  */
 


@@ -9,9 +11,6 @@
 
 #if !UCONFIG_NO_BREAK_ITERATION
 
 
 #if !UCONFIG_NO_BREAK_ITERATION
 

-#include "brkeng.h"
-#include "dictbe.h"
-#include "triedict.h"

 #include "unicode/uchar.h"
 #include "unicode/uniset.h"
 #include "unicode/chariter.h"
 #include "unicode/uchar.h"
 #include "unicode/uniset.h"
 #include "unicode/chariter.h"


@@ -20,6 +19,15 @@
 #include "unicode/putil.h"
 #include "unicode/ustring.h"
 #include "unicode/uscript.h"
 #include "unicode/putil.h"
 #include "unicode/ustring.h"
 #include "unicode/uscript.h"

+#include "unicode/ucharstrie.h"
+#include "unicode/bytestrie.h"
+
+#include "brkeng.h"
+#include "cmemory.h"
+#include "dictbe.h"
+#include "charstr.h"
+#include "dictionarydata.h"
+#include "mutex.h"

 #include "uvector.h"
 #include "umutex.h"
 #include "uresimp.h"
 #include "uvector.h"
 #include "umutex.h"
 #include "uresimp.h"


@@ -51,66 +59,47 @@ LanguageBreakFactory::~LanguageBreakFactory() {
  ******************************************************************
  */
 
  ******************************************************************
  */
 

-UnhandledEngine::UnhandledEngine(UErrorCode &/*status*/) {
-    for (int32_t i = 0; i < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0])); ++i) {
-        fHandled[i] = 0;
-    }
+UnhandledEngine::UnhandledEngine(UErrorCode &status) : fHandled(nullptr) {
+    (void)status;

 }
 
 UnhandledEngine::~UnhandledEngine() {
 }
 
 UnhandledEngine::~UnhandledEngine() {

-    for (int32_t i = 0; i < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0])); ++i) {
-        if (fHandled[i] != 0) {
-            delete fHandled[i];
-        }
-    }
+    delete fHandled;
+    fHandled = nullptr;

 }
 
 UBool
 }
 
 UBool

-UnhandledEngine::handles(UChar32 c, int32_t breakType) const {
-    return (breakType >= 0 && breakType < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0]))
-        && fHandled[breakType] != 0 && fHandled[breakType]->contains(c));
+UnhandledEngine::handles(UChar32 c) const {
+    return fHandled && fHandled->contains(c);

 }
 
 int32_t
 UnhandledEngine::findBreaks( UText *text,
 }
 
 int32_t
 UnhandledEngine::findBreaks( UText *text,

-                                 int32_t startPos,
-                                 int32_t endPos,
-                                 UBool reverse,
-                                 int32_t breakType,
-                                 UStack &/*foundBreaks*/ ) const {
-    if (breakType >= 0 && breakType < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0]))) {
-        UChar32 c = utext_current32(text); 
-        if (reverse) {
-            while((int32_t)utext_getNativeIndex(text) > startPos && fHandled[breakType]->contains(c)) {
-                c = utext_previous32(text);
-            }
-        }
-        else {
-            while((int32_t)utext_getNativeIndex(text) < endPos && fHandled[breakType]->contains(c)) {
-                utext_next32(text);            // TODO:  recast loop to work with post-increment operations.
-                c = utext_current32(text);
-            }
-        }
+                             int32_t /* startPos */,
+                             int32_t endPos,
+                             UVector32 &/*foundBreaks*/ ) const {
+    UChar32 c = utext_current32(text); 
+    while((int32_t)utext_getNativeIndex(text) < endPos && fHandled->contains(c)) {
+        utext_next32(text);            // TODO:  recast loop to work with post-increment operations.
+        c = utext_current32(text);

     }
     return 0;
 }
 
 void
     }
     return 0;
 }
 
 void

-UnhandledEngine::handleCharacter(UChar32 c, int32_t breakType) {
-    if (breakType >= 0 && breakType < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0]))) {
-        if (fHandled[breakType] == 0) {
-            fHandled[breakType] = new UnicodeSet();
-            if (fHandled[breakType] == 0) {
-                return;
-            }
-        }
-        if (!fHandled[breakType]->contains(c)) {
-            UErrorCode status = U_ZERO_ERROR;
-            // Apply the entire script of the character.
-            int32_t script = u_getIntPropertyValue(c, UCHAR_SCRIPT);
-            fHandled[breakType]->applyIntPropertyValue(UCHAR_SCRIPT, script, status);
+UnhandledEngine::handleCharacter(UChar32 c) {
+    if (fHandled == nullptr) {
+        fHandled = new UnicodeSet();
+        if (fHandled == nullptr) {
+            return;

         }
     }
         }
     }

+    if (!fHandled->contains(c)) {
+        UErrorCode status = U_ZERO_ERROR;
+        // Apply the entire script of the character.
+        int32_t script = u_getIntPropertyValue(c, UCHAR_SCRIPT);
+        fHandled->applyIntPropertyValue(UCHAR_SCRIPT, script, status);
+    }

 }
 
 /*
 }
 
 /*


@@ -130,107 +119,99 @@ ICULanguageBreakFactory::~ICULanguageBreakFactory() {
 U_NAMESPACE_END
 U_CDECL_BEGIN
 static void U_CALLCONV _deleteEngine(void *obj) {
 U_NAMESPACE_END
 U_CDECL_BEGIN
 static void U_CALLCONV _deleteEngine(void *obj) {

-    delete (const U_NAMESPACE_QUALIFIER LanguageBreakEngine *) obj;
+    delete (const icu::LanguageBreakEngine *) obj;

 }
 U_CDECL_END
 U_NAMESPACE_BEGIN
 
 const LanguageBreakEngine *
 }
 U_CDECL_END
 U_NAMESPACE_BEGIN
 
 const LanguageBreakEngine *

-ICULanguageBreakFactory::getEngineFor(UChar32 c, int32_t breakType) {
-    UBool       needsInit;
-    int32_t     i;
+ICULanguageBreakFactory::getEngineFor(UChar32 c) {

     const LanguageBreakEngine *lbe = NULL;
     UErrorCode  status = U_ZERO_ERROR;
 
     const LanguageBreakEngine *lbe = NULL;
     UErrorCode  status = U_ZERO_ERROR;
 

-    // TODO: The global mutex should not be used.
-    // The global mutex should only be used for short periods.
-    // A ICULanguageBreakFactory specific mutex should be used.
-    umtx_lock(NULL);
-    needsInit = (UBool)(fEngines == NULL);
-    if (!needsInit) {
-        i = fEngines->size();
-        while (--i >= 0) {
-            lbe = (const LanguageBreakEngine *)(fEngines->elementAt(i));
-            if (lbe != NULL && lbe->handles(c, breakType)) {
-                break;
-            }
-            lbe = NULL;
-        }
-    }
-    umtx_unlock(NULL);
-    
-    if (lbe != NULL) {
-        return lbe;
-    }
-    
-    if (needsInit) {
+    static UMutex *gBreakEngineMutex = STATIC_NEW(UMutex);
+    Mutex m(gBreakEngineMutex);
+
+    if (fEngines == NULL) {

         UStack  *engines = new UStack(_deleteEngine, NULL, status);
         UStack  *engines = new UStack(_deleteEngine, NULL, status);

-        if (U_SUCCESS(status) && engines == NULL) {
-            status = U_MEMORY_ALLOCATION_ERROR;
-        }
-        else if (U_FAILURE(status)) {
+        if (U_FAILURE(status) || engines == NULL) {
+            // Note: no way to return error code to caller.

             delete engines;
             delete engines;

-            engines = NULL;
+            return NULL;

         }
         }

-        else {
-            umtx_lock(NULL);
-            if (fEngines == NULL) {
-                fEngines = engines;
-                engines = NULL;
+        fEngines = engines;
+    } else {
+        int32_t i = fEngines->size();
+        while (--i >= 0) {
+            lbe = (const LanguageBreakEngine *)(fEngines->elementAt(i));
+            if (lbe != NULL && lbe->handles(c)) {
+                return lbe;

             }
             }

-            umtx_unlock(NULL);
-            delete engines;

         }
     }
     
         }
     }
     

-    if (fEngines == NULL) {
-        return NULL;
-    }
-
-    // We didn't find an engine the first time through, or there was no
-    // stack. Create an engine.
-    const LanguageBreakEngine *newlbe = loadEngineFor(c, breakType);
-    
-    // Now get the lock, and see if someone else has created it in the
-    // meantime
-    umtx_lock(NULL);
-    i = fEngines->size();
-    while (--i >= 0) {
-        lbe = (const LanguageBreakEngine *)(fEngines->elementAt(i));
-        if (lbe != NULL && lbe->handles(c, breakType)) {
-            break;
-        }
-        lbe = NULL;
-    }
-    if (lbe == NULL && newlbe != NULL) {
-        fEngines->push((void *)newlbe, status);
-        lbe = newlbe;
-        newlbe = NULL;
+    // We didn't find an engine. Create one.
+    lbe = loadEngineFor(c);
+    if (lbe != NULL) {
+        fEngines->push((void *)lbe, status);

     }
     }

-    umtx_unlock(NULL);
-    
-    delete newlbe;
-

     return lbe;
 }
 
 const LanguageBreakEngine *
     return lbe;
 }
 
 const LanguageBreakEngine *

-ICULanguageBreakFactory::loadEngineFor(UChar32 c, int32_t breakType) {
+ICULanguageBreakFactory::loadEngineFor(UChar32 c) {

     UErrorCode status = U_ZERO_ERROR;
     UScriptCode code = uscript_getScript(c, &status);
     if (U_SUCCESS(status)) {
     UErrorCode status = U_ZERO_ERROR;
     UScriptCode code = uscript_getScript(c, &status);
     if (U_SUCCESS(status)) {

-        const CompactTrieDictionary *dict = loadDictionaryFor(code, breakType);
-        if (dict != NULL) {
+        DictionaryMatcher *m = loadDictionaryMatcherFor(code);
+        if (m != NULL) {

             const LanguageBreakEngine *engine = NULL;
             switch(code) {
             case USCRIPT_THAI:
             const LanguageBreakEngine *engine = NULL;
             switch(code) {
             case USCRIPT_THAI:

-                engine = new ThaiBreakEngine(dict, status);
+                engine = new ThaiBreakEngine(m, status);
+                break;
+            case USCRIPT_LAO:
+                engine = new LaoBreakEngine(m, status);
+                break;
+            case USCRIPT_MYANMAR:
+                engine = new BurmeseBreakEngine(m, status);

                 break;
                 break;

+            case USCRIPT_KHMER:
+                engine = new KhmerBreakEngine(m, status);
+                break;
+
+#if !UCONFIG_NO_NORMALIZATION
+                // CJK not available w/o normalization
+            case USCRIPT_HANGUL:
+                engine = new CjkBreakEngine(m, kKorean, status);
+                break;
+
+            // use same BreakEngine and dictionary for both Chinese and Japanese
+            case USCRIPT_HIRAGANA:
+            case USCRIPT_KATAKANA:
+            case USCRIPT_HAN:
+                engine = new CjkBreakEngine(m, kChineseJapanese, status);
+                break;
+#if 0
+            // TODO: Have to get some characters with script=common handled
+            // by CjkBreakEngine (e.g. U+309B). Simply subjecting
+            // them to CjkBreakEngine does not work. The engine has to
+            // special-case them.
+            case USCRIPT_COMMON:
+            {
+                UBlockCode block = ublock_getCode(code);
+                if (block == UBLOCK_HIRAGANA || block == UBLOCK_KATAKANA)
+                   engine = new CjkBreakEngine(dict, kChineseJapanese, status);
+                break;
+            }
+#endif
+#endif
+

             default:
                 break;
             }
             if (engine == NULL) {
             default:
                 break;
             }
             if (engine == NULL) {

-                delete dict;
+                delete m;

             }
             else if (U_FAILURE(status)) {
                 delete engine;
             }
             else if (U_FAILURE(status)) {
                 delete engine;


@@ -242,45 +223,58 @@ ICULanguageBreakFactory::loadEngineFor(UChar32 c, int32_t breakType) {
     return NULL;
 }
 
     return NULL;
 }
 

-const CompactTrieDictionary *
-ICULanguageBreakFactory::loadDictionaryFor(UScriptCode script, int32_t /*breakType*/) {
+DictionaryMatcher *
+ICULanguageBreakFactory::loadDictionaryMatcherFor(UScriptCode script) { 

     UErrorCode status = U_ZERO_ERROR;
     UErrorCode status = U_ZERO_ERROR;

-    // Open root from brkitr tree.
-    char dictnbuff[256];
-    char ext[4]={'\0'};
-
+    // open root from brkitr tree.

     UResourceBundle *b = ures_open(U_ICUDATA_BRKITR, "", &status);
     b = ures_getByKeyWithFallback(b, "dictionaries", b, &status);
     UResourceBundle *b = ures_open(U_ICUDATA_BRKITR, "", &status);
     b = ures_getByKeyWithFallback(b, "dictionaries", b, &status);

-    b = ures_getByKeyWithFallback(b, uscript_getShortName(script), b, &status);

     int32_t dictnlength = 0;
     int32_t dictnlength = 0;

-    const UChar *dictfname = ures_getString(b, &dictnlength, &status);
-    if (U_SUCCESS(status) && (size_t)dictnlength >= sizeof(dictnbuff)) {
-        dictnlength = 0;
-        status = U_BUFFER_OVERFLOW_ERROR;
+    const UChar *dictfname =
+        ures_getStringByKeyWithFallback(b, uscript_getShortName(script), &dictnlength, &status);
+    if (U_FAILURE(status)) {
+        ures_close(b);
+        return NULL;

     }
     }

-    if (U_SUCCESS(status) && dictfname) {
-        UChar* extStart=u_strchr(dictfname, 0x002e);
-        int len = 0;
-        if(extStart!=NULL){
-            len = extStart-dictfname;
-            u_UCharsToChars(extStart+1, ext, sizeof(ext)); // nul terminates the buff
-            u_UCharsToChars(dictfname, dictnbuff, len);
-        }
-        dictnbuff[len]=0; // nul terminate
+    CharString dictnbuf;
+    CharString ext;
+    const UChar *extStart = u_memrchr(dictfname, 0x002e, dictnlength);  // last dot
+    if (extStart != NULL) {
+        int32_t len = (int32_t)(extStart - dictfname);
+        ext.appendInvariantChars(UnicodeString(FALSE, extStart + 1, dictnlength - len - 1), status);
+        dictnlength = len;

     }
     }

+    dictnbuf.appendInvariantChars(UnicodeString(FALSE, dictfname, dictnlength), status);

     ures_close(b);
     ures_close(b);

-    UDataMemory *file = udata_open(U_ICUDATA_BRKITR, ext, dictnbuff, &status);
+
+    UDataMemory *file = udata_open(U_ICUDATA_BRKITR, ext.data(), dictnbuf.data(), &status);

     if (U_SUCCESS(status)) {
     if (U_SUCCESS(status)) {

-        const CompactTrieDictionary *dict = new CompactTrieDictionary(
-            file, status);
-        if (U_SUCCESS(status) && dict == NULL) {
-            status = U_MEMORY_ALLOCATION_ERROR;
+        // build trie
+        const uint8_t *data = (const uint8_t *)udata_getMemory(file);
+        const int32_t *indexes = (const int32_t *)data;
+        const int32_t offset = indexes[DictionaryData::IX_STRING_TRIE_OFFSET];
+        const int32_t trieType = indexes[DictionaryData::IX_TRIE_TYPE] & DictionaryData::TRIE_TYPE_MASK;
+        DictionaryMatcher *m = NULL;
+        if (trieType == DictionaryData::TRIE_TYPE_BYTES) {
+            const int32_t transform = indexes[DictionaryData::IX_TRANSFORM];
+            const char *characters = (const char *)(data + offset);
+            m = new BytesDictionaryMatcher(characters, transform, file);
+        }
+        else if (trieType == DictionaryData::TRIE_TYPE_UCHARS) {
+            const UChar *characters = (const UChar *)(data + offset);
+            m = new UCharsDictionaryMatcher(characters, file);

         }
         }

-        if (U_FAILURE(status)) {
-            delete dict;
-            dict = NULL;
+        if (m == NULL) {
+            // no matcher exists to take ownership - either we are an invalid 
+            // type or memory allocation failed
+            udata_close(file);

         }
         }

-        return dict;
+        return m;
+    } else if (dictfname != NULL) {
+        // we don't have a dictionary matcher.
+        // returning NULL here will cause us to fail to find a dictionary break engine, as expected
+        status = U_ZERO_ERROR;
+        return NULL;

     }
     return NULL;
 }
     }
     return NULL;
 }