ICU-551.30.tar.gz

[apple/icu.git] / icuSources / i18n / csdetect.cpp

diff --git a/icuSources/i18n/csdetect.cpp b/icuSources/i18n/csdetect.cpp
index 13e0e913fd5f50403dcb01dddeb23d0006262aec..66d8f3a2ae3cc93142b509eb1c2d51638e93f7f4 100644 (file)
--- a/icuSources/i18n/csdetect.cpp
+++ b/icuSources/i18n/csdetect.cpp
@@ -1,6 +1,6 @@
 /*
  **********************************************************************
 /*
  **********************************************************************

- *   Copyright (C) 2005-2008, International Business Machines
+ *   Copyright (C) 2005-2015, International Business Machines

  *   Corporation and others.  All Rights Reserved.
  **********************************************************************
  */
  *   Corporation and others.  All Rights Reserved.
  **********************************************************************
  */


@@ -32,13 +32,28 @@
 #define NEW_ARRAY(type,count) (type *) uprv_malloc((count) * sizeof(type))
 #define DELETE_ARRAY(array) uprv_free((void *) (array))
 
 #define NEW_ARRAY(type,count) (type *) uprv_malloc((count) * sizeof(type))
 #define DELETE_ARRAY(array) uprv_free((void *) (array))
 

-U_CDECL_BEGIN
-static U_NAMESPACE_QUALIFIER CharsetRecognizer **fCSRecognizers = NULL;
+U_NAMESPACE_BEGIN
+
+struct CSRecognizerInfo : public UMemory {
+    CSRecognizerInfo(CharsetRecognizer *recognizer, UBool isDefaultEnabled)
+        : recognizer(recognizer), isDefaultEnabled(isDefaultEnabled) {};
+
+    ~CSRecognizerInfo() {delete recognizer;};
+
+    CharsetRecognizer *recognizer;
+    UBool isDefaultEnabled;
+};
+
+U_NAMESPACE_END

 
 

+static icu::CSRecognizerInfo **fCSRecognizers = NULL;
+static icu::UInitOnce gCSRecognizersInitOnce;

 static int32_t fCSRecognizers_size = 0;
 
 static int32_t fCSRecognizers_size = 0;
 

+U_CDECL_BEGIN

 static UBool U_CALLCONV csdet_cleanup(void)
 {
 static UBool U_CALLCONV csdet_cleanup(void)
 {

+    U_NAMESPACE_USE

     if (fCSRecognizers != NULL) {
         for(int32_t r = 0; r < fCSRecognizers_size; r += 1) {
             delete fCSRecognizers[r];
     if (fCSRecognizers != NULL) {
         for(int32_t r = 0; r < fCSRecognizers_size; r += 1) {
             delete fCSRecognizers[r];


@@ -49,6 +64,7 @@ static UBool U_CALLCONV csdet_cleanup(void)
         fCSRecognizers = NULL;
         fCSRecognizers_size = 0;
     }
         fCSRecognizers = NULL;
         fCSRecognizers_size = 0;
     }

+    gCSRecognizersInitOnce.reset();

 
     return TRUE;
 }
 
     return TRUE;
 }


@@ -65,108 +81,76 @@ charsetMatchComparator(const void * /*context*/, const void *left, const void *r
     return (*csm_r)->getConfidence() - (*csm_l)->getConfidence();
 }
 
     return (*csm_r)->getConfidence() - (*csm_l)->getConfidence();
 }
 

-U_CDECL_END
-
-U_NAMESPACE_BEGIN
-
-void CharsetDetector::setRecognizers(UErrorCode &status)
-{
-    UBool needsInit;
-    CharsetRecognizer **recognizers;
-
-    if (U_FAILURE(status)) {
-        return;
-    }
-
-    UMTX_CHECK(NULL, (UBool) (fCSRecognizers == NULL), needsInit);
-
-    if (needsInit) {
-        CharsetRecognizer *tempArray[] = {
-            new CharsetRecog_UTF8(),
-
-            new CharsetRecog_UTF_16_BE(),
-            new CharsetRecog_UTF_16_LE(),
-            new CharsetRecog_UTF_32_BE(),
-            new CharsetRecog_UTF_32_LE(),
-
-            new CharsetRecog_8859_1_en(),
-            new CharsetRecog_8859_1_da(),
-            new CharsetRecog_8859_1_de(),
-            new CharsetRecog_8859_1_es(),
-            new CharsetRecog_8859_1_fr(),
-            new CharsetRecog_8859_1_it(),
-            new CharsetRecog_8859_1_nl(),
-            new CharsetRecog_8859_1_no(),
-            new CharsetRecog_8859_1_pt(),
-            new CharsetRecog_8859_1_sv(),
-            new CharsetRecog_8859_2_cs(),
-            new CharsetRecog_8859_2_hu(),
-            new CharsetRecog_8859_2_pl(),
-            new CharsetRecog_8859_2_ro(),
-            new CharsetRecog_8859_5_ru(),
-            new CharsetRecog_8859_6_ar(),
-            new CharsetRecog_8859_7_el(),
-            new CharsetRecog_8859_8_I_he(),
-            new CharsetRecog_8859_8_he(),
-            new CharsetRecog_windows_1251(),
-            new CharsetRecog_windows_1256(),
-            new CharsetRecog_KOI8_R(),
-            new CharsetRecog_8859_9_tr(),
-            new CharsetRecog_sjis(),
-            new CharsetRecog_gb_18030(),
-            new CharsetRecog_euc_jp(),
-            new CharsetRecog_euc_kr(),
-            new CharsetRecog_big5(),
-
-            new CharsetRecog_2022JP(),
-            new CharsetRecog_2022KR(),
-            new CharsetRecog_2022CN()
-        };
-        int32_t rCount = ARRAY_SIZE(tempArray);
-        int32_t r;
-
-        recognizers = NEW_ARRAY(CharsetRecognizer *, rCount);
-
-        if (recognizers == NULL) {
-            status = U_MEMORY_ALLOCATION_ERROR;
-            return;
-        } else {
-            for (r = 0; r < rCount; r += 1) {
-                recognizers[r] = tempArray[r];
+static void U_CALLCONV initRecognizers(UErrorCode &status) {
+    U_NAMESPACE_USE
+    ucln_i18n_registerCleanup(UCLN_I18N_CSDET, csdet_cleanup);
+    CSRecognizerInfo *tempArray[] = {
+        new CSRecognizerInfo(new CharsetRecog_UTF8(), TRUE),
+
+        new CSRecognizerInfo(new CharsetRecog_UTF_16_BE(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_UTF_16_LE(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_UTF_32_BE(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_UTF_32_LE(), TRUE),
+
+        new CSRecognizerInfo(new CharsetRecog_8859_1(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_8859_2(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_8859_5_ru(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_8859_6_ar(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_8859_7_el(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_8859_8_I_he(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_8859_8_he(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_windows_1251(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_windows_1256(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_KOI8_R(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_8859_9_tr(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_sjis(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_gb_18030(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_euc_jp(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_euc_kr(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_big5(), TRUE),
+
+        new CSRecognizerInfo(new CharsetRecog_2022JP(), TRUE),
+#if !UCONFIG_ONLY_HTML_CONVERSION
+        new CSRecognizerInfo(new CharsetRecog_2022KR(), TRUE),
+        new CSRecognizerInfo(new CharsetRecog_2022CN(), TRUE),
+
+        new CSRecognizerInfo(new CharsetRecog_IBM424_he_rtl(), FALSE),
+        new CSRecognizerInfo(new CharsetRecog_IBM424_he_ltr(), FALSE),
+        new CSRecognizerInfo(new CharsetRecog_IBM420_ar_rtl(), FALSE),
+        new CSRecognizerInfo(new CharsetRecog_IBM420_ar_ltr(), FALSE)
+#endif
+    };
+    int32_t rCount = ARRAY_SIZE(tempArray);

 
 

-                if (recognizers[r] == NULL) {
-                    status = U_MEMORY_ALLOCATION_ERROR;
-                    break;
-                }
-            }
-        }
+    fCSRecognizers = NEW_ARRAY(CSRecognizerInfo *, rCount);

 
 

-        if (U_SUCCESS(status)) {
-            umtx_lock(NULL);
-            if (fCSRecognizers == NULL) {
-                fCSRecognizers = recognizers;
-                fCSRecognizers_size = rCount;
+    if (fCSRecognizers == NULL) {
+        status = U_MEMORY_ALLOCATION_ERROR;
+    } 
+    else {
+        fCSRecognizers_size = rCount;
+        for (int32_t r = 0; r < rCount; r += 1) {
+            fCSRecognizers[r] = tempArray[r];
+            if (fCSRecognizers[r] == NULL) {
+                status = U_MEMORY_ALLOCATION_ERROR;

             }
             }

-            umtx_unlock(NULL);

         }
         }

+    }
+}

 
 

-        if (fCSRecognizers != recognizers) {
-            for (r = 0; r < rCount; r += 1) {
-                delete recognizers[r];
-                recognizers[r] = NULL;
-            }
+U_CDECL_END

 
 

-            DELETE_ARRAY(recognizers);
-        }
+U_NAMESPACE_BEGIN

 
 

-        recognizers = NULL;
-        ucln_i18n_registerCleanup(UCLN_I18N_CSDET, csdet_cleanup);
-    }
+void CharsetDetector::setRecognizers(UErrorCode &status)
+{
+    umtx_initOnce(gCSRecognizersInitOnce, &initRecognizers, status);

 }
 
 CharsetDetector::CharsetDetector(UErrorCode &status)
   : textIn(new InputText(status)), resultArray(NULL),
 }
 
 CharsetDetector::CharsetDetector(UErrorCode &status)
   : textIn(new InputText(status)), resultArray(NULL),

-    resultCount(0), fStripTags(FALSE), fFreshTextSet(FALSE)
+    resultCount(0), fStripTags(FALSE), fFreshTextSet(FALSE),
+    fEnabledRecognizers(NULL)

 {
     if (U_FAILURE(status)) {
         return;
 {
     if (U_FAILURE(status)) {
         return;


@@ -204,6 +188,10 @@ CharsetDetector::~CharsetDetector()
     }
 
     uprv_free(resultArray);
     }
 
     uprv_free(resultArray);

+
+    if (fEnabledRecognizers) {
+        uprv_free(fEnabledRecognizers);
+    }

 }
 
 void CharsetDetector::setText(const char *in, int32_t len)
 }
 
 void CharsetDetector::setText(const char *in, int32_t len)


@@ -258,10 +246,8 @@ const CharsetMatch * const *CharsetDetector::detectAll(int32_t &maxMatchesFound,
         status = U_MISSING_RESOURCE_ERROR;// TODO:  Need to set proper status code for input text not set
 
         return NULL;
         status = U_MISSING_RESOURCE_ERROR;// TODO:  Need to set proper status code for input text not set
 
         return NULL;

-    } else if(fFreshTextSet) {
+    } else if (fFreshTextSet) {

         CharsetRecognizer *csr;
         CharsetRecognizer *csr;

-        int32_t            detectResults;
-        int32_t            confidence;

         int32_t            i;
 
         textIn->MungeInput(fStripTags);
         int32_t            i;
 
         textIn->MungeInput(fStripTags);


@@ -270,31 +256,15 @@ const CharsetMatch * const *CharsetDetector::detectAll(int32_t &maxMatchesFound,
         // give a match quality > 0.
         resultCount = 0;
         for (i = 0; i < fCSRecognizers_size; i += 1) {
         // give a match quality > 0.
         resultCount = 0;
         for (i = 0; i < fCSRecognizers_size; i += 1) {

-            csr = fCSRecognizers[i];
-            detectResults = csr->match(textIn);
-            confidence = detectResults;
-
-            if (confidence > 0)  {
-                resultArray[resultCount++]->set(textIn, csr, confidence);
+            csr = fCSRecognizers[i]->recognizer;
+            if (csr->match(textIn, resultArray[resultCount])) {
+                resultCount++;

             }
         }
 
             }
         }
 

-        for(i = resultCount; i < fCSRecognizers_size; i += 1) {
-            resultArray[i]->set(textIn, 0, 0);
+        if (resultCount > 1) {
+            uprv_sortArray(resultArray, resultCount, sizeof resultArray[0], charsetMatchComparator, NULL, TRUE, &status);

         }
         }

-
-        uprv_sortArray(resultArray, resultCount, sizeof resultArray[0], charsetMatchComparator, NULL, TRUE, &status);
-        ////Bubble sort
-        //for(int32_t i = resultCount; i > 1; i -= 1) {
-        //    for(int32_t j = 0; j < i-1; j += 1) {
-        //        if(resultArray[j]->getConfidence() < resultArray[j+1]->getConfidence()) {
-        //            CharsetMatch *temp = resultArray[j];
-        //            resultArray[j] = resultArray[j+1];
-        //            resultArray[j+1] = temp;
-        //        }
-        //    }
-        //}
-

         fFreshTextSet = FALSE;
     }
 
         fFreshTextSet = FALSE;
     }
 


@@ -303,6 +273,46 @@ const CharsetMatch * const *CharsetDetector::detectAll(int32_t &maxMatchesFound,
     return resultArray;
 }
 
     return resultArray;
 }
 

+void CharsetDetector::setDetectableCharset(const char *encoding, UBool enabled, UErrorCode &status)
+{
+    if (U_FAILURE(status)) {
+        return;
+    }
+
+    int32_t modIdx = -1;
+    UBool isDefaultVal = FALSE;
+    for (int32_t i = 0; i < fCSRecognizers_size; i++) {
+        CSRecognizerInfo *csrinfo = fCSRecognizers[i];
+        if (uprv_strcmp(csrinfo->recognizer->getName(), encoding) == 0) {
+            modIdx = i;
+            isDefaultVal = (csrinfo->isDefaultEnabled == enabled);
+            break;
+        }
+    }
+    if (modIdx < 0) {
+        // No matching encoding found
+        status = U_ILLEGAL_ARGUMENT_ERROR;
+        return;
+    }
+
+    if (fEnabledRecognizers == NULL && !isDefaultVal) {
+        // Create an array storing the non default setting
+        fEnabledRecognizers = NEW_ARRAY(UBool, fCSRecognizers_size);
+        if (fEnabledRecognizers == NULL) {
+            status = U_MEMORY_ALLOCATION_ERROR;
+            return;
+        }
+        // Initialize the array with default info
+        for (int32_t i = 0; i < fCSRecognizers_size; i++) {
+            fEnabledRecognizers[i] = fCSRecognizers[i]->isDefaultEnabled;
+        }
+    }
+
+    if (fEnabledRecognizers != NULL) {
+        fEnabledRecognizers[modIdx] = enabled;
+    }
+}
+

 /*const char *CharsetDetector::getCharsetName(int32_t index, UErrorCode &status) const
 {
     if( index > fCSRecognizers_size-1 || index < 0) {
 /*const char *CharsetDetector::getCharsetName(int32_t index, UErrorCode &status) const
 {
     if( index > fCSRecognizers_size-1 || index < 0) {


@@ -319,6 +329,8 @@ U_NAMESPACE_END
 U_CDECL_BEGIN
 typedef struct {
     int32_t currIndex;
 U_CDECL_BEGIN
 typedef struct {
     int32_t currIndex;

+    UBool all;
+    UBool *enabledRecognizers;

 } Context;
 
 
 } Context;
 
 


@@ -333,27 +345,73 @@ enumClose(UEnumeration *en) {
 }
 
 static int32_t U_CALLCONV
 }
 
 static int32_t U_CALLCONV

-enumCount(UEnumeration *, UErrorCode *) {
-    return fCSRecognizers_size;
+enumCount(UEnumeration *en, UErrorCode *) {
+    if (((Context *)en->context)->all) {
+        // ucsdet_getAllDetectableCharsets, all charset detector names
+        return fCSRecognizers_size;
+    }
+
+    // Otherwise, ucsdet_getDetectableCharsets - only enabled ones
+    int32_t count = 0;
+    UBool *enabledArray = ((Context *)en->context)->enabledRecognizers;
+    if (enabledArray != NULL) {
+        // custom set
+        for (int32_t i = 0; i < fCSRecognizers_size; i++) {
+            if (enabledArray[i]) {
+                count++;
+            }
+        }
+    } else {
+        // default set
+        for (int32_t i = 0; i < fCSRecognizers_size; i++) {
+            if (fCSRecognizers[i]->isDefaultEnabled) {
+                count++;
+            }
+        }
+    }
+    return count;

 }
 
 static const char* U_CALLCONV
 enumNext(UEnumeration *en, int32_t *resultLength, UErrorCode * /*status*/) {
 }
 
 static const char* U_CALLCONV
 enumNext(UEnumeration *en, int32_t *resultLength, UErrorCode * /*status*/) {

-    if(((Context *)en->context)->currIndex >= fCSRecognizers_size) {
-        if(resultLength != NULL) {
-            *resultLength = 0;
+    const char *currName = NULL;
+
+    if (((Context *)en->context)->currIndex < fCSRecognizers_size) {
+        if (((Context *)en->context)->all) {
+            // ucsdet_getAllDetectableCharsets, all charset detector names
+            currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
+            ((Context *)en->context)->currIndex++;
+        } else {
+            // ucsdet_getDetectableCharsets
+            UBool *enabledArray = ((Context *)en->context)->enabledRecognizers;
+            if (enabledArray != NULL) {
+                // custome set
+                while (currName == NULL && ((Context *)en->context)->currIndex < fCSRecognizers_size) {
+                    if (enabledArray[((Context *)en->context)->currIndex]) {
+                        currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
+                    }
+                    ((Context *)en->context)->currIndex++;
+                }
+            } else {
+                // default set
+                while (currName == NULL && ((Context *)en->context)->currIndex < fCSRecognizers_size) {
+                    if (fCSRecognizers[((Context *)en->context)->currIndex]->isDefaultEnabled) {
+                        currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
+                    }
+                    ((Context *)en->context)->currIndex++;
+                }
+            }

         }
         }

-        return NULL;

     }
     }

-    const char *currName = fCSRecognizers[((Context *)en->context)->currIndex]->getName();
+

     if(resultLength != NULL) {
     if(resultLength != NULL) {

-        *resultLength = (int32_t)uprv_strlen(currName);
+        *resultLength = currName == NULL ? 0 : (int32_t)uprv_strlen(currName);

     }
     }

-    ((Context *)en->context)->currIndex++;

 
     return currName;
 }
 
 
     return currName;
 }
 

+

 static void U_CALLCONV
 enumReset(UEnumeration *en, UErrorCode *) {
     ((Context *)en->context)->currIndex = 0;
 static void U_CALLCONV
 enumReset(UEnumeration *en, UErrorCode *) {
     ((Context *)en->context)->currIndex = 0;


@@ -369,25 +427,61 @@ static const UEnumeration gCSDetEnumeration = {
     enumReset
 };
 
     enumReset
 };
 

-U_CAPI  UEnumeration * U_EXPORT2
-ucsdet_getAllDetectableCharsets(const UCharsetDetector *ucsd, UErrorCode *status)
+U_CDECL_END
+
+U_NAMESPACE_BEGIN
+
+UEnumeration * CharsetDetector::getAllDetectableCharsets(UErrorCode &status)

 {
 {

-    U_NAMESPACE_USE

 
 

-    if(U_FAILURE(*status)) {
+    /* Initialize recognized charsets. */
+    setRecognizers(status);
+
+    if(U_FAILURE(status)) {

         return 0;
     }
 
         return 0;
     }
 

-    /* Initialize recognized charsets. */
-    CharsetDetector::getDetectableCount();
+    UEnumeration *en = NEW_ARRAY(UEnumeration, 1);
+    if (en == NULL) {
+        status = U_MEMORY_ALLOCATION_ERROR;
+        return 0;
+    }
+    memcpy(en, &gCSDetEnumeration, sizeof(UEnumeration));
+    en->context = (void*)NEW_ARRAY(Context, 1);
+    if (en->context == NULL) {
+        status = U_MEMORY_ALLOCATION_ERROR;
+        DELETE_ARRAY(en);
+        return 0;
+    }
+    uprv_memset(en->context, 0, sizeof(Context));
+    ((Context*)en->context)->all = TRUE;
+    return en;
+}
+
+UEnumeration * CharsetDetector::getDetectableCharsets(UErrorCode &status) const
+{
+    if(U_FAILURE(status)) {
+        return 0;
+    }

 
     UEnumeration *en = NEW_ARRAY(UEnumeration, 1);
 
     UEnumeration *en = NEW_ARRAY(UEnumeration, 1);

+    if (en == NULL) {
+        status = U_MEMORY_ALLOCATION_ERROR;
+        return 0;
+    }

     memcpy(en, &gCSDetEnumeration, sizeof(UEnumeration));
     en->context = (void*)NEW_ARRAY(Context, 1);
     memcpy(en, &gCSDetEnumeration, sizeof(UEnumeration));
     en->context = (void*)NEW_ARRAY(Context, 1);

+    if (en->context == NULL) {
+        status = U_MEMORY_ALLOCATION_ERROR;
+        DELETE_ARRAY(en);
+        return 0;
+    }

     uprv_memset(en->context, 0, sizeof(Context));
     uprv_memset(en->context, 0, sizeof(Context));

+    ((Context*)en->context)->all = FALSE;
+    ((Context*)en->context)->enabledRecognizers = fEnabledRecognizers;

     return en;
 }
     return en;
 }

-U_CDECL_END

 
 

-#endif
+U_NAMESPACE_END

 
 

+#endif