ICU-551.30.tar.gz

[apple/icu.git] / icuSources / i18n / csrucode.cpp

diff --git a/icuSources/i18n/csrucode.cpp b/icuSources/i18n/csrucode.cpp
index 99a76d850e668abb361c9b2a9fcf5597800cadf6..f09834306808107d482aa3dc8b8064a7eb9a5779 100644 (file)
--- a/icuSources/i18n/csrucode.cpp
+++ b/icuSources/i18n/csrucode.cpp
@@ -1,6 +1,6 @@
 /*
  **********************************************************************
 /*
  **********************************************************************

- *   Copyright (C) 2005-2006, International Business Machines
+ *   Copyright (C) 2005-2013, International Business Machines

  *   Corporation and others.  All Rights Reserved.
  **********************************************************************
  */
  *   Corporation and others.  All Rights Reserved.
  **********************************************************************
  */


@@ -10,6 +10,7 @@
 #if !UCONFIG_NO_CONVERSION
 
 #include "csrucode.h"
 #if !UCONFIG_NO_CONVERSION
 
 #include "csrucode.h"

+#include "csmatch.h"

 
 U_NAMESPACE_BEGIN
 
 
 U_NAMESPACE_BEGIN
 


@@ -28,16 +29,50 @@ const char *CharsetRecog_UTF_16_BE::getName() const
     return "UTF-16BE";
 }
 
     return "UTF-16BE";
 }
 

-int32_t CharsetRecog_UTF_16_BE::match(InputText* textIn)
+// UTF-16 confidence calculation. Very simple minded, but better than nothing.
+//   Any 8 bit non-control characters bump the confidence up. These have a zero high byte,
+//     and are very likely to be UTF-16, although they could also be part of a UTF-32 code.
+//   NULs are a contra-indication, they will appear commonly if the actual encoding is UTF-32.
+//   NULs should be rare in actual text. 
+
+static int32_t adjustConfidence(UChar codeUnit, int32_t confidence) {
+    if (codeUnit == 0) {
+        confidence -= 10;
+    } else if ((codeUnit >= 0x20 && codeUnit <= 0xff) || codeUnit == 0x0a) {
+        confidence += 10;
+    }
+    if (confidence < 0) {
+        confidence = 0;
+    } else if (confidence > 100) {
+        confidence = 100;
+    }
+    return confidence;
+}
+
+
+UBool CharsetRecog_UTF_16_BE::match(InputText* textIn, CharsetMatch *results) const

 {
     const uint8_t *input = textIn->fRawInput;
 {
     const uint8_t *input = textIn->fRawInput;

-
-    if (input[0] == 0xFE && input[1] == 0xFF) {
-        return 100;
+    int32_t confidence = 10;
+    int32_t length = textIn->fRawLength;
+
+    int32_t bytesToCheck = (length > 30) ? 30 : length;
+    for (int32_t charIndex=0; charIndex<bytesToCheck-1; charIndex+=2) {
+        UChar codeUnit = (input[charIndex] << 8) | input[charIndex + 1];
+        if (charIndex == 0 && codeUnit == 0xFEFF) {
+            confidence = 100;
+            break;
+        }
+        confidence = adjustConfidence(codeUnit, confidence);
+        if (confidence == 0 || confidence == 100) {
+            break;
+        }

     }
     }

-
-    // TODO: Do some statastics to check for unsigned UTF-16BE
-    return 0;
+    if (bytesToCheck < 4 && confidence < 100) {
+        confidence = 0;
+    }
+    results->set(textIn, this, confidence);
+    return (confidence > 0);

 }
 
 CharsetRecog_UTF_16_LE::~CharsetRecog_UTF_16_LE()
 }
 
 CharsetRecog_UTF_16_LE::~CharsetRecog_UTF_16_LE()


@@ -50,16 +85,32 @@ const char *CharsetRecog_UTF_16_LE::getName() const
     return "UTF-16LE";
 }
 
     return "UTF-16LE";
 }
 

-int32_t CharsetRecog_UTF_16_LE::match(InputText* textIn)
+UBool CharsetRecog_UTF_16_LE::match(InputText* textIn, CharsetMatch *results) const

 {
     const uint8_t *input = textIn->fRawInput;
 {
     const uint8_t *input = textIn->fRawInput;

-
-    if (input[0] == 0xFF && input[1] == 0xFE && (input[2] != 0x00 || input[3] != 0x00)) {
-        return 100;
+    int32_t confidence = 10;
+    int32_t length = textIn->fRawLength;
+
+    int32_t bytesToCheck = (length > 30) ? 30 : length;
+    for (int32_t charIndex=0; charIndex<bytesToCheck-1; charIndex+=2) {
+        UChar codeUnit = input[charIndex] | (input[charIndex + 1] << 8);
+        if (charIndex == 0 && codeUnit == 0xFEFF) {
+            confidence = 100;     // UTF-16 BOM
+            if (length >= 4 && input[2] == 0 && input[3] == 0) {
+                confidence = 0;   // UTF-32 BOM
+            }
+            break;
+        }
+        confidence = adjustConfidence(codeUnit, confidence);
+        if (confidence == 0 || confidence == 100) {
+            break;
+        }

     }
     }

-
-    // TODO: Do some statastics to check for unsigned UTF-16LE
-    return 0;
+    if (bytesToCheck < 4 && confidence < 100) {
+        confidence = 0;
+    }
+    results->set(textIn, this, confidence);
+    return (confidence > 0);

 }
 
 CharsetRecog_UTF_32::~CharsetRecog_UTF_32()
 }
 
 CharsetRecog_UTF_32::~CharsetRecog_UTF_32()


@@ -67,7 +118,7 @@ CharsetRecog_UTF_32::~CharsetRecog_UTF_32()
     // nothing to do
 }
 
     // nothing to do
 }
 

-int32_t CharsetRecog_UTF_32::match(InputText* textIn)
+UBool CharsetRecog_UTF_32::match(InputText* textIn, CharsetMatch *results) const

 {
     const uint8_t *input = textIn->fRawInput;
     int32_t limit = (textIn->fRawLength / 4) * 4;
 {
     const uint8_t *input = textIn->fRawInput;
     int32_t limit = (textIn->fRawLength / 4) * 4;


@@ -76,7 +127,7 @@ int32_t CharsetRecog_UTF_32::match(InputText* textIn)
     bool hasBOM = FALSE;
     int32_t confidence = 0;
 
     bool hasBOM = FALSE;
     int32_t confidence = 0;
 

-    if (getChar(input, 0) == 0x0000FEFFUL) {
+    if (limit > 0 && getChar(input, 0) == 0x0000FEFFUL) {

         hasBOM = TRUE;
     }
 
         hasBOM = TRUE;
     }
 


@@ -106,7 +157,8 @@ int32_t CharsetRecog_UTF_32::match(InputText* textIn)
         confidence = 25;
     }
 
         confidence = 25;
     }
 

-    return confidence;
+    results->set(textIn, this, confidence);
+    return (confidence > 0);

 }
 
 CharsetRecog_UTF_32_BE::~CharsetRecog_UTF_32_BE()
 }
 
 CharsetRecog_UTF_32_BE::~CharsetRecog_UTF_32_BE()