icuSources/i18n/csrucode.cpp

   1 /*
   2  **********************************************************************
   3  *   Copyright (C) 2005-2012, International Business Machines
   4  *   Corporation and others.  All Rights Reserved.
   5  **********************************************************************
   6  */
   7
   8 #include "unicode/utypes.h"
   9
  10 #if !UCONFIG_NO_CONVERSION
  11
  12 #include "csrucode.h"
  13 #include "csmatch.h"
  14
  15 U_NAMESPACE_BEGIN
  16
  17 CharsetRecog_Unicode::~CharsetRecog_Unicode()
  18 {
  19     // nothing to do
  20 }
  21
  22 CharsetRecog_UTF_16_BE::~CharsetRecog_UTF_16_BE()
  23 {
  24     // nothing to do
  25 }
  26
  27 const char *CharsetRecog_UTF_16_BE::getName() const
  28 {
  29     return "UTF-16BE";
  30 }
  31
  32 UBool CharsetRecog_UTF_16_BE::match(InputText* textIn, CharsetMatch *results) const
  33 {
  34     const uint8_t *input = textIn->fRawInput;
  35     int32_t confidence = 0;
  36
  37     if (input[0] == 0xFE && input[1] == 0xFF) {
  38         confidence = 100;
  39     }
  40
  41     // TODO: Do some statastics to check for unsigned UTF-16BE
  42     results->set(textIn, this, confidence);
  43     return (confidence > 0);
  44 }
  45
  46 CharsetRecog_UTF_16_LE::~CharsetRecog_UTF_16_LE()
  47 {
  48     // nothing to do
  49 }
  50
  51 const char *CharsetRecog_UTF_16_LE::getName() const
  52 {
  53     return "UTF-16LE";
  54 }
  55
  56 UBool CharsetRecog_UTF_16_LE::match(InputText* textIn, CharsetMatch *results) const
  57 {
  58     const uint8_t *input = textIn->fRawInput;
  59     int32_t confidence = 0;
  60
  61     if (input[0] == 0xFF && input[1] == 0xFE && (input[2] != 0x00 || input[3] != 0x00)) {
  62         confidence = 100;
  63     }
  64
  65     // TODO: Do some statastics to check for unsigned UTF-16LE
  66     results->set(textIn, this, confidence);
  67     return (confidence > 0);
  68 }
  69
  70 CharsetRecog_UTF_32::~CharsetRecog_UTF_32()
  71 {
  72     // nothing to do
  73 }
  74
  75 UBool CharsetRecog_UTF_32::match(InputText* textIn, CharsetMatch *results) const
  76 {
  77     const uint8_t *input = textIn->fRawInput;
  78     int32_t limit = (textIn->fRawLength / 4) * 4;
  79     int32_t numValid = 0;
  80     int32_t numInvalid = 0;
  81     bool hasBOM = FALSE;
  82     int32_t confidence = 0;
  83
  84     if (getChar(input, 0) == 0x0000FEFFUL) {
  85         hasBOM = TRUE;
  86     }
  87
  88     for(int32_t i = 0; i < limit; i += 4) {
  89         int32_t ch = getChar(input, i);
  90
  91         if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) {
  92             numInvalid += 1;
  93         } else {
  94             numValid += 1;
  95         }
  96     }
  97
  98
  99     // Cook up some sort of confidence score, based on presense of a BOM
 100     //    and the existence of valid and/or invalid multi-byte sequences.
 101     if (hasBOM && numInvalid==0) {
 102         confidence = 100;
 103     } else if (hasBOM && numValid > numInvalid*10) {
 104         confidence = 80;
 105     } else if (numValid > 3 && numInvalid == 0) {
 106         confidence = 100;
 107     } else if (numValid > 0 && numInvalid == 0) {
 108         confidence = 80;
 109     } else if (numValid > numInvalid*10) {
 110         // Probably corruput UTF-32BE data.  Valid sequences aren't likely by chance.
 111         confidence = 25;
 112     }
 113
 114     results->set(textIn, this, confidence);
 115     return (confidence > 0);
 116 }
 117
 118 CharsetRecog_UTF_32_BE::~CharsetRecog_UTF_32_BE()
 119 {
 120     // nothing to do
 121 }
 122
 123 const char *CharsetRecog_UTF_32_BE::getName() const
 124 {
 125     return "UTF-32BE";
 126 }
 127
 128 int32_t CharsetRecog_UTF_32_BE::getChar(const uint8_t *input, int32_t index) const
 129 {
 130     return input[index + 0] << 24 | input[index + 1] << 16 |
 131            input[index + 2] <<  8 | input[index + 3];
 132 }
 133
 134 CharsetRecog_UTF_32_LE::~CharsetRecog_UTF_32_LE()
 135 {
 136     // nothing to do
 137 }
 138
 139 const char *CharsetRecog_UTF_32_LE::getName() const
 140 {
 141     return "UTF-32LE";
 142 }
 143
 144 int32_t CharsetRecog_UTF_32_LE::getChar(const uint8_t *input, int32_t index) const
 145 {
 146     return input[index + 3] << 24 | input[index + 2] << 16 |
 147            input[index + 1] <<  8 | input[index + 0];
 148 }
 149
 150 U_NAMESPACE_END
 151 #endif
 152