icuSources/i18n/csdetect.cpp

   1 /*
   2  **********************************************************************
   3  *   Copyright (C) 2005-2013, International Business Machines
   4  *   Corporation and others.  All Rights Reserved.
   5  **********************************************************************
   6  */
   7
   8 #include "unicode/utypes.h"
   9
  10 #if !UCONFIG_NO_CONVERSION
  11
  12 #include "unicode/ucsdet.h"
  13
  14 #include "csdetect.h"
  15 #include "csmatch.h"
  16 #include "uenumimp.h"
  17
  18 #include "cmemory.h"
  19 #include "cstring.h"
  20 #include "umutex.h"
  21 #include "ucln_in.h"
  22 #include "uarrsort.h"
  23 #include "inputext.h"
  24 #include "csrsbcs.h"
  25 #include "csrmbcs.h"
  26 #include "csrutf8.h"
  27 #include "csrucode.h"
  28 #include "csr2022.h"
  29
  30 #define ARRAY_SIZE(array) (sizeof array / sizeof array[0])
  31
  32 #define NEW_ARRAY(type,count) (type *) uprv_malloc((count) * sizeof(type))
  33 #define DELETE_ARRAY(array) uprv_free((void *) (array))
  34
  35 U_NAMESPACE_BEGIN
  36
  37 struct CSRecognizerInfo : public UMemory {
  38     CSRecognizerInfo(CharsetRecognizer *recognizer, UBool isDefaultEnabled)
  39         : recognizer(recognizer), isDefaultEnabled(isDefaultEnabled) {};
  40
  41     ~CSRecognizerInfo() {delete recognizer;};
  42
  43     CharsetRecognizer *recognizer;
  44     UBool isDefaultEnabled;
  45 };
  46
  47 U_NAMESPACE_END
  48
  49 static icu::CSRecognizerInfo **fCSRecognizers = NULL;
  50 static icu::UInitOnce gCSRecognizersInitOnce;
  51 static int32_t fCSRecognizers_size = 0;
  52
  53 U_CDECL_BEGIN
  54 static UBool U_CALLCONV csdet_cleanup(void)
  55 {
  56     U_NAMESPACE_USE
  57     if (fCSRecognizers != NULL) {
  58         for(int32_t r = 0; r < fCSRecognizers_size; r += 1) {
  59             delete fCSRecognizers[r];
  60             fCSRecognizers[r] = NULL;
  61         }
  62
  63         DELETE_ARRAY(fCSRecognizers);
  64         fCSRecognizers = NULL;
  65         fCSRecognizers_size = 0;
  66     }
  67     gCSRecognizersInitOnce.reset();
  68
  69     return TRUE;
  70 }
  71
  72 static int32_t U_CALLCONV
  73 charsetMatchComparator(const void * /*context*/, const void *left, const void *right)
  74 {
  75     U_NAMESPACE_USE
  76
  77     const CharsetMatch **csm_l = (const CharsetMatch **) left;
  78     const CharsetMatch **csm_r = (const CharsetMatch **) right;
  79
  80     // NOTE: compare is backwards to sort from highest to lowest.
  81     return (*csm_r)->getConfidence() - (*csm_l)->getConfidence();
  82 }
  83
  84 static void U_CALLCONV initRecognizers(UErrorCode &status) {
  85     U_NAMESPACE_USE
  86     ucln_i18n_registerCleanup(UCLN_I18N_CSDET, csdet_cleanup);
  87     CSRecognizerInfo *tempArray[] = {
  88         new CSRecognizerInfo(new CharsetRecog_UTF8(), TRUE),
  89
  90         new CSRecognizerInfo(new CharsetRecog_UTF_16_BE(), TRUE),
  91         new CSRecognizerInfo(new CharsetRecog_UTF_16_LE(), TRUE),
  92         new CSRecognizerInfo(new CharsetRecog_UTF_32_BE(), TRUE),
  93         new CSRecognizerInfo(new CharsetRecog_UTF_32_LE(), TRUE),
  94
  95         new CSRecognizerInfo(new CharsetRecog_8859_1(), TRUE),
  96         new CSRecognizerInfo(new CharsetRecog_8859_2(), TRUE),
  97         new CSRecognizerInfo(new CharsetRecog_8859_5_ru(), TRUE),
  98         new CSRecognizerInfo(new CharsetRecog_8859_6_ar(), TRUE),
  99         new CSRecognizerInfo(new CharsetRecog_8859_7_el(), TRUE),
 100         new CSRecognizerInfo(new CharsetRecog_8859_8_I_he(), TRUE),
 101         new CSRecognizerInfo(new CharsetRecog_8859_8_he(), TRUE),
 102         new CSRecognizerInfo(new CharsetRecog_windows_1251(), TRUE),
 103         new CSRecognizerInfo(new CharsetRecog_windows_1256(), TRUE),
 104         new CSRecognizerInfo(new CharsetRecog_KOI8_R(), TRUE),
 105         new CSRecognizerInfo(new CharsetRecog_8859_9_tr(), TRUE),
 106         new CSRecognizerInfo(new CharsetRecog_sjis(), TRUE),
 107         new CSRecognizerInfo(new CharsetRecog_gb_18030(), TRUE),
 108         new CSRecognizerInfo(new CharsetRecog_euc_jp(), TRUE),
 109         new CSRecognizerInfo(new CharsetRecog_euc_kr(), TRUE),
 110         new CSRecognizerInfo(new CharsetRecog_big5(), TRUE),
 111
 112         new CSRecognizerInfo(new CharsetRecog_2022JP(), TRUE),
 113         new CSRecognizerInfo(new CharsetRecog_2022KR(), TRUE),
 114         new CSRecognizerInfo(new CharsetRecog_2022CN(), TRUE),
 115
 116         new CSRecognizerInfo(new CharsetRecog_IBM424_he_rtl(), FALSE),
 117         new CSRecognizerInfo(new CharsetRecog_IBM424_he_ltr(), FALSE),
 118         new CSRecognizerInfo(new CharsetRecog_IBM420_ar_rtl(), FALSE),
 119         new CSRecognizerInfo(new CharsetRecog_IBM420_ar_ltr(), FALSE)
 120     };
 121     int32_t rCount = ARRAY_SIZE(tempArray);
 122
 123     fCSRecognizers = NEW_ARRAY(CSRecognizerInfo *, rCount);
 124
 125     if (fCSRecognizers == NULL) {
 126         status = U_MEMORY_ALLOCATION_ERROR;
 127     }
 128     else {
 129         fCSRecognizers_size = rCount;
 130         for (int32_t r = 0; r < rCount; r += 1) {
 131             fCSRecognizers[r] = tempArray[r];
 132             if (fCSRecognizers[r] == NULL) {
 133                 status = U_MEMORY_ALLOCATION_ERROR;
 134             }
 135         }
 136     }
 137 }
 138
 139 U_CDECL_END
 140
 141 U_NAMESPACE_BEGIN
 142
 143 void CharsetDetector::setRecognizers(UErrorCode &status)
 144 {
 145     umtx_initOnce(gCSRecognizersInitOnce, &initRecognizers, status);
 146 }
 147
 148 CharsetDetector::CharsetDetector(UErrorCode &status)
 149   : textIn(new InputText(status)), resultArray(NULL),
 150     resultCount(0), fStripTags(FALSE), fFreshTextSet(FALSE),
 151     fEnabledRecognizers(NULL)
 152 {
 153     if (U_FAILURE(status)) {
 154         return;
 155     }
 156
 157     setRecognizers(status);
 158
 159     if (U_FAILURE(status)) {
 160         return;
 161     }
 162
 163     resultArray = (CharsetMatch **)uprv_malloc(sizeof(CharsetMatch *)*fCSRecognizers_size);
 164
 165     if (resultArray == NULL) {
 166         status = U_MEMORY_ALLOCATION_ERROR;
 167         return;
 168     }
 169
 170     for(int32_t i = 0; i < fCSRecognizers_size; i += 1) {
 171         resultArray[i] = new CharsetMatch();
 172
 173         if (resultArray[i] == NULL) {
 174             status = U_MEMORY_ALLOCATION_ERROR;
 175             break;
 176         }
 177     }
 178 }
 179
 180 CharsetDetector::~CharsetDetector()
 181 {
 182     delete textIn;
 183
 184     for(int32_t i = 0; i < fCSRecognizers_size; i += 1) {
 185         delete resultArray[i];
 186     }
 187
 188     uprv_free(resultArray);
 189
 190     if (fEnabledRecognizers) {
 191         uprv_free(fEnabledRecognizers);
 192     }
 193 }
 194
 195 void CharsetDetector::setText(const char *in, int32_t len)
 196 {
 197     textIn->setText(in, len);
 198     fFreshTextSet = TRUE;
 199 }
 200
 201 UBool CharsetDetector::setStripTagsFlag(UBool flag)
 202 {
 203     UBool temp = fStripTags;
 204     fStripTags = flag;
 205     fFreshTextSet = TRUE;
 206     return temp;
 207 }
 208
 209 UBool CharsetDetector::getStripTagsFlag() const
 210 {
 211     return fStripTags;
 212 }
 213
 214 void CharsetDetector::setDeclaredEncoding(const char *encoding, int32_t len) const
 215 {
 216     textIn->setDeclaredEncoding(encoding,len);
 217 }
 218
 219 int32_t CharsetDetector::getDetectableCount()
 220 {
 221     UErrorCode status = U_ZERO_ERROR;
 222
 223     setRecognizers(status);
 224
 225     return fCSRecognizers_size;
 226 }
 227
 228 const CharsetMatch *CharsetDetector::detect(UErrorCode &status)
 229 {
 230     int32_t maxMatchesFound = 0;
 231
 232     detectAll(maxMatchesFound, status);
 233
 234     if(maxMatchesFound > 0) {
 235         return resultArray[0];
 236     } else {
 237         return NULL;
 238     }
 239 }
 240
 241 const CharsetMatch * const *CharsetDetector::detectAll(int32_t &maxMatchesFound, UErrorCode &status)
 242 {
 243     if(!textIn->isSet()) {
 244         status = U_MISSING_RESOURCE_ERROR;// TODO:  Need to set proper status code for input text not set
 245
 246         return NULL;
 247     } else if (fFreshTextSet) {
 248         CharsetRecognizer *csr;
 249         int32_t            i;
 250
 251         textIn->MungeInput(fStripTags);
 252
 253         // Iterate over all possible charsets, remember all that
 254         // give a match quality > 0.
 255         resultCount = 0;
 256         for (i = 0; i < fCSRecognizers_size; i += 1) {
 257             csr = fCSRecognizers[i]->recognizer;
 258             if (csr->match(textIn, resultArray[resultCount])) {
 259                 resultCount++;
 260             }
 261         }
 262
 263         if (resultCount > 1) {
 264             uprv_sortArray(resultArray, resultCount, sizeof resultArray[0], charsetMatchComparator, NULL, TRUE, &status);
 265         }
 266         fFreshTextSet = FALSE;
 267     }
 268
 269     maxMatchesFound = resultCount;
 270
 271     return resultArray;
 272 }
 273
 274 void CharsetDetector::setDetectableCharset(const char *encoding, UBool enabled, UErrorCode &status)
 275 {
 276     if (U_FAILURE(status)) {
 277         return;
 278     }
 279
 280     int32_t modIdx = -1;
 281     UBool isDefaultVal = FALSE;
 282     for (int32_t i = 0; i < fCSRecognizers_size; i++) {
 283         CSRecognizerInfo *csrinfo = fCSRecognizers[i];
 284         if (uprv_strcmp(csrinfo->recognizer->getName(), encoding) == 0) {
 285             modIdx = i;
 286             isDefaultVal = (csrinfo->isDefaultEnabled == enabled);
 287             break;
 288         }
 289     }
 290     if (modIdx < 0) {
 291         // No matching encoding found
 292         status = U_ILLEGAL_ARGUMENT_ERROR;
 293         return;
 294     }
 295
 296     if (fEnabledRecognizers == NULL && !isDefaultVal) {
 297         // Create an array storing the non default setting
 298         fEnabledRecognizers = NEW_ARRAY(UBool, fCSRecognizers_size);
 299         if (fEnabledRecognizers == NULL) {
 300             status = U_MEMORY_ALLOCATION_ERROR;
 301             return;
 302         }
 303         // Initialize the array with default info
 304         for (int32_t i = 0; i < fCSRecognizers_size; i++) {
 305             fEnabledRecognizers[i] = fCSRecognizers[i]->isDefaultEnabled;
 306         }
 307     }
 308
 309     if (fEnabledRecognizers != NULL) {
 310         fEnabledRecognizers[modIdx] = enabled;
 311     }
 312 }
 313
 314 /*const char *CharsetDetector::getCharsetName(int32_t index, UErrorCode &status) const
 315 {
 316     if( index > fCSRecognizers_size-1 || index < 0) {
 317         status = U_INDEX_OUTOFBOUNDS_ERROR;
 318
 319         return 0;
 320     } else {
 321         return fCSRecognizers[index]->getName();
 322     }
 323 }*/
 324
 325 U_NAMESPACE_END
 326
 327 U_CDECL_BEGIN
 328 typedef struct {
 329     int32_t currIndex;
 330     UBool all;
 331     UBool *enabledRecognizers;
 332 } Context;
 333
 334
 335
 336 static void U_CALLCONV
 337 enumClose(UEnumeration *en) {
 338     if(en->context != NULL) {
 339         DELETE_ARRAY(en->context);
 340     }
 341
 342     DELETE_ARRAY(en);
 343 }
 344
 345 static int32_t U_CALLCONV
 346 enumCount(UEnumeration *en, UErrorCode *) {
 347     if (((Context *)en->context)->all) {
 348         // ucsdet_getAllDetectableCharsets, all charset detector names
 349         return fCSRecognizers_size;
 350     }
 351
 352     // Otherwise, ucsdet_getDetectableCharsets - only enabled ones
 353     int32_t count = 0;
 354     UBool *enabledArray = ((Context *)en->context)->enabledRecognizers;
 355     if (enabledArray != NULL) {
 356         // custom set
 357         for (int32_t i = 0; i < fCSRecognizers_size; i++) {
 358             if (enabledArray[i]) {
 359                 count++;
 360             }
 361         }
 362     } else {
 363         // default set
 364         for (int32_t i = 0; i < fCSRecognizers_size; i++) {
 365             if (fCSRecognizers[i]->isDefaultEnabled) {
 366                 count++;
 367             }
 368         }
 369     }
 370     return count;
 371 }
 372
 373 static const char* U_CALLCONV
 374 enumNext(UEnumeration *en, int32_t *resultLength, UErrorCode * /*status*/) {
 375     const char *currName = NULL;
 376
 377     if (((Context *)en->context)->currIndex < fCSRecognizers_size) {
 378         if (((Context *)en->context)->all) {
 379             // ucsdet_getAllDetectableCharsets, all charset detector names
 380             currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
 381             ((Context *)en->context)->currIndex++;
 382         } else {
 383             // ucsdet_getDetectableCharsets
 384             UBool *enabledArray = ((Context *)en->context)->enabledRecognizers;
 385             if (enabledArray != NULL) {
 386                 // custome set
 387                 while (currName == NULL && ((Context *)en->context)->currIndex < fCSRecognizers_size) {
 388                     if (enabledArray[((Context *)en->context)->currIndex]) {
 389                         currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
 390                     }
 391                     ((Context *)en->context)->currIndex++;
 392                 }
 393             } else {
 394                 // default set
 395                 while (currName == NULL && ((Context *)en->context)->currIndex < fCSRecognizers_size) {
 396                     if (fCSRecognizers[((Context *)en->context)->currIndex]->isDefaultEnabled) {
 397                         currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
 398                     }
 399                     ((Context *)en->context)->currIndex++;
 400                 }
 401             }
 402         }
 403     }
 404
 405     if(resultLength != NULL) {
 406         *resultLength = currName == NULL ? 0 : (int32_t)uprv_strlen(currName);
 407     }
 408
 409     return currName;
 410 }
 411
 412
 413 static void U_CALLCONV
 414 enumReset(UEnumeration *en, UErrorCode *) {
 415     ((Context *)en->context)->currIndex = 0;
 416 }
 417
 418 static const UEnumeration gCSDetEnumeration = {
 419     NULL,
 420     NULL,
 421     enumClose,
 422     enumCount,
 423     uenum_unextDefault,
 424     enumNext,
 425     enumReset
 426 };
 427
 428 U_CDECL_END
 429
 430 U_NAMESPACE_BEGIN
 431
 432 UEnumeration * CharsetDetector::getAllDetectableCharsets(UErrorCode &status)
 433 {
 434
 435     /* Initialize recognized charsets. */
 436     setRecognizers(status);
 437
 438     if(U_FAILURE(status)) {
 439         return 0;
 440     }
 441
 442     UEnumeration *en = NEW_ARRAY(UEnumeration, 1);
 443     if (en == NULL) {
 444         status = U_MEMORY_ALLOCATION_ERROR;
 445         return 0;
 446     }
 447     memcpy(en, &gCSDetEnumeration, sizeof(UEnumeration));
 448     en->context = (void*)NEW_ARRAY(Context, 1);
 449     if (en->context == NULL) {
 450         status = U_MEMORY_ALLOCATION_ERROR;
 451         DELETE_ARRAY(en);
 452         return 0;
 453     }
 454     uprv_memset(en->context, 0, sizeof(Context));
 455     ((Context*)en->context)->all = TRUE;
 456     return en;
 457 }
 458
 459 UEnumeration * CharsetDetector::getDetectableCharsets(UErrorCode &status) const
 460 {
 461     if(U_FAILURE(status)) {
 462         return 0;
 463     }
 464
 465     UEnumeration *en = NEW_ARRAY(UEnumeration, 1);
 466     if (en == NULL) {
 467         status = U_MEMORY_ALLOCATION_ERROR;
 468         return 0;
 469     }
 470     memcpy(en, &gCSDetEnumeration, sizeof(UEnumeration));
 471     en->context = (void*)NEW_ARRAY(Context, 1);
 472     if (en->context == NULL) {
 473         status = U_MEMORY_ALLOCATION_ERROR;
 474         DELETE_ARRAY(en);
 475         return 0;
 476     }
 477     uprv_memset(en->context, 0, sizeof(Context));
 478     ((Context*)en->context)->all = FALSE;
 479     ((Context*)en->context)->enabledRecognizers = fEnabledRecognizers;
 480     return en;
 481 }
 482
 483 U_NAMESPACE_END
 484
 485 #endif