icuSources/i18n/csrsbcs.cpp

   1 // © 2016 and later: Unicode, Inc. and others.
   2 // License & terms of use: http://www.unicode.org/copyright.html
   3 /*
   4  **********************************************************************
   5  *   Copyright (C) 2005-2016, International Business Machines
   6  *   Corporation and others.  All Rights Reserved.
   7  **********************************************************************
   8  */
   9
  10 #include "unicode/utypes.h"
  11
  12 #include "cmemory.h"
  13
  14 #if !UCONFIG_NO_CONVERSION
  15 #include "csrsbcs.h"
  16 #include "csmatch.h"
  17
  18 #define N_GRAM_SIZE 3
  19 #define N_GRAM_MASK 0xFFFFFF
  20
  21 U_NAMESPACE_BEGIN
  22
  23 NGramParser::NGramParser(const int32_t *theNgramList, const uint8_t *theCharMap)
  24  : ngram(0), byteIndex(0)
  25 {
  26     ngramList = theNgramList;
  27     charMap   = theCharMap;
  28
  29     ngramCount = hitCount = 0;
  30 }
  31
  32 NGramParser::~NGramParser()
  33 {
  34 }
  35
  36 /*
  37  * Binary search for value in table, which must have exactly 64 entries.
  38  */
  39
  40 int32_t NGramParser::search(const int32_t *table, int32_t value)
  41 {
  42     int32_t index = 0;
  43
  44     if (table[index + 32] <= value) {
  45         index += 32;
  46     }
  47
  48     if (table[index + 16] <= value) {
  49         index += 16;
  50     }
  51
  52     if (table[index + 8] <= value) {
  53         index += 8;
  54     }
  55
  56     if (table[index + 4] <= value) {
  57         index += 4;
  58     }
  59
  60     if (table[index + 2] <= value) {
  61         index += 2;
  62     }
  63
  64     if (table[index + 1] <= value) {
  65         index += 1;
  66     }
  67
  68     if (table[index] > value) {
  69         index -= 1;
  70     }
  71
  72     if (index < 0 || table[index] != value) {
  73         return -1;
  74     }
  75
  76     return index;
  77 }
  78
  79 void NGramParser::lookup(int32_t thisNgram)
  80 {
  81     ngramCount += 1;
  82
  83     if (search(ngramList, thisNgram) >= 0) {
  84         hitCount += 1;
  85     }
  86
  87 }
  88
  89 void NGramParser::addByte(int32_t b)
  90 {
  91     ngram = ((ngram << 8) + b) & N_GRAM_MASK;
  92     lookup(ngram);
  93 }
  94
  95 int32_t NGramParser::nextByte(InputText *det)
  96 {
  97     if (byteIndex >= det->fInputLen) {
  98         return -1;
  99     }
 100
 101     return det->fInputBytes[byteIndex++];
 102 }
 103
 104 void NGramParser::parseCharacters(InputText *det)
 105 {
 106     int32_t b;
 107     bool ignoreSpace = FALSE;
 108
 109     while ((b = nextByte(det)) >= 0) {
 110         uint8_t mb = charMap[b];
 111
 112         // TODO: 0x20 might not be a space in all character sets...
 113         if (mb != 0) {
 114             if (!(mb == 0x20 && ignoreSpace)) {
 115                 addByte(mb);
 116             }
 117
 118             ignoreSpace = (mb == 0x20);
 119         }
 120     }
 121 }
 122
 123 int32_t NGramParser::parse(InputText *det)
 124 {
 125     parseCharacters(det);
 126
 127     // TODO: Is this OK? The buffer could have ended in the middle of a word...
 128     addByte(0x20);
 129
 130     double rawPercent = (double) hitCount / (double) ngramCount;
 131
 132     //            if (rawPercent <= 2.0) {
 133     //                return 0;
 134     //            }
 135
 136     // TODO - This is a bit of a hack to take care of a case
 137     // were we were getting a confidence of 135...
 138     if (rawPercent > 0.33) {
 139         return 98;
 140     }
 141
 142     return (int32_t) (rawPercent * 300.0);
 143 }
 144
 145 #if !UCONFIG_ONLY_HTML_CONVERSION
 146 static const uint8_t unshapeMap_IBM420[] = {
 147 /*           -0    -1    -2    -3    -4    -5    -6    -7    -8    -9    -A    -B    -C    -D    -E    -F   */
 148 /* 0- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 149 /* 1- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 150 /* 2- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 151 /* 3- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 152 /* 4- */    0x40, 0x40, 0x42, 0x42, 0x44, 0x45, 0x46, 0x47, 0x47, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F,
 153 /* 5- */    0x50, 0x49, 0x52, 0x53, 0x54, 0x55, 0x56, 0x56, 0x58, 0x58, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F,
 154 /* 6- */    0x60, 0x61, 0x62, 0x63, 0x63, 0x65, 0x65, 0x67, 0x67, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 155 /* 7- */    0x69, 0x71, 0x71, 0x73, 0x74, 0x75, 0x76, 0x77, 0x77, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F,
 156 /* 8- */    0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x80, 0x8B, 0x8B, 0x8D, 0x8D, 0x8F,
 157 /* 9- */    0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9A, 0x9A, 0x9A, 0x9E, 0x9E,
 158 /* A- */    0x9E, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x9E, 0xAB, 0xAB, 0xAD, 0xAD, 0xAF,
 159 /* B- */    0xAF, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xB1, 0xBB, 0xBB, 0xBD, 0xBD, 0xBF,
 160 /* C- */    0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xBF, 0xCC, 0xBF, 0xCE, 0xCF,
 161 /* D- */    0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xDA, 0xDC, 0xDC, 0xDC, 0xDF,
 162 /* E- */    0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 163 /* F- */    0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
 164 };
 165
 166 NGramParser_IBM420::NGramParser_IBM420(const int32_t *theNgramList, const uint8_t *theCharMap):NGramParser(theNgramList, theCharMap)
 167 {
 168         alef = 0x00;
 169 }
 170
 171 NGramParser_IBM420::~NGramParser_IBM420() {}
 172
 173 int32_t NGramParser_IBM420::isLamAlef(int32_t b)
 174 {
 175         if(b == 0xB2 || b == 0xB3){
 176                 return 0x47;
 177         }else if(b == 0xB4 || b == 0xB5){
 178                 return 0x49;
 179         }else if(b == 0xB8 || b == 0xB9){
 180                 return 0x56;
 181         }else
 182                 return 0x00;
 183 }
 184
 185 /*
 186 * Arabic shaping needs to be done manually. Cannot call ArabicShaping class
 187 * because CharsetDetector is dealing with bytes not Unicode code points. We could
 188 * convert the bytes to Unicode code points but that would leave us dependent
 189 * on CharsetICU which we try to avoid. IBM420 converter amongst different versions
 190 * of JDK can produce different results and therefore is also avoided.
 191 */
 192 int32_t NGramParser_IBM420::nextByte(InputText *det)
 193 {
 194
 195     if (byteIndex >= det->fInputLen || det->fInputBytes[byteIndex] == 0) {
 196         return -1;
 197     }
 198     int next;
 199
 200     alef = isLamAlef(det->fInputBytes[byteIndex]);
 201     if(alef != 0x00)
 202         next = 0xB1 & 0xFF;
 203     else
 204         next = unshapeMap_IBM420[det->fInputBytes[byteIndex]& 0xFF] & 0xFF;
 205
 206     byteIndex++;
 207
 208     return next;
 209 }
 210
 211 void NGramParser_IBM420::parseCharacters(InputText *det)
 212 {
 213         int32_t b;
 214     bool ignoreSpace = FALSE;
 215
 216     while ((b = nextByte(det)) >= 0) {
 217         uint8_t mb = charMap[b];
 218
 219         // TODO: 0x20 might not be a space in all character sets...
 220         if (mb != 0) {
 221             if (!(mb == 0x20 && ignoreSpace)) {
 222                 addByte(mb);
 223             }
 224             ignoreSpace = (mb == 0x20);
 225         }
 226
 227                 if(alef != 0x00){
 228             mb = charMap[alef & 0xFF];
 229
 230             // TODO: 0x20 might not be a space in all character sets...
 231             if (mb != 0) {
 232                 if (!(mb == 0x20 && ignoreSpace)) {
 233                     addByte(mb);
 234                 }
 235
 236                 ignoreSpace = (mb == 0x20);
 237             }
 238
 239         }
 240     }
 241 }
 242 #endif
 243
 244 CharsetRecog_sbcs::CharsetRecog_sbcs()
 245 {
 246     // nothing else to do
 247 }
 248
 249 CharsetRecog_sbcs::~CharsetRecog_sbcs()
 250 {
 251     // nothing to do
 252 }
 253
 254 int32_t CharsetRecog_sbcs::match_sbcs(InputText *det, const int32_t ngrams[],  const uint8_t byteMap[]) const
 255 {
 256     NGramParser parser(ngrams, byteMap);
 257     int32_t result;
 258
 259     result = parser.parse(det);
 260
 261     return result;
 262 }
 263
 264 static const uint8_t charMap_8859_1[] = {
 265     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 266     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 267     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 268     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 269     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 270     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 271     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 272     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 273     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 274     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 275     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 276     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 277     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 278     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 279     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 280     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 281     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 282     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 283     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 284     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 285     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 286     0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20,
 287     0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
 288     0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20,
 289     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 290     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 291     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
 292     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF,
 293     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 294     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 295     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
 296     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
 297 };
 298
 299 static const uint8_t charMap_8859_2[] = {
 300     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 301     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 302     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 303     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 304     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 305     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 306     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 307     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 308     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 309     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 310     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 311     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 312     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 313     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 314     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 315     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 316     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 317     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 318     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 319     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 320     0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0x20,
 321     0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF,
 322     0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0xB7,
 323     0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF,
 324     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 325     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 326     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
 327     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF,
 328     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 329     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 330     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
 331     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20,
 332 };
 333
 334 static const uint8_t charMap_8859_5[] = {
 335     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 336     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 337     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 338     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 339     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 340     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 341     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 342     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 343     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 344     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 345     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 346     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 347     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 348     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 349     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 350     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 351     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 352     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 353     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 354     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 355     0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
 356     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF,
 357     0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
 358     0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
 359     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 360     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 361     0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
 362     0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
 363     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 364     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 365     0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
 366     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF,
 367 };
 368
 369 static const uint8_t charMap_8859_6[] = {
 370     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 371     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 372     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 373     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 374     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 375     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 376     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 377     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 378     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 379     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 380     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 381     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 382     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 383     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 384     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 385     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 386     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 387     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 388     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 389     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 390     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 391     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 392     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 393     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 394     0x20, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
 395     0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
 396     0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
 397     0xD8, 0xD9, 0xDA, 0x20, 0x20, 0x20, 0x20, 0x20,
 398     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 399     0xE8, 0xE9, 0xEA, 0x20, 0x20, 0x20, 0x20, 0x20,
 400     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 401     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 402 };
 403
 404 static const uint8_t charMap_8859_7[] = {
 405     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 406     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 407     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 408     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 409     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 410     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 411     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 412     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 413     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 414     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 415     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 416     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 417     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 418     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 419     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 420     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 421     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 422     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 423     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 424     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 425     0x20, 0xA1, 0xA2, 0x20, 0x20, 0x20, 0x20, 0x20,
 426     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 427     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xDC, 0x20,
 428     0xDD, 0xDE, 0xDF, 0x20, 0xFC, 0x20, 0xFD, 0xFE,
 429     0xC0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 430     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 431     0xF0, 0xF1, 0x20, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
 432     0xF8, 0xF9, 0xFA, 0xFB, 0xDC, 0xDD, 0xDE, 0xDF,
 433     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 434     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 435     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
 436     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20,
 437 };
 438
 439 static const uint8_t charMap_8859_8[] = {
 440     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 441     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 442     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 443     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 444     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 445     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 446     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 447     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 448     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 449     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 450     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 451     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 452     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 453     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 454     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 455     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 456     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 457     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 458     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 459     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 460     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 461     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 462     0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
 463     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 464     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 465     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 466     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 467     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 468     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 469     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 470     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
 471     0xF8, 0xF9, 0xFA, 0x20, 0x20, 0x20, 0x20, 0x20,
 472 };
 473
 474 static const uint8_t charMap_8859_9[] = {
 475     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 476     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 477     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 478     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 479     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 480     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 481     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 482     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 483     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 484     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 485     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 486     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 487     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 488     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 489     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 490     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 491     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 492     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 493     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 494     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 495     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 496     0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20,
 497     0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
 498     0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20,
 499     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 500     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 501     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
 502     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x69, 0xFE, 0xDF,
 503     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 504     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 505     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
 506     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
 507 };
 508
 509 static const int32_t ngrams_windows_1251[] = {
 510     0x20E220, 0x20E2EE, 0x20E4EE, 0x20E7E0, 0x20E820, 0x20EAE0, 0x20EAEE, 0x20EDE0, 0x20EDE5, 0x20EEE1, 0x20EFEE, 0x20EFF0, 0x20F0E0, 0x20F1EE, 0x20F1F2, 0x20F2EE,
 511     0x20F7F2, 0x20FDF2, 0xE0EDE8, 0xE0F2FC, 0xE3EE20, 0xE5EBFC, 0xE5EDE8, 0xE5F1F2, 0xE5F220, 0xE820EF, 0xE8E520, 0xE8E820, 0xE8FF20, 0xEBE5ED, 0xEBE820, 0xEBFCED,
 512     0xEDE020, 0xEDE520, 0xEDE8E5, 0xEDE8FF, 0xEDEE20, 0xEDEEE2, 0xEE20E2, 0xEE20EF, 0xEE20F1, 0xEEE220, 0xEEE2E0, 0xEEE3EE, 0xEEE920, 0xEEEBFC, 0xEEEC20, 0xEEF1F2,
 513     0xEFEEEB, 0xEFF0E5, 0xEFF0E8, 0xEFF0EE, 0xF0E0E2, 0xF0E5E4, 0xF1F2E0, 0xF1F2E2, 0xF1F2E8, 0xF1FF20, 0xF2E5EB, 0xF2EE20, 0xF2EEF0, 0xF2FC20, 0xF7F2EE, 0xFBF520,
 514 };
 515
 516 static const uint8_t charMap_windows_1251[] = {
 517     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 518     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 519     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 520     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 521     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 522     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 523     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 524     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 525     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 526     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 527     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 528     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 529     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 530     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 531     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 532     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 533     0x90, 0x83, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20,
 534     0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F,
 535     0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 536     0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F,
 537     0x20, 0xA2, 0xA2, 0xBC, 0x20, 0xB4, 0x20, 0x20,
 538     0xB8, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0xBF,
 539     0x20, 0x20, 0xB3, 0xB3, 0xB4, 0xB5, 0x20, 0x20,
 540     0xB8, 0x20, 0xBA, 0x20, 0xBC, 0xBE, 0xBE, 0xBF,
 541     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 542     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 543     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
 544     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
 545     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 546     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 547     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
 548     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
 549 };
 550
 551 static const int32_t ngrams_windows_1256[] = {
 552     0x20C7E1, 0x20C7E4, 0x20C8C7, 0x20DAE1, 0x20DDED, 0x20E1E1, 0x20E3E4, 0x20E6C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E120, 0xC7E1C3, 0xC7E1C7, 0xC7E1C8,
 553     0xC7E1CA, 0xC7E1CC, 0xC7E1CD, 0xC7E1CF, 0xC7E1D3, 0xC7E1DA, 0xC7E1DE, 0xC7E1E3, 0xC7E1E6, 0xC7E1ED, 0xC7E320, 0xC7E420, 0xC7E4CA, 0xC820C7, 0xC920C7, 0xC920DD,
 554     0xC920E1, 0xC920E3, 0xC920E6, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xDA20C7, 0xDAE1EC, 0xDDED20, 0xE120C7, 0xE1C920, 0xE1EC20, 0xE1ED20,
 555     0xE320C7, 0xE3C720, 0xE3C920, 0xE3E420, 0xE420C7, 0xE520C7, 0xE5C720, 0xE6C7E1, 0xE6E420, 0xEC20C7, 0xED20C7, 0xED20E3, 0xED20E6, 0xEDC920, 0xEDD120, 0xEDE420,
 556 };
 557
 558 static const uint8_t charMap_windows_1256[] = {
 559     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 560     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 561     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 562     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 563     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 564     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 565     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 566     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 567     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 568     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 569     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 570     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 571     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 572     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 573     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 574     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 575     0x20, 0x81, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20,
 576     0x88, 0x20, 0x8A, 0x20, 0x9C, 0x8D, 0x8E, 0x8F,
 577     0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 578     0x98, 0x20, 0x9A, 0x20, 0x9C, 0x20, 0x20, 0x9F,
 579     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 580     0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20,
 581     0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
 582     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 583     0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
 584     0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
 585     0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0x20,
 586     0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
 587     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 588     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 589     0x20, 0x20, 0x20, 0x20, 0xF4, 0x20, 0x20, 0x20,
 590     0x20, 0xF9, 0x20, 0xFB, 0xFC, 0x20, 0x20, 0xFF,
 591 };
 592
 593 static const int32_t ngrams_KOI8_R[] = {
 594     0x20C4CF, 0x20C920, 0x20CBC1, 0x20CBCF, 0x20CEC1, 0x20CEC5, 0x20CFC2, 0x20D0CF, 0x20D0D2, 0x20D2C1, 0x20D3CF, 0x20D3D4, 0x20D4CF, 0x20D720, 0x20D7CF, 0x20DAC1,
 595     0x20DCD4, 0x20DED4, 0xC1CEC9, 0xC1D4D8, 0xC5CCD8, 0xC5CEC9, 0xC5D3D4, 0xC5D420, 0xC7CF20, 0xC920D0, 0xC9C520, 0xC9C920, 0xC9D120, 0xCCC5CE, 0xCCC920, 0xCCD8CE,
 596     0xCEC120, 0xCEC520, 0xCEC9C5, 0xCEC9D1, 0xCECF20, 0xCECFD7, 0xCF20D0, 0xCF20D3, 0xCF20D7, 0xCFC7CF, 0xCFCA20, 0xCFCCD8, 0xCFCD20, 0xCFD3D4, 0xCFD720, 0xCFD7C1,
 597     0xD0CFCC, 0xD0D2C5, 0xD0D2C9, 0xD0D2CF, 0xD2C1D7, 0xD2C5C4, 0xD3D120, 0xD3D4C1, 0xD3D4C9, 0xD3D4D7, 0xD4C5CC, 0xD4CF20, 0xD4CFD2, 0xD4D820, 0xD9C820, 0xDED4CF,
 598 };
 599
 600 static const uint8_t charMap_KOI8_R[] = {
 601     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 602     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 603     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 604     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 605     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 606     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 607     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 608     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 609     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 610     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 611     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 612     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 613     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 614     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 615     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 616     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 617     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 618     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 619     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 620     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 621     0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20,
 622     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 623     0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20,
 624     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 625     0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
 626     0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
 627     0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
 628     0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
 629     0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
 630     0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
 631     0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
 632     0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
 633 };
 634
 635 #if !UCONFIG_ONLY_HTML_CONVERSION
 636 static const int32_t ngrams_IBM424_he_rtl[] = {
 637     0x404146, 0x404148, 0x404151, 0x404171, 0x404251, 0x404256, 0x404541, 0x404546, 0x404551, 0x404556, 0x404562, 0x404569, 0x404571, 0x405441, 0x405445, 0x405641,
 638     0x406254, 0x406954, 0x417140, 0x454041, 0x454042, 0x454045, 0x454054, 0x454056, 0x454069, 0x454641, 0x464140, 0x465540, 0x465740, 0x466840, 0x467140, 0x514045,
 639     0x514540, 0x514671, 0x515155, 0x515540, 0x515740, 0x516840, 0x517140, 0x544041, 0x544045, 0x544140, 0x544540, 0x554041, 0x554042, 0x554045, 0x554054, 0x554056,
 640     0x554069, 0x564540, 0x574045, 0x584540, 0x585140, 0x585155, 0x625440, 0x684045, 0x685155, 0x695440, 0x714041, 0x714042, 0x714045, 0x714054, 0x714056, 0x714069,
 641 };
 642
 643 static const int32_t ngrams_IBM424_he_ltr[] = {
 644     0x404146, 0x404154, 0x404551, 0x404554, 0x404556, 0x404558, 0x405158, 0x405462, 0x405469, 0x405546, 0x405551, 0x405746, 0x405751, 0x406846, 0x406851, 0x407141,
 645     0x407146, 0x407151, 0x414045, 0x414054, 0x414055, 0x414071, 0x414540, 0x414645, 0x415440, 0x415640, 0x424045, 0x424055, 0x424071, 0x454045, 0x454051, 0x454054,
 646     0x454055, 0x454057, 0x454068, 0x454071, 0x455440, 0x464140, 0x464540, 0x484140, 0x514140, 0x514240, 0x514540, 0x544045, 0x544055, 0x544071, 0x546240, 0x546940,
 647     0x555151, 0x555158, 0x555168, 0x564045, 0x564055, 0x564071, 0x564240, 0x564540, 0x624540, 0x694045, 0x694055, 0x694071, 0x694540, 0x714140, 0x714540, 0x714651,
 648 };
 649
 650 static const uint8_t charMap_IBM424_he[] = {
 651 /*           -0    -1    -2    -3    -4    -5    -6    -7    -8    -9    -A    -B    -C    -D    -E    -F   */
 652 /* 0- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 653 /* 1- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 654 /* 2- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 655 /* 3- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 656 /* 4- */    0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 657 /* 5- */    0x40, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 658 /* 6- */    0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 659 /* 7- */    0x40, 0x71, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x00, 0x40, 0x40,
 660 /* 8- */    0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 661 /* 9- */    0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 662 /* A- */    0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 663 /* B- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 664 /* C- */    0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 665 /* D- */    0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 666 /* E- */    0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 667 /* F- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 668 };
 669
 670 static const int32_t ngrams_IBM420_ar_rtl[] = {
 671     0x4056B1, 0x4056BD, 0x405856, 0x409AB1, 0x40ABDC, 0x40B1B1, 0x40BBBD, 0x40CF56, 0x564056, 0x564640, 0x566340, 0x567540, 0x56B140, 0x56B149, 0x56B156, 0x56B158,
 672     0x56B163, 0x56B167, 0x56B169, 0x56B173, 0x56B178, 0x56B19A, 0x56B1AD, 0x56B1BB, 0x56B1CF, 0x56B1DC, 0x56BB40, 0x56BD40, 0x56BD63, 0x584056, 0x624056, 0x6240AB,
 673     0x6240B1, 0x6240BB, 0x6240CF, 0x634056, 0x734056, 0x736240, 0x754056, 0x756240, 0x784056, 0x9A4056, 0x9AB1DA, 0xABDC40, 0xB14056, 0xB16240, 0xB1DA40, 0xB1DC40,
 674     0xBB4056, 0xBB5640, 0xBB6240, 0xBBBD40, 0xBD4056, 0xBF4056, 0xBF5640, 0xCF56B1, 0xCFBD40, 0xDA4056, 0xDC4056, 0xDC40BB, 0xDC40CF, 0xDC6240, 0xDC7540, 0xDCBD40,
 675 };
 676
 677 static const int32_t ngrams_IBM420_ar_ltr[] = {
 678     0x404656, 0x4056BB, 0x4056BF, 0x406273, 0x406275, 0x4062B1, 0x4062BB, 0x4062DC, 0x406356, 0x407556, 0x4075DC, 0x40B156, 0x40BB56, 0x40BD56, 0x40BDBB, 0x40BDCF,
 679     0x40BDDC, 0x40DAB1, 0x40DCAB, 0x40DCB1, 0x49B156, 0x564056, 0x564058, 0x564062, 0x564063, 0x564073, 0x564075, 0x564078, 0x56409A, 0x5640B1, 0x5640BB, 0x5640BD,
 680     0x5640BF, 0x5640DA, 0x5640DC, 0x565840, 0x56B156, 0x56CF40, 0x58B156, 0x63B156, 0x63BD56, 0x67B156, 0x69B156, 0x73B156, 0x78B156, 0x9AB156, 0xAB4062, 0xADB156,
 681     0xB14062, 0xB15640, 0xB156CF, 0xB19A40, 0xB1B140, 0xBB4062, 0xBB40DC, 0xBBB156, 0xBD5640, 0xBDBB40, 0xCF4062, 0xCF40DC, 0xCFB156, 0xDAB19A, 0xDCAB40, 0xDCB156
 682 };
 683
 684 static const uint8_t charMap_IBM420_ar[]= {
 685 /*           -0    -1    -2    -3    -4    -5    -6    -7    -8    -9    -A    -B    -C    -D    -E    -F   */
 686 /* 0- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 687 /* 1- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 688 /* 2- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 689 /* 3- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 690 /* 4- */    0x40, 0x40, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 691 /* 5- */    0x40, 0x51, 0x52, 0x40, 0x40, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 692 /* 6- */    0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 693 /* 7- */    0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 694 /* 8- */    0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F,
 695 /* 9- */    0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F,
 696 /* A- */    0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF,
 697 /* B- */    0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0x40, 0x40, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF,
 698 /* C- */    0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0xCB, 0x40, 0xCD, 0x40, 0xCF,
 699 /* D- */    0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
 700 /* E- */    0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xEA, 0xEB, 0x40, 0xED, 0xEE, 0xEF,
 701 /* F- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0xFB, 0xFC, 0xFD, 0xFE, 0x40,
 702 };
 703 #endif
 704
 705 //ISO-8859-1,2,5,6,7,8,9 Ngrams
 706
 707 struct NGramsPlusLang {
 708     const int32_t ngrams[64];
 709     const char *  lang;
 710 };
 711
 712 static const NGramsPlusLang ngrams_8859_1[] =  {
 713   {
 714     {
 715     0x206120, 0x20616E, 0x206265, 0x20636F, 0x20666F, 0x206861, 0x206865, 0x20696E, 0x206D61, 0x206F66, 0x207072, 0x207265, 0x207361, 0x207374, 0x207468, 0x20746F,
 716     0x207768, 0x616964, 0x616C20, 0x616E20, 0x616E64, 0x617320, 0x617420, 0x617465, 0x617469, 0x642061, 0x642074, 0x652061, 0x652073, 0x652074, 0x656420, 0x656E74,
 717     0x657220, 0x657320, 0x666F72, 0x686174, 0x686520, 0x686572, 0x696420, 0x696E20, 0x696E67, 0x696F6E, 0x697320, 0x6E2061, 0x6E2074, 0x6E6420, 0x6E6720, 0x6E7420,
 718     0x6F6620, 0x6F6E20, 0x6F7220, 0x726520, 0x727320, 0x732061, 0x732074, 0x736169, 0x737420, 0x742074, 0x746572, 0x746861, 0x746865, 0x74696F, 0x746F20, 0x747320,
 719     },
 720     "en"
 721   },
 722   {
 723     {
 724     0x206166, 0x206174, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207369, 0x207374, 0x207469, 0x207669, 0x616620,
 725     0x616E20, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646572, 0x646574, 0x652073, 0x656420, 0x656465, 0x656E20, 0x656E64, 0x657220, 0x657265, 0x657320,
 726     0x657420, 0x666F72, 0x676520, 0x67656E, 0x676572, 0x696765, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6572, 0x6C6967, 0x6C6C65, 0x6D6564, 0x6E6465, 0x6E6520,
 727     0x6E6720, 0x6E6765, 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722064, 0x722065, 0x722073, 0x726520, 0x737465, 0x742073, 0x746520, 0x746572, 0x74696C, 0x766572,
 728     },
 729     "da"
 730   },
 731   {
 732     {
 733     0x20616E, 0x206175, 0x206265, 0x206461, 0x206465, 0x206469, 0x206569, 0x206765, 0x206861, 0x20696E, 0x206D69, 0x207363, 0x207365, 0x20756E, 0x207665, 0x20766F,
 734     0x207765, 0x207A75, 0x626572, 0x636820, 0x636865, 0x636874, 0x646173, 0x64656E, 0x646572, 0x646965, 0x652064, 0x652073, 0x65696E, 0x656974, 0x656E20, 0x657220,
 735     0x657320, 0x67656E, 0x68656E, 0x687420, 0x696368, 0x696520, 0x696E20, 0x696E65, 0x697420, 0x6C6963, 0x6C6C65, 0x6E2061, 0x6E2064, 0x6E2073, 0x6E6420, 0x6E6465,
 736     0x6E6520, 0x6E6720, 0x6E6765, 0x6E7465, 0x722064, 0x726465, 0x726569, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x756E64, 0x756E67, 0x766572,
 737     },
 738     "de"
 739   },
 740   {
 741     {
 742     0x206120, 0x206361, 0x20636F, 0x206465, 0x20656C, 0x20656E, 0x206573, 0x20696E, 0x206C61, 0x206C6F, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365,
 743     0x20756E, 0x207920, 0x612063, 0x612064, 0x612065, 0x61206C, 0x612070, 0x616369, 0x61646F, 0x616C20, 0x617220, 0x617320, 0x6369F3, 0x636F6E, 0x646520, 0x64656C,
 744     0x646F20, 0x652064, 0x652065, 0x65206C, 0x656C20, 0x656E20, 0x656E74, 0x657320, 0x657374, 0x69656E, 0x69F36E, 0x6C6120, 0x6C6F73, 0x6E2065, 0x6E7465, 0x6F2064,
 745     0x6F2065, 0x6F6E20, 0x6F7220, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732064, 0x732065, 0x732070, 0x736520, 0x746520, 0x746F20, 0x756520, 0xF36E20,
 746     },
 747     "es"
 748   },
 749   {
 750     {
 751     0x206175, 0x20636F, 0x206461, 0x206465, 0x206475, 0x20656E, 0x206574, 0x206C61, 0x206C65, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207365, 0x20736F, 0x20756E,
 752     0x20E020, 0x616E74, 0x617469, 0x636520, 0x636F6E, 0x646520, 0x646573, 0x647520, 0x652061, 0x652063, 0x652064, 0x652065, 0x65206C, 0x652070, 0x652073, 0x656E20,
 753     0x656E74, 0x657220, 0x657320, 0x657420, 0x657572, 0x696F6E, 0x697320, 0x697420, 0x6C6120, 0x6C6520, 0x6C6573, 0x6D656E, 0x6E2064, 0x6E6520, 0x6E7320, 0x6E7420,
 754     0x6F6E20, 0x6F6E74, 0x6F7572, 0x717565, 0x72206C, 0x726520, 0x732061, 0x732064, 0x732065, 0x73206C, 0x732070, 0x742064, 0x746520, 0x74696F, 0x756520, 0x757220,
 755     },
 756     "fr"
 757   },
 758   {
 759     {
 760     0x20616C, 0x206368, 0x20636F, 0x206465, 0x206469, 0x206520, 0x20696C, 0x20696E, 0x206C61, 0x207065, 0x207072, 0x20756E, 0x612063, 0x612064, 0x612070, 0x612073,
 761     0x61746F, 0x636865, 0x636F6E, 0x64656C, 0x646920, 0x652061, 0x652063, 0x652064, 0x652069, 0x65206C, 0x652070, 0x652073, 0x656C20, 0x656C6C, 0x656E74, 0x657220,
 762     0x686520, 0x692061, 0x692063, 0x692064, 0x692073, 0x696120, 0x696C20, 0x696E20, 0x696F6E, 0x6C6120, 0x6C6520, 0x6C6920, 0x6C6C61, 0x6E6520, 0x6E6920, 0x6E6F20,
 763     0x6E7465, 0x6F2061, 0x6F2064, 0x6F2069, 0x6F2073, 0x6F6E20, 0x6F6E65, 0x706572, 0x726120, 0x726520, 0x736920, 0x746120, 0x746520, 0x746920, 0x746F20, 0x7A696F,
 764     },
 765     "it"
 766   },
 767   {
 768     {
 769     0x20616C, 0x206265, 0x206461, 0x206465, 0x206469, 0x206565, 0x20656E, 0x206765, 0x206865, 0x20696E, 0x206D61, 0x206D65, 0x206F70, 0x207465, 0x207661, 0x207665,
 770     0x20766F, 0x207765, 0x207A69, 0x61616E, 0x616172, 0x616E20, 0x616E64, 0x617220, 0x617420, 0x636874, 0x646520, 0x64656E, 0x646572, 0x652062, 0x652076, 0x65656E,
 771     0x656572, 0x656E20, 0x657220, 0x657273, 0x657420, 0x67656E, 0x686574, 0x696520, 0x696E20, 0x696E67, 0x697320, 0x6E2062, 0x6E2064, 0x6E2065, 0x6E2068, 0x6E206F,
 772     0x6E2076, 0x6E6465, 0x6E6720, 0x6F6E64, 0x6F6F72, 0x6F7020, 0x6F7220, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x76616E, 0x766572, 0x766F6F,
 773     },
 774     "nl"
 775   },
 776   {
 777     {
 778     0x206174, 0x206176, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207365, 0x20736B, 0x20736F, 0x207374, 0x207469,
 779     0x207669, 0x20E520, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646574, 0x652073, 0x656420, 0x656E20, 0x656E65, 0x657220, 0x657265, 0x657420, 0x657474,
 780     0x666F72, 0x67656E, 0x696B6B, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6520, 0x6C6C65, 0x6D6564, 0x6D656E, 0x6E2073, 0x6E6520, 0x6E6720, 0x6E6765, 0x6E6E65,
 781     0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722073, 0x726520, 0x736F6D, 0x737465, 0x742073, 0x746520, 0x74656E, 0x746572, 0x74696C, 0x747420, 0x747465, 0x766572,
 782     },
 783     "no"
 784   },
 785   {
 786     {
 787     0x206120, 0x20636F, 0x206461, 0x206465, 0x20646F, 0x206520, 0x206573, 0x206D61, 0x206E6F, 0x206F20, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365,
 788     0x20756D, 0x612061, 0x612063, 0x612064, 0x612070, 0x616465, 0x61646F, 0x616C20, 0x617220, 0x617261, 0x617320, 0x636F6D, 0x636F6E, 0x646120, 0x646520, 0x646F20,
 789     0x646F73, 0x652061, 0x652064, 0x656D20, 0x656E74, 0x657320, 0x657374, 0x696120, 0x696361, 0x6D656E, 0x6E7465, 0x6E746F, 0x6F2061, 0x6F2063, 0x6F2064, 0x6F2065,
 790     0x6F2070, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732061, 0x732064, 0x732065, 0x732070, 0x737461, 0x746520, 0x746F20, 0x756520, 0xE36F20, 0xE7E36F,
 791     },
 792     "pt"
 793   },
 794   {
 795     {
 796     0x206174, 0x206176, 0x206465, 0x20656E, 0x2066F6, 0x206861, 0x206920, 0x20696E, 0x206B6F, 0x206D65, 0x206F63, 0x2070E5, 0x20736B, 0x20736F, 0x207374, 0x207469,
 797     0x207661, 0x207669, 0x20E472, 0x616465, 0x616E20, 0x616E64, 0x617220, 0x617474, 0x636820, 0x646520, 0x64656E, 0x646572, 0x646574, 0x656420, 0x656E20, 0x657220,
 798     0x657420, 0x66F672, 0x67656E, 0x696C6C, 0x696E67, 0x6B6120, 0x6C6C20, 0x6D6564, 0x6E2073, 0x6E6120, 0x6E6465, 0x6E6720, 0x6E6765, 0x6E696E, 0x6F6368, 0x6F6D20,
 799     0x6F6E20, 0x70E520, 0x722061, 0x722073, 0x726120, 0x736B61, 0x736F6D, 0x742073, 0x746120, 0x746520, 0x746572, 0x74696C, 0x747420, 0x766172, 0xE47220, 0xF67220,
 800     },
 801     "sv"
 802   }
 803 };
 804
 805
 806 static const NGramsPlusLang ngrams_8859_2[] =  {
 807   {
 808     {
 809     0x206120, 0x206279, 0x20646F, 0x206A65, 0x206E61, 0x206E65, 0x206F20, 0x206F64, 0x20706F, 0x207072, 0x2070F8, 0x20726F, 0x207365, 0x20736F, 0x207374, 0x20746F,
 810     0x207620, 0x207679, 0x207A61, 0x612070, 0x636520, 0x636820, 0x652070, 0x652073, 0x652076, 0x656D20, 0x656EED, 0x686F20, 0x686F64, 0x697374, 0x6A6520, 0x6B7465,
 811     0x6C6520, 0x6C6920, 0x6E6120, 0x6EE920, 0x6EEC20, 0x6EED20, 0x6F2070, 0x6F646E, 0x6F6A69, 0x6F7374, 0x6F7520, 0x6F7661, 0x706F64, 0x706F6A, 0x70726F, 0x70F865,
 812     0x736520, 0x736F75, 0x737461, 0x737469, 0x73746E, 0x746572, 0x746EED, 0x746F20, 0x752070, 0xBE6520, 0xE16EED, 0xE9686F, 0xED2070, 0xED2073, 0xED6D20, 0xF86564,
 813     },
 814     "cs"
 815   },
 816   {
 817     {
 818     0x206120, 0x20617A, 0x206265, 0x206567, 0x20656C, 0x206665, 0x206861, 0x20686F, 0x206973, 0x206B65, 0x206B69, 0x206BF6, 0x206C65, 0x206D61, 0x206D65, 0x206D69,
 819     0x206E65, 0x20737A, 0x207465, 0x20E973, 0x612061, 0x61206B, 0x61206D, 0x612073, 0x616B20, 0x616E20, 0x617A20, 0x62616E, 0x62656E, 0x656779, 0x656B20, 0x656C20,
 820     0x656C65, 0x656D20, 0x656E20, 0x657265, 0x657420, 0x657465, 0x657474, 0x677920, 0x686F67, 0x696E74, 0x697320, 0x6B2061, 0x6BF67A, 0x6D6567, 0x6D696E, 0x6E2061,
 821     0x6E616B, 0x6E656B, 0x6E656D, 0x6E7420, 0x6F6779, 0x732061, 0x737A65, 0x737A74, 0x737AE1, 0x73E967, 0x742061, 0x747420, 0x74E173, 0x7A6572, 0xE16E20, 0xE97320,
 822     },
 823     "hu"
 824   },
 825   {
 826     {
 827     0x20637A, 0x20646F, 0x206920, 0x206A65, 0x206B6F, 0x206D61, 0x206D69, 0x206E61, 0x206E69, 0x206F64, 0x20706F, 0x207072, 0x207369, 0x207720, 0x207769, 0x207779,
 828     0x207A20, 0x207A61, 0x612070, 0x612077, 0x616E69, 0x636820, 0x637A65, 0x637A79, 0x646F20, 0x647A69, 0x652070, 0x652073, 0x652077, 0x65207A, 0x65676F, 0x656A20,
 829     0x656D20, 0x656E69, 0x676F20, 0x696120, 0x696520, 0x69656A, 0x6B6120, 0x6B6920, 0x6B6965, 0x6D6965, 0x6E6120, 0x6E6961, 0x6E6965, 0x6F2070, 0x6F7761, 0x6F7769,
 830     0x706F6C, 0x707261, 0x70726F, 0x70727A, 0x727A65, 0x727A79, 0x7369EA, 0x736B69, 0x737461, 0x776965, 0x796368, 0x796D20, 0x7A6520, 0x7A6965, 0x7A7920, 0xF37720,
 831     },
 832     "pl"
 833   },
 834   {
 835     {
 836     0x206120, 0x206163, 0x206361, 0x206365, 0x20636F, 0x206375, 0x206465, 0x206469, 0x206C61, 0x206D61, 0x207065, 0x207072, 0x207365, 0x2073E3, 0x20756E, 0x20BA69,
 837     0x20EE6E, 0x612063, 0x612064, 0x617265, 0x617420, 0x617465, 0x617520, 0x636172, 0x636F6E, 0x637520, 0x63E320, 0x646520, 0x652061, 0x652063, 0x652064, 0x652070,
 838     0x652073, 0x656120, 0x656920, 0x656C65, 0x656E74, 0x657374, 0x692061, 0x692063, 0x692064, 0x692070, 0x696520, 0x696920, 0x696E20, 0x6C6120, 0x6C6520, 0x6C6F72,
 839     0x6C7569, 0x6E6520, 0x6E7472, 0x6F7220, 0x70656E, 0x726520, 0x726561, 0x727520, 0x73E320, 0x746520, 0x747275, 0x74E320, 0x756920, 0x756C20, 0xBA6920, 0xEE6E20,
 840     },
 841     "ro"
 842   }
 843 };
 844
 845 static const int32_t ngrams_8859_5_ru[] = {
 846     0x20D220, 0x20D2DE, 0x20D4DE, 0x20D7D0, 0x20D820, 0x20DAD0, 0x20DADE, 0x20DDD0, 0x20DDD5, 0x20DED1, 0x20DFDE, 0x20DFE0, 0x20E0D0, 0x20E1DE, 0x20E1E2, 0x20E2DE,
 847     0x20E7E2, 0x20EDE2, 0xD0DDD8, 0xD0E2EC, 0xD3DE20, 0xD5DBEC, 0xD5DDD8, 0xD5E1E2, 0xD5E220, 0xD820DF, 0xD8D520, 0xD8D820, 0xD8EF20, 0xDBD5DD, 0xDBD820, 0xDBECDD,
 848     0xDDD020, 0xDDD520, 0xDDD8D5, 0xDDD8EF, 0xDDDE20, 0xDDDED2, 0xDE20D2, 0xDE20DF, 0xDE20E1, 0xDED220, 0xDED2D0, 0xDED3DE, 0xDED920, 0xDEDBEC, 0xDEDC20, 0xDEE1E2,
 849     0xDFDEDB, 0xDFE0D5, 0xDFE0D8, 0xDFE0DE, 0xE0D0D2, 0xE0D5D4, 0xE1E2D0, 0xE1E2D2, 0xE1E2D8, 0xE1EF20, 0xE2D5DB, 0xE2DE20, 0xE2DEE0, 0xE2EC20, 0xE7E2DE, 0xEBE520,
 850 };
 851
 852 static const int32_t ngrams_8859_6_ar[] = {
 853     0x20C7E4, 0x20C7E6, 0x20C8C7, 0x20D9E4, 0x20E1EA, 0x20E4E4, 0x20E5E6, 0x20E8C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E420, 0xC7E4C3, 0xC7E4C7, 0xC7E4C8,
 854     0xC7E4CA, 0xC7E4CC, 0xC7E4CD, 0xC7E4CF, 0xC7E4D3, 0xC7E4D9, 0xC7E4E2, 0xC7E4E5, 0xC7E4E8, 0xC7E4EA, 0xC7E520, 0xC7E620, 0xC7E6CA, 0xC820C7, 0xC920C7, 0xC920E1,
 855     0xC920E4, 0xC920E5, 0xC920E8, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xD920C7, 0xD9E4E9, 0xE1EA20, 0xE420C7, 0xE4C920, 0xE4E920, 0xE4EA20,
 856     0xE520C7, 0xE5C720, 0xE5C920, 0xE5E620, 0xE620C7, 0xE720C7, 0xE7C720, 0xE8C7E4, 0xE8E620, 0xE920C7, 0xEA20C7, 0xEA20E5, 0xEA20E8, 0xEAC920, 0xEAD120, 0xEAE620,
 857 };
 858
 859 static const int32_t ngrams_8859_7_el[] = {
 860     0x20E1ED, 0x20E1F0, 0x20E3E9, 0x20E4E9, 0x20E5F0, 0x20E720, 0x20EAE1, 0x20ECE5, 0x20EDE1, 0x20EF20, 0x20F0E1, 0x20F0EF, 0x20F0F1, 0x20F3F4, 0x20F3F5, 0x20F4E7,
 861     0x20F4EF, 0xDFE120, 0xE120E1, 0xE120F4, 0xE1E920, 0xE1ED20, 0xE1F0FC, 0xE1F220, 0xE3E9E1, 0xE5E920, 0xE5F220, 0xE720F4, 0xE7ED20, 0xE7F220, 0xE920F4, 0xE9E120,
 862     0xE9EADE, 0xE9F220, 0xEAE1E9, 0xEAE1F4, 0xECE520, 0xED20E1, 0xED20E5, 0xED20F0, 0xEDE120, 0xEFF220, 0xEFF520, 0xF0EFF5, 0xF0F1EF, 0xF0FC20, 0xF220E1, 0xF220E5,
 863     0xF220EA, 0xF220F0, 0xF220F4, 0xF3E520, 0xF3E720, 0xF3F4EF, 0xF4E120, 0xF4E1E9, 0xF4E7ED, 0xF4E7F2, 0xF4E9EA, 0xF4EF20, 0xF4EFF5, 0xF4F9ED, 0xF9ED20, 0xFEED20,
 864 };
 865
 866 static const int32_t ngrams_8859_8_I_he[] = {
 867     0x20E0E5, 0x20E0E7, 0x20E0E9, 0x20E0FA, 0x20E1E9, 0x20E1EE, 0x20E4E0, 0x20E4E5, 0x20E4E9, 0x20E4EE, 0x20E4F2, 0x20E4F9, 0x20E4FA, 0x20ECE0, 0x20ECE4, 0x20EEE0,
 868     0x20F2EC, 0x20F9EC, 0xE0FA20, 0xE420E0, 0xE420E1, 0xE420E4, 0xE420EC, 0xE420EE, 0xE420F9, 0xE4E5E0, 0xE5E020, 0xE5ED20, 0xE5EF20, 0xE5F820, 0xE5FA20, 0xE920E4,
 869     0xE9E420, 0xE9E5FA, 0xE9E9ED, 0xE9ED20, 0xE9EF20, 0xE9F820, 0xE9FA20, 0xEC20E0, 0xEC20E4, 0xECE020, 0xECE420, 0xED20E0, 0xED20E1, 0xED20E4, 0xED20EC, 0xED20EE,
 870     0xED20F9, 0xEEE420, 0xEF20E4, 0xF0E420, 0xF0E920, 0xF0E9ED, 0xF2EC20, 0xF820E4, 0xF8E9ED, 0xF9EC20, 0xFA20E0, 0xFA20E1, 0xFA20E4, 0xFA20EC, 0xFA20EE, 0xFA20F9,
 871 };
 872
 873 static const int32_t ngrams_8859_8_he[] = {
 874     0x20E0E5, 0x20E0EC, 0x20E4E9, 0x20E4EC, 0x20E4EE, 0x20E4F0, 0x20E9F0, 0x20ECF2, 0x20ECF9, 0x20EDE5, 0x20EDE9, 0x20EFE5, 0x20EFE9, 0x20F8E5, 0x20F8E9, 0x20FAE0,
 875     0x20FAE5, 0x20FAE9, 0xE020E4, 0xE020EC, 0xE020ED, 0xE020FA, 0xE0E420, 0xE0E5E4, 0xE0EC20, 0xE0EE20, 0xE120E4, 0xE120ED, 0xE120FA, 0xE420E4, 0xE420E9, 0xE420EC,
 876     0xE420ED, 0xE420EF, 0xE420F8, 0xE420FA, 0xE4EC20, 0xE5E020, 0xE5E420, 0xE7E020, 0xE9E020, 0xE9E120, 0xE9E420, 0xEC20E4, 0xEC20ED, 0xEC20FA, 0xECF220, 0xECF920,
 877     0xEDE9E9, 0xEDE9F0, 0xEDE9F8, 0xEE20E4, 0xEE20ED, 0xEE20FA, 0xEEE120, 0xEEE420, 0xF2E420, 0xF920E4, 0xF920ED, 0xF920FA, 0xF9E420, 0xFAE020, 0xFAE420, 0xFAE5E9,
 878 };
 879
 880 static const int32_t ngrams_8859_9_tr[] = {
 881     0x206261, 0x206269, 0x206275, 0x206461, 0x206465, 0x206765, 0x206861, 0x20696C, 0x206B61, 0x206B6F, 0x206D61, 0x206F6C, 0x207361, 0x207461, 0x207665, 0x207961,
 882     0x612062, 0x616B20, 0x616C61, 0x616D61, 0x616E20, 0x616EFD, 0x617220, 0x617261, 0x6172FD, 0x6173FD, 0x617961, 0x626972, 0x646120, 0x646520, 0x646920, 0x652062,
 883     0x65206B, 0x656469, 0x656E20, 0x657220, 0x657269, 0x657369, 0x696C65, 0x696E20, 0x696E69, 0x697220, 0x6C616E, 0x6C6172, 0x6C6520, 0x6C6572, 0x6E2061, 0x6E2062,
 884     0x6E206B, 0x6E6461, 0x6E6465, 0x6E6520, 0x6E6920, 0x6E696E, 0x6EFD20, 0x72696E, 0x72FD6E, 0x766520, 0x796120, 0x796F72, 0xFD6E20, 0xFD6E64, 0xFD6EFD, 0xFDF0FD,
 885 };
 886
 887 CharsetRecog_8859_1::~CharsetRecog_8859_1()
 888 {
 889     // nothing to do
 890 }
 891
 892 UBool CharsetRecog_8859_1::match(InputText *textIn, CharsetMatch *results) const {
 893     const char *name = textIn->fC1Bytes? "windows-1252" : "ISO-8859-1";
 894     uint32_t i;
 895     int32_t bestConfidenceSoFar = -1;
 896     for (i=0; i < UPRV_LENGTHOF(ngrams_8859_1) ; i++) {
 897         const int32_t *ngrams = ngrams_8859_1[i].ngrams;
 898         const char    *lang   = ngrams_8859_1[i].lang;
 899         int32_t confidence = match_sbcs(textIn, ngrams, charMap_8859_1);
 900         if (confidence > bestConfidenceSoFar) {
 901             results->set(textIn, this, confidence, name, lang);
 902             bestConfidenceSoFar = confidence;
 903         }
 904     }
 905     return (bestConfidenceSoFar > 0);
 906 }
 907
 908 const char *CharsetRecog_8859_1::getName() const
 909 {
 910     return "ISO-8859-1";
 911 }
 912
 913
 914 CharsetRecog_8859_2::~CharsetRecog_8859_2()
 915 {
 916     // nothing to do
 917 }
 918
 919 UBool CharsetRecog_8859_2::match(InputText *textIn, CharsetMatch *results) const {
 920     const char *name = textIn->fC1Bytes? "windows-1250" : "ISO-8859-2";
 921     uint32_t i;
 922     int32_t bestConfidenceSoFar = -1;
 923     for (i=0; i < UPRV_LENGTHOF(ngrams_8859_2) ; i++) {
 924         const int32_t *ngrams = ngrams_8859_2[i].ngrams;
 925         const char    *lang   = ngrams_8859_2[i].lang;
 926         int32_t confidence = match_sbcs(textIn, ngrams, charMap_8859_2);
 927         if (confidence > bestConfidenceSoFar) {
 928             results->set(textIn, this, confidence, name, lang);
 929             bestConfidenceSoFar = confidence;
 930         }
 931     }
 932     return (bestConfidenceSoFar > 0);
 933 }
 934
 935 const char *CharsetRecog_8859_2::getName() const
 936 {
 937     return "ISO-8859-2";
 938 }
 939
 940
 941 CharsetRecog_8859_5::~CharsetRecog_8859_5()
 942 {
 943     // nothing to do
 944 }
 945
 946 const char *CharsetRecog_8859_5::getName() const
 947 {
 948     return "ISO-8859-5";
 949 }
 950
 951 CharsetRecog_8859_5_ru::~CharsetRecog_8859_5_ru()
 952 {
 953     // nothing to do
 954 }
 955
 956 const char *CharsetRecog_8859_5_ru::getLanguage() const
 957 {
 958     return "ru";
 959 }
 960
 961 UBool CharsetRecog_8859_5_ru::match(InputText *textIn, CharsetMatch *results) const
 962 {
 963     int32_t confidence = match_sbcs(textIn, ngrams_8859_5_ru, charMap_8859_5);
 964     results->set(textIn, this, confidence);
 965     return (confidence > 0);
 966 }
 967
 968 CharsetRecog_8859_6::~CharsetRecog_8859_6()
 969 {
 970     // nothing to do
 971 }
 972
 973 const char *CharsetRecog_8859_6::getName() const
 974 {
 975     return "ISO-8859-6";
 976 }
 977
 978 CharsetRecog_8859_6_ar::~CharsetRecog_8859_6_ar()
 979 {
 980     // nothing to do
 981 }
 982
 983 const char *CharsetRecog_8859_6_ar::getLanguage() const
 984 {
 985     return "ar";
 986 }
 987
 988 UBool CharsetRecog_8859_6_ar::match(InputText *textIn, CharsetMatch *results) const
 989 {
 990     int32_t confidence = match_sbcs(textIn, ngrams_8859_6_ar, charMap_8859_6);
 991     results->set(textIn, this, confidence);
 992     return (confidence > 0);
 993 }
 994
 995 CharsetRecog_8859_7::~CharsetRecog_8859_7()
 996 {
 997     // nothing to do
 998 }
 999
1000 const char *CharsetRecog_8859_7::getName() const
1001 {
1002     return "ISO-8859-7";
1003 }
1004
1005 CharsetRecog_8859_7_el::~CharsetRecog_8859_7_el()
1006 {
1007     // nothing to do
1008 }
1009
1010 const char *CharsetRecog_8859_7_el::getLanguage() const
1011 {
1012     return "el";
1013 }
1014
1015 UBool CharsetRecog_8859_7_el::match(InputText *textIn, CharsetMatch *results) const
1016 {
1017     const char *name = textIn->fC1Bytes? "windows-1253" : "ISO-8859-7";
1018     int32_t confidence = match_sbcs(textIn, ngrams_8859_7_el, charMap_8859_7);
1019     results->set(textIn, this, confidence, name, "el");
1020     return (confidence > 0);
1021 }
1022
1023 CharsetRecog_8859_8::~CharsetRecog_8859_8()
1024 {
1025     // nothing to do
1026 }
1027
1028 const char *CharsetRecog_8859_8::getName() const
1029 {
1030     return "ISO-8859-8";
1031 }
1032
1033 CharsetRecog_8859_8_I_he::~CharsetRecog_8859_8_I_he ()
1034 {
1035     // nothing to do
1036 }
1037
1038 const char *CharsetRecog_8859_8_I_he::getName() const
1039 {
1040     return "ISO-8859-8-I";
1041 }
1042
1043 const char *CharsetRecog_8859_8_I_he::getLanguage() const
1044 {
1045     return "he";
1046 }
1047
1048 UBool CharsetRecog_8859_8_I_he::match(InputText *textIn, CharsetMatch *results) const
1049 {
1050     const char *name = textIn->fC1Bytes? "windows-1255" : "ISO-8859-8-I";
1051     int32_t confidence = match_sbcs(textIn, ngrams_8859_8_I_he, charMap_8859_8);
1052     results->set(textIn, this, confidence, name, "he");
1053     return (confidence > 0);
1054 }
1055
1056 CharsetRecog_8859_8_he::~CharsetRecog_8859_8_he()
1057 {
1058     // od ot gnihton
1059 }
1060
1061 const char *CharsetRecog_8859_8_he::getLanguage() const
1062 {
1063     return "he";
1064 }
1065
1066 UBool CharsetRecog_8859_8_he::match(InputText *textIn, CharsetMatch *results) const
1067 {
1068     const char *name = textIn->fC1Bytes? "windows-1255" : "ISO-8859-8";
1069     int32_t confidence = match_sbcs(textIn, ngrams_8859_8_he, charMap_8859_8);
1070     results->set(textIn, this, confidence, name, "he");
1071     return (confidence > 0);
1072 }
1073
1074 CharsetRecog_8859_9::~CharsetRecog_8859_9()
1075 {
1076     // nothing to do
1077 }
1078
1079 const char *CharsetRecog_8859_9::getName() const
1080 {
1081     return "ISO-8859-9";
1082 }
1083
1084 CharsetRecog_8859_9_tr::~CharsetRecog_8859_9_tr ()
1085 {
1086     // nothing to do
1087 }
1088
1089 const char *CharsetRecog_8859_9_tr::getLanguage() const
1090 {
1091     return "tr";
1092 }
1093
1094 UBool CharsetRecog_8859_9_tr::match(InputText *textIn, CharsetMatch *results) const
1095 {
1096     const char *name = textIn->fC1Bytes? "windows-1254" : "ISO-8859-9";
1097     int32_t confidence = match_sbcs(textIn, ngrams_8859_9_tr, charMap_8859_9);
1098     results->set(textIn, this, confidence, name, "tr");
1099     return (confidence > 0);
1100 }
1101
1102 CharsetRecog_windows_1256::~CharsetRecog_windows_1256()
1103 {
1104     // nothing to do
1105 }
1106
1107 const char *CharsetRecog_windows_1256::getName() const
1108 {
1109     return  "windows-1256";
1110 }
1111
1112 const char *CharsetRecog_windows_1256::getLanguage() const
1113 {
1114     return "ar";
1115 }
1116
1117 UBool CharsetRecog_windows_1256::match(InputText *textIn, CharsetMatch *results) const
1118 {
1119     int32_t confidence = match_sbcs(textIn, ngrams_windows_1256, charMap_windows_1256);
1120     results->set(textIn, this, confidence);
1121     return (confidence > 0);
1122 }
1123
1124 CharsetRecog_windows_1251::~CharsetRecog_windows_1251()
1125 {
1126     // nothing to do
1127 }
1128
1129 const char *CharsetRecog_windows_1251::getName() const
1130 {
1131     return  "windows-1251";
1132 }
1133
1134 const char *CharsetRecog_windows_1251::getLanguage() const
1135 {
1136     return "ru";
1137 }
1138
1139 UBool CharsetRecog_windows_1251::match(InputText *textIn, CharsetMatch *results) const
1140 {
1141     int32_t confidence = match_sbcs(textIn, ngrams_windows_1251, charMap_windows_1251);
1142     results->set(textIn, this, confidence);
1143     return (confidence > 0);
1144 }
1145
1146 CharsetRecog_KOI8_R::~CharsetRecog_KOI8_R()
1147 {
1148     // nothing to do
1149 }
1150
1151 const char *CharsetRecog_KOI8_R::getName() const
1152 {
1153     return  "KOI8-R";
1154 }
1155
1156 const char *CharsetRecog_KOI8_R::getLanguage() const
1157 {
1158     return "ru";
1159 }
1160
1161 UBool CharsetRecog_KOI8_R::match(InputText *textIn, CharsetMatch *results) const
1162 {
1163     int32_t confidence = match_sbcs(textIn, ngrams_KOI8_R, charMap_KOI8_R);
1164     results->set(textIn, this, confidence);
1165     return (confidence > 0);
1166 }
1167
1168 #if !UCONFIG_ONLY_HTML_CONVERSION
1169 CharsetRecog_IBM424_he::~CharsetRecog_IBM424_he()
1170 {
1171     // nothing to do
1172 }
1173
1174 const char *CharsetRecog_IBM424_he::getLanguage() const
1175 {
1176     return "he";
1177 }
1178
1179 CharsetRecog_IBM424_he_rtl::~CharsetRecog_IBM424_he_rtl()
1180 {
1181     // nothing to do
1182 }
1183
1184 const char *CharsetRecog_IBM424_he_rtl::getName() const
1185 {
1186     return  "IBM424_rtl";
1187 }
1188
1189 UBool CharsetRecog_IBM424_he_rtl::match(InputText *textIn, CharsetMatch *results) const
1190 {
1191     int32_t confidence = match_sbcs(textIn, ngrams_IBM424_he_rtl, charMap_IBM424_he);
1192     results->set(textIn, this, confidence);
1193     return (confidence > 0);
1194 }
1195
1196 CharsetRecog_IBM424_he_ltr::~CharsetRecog_IBM424_he_ltr()
1197 {
1198     // nothing to do
1199 }
1200
1201 const char *CharsetRecog_IBM424_he_ltr::getName() const
1202 {
1203     return  "IBM424_ltr";
1204 }
1205
1206 UBool CharsetRecog_IBM424_he_ltr::match(InputText *textIn, CharsetMatch *results) const
1207 {
1208     int32_t confidence = match_sbcs(textIn, ngrams_IBM424_he_ltr, charMap_IBM424_he);
1209     results->set(textIn, this, confidence);
1210     return (confidence > 0);
1211 }
1212
1213 CharsetRecog_IBM420_ar::~CharsetRecog_IBM420_ar()
1214 {
1215     // nothing to do
1216 }
1217
1218 const char *CharsetRecog_IBM420_ar::getLanguage() const
1219 {
1220     return "ar";
1221 }
1222
1223
1224 int32_t CharsetRecog_IBM420_ar::match_sbcs(InputText *det, const int32_t ngrams[],  const uint8_t byteMap[]) const
1225 {
1226     NGramParser_IBM420 parser(ngrams, byteMap);
1227     int32_t result;
1228
1229     result = parser.parse(det);
1230
1231     return result;
1232 }
1233
1234 CharsetRecog_IBM420_ar_rtl::~CharsetRecog_IBM420_ar_rtl()
1235 {
1236     // nothing to do
1237 }
1238
1239 const char *CharsetRecog_IBM420_ar_rtl::getName() const
1240 {
1241     return  "IBM420_rtl";
1242 }
1243
1244 UBool CharsetRecog_IBM420_ar_rtl::match(InputText *textIn, CharsetMatch *results) const
1245 {
1246     int32_t confidence = match_sbcs(textIn, ngrams_IBM420_ar_rtl, charMap_IBM420_ar);
1247     results->set(textIn, this, confidence);
1248     return (confidence > 0);
1249 }
1250
1251 CharsetRecog_IBM420_ar_ltr::~CharsetRecog_IBM420_ar_ltr()
1252 {
1253     // nothing to do
1254 }
1255
1256 const char *CharsetRecog_IBM420_ar_ltr::getName() const
1257 {
1258     return  "IBM420_ltr";
1259 }
1260
1261 UBool CharsetRecog_IBM420_ar_ltr::match(InputText *textIn, CharsetMatch *results) const
1262 {
1263     int32_t confidence = match_sbcs(textIn, ngrams_IBM420_ar_ltr, charMap_IBM420_ar);
1264     results->set(textIn, this, confidence);
1265     return (confidence > 0);
1266 }
1267 #endif
1268
1269 U_NAMESPACE_END
1270 #endif
1271