icuSources/i18n/csrsbcs.cpp

   1 /*
   2  **********************************************************************
   3  *   Copyright (C) 2005-2012, International Business Machines
   4  *   Corporation and others.  All Rights Reserved.
   5  **********************************************************************
   6  */
   7
   8 #include "unicode/utypes.h"
   9
  10 #include "cmemory.h"
  11
  12 #if !UCONFIG_NO_CONVERSION
  13 #include "csrsbcs.h"
  14 #include "csmatch.h"
  15
  16 #define N_GRAM_SIZE 3
  17 #define N_GRAM_MASK 0xFFFFFF
  18 #define ARRAY_SIZE(array) (sizeof array / sizeof array[0])
  19
  20 U_NAMESPACE_BEGIN
  21
  22 NGramParser::NGramParser(const int32_t *theNgramList, const uint8_t *theCharMap)
  23   :byteIndex(0), ngram(0)
  24 {
  25     ngramList = theNgramList;
  26     charMap   = theCharMap;
  27
  28     ngramCount = hitCount = 0;
  29 }
  30
  31 /*
  32  * Binary search for value in table, which must have exactly 64 entries.
  33  */
  34
  35 int32_t NGramParser::search(const int32_t *table, int32_t value)
  36 {
  37     int32_t index = 0;
  38
  39     if (table[index + 32] <= value) {
  40         index += 32;
  41     }
  42
  43     if (table[index + 16] <= value) {
  44         index += 16;
  45     }
  46
  47     if (table[index + 8] <= value) {
  48         index += 8;
  49     }
  50
  51     if (table[index + 4] <= value) {
  52         index += 4;
  53     }
  54
  55     if (table[index + 2] <= value) {
  56         index += 2;
  57     }
  58
  59     if (table[index + 1] <= value) {
  60         index += 1;
  61     }
  62
  63     if (table[index] > value) {
  64         index -= 1;
  65     }
  66
  67     if (index < 0 || table[index] != value) {
  68         return -1;
  69     }
  70
  71     return index;
  72 }
  73
  74 void NGramParser::lookup(int32_t thisNgram)
  75 {
  76     ngramCount += 1;
  77
  78     if (search(ngramList, thisNgram) >= 0) {
  79         hitCount += 1;
  80     }
  81
  82 }
  83
  84 void NGramParser::addByte(int32_t b)
  85 {
  86     ngram = ((ngram << 8) + b) & N_GRAM_MASK;
  87     lookup(ngram);
  88 }
  89
  90 int32_t NGramParser::nextByte(InputText *det)
  91 {
  92     if (byteIndex >= det->fInputLen) {
  93         return -1;
  94     }
  95
  96     return det->fInputBytes[byteIndex++];
  97 }
  98
  99 int32_t NGramParser::parse(InputText *det)
 100 {
 101     int32_t b;
 102     bool ignoreSpace = FALSE;
 103
 104     while ((b = nextByte(det)) >= 0) {
 105         uint8_t mb = charMap[b];
 106
 107         // TODO: 0x20 might not be a space in all character sets...
 108         if (mb != 0) {
 109             if (!(mb == 0x20 && ignoreSpace)) {
 110                 addByte(mb);
 111             }
 112
 113             ignoreSpace = (mb == 0x20);
 114         }
 115     }
 116
 117     // TODO: Is this OK? The buffer could have ended in the middle of a word...
 118     addByte(0x20);
 119
 120     double rawPercent = (double) hitCount / (double) ngramCount;
 121
 122     //            if (rawPercent <= 2.0) {
 123     //                return 0;
 124     //            }
 125
 126     // TODO - This is a bit of a hack to take care of a case
 127     // were we were getting a confidence of 135...
 128     if (rawPercent > 0.33) {
 129         return 98;
 130     }
 131
 132     return (int32_t) (rawPercent * 300.0);
 133 }
 134
 135 CharsetRecog_sbcs::CharsetRecog_sbcs()
 136 {
 137     // nothing else to do
 138 }
 139
 140 CharsetRecog_sbcs::~CharsetRecog_sbcs()
 141 {
 142     // nothing to do
 143 }
 144
 145 int32_t CharsetRecog_sbcs::match_sbcs(InputText *det, const int32_t ngrams[],  const uint8_t byteMap[]) const
 146 {
 147     NGramParser parser(ngrams, byteMap);
 148     int32_t result;
 149
 150     result = parser.parse(det);
 151
 152     return result;
 153 }
 154
 155 static const uint8_t charMap_8859_1[] = {
 156     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 157     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 158     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 159     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 160     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 161     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 162     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 163     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 164     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 165     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 166     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 167     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 168     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 169     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 170     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 171     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 172     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 173     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 174     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 175     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 176     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 177     0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20,
 178     0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
 179     0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20,
 180     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 181     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 182     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
 183     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF,
 184     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 185     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 186     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
 187     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
 188 };
 189
 190 static const uint8_t charMap_8859_2[] = {
 191     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 192     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 193     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 194     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 195     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 196     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 197     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 198     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 199     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 200     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 201     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 202     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 203     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 204     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 205     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 206     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 207     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 208     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 209     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 210     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 211     0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0x20,
 212     0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF,
 213     0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0xB7,
 214     0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF,
 215     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 216     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 217     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
 218     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF,
 219     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 220     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 221     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
 222     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20,
 223 };
 224
 225 static const uint8_t charMap_8859_5[] = {
 226     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 227     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 228     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 229     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 230     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 231     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 232     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 233     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 234     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 235     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 236     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 237     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 238     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 239     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 240     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 241     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 242     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 243     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 244     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 245     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 246     0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
 247     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF,
 248     0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
 249     0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
 250     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 251     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 252     0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
 253     0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
 254     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 255     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 256     0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
 257     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF,
 258 };
 259
 260 static const uint8_t charMap_8859_6[] = {
 261     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 262     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 263     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 264     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 265     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 266     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 267     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 268     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 269     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 270     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 271     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 272     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 273     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 274     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 275     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 276     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 277     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 278     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 279     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 280     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 281     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 282     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 283     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 284     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 285     0x20, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
 286     0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
 287     0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
 288     0xD8, 0xD9, 0xDA, 0x20, 0x20, 0x20, 0x20, 0x20,
 289     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 290     0xE8, 0xE9, 0xEA, 0x20, 0x20, 0x20, 0x20, 0x20,
 291     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 292     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 293 };
 294
 295 static const uint8_t charMap_8859_7[] = {
 296     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 297     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 298     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 299     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 300     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 301     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 302     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 303     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 304     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 305     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 306     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 307     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 308     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 309     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 310     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 311     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 312     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 313     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 314     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 315     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 316     0x20, 0xA1, 0xA2, 0x20, 0x20, 0x20, 0x20, 0x20,
 317     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 318     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xDC, 0x20,
 319     0xDD, 0xDE, 0xDF, 0x20, 0xFC, 0x20, 0xFD, 0xFE,
 320     0xC0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 321     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 322     0xF0, 0xF1, 0x20, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
 323     0xF8, 0xF9, 0xFA, 0xFB, 0xDC, 0xDD, 0xDE, 0xDF,
 324     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 325     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 326     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
 327     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20,
 328 };
 329
 330 static const uint8_t charMap_8859_8[] = {
 331     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 332     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 333     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 334     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 335     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 336     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 337     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 338     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 339     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 340     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 341     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 342     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 343     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 344     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 345     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 346     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 347     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 348     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 349     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 350     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 351     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 352     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 353     0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
 354     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 355     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 356     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 357     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 358     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 359     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 360     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 361     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
 362     0xF8, 0xF9, 0xFA, 0x20, 0x20, 0x20, 0x20, 0x20,
 363 };
 364
 365 static const uint8_t charMap_8859_9[] = {
 366     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 367     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 368     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 369     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 370     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 371     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 372     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 373     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 374     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 375     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 376     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 377     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 378     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 379     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 380     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 381     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 382     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 383     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 384     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 385     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 386     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 387     0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20,
 388     0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
 389     0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20,
 390     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 391     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 392     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
 393     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x69, 0xFE, 0xDF,
 394     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 395     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 396     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
 397     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
 398 };
 399
 400 static const int32_t ngrams_windows_1251[] = {
 401     0x20E220, 0x20E2EE, 0x20E4EE, 0x20E7E0, 0x20E820, 0x20EAE0, 0x20EAEE, 0x20EDE0, 0x20EDE5, 0x20EEE1, 0x20EFEE, 0x20EFF0, 0x20F0E0, 0x20F1EE, 0x20F1F2, 0x20F2EE,
 402     0x20F7F2, 0x20FDF2, 0xE0EDE8, 0xE0F2FC, 0xE3EE20, 0xE5EBFC, 0xE5EDE8, 0xE5F1F2, 0xE5F220, 0xE820EF, 0xE8E520, 0xE8E820, 0xE8FF20, 0xEBE5ED, 0xEBE820, 0xEBFCED,
 403     0xEDE020, 0xEDE520, 0xEDE8E5, 0xEDE8FF, 0xEDEE20, 0xEDEEE2, 0xEE20E2, 0xEE20EF, 0xEE20F1, 0xEEE220, 0xEEE2E0, 0xEEE3EE, 0xEEE920, 0xEEEBFC, 0xEEEC20, 0xEEF1F2,
 404     0xEFEEEB, 0xEFF0E5, 0xEFF0E8, 0xEFF0EE, 0xF0E0E2, 0xF0E5E4, 0xF1F2E0, 0xF1F2E2, 0xF1F2E8, 0xF1FF20, 0xF2E5EB, 0xF2EE20, 0xF2EEF0, 0xF2FC20, 0xF7F2EE, 0xFBF520,
 405 };
 406
 407 static const uint8_t charMap_windows_1251[] = {
 408     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 409     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 410     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 411     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 412     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 413     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 414     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 415     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 416     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 417     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 418     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 419     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 420     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 421     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 422     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 423     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 424     0x90, 0x83, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20,
 425     0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F,
 426     0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 427     0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F,
 428     0x20, 0xA2, 0xA2, 0xBC, 0x20, 0xB4, 0x20, 0x20,
 429     0xB8, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0xBF,
 430     0x20, 0x20, 0xB3, 0xB3, 0xB4, 0xB5, 0x20, 0x20,
 431     0xB8, 0x20, 0xBA, 0x20, 0xBC, 0xBE, 0xBE, 0xBF,
 432     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 433     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 434     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
 435     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
 436     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 437     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 438     0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
 439     0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
 440 };
 441
 442 static const int32_t ngrams_windows_1256[] = {
 443     0x20C7E1, 0x20C7E4, 0x20C8C7, 0x20DAE1, 0x20DDED, 0x20E1E1, 0x20E3E4, 0x20E6C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E120, 0xC7E1C3, 0xC7E1C7, 0xC7E1C8,
 444     0xC7E1CA, 0xC7E1CC, 0xC7E1CD, 0xC7E1CF, 0xC7E1D3, 0xC7E1DA, 0xC7E1DE, 0xC7E1E3, 0xC7E1E6, 0xC7E1ED, 0xC7E320, 0xC7E420, 0xC7E4CA, 0xC820C7, 0xC920C7, 0xC920DD,
 445     0xC920E1, 0xC920E3, 0xC920E6, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xDA20C7, 0xDAE1EC, 0xDDED20, 0xE120C7, 0xE1C920, 0xE1EC20, 0xE1ED20,
 446     0xE320C7, 0xE3C720, 0xE3C920, 0xE3E420, 0xE420C7, 0xE520C7, 0xE5C720, 0xE6C7E1, 0xE6E420, 0xEC20C7, 0xED20C7, 0xED20E3, 0xED20E6, 0xEDC920, 0xEDD120, 0xEDE420,
 447 };
 448
 449 static const uint8_t charMap_windows_1256[] = {
 450     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 451     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 452     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 453     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 454     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 455     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 456     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 457     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 458     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 459     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 460     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 461     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 462     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 463     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 464     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 465     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 466     0x20, 0x81, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20,
 467     0x88, 0x20, 0x8A, 0x20, 0x9C, 0x8D, 0x8E, 0x8F,
 468     0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 469     0x98, 0x20, 0x9A, 0x20, 0x9C, 0x20, 0x20, 0x9F,
 470     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 471     0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20,
 472     0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
 473     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 474     0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
 475     0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
 476     0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0x20,
 477     0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
 478     0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
 479     0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
 480     0x20, 0x20, 0x20, 0x20, 0xF4, 0x20, 0x20, 0x20,
 481     0x20, 0xF9, 0x20, 0xFB, 0xFC, 0x20, 0x20, 0xFF,
 482 };
 483
 484 static const int32_t ngrams_KOI8_R[] = {
 485     0x20C4CF, 0x20C920, 0x20CBC1, 0x20CBCF, 0x20CEC1, 0x20CEC5, 0x20CFC2, 0x20D0CF, 0x20D0D2, 0x20D2C1, 0x20D3CF, 0x20D3D4, 0x20D4CF, 0x20D720, 0x20D7CF, 0x20DAC1,
 486     0x20DCD4, 0x20DED4, 0xC1CEC9, 0xC1D4D8, 0xC5CCD8, 0xC5CEC9, 0xC5D3D4, 0xC5D420, 0xC7CF20, 0xC920D0, 0xC9C520, 0xC9C920, 0xC9D120, 0xCCC5CE, 0xCCC920, 0xCCD8CE,
 487     0xCEC120, 0xCEC520, 0xCEC9C5, 0xCEC9D1, 0xCECF20, 0xCECFD7, 0xCF20D0, 0xCF20D3, 0xCF20D7, 0xCFC7CF, 0xCFCA20, 0xCFCCD8, 0xCFCD20, 0xCFD3D4, 0xCFD720, 0xCFD7C1,
 488     0xD0CFCC, 0xD0D2C5, 0xD0D2C9, 0xD0D2CF, 0xD2C1D7, 0xD2C5C4, 0xD3D120, 0xD3D4C1, 0xD3D4C9, 0xD3D4D7, 0xD4C5CC, 0xD4CF20, 0xD4CFD2, 0xD4D820, 0xD9C820, 0xDED4CF,
 489 };
 490
 491 static const uint8_t charMap_KOI8_R[] = {
 492     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 493     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 494     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 495     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 496     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
 497     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 498     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 499     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 500     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 501     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 502     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 503     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 504     0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
 505     0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
 506     0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
 507     0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
 508     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 509     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 510     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 511     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 512     0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20,
 513     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 514     0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20,
 515     0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
 516     0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
 517     0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
 518     0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
 519     0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
 520     0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
 521     0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
 522     0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
 523     0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
 524 };
 525
 526 static const int32_t ngrams_IBM424_he_rtl[] = {
 527     0x404146, 0x404148, 0x404151, 0x404171, 0x404251, 0x404256, 0x404541, 0x404546, 0x404551, 0x404556, 0x404562, 0x404569, 0x404571, 0x405441, 0x405445, 0x405641,
 528     0x406254, 0x406954, 0x417140, 0x454041, 0x454042, 0x454045, 0x454054, 0x454056, 0x454069, 0x454641, 0x464140, 0x465540, 0x465740, 0x466840, 0x467140, 0x514045,
 529     0x514540, 0x514671, 0x515155, 0x515540, 0x515740, 0x516840, 0x517140, 0x544041, 0x544045, 0x544140, 0x544540, 0x554041, 0x554042, 0x554045, 0x554054, 0x554056,
 530     0x554069, 0x564540, 0x574045, 0x584540, 0x585140, 0x585155, 0x625440, 0x684045, 0x685155, 0x695440, 0x714041, 0x714042, 0x714045, 0x714054, 0x714056, 0x714069,
 531 };
 532
 533 static const int32_t ngrams_IBM424_he_ltr[] = {
 534     0x404146, 0x404154, 0x404551, 0x404554, 0x404556, 0x404558, 0x405158, 0x405462, 0x405469, 0x405546, 0x405551, 0x405746, 0x405751, 0x406846, 0x406851, 0x407141,
 535     0x407146, 0x407151, 0x414045, 0x414054, 0x414055, 0x414071, 0x414540, 0x414645, 0x415440, 0x415640, 0x424045, 0x424055, 0x424071, 0x454045, 0x454051, 0x454054,
 536     0x454055, 0x454057, 0x454068, 0x454071, 0x455440, 0x464140, 0x464540, 0x484140, 0x514140, 0x514240, 0x514540, 0x544045, 0x544055, 0x544071, 0x546240, 0x546940,
 537     0x555151, 0x555158, 0x555168, 0x564045, 0x564055, 0x564071, 0x564240, 0x564540, 0x624540, 0x694045, 0x694055, 0x694071, 0x694540, 0x714140, 0x714540, 0x714651,
 538 };
 539
 540 static const uint8_t charMap_IBM424_he[] = {
 541 /*           -0    -1    -2    -3    -4    -5    -6    -7    -8    -9    -A    -B    -C    -D    -E    -F   */
 542 /* 0- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 543 /* 1- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 544 /* 2- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 545 /* 3- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 546 /* 4- */    0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 547 /* 5- */    0x40, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 548 /* 6- */    0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 549 /* 7- */    0x40, 0x71, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x00, 0x40, 0x40,
 550 /* 8- */    0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 551 /* 9- */    0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 552 /* A- */    0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 553 /* B- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 554 /* C- */    0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 555 /* D- */    0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 556 /* E- */    0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 557 /* F- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 558 };
 559
 560 static const int32_t ngrams_IBM420_ar_rtl[] = {
 561     0x4056B1, 0x4056BD, 0x405856, 0x409AB1, 0x40ABDC, 0x40B1B1, 0x40BBBD, 0x40CF56, 0x564056, 0x564640, 0x566340, 0x567540, 0x56B140, 0x56B149, 0x56B156, 0x56B158,
 562     0x56B163, 0x56B167, 0x56B169, 0x56B173, 0x56B178, 0x56B19A, 0x56B1AD, 0x56B1BB, 0x56B1CF, 0x56B1DC, 0x56BB40, 0x56BD40, 0x56BD63, 0x584056, 0x624056, 0x6240AB,
 563     0x6240B1, 0x6240BB, 0x6240CF, 0x634056, 0x734056, 0x736240, 0x754056, 0x756240, 0x784056, 0x9A4056, 0x9AB1DA, 0xABDC40, 0xB14056, 0xB16240, 0xB1DA40, 0xB1DC40,
 564     0xBB4056, 0xBB5640, 0xBB6240, 0xBBBD40, 0xBD4056, 0xBF4056, 0xBF5640, 0xCF56B1, 0xCFBD40, 0xDA4056, 0xDC4056, 0xDC40BB, 0xDC40CF, 0xDC6240, 0xDC7540, 0xDCBD40,
 565 };
 566
 567 static const int32_t ngrams_IBM420_ar_ltr[] = {
 568     0x404656, 0x4056BB, 0x4056BF, 0x406273, 0x406275, 0x4062B1, 0x4062BB, 0x4062DC, 0x406356, 0x407556, 0x4075DC, 0x40B156, 0x40BB56, 0x40BD56, 0x40BDBB, 0x40BDCF,
 569     0x40BDDC, 0x40DAB1, 0x40DCAB, 0x40DCB1, 0x49B156, 0x564056, 0x564058, 0x564062, 0x564063, 0x564073, 0x564075, 0x564078, 0x56409A, 0x5640B1, 0x5640BB, 0x5640BD,
 570     0x5640BF, 0x5640DA, 0x5640DC, 0x565840, 0x56B156, 0x56CF40, 0x58B156, 0x63B156, 0x63BD56, 0x67B156, 0x69B156, 0x73B156, 0x78B156, 0x9AB156, 0xAB4062, 0xADB156,
 571     0xB14062, 0xB15640, 0xB156CF, 0xB19A40, 0xB1B140, 0xBB4062, 0xBB40DC, 0xBBB156, 0xBD5640, 0xBDBB40, 0xCF4062, 0xCF40DC, 0xCFB156, 0xDAB19A, 0xDCAB40, 0xDCB156
 572 };
 573
 574 static const uint8_t charMap_IBM420_ar[]= {
 575 /*           -0    -1    -2    -3    -4    -5    -6    -7    -8    -9    -A    -B    -C    -D    -E    -F   */
 576 /* 0- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 577 /* 1- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 578 /* 2- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 579 /* 3- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 580 /* 4- */    0x40, 0x40, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 581 /* 5- */    0x40, 0x51, 0x52, 0x40, 0x40, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 582 /* 6- */    0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 583 /* 7- */    0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
 584 /* 8- */    0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F,
 585 /* 9- */    0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F,
 586 /* A- */    0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF,
 587 /* B- */    0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0x40, 0x40, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF,
 588 /* C- */    0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0xCB, 0x40, 0xCD, 0x40, 0xCF,
 589 /* D- */    0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
 590 /* E- */    0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xEA, 0xEB, 0x40, 0xED, 0xEE, 0xEF,
 591 /* F- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0xFB, 0xFC, 0xFD, 0xFE, 0x40,
 592 };
 593
 594 //ISO-8859-1,2,5,6,7,8,9 Ngrams
 595
 596 struct NGramsPlusLang {
 597     const int32_t ngrams[64];
 598     const char *  lang;
 599 };
 600
 601 static const NGramsPlusLang ngrams_8859_1[] =  {
 602   {
 603     {
 604     0x206120, 0x20616E, 0x206265, 0x20636F, 0x20666F, 0x206861, 0x206865, 0x20696E, 0x206D61, 0x206F66, 0x207072, 0x207265, 0x207361, 0x207374, 0x207468, 0x20746F,
 605     0x207768, 0x616964, 0x616C20, 0x616E20, 0x616E64, 0x617320, 0x617420, 0x617465, 0x617469, 0x642061, 0x642074, 0x652061, 0x652073, 0x652074, 0x656420, 0x656E74,
 606     0x657220, 0x657320, 0x666F72, 0x686174, 0x686520, 0x686572, 0x696420, 0x696E20, 0x696E67, 0x696F6E, 0x697320, 0x6E2061, 0x6E2074, 0x6E6420, 0x6E6720, 0x6E7420,
 607     0x6F6620, 0x6F6E20, 0x6F7220, 0x726520, 0x727320, 0x732061, 0x732074, 0x736169, 0x737420, 0x742074, 0x746572, 0x746861, 0x746865, 0x74696F, 0x746F20, 0x747320,
 608     },
 609     "en"
 610   },
 611   {
 612     {
 613     0x206166, 0x206174, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207369, 0x207374, 0x207469, 0x207669, 0x616620,
 614     0x616E20, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646572, 0x646574, 0x652073, 0x656420, 0x656465, 0x656E20, 0x656E64, 0x657220, 0x657265, 0x657320,
 615     0x657420, 0x666F72, 0x676520, 0x67656E, 0x676572, 0x696765, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6572, 0x6C6967, 0x6C6C65, 0x6D6564, 0x6E6465, 0x6E6520,
 616     0x6E6720, 0x6E6765, 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722064, 0x722065, 0x722073, 0x726520, 0x737465, 0x742073, 0x746520, 0x746572, 0x74696C, 0x766572,
 617     },
 618     "da"
 619   },
 620   {
 621     {
 622     0x20616E, 0x206175, 0x206265, 0x206461, 0x206465, 0x206469, 0x206569, 0x206765, 0x206861, 0x20696E, 0x206D69, 0x207363, 0x207365, 0x20756E, 0x207665, 0x20766F,
 623     0x207765, 0x207A75, 0x626572, 0x636820, 0x636865, 0x636874, 0x646173, 0x64656E, 0x646572, 0x646965, 0x652064, 0x652073, 0x65696E, 0x656974, 0x656E20, 0x657220,
 624     0x657320, 0x67656E, 0x68656E, 0x687420, 0x696368, 0x696520, 0x696E20, 0x696E65, 0x697420, 0x6C6963, 0x6C6C65, 0x6E2061, 0x6E2064, 0x6E2073, 0x6E6420, 0x6E6465,
 625     0x6E6520, 0x6E6720, 0x6E6765, 0x6E7465, 0x722064, 0x726465, 0x726569, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x756E64, 0x756E67, 0x766572,
 626     },
 627     "de"
 628   },
 629   {
 630     {
 631     0x206120, 0x206361, 0x20636F, 0x206465, 0x20656C, 0x20656E, 0x206573, 0x20696E, 0x206C61, 0x206C6F, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365,
 632     0x20756E, 0x207920, 0x612063, 0x612064, 0x612065, 0x61206C, 0x612070, 0x616369, 0x61646F, 0x616C20, 0x617220, 0x617320, 0x6369F3, 0x636F6E, 0x646520, 0x64656C,
 633     0x646F20, 0x652064, 0x652065, 0x65206C, 0x656C20, 0x656E20, 0x656E74, 0x657320, 0x657374, 0x69656E, 0x69F36E, 0x6C6120, 0x6C6F73, 0x6E2065, 0x6E7465, 0x6F2064,
 634     0x6F2065, 0x6F6E20, 0x6F7220, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732064, 0x732065, 0x732070, 0x736520, 0x746520, 0x746F20, 0x756520, 0xF36E20,
 635     },
 636     "es"
 637   },
 638   {
 639     {
 640     0x206175, 0x20636F, 0x206461, 0x206465, 0x206475, 0x20656E, 0x206574, 0x206C61, 0x206C65, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207365, 0x20736F, 0x20756E,
 641     0x20E020, 0x616E74, 0x617469, 0x636520, 0x636F6E, 0x646520, 0x646573, 0x647520, 0x652061, 0x652063, 0x652064, 0x652065, 0x65206C, 0x652070, 0x652073, 0x656E20,
 642     0x656E74, 0x657220, 0x657320, 0x657420, 0x657572, 0x696F6E, 0x697320, 0x697420, 0x6C6120, 0x6C6520, 0x6C6573, 0x6D656E, 0x6E2064, 0x6E6520, 0x6E7320, 0x6E7420,
 643     0x6F6E20, 0x6F6E74, 0x6F7572, 0x717565, 0x72206C, 0x726520, 0x732061, 0x732064, 0x732065, 0x73206C, 0x732070, 0x742064, 0x746520, 0x74696F, 0x756520, 0x757220,
 644     },
 645     "fr"
 646   },
 647   {
 648     {
 649     0x20616C, 0x206368, 0x20636F, 0x206465, 0x206469, 0x206520, 0x20696C, 0x20696E, 0x206C61, 0x207065, 0x207072, 0x20756E, 0x612063, 0x612064, 0x612070, 0x612073,
 650     0x61746F, 0x636865, 0x636F6E, 0x64656C, 0x646920, 0x652061, 0x652063, 0x652064, 0x652069, 0x65206C, 0x652070, 0x652073, 0x656C20, 0x656C6C, 0x656E74, 0x657220,
 651     0x686520, 0x692061, 0x692063, 0x692064, 0x692073, 0x696120, 0x696C20, 0x696E20, 0x696F6E, 0x6C6120, 0x6C6520, 0x6C6920, 0x6C6C61, 0x6E6520, 0x6E6920, 0x6E6F20,
 652     0x6E7465, 0x6F2061, 0x6F2064, 0x6F2069, 0x6F2073, 0x6F6E20, 0x6F6E65, 0x706572, 0x726120, 0x726520, 0x736920, 0x746120, 0x746520, 0x746920, 0x746F20, 0x7A696F,
 653     },
 654     "it"
 655   },
 656   {
 657     {
 658     0x20616C, 0x206265, 0x206461, 0x206465, 0x206469, 0x206565, 0x20656E, 0x206765, 0x206865, 0x20696E, 0x206D61, 0x206D65, 0x206F70, 0x207465, 0x207661, 0x207665,
 659     0x20766F, 0x207765, 0x207A69, 0x61616E, 0x616172, 0x616E20, 0x616E64, 0x617220, 0x617420, 0x636874, 0x646520, 0x64656E, 0x646572, 0x652062, 0x652076, 0x65656E,
 660     0x656572, 0x656E20, 0x657220, 0x657273, 0x657420, 0x67656E, 0x686574, 0x696520, 0x696E20, 0x696E67, 0x697320, 0x6E2062, 0x6E2064, 0x6E2065, 0x6E2068, 0x6E206F,
 661     0x6E2076, 0x6E6465, 0x6E6720, 0x6F6E64, 0x6F6F72, 0x6F7020, 0x6F7220, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x76616E, 0x766572, 0x766F6F,
 662     },
 663     "nl"
 664   },
 665   {
 666     {
 667     0x206174, 0x206176, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207365, 0x20736B, 0x20736F, 0x207374, 0x207469,
 668     0x207669, 0x20E520, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646574, 0x652073, 0x656420, 0x656E20, 0x656E65, 0x657220, 0x657265, 0x657420, 0x657474,
 669     0x666F72, 0x67656E, 0x696B6B, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6520, 0x6C6C65, 0x6D6564, 0x6D656E, 0x6E2073, 0x6E6520, 0x6E6720, 0x6E6765, 0x6E6E65,
 670     0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722073, 0x726520, 0x736F6D, 0x737465, 0x742073, 0x746520, 0x74656E, 0x746572, 0x74696C, 0x747420, 0x747465, 0x766572,
 671     },
 672     "no"
 673   },
 674   {
 675     {
 676     0x206120, 0x20636F, 0x206461, 0x206465, 0x20646F, 0x206520, 0x206573, 0x206D61, 0x206E6F, 0x206F20, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365,
 677     0x20756D, 0x612061, 0x612063, 0x612064, 0x612070, 0x616465, 0x61646F, 0x616C20, 0x617220, 0x617261, 0x617320, 0x636F6D, 0x636F6E, 0x646120, 0x646520, 0x646F20,
 678     0x646F73, 0x652061, 0x652064, 0x656D20, 0x656E74, 0x657320, 0x657374, 0x696120, 0x696361, 0x6D656E, 0x6E7465, 0x6E746F, 0x6F2061, 0x6F2063, 0x6F2064, 0x6F2065,
 679     0x6F2070, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732061, 0x732064, 0x732065, 0x732070, 0x737461, 0x746520, 0x746F20, 0x756520, 0xE36F20, 0xE7E36F,
 680     },
 681     "pt"
 682   },
 683   {
 684     {
 685     0x206174, 0x206176, 0x206465, 0x20656E, 0x2066F6, 0x206861, 0x206920, 0x20696E, 0x206B6F, 0x206D65, 0x206F63, 0x2070E5, 0x20736B, 0x20736F, 0x207374, 0x207469,
 686     0x207661, 0x207669, 0x20E472, 0x616465, 0x616E20, 0x616E64, 0x617220, 0x617474, 0x636820, 0x646520, 0x64656E, 0x646572, 0x646574, 0x656420, 0x656E20, 0x657220,
 687     0x657420, 0x66F672, 0x67656E, 0x696C6C, 0x696E67, 0x6B6120, 0x6C6C20, 0x6D6564, 0x6E2073, 0x6E6120, 0x6E6465, 0x6E6720, 0x6E6765, 0x6E696E, 0x6F6368, 0x6F6D20,
 688     0x6F6E20, 0x70E520, 0x722061, 0x722073, 0x726120, 0x736B61, 0x736F6D, 0x742073, 0x746120, 0x746520, 0x746572, 0x74696C, 0x747420, 0x766172, 0xE47220, 0xF67220,
 689     },
 690     "sv"
 691   }
 692 };
 693
 694
 695 static const NGramsPlusLang ngrams_8859_2[] =  {
 696   {
 697     {
 698     0x206120, 0x206279, 0x20646F, 0x206A65, 0x206E61, 0x206E65, 0x206F20, 0x206F64, 0x20706F, 0x207072, 0x2070F8, 0x20726F, 0x207365, 0x20736F, 0x207374, 0x20746F,
 699     0x207620, 0x207679, 0x207A61, 0x612070, 0x636520, 0x636820, 0x652070, 0x652073, 0x652076, 0x656D20, 0x656EED, 0x686F20, 0x686F64, 0x697374, 0x6A6520, 0x6B7465,
 700     0x6C6520, 0x6C6920, 0x6E6120, 0x6EE920, 0x6EEC20, 0x6EED20, 0x6F2070, 0x6F646E, 0x6F6A69, 0x6F7374, 0x6F7520, 0x6F7661, 0x706F64, 0x706F6A, 0x70726F, 0x70F865,
 701     0x736520, 0x736F75, 0x737461, 0x737469, 0x73746E, 0x746572, 0x746EED, 0x746F20, 0x752070, 0xBE6520, 0xE16EED, 0xE9686F, 0xED2070, 0xED2073, 0xED6D20, 0xF86564,
 702     },
 703     "cs"
 704   },
 705   {
 706     {
 707     0x206120, 0x20617A, 0x206265, 0x206567, 0x20656C, 0x206665, 0x206861, 0x20686F, 0x206973, 0x206B65, 0x206B69, 0x206BF6, 0x206C65, 0x206D61, 0x206D65, 0x206D69,
 708     0x206E65, 0x20737A, 0x207465, 0x20E973, 0x612061, 0x61206B, 0x61206D, 0x612073, 0x616B20, 0x616E20, 0x617A20, 0x62616E, 0x62656E, 0x656779, 0x656B20, 0x656C20,
 709     0x656C65, 0x656D20, 0x656E20, 0x657265, 0x657420, 0x657465, 0x657474, 0x677920, 0x686F67, 0x696E74, 0x697320, 0x6B2061, 0x6BF67A, 0x6D6567, 0x6D696E, 0x6E2061,
 710     0x6E616B, 0x6E656B, 0x6E656D, 0x6E7420, 0x6F6779, 0x732061, 0x737A65, 0x737A74, 0x737AE1, 0x73E967, 0x742061, 0x747420, 0x74E173, 0x7A6572, 0xE16E20, 0xE97320,
 711     },
 712     "hu"
 713   },
 714   {
 715     {
 716     0x20637A, 0x20646F, 0x206920, 0x206A65, 0x206B6F, 0x206D61, 0x206D69, 0x206E61, 0x206E69, 0x206F64, 0x20706F, 0x207072, 0x207369, 0x207720, 0x207769, 0x207779,
 717     0x207A20, 0x207A61, 0x612070, 0x612077, 0x616E69, 0x636820, 0x637A65, 0x637A79, 0x646F20, 0x647A69, 0x652070, 0x652073, 0x652077, 0x65207A, 0x65676F, 0x656A20,
 718     0x656D20, 0x656E69, 0x676F20, 0x696120, 0x696520, 0x69656A, 0x6B6120, 0x6B6920, 0x6B6965, 0x6D6965, 0x6E6120, 0x6E6961, 0x6E6965, 0x6F2070, 0x6F7761, 0x6F7769,
 719     0x706F6C, 0x707261, 0x70726F, 0x70727A, 0x727A65, 0x727A79, 0x7369EA, 0x736B69, 0x737461, 0x776965, 0x796368, 0x796D20, 0x7A6520, 0x7A6965, 0x7A7920, 0xF37720,
 720     },
 721     "pl"
 722   },
 723   {
 724     {
 725     0x206120, 0x206163, 0x206361, 0x206365, 0x20636F, 0x206375, 0x206465, 0x206469, 0x206C61, 0x206D61, 0x207065, 0x207072, 0x207365, 0x2073E3, 0x20756E, 0x20BA69,
 726     0x20EE6E, 0x612063, 0x612064, 0x617265, 0x617420, 0x617465, 0x617520, 0x636172, 0x636F6E, 0x637520, 0x63E320, 0x646520, 0x652061, 0x652063, 0x652064, 0x652070,
 727     0x652073, 0x656120, 0x656920, 0x656C65, 0x656E74, 0x657374, 0x692061, 0x692063, 0x692064, 0x692070, 0x696520, 0x696920, 0x696E20, 0x6C6120, 0x6C6520, 0x6C6F72,
 728     0x6C7569, 0x6E6520, 0x6E7472, 0x6F7220, 0x70656E, 0x726520, 0x726561, 0x727520, 0x73E320, 0x746520, 0x747275, 0x74E320, 0x756920, 0x756C20, 0xBA6920, 0xEE6E20,
 729     },
 730     "ro"
 731   }
 732 };
 733
 734 static const int32_t ngrams_8859_5_ru[] = {
 735     0x20D220, 0x20D2DE, 0x20D4DE, 0x20D7D0, 0x20D820, 0x20DAD0, 0x20DADE, 0x20DDD0, 0x20DDD5, 0x20DED1, 0x20DFDE, 0x20DFE0, 0x20E0D0, 0x20E1DE, 0x20E1E2, 0x20E2DE,
 736     0x20E7E2, 0x20EDE2, 0xD0DDD8, 0xD0E2EC, 0xD3DE20, 0xD5DBEC, 0xD5DDD8, 0xD5E1E2, 0xD5E220, 0xD820DF, 0xD8D520, 0xD8D820, 0xD8EF20, 0xDBD5DD, 0xDBD820, 0xDBECDD,
 737     0xDDD020, 0xDDD520, 0xDDD8D5, 0xDDD8EF, 0xDDDE20, 0xDDDED2, 0xDE20D2, 0xDE20DF, 0xDE20E1, 0xDED220, 0xDED2D0, 0xDED3DE, 0xDED920, 0xDEDBEC, 0xDEDC20, 0xDEE1E2,
 738     0xDFDEDB, 0xDFE0D5, 0xDFE0D8, 0xDFE0DE, 0xE0D0D2, 0xE0D5D4, 0xE1E2D0, 0xE1E2D2, 0xE1E2D8, 0xE1EF20, 0xE2D5DB, 0xE2DE20, 0xE2DEE0, 0xE2EC20, 0xE7E2DE, 0xEBE520,
 739 };
 740
 741 static const int32_t ngrams_8859_6_ar[] = {
 742     0x20C7E4, 0x20C7E6, 0x20C8C7, 0x20D9E4, 0x20E1EA, 0x20E4E4, 0x20E5E6, 0x20E8C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E420, 0xC7E4C3, 0xC7E4C7, 0xC7E4C8,
 743     0xC7E4CA, 0xC7E4CC, 0xC7E4CD, 0xC7E4CF, 0xC7E4D3, 0xC7E4D9, 0xC7E4E2, 0xC7E4E5, 0xC7E4E8, 0xC7E4EA, 0xC7E520, 0xC7E620, 0xC7E6CA, 0xC820C7, 0xC920C7, 0xC920E1,
 744     0xC920E4, 0xC920E5, 0xC920E8, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xD920C7, 0xD9E4E9, 0xE1EA20, 0xE420C7, 0xE4C920, 0xE4E920, 0xE4EA20,
 745     0xE520C7, 0xE5C720, 0xE5C920, 0xE5E620, 0xE620C7, 0xE720C7, 0xE7C720, 0xE8C7E4, 0xE8E620, 0xE920C7, 0xEA20C7, 0xEA20E5, 0xEA20E8, 0xEAC920, 0xEAD120, 0xEAE620,
 746 };
 747
 748 static const int32_t ngrams_8859_7_el[] = {
 749     0x20E1ED, 0x20E1F0, 0x20E3E9, 0x20E4E9, 0x20E5F0, 0x20E720, 0x20EAE1, 0x20ECE5, 0x20EDE1, 0x20EF20, 0x20F0E1, 0x20F0EF, 0x20F0F1, 0x20F3F4, 0x20F3F5, 0x20F4E7,
 750     0x20F4EF, 0xDFE120, 0xE120E1, 0xE120F4, 0xE1E920, 0xE1ED20, 0xE1F0FC, 0xE1F220, 0xE3E9E1, 0xE5E920, 0xE5F220, 0xE720F4, 0xE7ED20, 0xE7F220, 0xE920F4, 0xE9E120,
 751     0xE9EADE, 0xE9F220, 0xEAE1E9, 0xEAE1F4, 0xECE520, 0xED20E1, 0xED20E5, 0xED20F0, 0xEDE120, 0xEFF220, 0xEFF520, 0xF0EFF5, 0xF0F1EF, 0xF0FC20, 0xF220E1, 0xF220E5,
 752     0xF220EA, 0xF220F0, 0xF220F4, 0xF3E520, 0xF3E720, 0xF3F4EF, 0xF4E120, 0xF4E1E9, 0xF4E7ED, 0xF4E7F2, 0xF4E9EA, 0xF4EF20, 0xF4EFF5, 0xF4F9ED, 0xF9ED20, 0xFEED20,
 753 };
 754
 755 static const int32_t ngrams_8859_8_I_he[] = {
 756     0x20E0E5, 0x20E0E7, 0x20E0E9, 0x20E0FA, 0x20E1E9, 0x20E1EE, 0x20E4E0, 0x20E4E5, 0x20E4E9, 0x20E4EE, 0x20E4F2, 0x20E4F9, 0x20E4FA, 0x20ECE0, 0x20ECE4, 0x20EEE0,
 757     0x20F2EC, 0x20F9EC, 0xE0FA20, 0xE420E0, 0xE420E1, 0xE420E4, 0xE420EC, 0xE420EE, 0xE420F9, 0xE4E5E0, 0xE5E020, 0xE5ED20, 0xE5EF20, 0xE5F820, 0xE5FA20, 0xE920E4,
 758     0xE9E420, 0xE9E5FA, 0xE9E9ED, 0xE9ED20, 0xE9EF20, 0xE9F820, 0xE9FA20, 0xEC20E0, 0xEC20E4, 0xECE020, 0xECE420, 0xED20E0, 0xED20E1, 0xED20E4, 0xED20EC, 0xED20EE,
 759     0xED20F9, 0xEEE420, 0xEF20E4, 0xF0E420, 0xF0E920, 0xF0E9ED, 0xF2EC20, 0xF820E4, 0xF8E9ED, 0xF9EC20, 0xFA20E0, 0xFA20E1, 0xFA20E4, 0xFA20EC, 0xFA20EE, 0xFA20F9,
 760 };
 761
 762 static const int32_t ngrams_8859_8_he[] = {
 763     0x20E0E5, 0x20E0EC, 0x20E4E9, 0x20E4EC, 0x20E4EE, 0x20E4F0, 0x20E9F0, 0x20ECF2, 0x20ECF9, 0x20EDE5, 0x20EDE9, 0x20EFE5, 0x20EFE9, 0x20F8E5, 0x20F8E9, 0x20FAE0,
 764     0x20FAE5, 0x20FAE9, 0xE020E4, 0xE020EC, 0xE020ED, 0xE020FA, 0xE0E420, 0xE0E5E4, 0xE0EC20, 0xE0EE20, 0xE120E4, 0xE120ED, 0xE120FA, 0xE420E4, 0xE420E9, 0xE420EC,
 765     0xE420ED, 0xE420EF, 0xE420F8, 0xE420FA, 0xE4EC20, 0xE5E020, 0xE5E420, 0xE7E020, 0xE9E020, 0xE9E120, 0xE9E420, 0xEC20E4, 0xEC20ED, 0xEC20FA, 0xECF220, 0xECF920,
 766     0xEDE9E9, 0xEDE9F0, 0xEDE9F8, 0xEE20E4, 0xEE20ED, 0xEE20FA, 0xEEE120, 0xEEE420, 0xF2E420, 0xF920E4, 0xF920ED, 0xF920FA, 0xF9E420, 0xFAE020, 0xFAE420, 0xFAE5E9,
 767 };
 768
 769 static const int32_t ngrams_8859_9_tr[] = {
 770     0x206261, 0x206269, 0x206275, 0x206461, 0x206465, 0x206765, 0x206861, 0x20696C, 0x206B61, 0x206B6F, 0x206D61, 0x206F6C, 0x207361, 0x207461, 0x207665, 0x207961,
 771     0x612062, 0x616B20, 0x616C61, 0x616D61, 0x616E20, 0x616EFD, 0x617220, 0x617261, 0x6172FD, 0x6173FD, 0x617961, 0x626972, 0x646120, 0x646520, 0x646920, 0x652062,
 772     0x65206B, 0x656469, 0x656E20, 0x657220, 0x657269, 0x657369, 0x696C65, 0x696E20, 0x696E69, 0x697220, 0x6C616E, 0x6C6172, 0x6C6520, 0x6C6572, 0x6E2061, 0x6E2062,
 773     0x6E206B, 0x6E6461, 0x6E6465, 0x6E6520, 0x6E6920, 0x6E696E, 0x6EFD20, 0x72696E, 0x72FD6E, 0x766520, 0x796120, 0x796F72, 0xFD6E20, 0xFD6E64, 0xFD6EFD, 0xFDF0FD,
 774 };
 775
 776 CharsetRecog_8859_1::~CharsetRecog_8859_1()
 777 {
 778     // nothing to do
 779 }
 780
 781 UBool CharsetRecog_8859_1::match(InputText *textIn, CharsetMatch *results) const {
 782     const char *name = textIn->fC1Bytes? "windows-1252" : "ISO-8859-1";
 783     uint32_t i;
 784     int32_t bestConfidenceSoFar = -1;
 785     for (i=0; i < ARRAY_SIZE(ngrams_8859_1) ; i++) {
 786         const int32_t *ngrams = ngrams_8859_1[i].ngrams;
 787         const char    *lang   = ngrams_8859_1[i].lang;
 788         int32_t confidence = match_sbcs(textIn, ngrams, charMap_8859_1);
 789         if (confidence > bestConfidenceSoFar) {
 790             results->set(textIn, this, confidence, name, lang);
 791             bestConfidenceSoFar = confidence;
 792         }
 793     }
 794     return (bestConfidenceSoFar > 0);
 795 }
 796
 797 const char *CharsetRecog_8859_1::getName() const
 798 {
 799     return "ISO-8859-1";
 800 }
 801
 802
 803 CharsetRecog_8859_2::~CharsetRecog_8859_2()
 804 {
 805     // nothing to do
 806 }
 807
 808 UBool CharsetRecog_8859_2::match(InputText *textIn, CharsetMatch *results) const {
 809     const char *name = textIn->fC1Bytes? "windows-1250" : "ISO-8859-2";
 810     uint32_t i;
 811     int32_t bestConfidenceSoFar = -1;
 812     for (i=0; i < ARRAY_SIZE(ngrams_8859_2) ; i++) {
 813         const int32_t *ngrams = ngrams_8859_2[i].ngrams;
 814         const char    *lang   = ngrams_8859_2[i].lang;
 815         int32_t confidence = match_sbcs(textIn, ngrams, charMap_8859_2);
 816         if (confidence > bestConfidenceSoFar) {
 817             results->set(textIn, this, confidence, name, lang);
 818             bestConfidenceSoFar = confidence;
 819         }
 820     }
 821     return (bestConfidenceSoFar > 0);
 822 }
 823
 824 const char *CharsetRecog_8859_2::getName() const
 825 {
 826     return "ISO-8859-2";
 827 }
 828
 829
 830 CharsetRecog_8859_5::~CharsetRecog_8859_5()
 831 {
 832     // nothing to do
 833 }
 834
 835 const char *CharsetRecog_8859_5::getName() const
 836 {
 837     return "ISO-8859-5";
 838 }
 839
 840 CharsetRecog_8859_5_ru::~CharsetRecog_8859_5_ru()
 841 {
 842     // nothing to do
 843 }
 844
 845 const char *CharsetRecog_8859_5_ru::getLanguage() const
 846 {
 847     return "ru";
 848 }
 849
 850 UBool CharsetRecog_8859_5_ru::match(InputText *textIn, CharsetMatch *results) const
 851 {
 852     int32_t confidence = match_sbcs(textIn, ngrams_8859_5_ru, charMap_8859_5);
 853     results->set(textIn, this, confidence);
 854     return (confidence > 0);
 855 }
 856
 857 CharsetRecog_8859_6::~CharsetRecog_8859_6()
 858 {
 859     // nothing to do
 860 }
 861
 862 const char *CharsetRecog_8859_6::getName() const
 863 {
 864     return "ISO-8859-6";
 865 }
 866
 867 CharsetRecog_8859_6_ar::~CharsetRecog_8859_6_ar()
 868 {
 869     // nothing to do
 870 }
 871
 872 const char *CharsetRecog_8859_6_ar::getLanguage() const
 873 {
 874     return "ar";
 875 }
 876
 877 UBool CharsetRecog_8859_6_ar::match(InputText *textIn, CharsetMatch *results) const
 878 {
 879     int32_t confidence = match_sbcs(textIn, ngrams_8859_6_ar, charMap_8859_6);
 880     results->set(textIn, this, confidence);
 881     return (confidence > 0);
 882 }
 883
 884 CharsetRecog_8859_7::~CharsetRecog_8859_7()
 885 {
 886     // nothing to do
 887 }
 888
 889 const char *CharsetRecog_8859_7::getName() const
 890 {
 891     return "ISO-8859-7";
 892 }
 893
 894 CharsetRecog_8859_7_el::~CharsetRecog_8859_7_el()
 895 {
 896     // nothing to do
 897 }
 898
 899 const char *CharsetRecog_8859_7_el::getLanguage() const
 900 {
 901     return "el";
 902 }
 903
 904 UBool CharsetRecog_8859_7_el::match(InputText *textIn, CharsetMatch *results) const
 905 {
 906     const char *name = textIn->fC1Bytes? "windows-1253" : "ISO-8859-7";
 907     int32_t confidence = match_sbcs(textIn, ngrams_8859_7_el, charMap_8859_7);
 908     results->set(textIn, this, confidence, name, "el");
 909     return (confidence > 0);
 910 }
 911
 912 CharsetRecog_8859_8::~CharsetRecog_8859_8()
 913 {
 914     // nothing to do
 915 }
 916
 917 const char *CharsetRecog_8859_8::getName() const
 918 {
 919     return "ISO-8859-8";
 920 }
 921
 922 CharsetRecog_8859_8_I_he::~CharsetRecog_8859_8_I_he ()
 923 {
 924     // nothing to do
 925 }
 926
 927 const char *CharsetRecog_8859_8_I_he::getName() const
 928 {
 929     return "ISO-8859-8-I";
 930 }
 931
 932 const char *CharsetRecog_8859_8_I_he::getLanguage() const
 933 {
 934     return "he";
 935 }
 936
 937 UBool CharsetRecog_8859_8_I_he::match(InputText *textIn, CharsetMatch *results) const
 938 {
 939     const char *name = textIn->fC1Bytes? "windows-1255" : "ISO-8859-8-I";
 940     int32_t confidence = match_sbcs(textIn, ngrams_8859_8_I_he, charMap_8859_8);
 941     results->set(textIn, this, confidence, name, "he");
 942     return (confidence > 0);
 943 }
 944
 945 CharsetRecog_8859_8_he::~CharsetRecog_8859_8_he()
 946 {
 947     // od ot gnihton
 948 }
 949
 950 const char *CharsetRecog_8859_8_he::getLanguage() const
 951 {
 952     return "he";
 953 }
 954
 955 UBool CharsetRecog_8859_8_he::match(InputText *textIn, CharsetMatch *results) const
 956 {
 957     const char *name = textIn->fC1Bytes? "windows-1255" : "ISO-8859-8";
 958     int32_t confidence = match_sbcs(textIn, ngrams_8859_8_he, charMap_8859_8);
 959     results->set(textIn, this, confidence, name, "he");
 960     return (confidence > 0);
 961 }
 962
 963 CharsetRecog_8859_9::~CharsetRecog_8859_9()
 964 {
 965     // nothing to do
 966 }
 967
 968 const char *CharsetRecog_8859_9::getName() const
 969 {
 970     return "ISO-8859-9";
 971 }
 972
 973 CharsetRecog_8859_9_tr::~CharsetRecog_8859_9_tr ()
 974 {
 975     // nothing to do
 976 }
 977
 978 const char *CharsetRecog_8859_9_tr::getLanguage() const
 979 {
 980     return "tr";
 981 }
 982
 983 UBool CharsetRecog_8859_9_tr::match(InputText *textIn, CharsetMatch *results) const
 984 {
 985     const char *name = textIn->fC1Bytes? "windows-1254" : "ISO-8859-9";
 986     int32_t confidence = match_sbcs(textIn, ngrams_8859_9_tr, charMap_8859_9);
 987     results->set(textIn, this, confidence, name, "tr");
 988     return (confidence > 0);
 989 }
 990
 991 CharsetRecog_windows_1256::~CharsetRecog_windows_1256()
 992 {
 993     // nothing to do
 994 }
 995
 996 const char *CharsetRecog_windows_1256::getName() const
 997 {
 998     return  "windows-1256";
 999 }
1000
1001 const char *CharsetRecog_windows_1256::getLanguage() const
1002 {
1003     return "ar";
1004 }
1005
1006 UBool CharsetRecog_windows_1256::match(InputText *textIn, CharsetMatch *results) const
1007 {
1008     int32_t confidence = match_sbcs(textIn, ngrams_windows_1256, charMap_windows_1256);
1009     results->set(textIn, this, confidence);
1010     return (confidence > 0);
1011 }
1012
1013 CharsetRecog_windows_1251::~CharsetRecog_windows_1251()
1014 {
1015     // nothing to do
1016 }
1017
1018 const char *CharsetRecog_windows_1251::getName() const
1019 {
1020     return  "windows-1251";
1021 }
1022
1023 const char *CharsetRecog_windows_1251::getLanguage() const
1024 {
1025     return "ru";
1026 }
1027
1028 UBool CharsetRecog_windows_1251::match(InputText *textIn, CharsetMatch *results) const
1029 {
1030     int32_t confidence = match_sbcs(textIn, ngrams_windows_1251, charMap_windows_1251);
1031     results->set(textIn, this, confidence);
1032     return (confidence > 0);
1033 }
1034
1035 CharsetRecog_KOI8_R::~CharsetRecog_KOI8_R()
1036 {
1037     // nothing to do
1038 }
1039
1040 const char *CharsetRecog_KOI8_R::getName() const
1041 {
1042     return  "KOI8-R";
1043 }
1044
1045 const char *CharsetRecog_KOI8_R::getLanguage() const
1046 {
1047     return "ru";
1048 }
1049
1050 UBool CharsetRecog_KOI8_R::match(InputText *textIn, CharsetMatch *results) const
1051 {
1052     int32_t confidence = match_sbcs(textIn, ngrams_KOI8_R, charMap_KOI8_R);
1053     results->set(textIn, this, confidence);
1054     return (confidence > 0);
1055 }
1056
1057 CharsetRecog_IBM424_he::~CharsetRecog_IBM424_he()
1058 {
1059     // nothing to do
1060 }
1061
1062 const char *CharsetRecog_IBM424_he::getLanguage() const
1063 {
1064     return "he";
1065 }
1066
1067 CharsetRecog_IBM424_he_rtl::~CharsetRecog_IBM424_he_rtl()
1068 {
1069     // nothing to do
1070 }
1071
1072 const char *CharsetRecog_IBM424_he_rtl::getName() const
1073 {
1074     return  "IBM424_rtl";
1075 }
1076
1077 UBool CharsetRecog_IBM424_he_rtl::match(InputText *textIn, CharsetMatch *results) const
1078 {
1079     int32_t confidence = match_sbcs(textIn, ngrams_IBM424_he_rtl, charMap_IBM424_he);
1080     results->set(textIn, this, confidence);
1081     return (confidence > 0);
1082 }
1083
1084 CharsetRecog_IBM424_he_ltr::~CharsetRecog_IBM424_he_ltr()
1085 {
1086     // nothing to do
1087 }
1088
1089 const char *CharsetRecog_IBM424_he_ltr::getName() const
1090 {
1091     return  "IBM424_ltr";
1092 }
1093
1094 UBool CharsetRecog_IBM424_he_ltr::match(InputText *textIn, CharsetMatch *results) const
1095 {
1096     int32_t confidence = match_sbcs(textIn, ngrams_IBM424_he_ltr, charMap_IBM424_he);
1097     results->set(textIn, this, confidence);
1098     return (confidence > 0);
1099 }
1100
1101 static const uint8_t unshapeMap_IBM420[] = {
1102 /*           -0    -1    -2    -3    -4    -5    -6    -7    -8    -9    -A    -B    -C    -D    -E    -F   */
1103 /* 0- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1104 /* 1- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1105 /* 2- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1106 /* 3- */    0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
1107 /* 4- */    0x40, 0x40, 0x42, 0x42, 0x44, 0x45, 0x46, 0x47, 0x47, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F,
1108 /* 5- */    0x50, 0x49, 0x52, 0x53, 0x54, 0x55, 0x56, 0x56, 0x58, 0x58, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F,
1109 /* 6- */    0x60, 0x61, 0x62, 0x63, 0x63, 0x65, 0x65, 0x67, 0x67, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
1110 /* 7- */    0x69, 0x71, 0x71, 0x73, 0x74, 0x75, 0x76, 0x77, 0x77, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F,
1111 /* 8- */    0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x80, 0x8B, 0x8B, 0x8D, 0x8D, 0x8F,
1112 /* 9- */    0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9A, 0x9A, 0x9A, 0x9E, 0x9E,
1113 /* A- */    0x9E, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x9E, 0xAB, 0xAB, 0xAD, 0xAD, 0xAF,
1114 /* B- */    0xAF, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xB1, 0xBB, 0xBB, 0xBD, 0xBD, 0xBF,
1115 /* C- */    0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xBF, 0xCC, 0xBF, 0xCE, 0xCF,
1116 /* D- */    0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xDA, 0xDC, 0xDC, 0xDC, 0xDF,
1117 /* E- */    0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
1118 /* F- */    0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
1119 };
1120
1121 CharsetRecog_IBM420_ar::~CharsetRecog_IBM420_ar()
1122 {
1123     // nothing to do
1124 }
1125
1126 const char *CharsetRecog_IBM420_ar::getLanguage() const
1127 {
1128     return "ar";
1129 }
1130
1131 void CharsetRecog_IBM420_ar::matchInit(InputText *textIn) {
1132     prev_fInputBytesLength = textIn->fInputLen;
1133     prev_fInputBytes = textIn->fInputBytes;
1134
1135     int32_t length = 0;
1136     uint8_t *bb = unshape(prev_fInputBytes, prev_fInputBytesLength, length);
1137
1138     if (bb != NULL) {
1139         textIn->fInputBytes = bb;
1140         textIn->fInputLen = length;
1141
1142         deleteBuffer = TRUE;
1143     } else {
1144         deleteBuffer = FALSE;
1145     }
1146 }
1147
1148 uint8_t *CharsetRecog_IBM420_ar::unshape(const uint8_t *inputBytes, int32_t inputBytesLength, int32_t &length) {
1149     uint8_t *resultArray = unshapeLamAlef(inputBytes, inputBytesLength, length);
1150
1151     if (resultArray != NULL) {
1152         for (int32_t i = 0; i < inputBytesLength; i++) {
1153             resultArray[i] = unshapeMap_IBM420[resultArray[i]];
1154         }
1155     }
1156
1157     return resultArray;
1158 }
1159
1160 uint8_t *CharsetRecog_IBM420_ar::unshapeLamAlef(const uint8_t *inputBytes, int32_t inputBytesLength, int32_t &length) {
1161     int32_t bigBufferLength = inputBytesLength * 2;
1162     uint8_t *bigBuffer = (uint8_t *)uprv_malloc(bigBufferLength);
1163     uint8_t *resultBuffer = NULL;
1164
1165     if (bigBuffer != NULL) {
1166         int32_t bufferIndex;
1167         static const uint8_t unshapedLamAlef[] = { 0xb1, 0x56 };
1168
1169         for (int32_t i = bufferIndex = 0; i < inputBytesLength; i++) {
1170             if (isLamAlef(inputBytes[i])) {
1171                 bigBuffer[bufferIndex++] = unshapedLamAlef[0];
1172                 bigBuffer[bufferIndex++] = unshapedLamAlef[1];
1173             } else {
1174                 bigBuffer[bufferIndex++] = inputBytes[i];
1175             }
1176         }
1177
1178         length = bufferIndex;
1179         resultBuffer = (uint8_t *)uprv_malloc(length);
1180         if (resultBuffer != NULL) {
1181             uprv_memcpy(resultBuffer, bigBuffer, length);
1182         }
1183     }
1184
1185     if (bigBuffer != NULL) {
1186         uprv_free(bigBuffer);
1187     }
1188
1189     return resultBuffer;
1190 }
1191
1192 void CharsetRecog_IBM420_ar::matchFinish(InputText *textIn) {
1193     if (deleteBuffer) {
1194         uprv_free(textIn->fInputBytes);
1195
1196         textIn->fInputBytes = prev_fInputBytes;
1197         textIn->fInputLen = prev_fInputBytesLength;
1198     }
1199 }
1200
1201 UBool CharsetRecog_IBM420_ar::isLamAlef(uint8_t b) {
1202     static const uint8_t shapedLamAlef[] = {
1203         0xb2, 0xb3, 0xb4, 0xb5, 0xb7, 0xb8
1204     };
1205
1206     for (uint32_t i = 0; i < sizeof(shapedLamAlef); i++) {
1207         if (b == shapedLamAlef[i]) {
1208             return TRUE;
1209         }
1210     }
1211
1212     return FALSE;
1213 }
1214
1215 CharsetRecog_IBM420_ar_rtl::~CharsetRecog_IBM420_ar_rtl()
1216 {
1217     // nothing to do
1218 }
1219
1220 const char *CharsetRecog_IBM420_ar_rtl::getName() const
1221 {
1222     return  "IBM420_rtl";
1223 }
1224
1225 UBool CharsetRecog_IBM420_ar_rtl::match(InputText *textIn, CharsetMatch *results) const
1226 {
1227     int32_t confidence = match_sbcs(textIn, ngrams_IBM420_ar_rtl, charMap_IBM420_ar);
1228     results->set(textIn, this, confidence);
1229     return (confidence > 0);
1230 }
1231
1232 CharsetRecog_IBM420_ar_ltr::~CharsetRecog_IBM420_ar_ltr()
1233 {
1234     // nothing to do
1235 }
1236
1237 const char *CharsetRecog_IBM420_ar_ltr::getName() const
1238 {
1239     return  "IBM420_ltr";
1240 }
1241
1242 UBool CharsetRecog_IBM420_ar_ltr::match(InputText *textIn, CharsetMatch *results) const
1243 {
1244     int32_t confidence = match_sbcs(textIn, ngrams_IBM420_ar_ltr, charMap_IBM420_ar);
1245     results->set(textIn, this, confidence);
1246     return (confidence > 0);
1247 }
1248
1249 U_NAMESPACE_END
1250 #endif
1251