]> git.saurik.com Git - apple/icu.git/blame - icuSources/i18n/csrsbcs.cpp
projects / apple / icu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ICU-551.51.4.tar.gz
[apple/icu.git] / icuSources / i18n / csrsbcs.cpp
CommitLineData
73c04bcf
A		 1/*
2 **********************************************************************
b331163b 3 * Copyright (C) 2005-2015, International Business Machines
73c04bcf
A		 4 * Corporation and others. All Rights Reserved.
5 **********************************************************************
6 */
7
8#include "unicode/utypes.h"
9
729e4ab9
A		 10#include "cmemory.h"
11
73c04bcf
A		 12#if !UCONFIG_NO_CONVERSION
13#include "csrsbcs.h"
51004dcb 14#include "csmatch.h"
73c04bcf 15
73c04bcf
A		 16#define N_GRAM_SIZE 3
17#define N_GRAM_MASK 0xFFFFFF
51004dcb 18#define ARRAY_SIZE(array) (sizeof array / sizeof array[0])
73c04bcf
A		 19
20U_NAMESPACE_BEGIN
21
22NGramParser::NGramParser(const int32_t *theNgramList, const uint8_t *theCharMap)
57a6839d 23 : ngram(0), byteIndex(0)
73c04bcf
A		 24{
25 ngramList = theNgramList;
26 charMap = theCharMap;
27
28 ngramCount = hitCount = 0;
29}
30
b331163b
A		 31NGramParser::~NGramParser()
32{
33}
34
73c04bcf
A		 35/*
36 * Binary search for value in table, which must have exactly 64 entries.
37 */
38
39int32_t NGramParser::search(const int32_t *table, int32_t value)
40{
41 int32_t index = 0;
42
43 if (table[index + 32] <= value) {
44 index += 32;
45 }
46
47 if (table[index + 16] <= value) {
48 index += 16;
49 }
50
51 if (table[index + 8] <= value) {
52 index += 8;
53 }
54
55 if (table[index + 4] <= value) {
56 index += 4;
57 }
58
59 if (table[index + 2] <= value) {
60 index += 2;
61 }
62
63 if (table[index + 1] <= value) {
64 index += 1;
65 }
66
67 if (table[index] > value) {
68 index -= 1;
69 }
70
71 if (index < 0 || table[index] != value) {
72 return -1;
73 }
74
75 return index;
76}
77
78void NGramParser::lookup(int32_t thisNgram)
79{
80 ngramCount += 1;
81
82 if (search(ngramList, thisNgram) >= 0) {
83 hitCount += 1;
84 }
85
86}
87
88void NGramParser::addByte(int32_t b)
89{
90 ngram = ((ngram << 8) + b) & N_GRAM_MASK;
91 lookup(ngram);
92}
93
94int32_t NGramParser::nextByte(InputText *det)
95{
96 if (byteIndex >= det->fInputLen) {
97 return -1;
98 }
99
100 return det->fInputBytes[byteIndex++];
101}
102
57a6839d 103void NGramParser::parseCharacters(InputText *det)
73c04bcf
A		 104{
105 int32_t b;
106 bool ignoreSpace = FALSE;
107
108 while ((b = nextByte(det)) >= 0) {
109 uint8_t mb = charMap[b];
110
111 // TODO: 0x20 might not be a space in all character sets...
112 if (mb != 0) {
113 if (!(mb == 0x20 && ignoreSpace)) {
114 addByte(mb);
115 }
116
117 ignoreSpace = (mb == 0x20);
118 }
119 }
57a6839d
A		 120}
121
122int32_t NGramParser::parse(InputText *det)
123{
124 parseCharacters(det);
73c04bcf
A		 125
126 // TODO: Is this OK? The buffer could have ended in the middle of a word...
127 addByte(0x20);
128
129 double rawPercent = (double) hitCount / (double) ngramCount;
130
131 // if (rawPercent <= 2.0) {
132 // return 0;
133 // }
134
135 // TODO - This is a bit of a hack to take care of a case
136 // were we were getting a confidence of 135...
137 if (rawPercent > 0.33) {
138 return 98;
139 }
140
141 return (int32_t) (rawPercent * 300.0);
142}
143
b331163b 144#if !UCONFIG_ONLY_HTML_CONVERSION
57a6839d
A		 145static const uint8_t unshapeMap_IBM420[] = {
146/* -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -A -B -C -D -E -F */
147/* 0- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
148/* 1- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
149/* 2- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
150/* 3- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
151/* 4- */ 0x40, 0x40, 0x42, 0x42, 0x44, 0x45, 0x46, 0x47, 0x47, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F,
152/* 5- */ 0x50, 0x49, 0x52, 0x53, 0x54, 0x55, 0x56, 0x56, 0x58, 0x58, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F,
153/* 6- */ 0x60, 0x61, 0x62, 0x63, 0x63, 0x65, 0x65, 0x67, 0x67, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
154/* 7- */ 0x69, 0x71, 0x71, 0x73, 0x74, 0x75, 0x76, 0x77, 0x77, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F,
155/* 8- */ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x80, 0x8B, 0x8B, 0x8D, 0x8D, 0x8F,
156/* 9- */ 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9A, 0x9A, 0x9A, 0x9E, 0x9E,
157/* A- */ 0x9E, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x9E, 0xAB, 0xAB, 0xAD, 0xAD, 0xAF,
158/* B- */ 0xAF, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xB1, 0xBB, 0xBB, 0xBD, 0xBD, 0xBF,
159/* C- */ 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xBF, 0xCC, 0xBF, 0xCE, 0xCF,
160/* D- */ 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xDA, 0xDC, 0xDC, 0xDC, 0xDF,
161/* E- */ 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
162/* F- */ 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
163};
164
165NGramParser_IBM420::NGramParser_IBM420(const int32_t *theNgramList, const uint8_t *theCharMap):NGramParser(theNgramList, theCharMap)
166{
167 alef = 0x00;
168}
169
170
171int32_t NGramParser_IBM420::isLamAlef(int32_t b)
172{
173 if(b == 0xB2 || b == 0xB3){
174 return 0x47;
175 }else if(b == 0xB4 || b == 0xB5){
176 return 0x49;
177 }else if(b == 0xB8 || b == 0xB9){
178 return 0x56;
179 }else
180 return 0x00;
181}
182
183/*
184* Arabic shaping needs to be done manually. Cannot call ArabicShaping class
185* because CharsetDetector is dealing with bytes not Unicode code points. We could
186* convert the bytes to Unicode code points but that would leave us dependent
187* on CharsetICU which we try to avoid. IBM420 converter amongst different versions
188* of JDK can produce different results and therefore is also avoided.
189*/
190int32_t NGramParser_IBM420::nextByte(InputText *det)
191{
192
193 if (byteIndex >= det->fInputLen || det->fInputBytes[byteIndex] == 0) {
194 return -1;
195 }
196 int next;
197
198 alef = isLamAlef(det->fInputBytes[byteIndex]);
199 if(alef != 0x00)
200 next = 0xB1 & 0xFF;
201 else
202 next = unshapeMap_IBM420[det->fInputBytes[byteIndex]& 0xFF] & 0xFF;
203
204 byteIndex++;
205
206 return next;
207}
208
209void NGramParser_IBM420::parseCharacters(InputText *det)
210{
211 int32_t b;
212 bool ignoreSpace = FALSE;
213
214 while ((b = nextByte(det)) >= 0) {
215 uint8_t mb = charMap[b];
216
217 // TODO: 0x20 might not be a space in all character sets...
218 if (mb != 0) {
219 if (!(mb == 0x20 && ignoreSpace)) {
220 addByte(mb);
221 }
222 ignoreSpace = (mb == 0x20);
223 }
224
225 if(alef != 0x00){
226 mb = charMap[alef & 0xFF];
227
228 // TODO: 0x20 might not be a space in all character sets...
229 if (mb != 0) {
230 if (!(mb == 0x20 && ignoreSpace)) {
231 addByte(mb);
232 }
233
234 ignoreSpace = (mb == 0x20);
235 }
236
237 }
238 }
239}
b331163b 240#endif
57a6839d 241
73c04bcf 242CharsetRecog_sbcs::CharsetRecog_sbcs()
73c04bcf
A		 243{
244 // nothing else to do
245}
246
247CharsetRecog_sbcs::~CharsetRecog_sbcs()
248{
249 // nothing to do
250}
251
51004dcb 252int32_t CharsetRecog_sbcs::match_sbcs(InputText *det, const int32_t ngrams[], const uint8_t byteMap[]) const
73c04bcf 253{
46f4442e 254 NGramParser parser(ngrams, byteMap);
73c04bcf
A		 255 int32_t result;
256
46f4442e 257 result = parser.parse(det);
73c04bcf
A		 258
259 return result;
260}
261
262static const uint8_t charMap_8859_1[] = {
263 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
264 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
265 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
266 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
267 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
268 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
269 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
270 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
271 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
272 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
273 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
274 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
275 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
276 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
277 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
278 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
279 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
280 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
281 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
282 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
283 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
284 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20,
285 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
286 0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20,
287 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
288 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
289 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
290 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF,
291 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
292 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
293 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
294 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
295};
296
297static const uint8_t charMap_8859_2[] = {
298 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
299 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
300 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
301 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
302 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
303 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
304 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
305 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
306 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
307 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
308 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
309 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
310 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
311 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
312 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
313 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
314 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
315 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
316 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
317 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
318 0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0x20,
319 0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF,
320 0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0xB7,
321 0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF,
322 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
323 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
324 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
325 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF,
326 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
327 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
328 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
329 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20,
330};
331
332static const uint8_t charMap_8859_5[] = {
333 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
334 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
335 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
336 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
337 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
338 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
339 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
340 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
341 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
342 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
343 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
344 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
345 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
346 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
347 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
348 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
349 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
350 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
351 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
352 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
353 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
354 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF,
355 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
356 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
357 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
358 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
359 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
360 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
361 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
362 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
363 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
364 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF,
365};
366
367static const uint8_t charMap_8859_6[] = {
368 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
369 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
370 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
371 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
372 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
373 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
374 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
375 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
376 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
377 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
378 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
379 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
380 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
381 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
382 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
383 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
384 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
385 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
386 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
387 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
388 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
389 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
390 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
391 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
392 0x20, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
393 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
394 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
395 0xD8, 0xD9, 0xDA, 0x20, 0x20, 0x20, 0x20, 0x20,
396 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
397 0xE8, 0xE9, 0xEA, 0x20, 0x20, 0x20, 0x20, 0x20,
398 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
399 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
400};
401
402static const uint8_t charMap_8859_7[] = {
403 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
404 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
405 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
406 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
407 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
408 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
409 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
410 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
411 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
412 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
413 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
414 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
415 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
416 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
417 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
418 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
419 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
420 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
421 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
422 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
423 0x20, 0xA1, 0xA2, 0x20, 0x20, 0x20, 0x20, 0x20,
424 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
425 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xDC, 0x20,
426 0xDD, 0xDE, 0xDF, 0x20, 0xFC, 0x20, 0xFD, 0xFE,
427 0xC0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
428 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
429 0xF0, 0xF1, 0x20, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
430 0xF8, 0xF9, 0xFA, 0xFB, 0xDC, 0xDD, 0xDE, 0xDF,
431 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
432 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
433 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
434 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20,
435};
436
437static const uint8_t charMap_8859_8[] = {
438 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
439 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
440 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
441 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
442 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
443 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
444 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
445 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
446 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
447 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
448 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
449 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
450 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
451 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
452 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
453 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
454 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
455 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
456 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
457 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
458 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
459 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
460 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
461 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
462 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
463 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
464 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
465 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
466 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
467 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
468 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
469 0xF8, 0xF9, 0xFA, 0x20, 0x20, 0x20, 0x20, 0x20,
470};
471
472static const uint8_t charMap_8859_9[] = {
473 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
474 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
475 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
476 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
477 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
478 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
479 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
480 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
481 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
482 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
483 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
484 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
485 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
486 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
487 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
488 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
489 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
490 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
491 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
492 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
493 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
494 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20,
495 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
496 0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20,
497 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
498 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
499 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
500 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x69, 0xFE, 0xDF,
501 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
502 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
503 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20,
504 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
505};
506
507static const int32_t ngrams_windows_1251[] = {
508 0x20E220, 0x20E2EE, 0x20E4EE, 0x20E7E0, 0x20E820, 0x20EAE0, 0x20EAEE, 0x20EDE0, 0x20EDE5, 0x20EEE1, 0x20EFEE, 0x20EFF0, 0x20F0E0, 0x20F1EE, 0x20F1F2, 0x20F2EE,
509 0x20F7F2, 0x20FDF2, 0xE0EDE8, 0xE0F2FC, 0xE3EE20, 0xE5EBFC, 0xE5EDE8, 0xE5F1F2, 0xE5F220, 0xE820EF, 0xE8E520, 0xE8E820, 0xE8FF20, 0xEBE5ED, 0xEBE820, 0xEBFCED,
510 0xEDE020, 0xEDE520, 0xEDE8E5, 0xEDE8FF, 0xEDEE20, 0xEDEEE2, 0xEE20E2, 0xEE20EF, 0xEE20F1, 0xEEE220, 0xEEE2E0, 0xEEE3EE, 0xEEE920, 0xEEEBFC, 0xEEEC20, 0xEEF1F2,
511 0xEFEEEB, 0xEFF0E5, 0xEFF0E8, 0xEFF0EE, 0xF0E0E2, 0xF0E5E4, 0xF1F2E0, 0xF1F2E2, 0xF1F2E8, 0xF1FF20, 0xF2E5EB, 0xF2EE20, 0xF2EEF0, 0xF2FC20, 0xF7F2EE, 0xFBF520,
512};
513
514static const uint8_t charMap_windows_1251[] = {
515 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
516 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
517 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
518 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
519 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
520 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
521 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
522 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
523 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
524 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
525 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
526 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
527 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
528 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
529 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
530 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
531 0x90, 0x83, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20,
532 0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F,
533 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
534 0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F,
535 0x20, 0xA2, 0xA2, 0xBC, 0x20, 0xB4, 0x20, 0x20,
536 0xB8, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0xBF,
537 0x20, 0x20, 0xB3, 0xB3, 0xB4, 0xB5, 0x20, 0x20,
538 0xB8, 0x20, 0xBA, 0x20, 0xBC, 0xBE, 0xBE, 0xBF,
539 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
540 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
541 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
542 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
543 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
544 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
545 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
546 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF,
547};
548
549static const int32_t ngrams_windows_1256[] = {
550 0x20C7E1, 0x20C7E4, 0x20C8C7, 0x20DAE1, 0x20DDED, 0x20E1E1, 0x20E3E4, 0x20E6C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E120, 0xC7E1C3, 0xC7E1C7, 0xC7E1C8,
551 0xC7E1CA, 0xC7E1CC, 0xC7E1CD, 0xC7E1CF, 0xC7E1D3, 0xC7E1DA, 0xC7E1DE, 0xC7E1E3, 0xC7E1E6, 0xC7E1ED, 0xC7E320, 0xC7E420, 0xC7E4CA, 0xC820C7, 0xC920C7, 0xC920DD,
552 0xC920E1, 0xC920E3, 0xC920E6, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xDA20C7, 0xDAE1EC, 0xDDED20, 0xE120C7, 0xE1C920, 0xE1EC20, 0xE1ED20,
553 0xE320C7, 0xE3C720, 0xE3C920, 0xE3E420, 0xE420C7, 0xE520C7, 0xE5C720, 0xE6C7E1, 0xE6E420, 0xEC20C7, 0xED20C7, 0xED20E3, 0xED20E6, 0xEDC920, 0xEDD120, 0xEDE420,
554};
555
556static const uint8_t charMap_windows_1256[] = {
557 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
558 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
559 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
560 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
561 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
562 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
563 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
564 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
565 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
566 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
567 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
568 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
569 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
570 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
571 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
572 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
573 0x20, 0x81, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20,
574 0x88, 0x20, 0x8A, 0x20, 0x9C, 0x8D, 0x8E, 0x8F,
575 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
576 0x98, 0x20, 0x9A, 0x20, 0x9C, 0x20, 0x20, 0x9F,
577 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
578 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20,
579 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20,
580 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
581 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
582 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
583 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0x20,
584 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
585 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
586 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
587 0x20, 0x20, 0x20, 0x20, 0xF4, 0x20, 0x20, 0x20,
588 0x20, 0xF9, 0x20, 0xFB, 0xFC, 0x20, 0x20, 0xFF,
589};
590
591static const int32_t ngrams_KOI8_R[] = {
592 0x20C4CF, 0x20C920, 0x20CBC1, 0x20CBCF, 0x20CEC1, 0x20CEC5, 0x20CFC2, 0x20D0CF, 0x20D0D2, 0x20D2C1, 0x20D3CF, 0x20D3D4, 0x20D4CF, 0x20D720, 0x20D7CF, 0x20DAC1,
593 0x20DCD4, 0x20DED4, 0xC1CEC9, 0xC1D4D8, 0xC5CCD8, 0xC5CEC9, 0xC5D3D4, 0xC5D420, 0xC7CF20, 0xC920D0, 0xC9C520, 0xC9C920, 0xC9D120, 0xCCC5CE, 0xCCC920, 0xCCD8CE,
594 0xCEC120, 0xCEC520, 0xCEC9C5, 0xCEC9D1, 0xCECF20, 0xCECFD7, 0xCF20D0, 0xCF20D3, 0xCF20D7, 0xCFC7CF, 0xCFCA20, 0xCFCCD8, 0xCFCD20, 0xCFD3D4, 0xCFD720, 0xCFD7C1,
595 0xD0CFCC, 0xD0D2C5, 0xD0D2C9, 0xD0D2CF, 0xD2C1D7, 0xD2C5C4, 0xD3D120, 0xD3D4C1, 0xD3D4C9, 0xD3D4D7, 0xD4C5CC, 0xD4CF20, 0xD4CFD2, 0xD4D820, 0xD9C820, 0xDED4CF,
596};
597
598static const uint8_t charMap_KOI8_R[] = {
599 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
600 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
601 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
602 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
603 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00,
604 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
605 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
606 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
607 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
608 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
609 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
610 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
611 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
612 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F,
613 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
614 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20,
615 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
616 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
617 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
618 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
619 0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20,
620 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
621 0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20,
622 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
623 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
624 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
625 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
626 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
627 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
628 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
629 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
630 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
631};
632
b331163b 633#if !UCONFIG_ONLY_HTML_CONVERSION
729e4ab9
A		 634static const int32_t ngrams_IBM424_he_rtl[] = {
635 0x404146, 0x404148, 0x404151, 0x404171, 0x404251, 0x404256, 0x404541, 0x404546, 0x404551, 0x404556, 0x404562, 0x404569, 0x404571, 0x405441, 0x405445, 0x405641,
636 0x406254, 0x406954, 0x417140, 0x454041, 0x454042, 0x454045, 0x454054, 0x454056, 0x454069, 0x454641, 0x464140, 0x465540, 0x465740, 0x466840, 0x467140, 0x514045,
637 0x514540, 0x514671, 0x515155, 0x515540, 0x515740, 0x516840, 0x517140, 0x544041, 0x544045, 0x544140, 0x544540, 0x554041, 0x554042, 0x554045, 0x554054, 0x554056,
638 0x554069, 0x564540, 0x574045, 0x584540, 0x585140, 0x585155, 0x625440, 0x684045, 0x685155, 0x695440, 0x714041, 0x714042, 0x714045, 0x714054, 0x714056, 0x714069,
639};
640
641static const int32_t ngrams_IBM424_he_ltr[] = {
642 0x404146, 0x404154, 0x404551, 0x404554, 0x404556, 0x404558, 0x405158, 0x405462, 0x405469, 0x405546, 0x405551, 0x405746, 0x405751, 0x406846, 0x406851, 0x407141,
643 0x407146, 0x407151, 0x414045, 0x414054, 0x414055, 0x414071, 0x414540, 0x414645, 0x415440, 0x415640, 0x424045, 0x424055, 0x424071, 0x454045, 0x454051, 0x454054,
644 0x454055, 0x454057, 0x454068, 0x454071, 0x455440, 0x464140, 0x464540, 0x484140, 0x514140, 0x514240, 0x514540, 0x544045, 0x544055, 0x544071, 0x546240, 0x546940,
645 0x555151, 0x555158, 0x555168, 0x564045, 0x564055, 0x564071, 0x564240, 0x564540, 0x624540, 0x694045, 0x694055, 0x694071, 0x694540, 0x714140, 0x714540, 0x714651,
646};
647
648static const uint8_t charMap_IBM424_he[] = {
649/* -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -A -B -C -D -E -F */
650/* 0- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
651/* 1- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
652/* 2- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
653/* 3- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
654/* 4- */ 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
655/* 5- */ 0x40, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
656/* 6- */ 0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
657/* 7- */ 0x40, 0x71, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x00, 0x40, 0x40,
658/* 8- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
659/* 9- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
660/* A- */ 0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
661/* B- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
662/* C- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
663/* D- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
664/* E- */ 0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
665/* F- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
666};
667
668static const int32_t ngrams_IBM420_ar_rtl[] = {
669 0x4056B1, 0x4056BD, 0x405856, 0x409AB1, 0x40ABDC, 0x40B1B1, 0x40BBBD, 0x40CF56, 0x564056, 0x564640, 0x566340, 0x567540, 0x56B140, 0x56B149, 0x56B156, 0x56B158,
670 0x56B163, 0x56B167, 0x56B169, 0x56B173, 0x56B178, 0x56B19A, 0x56B1AD, 0x56B1BB, 0x56B1CF, 0x56B1DC, 0x56BB40, 0x56BD40, 0x56BD63, 0x584056, 0x624056, 0x6240AB,
671 0x6240B1, 0x6240BB, 0x6240CF, 0x634056, 0x734056, 0x736240, 0x754056, 0x756240, 0x784056, 0x9A4056, 0x9AB1DA, 0xABDC40, 0xB14056, 0xB16240, 0xB1DA40, 0xB1DC40,
672 0xBB4056, 0xBB5640, 0xBB6240, 0xBBBD40, 0xBD4056, 0xBF4056, 0xBF5640, 0xCF56B1, 0xCFBD40, 0xDA4056, 0xDC4056, 0xDC40BB, 0xDC40CF, 0xDC6240, 0xDC7540, 0xDCBD40,
673};
674
675static const int32_t ngrams_IBM420_ar_ltr[] = {
676 0x404656, 0x4056BB, 0x4056BF, 0x406273, 0x406275, 0x4062B1, 0x4062BB, 0x4062DC, 0x406356, 0x407556, 0x4075DC, 0x40B156, 0x40BB56, 0x40BD56, 0x40BDBB, 0x40BDCF,
677 0x40BDDC, 0x40DAB1, 0x40DCAB, 0x40DCB1, 0x49B156, 0x564056, 0x564058, 0x564062, 0x564063, 0x564073, 0x564075, 0x564078, 0x56409A, 0x5640B1, 0x5640BB, 0x5640BD,
678 0x5640BF, 0x5640DA, 0x5640DC, 0x565840, 0x56B156, 0x56CF40, 0x58B156, 0x63B156, 0x63BD56, 0x67B156, 0x69B156, 0x73B156, 0x78B156, 0x9AB156, 0xAB4062, 0xADB156,
679 0xB14062, 0xB15640, 0xB156CF, 0xB19A40, 0xB1B140, 0xBB4062, 0xBB40DC, 0xBBB156, 0xBD5640, 0xBDBB40, 0xCF4062, 0xCF40DC, 0xCFB156, 0xDAB19A, 0xDCAB40, 0xDCB156
680};
681
682static const uint8_t charMap_IBM420_ar[]= {
683/* -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -A -B -C -D -E -F */
684/* 0- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
685/* 1- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
686/* 2- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
687/* 3- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
688/* 4- */ 0x40, 0x40, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
689/* 5- */ 0x40, 0x51, 0x52, 0x40, 0x40, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
690/* 6- */ 0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
691/* 7- */ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40,
692/* 8- */ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F,
693/* 9- */ 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F,
694/* A- */ 0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF,
695/* B- */ 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0x40, 0x40, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF,
696/* C- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0xCB, 0x40, 0xCD, 0x40, 0xCF,
697/* D- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
698/* E- */ 0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xEA, 0xEB, 0x40, 0xED, 0xEE, 0xEF,
699/* F- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0xFB, 0xFC, 0xFD, 0xFE, 0x40,
700};
b331163b 701#endif
729e4ab9 702
51004dcb
A		 703//ISO-8859-1,2,5,6,7,8,9 Ngrams
704
705struct NGramsPlusLang {
706 const int32_t ngrams[64];
707 const char * lang;
708};
709
710static const NGramsPlusLang ngrams_8859_1[] = {
711 {
712 {
73c04bcf
A		 713 0x206120, 0x20616E, 0x206265, 0x20636F, 0x20666F, 0x206861, 0x206865, 0x20696E, 0x206D61, 0x206F66, 0x207072, 0x207265, 0x207361, 0x207374, 0x207468, 0x20746F,
714 0x207768, 0x616964, 0x616C20, 0x616E20, 0x616E64, 0x617320, 0x617420, 0x617465, 0x617469, 0x642061, 0x642074, 0x652061, 0x652073, 0x652074, 0x656420, 0x656E74,
715 0x657220, 0x657320, 0x666F72, 0x686174, 0x686520, 0x686572, 0x696420, 0x696E20, 0x696E67, 0x696F6E, 0x697320, 0x6E2061, 0x6E2074, 0x6E6420, 0x6E6720, 0x6E7420,
716 0x6F6620, 0x6F6E20, 0x6F7220, 0x726520, 0x727320, 0x732061, 0x732074, 0x736169, 0x737420, 0x742074, 0x746572, 0x746861, 0x746865, 0x74696F, 0x746F20, 0x747320,
51004dcb
A		 717 },
718 "en"
719 },
720 {
721 {
73c04bcf
A		 722 0x206166, 0x206174, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207369, 0x207374, 0x207469, 0x207669, 0x616620,
723 0x616E20, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646572, 0x646574, 0x652073, 0x656420, 0x656465, 0x656E20, 0x656E64, 0x657220, 0x657265, 0x657320,
724 0x657420, 0x666F72, 0x676520, 0x67656E, 0x676572, 0x696765, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6572, 0x6C6967, 0x6C6C65, 0x6D6564, 0x6E6465, 0x6E6520,
725 0x6E6720, 0x6E6765, 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722064, 0x722065, 0x722073, 0x726520, 0x737465, 0x742073, 0x746520, 0x746572, 0x74696C, 0x766572,
51004dcb
A		 726 },
727 "da"
728 },
729 {
730 {
73c04bcf
A		 731 0x20616E, 0x206175, 0x206265, 0x206461, 0x206465, 0x206469, 0x206569, 0x206765, 0x206861, 0x20696E, 0x206D69, 0x207363, 0x207365, 0x20756E, 0x207665, 0x20766F,
732 0x207765, 0x207A75, 0x626572, 0x636820, 0x636865, 0x636874, 0x646173, 0x64656E, 0x646572, 0x646965, 0x652064, 0x652073, 0x65696E, 0x656974, 0x656E20, 0x657220,
733 0x657320, 0x67656E, 0x68656E, 0x687420, 0x696368, 0x696520, 0x696E20, 0x696E65, 0x697420, 0x6C6963, 0x6C6C65, 0x6E2061, 0x6E2064, 0x6E2073, 0x6E6420, 0x6E6465,
734 0x6E6520, 0x6E6720, 0x6E6765, 0x6E7465, 0x722064, 0x726465, 0x726569, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x756E64, 0x756E67, 0x766572,
51004dcb
A		 735 },
736 "de"
737 },
738 {
739 {
73c04bcf
A		 740 0x206120, 0x206361, 0x20636F, 0x206465, 0x20656C, 0x20656E, 0x206573, 0x20696E, 0x206C61, 0x206C6F, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365,
741 0x20756E, 0x207920, 0x612063, 0x612064, 0x612065, 0x61206C, 0x612070, 0x616369, 0x61646F, 0x616C20, 0x617220, 0x617320, 0x6369F3, 0x636F6E, 0x646520, 0x64656C,
742 0x646F20, 0x652064, 0x652065, 0x65206C, 0x656C20, 0x656E20, 0x656E74, 0x657320, 0x657374, 0x69656E, 0x69F36E, 0x6C6120, 0x6C6F73, 0x6E2065, 0x6E7465, 0x6F2064,
743 0x6F2065, 0x6F6E20, 0x6F7220, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732064, 0x732065, 0x732070, 0x736520, 0x746520, 0x746F20, 0x756520, 0xF36E20,
51004dcb
A		 744 },
745 "es"
746 },
747 {
748 {
73c04bcf
A		 749 0x206175, 0x20636F, 0x206461, 0x206465, 0x206475, 0x20656E, 0x206574, 0x206C61, 0x206C65, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207365, 0x20736F, 0x20756E,
750 0x20E020, 0x616E74, 0x617469, 0x636520, 0x636F6E, 0x646520, 0x646573, 0x647520, 0x652061, 0x652063, 0x652064, 0x652065, 0x65206C, 0x652070, 0x652073, 0x656E20,
751 0x656E74, 0x657220, 0x657320, 0x657420, 0x657572, 0x696F6E, 0x697320, 0x697420, 0x6C6120, 0x6C6520, 0x6C6573, 0x6D656E, 0x6E2064, 0x6E6520, 0x6E7320, 0x6E7420,
752 0x6F6E20, 0x6F6E74, 0x6F7572, 0x717565, 0x72206C, 0x726520, 0x732061, 0x732064, 0x732065, 0x73206C, 0x732070, 0x742064, 0x746520, 0x74696F, 0x756520, 0x757220,
51004dcb
A		 753 },
754 "fr"
755 },
756 {
757 {
73c04bcf
A		 758 0x20616C, 0x206368, 0x20636F, 0x206465, 0x206469, 0x206520, 0x20696C, 0x20696E, 0x206C61, 0x207065, 0x207072, 0x20756E, 0x612063, 0x612064, 0x612070, 0x612073,
759 0x61746F, 0x636865, 0x636F6E, 0x64656C, 0x646920, 0x652061, 0x652063, 0x652064, 0x652069, 0x65206C, 0x652070, 0x652073, 0x656C20, 0x656C6C, 0x656E74, 0x657220,
760 0x686520, 0x692061, 0x692063, 0x692064, 0x692073, 0x696120, 0x696C20, 0x696E20, 0x696F6E, 0x6C6120, 0x6C6520, 0x6C6920, 0x6C6C61, 0x6E6520, 0x6E6920, 0x6E6F20,
761 0x6E7465, 0x6F2061, 0x6F2064, 0x6F2069, 0x6F2073, 0x6F6E20, 0x6F6E65, 0x706572, 0x726120, 0x726520, 0x736920, 0x746120, 0x746520, 0x746920, 0x746F20, 0x7A696F,
51004dcb
A		 762 },
763 "it"
764 },
765 {
766 {
73c04bcf
A		 767 0x20616C, 0x206265, 0x206461, 0x206465, 0x206469, 0x206565, 0x20656E, 0x206765, 0x206865, 0x20696E, 0x206D61, 0x206D65, 0x206F70, 0x207465, 0x207661, 0x207665,
768 0x20766F, 0x207765, 0x207A69, 0x61616E, 0x616172, 0x616E20, 0x616E64, 0x617220, 0x617420, 0x636874, 0x646520, 0x64656E, 0x646572, 0x652062, 0x652076, 0x65656E,
769 0x656572, 0x656E20, 0x657220, 0x657273, 0x657420, 0x67656E, 0x686574, 0x696520, 0x696E20, 0x696E67, 0x697320, 0x6E2062, 0x6E2064, 0x6E2065, 0x6E2068, 0x6E206F,
770 0x6E2076, 0x6E6465, 0x6E6720, 0x6F6E64, 0x6F6F72, 0x6F7020, 0x6F7220, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x76616E, 0x766572, 0x766F6F,
51004dcb
A		 771 },
772 "nl"
773 },
774 {
775 {
73c04bcf
A		 776 0x206174, 0x206176, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207365, 0x20736B, 0x20736F, 0x207374, 0x207469,
777 0x207669, 0x20E520, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646574, 0x652073, 0x656420, 0x656E20, 0x656E65, 0x657220, 0x657265, 0x657420, 0x657474,
778 0x666F72, 0x67656E, 0x696B6B, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6520, 0x6C6C65, 0x6D6564, 0x6D656E, 0x6E2073, 0x6E6520, 0x6E6720, 0x6E6765, 0x6E6E65,
779 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722073, 0x726520, 0x736F6D, 0x737465, 0x742073, 0x746520, 0x74656E, 0x746572, 0x74696C, 0x747420, 0x747465, 0x766572,
51004dcb
A		 780 },
781 "no"
782 },
783 {
784 {
73c04bcf
A		 785 0x206120, 0x20636F, 0x206461, 0x206465, 0x20646F, 0x206520, 0x206573, 0x206D61, 0x206E6F, 0x206F20, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365,
786 0x20756D, 0x612061, 0x612063, 0x612064, 0x612070, 0x616465, 0x61646F, 0x616C20, 0x617220, 0x617261, 0x617320, 0x636F6D, 0x636F6E, 0x646120, 0x646520, 0x646F20,
787 0x646F73, 0x652061, 0x652064, 0x656D20, 0x656E74, 0x657320, 0x657374, 0x696120, 0x696361, 0x6D656E, 0x6E7465, 0x6E746F, 0x6F2061, 0x6F2063, 0x6F2064, 0x6F2065,
788 0x6F2070, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732061, 0x732064, 0x732065, 0x732070, 0x737461, 0x746520, 0x746F20, 0x756520, 0xE36F20, 0xE7E36F,
51004dcb
A		 789 },
790 "pt"
791 },
792 {
793 {
73c04bcf
A		 794 0x206174, 0x206176, 0x206465, 0x20656E, 0x2066F6, 0x206861, 0x206920, 0x20696E, 0x206B6F, 0x206D65, 0x206F63, 0x2070E5, 0x20736B, 0x20736F, 0x207374, 0x207469,
795 0x207661, 0x207669, 0x20E472, 0x616465, 0x616E20, 0x616E64, 0x617220, 0x617474, 0x636820, 0x646520, 0x64656E, 0x646572, 0x646574, 0x656420, 0x656E20, 0x657220,
796 0x657420, 0x66F672, 0x67656E, 0x696C6C, 0x696E67, 0x6B6120, 0x6C6C20, 0x6D6564, 0x6E2073, 0x6E6120, 0x6E6465, 0x6E6720, 0x6E6765, 0x6E696E, 0x6F6368, 0x6F6D20,
797 0x6F6E20, 0x70E520, 0x722061, 0x722073, 0x726120, 0x736B61, 0x736F6D, 0x742073, 0x746120, 0x746520, 0x746572, 0x74696C, 0x747420, 0x766172, 0xE47220, 0xF67220,
51004dcb
A		 798 },
799 "sv"
800 }
73c04bcf
A		 801};
802
51004dcb
A		 803
804static const NGramsPlusLang ngrams_8859_2[] = {
805 {
806 {
73c04bcf
A		 807 0x206120, 0x206279, 0x20646F, 0x206A65, 0x206E61, 0x206E65, 0x206F20, 0x206F64, 0x20706F, 0x207072, 0x2070F8, 0x20726F, 0x207365, 0x20736F, 0x207374, 0x20746F,
808 0x207620, 0x207679, 0x207A61, 0x612070, 0x636520, 0x636820, 0x652070, 0x652073, 0x652076, 0x656D20, 0x656EED, 0x686F20, 0x686F64, 0x697374, 0x6A6520, 0x6B7465,
809 0x6C6520, 0x6C6920, 0x6E6120, 0x6EE920, 0x6EEC20, 0x6EED20, 0x6F2070, 0x6F646E, 0x6F6A69, 0x6F7374, 0x6F7520, 0x6F7661, 0x706F64, 0x706F6A, 0x70726F, 0x70F865,
810 0x736520, 0x736F75, 0x737461, 0x737469, 0x73746E, 0x746572, 0x746EED, 0x746F20, 0x752070, 0xBE6520, 0xE16EED, 0xE9686F, 0xED2070, 0xED2073, 0xED6D20, 0xF86564,
51004dcb
A		 811 },
812 "cs"
813 },
814 {
815 {
73c04bcf
A		 816 0x206120, 0x20617A, 0x206265, 0x206567, 0x20656C, 0x206665, 0x206861, 0x20686F, 0x206973, 0x206B65, 0x206B69, 0x206BF6, 0x206C65, 0x206D61, 0x206D65, 0x206D69,
817 0x206E65, 0x20737A, 0x207465, 0x20E973, 0x612061, 0x61206B, 0x61206D, 0x612073, 0x616B20, 0x616E20, 0x617A20, 0x62616E, 0x62656E, 0x656779, 0x656B20, 0x656C20,
818 0x656C65, 0x656D20, 0x656E20, 0x657265, 0x657420, 0x657465, 0x657474, 0x677920, 0x686F67, 0x696E74, 0x697320, 0x6B2061, 0x6BF67A, 0x6D6567, 0x6D696E, 0x6E2061,
819 0x6E616B, 0x6E656B, 0x6E656D, 0x6E7420, 0x6F6779, 0x732061, 0x737A65, 0x737A74, 0x737AE1, 0x73E967, 0x742061, 0x747420, 0x74E173, 0x7A6572, 0xE16E20, 0xE97320,
51004dcb
A		 820 },
821 "hu"
822 },
823 {
824 {
73c04bcf
A		 825 0x20637A, 0x20646F, 0x206920, 0x206A65, 0x206B6F, 0x206D61, 0x206D69, 0x206E61, 0x206E69, 0x206F64, 0x20706F, 0x207072, 0x207369, 0x207720, 0x207769, 0x207779,
826 0x207A20, 0x207A61, 0x612070, 0x612077, 0x616E69, 0x636820, 0x637A65, 0x637A79, 0x646F20, 0x647A69, 0x652070, 0x652073, 0x652077, 0x65207A, 0x65676F, 0x656A20,
827 0x656D20, 0x656E69, 0x676F20, 0x696120, 0x696520, 0x69656A, 0x6B6120, 0x6B6920, 0x6B6965, 0x6D6965, 0x6E6120, 0x6E6961, 0x6E6965, 0x6F2070, 0x6F7761, 0x6F7769,
828 0x706F6C, 0x707261, 0x70726F, 0x70727A, 0x727A65, 0x727A79, 0x7369EA, 0x736B69, 0x737461, 0x776965, 0x796368, 0x796D20, 0x7A6520, 0x7A6965, 0x7A7920, 0xF37720,
51004dcb
A		 829 },
830 "pl"
831 },
832 {
833 {
73c04bcf
A		 834 0x206120, 0x206163, 0x206361, 0x206365, 0x20636F, 0x206375, 0x206465, 0x206469, 0x206C61, 0x206D61, 0x207065, 0x207072, 0x207365, 0x2073E3, 0x20756E, 0x20BA69,
835 0x20EE6E, 0x612063, 0x612064, 0x617265, 0x617420, 0x617465, 0x617520, 0x636172, 0x636F6E, 0x637520, 0x63E320, 0x646520, 0x652061, 0x652063, 0x652064, 0x652070,
836 0x652073, 0x656120, 0x656920, 0x656C65, 0x656E74, 0x657374, 0x692061, 0x692063, 0x692064, 0x692070, 0x696520, 0x696920, 0x696E20, 0x6C6120, 0x6C6520, 0x6C6F72,
837 0x6C7569, 0x6E6520, 0x6E7472, 0x6F7220, 0x70656E, 0x726520, 0x726561, 0x727520, 0x73E320, 0x746520, 0x747275, 0x74E320, 0x756920, 0x756C20, 0xBA6920, 0xEE6E20,
51004dcb
A		 838 },
839 "ro"
840 }
73c04bcf
A		 841};
842
843static const int32_t ngrams_8859_5_ru[] = {
844 0x20D220, 0x20D2DE, 0x20D4DE, 0x20D7D0, 0x20D820, 0x20DAD0, 0x20DADE, 0x20DDD0, 0x20DDD5, 0x20DED1, 0x20DFDE, 0x20DFE0, 0x20E0D0, 0x20E1DE, 0x20E1E2, 0x20E2DE,
845 0x20E7E2, 0x20EDE2, 0xD0DDD8, 0xD0E2EC, 0xD3DE20, 0xD5DBEC, 0xD5DDD8, 0xD5E1E2, 0xD5E220, 0xD820DF, 0xD8D520, 0xD8D820, 0xD8EF20, 0xDBD5DD, 0xDBD820, 0xDBECDD,
846 0xDDD020, 0xDDD520, 0xDDD8D5, 0xDDD8EF, 0xDDDE20, 0xDDDED2, 0xDE20D2, 0xDE20DF, 0xDE20E1, 0xDED220, 0xDED2D0, 0xDED3DE, 0xDED920, 0xDEDBEC, 0xDEDC20, 0xDEE1E2,
847 0xDFDEDB, 0xDFE0D5, 0xDFE0D8, 0xDFE0DE, 0xE0D0D2, 0xE0D5D4, 0xE1E2D0, 0xE1E2D2, 0xE1E2D8, 0xE1EF20, 0xE2D5DB, 0xE2DE20, 0xE2DEE0, 0xE2EC20, 0xE7E2DE, 0xEBE520,
848};
849
850static const int32_t ngrams_8859_6_ar[] = {
851 0x20C7E4, 0x20C7E6, 0x20C8C7, 0x20D9E4, 0x20E1EA, 0x20E4E4, 0x20E5E6, 0x20E8C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E420, 0xC7E4C3, 0xC7E4C7, 0xC7E4C8,
852 0xC7E4CA, 0xC7E4CC, 0xC7E4CD, 0xC7E4CF, 0xC7E4D3, 0xC7E4D9, 0xC7E4E2, 0xC7E4E5, 0xC7E4E8, 0xC7E4EA, 0xC7E520, 0xC7E620, 0xC7E6CA, 0xC820C7, 0xC920C7, 0xC920E1,
853 0xC920E4, 0xC920E5, 0xC920E8, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xD920C7, 0xD9E4E9, 0xE1EA20, 0xE420C7, 0xE4C920, 0xE4E920, 0xE4EA20,
854 0xE520C7, 0xE5C720, 0xE5C920, 0xE5E620, 0xE620C7, 0xE720C7, 0xE7C720, 0xE8C7E4, 0xE8E620, 0xE920C7, 0xEA20C7, 0xEA20E5, 0xEA20E8, 0xEAC920, 0xEAD120, 0xEAE620,
855};
856
857static const int32_t ngrams_8859_7_el[] = {
858 0x20E1ED, 0x20E1F0, 0x20E3E9, 0x20E4E9, 0x20E5F0, 0x20E720, 0x20EAE1, 0x20ECE5, 0x20EDE1, 0x20EF20, 0x20F0E1, 0x20F0EF, 0x20F0F1, 0x20F3F4, 0x20F3F5, 0x20F4E7,
859 0x20F4EF, 0xDFE120, 0xE120E1, 0xE120F4, 0xE1E920, 0xE1ED20, 0xE1F0FC, 0xE1F220, 0xE3E9E1, 0xE5E920, 0xE5F220, 0xE720F4, 0xE7ED20, 0xE7F220, 0xE920F4, 0xE9E120,
860 0xE9EADE, 0xE9F220, 0xEAE1E9, 0xEAE1F4, 0xECE520, 0xED20E1, 0xED20E5, 0xED20F0, 0xEDE120, 0xEFF220, 0xEFF520, 0xF0EFF5, 0xF0F1EF, 0xF0FC20, 0xF220E1, 0xF220E5,
861 0xF220EA, 0xF220F0, 0xF220F4, 0xF3E520, 0xF3E720, 0xF3F4EF, 0xF4E120, 0xF4E1E9, 0xF4E7ED, 0xF4E7F2, 0xF4E9EA, 0xF4EF20, 0xF4EFF5, 0xF4F9ED, 0xF9ED20, 0xFEED20,
862};
863
864static const int32_t ngrams_8859_8_I_he[] = {
865 0x20E0E5, 0x20E0E7, 0x20E0E9, 0x20E0FA, 0x20E1E9, 0x20E1EE, 0x20E4E0, 0x20E4E5, 0x20E4E9, 0x20E4EE, 0x20E4F2, 0x20E4F9, 0x20E4FA, 0x20ECE0, 0x20ECE4, 0x20EEE0,
866 0x20F2EC, 0x20F9EC, 0xE0FA20, 0xE420E0, 0xE420E1, 0xE420E4, 0xE420EC, 0xE420EE, 0xE420F9, 0xE4E5E0, 0xE5E020, 0xE5ED20, 0xE5EF20, 0xE5F820, 0xE5FA20, 0xE920E4,
867 0xE9E420, 0xE9E5FA, 0xE9E9ED, 0xE9ED20, 0xE9EF20, 0xE9F820, 0xE9FA20, 0xEC20E0, 0xEC20E4, 0xECE020, 0xECE420, 0xED20E0, 0xED20E1, 0xED20E4, 0xED20EC, 0xED20EE,
868 0xED20F9, 0xEEE420, 0xEF20E4, 0xF0E420, 0xF0E920, 0xF0E9ED, 0xF2EC20, 0xF820E4, 0xF8E9ED, 0xF9EC20, 0xFA20E0, 0xFA20E1, 0xFA20E4, 0xFA20EC, 0xFA20EE, 0xFA20F9,
869};
870
871static const int32_t ngrams_8859_8_he[] = {
872 0x20E0E5, 0x20E0EC, 0x20E4E9, 0x20E4EC, 0x20E4EE, 0x20E4F0, 0x20E9F0, 0x20ECF2, 0x20ECF9, 0x20EDE5, 0x20EDE9, 0x20EFE5, 0x20EFE9, 0x20F8E5, 0x20F8E9, 0x20FAE0,
873 0x20FAE5, 0x20FAE9, 0xE020E4, 0xE020EC, 0xE020ED, 0xE020FA, 0xE0E420, 0xE0E5E4, 0xE0EC20, 0xE0EE20, 0xE120E4, 0xE120ED, 0xE120FA, 0xE420E4, 0xE420E9, 0xE420EC,
874 0xE420ED, 0xE420EF, 0xE420F8, 0xE420FA, 0xE4EC20, 0xE5E020, 0xE5E420, 0xE7E020, 0xE9E020, 0xE9E120, 0xE9E420, 0xEC20E4, 0xEC20ED, 0xEC20FA, 0xECF220, 0xECF920,
875 0xEDE9E9, 0xEDE9F0, 0xEDE9F8, 0xEE20E4, 0xEE20ED, 0xEE20FA, 0xEEE120, 0xEEE420, 0xF2E420, 0xF920E4, 0xF920ED, 0xF920FA, 0xF9E420, 0xFAE020, 0xFAE420, 0xFAE5E9,
876};
877
878static const int32_t ngrams_8859_9_tr[] = {
879 0x206261, 0x206269, 0x206275, 0x206461, 0x206465, 0x206765, 0x206861, 0x20696C, 0x206B61, 0x206B6F, 0x206D61, 0x206F6C, 0x207361, 0x207461, 0x207665, 0x207961,
880 0x612062, 0x616B20, 0x616C61, 0x616D61, 0x616E20, 0x616EFD, 0x617220, 0x617261, 0x6172FD, 0x6173FD, 0x617961, 0x626972, 0x646120, 0x646520, 0x646920, 0x652062,
881 0x65206B, 0x656469, 0x656E20, 0x657220, 0x657269, 0x657369, 0x696C65, 0x696E20, 0x696E69, 0x697220, 0x6C616E, 0x6C6172, 0x6C6520, 0x6C6572, 0x6E2061, 0x6E2062,
882 0x6E206B, 0x6E6461, 0x6E6465, 0x6E6520, 0x6E6920, 0x6E696E, 0x6EFD20, 0x72696E, 0x72FD6E, 0x766520, 0x796120, 0x796F72, 0xFD6E20, 0xFD6E64, 0xFD6EFD, 0xFDF0FD,
883};
884
885CharsetRecog_8859_1::~CharsetRecog_8859_1()
886{
887 // nothing to do
888}
889
51004dcb
A		 890UBool CharsetRecog_8859_1::match(InputText *textIn, CharsetMatch *results) const {
891 const char *name = textIn->fC1Bytes? "windows-1252" : "ISO-8859-1";
892 uint32_t i;
893 int32_t bestConfidenceSoFar = -1;
894 for (i=0; i < ARRAY_SIZE(ngrams_8859_1) ; i++) {
895 const int32_t *ngrams = ngrams_8859_1[i].ngrams;
896 const char *lang = ngrams_8859_1[i].lang;
897 int32_t confidence = match_sbcs(textIn, ngrams, charMap_8859_1);
898 if (confidence > bestConfidenceSoFar) {
899 results->set(textIn, this, confidence, name, lang);
900 bestConfidenceSoFar = confidence;
901 }
902 }
903 return (bestConfidenceSoFar > 0);
73c04bcf
A		 904}
905
51004dcb 906const char *CharsetRecog_8859_1::getName() const
73c04bcf 907{
51004dcb 908 return "ISO-8859-1";
73c04bcf
A		 909}
910
73c04bcf
A		 911
912CharsetRecog_8859_2::~CharsetRecog_8859_2()
913{
914 // nothing to do
915}
916
51004dcb
A		 917UBool CharsetRecog_8859_2::match(InputText *textIn, CharsetMatch *results) const {
918 const char *name = textIn->fC1Bytes? "windows-1250" : "ISO-8859-2";
919 uint32_t i;
920 int32_t bestConfidenceSoFar = -1;
921 for (i=0; i < ARRAY_SIZE(ngrams_8859_2) ; i++) {
922 const int32_t *ngrams = ngrams_8859_2[i].ngrams;
923 const char *lang = ngrams_8859_2[i].lang;
924 int32_t confidence = match_sbcs(textIn, ngrams, charMap_8859_2);
925 if (confidence > bestConfidenceSoFar) {
926 results->set(textIn, this, confidence, name, lang);
927 bestConfidenceSoFar = confidence;
928 }
929 }
930 return (bestConfidenceSoFar > 0);
73c04bcf
A		 931}
932
51004dcb 933const char *CharsetRecog_8859_2::getName() const
73c04bcf 934{
51004dcb 935 return "ISO-8859-2";
73c04bcf
A		 936}
937
73c04bcf
A		 938
939CharsetRecog_8859_5::~CharsetRecog_8859_5()
940{
941 // nothing to do
942}
943
944const char *CharsetRecog_8859_5::getName() const
945{
946 return "ISO-8859-5";
947}
948
949CharsetRecog_8859_5_ru::~CharsetRecog_8859_5_ru()
950{
951 // nothing to do
952}
953
954const char *CharsetRecog_8859_5_ru::getLanguage() const
955{
956 return "ru";
957}
958
51004dcb 959UBool CharsetRecog_8859_5_ru::match(InputText *textIn, CharsetMatch *results) const
73c04bcf 960{
51004dcb
A		 961 int32_t confidence = match_sbcs(textIn, ngrams_8859_5_ru, charMap_8859_5);
962 results->set(textIn, this, confidence);
963 return (confidence > 0);
73c04bcf
A		 964}
965
966CharsetRecog_8859_6::~CharsetRecog_8859_6()
967{
968 // nothing to do
969}
970
971const char *CharsetRecog_8859_6::getName() const
972{
973 return "ISO-8859-6";
974}
975
976CharsetRecog_8859_6_ar::~CharsetRecog_8859_6_ar()
977{
978 // nothing to do
979}
980
981const char *CharsetRecog_8859_6_ar::getLanguage() const
982{
983 return "ar";
984}
985
51004dcb 986UBool CharsetRecog_8859_6_ar::match(InputText *textIn, CharsetMatch *results) const
73c04bcf 987{
51004dcb
A		 988 int32_t confidence = match_sbcs(textIn, ngrams_8859_6_ar, charMap_8859_6);
989 results->set(textIn, this, confidence);
990 return (confidence > 0);
73c04bcf
A		 991}
992
993CharsetRecog_8859_7::~CharsetRecog_8859_7()
994{
995 // nothing to do
996}
997
998const char *CharsetRecog_8859_7::getName() const
999{
51004dcb 1000 return "ISO-8859-7";
73c04bcf
A		 1001}
1002
1003CharsetRecog_8859_7_el::~CharsetRecog_8859_7_el()
1004{
1005 // nothing to do
1006}
1007
1008const char *CharsetRecog_8859_7_el::getLanguage() const
1009{
1010 return "el";
1011}
1012
51004dcb 1013UBool CharsetRecog_8859_7_el::match(InputText *textIn, CharsetMatch *results) const
73c04bcf 1014{
51004dcb
A		 1015 const char *name = textIn->fC1Bytes? "windows-1253" : "ISO-8859-7";
1016 int32_t confidence = match_sbcs(textIn, ngrams_8859_7_el, charMap_8859_7);
1017 results->set(textIn, this, confidence, name, "el");
1018 return (confidence > 0);
73c04bcf
A		 1019}
1020
1021CharsetRecog_8859_8::~CharsetRecog_8859_8()
1022{
1023 // nothing to do
1024}
1025
1026const char *CharsetRecog_8859_8::getName() const
1027{
51004dcb 1028 return "ISO-8859-8";
73c04bcf
A		 1029}
1030
1031CharsetRecog_8859_8_I_he::~CharsetRecog_8859_8_I_he ()
1032{
1033 // nothing to do
1034}
1035
1036const char *CharsetRecog_8859_8_I_he::getName() const
1037{
51004dcb 1038 return "ISO-8859-8-I";
73c04bcf
A		 1039}
1040
1041const char *CharsetRecog_8859_8_I_he::getLanguage() const
1042{
1043 return "he";
1044}
1045
51004dcb 1046UBool CharsetRecog_8859_8_I_he::match(InputText *textIn, CharsetMatch *results) const
73c04bcf 1047{
51004dcb
A		 1048 const char *name = textIn->fC1Bytes? "windows-1255" : "ISO-8859-8-I";
1049 int32_t confidence = match_sbcs(textIn, ngrams_8859_8_I_he, charMap_8859_8);
1050 results->set(textIn, this, confidence, name, "he");
1051 return (confidence > 0);
73c04bcf
A		 1052}
1053
1054CharsetRecog_8859_8_he::~CharsetRecog_8859_8_he()
1055{
1056 // od ot gnihton
1057}
1058
1059const char *CharsetRecog_8859_8_he::getLanguage() const
1060{
1061 return "he";
1062}
1063
51004dcb 1064UBool CharsetRecog_8859_8_he::match(InputText *textIn, CharsetMatch *results) const
73c04bcf 1065{
51004dcb
A		 1066 const char *name = textIn->fC1Bytes? "windows-1255" : "ISO-8859-8";
1067 int32_t confidence = match_sbcs(textIn, ngrams_8859_8_he, charMap_8859_8);
1068 results->set(textIn, this, confidence, name, "he");
1069 return (confidence > 0);
73c04bcf
A		 1070}
1071
1072CharsetRecog_8859_9::~CharsetRecog_8859_9()
1073{
1074 // nothing to do
1075}
1076
1077const char *CharsetRecog_8859_9::getName() const
1078{
51004dcb 1079 return "ISO-8859-9";
73c04bcf
A		 1080}
1081
1082CharsetRecog_8859_9_tr::~CharsetRecog_8859_9_tr ()
1083{
1084 // nothing to do
1085}
1086
1087const char *CharsetRecog_8859_9_tr::getLanguage() const
1088{
1089 return "tr";
1090}
1091
51004dcb 1092UBool CharsetRecog_8859_9_tr::match(InputText *textIn, CharsetMatch *results) const
73c04bcf 1093{
51004dcb
A		 1094 const char *name = textIn->fC1Bytes? "windows-1254" : "ISO-8859-9";
1095 int32_t confidence = match_sbcs(textIn, ngrams_8859_9_tr, charMap_8859_9);
1096 results->set(textIn, this, confidence, name, "tr");
1097 return (confidence > 0);
73c04bcf
A		 1098}
1099
1100CharsetRecog_windows_1256::~CharsetRecog_windows_1256()
1101{
1102 // nothing to do
1103}
1104
1105const char *CharsetRecog_windows_1256::getName() const
1106{
1107 return "windows-1256";
1108}
1109
1110const char *CharsetRecog_windows_1256::getLanguage() const
1111{
1112 return "ar";
1113}
1114
51004dcb 1115UBool CharsetRecog_windows_1256::match(InputText *textIn, CharsetMatch *results) const
73c04bcf 1116{
51004dcb
A		 1117 int32_t confidence = match_sbcs(textIn, ngrams_windows_1256, charMap_windows_1256);
1118 results->set(textIn, this, confidence);
1119 return (confidence > 0);
73c04bcf
A		 1120}
1121
1122CharsetRecog_windows_1251::~CharsetRecog_windows_1251()
1123{
1124 // nothing to do
1125}
1126
1127const char *CharsetRecog_windows_1251::getName() const
1128{
1129 return "windows-1251";
1130}
1131
1132const char *CharsetRecog_windows_1251::getLanguage() const
1133{
1134 return "ru";
1135}
1136
51004dcb 1137UBool CharsetRecog_windows_1251::match(InputText *textIn, CharsetMatch *results) const
73c04bcf 1138{
51004dcb
A		 1139 int32_t confidence = match_sbcs(textIn, ngrams_windows_1251, charMap_windows_1251);
1140 results->set(textIn, this, confidence);
1141 return (confidence > 0);
73c04bcf
A		 1142}
1143
1144CharsetRecog_KOI8_R::~CharsetRecog_KOI8_R()
1145{
1146 // nothing to do
1147}
1148
1149const char *CharsetRecog_KOI8_R::getName() const
1150{
1151 return "KOI8-R";
1152}
1153
1154const char *CharsetRecog_KOI8_R::getLanguage() const
1155{
1156 return "ru";
1157}
1158
51004dcb 1159UBool CharsetRecog_KOI8_R::match(InputText *textIn, CharsetMatch *results) const
73c04bcf 1160{
51004dcb
A		 1161 int32_t confidence = match_sbcs(textIn, ngrams_KOI8_R, charMap_KOI8_R);
1162 results->set(textIn, this, confidence);
1163 return (confidence > 0);
73c04bcf
A		 1164}
1165
b331163b 1166#if !UCONFIG_ONLY_HTML_CONVERSION
729e4ab9
A		 1167CharsetRecog_IBM424_he::~CharsetRecog_IBM424_he()
1168{
1169 // nothing to do
1170}
1171
1172const char *CharsetRecog_IBM424_he::getLanguage() const
1173{
1174 return "he";
1175}
1176
1177CharsetRecog_IBM424_he_rtl::~CharsetRecog_IBM424_he_rtl()
1178{
1179 // nothing to do
1180}
1181
1182const char *CharsetRecog_IBM424_he_rtl::getName() const
1183{
1184 return "IBM424_rtl";
1185}
1186
51004dcb 1187UBool CharsetRecog_IBM424_he_rtl::match(InputText *textIn, CharsetMatch *results) const
729e4ab9 1188{
51004dcb
A		 1189 int32_t confidence = match_sbcs(textIn, ngrams_IBM424_he_rtl, charMap_IBM424_he);
1190 results->set(textIn, this, confidence);
1191 return (confidence > 0);
729e4ab9
A		 1192}
1193
1194CharsetRecog_IBM424_he_ltr::~CharsetRecog_IBM424_he_ltr()
1195{
1196 // nothing to do
1197}
1198
1199const char *CharsetRecog_IBM424_he_ltr::getName() const
1200{
1201 return "IBM424_ltr";
1202}
1203
51004dcb 1204UBool CharsetRecog_IBM424_he_ltr::match(InputText *textIn, CharsetMatch *results) const
729e4ab9 1205{
51004dcb
A		 1206 int32_t confidence = match_sbcs(textIn, ngrams_IBM424_he_ltr, charMap_IBM424_he);
1207 results->set(textIn, this, confidence);
1208 return (confidence > 0);
729e4ab9
A		 1209}
1210
729e4ab9
A		 1211CharsetRecog_IBM420_ar::~CharsetRecog_IBM420_ar()
1212{
1213 // nothing to do
1214}
1215
1216const char *CharsetRecog_IBM420_ar::getLanguage() const
1217{
1218 return "ar";
1219}
1220
729e4ab9 1221
57a6839d
A		 1222int32_t CharsetRecog_IBM420_ar::match_sbcs(InputText *det, const int32_t ngrams[], const uint8_t byteMap[]) const
1223{
1224 NGramParser_IBM420 parser(ngrams, byteMap);
1225 int32_t result;
729e4ab9 1226
57a6839d 1227 result = parser.parse(det);
729e4ab9 1228
57a6839d 1229 return result;
729e4ab9
A		 1230}
1231
1232CharsetRecog_IBM420_ar_rtl::~CharsetRecog_IBM420_ar_rtl()
1233{
1234 // nothing to do
1235}
1236
1237const char *CharsetRecog_IBM420_ar_rtl::getName() const
1238{
1239 return "IBM420_rtl";
1240}
1241
51004dcb 1242UBool CharsetRecog_IBM420_ar_rtl::match(InputText *textIn, CharsetMatch *results) const
729e4ab9 1243{
51004dcb
A		 1244 int32_t confidence = match_sbcs(textIn, ngrams_IBM420_ar_rtl, charMap_IBM420_ar);
1245 results->set(textIn, this, confidence);
1246 return (confidence > 0);
729e4ab9
A		 1247}
1248
1249CharsetRecog_IBM420_ar_ltr::~CharsetRecog_IBM420_ar_ltr()
1250{
1251 // nothing to do
1252}
1253
1254const char *CharsetRecog_IBM420_ar_ltr::getName() const
1255{
1256 return "IBM420_ltr";
1257}
1258
51004dcb 1259UBool CharsetRecog_IBM420_ar_ltr::match(InputText *textIn, CharsetMatch *results) const
729e4ab9 1260{
51004dcb
A		 1261 int32_t confidence = match_sbcs(textIn, ngrams_IBM420_ar_ltr, charMap_IBM420_ar);
1262 results->set(textIn, this, confidence);
1263 return (confidence > 0);
729e4ab9 1264}
b331163b 1265#endif
729e4ab9 1266
73c04bcf
A		 1267U_NAMESPACE_END
1268#endif
1269
mirror of ICU