icuSources/test/cintltst/ucsdetst.c

   1 /*
   2  ****************************************************************************
   3  * Copyright (c) 2005-2009, International Business Machines Corporation and *
   4  * others. All Rights Reserved.                                             *
   5  ****************************************************************************
   6  */
   7
   8 #include "unicode/utypes.h"
   9
  10 #include "unicode/ucsdet.h"
  11 #include "unicode/ucnv.h"
  12 #include "unicode/ustring.h"
  13
  14 #include "cintltst.h"
  15
  16 #include <stdlib.h>
  17 #include <string.h>
  18
  19 #define ARRAY_SIZE(array) (sizeof(array)/sizeof(array[0]))
  20
  21 #define NEW_ARRAY(type,count) (type *) malloc((count) * sizeof(type))
  22 #define DELETE_ARRAY(array) free(array)
  23
  24 static void TestConstruction(void);
  25 static void TestUTF8(void);
  26 static void TestUTF16(void);
  27 static void TestC1Bytes(void);
  28 static void TestInputFilter(void);
  29 static void TestChaining(void);
  30 static void TestBufferOverflow(void);
  31 static void TestIBM424(void);
  32 static void TestIBM420(void);
  33
  34 void addUCsdetTest(TestNode** root);
  35
  36 void addUCsdetTest(TestNode** root)
  37 {
  38     addTest(root, &TestConstruction, "ucsdetst/TestConstruction");
  39     addTest(root, &TestUTF8, "ucsdetst/TestUTF8");
  40     addTest(root, &TestUTF16, "ucsdetst/TestUTF16");
  41     addTest(root, &TestC1Bytes, "ucsdetst/TestC1Bytes");
  42     addTest(root, &TestInputFilter, "ucsdetst/TestInputFilter");
  43     addTest(root, &TestChaining, "ucsdetst/TestErrorChaining");
  44     addTest(root, &TestBufferOverflow, "ucsdetst/TestBufferOverflow");
  45 #if !UCONFIG_NO_LEGACY_CONVERSION
  46     addTest(root, &TestIBM424, "ucsdetst/TestIBM424");
  47     addTest(root, &TestIBM420, "ucsdetst/TestIBM420");
  48 #endif
  49 }
  50
  51 static int32_t preflight(const UChar *src, int32_t length, UConverter *cnv)
  52 {
  53     UErrorCode status;
  54     char buffer[1024];
  55     char *dest, *destLimit = buffer + sizeof(buffer);
  56     const UChar *srcLimit = src + length;
  57     int32_t result = 0;
  58
  59     do {
  60         dest = buffer;
  61         status = U_ZERO_ERROR;
  62         ucnv_fromUnicode(cnv, &dest, destLimit, &src, srcLimit, 0, TRUE, &status);
  63         result += (int32_t) (dest - buffer);
  64     } while (status == U_BUFFER_OVERFLOW_ERROR);
  65
  66     return result;
  67 }
  68
  69 static char *extractBytes(const UChar *src, int32_t length, const char *codepage, int32_t *byteLength)
  70 {
  71     UErrorCode status = U_ZERO_ERROR;
  72     UConverter *cnv = ucnv_open(codepage, &status);
  73     int32_t byteCount = preflight(src, length, cnv);
  74     const UChar *srcLimit = src + length;
  75     char *bytes = NEW_ARRAY(char, byteCount + 1);
  76     char *dest = bytes, *destLimit = bytes + byteCount + 1;
  77
  78     ucnv_fromUnicode(cnv, &dest, destLimit, &src, srcLimit, 0, TRUE, &status);
  79     ucnv_close(cnv);
  80
  81     *byteLength = byteCount;
  82     return bytes;
  83 }
  84
  85 static void freeBytes(char *bytes)
  86 {
  87     DELETE_ARRAY(bytes);
  88 }
  89
  90 static void TestConstruction(void)
  91 {
  92     UErrorCode status = U_ZERO_ERROR;
  93     UCharsetDetector *csd = ucsdet_open(&status);
  94     UEnumeration *e = ucsdet_getAllDetectableCharsets(csd, &status);
  95     const char *name;
  96     int32_t count = uenum_count(e, &status);
  97     int32_t i, length;
  98
  99     for(i = 0; i < count; i += 1) {
 100         name = uenum_next(e, &length, &status);
 101
 102         if(name == NULL || length <= 0) {
 103             log_err("ucsdet_getAllDetectableCharsets() returned a null or empty name!\n");
 104         }
 105     }
 106     /* one past the list of all names must return NULL */
 107     name = uenum_next(e, &length, &status);
 108     if(name != NULL || length != 0 || U_FAILURE(status)) {
 109         log_err("ucsdet_getAllDetectableCharsets(past the list) returned a non-null name!\n");
 110     }
 111
 112     uenum_close(e);
 113     ucsdet_close(csd);
 114 }
 115
 116 static void TestUTF8(void)
 117 {
 118     UErrorCode status = U_ZERO_ERROR;
 119     static const char ss[] = "This is a string with some non-ascii characters that will "
 120                "be converted to UTF-8, then shoved through the detection process.  "
 121                "\\u0391\\u0392\\u0393\\u0394\\u0395"
 122                "Sure would be nice if our source could contain Unicode directly!";
 123     int32_t byteLength = 0, sLength = 0, dLength = 0;
 124     UChar s[sizeof(ss)];
 125     char *bytes;
 126     UCharsetDetector *csd = ucsdet_open(&status);
 127     const UCharsetMatch *match;
 128     UChar detected[sizeof(ss)];
 129
 130     sLength = u_unescape(ss, s, sizeof(ss));
 131     bytes = extractBytes(s, sLength, "UTF-8", &byteLength);
 132
 133     ucsdet_setText(csd, bytes, byteLength, &status);
 134     if (U_FAILURE(status)) {
 135         log_err("status is %s\n", u_errorName(status));
 136         goto bail;
 137     }
 138
 139     match = ucsdet_detect(csd, &status);
 140
 141     if (match == NULL) {
 142         log_err("Detection failure for UTF-8: got no matches.\n");
 143         goto bail;
 144     }
 145
 146     dLength = ucsdet_getUChars(match, detected, sLength, &status);
 147
 148     if (u_strCompare(detected, dLength, s, sLength, FALSE) != 0) {
 149         log_err("Round-trip test failed!\n");
 150     }
 151
 152     ucsdet_setDeclaredEncoding(csd, "UTF-8", 5, &status); /* for coverage */
 153
 154 bail:
 155     freeBytes(bytes);
 156     ucsdet_close(csd);
 157 }
 158
 159 static void TestUTF16(void)
 160 {
 161     UErrorCode status = U_ZERO_ERROR;
 162     /* Notice the BOM on the start of this string */
 163     static const UChar chars[] = {
 164         0xFEFF, 0x0623, 0x0648, 0x0631, 0x0648, 0x0628, 0x0627, 0x002C,
 165         0x0020, 0x0628, 0x0631, 0x0645, 0x062c, 0x064a, 0x0627, 0x062a,
 166         0x0020, 0x0627, 0x0644, 0x062d, 0x0627, 0x0633, 0x0648, 0x0628,
 167         0x0020, 0x002b, 0x0020, 0x0627, 0x0646, 0x062a, 0x0631, 0x0646,
 168         0x064a, 0x062a, 0x0000};
 169     int32_t beLength = 0, leLength = 0, cLength = ARRAY_SIZE(chars);
 170     char *beBytes = extractBytes(chars, cLength, "UTF-16BE", &beLength);
 171     char *leBytes = extractBytes(chars, cLength, "UTF-16LE", &leLength);
 172     UCharsetDetector *csd = ucsdet_open(&status);
 173     const UCharsetMatch *match;
 174     const char *name;
 175     int32_t conf;
 176
 177     ucsdet_setText(csd, beBytes, beLength, &status);
 178     match = ucsdet_detect(csd, &status);
 179
 180     if (match == NULL) {
 181         log_err("Encoding detection failure for UTF-16BE: got no matches.\n");
 182         goto try_le;
 183     }
 184
 185     name  = ucsdet_getName(match, &status);
 186     conf  = ucsdet_getConfidence(match, &status);
 187
 188     if (strcmp(name, "UTF-16BE") != 0) {
 189         log_err("Encoding detection failure for UTF-16BE: got %s\n", name);
 190     }
 191
 192     if (conf != 100) {
 193         log_err("Did not get 100%% confidence for UTF-16BE: got %d\n", conf);
 194     }
 195
 196 try_le:
 197     ucsdet_setText(csd, leBytes, leLength, &status);
 198     match = ucsdet_detect(csd, &status);
 199
 200     if (match == NULL) {
 201         log_err("Encoding detection failure for UTF-16LE: got no matches.\n");
 202         goto bail;
 203     }
 204
 205     name  = ucsdet_getName(match, &status);
 206     conf = ucsdet_getConfidence(match, &status);
 207
 208
 209     if (strcmp(name, "UTF-16LE") != 0) {
 210         log_err("Enconding detection failure for UTF-16LE: got %s\n", name);
 211     }
 212
 213     if (conf != 100) {
 214         log_err("Did not get 100%% confidence for UTF-16LE: got %d\n", conf);
 215     }
 216
 217 bail:
 218     freeBytes(leBytes);
 219     freeBytes(beBytes);
 220     ucsdet_close(csd);
 221 }
 222
 223 static void TestC1Bytes(void)
 224 {
 225 #if !UCONFIG_NO_LEGACY_CONVERSION
 226     UErrorCode status = U_ZERO_ERROR;
 227     static const char ssISO[] = "This is a small sample of some English text. Just enough to be sure that it detects correctly.";
 228     static const char ssWindows[] = "This is another small sample of some English text. Just enough to be sure that it detects correctly. It also includes some \\u201CC1\\u201D bytes.";
 229     int32_t sISOLength = 0, sWindowsLength = 0;
 230     UChar sISO[sizeof(ssISO)];
 231     UChar sWindows[sizeof(ssWindows)];
 232     int32_t lISO = 0, lWindows = 0;
 233     char *bISO;
 234     char *bWindows;
 235     UCharsetDetector *csd = ucsdet_open(&status);
 236     const UCharsetMatch *match;
 237     const char *name;
 238
 239     sISOLength = u_unescape(ssISO, sISO, sizeof(ssISO));
 240     sWindowsLength = u_unescape(ssWindows, sWindows, sizeof(ssWindows));
 241     bISO = extractBytes(sISO, sISOLength, "ISO-8859-1", &lISO);
 242     bWindows = extractBytes(sWindows, sWindowsLength, "windows-1252", &lWindows);
 243
 244     ucsdet_setText(csd, bWindows, lWindows, &status);
 245     match = ucsdet_detect(csd, &status);
 246
 247     if (match == NULL) {
 248         log_err("English test with C1 bytes got no matches.\n");
 249         goto bail;
 250     }
 251
 252     name  = ucsdet_getName(match, &status);
 253
 254     if (strcmp(name, "windows-1252") != 0) {
 255         log_data_err("English text with C1 bytes does not detect as windows-1252, but as %s. (Are you missing data?)\n", name);
 256     }
 257
 258     ucsdet_setText(csd, bISO, lISO, &status);
 259     match = ucsdet_detect(csd, &status);
 260
 261     if (match == NULL) {
 262         log_err("English text without C1 bytes got no matches.\n");
 263         goto bail;
 264     }
 265
 266     name  = ucsdet_getName(match, &status);
 267
 268     if (strcmp(name, "ISO-8859-1") != 0) {
 269         log_err("English text without C1 bytes does not detect as ISO-8859-1, but as %s\n", name);
 270     }
 271
 272 bail:
 273     freeBytes(bWindows);
 274     freeBytes(bISO);
 275
 276     ucsdet_close(csd);
 277 #endif
 278 }
 279
 280 static void TestInputFilter(void)
 281 {
 282     UErrorCode status = U_ZERO_ERROR;
 283     static const char ss[] = "<a> <lot> <of> <English> <inside> <the> <markup> Un tr\\u00E8s petit peu de Fran\\u00E7ais. <to> <confuse> <the> <detector>";
 284     int32_t sLength = 0;
 285     UChar s[sizeof(ss)];
 286     int32_t byteLength = 0;
 287     char *bytes;
 288     UCharsetDetector *csd = ucsdet_open(&status);
 289     const UCharsetMatch *match;
 290     const char *lang, *name;
 291
 292     sLength = u_unescape(ss, s, sizeof(ss));
 293     bytes = extractBytes(s, sLength, "ISO-8859-1", &byteLength);
 294
 295     ucsdet_enableInputFilter(csd, TRUE);
 296
 297     if (!ucsdet_isInputFilterEnabled(csd)) {
 298         log_err("ucsdet_enableInputFilter(csd, TRUE) did not enable input filter!\n");
 299     }
 300
 301
 302     ucsdet_setText(csd, bytes, byteLength, &status);
 303     match = ucsdet_detect(csd, &status);
 304
 305     if (match == NULL) {
 306         log_err("Turning on the input filter resulted in no matches.\n");
 307         goto turn_off;
 308     }
 309
 310     name = ucsdet_getName(match, &status);
 311
 312     if (name == NULL || strcmp(name, "ISO-8859-1") != 0) {
 313         log_err("Turning on the input filter resulted in %s rather than ISO-8859-1\n", name);
 314     } else {
 315         lang = ucsdet_getLanguage(match, &status);
 316
 317         if (lang == NULL || strcmp(lang, "fr") != 0) {
 318             log_err("Input filter did not strip markup!\n");
 319         }
 320     }
 321
 322 turn_off:
 323     ucsdet_enableInputFilter(csd, FALSE);
 324     ucsdet_setText(csd, bytes, byteLength, &status);
 325     match = ucsdet_detect(csd, &status);
 326
 327     if (match == NULL) {
 328         log_err("Turning off the input filter resulted in no matches.\n");
 329         goto bail;
 330     }
 331
 332     name = ucsdet_getName(match, &status);
 333
 334     if (name == NULL || strcmp(name, "ISO-8859-1") != 0) {
 335         log_err("Turning off the input filter resulted in %s rather than ISO-8859-1\n", name);
 336     } else {
 337         lang = ucsdet_getLanguage(match, &status);
 338
 339         if (lang == NULL || strcmp(lang, "en") != 0) {
 340             log_err("Unfiltered input did not detect as English!\n");
 341         }
 342     }
 343
 344 bail:
 345     freeBytes(bytes);
 346     ucsdet_close(csd);
 347 }
 348
 349 static void TestChaining(void) {
 350     UErrorCode status = U_USELESS_COLLATOR_ERROR;
 351
 352     ucsdet_open(&status);
 353     ucsdet_setText(NULL, NULL, 0, &status);
 354     ucsdet_getName(NULL, &status);
 355     ucsdet_getConfidence(NULL, &status);
 356     ucsdet_getLanguage(NULL, &status);
 357     ucsdet_detect(NULL, &status);
 358     ucsdet_setDeclaredEncoding(NULL, NULL, 0, &status);
 359     ucsdet_detectAll(NULL, NULL, &status);
 360     ucsdet_getUChars(NULL, NULL, 0, &status);
 361     ucsdet_getUChars(NULL, NULL, 0, &status);
 362     ucsdet_close(NULL);
 363
 364     /* All of this code should have done nothing. */
 365     if (status != U_USELESS_COLLATOR_ERROR) {
 366         log_err("Status got changed to %s\n", u_errorName(status));
 367     }
 368 }
 369
 370 static void TestBufferOverflow(void) {
 371     UErrorCode status = U_ZERO_ERROR;
 372     static const char *testStrings[] = {
 373         "\x80\x20\x54\x68\x69\x73\x20\x69\x73\x20\x45\x6E\x67\x6C\x69\x73\x68\x20\x1b", /* A partial ISO-2022 shift state at the end */
 374         "\x80\x20\x54\x68\x69\x73\x20\x69\x73\x20\x45\x6E\x67\x6C\x69\x73\x68\x20\x1b\x24", /* A partial ISO-2022 shift state at the end */
 375         "\x80\x20\x54\x68\x69\x73\x20\x69\x73\x20\x45\x6E\x67\x6C\x69\x73\x68\x20\x1b\x24\x28", /* A partial ISO-2022 shift state at the end */
 376         "\x80\x20\x54\x68\x69\x73\x20\x69\x73\x20\x45\x6E\x67\x6C\x69\x73\x68\x20\x1b\x24\x28\x44", /* A complete ISO-2022 shift state at the end with a bad one at the start */
 377         "\x1b\x24\x28\x44", /* A complete ISO-2022 shift state at the end */
 378         "\xa1", /* Could be a single byte shift-jis at the end */
 379         "\x74\x68\xa1", /* Could be a single byte shift-jis at the end */
 380         "\x74\x68\x65\xa1" /* Could be a single byte shift-jis at the end, but now we have English creeping in. */
 381     };
 382     static const char *testResults[] = {
 383         "windows-1252",
 384         "windows-1252",
 385         "windows-1252",
 386         "windows-1252",
 387         "ISO-2022-JP",
 388         NULL,
 389         NULL,
 390         "ISO-8859-1"
 391     };
 392     int32_t idx = 0;
 393     UCharsetDetector *csd = ucsdet_open(&status);
 394     const UCharsetMatch *match;
 395
 396     ucsdet_setDeclaredEncoding(csd, "ISO-2022-JP", -1, &status);
 397
 398     if (U_FAILURE(status)) {
 399         log_err("Couldn't open detector. %s\n", u_errorName(status));
 400         goto bail;
 401     }
 402
 403     for (idx = 0; idx < ARRAY_SIZE(testStrings); idx++) {
 404         ucsdet_setText(csd, testStrings[idx], -1, &status);
 405         match = ucsdet_detect(csd, &status);
 406
 407         if (match == NULL) {
 408             if (testResults[idx] != NULL) {
 409                 log_err("Unexpectedly got no results at index %d.\n", idx);
 410             }
 411             else {
 412                 log_verbose("Got no result as expected at index %d.\n", idx);
 413             }
 414             continue;
 415         }
 416
 417         if (testResults[idx] == NULL || strcmp(ucsdet_getName(match, &status), testResults[idx]) != 0) {
 418             log_err("Unexpectedly got %s instead of %s at index %d with confidence %d.\n",
 419                 ucsdet_getName(match, &status), testResults[idx], idx, ucsdet_getConfidence(match, &status));
 420             goto bail;
 421         }
 422     }
 423
 424 bail:
 425     ucsdet_close(csd);
 426 }
 427
 428 static void TestIBM424(void)
 429 {
 430     UErrorCode status = U_ZERO_ERROR;
 431
 432     static const UChar chars[] = {
 433             0x05D4, 0x05E4, 0x05E8, 0x05E7, 0x05DC, 0x05D9, 0x05D8, 0x0020, 0x05D4, 0x05E6, 0x05D1, 0x05D0, 0x05D9, 0x0020, 0x05D4, 0x05E8,
 434             0x05D0, 0x05E9, 0x05D9, 0x002C, 0x0020, 0x05EA, 0x05EA, 0x0020, 0x05D0, 0x05DC, 0x05D5, 0x05E3, 0x0020, 0x05D0, 0x05D1, 0x05D9,
 435             0x05D7, 0x05D9, 0x0020, 0x05DE, 0x05E0, 0x05D3, 0x05DC, 0x05D1, 0x05DC, 0x05D9, 0x05D8, 0x002C, 0x0020, 0x05D4, 0x05D5, 0x05E8,
 436             0x05D4, 0x0020, 0x05E2, 0x05DC, 0x0020, 0x05E4, 0x05EA, 0x05D9, 0x05D7, 0x05EA, 0x0020, 0x05D7, 0x05E7, 0x05D9, 0x05E8, 0x05EA,
 437             0x0020, 0x05DE, 0x05E6, 0x0022, 0x05D7, 0x0020, 0x05D1, 0x05E2, 0x05E7, 0x05D1, 0x05D5, 0x05EA, 0x0020, 0x05E2, 0x05D3, 0x05D5,
 438             0x05D9, 0x05D5, 0x05EA, 0x0020, 0x05D7, 0x05D9, 0x05D9, 0x05DC, 0x05D9, 0x0020, 0x05E6, 0x05D4, 0x0022, 0x05DC, 0x0020, 0x05DE,
 439             0x05DE, 0x05D1, 0x05E6, 0x05E2, 0x0020, 0x05E2, 0x05D5, 0x05E4, 0x05E8, 0x05EA, 0x0020, 0x05D9, 0x05E6, 0x05D5, 0x05E7, 0x05D4,
 440             0x0020, 0x05D1, 0x002B, 0x0020, 0x05E8, 0x05E6, 0x05D5, 0x05E2, 0x05EA, 0x0020, 0x05E2, 0x05D6, 0x05D4, 0x002E, 0x0020, 0x05DC,
 441             0x05D3, 0x05D1, 0x05E8, 0x05D9, 0x0020, 0x05D4, 0x05E4, 0x05E6, 0x0022, 0x05E8, 0x002C, 0x0020, 0x05DE, 0x05D4, 0x05E2, 0x05D3,
 442             0x05D5, 0x05D9, 0x05D5, 0x05EA, 0x0020, 0x05E2, 0x05D5, 0x05DC, 0x05D4, 0x0020, 0x05EA, 0x05DE, 0x05D5, 0x05E0, 0x05D4, 0x0020,
 443             0x05E9, 0x05DC, 0x0020, 0x0022, 0x05D4, 0x05EA, 0x05E0, 0x05D4, 0x05D2, 0x05D5, 0x05EA, 0x0020, 0x05E4, 0x05E1, 0x05D5, 0x05DC,
 444             0x05D4, 0x0020, 0x05DC, 0x05DB, 0x05D0, 0x05D5, 0x05E8, 0x05D4, 0x0020, 0x05E9, 0x05DC, 0x0020, 0x05D7, 0x05D9, 0x05D9, 0x05DC,
 445             0x05D9, 0x05DD, 0x0020, 0x05D1, 0x05DE, 0x05D4, 0x05DC, 0x05DA, 0x0020, 0x05DE, 0x05D1, 0x05E6, 0x05E2, 0x0020, 0x05E2, 0x05D5,
 446             0x05E4, 0x05E8, 0x05EA, 0x0020, 0x05D9, 0x05E6, 0x05D5, 0x05E7, 0x05D4, 0x0022, 0x002E, 0x0020, 0x05DE, 0x05E0, 0x05D3, 0x05DC,
 447             0x05D1, 0x05DC, 0x05D9, 0x05D8, 0x0020, 0x05E7, 0x05D9, 0x05D1, 0x05DC, 0x0020, 0x05D0, 0x05EA, 0x0020, 0x05D4, 0x05D7, 0x05DC,
 448             0x05D8, 0x05EA, 0x05D5, 0x0020, 0x05DC, 0x05D0, 0x05D7, 0x05E8, 0x0020, 0x05E9, 0x05E2, 0x05D9, 0x05D9, 0x05DF, 0x0020, 0x05D1,
 449             0x05EA, 0x05DE, 0x05DC, 0x05D9, 0x05DC, 0x0020, 0x05D4, 0x05E2, 0x05D3, 0x05D5, 0x05D9, 0x05D5, 0x05EA, 0x0000
 450     };
 451
 452     static const UChar chars_reverse[] = {
 453             0x05EA, 0x05D5, 0x05D9, 0x05D5, 0x05D3, 0x05E2, 0x05D4, 0x0020, 0x05DC, 0x05D9, 0x05DC, 0x05DE, 0x05EA,
 454             0x05D1, 0x0020, 0x05DF, 0x05D9, 0x05D9, 0x05E2, 0x05E9, 0x0020, 0x05E8, 0x05D7, 0x05D0, 0x05DC, 0x0020, 0x05D5, 0x05EA, 0x05D8,
 455             0x05DC, 0x05D7, 0x05D4, 0x0020, 0x05EA, 0x05D0, 0x0020, 0x05DC, 0x05D1, 0x05D9, 0x05E7, 0x0020, 0x05D8, 0x05D9, 0x05DC, 0x05D1,
 456             0x05DC, 0x05D3, 0x05E0, 0x05DE, 0x0020, 0x002E, 0x0022, 0x05D4, 0x05E7, 0x05D5, 0x05E6, 0x05D9, 0x0020, 0x05EA, 0x05E8, 0x05E4,
 457             0x05D5, 0x05E2, 0x0020, 0x05E2, 0x05E6, 0x05D1, 0x05DE, 0x0020, 0x05DA, 0x05DC, 0x05D4, 0x05DE, 0x05D1, 0x0020, 0x05DD, 0x05D9,
 458             0x05DC, 0x05D9, 0x05D9, 0x05D7, 0x0020, 0x05DC, 0x05E9, 0x0020, 0x05D4, 0x05E8, 0x05D5, 0x05D0, 0x05DB, 0x05DC, 0x0020, 0x05D4,
 459             0x05DC, 0x05D5, 0x05E1, 0x05E4, 0x0020, 0x05EA, 0x05D5, 0x05D2, 0x05D4, 0x05E0, 0x05EA, 0x05D4, 0x0022, 0x0020, 0x05DC, 0x05E9,
 460             0x0020, 0x05D4, 0x05E0, 0x05D5, 0x05DE, 0x05EA, 0x0020, 0x05D4, 0x05DC, 0x05D5, 0x05E2, 0x0020, 0x05EA, 0x05D5, 0x05D9, 0x05D5,
 461             0x05D3, 0x05E2, 0x05D4, 0x05DE, 0x0020, 0x002C, 0x05E8, 0x0022, 0x05E6, 0x05E4, 0x05D4, 0x0020, 0x05D9, 0x05E8, 0x05D1, 0x05D3,
 462             0x05DC, 0x0020, 0x002E, 0x05D4, 0x05D6, 0x05E2, 0x0020, 0x05EA, 0x05E2, 0x05D5, 0x05E6, 0x05E8, 0x0020, 0x002B, 0x05D1, 0x0020,
 463             0x05D4, 0x05E7, 0x05D5, 0x05E6, 0x05D9, 0x0020, 0x05EA, 0x05E8, 0x05E4, 0x05D5, 0x05E2, 0x0020, 0x05E2, 0x05E6, 0x05D1, 0x05DE,
 464             0x05DE, 0x0020, 0x05DC, 0x0022, 0x05D4, 0x05E6, 0x0020, 0x05D9, 0x05DC, 0x05D9, 0x05D9, 0x05D7, 0x0020, 0x05EA, 0x05D5, 0x05D9,
 465             0x05D5, 0x05D3, 0x05E2, 0x0020, 0x05EA, 0x05D5, 0x05D1, 0x05E7, 0x05E2, 0x05D1, 0x0020, 0x05D7, 0x0022, 0x05E6, 0x05DE, 0x0020,
 466             0x05EA, 0x05E8, 0x05D9, 0x05E7, 0x05D7, 0x0020, 0x05EA, 0x05D7, 0x05D9, 0x05EA, 0x05E4, 0x0020, 0x05DC, 0x05E2, 0x0020, 0x05D4,
 467             0x05E8, 0x05D5, 0x05D4, 0x0020, 0x002C, 0x05D8, 0x05D9, 0x05DC, 0x05D1, 0x05DC, 0x05D3, 0x05E0, 0x05DE, 0x0020, 0x05D9, 0x05D7,
 468             0x05D9, 0x05D1, 0x05D0, 0x0020, 0x05E3, 0x05D5, 0x05DC, 0x05D0, 0x0020, 0x05EA, 0x05EA, 0x0020, 0x002C, 0x05D9, 0x05E9, 0x05D0,
 469             0x05E8, 0x05D4, 0x0020, 0x05D9, 0x05D0, 0x05D1, 0x05E6, 0x05D4, 0x0020, 0x05D8, 0x05D9, 0x05DC, 0x05E7, 0x05E8, 0x05E4, 0x05D4,
 470             0x0000
 471     };
 472
 473     int32_t bLength = 0, brLength = 0, cLength = ARRAY_SIZE(chars), crLength = ARRAY_SIZE(chars_reverse);
 474
 475     char *bytes = extractBytes(chars, cLength, "IBM424", &bLength);
 476     char *bytes_r = extractBytes(chars_reverse, crLength, "IBM424", &brLength);
 477
 478     UCharsetDetector *csd = ucsdet_open(&status);
 479     const UCharsetMatch *match;
 480     const char *name;
 481
 482     ucsdet_setText(csd, bytes, bLength, &status);
 483     match = ucsdet_detect(csd, &status);
 484
 485     if (match == NULL) {
 486         log_err("Encoding detection failure for IBM424_rtl: got no matches.\n");
 487         goto bail;
 488     }
 489
 490     name  = ucsdet_getName(match, &status);
 491     if (strcmp(name, "IBM424_rtl") != 0) {
 492         log_data_err("Encoding detection failure for IBM424_rtl: got %s. (Are you missing data?)\n", name);
 493     }
 494
 495     ucsdet_setText(csd, bytes_r, brLength, &status);
 496     match = ucsdet_detect(csd, &status);
 497
 498     if (match == NULL) {
 499         log_err("Encoding detection failure for IBM424_ltr: got no matches.\n");
 500         goto bail;
 501     }
 502
 503     name  = ucsdet_getName(match, &status);
 504     if (strcmp(name, "IBM424_ltr") != 0) {
 505         log_data_err("Encoding detection failure for IBM424_ltr: got %s. (Are you missing data?)\n", name);
 506     }
 507
 508 bail:
 509     freeBytes(bytes);
 510     freeBytes(bytes_r);
 511     ucsdet_close(csd);
 512 }
 513
 514 static void TestIBM420(void)
 515 {
 516     UErrorCode status = U_ZERO_ERROR;
 517
 518     static const UChar chars[] = {
 519         0x0648, 0x064F, 0x0636, 0x0639, 0x062A, 0x0020, 0x0648, 0x0646, 0x064F, 0x0641, 0x0630, 0x062A, 0x0020, 0x0628, 0x0631, 0x0627,
 520         0x0645, 0x062C, 0x0020, 0x062A, 0x0623, 0x0645, 0x064A, 0x0646, 0x0020, 0x0639, 0x062F, 0x064A, 0x062F, 0x0629, 0x0020, 0x0641,
 521         0x064A, 0x0020, 0x0645, 0x0624, 0x0633, 0x0633, 0x0629, 0x0020, 0x0627, 0x0644, 0x062A, 0x0623, 0x0645, 0x064A, 0x0646, 0x0020,
 522         0x0627, 0x0644, 0x0648, 0x0637, 0x0646, 0x064A, 0x002C, 0x0020, 0x0645, 0x0639, 0x0020, 0x0645, 0x0644, 0x0627, 0x0626, 0x0645,
 523         0x062A, 0x0647, 0x0627, 0x0020, 0x062F, 0x0627, 0x0626, 0x0645, 0x0627, 0x064B, 0x0020, 0x0644, 0x0644, 0x0627, 0x062D, 0x062A,
 524         0x064A, 0x0627, 0x062C, 0x0627, 0x062A, 0x0020, 0x0627, 0x0644, 0x0645, 0x062A, 0x063A, 0x064A, 0x0631, 0x0629, 0x0020, 0x0644,
 525         0x0644, 0x0645, 0x062C, 0x062A, 0x0645, 0x0639, 0x0020, 0x0648, 0x0644, 0x0644, 0x062F, 0x0648, 0x0644, 0x0629, 0x002E, 0x0020,
 526         0x062A, 0x0648, 0x0633, 0x0639, 0x062A, 0x0020, 0x0648, 0x062A, 0x0637, 0x0648, 0x0631, 0x062A, 0x0020, 0x0627, 0x0644, 0x0645,
 527         0x0624, 0x0633, 0x0633, 0x0629, 0x0020, 0x0628, 0x0647, 0x062F, 0x0641, 0x0020, 0x0636, 0x0645, 0x0627, 0x0646, 0x0020, 0x0634,
 528         0x0628, 0x0643, 0x0629, 0x0020, 0x0623, 0x0645, 0x0627, 0x0646, 0x0020, 0x0644, 0x0633, 0x0643, 0x0627, 0x0646, 0x0020, 0x062F,
 529         0x0648, 0x0644, 0x0629, 0x0020, 0x0627, 0x0633, 0x0631, 0x0627, 0x0626, 0x064A, 0x0644, 0x0020, 0x0628, 0x0648, 0x062C, 0x0647,
 530         0x0020, 0x0627, 0x0644, 0x0645, 0x062E, 0x0627, 0x0637, 0x0631, 0x0020, 0x0627, 0x0644, 0x0627, 0x0642, 0x062A, 0x0635, 0x0627,
 531         0x062F, 0x064A, 0x0629, 0x0020, 0x0648, 0x0627, 0x0644, 0x0627, 0x062C, 0x062A, 0x0645, 0x0627, 0x0639, 0x064A, 0x0629, 0x002E,
 532         0x0000
 533     };
 534     static const UChar chars_reverse[] = {
 535         0x002E, 0x0629, 0x064A, 0x0639, 0x0627, 0x0645, 0x062A, 0x062C, 0x0627, 0x0644, 0x0627, 0x0648, 0x0020, 0x0629, 0x064A, 0x062F,
 536         0x0627, 0x0635, 0x062A, 0x0642, 0x0627, 0x0644, 0x0627, 0x0020, 0x0631, 0x0637, 0x0627, 0x062E, 0x0645, 0x0644, 0x0627, 0x0020,
 537         0x0647, 0x062C, 0x0648, 0x0628, 0x0020, 0x0644, 0x064A, 0x0626, 0x0627, 0x0631, 0x0633, 0x0627, 0x0020, 0x0629, 0x0644, 0x0648,
 538         0x062F, 0x0020, 0x0646, 0x0627, 0x0643, 0x0633, 0x0644, 0x0020, 0x0646, 0x0627, 0x0645, 0x0623, 0x0020, 0x0629, 0x0643, 0x0628,
 539         0x0634, 0x0020, 0x0646, 0x0627, 0x0645, 0x0636, 0x0020, 0x0641, 0x062F, 0x0647, 0x0628, 0x0020, 0x0629, 0x0633, 0x0633, 0x0624,
 540         0x0645, 0x0644, 0x0627, 0x0020, 0x062A, 0x0631, 0x0648, 0x0637, 0x062A, 0x0648, 0x0020, 0x062A, 0x0639, 0x0633, 0x0648, 0x062A,
 541         0x0020, 0x002E, 0x0629, 0x0644, 0x0648, 0x062F, 0x0644, 0x0644, 0x0648, 0x0020, 0x0639, 0x0645, 0x062A, 0x062C, 0x0645, 0x0644,
 542         0x0644, 0x0020, 0x0629, 0x0631, 0x064A, 0x063A, 0x062A, 0x0645, 0x0644, 0x0627, 0x0020, 0x062A, 0x0627, 0x062C, 0x0627, 0x064A,
 543         0x062A, 0x062D, 0x0627, 0x0644, 0x0644, 0x0020, 0x064B, 0x0627, 0x0645, 0x0626, 0x0627, 0x062F, 0x0020, 0x0627, 0x0647, 0x062A,
 544         0x0645, 0x0626, 0x0627, 0x0644, 0x0645, 0x0020, 0x0639, 0x0645, 0x0020, 0x002C, 0x064A, 0x0646, 0x0637, 0x0648, 0x0644, 0x0627,
 545         0x0020, 0x0646, 0x064A, 0x0645, 0x0623, 0x062A, 0x0644, 0x0627, 0x0020, 0x0629, 0x0633, 0x0633, 0x0624, 0x0645, 0x0020, 0x064A,
 546         0x0641, 0x0020, 0x0629, 0x062F, 0x064A, 0x062F, 0x0639, 0x0020, 0x0646, 0x064A, 0x0645, 0x0623, 0x062A, 0x0020, 0x062C, 0x0645,
 547         0x0627, 0x0631, 0x0628, 0x0020, 0x062A, 0x0630, 0x0641, 0x064F, 0x0646, 0x0648, 0x0020, 0x062A, 0x0639, 0x0636, 0x064F, 0x0648,
 548         0x0000,
 549     };
 550
 551     int32_t bLength = 0, brLength = 0, cLength = ARRAY_SIZE(chars), crLength = ARRAY_SIZE(chars_reverse);
 552
 553     char *bytes = extractBytes(chars, cLength, "IBM420", &bLength);
 554     char *bytes_r = extractBytes(chars_reverse, crLength, "IBM420", &brLength);
 555
 556     UCharsetDetector *csd = ucsdet_open(&status);
 557     const UCharsetMatch *match;
 558     const char *name;
 559
 560     ucsdet_setText(csd, bytes, bLength, &status);
 561     match = ucsdet_detect(csd, &status);
 562
 563     if (match == NULL) {
 564         log_err("Encoding detection failure for IBM420_rtl: got no matches.\n");
 565         goto bail;
 566     }
 567
 568     name  = ucsdet_getName(match, &status);
 569     if (strcmp(name, "IBM420_rtl") != 0) {
 570         log_data_err("Encoding detection failure for IBM420_rtl: got %s. (Are you missing data?)\n", name);
 571     }
 572
 573     ucsdet_setText(csd, bytes_r, brLength, &status);
 574     match = ucsdet_detect(csd, &status);
 575
 576     if (match == NULL) {
 577         log_err("Encoding detection failure for IBM420_ltr: got no matches.\n");
 578         goto bail;
 579     }
 580
 581     name  = ucsdet_getName(match, &status);
 582     if (strcmp(name, "IBM420_ltr") != 0) {
 583         log_data_err("Encoding detection failure for IBM420_ltr: got %s. (Are you missing data?)\n", name);
 584     }
 585
 586 bail:
 587     freeBytes(bytes);
 588     freeBytes(bytes_r);
 589     ucsdet_close(csd);
 590 }