+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
/*
****************************************************************************
- * Copyright (c) 2005-2006, International Business Machines Corporation and *
+ * Copyright (c) 2005-2016, International Business Machines Corporation and *
* others. All Rights Reserved. *
****************************************************************************
*/
#include "unicode/ustring.h"
#include "cintltst.h"
+#include "cmemory.h"
#include <stdlib.h>
#include <string.h>
-#define ARRAY_SIZE(array) (sizeof array / sizeof array[0])
-
-#define NEW_ARRAY(type,count) (type *) ctst_malloc((count) * sizeof(type))
-#define DELETE_ARRAY(array)
+#define NEW_ARRAY(type,count) (type *) malloc((count) * sizeof(type))
+#define DELETE_ARRAY(array) free(array)
static void TestConstruction(void);
static void TestUTF8(void);
static void TestC1Bytes(void);
static void TestInputFilter(void);
static void TestChaining(void);
+static void TestBufferOverflow(void);
+static void TestIBM424(void);
+static void TestIBM420(void);
void addUCsdetTest(TestNode** root);
addTest(root, &TestC1Bytes, "ucsdetst/TestC1Bytes");
addTest(root, &TestInputFilter, "ucsdetst/TestInputFilter");
addTest(root, &TestChaining, "ucsdetst/TestErrorChaining");
+ addTest(root, &TestBufferOverflow, "ucsdetst/TestBufferOverflow");
+#if !UCONFIG_NO_LEGACY_CONVERSION
+ addTest(root, &TestIBM424, "ucsdetst/TestIBM424");
+ addTest(root, &TestIBM420, "ucsdetst/TestIBM420");
+#endif
}
static int32_t preflight(const UChar *src, int32_t length, UConverter *cnv)
return result;
}
-static UChar *unescape(const char *src, int32_t *length)
-{
- int32_t charCount = u_unescape(src, NULL, 0);
- UChar *chars = NEW_ARRAY(UChar, charCount + 1);
-
- u_unescape(src, chars, charCount);
-
- *length = charCount;
- return chars;
-}
-
static char *extractBytes(const UChar *src, int32_t length, const char *codepage, int32_t *byteLength)
{
UErrorCode status = U_ZERO_ERROR;
static void TestUTF8(void)
{
UErrorCode status = U_ZERO_ERROR;
- const char *ss = "This is a string with some non-ascii characters that will "
+ static const char ss[] = "This is a string with some non-ascii characters that will "
"be converted to UTF-8, then shoved through the detection process. "
"\\u0391\\u0392\\u0393\\u0394\\u0395"
"Sure would be nice if our source could contain Unicode directly!";
int32_t byteLength = 0, sLength = 0, dLength = 0;
- UChar *s = unescape(ss, &sLength);
- char *bytes = extractBytes(s, sLength, "UTF-8", &byteLength);
+ UChar s[sizeof(ss)];
+ char *bytes;
UCharsetDetector *csd = ucsdet_open(&status);
const UCharsetMatch *match;
- UChar *detected = NEW_ARRAY(UChar, sLength);
+ UChar detected[sizeof(ss)];
+
+ sLength = u_unescape(ss, s, sizeof(ss));
+ bytes = extractBytes(s, sLength, "UTF-8", &byteLength);
ucsdet_setText(csd, bytes, byteLength, &status);
+ if (U_FAILURE(status)) {
+ log_err("status is %s\n", u_errorName(status));
+ goto bail;
+ }
+
match = ucsdet_detect(csd, &status);
if (match == NULL) {
ucsdet_setDeclaredEncoding(csd, "UTF-8", 5, &status); /* for coverage */
bail:
- DELETE_ARRAY(detected);
freeBytes(bytes);
ucsdet_close(csd);
}
{
UErrorCode status = U_ZERO_ERROR;
/* Notice the BOM on the start of this string */
- UChar chars[] = {
+ static const UChar chars[] = {
0xFEFF, 0x0623, 0x0648, 0x0631, 0x0648, 0x0628, 0x0627, 0x002C,
0x0020, 0x0628, 0x0631, 0x0645, 0x062c, 0x064a, 0x0627, 0x062a,
0x0020, 0x0627, 0x0644, 0x062d, 0x0627, 0x0633, 0x0648, 0x0628,
0x0020, 0x002b, 0x0020, 0x0627, 0x0646, 0x062a, 0x0631, 0x0646,
0x064a, 0x062a, 0x0000};
- int32_t beLength = 0, leLength = 0, cLength = ARRAY_SIZE(chars);
+ int32_t beLength = 0, leLength = 0, cLength = UPRV_LENGTHOF(chars);
char *beBytes = extractBytes(chars, cLength, "UTF-16BE", &beLength);
char *leBytes = extractBytes(chars, cLength, "UTF-16LE", &leLength);
UCharsetDetector *csd = ucsdet_open(&status);
{
#if !UCONFIG_NO_LEGACY_CONVERSION
UErrorCode status = U_ZERO_ERROR;
- const char *ssISO = "This is a small sample of some English text. Just enough to be sure that it detects correctly.";
- const char *ssWindows = "This is another small sample of some English text. Just enough to be sure that it detects correctly. It also includes some \\u201CC1\\u201D bytes.";
+ static const char ssISO[] = "This is a small sample of some English text. Just enough to be sure that it detects correctly.";
+ static const char ssWindows[] = "This is another small sample of some English text. Just enough to be sure that it detects correctly. It also includes some \\u201CC1\\u201D bytes.";
int32_t sISOLength = 0, sWindowsLength = 0;
- UChar *sISO = unescape(ssISO, &sISOLength);
- UChar *sWindows = unescape(ssWindows, &sWindowsLength);
+ UChar sISO[sizeof(ssISO)];
+ UChar sWindows[sizeof(ssWindows)];
int32_t lISO = 0, lWindows = 0;
- char *bISO = extractBytes(sISO, sISOLength, "ISO-8859-1", &lISO);
- char *bWindows = extractBytes(sWindows, sWindowsLength, "windows-1252", &lWindows);
+ char *bISO;
+ char *bWindows;
UCharsetDetector *csd = ucsdet_open(&status);
const UCharsetMatch *match;
const char *name;
+ sISOLength = u_unescape(ssISO, sISO, sizeof(ssISO));
+ sWindowsLength = u_unescape(ssWindows, sWindows, sizeof(ssWindows));
+ bISO = extractBytes(sISO, sISOLength, "ISO-8859-1", &lISO);
+ bWindows = extractBytes(sWindows, sWindowsLength, "windows-1252", &lWindows);
+
ucsdet_setText(csd, bWindows, lWindows, &status);
match = ucsdet_detect(csd, &status);
name = ucsdet_getName(match, &status);
if (strcmp(name, "windows-1252") != 0) {
- log_err("English text with C1 bytes does not detect as windows-1252, but as %s\n", name);
+ log_data_err("English text with C1 bytes does not detect as windows-1252, but as %s. (Are you missing data?)\n", name);
}
ucsdet_setText(csd, bISO, lISO, &status);
static void TestInputFilter(void)
{
UErrorCode status = U_ZERO_ERROR;
-
const char *ss = "<a> <lot> <of> <English> <inside> <the> <markup> Un tr\\u00E8s petit peu de Fran\\u00E7ais. <to> <confuse> <the> <detector>";
+
static const char ss[] = "<a> <lot> <of> <English> <inside> <the> <markup> Un tr\\u00E8s petit peu de Fran\\u00E7ais. <to> <confuse> <the> <detector>";
int32_t sLength = 0;
- UChar *s = unescape(ss, &sLength);
+ UChar s[sizeof(ss)];
int32_t byteLength = 0;
- char *bytes = extractBytes(s, sLength, "ISO-8859-1", &byteLength);
+ char *bytes;
UCharsetDetector *csd = ucsdet_open(&status);
const UCharsetMatch *match;
const char *lang, *name;
+ sLength = u_unescape(ss, s, sizeof(ss));
+ bytes = extractBytes(s, sLength, "ISO-8859-1", &byteLength);
+
ucsdet_enableInputFilter(csd, TRUE);
if (!ucsdet_isInputFilterEnabled(csd)) {
log_err("Status got changed to %s\n", u_errorName(status));
}
}
+
+static void TestBufferOverflow(void) {
+ UErrorCode status = U_ZERO_ERROR;
+ static const char *testStrings[] = {
+ "\x80\x20\x54\x68\x69\x73\x20\x69\x73\x20\x45\x6E\x67\x6C\x69\x73\x68\x20\x1b", /* A partial ISO-2022 shift state at the end */
+ "\x80\x20\x54\x68\x69\x73\x20\x69\x73\x20\x45\x6E\x67\x6C\x69\x73\x68\x20\x1b\x24", /* A partial ISO-2022 shift state at the end */
+ "\x80\x20\x54\x68\x69\x73\x20\x69\x73\x20\x45\x6E\x67\x6C\x69\x73\x68\x20\x1b\x24\x28", /* A partial ISO-2022 shift state at the end */
+ "\x80\x20\x54\x68\x69\x73\x20\x69\x73\x20\x45\x6E\x67\x6C\x69\x73\x68\x20\x1b\x24\x28\x44", /* A complete ISO-2022 shift state at the end with a bad one at the start */
+ "\x1b\x24\x28\x44", /* A complete ISO-2022 shift state at the end */
+ "\xa1", /* Could be a single byte shift-jis at the end */
+ "\x74\x68\xa1", /* Could be a single byte shift-jis at the end */
+ "\x74\x68\x65\xa1" /* Could be a single byte shift-jis at the end, but now we have English creeping in. */
+ };
+ static const char *testResults[] = {
+ "windows-1252",
+ "windows-1252",
+ "windows-1252",
+ "windows-1252",
+ "ISO-2022-JP",
+ NULL,
+ NULL,
+ "ISO-8859-1"
+ };
+ int32_t idx = 0;
+ UCharsetDetector *csd = ucsdet_open(&status);
+ const UCharsetMatch *match;
+
+ ucsdet_setDeclaredEncoding(csd, "ISO-2022-JP", -1, &status);
+
+ if (U_FAILURE(status)) {
+ log_err("Couldn't open detector. %s\n", u_errorName(status));
+ goto bail;
+ }
+
+ for (idx = 0; idx < UPRV_LENGTHOF(testStrings); idx++) {
+ ucsdet_setText(csd, testStrings[idx], -1, &status);
+ match = ucsdet_detect(csd, &status);
+
+ if (match == NULL) {
+ if (testResults[idx] != NULL) {
+ log_err("Unexpectedly got no results at index %d.\n", idx);
+ }
+ else {
+ log_verbose("Got no result as expected at index %d.\n", idx);
+ }
+ continue;
+ }
+
+ if (testResults[idx] == NULL || strcmp(ucsdet_getName(match, &status), testResults[idx]) != 0) {
+ log_err("Unexpectedly got %s instead of %s at index %d with confidence %d.\n",
+ ucsdet_getName(match, &status), testResults[idx], idx, ucsdet_getConfidence(match, &status));
+ goto bail;
+ }
+ }
+
+bail:
+ ucsdet_close(csd);
+}
+
+static void TestIBM424(void)
+{
+ UErrorCode status = U_ZERO_ERROR;
+
+ static const UChar chars[] = {
+ 0x05D4, 0x05E4, 0x05E8, 0x05E7, 0x05DC, 0x05D9, 0x05D8, 0x0020, 0x05D4, 0x05E6, 0x05D1, 0x05D0, 0x05D9, 0x0020, 0x05D4, 0x05E8,
+ 0x05D0, 0x05E9, 0x05D9, 0x002C, 0x0020, 0x05EA, 0x05EA, 0x0020, 0x05D0, 0x05DC, 0x05D5, 0x05E3, 0x0020, 0x05D0, 0x05D1, 0x05D9,
+ 0x05D7, 0x05D9, 0x0020, 0x05DE, 0x05E0, 0x05D3, 0x05DC, 0x05D1, 0x05DC, 0x05D9, 0x05D8, 0x002C, 0x0020, 0x05D4, 0x05D5, 0x05E8,
+ 0x05D4, 0x0020, 0x05E2, 0x05DC, 0x0020, 0x05E4, 0x05EA, 0x05D9, 0x05D7, 0x05EA, 0x0020, 0x05D7, 0x05E7, 0x05D9, 0x05E8, 0x05EA,
+ 0x0020, 0x05DE, 0x05E6, 0x0022, 0x05D7, 0x0020, 0x05D1, 0x05E2, 0x05E7, 0x05D1, 0x05D5, 0x05EA, 0x0020, 0x05E2, 0x05D3, 0x05D5,
+ 0x05D9, 0x05D5, 0x05EA, 0x0020, 0x05D7, 0x05D9, 0x05D9, 0x05DC, 0x05D9, 0x0020, 0x05E6, 0x05D4, 0x0022, 0x05DC, 0x0020, 0x05DE,
+ 0x05DE, 0x05D1, 0x05E6, 0x05E2, 0x0020, 0x05E2, 0x05D5, 0x05E4, 0x05E8, 0x05EA, 0x0020, 0x05D9, 0x05E6, 0x05D5, 0x05E7, 0x05D4,
+ 0x0020, 0x05D1, 0x002B, 0x0020, 0x05E8, 0x05E6, 0x05D5, 0x05E2, 0x05EA, 0x0020, 0x05E2, 0x05D6, 0x05D4, 0x002E, 0x0020, 0x05DC,
+ 0x05D3, 0x05D1, 0x05E8, 0x05D9, 0x0020, 0x05D4, 0x05E4, 0x05E6, 0x0022, 0x05E8, 0x002C, 0x0020, 0x05DE, 0x05D4, 0x05E2, 0x05D3,
+ 0x05D5, 0x05D9, 0x05D5, 0x05EA, 0x0020, 0x05E2, 0x05D5, 0x05DC, 0x05D4, 0x0020, 0x05EA, 0x05DE, 0x05D5, 0x05E0, 0x05D4, 0x0020,
+ 0x05E9, 0x05DC, 0x0020, 0x0022, 0x05D4, 0x05EA, 0x05E0, 0x05D4, 0x05D2, 0x05D5, 0x05EA, 0x0020, 0x05E4, 0x05E1, 0x05D5, 0x05DC,
+ 0x05D4, 0x0020, 0x05DC, 0x05DB, 0x05D0, 0x05D5, 0x05E8, 0x05D4, 0x0020, 0x05E9, 0x05DC, 0x0020, 0x05D7, 0x05D9, 0x05D9, 0x05DC,
+ 0x05D9, 0x05DD, 0x0020, 0x05D1, 0x05DE, 0x05D4, 0x05DC, 0x05DA, 0x0020, 0x05DE, 0x05D1, 0x05E6, 0x05E2, 0x0020, 0x05E2, 0x05D5,
+ 0x05E4, 0x05E8, 0x05EA, 0x0020, 0x05D9, 0x05E6, 0x05D5, 0x05E7, 0x05D4, 0x0022, 0x002E, 0x0020, 0x05DE, 0x05E0, 0x05D3, 0x05DC,
+ 0x05D1, 0x05DC, 0x05D9, 0x05D8, 0x0020, 0x05E7, 0x05D9, 0x05D1, 0x05DC, 0x0020, 0x05D0, 0x05EA, 0x0020, 0x05D4, 0x05D7, 0x05DC,
+ 0x05D8, 0x05EA, 0x05D5, 0x0020, 0x05DC, 0x05D0, 0x05D7, 0x05E8, 0x0020, 0x05E9, 0x05E2, 0x05D9, 0x05D9, 0x05DF, 0x0020, 0x05D1,
+ 0x05EA, 0x05DE, 0x05DC, 0x05D9, 0x05DC, 0x0020, 0x05D4, 0x05E2, 0x05D3, 0x05D5, 0x05D9, 0x05D5, 0x05EA, 0x0000
+ };
+
+ static const UChar chars_reverse[] = {
+ 0x05EA, 0x05D5, 0x05D9, 0x05D5, 0x05D3, 0x05E2, 0x05D4, 0x0020, 0x05DC, 0x05D9, 0x05DC, 0x05DE, 0x05EA,
+ 0x05D1, 0x0020, 0x05DF, 0x05D9, 0x05D9, 0x05E2, 0x05E9, 0x0020, 0x05E8, 0x05D7, 0x05D0, 0x05DC, 0x0020, 0x05D5, 0x05EA, 0x05D8,
+ 0x05DC, 0x05D7, 0x05D4, 0x0020, 0x05EA, 0x05D0, 0x0020, 0x05DC, 0x05D1, 0x05D9, 0x05E7, 0x0020, 0x05D8, 0x05D9, 0x05DC, 0x05D1,
+ 0x05DC, 0x05D3, 0x05E0, 0x05DE, 0x0020, 0x002E, 0x0022, 0x05D4, 0x05E7, 0x05D5, 0x05E6, 0x05D9, 0x0020, 0x05EA, 0x05E8, 0x05E4,
+ 0x05D5, 0x05E2, 0x0020, 0x05E2, 0x05E6, 0x05D1, 0x05DE, 0x0020, 0x05DA, 0x05DC, 0x05D4, 0x05DE, 0x05D1, 0x0020, 0x05DD, 0x05D9,
+ 0x05DC, 0x05D9, 0x05D9, 0x05D7, 0x0020, 0x05DC, 0x05E9, 0x0020, 0x05D4, 0x05E8, 0x05D5, 0x05D0, 0x05DB, 0x05DC, 0x0020, 0x05D4,
+ 0x05DC, 0x05D5, 0x05E1, 0x05E4, 0x0020, 0x05EA, 0x05D5, 0x05D2, 0x05D4, 0x05E0, 0x05EA, 0x05D4, 0x0022, 0x0020, 0x05DC, 0x05E9,
+ 0x0020, 0x05D4, 0x05E0, 0x05D5, 0x05DE, 0x05EA, 0x0020, 0x05D4, 0x05DC, 0x05D5, 0x05E2, 0x0020, 0x05EA, 0x05D5, 0x05D9, 0x05D5,
+ 0x05D3, 0x05E2, 0x05D4, 0x05DE, 0x0020, 0x002C, 0x05E8, 0x0022, 0x05E6, 0x05E4, 0x05D4, 0x0020, 0x05D9, 0x05E8, 0x05D1, 0x05D3,
+ 0x05DC, 0x0020, 0x002E, 0x05D4, 0x05D6, 0x05E2, 0x0020, 0x05EA, 0x05E2, 0x05D5, 0x05E6, 0x05E8, 0x0020, 0x002B, 0x05D1, 0x0020,
+ 0x05D4, 0x05E7, 0x05D5, 0x05E6, 0x05D9, 0x0020, 0x05EA, 0x05E8, 0x05E4, 0x05D5, 0x05E2, 0x0020, 0x05E2, 0x05E6, 0x05D1, 0x05DE,
+ 0x05DE, 0x0020, 0x05DC, 0x0022, 0x05D4, 0x05E6, 0x0020, 0x05D9, 0x05DC, 0x05D9, 0x05D9, 0x05D7, 0x0020, 0x05EA, 0x05D5, 0x05D9,
+ 0x05D5, 0x05D3, 0x05E2, 0x0020, 0x05EA, 0x05D5, 0x05D1, 0x05E7, 0x05E2, 0x05D1, 0x0020, 0x05D7, 0x0022, 0x05E6, 0x05DE, 0x0020,
+ 0x05EA, 0x05E8, 0x05D9, 0x05E7, 0x05D7, 0x0020, 0x05EA, 0x05D7, 0x05D9, 0x05EA, 0x05E4, 0x0020, 0x05DC, 0x05E2, 0x0020, 0x05D4,
+ 0x05E8, 0x05D5, 0x05D4, 0x0020, 0x002C, 0x05D8, 0x05D9, 0x05DC, 0x05D1, 0x05DC, 0x05D3, 0x05E0, 0x05DE, 0x0020, 0x05D9, 0x05D7,
+ 0x05D9, 0x05D1, 0x05D0, 0x0020, 0x05E3, 0x05D5, 0x05DC, 0x05D0, 0x0020, 0x05EA, 0x05EA, 0x0020, 0x002C, 0x05D9, 0x05E9, 0x05D0,
+ 0x05E8, 0x05D4, 0x0020, 0x05D9, 0x05D0, 0x05D1, 0x05E6, 0x05D4, 0x0020, 0x05D8, 0x05D9, 0x05DC, 0x05E7, 0x05E8, 0x05E4, 0x05D4,
+ 0x0000
+ };
+
+ int32_t bLength = 0, brLength = 0, cLength = UPRV_LENGTHOF(chars), crLength = UPRV_LENGTHOF(chars_reverse);
+
+ char *bytes = extractBytes(chars, cLength, "IBM424", &bLength);
+ char *bytes_r = extractBytes(chars_reverse, crLength, "IBM424", &brLength);
+
+ UCharsetDetector *csd = ucsdet_open(&status);
+ const UCharsetMatch *match;
+ const char *name;
+
+ ucsdet_setText(csd, bytes, bLength, &status);
+ match = ucsdet_detect(csd, &status);
+
+ if (match == NULL) {
+ log_err("Encoding detection failure for IBM424_rtl: got no matches.\n");
+ goto bail;
+ }
+
+ name = ucsdet_getName(match, &status);
+ if (strcmp(name, "IBM424_rtl") != 0) {
+ log_data_err("Encoding detection failure for IBM424_rtl: got %s. (Are you missing data?)\n", name);
+ }
+
+ ucsdet_setText(csd, bytes_r, brLength, &status);
+ match = ucsdet_detect(csd, &status);
+
+ if (match == NULL) {
+ log_err("Encoding detection failure for IBM424_ltr: got no matches.\n");
+ goto bail;
+ }
+
+ name = ucsdet_getName(match, &status);
+ if (strcmp(name, "IBM424_ltr") != 0) {
+ log_data_err("Encoding detection failure for IBM424_ltr: got %s. (Are you missing data?)\n", name);
+ }
+
+bail:
+ freeBytes(bytes);
+ freeBytes(bytes_r);
+ ucsdet_close(csd);
+}
+
+static void TestIBM420(void)
+{
+ UErrorCode status = U_ZERO_ERROR;
+
+ static const UChar chars[] = {
+ 0x0648, 0x064F, 0x0636, 0x0639, 0x062A, 0x0020, 0x0648, 0x0646, 0x064F, 0x0641, 0x0630, 0x062A, 0x0020, 0x0628, 0x0631, 0x0627,
+ 0x0645, 0x062C, 0x0020, 0x062A, 0x0623, 0x0645, 0x064A, 0x0646, 0x0020, 0x0639, 0x062F, 0x064A, 0x062F, 0x0629, 0x0020, 0x0641,
+ 0x064A, 0x0020, 0x0645, 0x0624, 0x0633, 0x0633, 0x0629, 0x0020, 0x0627, 0x0644, 0x062A, 0x0623, 0x0645, 0x064A, 0x0646, 0x0020,
+ 0x0627, 0x0644, 0x0648, 0x0637, 0x0646, 0x064A, 0x002C, 0x0020, 0x0645, 0x0639, 0x0020, 0x0645, 0x0644, 0x0627, 0x0626, 0x0645,
+ 0x062A, 0x0647, 0x0627, 0x0020, 0x062F, 0x0627, 0x0626, 0x0645, 0x0627, 0x064B, 0x0020, 0x0644, 0x0644, 0x0627, 0x062D, 0x062A,
+ 0x064A, 0x0627, 0x062C, 0x0627, 0x062A, 0x0020, 0x0627, 0x0644, 0x0645, 0x062A, 0x063A, 0x064A, 0x0631, 0x0629, 0x0020, 0x0644,
+ 0x0644, 0x0645, 0x062C, 0x062A, 0x0645, 0x0639, 0x0020, 0x0648, 0x0644, 0x0644, 0x062F, 0x0648, 0x0644, 0x0629, 0x002E, 0x0020,
+ 0x062A, 0x0648, 0x0633, 0x0639, 0x062A, 0x0020, 0x0648, 0x062A, 0x0637, 0x0648, 0x0631, 0x062A, 0x0020, 0x0627, 0x0644, 0x0645,
+ 0x0624, 0x0633, 0x0633, 0x0629, 0x0020, 0x0628, 0x0647, 0x062F, 0x0641, 0x0020, 0x0636, 0x0645, 0x0627, 0x0646, 0x0020, 0x0634,
+ 0x0628, 0x0643, 0x0629, 0x0020, 0x0623, 0x0645, 0x0627, 0x0646, 0x0020, 0x0644, 0x0633, 0x0643, 0x0627, 0x0646, 0x0020, 0x062F,
+ 0x0648, 0x0644, 0x0629, 0x0020, 0x0627, 0x0633, 0x0631, 0x0627, 0x0626, 0x064A, 0x0644, 0x0020, 0x0628, 0x0648, 0x062C, 0x0647,
+ 0x0020, 0x0627, 0x0644, 0x0645, 0x062E, 0x0627, 0x0637, 0x0631, 0x0020, 0x0627, 0x0644, 0x0627, 0x0642, 0x062A, 0x0635, 0x0627,
+ 0x062F, 0x064A, 0x0629, 0x0020, 0x0648, 0x0627, 0x0644, 0x0627, 0x062C, 0x062A, 0x0645, 0x0627, 0x0639, 0x064A, 0x0629, 0x002E,
+ 0x0000
+ };
+ static const UChar chars_reverse[] = {
+ 0x002E, 0x0629, 0x064A, 0x0639, 0x0627, 0x0645, 0x062A, 0x062C, 0x0627, 0x0644, 0x0627, 0x0648, 0x0020, 0x0629, 0x064A, 0x062F,
+ 0x0627, 0x0635, 0x062A, 0x0642, 0x0627, 0x0644, 0x0627, 0x0020, 0x0631, 0x0637, 0x0627, 0x062E, 0x0645, 0x0644, 0x0627, 0x0020,
+ 0x0647, 0x062C, 0x0648, 0x0628, 0x0020, 0x0644, 0x064A, 0x0626, 0x0627, 0x0631, 0x0633, 0x0627, 0x0020, 0x0629, 0x0644, 0x0648,
+ 0x062F, 0x0020, 0x0646, 0x0627, 0x0643, 0x0633, 0x0644, 0x0020, 0x0646, 0x0627, 0x0645, 0x0623, 0x0020, 0x0629, 0x0643, 0x0628,
+ 0x0634, 0x0020, 0x0646, 0x0627, 0x0645, 0x0636, 0x0020, 0x0641, 0x062F, 0x0647, 0x0628, 0x0020, 0x0629, 0x0633, 0x0633, 0x0624,
+ 0x0645, 0x0644, 0x0627, 0x0020, 0x062A, 0x0631, 0x0648, 0x0637, 0x062A, 0x0648, 0x0020, 0x062A, 0x0639, 0x0633, 0x0648, 0x062A,
+ 0x0020, 0x002E, 0x0629, 0x0644, 0x0648, 0x062F, 0x0644, 0x0644, 0x0648, 0x0020, 0x0639, 0x0645, 0x062A, 0x062C, 0x0645, 0x0644,
+ 0x0644, 0x0020, 0x0629, 0x0631, 0x064A, 0x063A, 0x062A, 0x0645, 0x0644, 0x0627, 0x0020, 0x062A, 0x0627, 0x062C, 0x0627, 0x064A,
+ 0x062A, 0x062D, 0x0627, 0x0644, 0x0644, 0x0020, 0x064B, 0x0627, 0x0645, 0x0626, 0x0627, 0x062F, 0x0020, 0x0627, 0x0647, 0x062A,
+ 0x0645, 0x0626, 0x0627, 0x0644, 0x0645, 0x0020, 0x0639, 0x0645, 0x0020, 0x002C, 0x064A, 0x0646, 0x0637, 0x0648, 0x0644, 0x0627,
+ 0x0020, 0x0646, 0x064A, 0x0645, 0x0623, 0x062A, 0x0644, 0x0627, 0x0020, 0x0629, 0x0633, 0x0633, 0x0624, 0x0645, 0x0020, 0x064A,
+ 0x0641, 0x0020, 0x0629, 0x062F, 0x064A, 0x062F, 0x0639, 0x0020, 0x0646, 0x064A, 0x0645, 0x0623, 0x062A, 0x0020, 0x062C, 0x0645,
+ 0x0627, 0x0631, 0x0628, 0x0020, 0x062A, 0x0630, 0x0641, 0x064F, 0x0646, 0x0648, 0x0020, 0x062A, 0x0639, 0x0636, 0x064F, 0x0648,
+ 0x0000,
+ };
+
+ int32_t bLength = 0, brLength = 0, cLength = UPRV_LENGTHOF(chars), crLength = UPRV_LENGTHOF(chars_reverse);
+
+ char *bytes = extractBytes(chars, cLength, "IBM420", &bLength);
+ char *bytes_r = extractBytes(chars_reverse, crLength, "IBM420", &brLength);
+
+ UCharsetDetector *csd = ucsdet_open(&status);
+ const UCharsetMatch *match;
+ const char *name;
+
+ ucsdet_setText(csd, bytes, bLength, &status);
+ match = ucsdet_detect(csd, &status);
+
+ if (match == NULL) {
+ log_err("Encoding detection failure for IBM420_rtl: got no matches.\n");
+ goto bail;
+ }
+
+ name = ucsdet_getName(match, &status);
+ if (strcmp(name, "IBM420_rtl") != 0) {
+ log_data_err("Encoding detection failure for IBM420_rtl: got %s. (Are you missing data?)\n", name);
+ }
+
+ ucsdet_setText(csd, bytes_r, brLength, &status);
+ match = ucsdet_detect(csd, &status);
+
+ if (match == NULL) {
+ log_err("Encoding detection failure for IBM420_ltr: got no matches.\n");
+ goto bail;
+ }
+
+ name = ucsdet_getName(match, &status);
+ if (strcmp(name, "IBM420_ltr") != 0) {
+ log_data_err("Encoding detection failure for IBM420_ltr: got %s. (Are you missing data?)\n", name);
+ }
+
+bail:
+ freeBytes(bytes);
+ freeBytes(bytes_r);
+ ucsdet_close(csd);
+}
projects / apple / icu.git / blobdiff