--- /dev/null
+/*
+*******************************************************************************
+*
+* Copyright (C) 2009-2010, International Business Machines
+* Corporation and others. All Rights Reserved.
+*
+*******************************************************************************
+* file name: bidiconf.cpp
+* encoding: US-ASCII
+* tab size: 8 (not used)
+* indentation:4
+*
+* created on: 2009oct16
+* created by: Markus W. Scherer
+*
+* BiDi conformance test, using the Unicode BidiTest.txt file.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "unicode/utypes.h"
+#include "unicode/ubidi.h"
+#include "unicode/errorcode.h"
+#include "unicode/localpointer.h"
+#include "unicode/putil.h"
+#include "unicode/unistr.h"
+#include "intltest.h"
+#include "uparse.h"
+
+class BiDiConformanceTest : public IntlTest {
+public:
+ BiDiConformanceTest() :
+ directionBits(0), lineNumber(0), levelsCount(0), orderingCount(0),
+ errorCount(0) {}
+
+ void runIndexedTest(int32_t index, UBool exec, const char *&name, char *par=NULL);
+
+ void TestBidiTest();
+private:
+ char *getUnidataPath(char path[]);
+
+ UBool parseLevels(const char *start);
+ UBool parseOrdering(const char *start);
+ UBool parseInputStringFromBiDiClasses(const char *&start);
+
+ UBool checkLevels(const UBiDiLevel actualLevels[], int32_t actualCount,
+ const char *paraLevelName);
+ UBool checkOrdering(UBiDi *ubidi, const char *paraLevelName);
+
+ void printErrorLine(const char *paraLevelName);
+
+ char line[10000];
+ UBiDiLevel levels[1000];
+ uint32_t directionBits;
+ int32_t ordering[1000];
+ int32_t lineNumber;
+ int32_t levelsCount;
+ int32_t orderingCount;
+ int32_t errorCount;
+ UnicodeString inputString;
+};
+
+extern IntlTest *createBiDiConformanceTest() {
+ return new BiDiConformanceTest();
+}
+
+void BiDiConformanceTest::runIndexedTest(int32_t index, UBool exec, const char *&name, char * /*par*/) {
+ if(exec) {
+ logln("TestSuite BiDiConformanceTest: ");
+ }
+ switch (index) {
+ TESTCASE(0, TestBidiTest);
+ default:
+ name="";
+ break; // needed to end the loop
+ }
+}
+
+// TODO: Move to a common place (IntlTest?) to avoid duplication with UnicodeTest (ucdtest.cpp).
+char *BiDiConformanceTest::getUnidataPath(char path[]) {
+ IcuTestErrorCode errorCode(*this, "getUnidataPath");
+ const int kUnicodeDataTxtLength=15; // strlen("UnicodeData.txt")
+
+ // Look inside ICU_DATA first.
+ strcpy(path, pathToDataDirectory());
+ strcat(path, "unidata" U_FILE_SEP_STRING "UnicodeData.txt");
+ FILE *f=fopen(path, "r");
+ if(f!=NULL) {
+ fclose(f);
+ *(strchr(path, 0)-kUnicodeDataTxtLength)=0; // Remove the basename.
+ return path;
+ }
+
+ // As a fallback, try to guess where the source data was located
+ // at the time ICU was built, and look there.
+# ifdef U_TOPSRCDIR
+ strcpy(path, U_TOPSRCDIR U_FILE_SEP_STRING "data");
+# else
+ strcpy(path, loadTestData(errorCode));
+ strcat(path, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".."
+ U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".."
+ U_FILE_SEP_STRING "data");
+# endif
+ strcat(path, U_FILE_SEP_STRING);
+ strcat(path, "unidata" U_FILE_SEP_STRING "UnicodeData.txt");
+ f=fopen(path, "r");
+ if(f!=NULL) {
+ fclose(f);
+ *(strchr(path, 0)-kUnicodeDataTxtLength)=0; // Remove the basename.
+ return path;
+ }
+ return NULL;
+}
+
+U_DEFINE_LOCAL_OPEN_POINTER(LocalStdioFilePointer, FILE, fclose);
+
+UBool BiDiConformanceTest::parseLevels(const char *start) {
+ directionBits=0;
+ levelsCount=0;
+ while(*start!=0 && *(start=u_skipWhitespace(start))!=0) {
+ if(*start=='x') {
+ levels[levelsCount++]=UBIDI_DEFAULT_LTR;
+ ++start;
+ } else {
+ char *end;
+ uint32_t value=(uint32_t)strtoul(start, &end, 10);
+ if(end<=start || (!U_IS_INV_WHITESPACE(*end) && *end!=0) || value>(UBIDI_MAX_EXPLICIT_LEVEL+1)) {
+ errln("@Levels: parse error at %s", start);
+ return FALSE;
+ }
+ levels[levelsCount++]=(UBiDiLevel)value;
+ directionBits|=(1<<(value&1));
+ start=end;
+ }
+ }
+ return TRUE;
+}
+
+UBool BiDiConformanceTest::parseOrdering(const char *start) {
+ orderingCount=0;
+ while(*start!=0 && *(start=u_skipWhitespace(start))!=0) {
+ char *end;
+ uint32_t value=(uint32_t)strtoul(start, &end, 10);
+ if(end<=start || (!U_IS_INV_WHITESPACE(*end) && *end!=0) || value>=1000) {
+ errln("@Reorder: parse error at %s", start);
+ return FALSE;
+ }
+ ordering[orderingCount++]=(int32_t)value;
+ start=end;
+ }
+ return TRUE;
+}
+
+static const UChar charFromBiDiClass[U_CHAR_DIRECTION_COUNT]={
+ 0x6c, // 'l' for L
+ 0x52, // 'R' for R
+ 0x33, // '3' for EN
+ 0x2d, // '-' for ES
+ 0x25, // '%' for ET
+ 0x39, // '9' for AN
+ 0x2c, // ',' for CS
+ 0x2f, // '/' for B
+ 0x5f, // '_' for S
+ 0x20, // ' ' for WS
+ 0x3d, // '=' for ON
+ 0x65, // 'e' for LRE
+ 0x6f, // 'o' for LRO
+ 0x41, // 'A' for AL
+ 0x45, // 'E' for RLE
+ 0x4f, // 'O' for RLO
+ 0x2a, // '*' for PDF
+ 0x60, // '`' for NSM
+ 0x7c // '|' for BN
+};
+
+U_CDECL_BEGIN
+
+static UCharDirection U_CALLCONV
+biDiConfUBiDiClassCallback(const void * /*context*/, UChar32 c) {
+ for(int i=0; i<U_CHAR_DIRECTION_COUNT; ++i) {
+ if(c==charFromBiDiClass[i]) {
+ return (UCharDirection)i;
+ }
+ }
+ // Character not in our hardcoded table.
+ // Should not occur during testing.
+ return U_BIDI_CLASS_DEFAULT;
+}
+
+U_CDECL_END
+
+static const int8_t biDiClassNameLengths[U_CHAR_DIRECTION_COUNT+1]={
+ 1, 1, 2, 2, 2, 2, 2, 1, 1, 2, 2, 3, 3, 2, 3, 3, 3, 3, 2, 0
+};
+
+UBool BiDiConformanceTest::parseInputStringFromBiDiClasses(const char *&start) {
+ inputString.remove();
+ /*
+ * Lengthy but fast BiDi class parser.
+ * A simple parser could terminate or extract the name string and use
+ * int32_t biDiClassInt=u_getPropertyValueEnum(UCHAR_BIDI_CLASS, bidiClassString);
+ * but that makes this test take significantly more time.
+ */
+ while(*start!=0 && *(start=u_skipWhitespace(start))!=0 && *start!=';') {
+ UCharDirection biDiClass=U_CHAR_DIRECTION_COUNT;
+ // Compare each character once until we have a match on
+ // a complete, short BiDi class name.
+ if(start[0]=='L') {
+ if(start[1]=='R') {
+ if(start[2]=='E') {
+ biDiClass=U_LEFT_TO_RIGHT_EMBEDDING;
+ } else if(start[2]=='O') {
+ biDiClass=U_LEFT_TO_RIGHT_OVERRIDE;
+ }
+ } else {
+ biDiClass=U_LEFT_TO_RIGHT;
+ }
+ } else if(start[0]=='R') {
+ if(start[1]=='L') {
+ if(start[2]=='E') {
+ biDiClass=U_RIGHT_TO_LEFT_EMBEDDING;
+ } else if(start[2]=='O') {
+ biDiClass=U_RIGHT_TO_LEFT_OVERRIDE;
+ }
+ } else {
+ biDiClass=U_RIGHT_TO_LEFT;
+ }
+ } else if(start[0]=='E') {
+ if(start[1]=='N') {
+ biDiClass=U_EUROPEAN_NUMBER;
+ } else if(start[1]=='S') {
+ biDiClass=U_EUROPEAN_NUMBER_SEPARATOR;
+ } else if(start[1]=='T') {
+ biDiClass=U_EUROPEAN_NUMBER_TERMINATOR;
+ }
+ } else if(start[0]=='A') {
+ if(start[1]=='L') {
+ biDiClass=U_RIGHT_TO_LEFT_ARABIC;
+ } else if(start[1]=='N') {
+ biDiClass=U_ARABIC_NUMBER;
+ }
+ } else if(start[0]=='C' && start[1]=='S') {
+ biDiClass=U_COMMON_NUMBER_SEPARATOR;
+ } else if(start[0]=='B') {
+ if(start[1]=='N') {
+ biDiClass=U_BOUNDARY_NEUTRAL;
+ } else {
+ biDiClass=U_BLOCK_SEPARATOR;
+ }
+ } else if(start[0]=='S') {
+ biDiClass=U_SEGMENT_SEPARATOR;
+ } else if(start[0]=='W' && start[1]=='S') {
+ biDiClass=U_WHITE_SPACE_NEUTRAL;
+ } else if(start[0]=='O' && start[1]=='N') {
+ biDiClass=U_OTHER_NEUTRAL;
+ } else if(start[0]=='P' && start[1]=='D' && start[2]=='F') {
+ biDiClass=U_POP_DIRECTIONAL_FORMAT;
+ } else if(start[0]=='N' && start[1]=='S' && start[2]=='M') {
+ biDiClass=U_DIR_NON_SPACING_MARK;
+ }
+ // Now we verify that the class name is terminated properly,
+ // and not just the start of a longer word.
+ int8_t biDiClassNameLength=biDiClassNameLengths[biDiClass];
+ char c=start[biDiClassNameLength];
+ if(biDiClass==U_CHAR_DIRECTION_COUNT || (!U_IS_INV_WHITESPACE(c) && c!=';' && c!=0)) {
+ errln("BiDi class string not recognized at %s", start);
+ return FALSE;
+ }
+ inputString.append(charFromBiDiClass[biDiClass]);
+ start+=biDiClassNameLength;
+ }
+ return TRUE;
+}
+
+void BiDiConformanceTest::TestBidiTest() {
+ IcuTestErrorCode errorCode(*this, "TestBidiTest");
+ const char *sourceTestDataPath=getSourceTestData(errorCode);
+ if(errorCode.logIfFailureAndReset("unable to find the source/test/testdata "
+ "folder (getSourceTestData())")) {
+ return;
+ }
+ char bidiTestPath[400];
+ strcpy(bidiTestPath, sourceTestDataPath);
+ strcat(bidiTestPath, "BidiTest.txt");
+ LocalStdioFilePointer bidiTestFile(fopen(bidiTestPath, "r"));
+ if(bidiTestFile.isNull()) {
+ errln("unable to open %s", bidiTestPath);
+ return;
+ }
+ LocalUBiDiPointer ubidi(ubidi_open());
+ ubidi_setClassCallback(ubidi.getAlias(), biDiConfUBiDiClassCallback, NULL,
+ NULL, NULL, errorCode);
+ if(errorCode.logIfFailureAndReset("ubidi_setClassCallback()")) {
+ return;
+ }
+ lineNumber=0;
+ levelsCount=0;
+ orderingCount=0;
+ errorCount=0;
+ while(errorCount<10 && fgets(line, (int)sizeof(line), bidiTestFile.getAlias())!=NULL) {
+ ++lineNumber;
+ // Remove trailing comments and whitespace.
+ char *commentStart=strchr(line, '#');
+ if(commentStart!=NULL) {
+ *commentStart=0;
+ }
+ u_rtrim(line);
+ const char *start=u_skipWhitespace(line);
+ if(*start==0) {
+ continue; // Skip empty and comment-only lines.
+ }
+ if(*start=='@') {
+ ++start;
+ if(0==strncmp(start, "Levels:", 7)) {
+ if(!parseLevels(start+7)) {
+ return;
+ }
+ } else if(0==strncmp(start, "Reorder:", 8)) {
+ if(!parseOrdering(start+8)) {
+ return;
+ }
+ }
+ // Skip unknown @Xyz: ...
+ } else {
+ if(!parseInputStringFromBiDiClasses(start)) {
+ return;
+ }
+ start=u_skipWhitespace(start);
+ if(*start!=';') {
+ errln("missing ; separator on input line %s", line);
+ return;
+ }
+ start=u_skipWhitespace(start+1);
+ char *end;
+ uint32_t bitset=(uint32_t)strtoul(start, &end, 16);
+ if(end<=start || (!U_IS_INV_WHITESPACE(*end) && *end!=';' && *end!=0)) {
+ errln("input bitset parse error at %s", start);
+ return;
+ }
+ // Loop over the bitset.
+ static const UBiDiLevel paraLevels[]={ UBIDI_DEFAULT_LTR, 0, 1, UBIDI_DEFAULT_RTL };
+ static const char *const paraLevelNames[]={ "auto/LTR", "LTR", "RTL", "auto/RTL" };
+ for(int i=0; i<=3; ++i) {
+ if(bitset&(1<<i)) {
+ ubidi_setPara(ubidi.getAlias(), inputString.getBuffer(), inputString.length(),
+ paraLevels[i], NULL, errorCode);
+ const UBiDiLevel *actualLevels=ubidi_getLevels(ubidi.getAlias(), errorCode);
+ if(errorCode.logIfFailureAndReset("ubidi_setPara() or ubidi_getLevels()")) {
+ errln("Input line %d: %s", (int)lineNumber, line);
+ return;
+ }
+ if(!checkLevels(actualLevels, ubidi_getProcessedLength(ubidi.getAlias()),
+ paraLevelNames[i])) {
+ // continue outerLoop; does not exist in C++
+ // so just break out of the inner loop.
+ break;
+ }
+ if(!checkOrdering(ubidi.getAlias(), paraLevelNames[i])) {
+ // continue outerLoop; does not exist in C++
+ // so just break out of the inner loop.
+ break;
+ }
+ }
+ }
+ }
+ }
+}
+
+static UChar printLevel(UBiDiLevel level) {
+ if(level<UBIDI_DEFAULT_LTR) {
+ return 0x30+level;
+ } else {
+ return 0x78; // 'x'
+ }
+}
+
+static uint32_t getDirectionBits(const UBiDiLevel actualLevels[], int32_t actualCount) {
+ uint32_t actualDirectionBits=0;
+ for(int32_t i=0; i<actualCount; ++i) {
+ actualDirectionBits|=(1<<(actualLevels[i]&1));
+ }
+ return actualDirectionBits;
+}
+
+UBool BiDiConformanceTest::checkLevels(const UBiDiLevel actualLevels[], int32_t actualCount,
+ const char *paraLevelName) {
+ UBool isOk=TRUE;
+ if(levelsCount!=actualCount) {
+ errln("Wrong number of level values; expected %d actual %d",
+ (int)levelsCount, (int)actualCount);
+ isOk=FALSE;
+ } else {
+ for(int32_t i=0; i<actualCount; ++i) {
+ if(levels[i]!=actualLevels[i] && levels[i]<UBIDI_DEFAULT_LTR) {
+ if(directionBits!=3 && directionBits==getDirectionBits(actualLevels, actualCount)) {
+ // ICU used a shortcut:
+ // Since the text is unidirectional, it did not store the resolved
+ // levels but just returns all levels as the paragraph level 0 or 1.
+ // The reordering result is the same, so this is fine.
+ break;
+ } else {
+ errln("Wrong level value at index %d; expected %d actual %d",
+ (int)i, levels[i], actualLevels[i]);
+ isOk=FALSE;
+ break;
+ }
+ }
+ }
+ }
+ if(!isOk) {
+ printErrorLine(paraLevelName);
+ UnicodeString els("Expected levels: ");
+ int32_t i;
+ for(i=0; i<levelsCount; ++i) {
+ els.append((UChar)0x20).append(printLevel(levels[i]));
+ }
+ UnicodeString als("Actual levels: ");
+ for(i=0; i<actualCount; ++i) {
+ als.append((UChar)0x20).append(printLevel(actualLevels[i]));
+ }
+ errln(els);
+ errln(als);
+ }
+ return isOk;
+}
+
+// Note: ubidi_setReorderingOptions(ubidi, UBIDI_OPTION_REMOVE_CONTROLS);
+// does not work for custom BiDi class assignments
+// and anyway also removes LRM/RLM/ZWJ/ZWNJ which is not desirable here.
+// Therefore we just skip the indexes for BiDi controls while comparing
+// with the expected ordering that has them omitted.
+UBool BiDiConformanceTest::checkOrdering(UBiDi *ubidi, const char *paraLevelName) {
+ UBool isOk=TRUE;
+ IcuTestErrorCode errorCode(*this, "TestBidiTest/checkOrdering()");
+ int32_t resultLength=ubidi_getResultLength(ubidi); // visual length including BiDi controls
+ int32_t i, visualIndex;
+ // Note: It should be faster to call ubidi_countRuns()/ubidi_getVisualRun()
+ // and loop over each run's indexes, but that seems unnecessary for this test code.
+ for(i=visualIndex=0; i<resultLength; ++i) {
+ int32_t logicalIndex=ubidi_getLogicalIndex(ubidi, i, errorCode);
+ if(errorCode.logIfFailureAndReset("ubidi_getLogicalIndex()")) {
+ errln("Input line %d: %s", (int)lineNumber, line);
+ return FALSE;
+ }
+ if(levels[logicalIndex]>=UBIDI_DEFAULT_LTR) {
+ continue; // BiDi control, omitted from expected ordering.
+ }
+ if(visualIndex<orderingCount && logicalIndex!=ordering[visualIndex]) {
+ errln("Wrong ordering value at visual index %d; expected %d actual %d",
+ (int)visualIndex, ordering[visualIndex], logicalIndex);
+ isOk=FALSE;
+ break;
+ }
+ ++visualIndex;
+ }
+ // visualIndex is now the visual length minus the BiDi controls,
+ // which should match the length of the BidiTest.txt ordering.
+ if(isOk && orderingCount!=visualIndex) {
+ errln("Wrong number of ordering values; expected %d actual %d",
+ (int)orderingCount, (int)visualIndex);
+ isOk=FALSE;
+ }
+ if(!isOk) {
+ printErrorLine(paraLevelName);
+ UnicodeString eord("Expected ordering: ");
+ for(i=0; i<orderingCount; ++i) {
+ eord.append((UChar)0x20).append((UChar)(0x30+ordering[i]));
+ }
+ UnicodeString aord("Actual ordering: ");
+ for(i=0; i<resultLength; ++i) {
+ int32_t logicalIndex=ubidi_getLogicalIndex(ubidi, i, errorCode);
+ if(levels[logicalIndex]<UBIDI_DEFAULT_LTR) {
+ aord.append((UChar)0x20).append((UChar)(0x30+logicalIndex));
+ }
+ }
+ errln(eord);
+ errln(aord);
+ }
+ return isOk;
+}
+
+void BiDiConformanceTest::printErrorLine(const char *paraLevelName) {
+ ++errorCount;
+ errln("Input line %5d: %s", (int)lineNumber, line);
+ errln(UnicodeString("Input string: ")+inputString);
+ errln("Para level: %s", paraLevelName);
+}
projects / apple / icu.git / blobdiff