[apple/icu.git] / icuSources / test / intltest / normconf.cpp

/*
************************************************************************
* Copyright (c) 1997-2016, International Business Machines
* Corporation and others.  All Rights Reserved.
************************************************************************
*/

#include "unicode/utypes.h"

#if !UCONFIG_NO_NORMALIZATION

#include "unicode/uchar.h"
#include "unicode/normlzr.h"
#include "unicode/uniset.h"
#include "unicode/putil.h"
#include "cmemory.h"
#include "cstring.h"
#include "filestrm.h"
#include "normconf.h"
#include <stdio.h>

#define CASE(id,test,exec) case id:                          \
                          name = #test;                 \
                          if (exec) {                   \
                              logln(#test "---");       \
                              logln((UnicodeString)""); \
                              test();                   \
                          }                             \
                          break

void NormalizerConformanceTest::runIndexedTest(int32_t index, UBool exec, const char* &name, char* /*par*/) {
    switch (index) {
        CASE(0, TestConformance, exec);
#if !UCONFIG_NO_FILE_IO && !UCONFIG_NO_LEGACY_CONVERSION
        CASE(1, TestConformance32, exec);
#endif
        // CASE(2, TestCase6);
        default: name = ""; break;
    }
}

#define FIELD_COUNT 5

NormalizerConformanceTest::NormalizerConformanceTest() :
    normalizer(UnicodeString(), UNORM_NFC) {}

NormalizerConformanceTest::~NormalizerConformanceTest() {}

// more interesting conformance test cases, not in the unicode.org NormalizationTest.txt
static const char *moreCases[]={
    // Markus 2001aug30
    "0061 0332 0308;00E4 0332;0061 0332 0308;00E4 0332;0061 0332 0308; # Markus 0",

    // Markus 2001oct26 - test edge case for iteration: U+0f73.cc==0 but decomposition.lead.cc==129
    "0061 0301 0F73;00E1 0F71 0F72;0061 0F71 0F72 0301;00E1 0F71 0F72;0061 0F71 0F72 0301; # Markus 1"
};

void NormalizerConformanceTest::compare(const UnicodeString& s1, const UnicodeString& s2){
    UErrorCode status=U_ZERO_ERROR;
     // TODO: Re-enable this tests after UTC fixes UAX 21
    if(s1.indexOf((UChar32)0x0345)>=0)return;
    if(Normalizer::compare(s1,s2,U_FOLD_CASE_DEFAULT,status)!=0){
        errln("Normalizer::compare() failed for s1: " + prettify(s1) + " s2: " +prettify(s2));
    }
}

FileStream *
NormalizerConformanceTest::openNormalizationTestFile(const char *filename) {
    char unidataPath[2000];
    const char *folder;
    FileStream *input;
    UErrorCode errorCode;

    // look inside ICU_DATA first
    folder=pathToDataDirectory();
    if(folder!=NULL) {
        strcpy(unidataPath, folder);
        strcat(unidataPath, "unidata" U_FILE_SEP_STRING);
        strcat(unidataPath, filename);
        input=T_FileStream_open(unidataPath, "rb");
        if(input!=NULL) {
            return input;
        }
    }

    // find icu/source/data/unidata relative to the test data
    errorCode=U_ZERO_ERROR;
    folder=loadTestData(errorCode);
    if(U_SUCCESS(errorCode)) {
        strcpy(unidataPath, folder);
        strcat(unidataPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".."
                     U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".."
                     U_FILE_SEP_STRING "data" U_FILE_SEP_STRING "unidata" U_FILE_SEP_STRING);
        strcat(unidataPath, filename);
        input=T_FileStream_open(unidataPath, "rb");
        if(input!=NULL) {
            return input;
        }
    }

    // look in icu/source/test/testdata/out/build
    errorCode=U_ZERO_ERROR;
    folder=loadTestData(errorCode);
    if(U_SUCCESS(errorCode)) {
        strcpy(unidataPath, folder);
        strcat(unidataPath, U_FILE_SEP_STRING);
        strcat(unidataPath, filename);
        input=T_FileStream_open(unidataPath, "rb");
        if(input!=NULL) {
            return input;
        }
    }

    // look in icu/source/test/testdata
    errorCode=U_ZERO_ERROR;
    folder=loadTestData(errorCode);
    if(U_SUCCESS(errorCode)) {
        strcpy(unidataPath, folder);
        strcat(unidataPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING);
        strcat(unidataPath, filename);
        input=T_FileStream_open(unidataPath, "rb");
        if(input!=NULL) {
            return input;
        }
    }

    // find icu/source/data/unidata relative to U_TOPSRCDIR
#if defined(U_TOPSRCDIR)
    strcpy(unidataPath, U_TOPSRCDIR U_FILE_SEP_STRING "data" U_FILE_SEP_STRING "unidata" U_FILE_SEP_STRING);
    strcat(unidataPath, filename);
    input=T_FileStream_open(unidataPath, "rb");
    if(input!=NULL) {
        return input;
    }

    strcpy(unidataPath, U_TOPSRCDIR U_FILE_SEP_STRING "test" U_FILE_SEP_STRING "testdata" U_FILE_SEP_STRING);
    strcat(unidataPath, filename);
    input=T_FileStream_open(unidataPath, "rb");
    if(input!=NULL) {
        return input;
    }
#endif

    dataerrln("Failed to open %s", filename);
    return NULL;
}

/**
 * Test the conformance of Normalizer to
 * http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt
 */
void NormalizerConformanceTest::TestConformance() {
    TestConformance(openNormalizationTestFile("NormalizationTest.txt"), 0);
}

void NormalizerConformanceTest::TestConformance32() {
    TestConformance(openNormalizationTestFile("NormalizationTest-3.2.0.txt"), UNORM_UNICODE_3_2);
}

void NormalizerConformanceTest::TestConformance(FileStream *input, int32_t options) {
    enum { BUF_SIZE = 1024 };
    char lineBuf[BUF_SIZE];
    UnicodeString fields[FIELD_COUNT];
    UErrorCode status = U_ZERO_ERROR;
    int32_t passCount = 0;
    int32_t failCount = 0;
    UChar32 c;

    if(input==NULL) {
        return;
    }

    // UnicodeSet for all code points that are not mentioned in NormalizationTest.txt
    UnicodeSet other(0, 0x10ffff);

    int32_t count, countMoreCases = UPRV_LENGTHOF(moreCases);
    for (count = 1;;++count) {
        if (!T_FileStream_eof(input)) {
            T_FileStream_readLine(input, lineBuf, (int32_t)sizeof(lineBuf));
        } else {
            // once NormalizationTest.txt is finished, use moreCases[]
            if(count > countMoreCases) {
                count = 0;
            } else if(count == countMoreCases) {
                // all done
                break;
            }
            uprv_strcpy(lineBuf, moreCases[count]);
        }
        if (lineBuf[0] == 0 || lineBuf[0] == '\n' || lineBuf[0] == '\r') continue;

        // Expect 5 columns of this format:
        // 1E0C;1E0C;0044 0323;1E0C;0044 0323; # <comments>

        // Parse out the comment.
        if (lineBuf[0] == '#') continue;

        // Read separator lines starting with '@'
        if (lineBuf[0] == '@') {
            logln(lineBuf);
            continue;
        }

        // Parse out the fields
        if (!hexsplit(lineBuf, ';', fields, FIELD_COUNT)) {
            errln((UnicodeString)"Unable to parse line " + count);
            break; // Syntax error
        }

        // Remove a single code point from the "other" UnicodeSet
        if(fields[0].length()==fields[0].moveIndex32(0, 1)) {
            c=fields[0].char32At(0);
            if(0xac20<=c && c<=0xd73f && quick) {
                // not an exhaustive test run: skip most Hangul syllables
                if(c==0xac20) {
                    other.remove(0xac20, 0xd73f);
                }
                continue;
            }
            other.remove(c);
        }

        if (checkConformance(fields, lineBuf, options, status)) {
            ++passCount;
        } else {
            ++failCount;
            if(status == U_FILE_ACCESS_ERROR) {
              dataerrln("Something is wrong with the normalizer, skipping the rest of the test.");
              break;
            }
        }
        if ((count % 1000) == 0) {
            logln("Line %d", count);
        }
    }

    T_FileStream_close(input);

    /*
     * Test that all characters that are not mentioned
     * as single code points in column 1
     * do not change under any normalization.
     */

    // remove U+ffff because that is the end-of-iteration sentinel value
    other.remove(0xffff);

    for(c=0; c<=0x10ffff; quick ? c+=113 : ++c) {
        if(0x30000<=c && c<0xe0000) {
            c=0xe0000;
        }
        if(!other.contains(c)) {
            continue;
        }

        fields[0]=fields[1]=fields[2]=fields[3]=fields[4].setTo(c);
        sprintf(lineBuf, "not mentioned code point U+%04lx", (long)c);

        if (checkConformance(fields, lineBuf, options, status)) {
            ++passCount;
        } else {
            ++failCount;
            if(status == U_FILE_ACCESS_ERROR) {
              dataerrln("Something is wrong with the normalizer, skipping the rest of the test.: %s", u_errorName(status));
              break;
            }
        }
        if ((c % 0x1000) == 0) {
            logln("Code point U+%04lx", c);
        }
    }

    if (failCount != 0) {
        dataerrln((UnicodeString)"Total: " + failCount + " lines/code points failed, " +
              passCount + " lines/code points passed");
    } else {
        logln((UnicodeString)"Total: " + passCount + " lines/code points passed");
    }
}

/**
 * Verify the conformance of the given line of the Unicode
 * normalization (UTR 15) test suite file.  For each line,
 * there are five columns, corresponding to field[0]..field[4].
 *
 * The following invariants must be true for all conformant implementations
 *  c2 == NFC(c1) == NFC(c2) == NFC(c3)
 *  c3 == NFD(c1) == NFD(c2) == NFD(c3)
 *  c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
 *  c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
 *
 * @param field the 5 columns
 * @param line the source line from the test suite file
 * @return true if the test passes
 */
UBool NormalizerConformanceTest::checkConformance(const UnicodeString* field,
                                                  const char *line,
                                                  int32_t options,
                                                  UErrorCode &status) {
    UBool pass = TRUE, result;
    //UErrorCode status = U_ZERO_ERROR;
    UnicodeString out, fcd;
    int32_t fieldNum;

    for (int32_t i=0; i<FIELD_COUNT; ++i) {
        fieldNum = i+1;
        if (i<3) {
            Normalizer::normalize(field[i], UNORM_NFC, options, out, status);
            if (U_FAILURE(status)) {
                dataerrln("Error running normalize UNORM_NFC: %s", u_errorName(status));
            } else {
                pass &= assertEqual("C", field[i], out, field[1], "c2!=C(c", fieldNum);
                iterativeNorm(field[i], UNORM_NFC, options, out, +1);
                pass &= assertEqual("C(+1)", field[i], out, field[1], "c2!=C(c", fieldNum);
                iterativeNorm(field[i], UNORM_NFC, options, out, -1);
                pass &= assertEqual("C(-1)", field[i], out, field[1], "c2!=C(c", fieldNum);
            }

            Normalizer::normalize(field[i], UNORM_NFD, options, out, status);
            if (U_FAILURE(status)) {
                dataerrln("Error running normalize UNORM_NFD: %s", u_errorName(status));
            } else {
                pass &= assertEqual("D", field[i], out, field[2], "c3!=D(c", fieldNum);
                iterativeNorm(field[i], UNORM_NFD, options, out, +1);
                pass &= assertEqual("D(+1)", field[i], out, field[2], "c3!=D(c", fieldNum);
                iterativeNorm(field[i], UNORM_NFD, options, out, -1);
                pass &= assertEqual("D(-1)", field[i], out, field[2], "c3!=D(c", fieldNum);
            }
        }
        Normalizer::normalize(field[i], UNORM_NFKC, options, out, status);
        if (U_FAILURE(status)) {
            dataerrln("Error running normalize UNORM_NFKC: %s", u_errorName(status));
        } else {
            pass &= assertEqual("KC", field[i], out, field[3], "c4!=KC(c", fieldNum);
            iterativeNorm(field[i], UNORM_NFKC, options, out, +1);
            pass &= assertEqual("KC(+1)", field[i], out, field[3], "c4!=KC(c", fieldNum);
            iterativeNorm(field[i], UNORM_NFKC, options, out, -1);
            pass &= assertEqual("KC(-1)", field[i], out, field[3], "c4!=KC(c", fieldNum);
        }

        Normalizer::normalize(field[i], UNORM_NFKD, options, out, status);
        if (U_FAILURE(status)) {
            dataerrln("Error running normalize UNORM_NFKD: %s", u_errorName(status));
        } else {
            pass &= assertEqual("KD", field[i], out, field[4], "c5!=KD(c", fieldNum);
            iterativeNorm(field[i], UNORM_NFKD, options, out, +1);
            pass &= assertEqual("KD(+1)", field[i], out, field[4], "c5!=KD(c", fieldNum);
            iterativeNorm(field[i], UNORM_NFKD, options, out, -1);
            pass &= assertEqual("KD(-1)", field[i], out, field[4], "c5!=KD(c", fieldNum);
        }
    }
    compare(field[1],field[2]);
    compare(field[0],field[1]);
    // test quick checks
    if(UNORM_NO == Normalizer::quickCheck(field[1], UNORM_NFC, options, status)) {
        errln("Normalizer error: quickCheck(NFC(s), UNORM_NFC) is UNORM_NO");
        pass = FALSE;
    }
    if(UNORM_NO == Normalizer::quickCheck(field[2], UNORM_NFD, options, status)) {
        errln("Normalizer error: quickCheck(NFD(s), UNORM_NFD) is UNORM_NO");
        pass = FALSE;
    }
    if(UNORM_NO == Normalizer::quickCheck(field[3], UNORM_NFKC, options, status)) {
        errln("Normalizer error: quickCheck(NFKC(s), UNORM_NFKC) is UNORM_NO");
        pass = FALSE;
    }
    if(UNORM_NO == Normalizer::quickCheck(field[4], UNORM_NFKD, options, status)) {
        errln("Normalizer error: quickCheck(NFKD(s), UNORM_NFKD) is UNORM_NO");
        pass = FALSE;
    }

    // branch on options==0 for better code coverage
    if(options==0) {
        result = Normalizer::isNormalized(field[1], UNORM_NFC, status);
    } else {
        result = Normalizer::isNormalized(field[1], UNORM_NFC, options, status);
    }
    if(!result) {
        dataerrln("Normalizer error: isNormalized(NFC(s), UNORM_NFC) is FALSE");
        pass = FALSE;
    }
    if(field[0]!=field[1] && Normalizer::isNormalized(field[0], UNORM_NFC, options, status)) {
        errln("Normalizer error: isNormalized(s, UNORM_NFC) is TRUE");
        pass = FALSE;
    }
    if(!Normalizer::isNormalized(field[3], UNORM_NFKC, options, status)) {
        dataerrln("Normalizer error: isNormalized(NFKC(s), UNORM_NFKC) is FALSE");
        pass = FALSE;
    }
    if(field[0]!=field[3] && Normalizer::isNormalized(field[0], UNORM_NFKC, options, status)) {
        errln("Normalizer error: isNormalized(s, UNORM_NFKC) is TRUE");
        pass = FALSE;
    }

    // test FCD quick check and "makeFCD"
    Normalizer::normalize(field[0], UNORM_FCD, options, fcd, status);
    if(UNORM_NO == Normalizer::quickCheck(fcd, UNORM_FCD, options, status)) {
        errln("Normalizer error: quickCheck(FCD(s), UNORM_FCD) is UNORM_NO");
        pass = FALSE;
    }
    if(UNORM_NO == Normalizer::quickCheck(field[2], UNORM_FCD, options, status)) {
        errln("Normalizer error: quickCheck(NFD(s), UNORM_FCD) is UNORM_NO");
        pass = FALSE;
    }
    if(UNORM_NO == Normalizer::quickCheck(field[4], UNORM_FCD, options, status)) {
        errln("Normalizer error: quickCheck(NFKD(s), UNORM_FCD) is UNORM_NO");
        pass = FALSE;
    }

    Normalizer::normalize(fcd, UNORM_NFD, options, out, status);
    if(out != field[2]) {
        dataerrln("Normalizer error: NFD(FCD(s))!=NFD(s)");
        pass = FALSE;
    }

    if (U_FAILURE(status)) {
        dataerrln("Normalizer::normalize returned error status: %s", u_errorName(status));
        pass = FALSE;
    }

    if(field[0]!=field[2]) {
        // two strings that are canonically equivalent must test
        // equal under a canonical caseless match
        // see UAX #21 Case Mappings and Jitterbug 2021 and
        // Unicode Technical Committee meeting consensus 92-C31
        int32_t rc;

        status=U_ZERO_ERROR;
        rc=Normalizer::compare(field[0], field[2], (options<<UNORM_COMPARE_NORM_OPTIONS_SHIFT)|U_COMPARE_IGNORE_CASE, status);
        if(U_FAILURE(status)) {
            dataerrln("Normalizer::compare(case-insensitive) sets %s", u_errorName(status));
            pass=FALSE;
        } else if(rc!=0) {
            errln("Normalizer::compare(original, NFD, case-insensitive) returned %d instead of 0 for equal", rc);
            pass=FALSE;
        }
    }

    if (!pass) {
        dataerrln("FAIL: %s", line);
    }
    return pass;
}

/**
 * Do a normalization using the iterative API in the given direction.
 * @param dir either +1 or -1
 */
void NormalizerConformanceTest::iterativeNorm(const UnicodeString& str,
                                              UNormalizationMode mode, int32_t options,
                                              UnicodeString& result,
                                              int8_t dir) {
    UErrorCode status = U_ZERO_ERROR;
    normalizer.setText(str, status);
    normalizer.setMode(mode);
    normalizer.setOption(-1, 0);        // reset all options
    normalizer.setOption(options, 1);   // set desired options
    result.truncate(0);
    if (U_FAILURE(status)) {
        return;
    }
    UChar32 ch;
    if (dir > 0) {
        for (ch = normalizer.first(); ch != Normalizer::DONE;
             ch = normalizer.next()) {
            result.append(ch);
        }
    } else {
        for (ch = normalizer.last(); ch != Normalizer::DONE;
             ch = normalizer.previous()) {
            result.insert(0, ch);
        }
    }
}

/**
 * @param op name of normalization form, e.g., "KC"
 * @param s string being normalized
 * @param got value received
 * @param exp expected value
 * @param msg description of this test
 * @param return true if got == exp
 */
UBool NormalizerConformanceTest::assertEqual(const char *op,
                                             const UnicodeString& s,
                                             const UnicodeString& got,
                                             const UnicodeString& exp,
                                             const char *msg,
                                             int32_t field)
{
    if (exp == got)
        return TRUE;

    char *sChars, *gotChars, *expChars;
    UnicodeString sPretty(prettify(s));
    UnicodeString gotPretty(prettify(got));
    UnicodeString expPretty(prettify(exp));

    sChars = new char[sPretty.length() + 1];
    gotChars = new char[gotPretty.length() + 1];
    expChars = new char[expPretty.length() + 1];

    sPretty.extract(0, sPretty.length(), sChars, sPretty.length() + 1);
    sChars[sPretty.length()] = 0;
    gotPretty.extract(0, gotPretty.length(), gotChars, gotPretty.length() + 1);
    gotChars[gotPretty.length()] = 0;
    expPretty.extract(0, expPretty.length(), expChars, expPretty.length() + 1);
    expChars[expPretty.length()] = 0;

    errln("    %s%d)%s(%s)=%s, exp. %s", msg, field, op, sChars, gotChars, expChars);

    delete []sChars;
    delete []gotChars;
    delete []expChars;
    return FALSE;
}

/**
 * Split a string into pieces based on the given delimiter
 * character.  Then, parse the resultant fields from hex into
 * characters.  That is, "0040 0400;0C00;0899" -> new String[] {
 * "\u0040\u0400", "\u0C00", "\u0899" }.  The output is assumed to
 * be of the proper length already, and exactly output.length
 * fields are parsed.  If there are too few an exception is
 * thrown.  If there are too many the extras are ignored.
 *
 * @return FALSE upon failure
 */
UBool NormalizerConformanceTest::hexsplit(const char *s, char delimiter,
                                          UnicodeString output[], int32_t outputLength) {
    const char *t = s;
    char *end = NULL;
    UChar32 c;
    int32_t i;
    for (i=0; i<outputLength; ++i) {
        // skip whitespace
        while(*t == ' ' || *t == '\t') {
            ++t;
        }

        // read a sequence of code points
        output[i].remove();
        for(;;) {
            c = (UChar32)uprv_strtoul(t, &end, 16);

            if( (char *)t == end ||
                (uint32_t)c > 0x10ffff ||
                (*end != ' ' && *end != '\t' && *end != delimiter)
            ) {
                errln(UnicodeString("Bad field ", "") + (i + 1) + " in " + UnicodeString(s, ""));
                return FALSE;
            }

            output[i].append(c);

            t = (const char *)end;

            // skip whitespace
            while(*t == ' ' || *t == '\t') {
                ++t;
            }

            if(*t == delimiter) {
                ++t;
                break;
            }
            if(*t == 0) {
                if((i + 1) == outputLength) {
                    return TRUE;
                } else {
                    errln(UnicodeString("Missing field(s) in ", "") + s + " only " + (i + 1) + " out of " + outputLength);
                    return FALSE;
                }
            }
        }
    }
    return TRUE;
}

// Specific tests for debugging.  These are generally failures taken from
// the conformance file, but culled out to make debugging easier.

void NormalizerConformanceTest::TestCase6(void) {
    _testOneLine("0385;0385;00A8 0301;0020 0308 0301;0020 0308 0301;");
}

void NormalizerConformanceTest::_testOneLine(const char *line) {
  UErrorCode status = U_ZERO_ERROR;
    UnicodeString fields[FIELD_COUNT];
    if (!hexsplit(line, ';', fields, FIELD_COUNT)) {
        errln((UnicodeString)"Unable to parse line " + line);
    } else {
        checkConformance(fields, line, 0, status);
    }
}

#endif /* #if !UCONFIG_NO_NORMALIZATION */
Commit	Line	Data
b75a7d8f A	1	/*
b75a7d8f A	2	************************************************************************
2ca993e8	3	* Copyright (c) 1997-2016, International Business Machines
b75a7d8f A	4	* Corporation and others. All Rights Reserved.
	5	************************************************************************
	6	*/
	7
	8	#include "unicode/utypes.h"
	9
	10	#if !UCONFIG_NO_NORMALIZATION
	11
	12	#include "unicode/uchar.h"
	13	#include "unicode/normlzr.h"
	14	#include "unicode/uniset.h"
374ca955	15	#include "unicode/putil.h"
2ca993e8	16	#include "cmemory.h"
b75a7d8f A	17	#include "cstring.h"
	18	#include "filestrm.h"
	19	#include "normconf.h"
	20	#include <stdio.h>
	21
729e4ab9	22	#define CASE(id,test,exec) case id: \
b75a7d8f A	23	name = #test; \
	24	if (exec) { \
	25	logln(#test "---"); \
	26	logln((UnicodeString)""); \
	27	test(); \
	28	} \
	29	break
	30
	31	void NormalizerConformanceTest::runIndexedTest(int32_t index, UBool exec, const char* &name, char* /par/) {
	32	switch (index) {
729e4ab9 A	33	CASE(0, TestConformance, exec);
	34	#if !UCONFIG_NO_FILE_IO && !UCONFIG_NO_LEGACY_CONVERSION
	35	CASE(1, TestConformance32, exec);
	36	#endif
b75a7d8f A	37	// CASE(2, TestCase6);
	38	default: name = ""; break;
	39	}
	40	}
	41
	42	#define FIELD_COUNT 5
	43
	44	NormalizerConformanceTest::NormalizerConformanceTest() :
	45	normalizer(UnicodeString(), UNORM_NFC) {}
	46
	47	NormalizerConformanceTest::~NormalizerConformanceTest() {}
	48
	49	// more interesting conformance test cases, not in the unicode.org NormalizationTest.txt
	50	static const char *moreCases[]={
	51	// Markus 2001aug30
	52	"0061 0332 0308;00E4 0332;0061 0332 0308;00E4 0332;0061 0332 0308; # Markus 0",
	53
	54	// Markus 2001oct26 - test edge case for iteration: U+0f73.cc==0 but decomposition.lead.cc==129
	55	"0061 0301 0F73;00E1 0F71 0F72;0061 0F71 0F72 0301;00E1 0F71 0F72;0061 0F71 0F72 0301; # Markus 1"
	56	};
	57
	58	void NormalizerConformanceTest::compare(const UnicodeString& s1, const UnicodeString& s2){
	59	UErrorCode status=U_ZERO_ERROR;
	60	// TODO: Re-enable this tests after UTC fixes UAX 21
	61	if(s1.indexOf((UChar32)0x0345)>=0)return;
	62	if(Normalizer::compare(s1,s2,U_FOLD_CASE_DEFAULT,status)!=0){
	63	errln("Normalizer::compare() failed for s1: " + prettify(s1) + " s2: " +prettify(s2));
	64	}
	65	}
	66
	67	FileStream *
	68	NormalizerConformanceTest::openNormalizationTestFile(const char *filename) {
	69	char unidataPath[2000];
	70	const char *folder;
	71	FileStream *input;
	72	UErrorCode errorCode;
	73
	74	// look inside ICU_DATA first
374ca955	75	folder=pathToDataDirectory();
b75a7d8f A	76	if(folder!=NULL) {
	77	strcpy(unidataPath, folder);
	78	strcat(unidataPath, "unidata" U_FILE_SEP_STRING);
	79	strcat(unidataPath, filename);
	80	input=T_FileStream_open(unidataPath, "rb");
	81	if(input!=NULL) {
	82	return input;
	83	}
	84	}
	85
	86	// find icu/source/data/unidata relative to the test data
	87	errorCode=U_ZERO_ERROR;
	88	folder=loadTestData(errorCode);
	89	if(U_SUCCESS(errorCode)) {
	90	strcpy(unidataPath, folder);
	91	strcat(unidataPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".."
	92	U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".."
	93	U_FILE_SEP_STRING "data" U_FILE_SEP_STRING "unidata" U_FILE_SEP_STRING);
	94	strcat(unidataPath, filename);
	95	input=T_FileStream_open(unidataPath, "rb");
	96	if(input!=NULL) {
	97	return input;
	98	}
	99	}
	100
	101	// look in icu/source/test/testdata/out/build
	102	errorCode=U_ZERO_ERROR;
	103	folder=loadTestData(errorCode);
	104	if(U_SUCCESS(errorCode)) {
	105	strcpy(unidataPath, folder);
	106	strcat(unidataPath, U_FILE_SEP_STRING);
	107	strcat(unidataPath, filename);
	108	input=T_FileStream_open(unidataPath, "rb");
	109	if(input!=NULL) {
	110	return input;
	111	}
	112	}
	113
	114	// look in icu/source/test/testdata
	115	errorCode=U_ZERO_ERROR;
	116	folder=loadTestData(errorCode);
	117	if(U_SUCCESS(errorCode)) {
	118	strcpy(unidataPath, folder);
	119	strcat(unidataPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING);
	120	strcat(unidataPath, filename);
	121	input=T_FileStream_open(unidataPath, "rb");
	122	if(input!=NULL) {
	123	return input;
	124	}
	125	}
	126
	127	// find icu/source/data/unidata relative to U_TOPSRCDIR
	128	#if defined(U_TOPSRCDIR)
	129	strcpy(unidataPath, U_TOPSRCDIR U_FILE_SEP_STRING "data" U_FILE_SEP_STRING "unidata" U_FILE_SEP_STRING);
	130	strcat(unidataPath, filename);
	131	input=T_FileStream_open(unidataPath, "rb");
	132	if(input!=NULL) {
	133	return input;
	134	}
	135
	136	strcpy(unidataPath, U_TOPSRCDIR U_FILE_SEP_STRING "test" U_FILE_SEP_STRING "testdata" U_FILE_SEP_STRING);
	137	strcat(unidataPath, filename);
	138	input=T_FileStream_open(unidataPath, "rb");
	139	if(input!=NULL) {
140	return input;
141	}
142	#endif
143
729e4ab9	144	dataerrln("Failed to open %s", filename);
b75a7d8f A	145	return NULL;
	146	}
	147
	148	/**
	149	* Test the conformance of Normalizer to
	150	* http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt
	151	*/
	152	void NormalizerConformanceTest::TestConformance() {
	153	TestConformance(openNormalizationTestFile("NormalizationTest.txt"), 0);
	154	}
	155
	156	void NormalizerConformanceTest::TestConformance32() {
	157	TestConformance(openNormalizationTestFile("NormalizationTest-3.2.0.txt"), UNORM_UNICODE_3_2);
	158	}
	159
	160	void NormalizerConformanceTest::TestConformance(FileStream *input, int32_t options) {
	161	enum { BUF_SIZE = 1024 };
	162	char lineBuf[BUF_SIZE];
	163	UnicodeString fields[FIELD_COUNT];
	164	UErrorCode status = U_ZERO_ERROR;
	165	int32_t passCount = 0;
	166	int32_t failCount = 0;
	167	UChar32 c;
	168
	169	if(input==NULL) {
	170	return;
	171	}
	172
	173	// UnicodeSet for all code points that are not mentioned in NormalizationTest.txt
	174	UnicodeSet other(0, 0x10ffff);
	175
2ca993e8	176	int32_t count, countMoreCases = UPRV_LENGTHOF(moreCases);
b75a7d8f A	177	for (count = 1;;++count) {
	178	if (!T_FileStream_eof(input)) {
	179	T_FileStream_readLine(input, lineBuf, (int32_t)sizeof(lineBuf));
	180	} else {
	181	// once NormalizationTest.txt is finished, use moreCases[]
	182	if(count > countMoreCases) {
	183	count = 0;
	184	} else if(count == countMoreCases) {
	185	// all done
	186	break;
	187	}
	188	uprv_strcpy(lineBuf, moreCases[count]);
	189	}
	190	if (lineBuf[0] == 0 \|\| lineBuf[0] == '\n' \|\| lineBuf[0] == '\r') continue;
	191
	192	// Expect 5 columns of this format:
	193	// 1E0C;1E0C;0044 0323;1E0C;0044 0323; # <comments>
	194
	195	// Parse out the comment.
	196	if (lineBuf[0] == '#') continue;
	197
	198	// Read separator lines starting with '@'
	199	if (lineBuf[0] == '@') {
	200	logln(lineBuf);
	201	continue;
	202	}
	203
	204	// Parse out the fields
	205	if (!hexsplit(lineBuf, ';', fields, FIELD_COUNT)) {
	206	errln((UnicodeString)"Unable to parse line " + count);
	207	break; // Syntax error
	208	}
	209
	210	// Remove a single code point from the "other" UnicodeSet
	211	if(fields[0].length()==fields[0].moveIndex32(0, 1)) {
	212	c=fields[0].char32At(0);
	213	if(0xac20<=c && c<=0xd73f && quick) {
	214	// not an exhaustive test run: skip most Hangul syllables
	215	if(c==0xac20) {
	216	other.remove(0xac20, 0xd73f);
	217	}
	218	continue;
	219	}
	220	other.remove(c);
	221	}
	222
	223	if (checkConformance(fields, lineBuf, options, status)) {
	224	++passCount;
	225	} else {
	226	++failCount;
	227	if(status == U_FILE_ACCESS_ERROR) {
729e4ab9	228	dataerrln("Something is wrong with the normalizer, skipping the rest of the test.");
b75a7d8f A	229	break;
	230	}
	231	}
	232	if ((count % 1000) == 0) {
	233	logln("Line %d", count);
	234	}
	235	}
	236
	237	T_FileStream_close(input);
	238
	239	/*
	240	* Test that all characters that are not mentioned
	241	* as single code points in column 1
	242	* do not change under any normalization.
	243	*/
	244
	245	// remove U+ffff because that is the end-of-iteration sentinel value
	246	other.remove(0xffff);
	247
	248	for(c=0; c<=0x10ffff; quick ? c+=113 : ++c) {
	249	if(0x30000<=c && c<0xe0000) {
	250	c=0xe0000;
	251	}
	252	if(!other.contains(c)) {
	253	continue;
	254	}
	255
	256	fields[0]=fields[1]=fields[2]=fields[3]=fields[4].setTo(c);
	257	sprintf(lineBuf, "not mentioned code point U+%04lx", (long)c);
	258
	259	if (checkConformance(fields, lineBuf, options, status)) {
	260	++passCount;
	261	} else {
	262	++failCount;
	263	if(status == U_FILE_ACCESS_ERROR) {
729e4ab9	264	dataerrln("Something is wrong with the normalizer, skipping the rest of the test.: %s", u_errorName(status));
b75a7d8f A	265	break;
	266	}
	267	}
	268	if ((c % 0x1000) == 0) {
	269	logln("Code point U+%04lx", c);
	270	}
	271	}
	272
	273	if (failCount != 0) {
729e4ab9	274	dataerrln((UnicodeString)"Total: " + failCount + " lines/code points failed, " +
b75a7d8f A	275	passCount + " lines/code points passed");
	276	} else {
	277	logln((UnicodeString)"Total: " + passCount + " lines/code points passed");
	278	}
	279	}
	280
	281	/**
	282	* Verify the conformance of the given line of the Unicode
	283	* normalization (UTR 15) test suite file. For each line,
	284	* there are five columns, corresponding to field[0]..field[4].
	285	*
	286	* The following invariants must be true for all conformant implementations
	287	* c2 == NFC(c1) == NFC(c2) == NFC(c3)
	288	* c3 == NFD(c1) == NFD(c2) == NFD(c3)
	289	* c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
	290	* c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
	291	*
	292	* @param field the 5 columns
	293	* @param line the source line from the test suite file
	294	* @return true if the test passes
	295	*/
	296	UBool NormalizerConformanceTest::checkConformance(const UnicodeString* field,
	297	const char *line,
	298	int32_t options,
	299	UErrorCode &status) {
	300	UBool pass = TRUE, result;
	301	//UErrorCode status = U_ZERO_ERROR;
	302	UnicodeString out, fcd;
	303	int32_t fieldNum;
	304
	305	for (int32_t i=0; i<FIELD_COUNT; ++i) {
	306	fieldNum = i+1;
	307	if (i<3) {
	308	Normalizer::normalize(field[i], UNORM_NFC, options, out, status);
729e4ab9 A	309	if (U_FAILURE(status)) {
	310	dataerrln("Error running normalize UNORM_NFC: %s", u_errorName(status));
	311	} else {
	312	pass &= assertEqual("C", field[i], out, field[1], "c2!=C(c", fieldNum);
	313	iterativeNorm(field[i], UNORM_NFC, options, out, +1);
	314	pass &= assertEqual("C(+1)", field[i], out, field[1], "c2!=C(c", fieldNum);
	315	iterativeNorm(field[i], UNORM_NFC, options, out, -1);
	316	pass &= assertEqual("C(-1)", field[i], out, field[1], "c2!=C(c", fieldNum);
	317	}
b75a7d8f A	318
b75a7d8f A	319	Normalizer::normalize(field[i], UNORM_NFD, options, out, status);
729e4ab9 A	320	if (U_FAILURE(status)) {
	321	dataerrln("Error running normalize UNORM_NFD: %s", u_errorName(status));
	322	} else {
	323	pass &= assertEqual("D", field[i], out, field[2], "c3!=D(c", fieldNum);
	324	iterativeNorm(field[i], UNORM_NFD, options, out, +1);
	325	pass &= assertEqual("D(+1)", field[i], out, field[2], "c3!=D(c", fieldNum);
	326	iterativeNorm(field[i], UNORM_NFD, options, out, -1);
	327	pass &= assertEqual("D(-1)", field[i], out, field[2], "c3!=D(c", fieldNum);
	328	}
b75a7d8f A	329	}
b75a7d8f A	330	Normalizer::normalize(field[i], UNORM_NFKC, options, out, status);
729e4ab9 A	331	if (U_FAILURE(status)) {
	332	dataerrln("Error running normalize UNORM_NFKC: %s", u_errorName(status));
	333	} else {
	334	pass &= assertEqual("KC", field[i], out, field[3], "c4!=KC(c", fieldNum);
	335	iterativeNorm(field[i], UNORM_NFKC, options, out, +1);
	336	pass &= assertEqual("KC(+1)", field[i], out, field[3], "c4!=KC(c", fieldNum);
	337	iterativeNorm(field[i], UNORM_NFKC, options, out, -1);
	338	pass &= assertEqual("KC(-1)", field[i], out, field[3], "c4!=KC(c", fieldNum);
	339	}
b75a7d8f A	340
b75a7d8f A	341	Normalizer::normalize(field[i], UNORM_NFKD, options, out, status);
729e4ab9 A	342	if (U_FAILURE(status)) {
	343	dataerrln("Error running normalize UNORM_NFKD: %s", u_errorName(status));
	344	} else {
	345	pass &= assertEqual("KD", field[i], out, field[4], "c5!=KD(c", fieldNum);
	346	iterativeNorm(field[i], UNORM_NFKD, options, out, +1);
	347	pass &= assertEqual("KD(+1)", field[i], out, field[4], "c5!=KD(c", fieldNum);
	348	iterativeNorm(field[i], UNORM_NFKD, options, out, -1);
	349	pass &= assertEqual("KD(-1)", field[i], out, field[4], "c5!=KD(c", fieldNum);
	350	}
b75a7d8f A	351	}
	352	compare(field[1],field[2]);
	353	compare(field[0],field[1]);
	354	// test quick checks
	355	if(UNORM_NO == Normalizer::quickCheck(field[1], UNORM_NFC, options, status)) {
	356	errln("Normalizer error: quickCheck(NFC(s), UNORM_NFC) is UNORM_NO");
	357	pass = FALSE;
	358	}
	359	if(UNORM_NO == Normalizer::quickCheck(field[2], UNORM_NFD, options, status)) {
	360	errln("Normalizer error: quickCheck(NFD(s), UNORM_NFD) is UNORM_NO");
	361	pass = FALSE;
	362	}
	363	if(UNORM_NO == Normalizer::quickCheck(field[3], UNORM_NFKC, options, status)) {
	364	errln("Normalizer error: quickCheck(NFKC(s), UNORM_NFKC) is UNORM_NO");
	365	pass = FALSE;
	366	}
	367	if(UNORM_NO == Normalizer::quickCheck(field[4], UNORM_NFKD, options, status)) {
	368	errln("Normalizer error: quickCheck(NFKD(s), UNORM_NFKD) is UNORM_NO");
	369	pass = FALSE;
	370	}
	371
	372	// branch on options==0 for better code coverage
	373	if(options==0) {
	374	result = Normalizer::isNormalized(field[1], UNORM_NFC, status);
	375	} else {
	376	result = Normalizer::isNormalized(field[1], UNORM_NFC, options, status);
	377	}
	378	if(!result) {
729e4ab9	379	dataerrln("Normalizer error: isNormalized(NFC(s), UNORM_NFC) is FALSE");
b75a7d8f A	380	pass = FALSE;
	381	}
	382	if(field[0]!=field[1] && Normalizer::isNormalized(field[0], UNORM_NFC, options, status)) {
	383	errln("Normalizer error: isNormalized(s, UNORM_NFC) is TRUE");
	384	pass = FALSE;
	385	}
	386	if(!Normalizer::isNormalized(field[3], UNORM_NFKC, options, status)) {
729e4ab9	387	dataerrln("Normalizer error: isNormalized(NFKC(s), UNORM_NFKC) is FALSE");
b75a7d8f A	388	pass = FALSE;
	389	}
	390	if(field[0]!=field[3] && Normalizer::isNormalized(field[0], UNORM_NFKC, options, status)) {
	391	errln("Normalizer error: isNormalized(s, UNORM_NFKC) is TRUE");
	392	pass = FALSE;
	393	}
	394
	395	// test FCD quick check and "makeFCD"
	396	Normalizer::normalize(field[0], UNORM_FCD, options, fcd, status);
	397	if(UNORM_NO == Normalizer::quickCheck(fcd, UNORM_FCD, options, status)) {
	398	errln("Normalizer error: quickCheck(FCD(s), UNORM_FCD) is UNORM_NO");
	399	pass = FALSE;
	400	}
	401	if(UNORM_NO == Normalizer::quickCheck(field[2], UNORM_FCD, options, status)) {
	402	errln("Normalizer error: quickCheck(NFD(s), UNORM_FCD) is UNORM_NO");
	403	pass = FALSE;
	404	}
	405	if(UNORM_NO == Normalizer::quickCheck(field[4], UNORM_FCD, options, status)) {
	406	errln("Normalizer error: quickCheck(NFKD(s), UNORM_FCD) is UNORM_NO");
	407	pass = FALSE;
	408	}
	409
	410	Normalizer::normalize(fcd, UNORM_NFD, options, out, status);
	411	if(out != field[2]) {
729e4ab9	412	dataerrln("Normalizer error: NFD(FCD(s))!=NFD(s)");
b75a7d8f A	413	pass = FALSE;
	414	}
	415
	416	if (U_FAILURE(status)) {
729e4ab9	417	dataerrln("Normalizer::normalize returned error status: %s", u_errorName(status));
b75a7d8f A	418	pass = FALSE;
	419	}
	420
	421	if(field[0]!=field[2]) {
	422	// two strings that are canonically equivalent must test
	423	// equal under a canonical caseless match
	424	// see UAX #21 Case Mappings and Jitterbug 2021 and
	425	// Unicode Technical Committee meeting consensus 92-C31
	426	int32_t rc;
	427
	428	status=U_ZERO_ERROR;
	429	rc=Normalizer::compare(field[0], field[2], (options<<UNORM_COMPARE_NORM_OPTIONS_SHIFT)\|U_COMPARE_IGNORE_CASE, status);
	430	if(U_FAILURE(status)) {
729e4ab9	431	dataerrln("Normalizer::compare(case-insensitive) sets %s", u_errorName(status));
b75a7d8f A	432	pass=FALSE;
	433	} else if(rc!=0) {
	434	errln("Normalizer::compare(original, NFD, case-insensitive) returned %d instead of 0 for equal", rc);
	435	pass=FALSE;
	436	}
	437	}
	438
	439	if (!pass) {
729e4ab9	440	dataerrln("FAIL: %s", line);
b75a7d8f A	441	}
	442	return pass;
	443	}
	444
	445	/**
	446	* Do a normalization using the iterative API in the given direction.
	447	* @param dir either +1 or -1
	448	*/
	449	void NormalizerConformanceTest::iterativeNorm(const UnicodeString& str,
	450	UNormalizationMode mode, int32_t options,
	451	UnicodeString& result,
	452	int8_t dir) {
	453	UErrorCode status = U_ZERO_ERROR;
	454	normalizer.setText(str, status);
	455	normalizer.setMode(mode);
	456	normalizer.setOption(-1, 0); // reset all options
	457	normalizer.setOption(options, 1); // set desired options
	458	result.truncate(0);
	459	if (U_FAILURE(status)) {
	460	return;
	461	}
	462	UChar32 ch;
	463	if (dir > 0) {
	464	for (ch = normalizer.first(); ch != Normalizer::DONE;
	465	ch = normalizer.next()) {
	466	result.append(ch);
	467	}
	468	} else {
	469	for (ch = normalizer.last(); ch != Normalizer::DONE;
	470	ch = normalizer.previous()) {
	471	result.insert(0, ch);
	472	}
	473	}
	474	}
	475
	476	/**
	477	* @param op name of normalization form, e.g., "KC"
	478	* @param s string being normalized
	479	* @param got value received
	480	* @param exp expected value
	481	* @param msg description of this test
	482	* @param return true if got == exp
	483	*/
	484	UBool NormalizerConformanceTest::assertEqual(const char *op,
	485	const UnicodeString& s,
	486	const UnicodeString& got,
	487	const UnicodeString& exp,
	488	const char *msg,
	489	int32_t field)
	490	{
	491	if (exp == got)
	492	return TRUE;
	493
	494	char sChars, gotChars, *expChars;
	495	UnicodeString sPretty(prettify(s));
	496	UnicodeString gotPretty(prettify(got));
	497	UnicodeString expPretty(prettify(exp));
	498
	499	sChars = new char[sPretty.length() + 1];
	500	gotChars = new char[gotPretty.length() + 1];
	501	expChars = new char[expPretty.length() + 1];
	502
	503	sPretty.extract(0, sPretty.length(), sChars, sPretty.length() + 1);
	504	sChars[sPretty.length()] = 0;
505	gotPretty.extract(0, gotPretty.length(), gotChars, gotPretty.length() + 1);
506	gotChars[gotPretty.length()] = 0;
507	expPretty.extract(0, expPretty.length(), expChars, expPretty.length() + 1);
508	expChars[expPretty.length()] = 0;
509
510	errln(" %s%d)%s(%s)=%s, exp. %s", msg, field, op, sChars, gotChars, expChars);
511
512	delete []sChars;
513	delete []gotChars;
514	delete []expChars;
515	return FALSE;
516	}
517
518	/**
519	* Split a string into pieces based on the given delimiter
520	* character. Then, parse the resultant fields from hex into
521	* characters. That is, "0040 0400;0C00;0899" -> new String[] {
522	* "\u0040\u0400", "\u0C00", "\u0899" }. The output is assumed to
523	* be of the proper length already, and exactly output.length
524	* fields are parsed. If there are too few an exception is
525	* thrown. If there are too many the extras are ignored.
526	*
527	* @return FALSE upon failure
528	*/
529	UBool NormalizerConformanceTest::hexsplit(const char *s, char delimiter,
530	UnicodeString output[], int32_t outputLength) {
531	const char *t = s;
532	char *end = NULL;
533	UChar32 c;
534	int32_t i;
535	for (i=0; i<outputLength; ++i) {
536	// skip whitespace
537	while(t == ' ' \|\| t == '\t') {
538	++t;
539	}
540
541	// read a sequence of code points
542	output[i].remove();
543	for(;;) {
544	c = (UChar32)uprv_strtoul(t, &end, 16);
545
546	if( (char *)t == end \|\|
547	(uint32_t)c > 0x10ffff \|\|
548	(end != ' ' && end != '\t' && *end != delimiter)
549	) {
550	errln(UnicodeString("Bad field ", "") + (i + 1) + " in " + UnicodeString(s, ""));
551	return FALSE;
552	}
553
554	output[i].append(c);
555
556	t = (const char *)end;
557
558	// skip whitespace
559	while(t == ' ' \|\| t == '\t') {
560	++t;
561	}
562
563	if(*t == delimiter) {
564	++t;
565	break;
566	}
567	if(*t == 0) {
568	if((i + 1) == outputLength) {
569	return TRUE;
570	} else {
571	errln(UnicodeString("Missing field(s) in ", "") + s + " only " + (i + 1) + " out of " + outputLength);
572	return FALSE;
573	}
574	}
575	}
576	}
577	return TRUE;
578	}
579
580	// Specific tests for debugging. These are generally failures taken from
581	// the conformance file, but culled out to make debugging easier.
582
583	void NormalizerConformanceTest::TestCase6(void) {
584	_testOneLine("0385;0385;00A8 0301;0020 0308 0301;0020 0308 0301;");
585	}
586
587	void NormalizerConformanceTest::_testOneLine(const char *line) {
588	UErrorCode status = U_ZERO_ERROR;
589	UnicodeString fields[FIELD_COUNT];
590	if (!hexsplit(line, ';', fields, FIELD_COUNT)) {
591	errln((UnicodeString)"Unable to parse line " + line);
592	} else {
593	checkConformance(fields, line, 0, status);
594	}
595	}
596
597	#endif /* #if !UCONFIG_NO_NORMALIZATION */